diff options
489 files changed, 18274 insertions, 4950 deletions
diff --git a/.gitignore b/.gitignore index 0865f981b1..77610b3193 100644 --- a/.gitignore +++ b/.gitignore @@ -162,6 +162,8 @@ uptime-*.json /src/lib/libtor-buf-testing.a /src/lib/libtor-compress.a /src/lib/libtor-compress-testing.a +/src/lib/libtor-confmgt.a +/src/lib/libtor-confmgt-testing.a /src/lib/libtor-container.a /src/lib/libtor-container-testing.a /src/lib/libtor-crypt-ops.a diff --git a/.travis.yml b/.travis.yml index 375796799b..368910f1f1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,6 +29,8 @@ env: - HARDENING_OPTIONS="--enable-expensive-hardening" ## We turn off asciidoc by default, because it's slow - ASCIIDOC_OPTIONS="--disable-asciidoc" + ## Our default rust version is the minimum supported version + - RUST_VERSION="1.31.0" ## Turn off tor's sandbox in chutney, until we fix sandbox errors that are ## triggered by Ubuntu Xenial and Bionic. See #32722. - CHUTNEY_TOR_SANDBOX="0" @@ -69,7 +71,7 @@ matrix: ## concurrent macOS jobs. We're not actively developing Rust, so it is ## the lowest priority. ## We run rust on macOS, because we have seen macOS rust failures before - #- env: RUST_OPTIONS="--enable-rust --enable-cargo-online-mode" + #- env: RUST_VERSION="nightly" RUST_OPTIONS="--enable-rust --enable-cargo-online-mode" # compiler: clang # os: osx ## We run chutney on macOS, because macOS Travis has IPv6 @@ -88,7 +90,7 @@ matrix: ## macOS rust and chutney are very slow, so we let the build finish before ## they are done. We'd like to fast finish, but still eventually show ## any failures in the build status. But Travis doesn't have that ability. - - env: RUST_OPTIONS="--enable-rust --enable-cargo-online-mode" + - env: RUST_VERSION="nightly" RUST_OPTIONS="--enable-rust --enable-cargo-online-mode" compiler: clang os: osx - env: CHUTNEY="yes" CHUTNEY_ALLOW_FAILURES="2" SKIP_MAKE_CHECK="yes" @@ -176,8 +178,8 @@ install: - if [[ "$ASCIIDOC_OPTIONS" == "" ]] && [[ "$TRAVIS_OS_NAME" == "osx" ]]; then export XML_CATALOG_FILES="/usr/local/etc/xml/catalog"; fi ## If we're using Rust, download rustup - if [[ "$RUST_OPTIONS" != "" ]]; then curl -Ssf -o rustup.sh https://sh.rustup.rs; fi - ## Install the nightly channels of rustc and cargo and setup our toolchain environment - - if [[ "$RUST_OPTIONS" != "" ]]; then sh rustup.sh -y --default-toolchain nightly; fi + ## Install the stable channels of rustc and cargo and setup our toolchain environment + - if [[ "$RUST_OPTIONS" != "" ]]; then sh rustup.sh -y --default-toolchain $RUST_VERSION; fi - if [[ "$RUST_OPTIONS" != "" ]]; then source $HOME/.cargo/env; fi ## If we're testing rust builds in offline-mode, then set up our vendored dependencies - if [[ "$TOR_RUST_DEPENDENCIES" == "true" ]]; then export TOR_RUST_DEPENDENCIES=$PWD/src/ext/rust/crates; fi @@ -202,6 +204,10 @@ install: - if [[ "$CHUTNEY" != "" ]]; then pushd "$CHUTNEY_PATH"; git log -1 ; popd ; fi ## If we're running stem, show the stem version and commit - if [[ "$TEST_STEM" != "" ]]; then pushd stem; python -c "from stem import stem; print(stem.__version__);"; git log -1; popd; fi + ## We don't want Tor tests to depend on default configuration file at + ## ~/.torrc. So we put some random bytes in there, to make sure we get build + ## failures in case Tor is reading it during CI jobs. + - dd ibs=1 count=1024 if=/dev/urandom > ~/.torrc script: # Skip test_rebind on macOS @@ -1,3 +1,790 @@ +Changes in version 0.4.2.2-alpha - 2019-10-07 + This release fixes several bugs from the previous alpha release, and + from earlier versions. It also includes a change in authorities, so + that they begin to reject the currently unsupported release series. + + o Major features (directory authorities): + - Directory authorities now reject relays running all currently + deprecated release series. The currently supported release series + are: 0.2.9, 0.3.5, 0.4.0, 0.4.1, and 0.4.2. Closes ticket 31549. + + o Major bugfixes (embedded Tor): + - Avoid a possible crash when restarting Tor in embedded mode and + enabling a different set of publish/subscribe messages. Fixes bug + 31898; bugfix on 0.4.1.1-alpha. + + o Major bugfixes (torrc parsing): + - Stop ignoring torrc options after an %include directive, when the + included directory ends with a file that does not contain any + config options (but does contain comments or whitespace). Fixes + bug 31408; bugfix on 0.3.1.1-alpha. + + o Minor features (auto-formatting scripts): + - When annotating C macros, never generate a line that our check- + spaces script would reject. Closes ticket 31759. + - When annotating C macros, try to remove cases of double-negation. + Closes ticket 31779. + + o Minor features (continuous integration): + - When building on Appveyor and Travis, pass the "-k" flag to make, + so that we are informed of all compilation failures, not just the + first one or two. Closes ticket 31372. + + o Minor features (geoip): + - Update geoip and geoip6 to the October 1 2019 Maxmind GeoLite2 + Country database. Closes ticket 31931. + + o Minor features (maintenance scripts): + - Add a Coccinelle script to detect bugs caused by incrementing or + decrementing a variable inside a call to log_debug(). Since + log_debug() is a macro whose arguments are conditionally + evaluated, it is usually an error to do this. One such bug was + 30628, in which SENDME cells were miscounted by a decrement + operator inside a log_debug() call. Closes ticket 30743. + + o Minor features (onion services v3): + - Assist users who try to setup v2 client authorization in v3 onion + services by pointing them to the right documentation. Closes + ticket 28966. + + o Minor bugfixes (Appveyor continuous integration): + - Avoid spurious errors when Appveyor CI fails before the install + step. Fixes bug 31884; bugfix on 0.3.4.2-alpha. + + o Minor bugfixes (best practices tracker): + - When listing overbroad exceptions, do not also list problems, and + do not list insufficiently broad exceptions. Fixes bug 31338; + bugfix on 0.4.2.1-alpha. + + o Minor bugfixes (controller protocol): + - Fix the MAPADDRESS controller command to accept one or more + arguments. Previously, it required two or more arguments, and + ignored the first. Fixes bug 31772; bugfix on 0.4.1.1-alpha. + + o Minor bugfixes (logging): + - Add a missing check for HAVE_PTHREAD_H, because the backtrace code + uses mutexes. Fixes bug 31614; bugfix on 0.2.5.2-alpha. + - Disable backtrace signal handlers when shutting down tor. Fixes + bug 31614; bugfix on 0.2.5.2-alpha. + - Rate-limit our the logging message about the obsolete .exit + notation. Previously, there was no limit on this warning, which + could potentially be triggered many times by a hostile website. + Fixes bug 31466; bugfix on 0.2.2.1-alpha. + - When initialising log domain masks, only set known log domains. + Fixes bug 31854; bugfix on 0.2.1.1-alpha. + + o Minor bugfixes (logging, protocol violations): + - Do not log a nonfatal assertion failure when receiving a VERSIONS + cell on a connection using the obsolete v1 link protocol. Log a + protocol_warn instead. Fixes bug 31107; bugfix on 0.2.4.4-alpha. + + o Minor bugfixes (modules): + - Explain what the optional Directory Authority module is, and what + happens when it is disabled. Fixes bug 31825; bugfix + on 0.3.4.1-alpha. + + o Minor bugfixes (multithreading): + - Avoid some undefined behaviour when freeing mutexes. Fixes bug + 31736; bugfix on 0.0.7. + + o Minor bugfixes (relay): + - Avoid crashing when starting with a corrupt keys directory where + the old ntor key and the new ntor key are identical. Fixes bug + 30916; bugfix on 0.2.4.8-alpha. + + o Minor bugfixes (tests, SunOS): + - Avoid a map_anon_nofork test failure due to a signed/unsigned + integer comparison. Fixes bug 31897; bugfix on 0.4.1.1-alpha. + + o Code simplification and refactoring: + - Refactor connection_control_process_inbuf() to reduce the size of + a practracker exception. Closes ticket 31840. + - Refactor the microdescs_parse_from_string() function into smaller + pieces, for better comprehensibility. Closes ticket 31675. + - Use SEVERITY_MASK_IDX() to find the LOG_* mask indexes in the unit + tests and fuzzers, rather than using hard-coded values. Closes + ticket 31334. + - Interface for function `decrypt_desc_layer` cleaned up. Closes + ticket 31589. + + o Documentation: + - Document the signal-safe logging behaviour in the tor man page. + Also add some comments to the relevant functions. Closes + ticket 31839. + - Explain why we can't destroy the backtrace buffer mutex. Explain + why we don't need to destroy the log mutex. Closes ticket 31736. + - The Tor source code repository now includes a (somewhat dated) + description of Tor's modular architecture, in doc/HACKING/design. + This is based on the old "tor-guts.git" repository, which we are + adopting and superseding. Closes ticket 31849. + + +Changes in version 0.4.1.6 - 2019-09-19 + This release backports several bugfixes to improve stability and + correctness. Anyone experiencing build problems or crashes with 0.4.1.5, + or experiencing reliability issues with single onion services, should + upgrade. + + o Major bugfixes (crash, Linux, Android, backport from 0.4.2.1-alpha): + - Tolerate systems (including some Android installations) where + madvise and MADV_DONTDUMP are available at build-time, but not at + run time. Previously, these systems would notice a failed syscall + and abort. Fixes bug 31570; bugfix on 0.4.1.1-alpha. + - Tolerate systems (including some Linux installations) where + madvise and/or MADV_DONTFORK are available at build-time, but not + at run time. Previously, these systems would notice a failed + syscall and abort. Fixes bug 31696; bugfix on 0.4.1.1-alpha. + + o Minor features (stem tests, backport from 0.4.2.1-alpha): + - Change "make test-stem" so it only runs the stem tests that use + tor. This change makes test-stem faster and more reliable. Closes + ticket 31554. + + o Minor bugfixes (build system, backport form 0.4.2.1-alpha): + - Do not include the deprecated <sys/sysctl.h> on Linux or Windows + systems. Fixes bug 31673; bugfix on 0.2.5.4-alpha. + + o Minor bugfixes (compilation, backport from 0.4.2.1-alpha): + - Add more stub functions to fix compilation on Android with link- + time optimization when --disable-module-dirauth is used. + Previously, these compilation settings would make the compiler + look for functions that didn't exist. Fixes bug 31552; bugfix + on 0.4.1.1-alpha. + - Suppress spurious float-conversion warnings from GCC when calling + floating-point classifier functions on FreeBSD. Fixes part of bug + 31687; bugfix on 0.3.1.5-alpha. + + o Minor bugfixes (controller protocol): + - Fix the MAPADDRESS controller command to accept one or more + arguments. Previously, it required two or more arguments, and ignored + the first. Fixes bug 31772; bugfix on 0.4.1.1-alpha. + + o Minor bugfixes (guards, backport from 0.4.2.1-alpha): + - When tor is missing descriptors for some primary entry guards, + make the log message less alarming. It's normal for descriptors to + expire, as long as tor fetches new ones soon after. Fixes bug + 31657; bugfix on 0.3.3.1-alpha. + + o Minor bugfixes (logging, backport from 0.4.2.1-alpha): + - Change log level of message "Hash of session info was not as + expected" to LOG_PROTOCOL_WARN. Fixes bug 12399; bugfix + on 0.1.1.10-alpha. + + o Minor bugfixes (rust, backport from 0.4.2.1-alpha): + - Correctly exclude a redundant rust build job in Travis. Fixes bug + 31463; bugfix on 0.3.5.4-alpha. + + o Minor bugfixes (v2 single onion services, backport from 0.4.2.1-alpha): + - Always retry v2 single onion service intro and rend circuits with + a 3-hop path. Previously, v2 single onion services used a 3-hop + path when rendezvous circuits were retried after a remote or + delayed failure, but a 1-hop path for immediate retries. Fixes bug + 23818; bugfix on 0.2.9.3-alpha. + + o Minor bugfixes (v3 single onion services, backport from 0.4.2.1-alpha): + - Always retry v3 single onion service intro and rend circuits with + a 3-hop path. Previously, v3 single onion services used a 3-hop + path when rend circuits were retried after a remote or delayed + failure, but a 1-hop path for immediate retries. Fixes bug 23818; + bugfix on 0.3.2.1-alpha. + - Make v3 single onion services fall back to a 3-hop intro, when all + intro points are unreachable via a 1-hop path. Previously, v3 + single onion services failed when all intro nodes were unreachable + via a 1-hop path. Fixes bug 23507; bugfix on 0.3.2.1-alpha. + + o Documentation (backport from 0.4.2.1-alpha): + - Use RFC 2397 data URL scheme to embed an image into tor-exit- + notice.html so that operators no longer have to host it + themselves. Closes ticket 31089. + + +Changes in version 0.4.2.1-alpha - 2019-09-17 + This is the first alpha release in the 0.4.2.x series. It adds new + defenses for denial-of-service attacks against onion services. It also + includes numerous kinds of bugfixes and refactoring to help improve + Tor's stability and ease of development. + + o Major features (onion service v3, denial of service): + - Add onion service introduction denial of service defenses. Intro + points can now rate-limit client introduction requests, using + parameters that can be sent by the service within the + ESTABLISH_INTRO cell. If the cell extension for this is not used, + the intro point will honor the consensus parameters. Closes + ticket 30924. + + o Major bugfixes (circuit build, guard): + - When considering upgrading circuits from "waiting for guard" to + "open", always ignore circuits that are marked for close. + Previously we could end up in the situation where a subsystem is + notified of a circuit opening, but the circuit is still marked for + close, leading to undesirable behavior. Fixes bug 30871; bugfix + on 0.3.0.1-alpha. + + o Major bugfixes (crash, Linux, Android): + - Tolerate systems (including some Android installations) where + madvise and MADV_DONTDUMP are available at build-time, but not at + run time. Previously, these systems would notice a failed syscall + and abort. Fixes bug 31570; bugfix on 0.4.1.1-alpha. + - Tolerate systems (including some Linux installations) where + madvise and/or MADV_DONTFORK are available at build-time, but not + at run time. Previously, these systems would notice a failed + syscall and abort. Fixes bug 31696; bugfix on 0.4.1.1-alpha. + + o Minor features (best practices tracker): + - Our best-practices tracker now integrates with our include-checker + tool to keep track of how many layering violations we have not yet + fixed. We hope to reduce this number over time to improve Tor's + modularity. Closes ticket 31176. + - Add a TOR_PRACTRACKER_OPTIONS variable for passing arguments to + practracker from the environment. We may want this for continuous + integration. Closes ticket 31309. + - Give a warning rather than an error when a practracker exception + is violated by a small amount, add a --list-overbroad option to + practracker that lists exceptions that are stricter than they need + to be, and provide an environment variable for disabling + practracker. Closes ticket 30752. + - Our best-practices tracker now looks at headers as well as C + files. Closes ticket 31175. + + o Minor features (build system): + - Add --disable-manpage and --disable-html-manual options to + configure script. This will enable shortening build times by not + building documentation. Resolves issue 19381. + + o Minor features (compilation): + - Log a more useful error message when we are compiling and one of + the compile-time hardening options we have selected can be linked + but not executed. Closes ticket 27530. + + o Minor features (configuration): + - The configuration code has been extended to allow splitting + configuration data across multiple objects. Previously, all + configuration data needed to be kept in a single object, which + tended to become bloated. Closes ticket 31240. + + o Minor features (continuous integration): + - When running CI builds on Travis, put some random data in + ~/.torrc, to make sure no tests are reading the Tor configuration + file from its default location. Resolves issue 30102. + + o Minor features (debugging): + - Log a nonfatal assertion failure if we encounter a configuration + line whose command is "CLEAR" but which has a nonempty value. This + should be impossible, according to the rules of our configuration + line parsing. Closes ticket 31529. + + o Minor features (git hooks): + - Our pre-commit git hook now checks for a special file before + running practracker, so that practracker only runs on branches + that are based on master. Since the pre-push hook calls the pre- + commit hook, practracker will also only run before pushes of + branches based on master. Closes ticket 30979. + + o Minor features (git scripts): + - Add a "--" command-line argument, to separate git-push-all.sh + script arguments from arguments that are passed through to git + push. Closes ticket 31314. + - Add a -r <remote-name> argument to git-push-all.sh, so the script + can push test branches to a personal remote. Closes ticket 31314. + - Add a -t <test-branch-prefix> argument to git-merge-forward.sh and + git-push-all.sh, which makes these scripts create, merge forward, + and push test branches. Closes ticket 31314. + - Add a -u argument to git-merge-forward.sh, so that the script can + re-use existing test branches after a merge failure and fix. + Closes ticket 31314. + - Add a TOR_GIT_PUSH env var, which sets the default git push + command and arguments for git-push-all.sh. Closes ticket 31314. + - Add a TOR_PUSH_DELAY variable to git-push-all.sh, which makes the + script push master and maint branches with a delay between each + branch. These delays trigger the CI jobs in a set order, which + should show the most likely failures first. Also make pushes + atomic by default, and make the script pass any command-line + arguments to git push. Closes ticket 29879. + - Call the shellcheck script from the pre-commit hook. Closes + ticket 30967. + - Skip pushing test branches that are the same as a remote + maint/release/master branch in git-push-all.sh by default. Add a + -s argument, so git-push-all.sh can push all test branches. Closes + ticket 31314. + + o Minor features (IPv6, logging): + - Log IPv6 addresses as well as IPv4 addresses when describing + routerinfos, routerstatuses, and nodes. Closes ticket 21003. + + o Minor features (onion service v3): + - Do not allow single hop clients to fetch or post an HS descriptor + from an HSDir. Closes ticket 24964. + + o Minor features (onion service): + - Disallow single-hop clients at the introduction point. We've + removed Tor2web support a while back and single-hop rendezvous + attempts are blocked at the relays. This change should remove load + off the network from spammy clients. Close ticket 24963. + + o Minor features (stem tests): + - Change "make test-stem" so it only runs the stem tests that use + tor. This change makes test-stem faster and more reliable. Closes + ticket 31554. + + o Minor features (testing): + - Add a script to invoke "tor --dump-config" and "tor + --verify-config" with various configuration options, and see + whether tor's resulting configuration or error messages are what + we expect. Use it for integration testing of our +Option and + /Option flags. Closes ticket 31637. + - Improve test coverage for our existing configuration parsing and + management API. Closes ticket 30893. + - Add integration tests to make sure that practracker gives the + outputs we expect. Closes ticket 31477. + - The practracker self-tests are now run as part of the Tor test + suite. Closes ticket 31304. + + o Minor features (token bucket): + - Implement a generic token bucket that uses a single counter, for + use in anti-DoS onion service work. Closes ticket 30687. + + o Minor bugfixes (best practices tracker): + - Fix a few issues in the best-practices script, including tests, + tab tolerance, error reporting, and directory-exclusion logic. + Fixes bug 29746; bugfix on 0.4.1.1-alpha. + - When running check-best-practices, only consider files in the src + subdirectory. Previously we had recursively considered all + subdirectories, which made us get confused by the temporary + directories made by "make distcheck". Fixes bug 31578; bugfix + on 0.4.1.1-alpha. + + o Minor bugfixes (build system): + - Do not include the deprecated <sys/sysctl.h> on Linux or Windows + systems. Fixes bug 31673; bugfix on 0.2.5.4-alpha. + + o Minor bugfixes (chutney, makefiles, documentation): + - "make test-network-all" now shows the warnings from each test- + network.sh run on the console, so developers see new warnings + early. We've also improved the documentation for this feature, and + renamed a Makefile variable so the code is self-documenting. Fixes + bug 30455; bugfix on 0.3.0.4-rc. + + o Minor bugfixes (compilation): + - Add more stub functions to fix compilation on Android with link- + time optimization when --disable-module-dirauth is used. + Previously, these compilation settings would make the compiler + look for functions that didn't exist. Fixes bug 31552; bugfix + on 0.4.1.1-alpha. + - Suppress spurious float-conversion warnings from GCC when calling + floating-point classifier functions on FreeBSD. Fixes part of bug + 31687; bugfix on 0.3.1.5-alpha. + + o Minor bugfixes (configuration): + - Invalid floating-point values in the configuration file are now + treated as errors in the configuration. Previously, they were + ignored and treated as zero. Fixes bug 31475; bugfix on 0.0.1. + + o Minor bugfixes (coverity): + - Add an assertion when parsing a BEGIN cell so that coverity can be + sure that we are not about to dereference a NULL address. Fixes + bug 31026; bugfix on 0.2.4.7-alpha. This is CID 1447296. + - In our siphash implementation, when building for coverity, use + memcpy in place of a switch statement, so that coverity can tell + we are not accessing out-of-bounds memory. Fixes bug 31025; bugfix + on 0.2.8.1-alpha. This is tracked as CID 1447293 and 1447295. + - Fix several coverity warnings from our unit tests. Fixes bug + 31030; bugfix on 0.2.4.1-alpha, 0.3.2.1-alpha, and 0.4.0.1-alpha. + + o Minor bugfixes (developer tooling): + - Only log git script changes in the post-merge script when the + merge was to the master branch. Fixes bug 31040; bugfix + on 0.4.1.1-alpha. + + o Minor bugfixes (directory authorities): + - Return a distinct status when formatting annotations fails. Fixes + bug 30780; bugfix on 0.2.0.8-alpha. + + o Minor bugfixes (error handling): + - On abort, try harder to flush the output buffers of log messages. + On some platforms (macOS), log messages could be discarded when + the process terminates. Fixes bug 31571; bugfix on 0.3.5.1-alpha. + - Report the tor version whenever an assertion fails. Previously, we + only reported the Tor version on some crashes, and some non-fatal + assertions. Fixes bug 31571; bugfix on 0.3.5.1-alpha. + - When tor aborts due to an error, close log file descriptors before + aborting. Closing the logs makes some OSes flush log file buffers, + rather than deleting buffered log lines. Fixes bug 31594; bugfix + on 0.2.5.2-alpha. + + o Minor bugfixes (FreeBSD, PF-based proxy, IPv6): + - When extracting an IPv6 address from a PF-based proxy, verify that + we are actually configured to receive an IPv6 address, and log an + internal error if not. Fixes part of bug 31687; bugfix + on 0.2.3.4-alpha. + + o Minor bugfixes (git hooks): + - Remove a duplicate call to practracker from the pre-push hook. The + pre-push hook already calls the pre-commit hook, which calls + practracker. Fixes bug 31462; bugfix on 0.4.1.1-alpha. + + o Minor bugfixes (git scripts): + - Stop hard-coding the bash path in the git scripts. Some OSes don't + have bash in /usr/bin, others have an ancient bash at this path. + Fixes bug 30840; bugfix on 0.4.0.1-alpha. + - Stop hard-coding the tor master branch name and worktree path in + the git scripts. Fixes bug 30841; bugfix on 0.4.0.1-alpha. + - Allow git-push-all.sh to be run from any directory. Previously, + the script only worked if run from an upstream worktree directory. + Closes ticket 31678. + + o Minor bugfixes (guards): + - When tor is missing descriptors for some primary entry guards, + make the log message less alarming. It's normal for descriptors to + expire, as long as tor fetches new ones soon after. Fixes bug + 31657; bugfix on 0.3.3.1-alpha. + + o Minor bugfixes (ipv6): + - Check for private IPv6 addresses alongside their IPv4 equivalents + when authorities check descriptors. Previously, we only checked + for private IPv4 addresses. Fixes bug 31088; bugfix on + 0.2.3.21-rc. Patch by Neel Chauhan. + - When parsing microdescriptors, we should check the IPv6 exit + policy alongside IPv4. Previously, we checked both exit policies + for only router info structures, while microdescriptors were + IPv4-only. Fixes bug 27284; bugfix on 0.2.3.1-alpha. Patch by + Neel Chauhan. + + o Minor bugfixes (logging): + - Change log level of message "Hash of session info was not as + expected" to LOG_PROTOCOL_WARN. Fixes bug 12399; bugfix + on 0.1.1.10-alpha. + - Fix a code issue that would have broken our parsing of log domains + as soon as we had 33 of them. Fortunately, we still only have 29. + Fixes bug 31451; bugfix on 0.4.1.4-rc. + + o Minor bugfixes (memory management): + - Stop leaking a small amount of memory in nt_service_install(), in + unreachable code. Fixes bug 30799; bugfix on 0.2.0.7-alpha. Patch + by Xiaoyin Liu. + + o Minor bugfixes (networking, IP addresses): + - When parsing addresses via Tor's internal DNS lookup API, reject + IPv4 addresses in square brackets, and accept IPv6 addresses in + square brackets. This change completes the work started in 23082, + making address parsing consistent between tor's internal DNS + lookup and address parsing APIs. Fixes bug 30721; bugfix + on 0.2.1.5-alpha. + - When parsing addresses via Tor's internal address:port parsing and + DNS lookup APIs, require IPv6 addresses with ports to have square + brackets. But allow IPv6 addresses without ports, whether or not + they have square brackets. Fixes bug 30721; bugfix + on 0.2.1.5-alpha. + + o Minor bugfixes (onion service v3): + - When purging the client descriptor cache, close any introduction + point circuits associated with purged cache entries. This avoids + picking those circuits later when connecting to the same + introduction points. Fixes bug 30921; bugfix on 0.3.2.1-alpha. + + o Minor bugfixes (onion services): + - In the hs_ident_circuit_t data structure, remove the unused field + circuit_type and the respective argument in hs_ident_circuit_new(). + This field was set by clients (for introduction) and services (for + introduction and rendezvous) but was never used afterwards. Fixes + bug 31490; bugfix on 0.3.2.1-alpha. Patch by Neel Chauhan. + + o Minor bugfixes (operator tools): + - Make tor-print-ed-signing-cert(1) print certificate expiration + date in RFC 1123 and UNIX timestamp formats, to make output + machine readable. Fixes bug 31012; bugfix on 0.3.5.1-alpha. + + o Minor bugfixes (rust): + - Correctly exclude a redundant rust build job in Travis. Fixes bug + 31463; bugfix on 0.3.5.4-alpha. + - Raise the minimum rustc version to 1.31.0, as checked by configure + and CI. Fixes bug 31442; bugfix on 0.3.5.4-alpha. + + o Minor bugfixes (sendme, code structure): + - Rename the trunnel SENDME file definition from sendme.trunnel to + sendme_cell.trunnel to avoid having twice sendme.{c|h} in the + repository. Fixes bug 30769; bugfix on 0.4.1.1-alpha. + + o Minor bugfixes (statistics): + - Stop removing the ed25519 signature if the extra info file is too + big. If the signature data was removed, but the keyword was kept, + this could result in an unparseable extra info file. Fixes bug + 30958; bugfix on 0.2.7.2-alpha. + + o Minor bugfixes (subsystems): + - Make the subsystem init order match the subsystem module + dependencies. Call windows process security APIs as early as + possible. Initialize logging before network and time, so that + network and time can use logging. Fixes bug 31615; bugfix + on 0.4.0.1-alpha. + + o Minor bugfixes (testing): + - Teach the util/socketpair_ersatz test to work correctly when we + have no network stack configured. Fixes bug 30804; bugfix + on 0.2.5.1-alpha. + + o Minor bugfixes (v2 single onion services): + - Always retry v2 single onion service intro and rend circuits with + a 3-hop path. Previously, v2 single onion services used a 3-hop + path when rendezvous circuits were retried after a remote or + delayed failure, but a 1-hop path for immediate retries. Fixes bug + 23818; bugfix on 0.2.9.3-alpha. + + o Minor bugfixes (v3 single onion services): + - Always retry v3 single onion service intro and rend circuits with + a 3-hop path. Previously, v3 single onion services used a 3-hop + path when rend circuits were retried after a remote or delayed + failure, but a 1-hop path for immediate retries. Fixes bug 23818; + bugfix on 0.3.2.1-alpha. + - Make v3 single onion services fall back to a 3-hop intro, when all + intro points are unreachable via a 1-hop path. Previously, v3 + single onion services failed when all intro nodes were unreachable + via a 1-hop path. Fixes bug 23507; bugfix on 0.3.2.1-alpha. + + o Documentation: + - Improve documentation in circuit padding subsystem. Patch by + Tobias Pulls. Closes ticket 31113. + - Include an example usage for IPv6 ORPort in our sample torrc. + Closes ticket 31320; patch from Ali Raheem. + - Use RFC 2397 data URL scheme to embed an image into tor-exit- + notice.html so that operators no longer have to host it + themselves. Closes ticket 31089. + + o Removed features: + - No longer include recommended package digests in votes as detailed + in proposal 301. The RecommendedPackages torrc option is + deprecated and will no longer have any effect. "package" lines + will still be considered when computing consensuses for consensus + methods that include them. (This change has no effect on the list + of recommended Tor versions, which is still in use.) Closes + ticket 29738. + - Remove torctl.in from contrib/dist directory. Resolves + ticket 30550. + + o Testing: + - Run shellcheck for all non-third-party shell scripts that are + shipped with Tor. Closes ticket 29533. + - When checking shell scripts, ignore any user-created directories. + Closes ticket 30967. + + o Code simplification and refactoring (config handling): + - Extract our variable manipulation code from confparse.c to a new + lower-level typedvar.h module. Closes ticket 30864. + - Lower another layer of object management from confparse.c to a + more general tool. Now typed structure members are accessible via + an abstract type. Implements ticket 30914. + - Move our backend logic for working with configuration and state + files into a lower-level library, since it no longer depends on + any tor-specific functionality. Closes ticket 31626. + - Numerous simplifications in configuration-handling logic: remove + duplicated macro definitions, replace magical names with flags, + and refactor "TestingTorNetwork" to use the same default-option + logic as the rest of Tor. Closes ticket 30935. + - Replace our ad-hoc set of flags for configuration variables and + configuration variable types with fine-grained orthogonal flags + corresponding to the actual behavior we want. Closes ticket 31625. + + o Code simplification and refactoring (misc): + - Eliminate some uses of lower-level control reply abstractions, + primarily in the onion_helper functions. Closes ticket 30889. + - Rework bootstrap tracking to use the new publish-subscribe + subsystem. Closes ticket 29976. + - Rewrite format_node_description() and router_get_verbose_nickname() + to use strlcpy() and strlcat(). The previous implementation used + memcpy() and pointer arithmetic, which was error-prone. Closes + ticket 31545. This is CID 1452819. + - Split extrainfo_dump_to_string() into smaller functions. Closes + ticket 30956. + - Use the ptrdiff_t type consistently for expressing variable + offsets and pointer differences. Previously we incorrectly (but + harmlessly) used int and sometimes off_t for these cases. Closes + ticket 31532. + - Use the subsystems mechanism to manage the main event loop code. + Closes ticket 30806. + - Various simplifications and minor improvements to the circuit + padding machines. Patch by Tobias Pulls. Closes tickets 31112 + and 31098. + + o Documentation (hard-coded directories): + - Improve the documentation for the DirAuthority and FallbackDir + torrc options. Closes ticket 30955. + + o Documentation (tor.1 man page): + - Fix typo in tor.1 man page: the option is "--help", not "-help". + Fixes bug 31008; bugfix on 0.2.2.9-alpha. + + +Changes in version 0.4.1.5 - 2019-08-20 + This is the first stable release in the 0.4.1.x series. This series + adds experimental circuit-level padding, authenticated SENDME cells to + defend against certain attacks, and several performance improvements + to save on CPU consumption. It fixes bugs in bootstrapping and v3 + onion services. It also includes numerous smaller features and + bugfixes on earlier versions. + + Per our support policy, we will support the 0.4.1.x series for nine + months, or until three months after the release of a stable 0.4.2.x: + whichever is longer. If you need longer-term support, please stick + with 0.3.5.x, which will we plan to support until Feb 2022. + + Below are the changes since 0.4.1.4-rc. For a complete list of changes + since 0.4.0.5, see the ReleaseNotes file. + + o Directory authority changes: + - The directory authority "dizum" has a new IP address. Closes + ticket 31406. + + o Minor features (circuit padding logging): + - Demote noisy client-side warn logs about circuit padding to + protocol warnings. Add additional log messages and circuit ID + fields to help with bug 30992 and any other future issues. + + o Minor bugfixes (circuit padding negotiation): + - Bump the circuit padding protocol version to explicitly signify + that the HS setup machine support is finalized in 0.4.1.x-stable. + This also means that 0.4.1.x-alpha clients will not negotiate + padding with 0.4.1.x-stable relays, and 0.4.1.x-stable clients + will not negotiate padding with 0.4.1.x-alpha relays (or 0.4.0.x + relays). Fixes bug 31356; bugfix on 0.4.1.1-alpha. + + o Minor bugfixes (circuit padding): + - Ignore non-padding cells on padding circuits. This addresses + various warning messages from subsystems that were not expecting + padding circuits. Fixes bug 30942; bugfix on 0.4.1.1-alpha. + + o Minor bugfixes (clock skew detection): + - Don't believe clock skew results from NETINFO cells that appear to + arrive before we sent the VERSIONS cells they are responding to. + Previously, we would accept them up to 3 minutes "in the past". + Fixes bug 31343; bugfix on 0.2.4.4-alpha. + + o Minor bugfixes (compatibility, standards compliance): + - Fix a bug that would invoke undefined behavior on certain + operating systems when trying to asprintf() a string exactly + INT_MAX bytes long. We don't believe this is exploitable, but it's + better to fix it anyway. Fixes bug 31001; bugfix on 0.2.2.11-alpha. + Found and fixed by Tobias Stoeckmann. + + o Minor bugfixes (compilation warning): + - Fix a compilation warning on Windows about casting a function + pointer for GetTickCount64(). Fixes bug 31374; bugfix + on 0.2.9.1-alpha. + + o Minor bugfixes (compilation): + - Avoid using labs() on time_t, which can cause compilation warnings + on 64-bit Windows builds. Fixes bug 31343; bugfix on 0.2.4.4-alpha. + + o Minor bugfixes (distribution): + - Do not ship any temporary files found in the + scripts/maint/practracker directory. Fixes bug 31311; bugfix + on 0.4.1.1-alpha. + + o Testing (continuous integration): + - In Travis, make stem log a controller trace to the console, and + tail stem's tor log after failure. Closes ticket 30591. + - In Travis, only run the stem tests that use a tor binary. Closes + ticket 30694. + + +Changes in version 0.4.1.4-rc - 2019-07-25 + Tor 0.4.1.4-rc fixes a few bugs from previous versions of Tor, and + updates to a new list of fallback directories. If no new bugs are + found, the next release in the 0.4.1.x serious should be stable. + + o Major bugfixes (circuit build, guard): + - When considering upgrading circuits from "waiting for guard" to + "open", always ignore circuits that are marked for close. Otherwise, + we can end up in the situation where a subsystem is notified that + a closing circuit has just opened, leading to undesirable + behavior. Fixes bug 30871; bugfix on 0.3.0.1-alpha. + + o Minor features (continuous integration): + - Our Travis configuration now uses Chutney to run some network + integration tests automatically. Closes ticket 29280. + + o Minor features (fallback directory list): + - Replace the 157 fallbacks originally introduced in Tor 0.3.5.6-rc + in December 2018 (of which ~122 were still functional), with a + list of 148 fallbacks (70 new, 78 existing, 79 removed) generated + in June 2019. Closes ticket 28795. + + o Minor bugfixes (circuit padding): + - On relays, properly check that a padding machine is absent before + logging a warning about it being absent. Fixes bug 30649; bugfix + on 0.4.0.1-alpha. + - Add two NULL checks in unreachable places to silence Coverity (CID + 144729 and 1447291) and better future-proof ourselves. Fixes bug + 31024; bugfix on 0.4.1.1-alpha. + + o Minor bugfixes (crash on exit): + - Avoid a set of possible code paths that could try to use freed + memory in routerlist_free() while Tor was exiting. Fixes bug + 31003; bugfix on 0.1.2.2-alpha. + + o Minor bugfixes (logging): + - Fix a conflict between the flag used for messaging-domain log + messages, and the LD_NO_MOCK testing flag. Fixes bug 31080; bugfix + on 0.4.1.1-alpha. + + o Minor bugfixes (memory leaks): + - Fix a trivial memory leak when parsing an invalid value from a + download schedule in the configuration. Fixes bug 30894; bugfix + on 0.3.4.1-alpha. + + o Code simplification and refactoring: + - Remove some dead code from circpad_machine_remove_token() to fix + some Coverity warnings (CID 1447298). Fixes bug 31027; bugfix + on 0.4.1.1-alpha. + + +Changes in version 0.4.1.3-alpha - 2019-06-25 + Tor 0.4.1.3-alpha resolves numerous bugs left over from the previous + alpha, most of them from earlier release series. + + o Major bugfixes (Onion service reachability): + - Properly clean up the introduction point map when circuits change + purpose from onion service circuits to pathbias, measurement, or + other circuit types. This should fix some service-side instances + of introduction point failure. Fixes bug 29034; bugfix + on 0.3.2.1-alpha. + + o Minor features (geoip): + - Update geoip and geoip6 to the June 10 2019 Maxmind GeoLite2 + Country database. Closes ticket 30852. + + o Minor features (logging): + - Give a more useful assertion failure message if we think we have + minherit() but we fail to make a region non-inheritable. Give a + compile-time warning if our support for minherit() is incomplete. + Closes ticket 30686. + + o Minor bugfixes (circuit isolation): + - Fix a logic error that prevented the SessionGroup sub-option from + being accepted. Fixes bug 22619; bugfix on 0.2.7.2-alpha. + + o Minor bugfixes (continuous integration): + - Allow the test-stem job to fail in Travis, because it sometimes + hangs. Fixes bug 30744; bugfix on 0.3.5.4-alpha. + - Skip test_rebind on macOS in Travis, because it is unreliable on + macOS on Travis. Fixes bug 30713; bugfix on 0.3.5.1-alpha. + - Skip test_rebind when the TOR_SKIP_TEST_REBIND environment + variable is set. Fixes bug 30713; bugfix on 0.3.5.1-alpha. + + o Minor bugfixes (directory authorities): + - Stop crashing after parsing an unknown descriptor purpose + annotation. We think this bug can only be triggered by modifying a + local file. Fixes bug 30781; bugfix on 0.2.0.8-alpha. + + o Minor bugfixes (pluggable transports): + - When running as a bridge with pluggable transports, always publish + pluggable transport information in our extrainfo descriptor, even + if ExtraInfoStatistics is 0. This information is needed by + BridgeDB. Fixes bug 30956; bugfix on 0.4.1.1-alpha. + + o Documentation: + - Mention URLs for Travis/Appveyor/Jenkins in ReleasingTor.md. + Closes ticket 30630. + + Changes in version 0.4.1.2-alpha - 2019-06-06 Tor 0.4.1.2-alpha resolves numerous bugs--some of them from the previous alpha, and some much older. It also contains minor testing @@ -134,7 +921,7 @@ Changes in version 0.4.1.1-alpha - 2019-05-22 circuits. This feature is only enabled when also supported by the circuit's middle node. (Clients may specify fixed middle nodes with the MiddleNodes option, and may force-disable this feature - with the CircuitPadding torrc.) Closes ticket 28634. + with the CircuitPadding option.) Closes ticket 28634. o Major features (code organization): - Tor now includes a generic publish-subscribe message-passing @@ -318,7 +1105,7 @@ Changes in version 0.4.1.1-alpha - 2019-05-22 o Minor bugfixes (directory authority, ipv6): - Directory authorities with IPv6 support now always mark themselves - as reachable via IPv6. Fixes bug 24338; bugfix on 0.4.0.2-alpha. + as reachable via IPv6. Fixes bug 24338; bugfix on 0.2.4.1-alpha. Patch by Neel Chauhan. o Minor bugfixes (documentation): @@ -356,7 +1143,7 @@ Changes in version 0.4.1.1-alpha - 2019-05-22 Neel Chauhan. - When relaunching a circuit to a rendezvous service, mark the circuit as needing high-uptime routers as appropriate. Fixes bug - 17357; bugfix on 0.4.0.2-alpha. Patch by Neel Chauhan. + 17357; bugfix on 0.1.0.1-rc. Patch by Neel Chauhan. - Stop ignoring IPv6 link specifiers sent to v3 onion services. (IPv6 support for v3 onion services is still incomplete: see ticket 23493 for details.) Fixes bug 23588; bugfix on @@ -1290,7 +2077,7 @@ Changes in version 0.4.0.1-alpha - 2019-01-18 we had added up the sum of all nodes with a descriptor, but that could cause us to build failing circuits when we had either too many bridges or not enough guard nodes. Fixes bug 25885; bugfix on - 0.3.6.1-alpha. Patch by Neel Chauhan. + 0.2.3.1-alpha. Patch by Neel Chauhan. o Minor bugfixes (IPv6): - Fix tor_ersatz_socketpair on IPv6-only systems. Previously, the @@ -4513,7 +5300,7 @@ Changes in version 0.3.4.1-alpha - 2018-05-17 - Remove duplicate code in parse_{c,s}method_line and bootstrap their functionalities into a single function. Fixes bug 6236; bugfix on 0.2.3.6-alpha. - - We remove the PortForwsrding and PortForwardingHelper options, + - We remove the PortForwarding and PortForwardingHelper options, related functions, and the port_forwarding tests. These options were used by the now-deprecated Vidalia to help ordinary users become Tor relays or bridges. Closes ticket 25409. Patch by @@ -11115,7 +11902,7 @@ Changes in version 0.2.9.3-alpha - 2016-09-23 HiddenServiceNonAnonymousMode options. When both are set to 1, every hidden service on a Tor instance becomes a non-anonymous Single Onion Service. Single Onions make one-hop (direct) - connections to their introduction and renzedvous points. One-hop + connections to their introduction and rendezvous points. One-hop circuits make Single Onion servers easily locatable, but clients remain location-anonymous. This is compatible with the existing hidden service implementation, and works on the current tor @@ -11618,13 +12405,12 @@ Changes in version 0.2.9.1-alpha - 2016-08-08 needless crash bugs. Closes ticket 18613. o Minor features (performance): - - Changer the "optimistic data" extension from "off by default" to + - Change the "optimistic data" extension from "off by default" to "on by default". The default was ordinarily overridden by a consensus option, but when clients were bootstrapping for the first time, they would not have a consensus to get the option - from. Changing this default When fetching a consensus for the - first time, use optimistic data. This saves a round-trip during - startup. Closes ticket 18815. + from. Changing this default saves a round-trip during startup. + Closes ticket 18815. o Minor features (relay, usability): - When the directory authorities refuse a bad relay's descriptor, @@ -9,7 +9,8 @@ there may be other license terms that you should be aware of. =============================================================================== -Tor is distributed under this license: +Tor is distributed under the "3-clause BSD" license, a commonly used +software license that means Tor is both free software and open source: Copyright (c) 2001-2004, Roger Dingledine Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson diff --git a/Makefile.am b/Makefile.am index 3cac5f5331..03593df161 100644 --- a/Makefile.am +++ b/Makefile.am @@ -41,6 +41,7 @@ TOR_UTIL_LIBS = \ src/lib/libtor-geoip.a \ src/lib/libtor-process.a \ src/lib/libtor-buf.a \ + src/lib/libtor-confmgt.a \ src/lib/libtor-pubsub.a \ src/lib/libtor-dispatch.a \ src/lib/libtor-time.a \ @@ -74,6 +75,7 @@ TOR_UTIL_TESTING_LIBS = \ src/lib/libtor-geoip-testing.a \ src/lib/libtor-process-testing.a \ src/lib/libtor-buf-testing.a \ + src/lib/libtor-confmgt-testing.a \ src/lib/libtor-pubsub-testing.a \ src/lib/libtor-dispatch-testing.a \ src/lib/libtor-time-testing.a \ @@ -166,10 +168,26 @@ EXTRA_DIST+= \ ReleaseNotes \ scripts/maint/checkIncludes.py \ scripts/maint/checkSpace.pl \ + scripts/maint/checkShellScripts.sh \ + scripts/maint/practracker/README \ scripts/maint/practracker/exceptions.txt \ + scripts/maint/practracker/includes.py \ scripts/maint/practracker/metrics.py \ scripts/maint/practracker/practracker.py \ + scripts/maint/practracker/practracker_tests.py \ scripts/maint/practracker/problem.py \ + scripts/maint/practracker/testdata/.may_include \ + scripts/maint/practracker/testdata/a.c \ + scripts/maint/practracker/testdata/b.c \ + scripts/maint/practracker/testdata/ex0-expected.txt \ + scripts/maint/practracker/testdata/ex0.txt \ + scripts/maint/practracker/testdata/ex1-expected.txt \ + scripts/maint/practracker/testdata/ex1.txt \ + scripts/maint/practracker/testdata/ex1-overbroad-expected.txt \ + scripts/maint/practracker/testdata/ex.txt \ + scripts/maint/practracker/testdata/header.h \ + scripts/maint/practracker/testdata/not_c_file \ + scripts/maint/practracker/test_practracker.sh \ scripts/maint/practracker/util.py ## This tells etags how to find mockable function definitions. @@ -188,7 +206,7 @@ TEST_CFLAGS= TEST_CPPFLAGS=-DTOR_UNIT_TESTS @TOR_MODULES_ALL_ENABLED@ TEST_NETWORK_FLAGS=--hs-multi-client 1 endif -TEST_NETWORK_WARNING_FLAGS=--quiet --only-warnings +TEST_NETWORK_SHOW_WARNINGS_FOR_LAST_RUN_FLAGS=--quiet --only-warnings if LIBFUZZER_ENABLED TEST_CFLAGS += -fsanitize-coverage=trace-pc-guard,trace-cmp,trace-div @@ -225,15 +243,9 @@ test: all $(top_builddir)/src/test/test shellcheck: - # Only use shellcheck if it is present - if command -v shellcheck; then \ - find $(top_srcdir)/scripts/ -name "*.sh" -exec shellcheck {} +; \ - if [ -d "$(top_srcdir)/scripts/test" ]; then \ - shellcheck $(top_srcdir)/scripts/test/cov-diff $(top_srcdir)/scripts/test/coverage; \ - fi; \ - fi + $(top_srcdir)/scripts/maint/checkShellScripts.sh -check-local: check-spaces check-changes check-includes shellcheck +check-local: check-spaces check-changes check-includes check-best-practices shellcheck need-chutney-path: @if test ! -d "$$CHUTNEY_PATH"; then \ @@ -253,12 +265,15 @@ test-network: need-chutney-path $(TESTING_TOR_BINARY) src/tools/tor-gencert $(top_srcdir)/src/test/test-network.sh $(TEST_NETWORK_FLAGS) # Run all available tests using automake's test-driver -# only run IPv6 tests if we can ping6 ::1 (localhost) -# only run IPv6 tests if we can ping ::1 (localhost) -# some IPv6 tests will fail without an IPv6 DNS server (see #16971 and #17011) -# only run mixed tests if we have a tor-stable binary -# Try the syntax for BSD ping6, Linux ping6, and Linux ping -6, -# because they're incompatible +# - only run IPv6 tests if we can ping6 or ping -6 ::1 (localhost) +# we try the syntax for BSD ping6, Linux ping6, and Linux ping -6, +# because they're incompatible +# - some IPv6 tests may fail without an IPv6 DNS server +# (see #16971 and #17011) +# - only run mixed tests if we have a tor-stable binary +# - show tor warnings on the console after each network run +# (otherwise, warnings go to the logs, and people don't see them unless +# there is a network failure) test-network-all: need-chutney-path test-driver $(TESTING_TOR_BINARY) src/tools/tor-gencert mkdir -p $(TEST_NETWORK_ALL_LOG_DIR) rm -f $(TEST_NETWORK_ALL_LOG_DIR)/*.log $(TEST_NETWORK_ALL_LOG_DIR)/*.trs @@ -282,7 +297,7 @@ test-network-all: need-chutney-path test-driver $(TESTING_TOR_BINARY) src/tools/ done; \ for f in $$flavors; do \ $(SHELL) $(top_srcdir)/test-driver --test-name $$f --log-file $(TEST_NETWORK_ALL_LOG_DIR)/$$f.log --trs-file $(TEST_NETWORK_ALL_LOG_DIR)/$$f.trs $(TEST_NETWORK_ALL_DRIVER_FLAGS) $(top_srcdir)/src/test/test-network.sh --flavor $$f $(TEST_NETWORK_FLAGS); \ - $(top_srcdir)/src/test/test-network.sh $(TEST_NETWORK_WARNING_FLAGS); \ + $(top_srcdir)/src/test/test-network.sh $(TEST_NETWORK_SHOW_WARNINGS_FOR_LAST_RUN_FLAGS); \ done; \ echo "Log and result files are available in $(TEST_NETWORK_ALL_LOG_DIR)."; \ ! grep -q FAIL $(TEST_NETWORK_ALL_LOG_DIR)/*.trs @@ -355,12 +370,12 @@ endif check-includes: if USEPYTHON - $(PYTHON) $(top_srcdir)/scripts/maint/checkIncludes.py + $(PYTHON) $(top_srcdir)/scripts/maint/practracker/includes.py $(top_srcdir) endif check-best-practices: if USEPYTHON - $(PYTHON) $(top_srcdir)/scripts/maint/practracker/practracker.py $(top_srcdir) + @$(PYTHON) $(top_srcdir)/scripts/maint/practracker/practracker.py $(top_srcdir) $(TOR_PRACTRACKER_OPTIONS) endif practracker-regen: @@ -463,7 +478,7 @@ version: .PHONY: autostyle-ifdefs autostyle-ifdefs: - $(PYTHON) scripts/maint/annotate_ifdef_directives $(OWNED_TOR_C_FILES) + $(PYTHON) scripts/maint/annotate_ifdef_directives.py $(OWNED_TOR_C_FILES) .PHONY: autostyle-ifdefs autostyle-operators: diff --git a/ReleaseNotes b/ReleaseNotes index 788804236b..ad24efd606 100644 --- a/ReleaseNotes +++ b/ReleaseNotes @@ -2,6 +2,639 @@ This document summarizes new features and bugfixes in each stable release of Tor. If you want to see more detailed descriptions of the changes in each development snapshot, see the ChangeLog file. +Changes in version 0.4.1.6 - 2019-09-19 + This release backports several bugfixes to improve stability and + correctness. Anyone experiencing build problems or crashes with 0.4.1.5, + or experiencing reliability issues with single onion services, should + upgrade. + + o Major bugfixes (crash, Linux, Android, backport from 0.4.2.1-alpha): + - Tolerate systems (including some Android installations) where + madvise and MADV_DONTDUMP are available at build-time, but not at + run time. Previously, these systems would notice a failed syscall + and abort. Fixes bug 31570; bugfix on 0.4.1.1-alpha. + - Tolerate systems (including some Linux installations) where + madvise and/or MADV_DONTFORK are available at build-time, but not + at run time. Previously, these systems would notice a failed + syscall and abort. Fixes bug 31696; bugfix on 0.4.1.1-alpha. + + o Minor features (stem tests, backport from 0.4.2.1-alpha): + - Change "make test-stem" so it only runs the stem tests that use + tor. This change makes test-stem faster and more reliable. Closes + ticket 31554. + + o Minor bugfixes (build system, backport form 0.4.2.1-alpha): + - Do not include the deprecated <sys/sysctl.h> on Linux or Windows + systems. Fixes bug 31673; bugfix on 0.2.5.4-alpha. + + o Minor bugfixes (compilation, backport from 0.4.2.1-alpha): + - Add more stub functions to fix compilation on Android with link- + time optimization when --disable-module-dirauth is used. + Previously, these compilation settings would make the compiler + look for functions that didn't exist. Fixes bug 31552; bugfix + on 0.4.1.1-alpha. + - Suppress spurious float-conversion warnings from GCC when calling + floating-point classifier functions on FreeBSD. Fixes part of bug + 31687; bugfix on 0.3.1.5-alpha. + + o Minor bugfixes (controller protocol): + - Fix the MAPADDRESS controller command to accept one or more + arguments. Previously, it required two or more arguments, and ignored + the first. Fixes bug 31772; bugfix on 0.4.1.1-alpha. + + o Minor bugfixes (guards, backport from 0.4.2.1-alpha): + - When tor is missing descriptors for some primary entry guards, + make the log message less alarming. It's normal for descriptors to + expire, as long as tor fetches new ones soon after. Fixes bug + 31657; bugfix on 0.3.3.1-alpha. + + o Minor bugfixes (logging, backport from 0.4.2.1-alpha): + - Change log level of message "Hash of session info was not as + expected" to LOG_PROTOCOL_WARN. Fixes bug 12399; bugfix + on 0.1.1.10-alpha. + + o Minor bugfixes (rust, backport from 0.4.2.1-alpha): + - Correctly exclude a redundant rust build job in Travis. Fixes bug + 31463; bugfix on 0.3.5.4-alpha. + + o Minor bugfixes (v2 single onion services, backport from 0.4.2.1-alpha): + - Always retry v2 single onion service intro and rend circuits with + a 3-hop path. Previously, v2 single onion services used a 3-hop + path when rendezvous circuits were retried after a remote or + delayed failure, but a 1-hop path for immediate retries. Fixes bug + 23818; bugfix on 0.2.9.3-alpha. + + o Minor bugfixes (v3 single onion services, backport from 0.4.2.1-alpha): + - Always retry v3 single onion service intro and rend circuits with + a 3-hop path. Previously, v3 single onion services used a 3-hop + path when rend circuits were retried after a remote or delayed + failure, but a 1-hop path for immediate retries. Fixes bug 23818; + bugfix on 0.3.2.1-alpha. + - Make v3 single onion services fall back to a 3-hop intro, when all + intro points are unreachable via a 1-hop path. Previously, v3 + single onion services failed when all intro nodes were unreachable + via a 1-hop path. Fixes bug 23507; bugfix on 0.3.2.1-alpha. + + o Documentation (backport from 0.4.2.1-alpha): + - Use RFC 2397 data URL scheme to embed an image into tor-exit- + notice.html so that operators no longer have to host it + themselves. Closes ticket 31089. + + +Changes in version 0.4.1.5 - 2019-08-20 + This is the first stable release in the 0.4.1.x series. This series + adds experimental circuit-level padding, authenticated SENDME cells to + defend against certain attacks, and several performance improvements + to save on CPU consumption. It fixes bugs in bootstrapping and v3 + onion services. It also includes numerous smaller features and + bugfixes on earlier versions. + + Per our support policy, we will support the 0.4.1.x series for nine + months, or until three months after the release of a stable 0.4.2.x: + whichever is longer. If you need longer-term support, please stick + with 0.3.5.x, which will we plan to support until Feb 2022. + + Below are the changes since 0.4.0.5. For a list of only the changes + since 0.4.1.4-rc, see the ChangeLog file. + + o Directory authority changes: + - The directory authority "dizum" has a new IP address. Closes + ticket 31406. + + o Major features (circuit padding): + - Onion service clients now add padding cells at the start of their + INTRODUCE and RENDEZVOUS circuits, to make those circuits' traffic + look more like general purpose Exit traffic. The overhead for this + is 2 extra cells in each direction for RENDEZVOUS circuits, and 1 + extra upstream cell and 10 downstream cells for INTRODUCE + circuits. This feature is only enabled when also supported by the + circuit's middle node. (Clients may specify fixed middle nodes + with the MiddleNodes option, and may force-disable this feature + with the CircuitPadding option.) Closes ticket 28634. + + o Major features (code organization): + - Tor now includes a generic publish-subscribe message-passing + subsystem that we can use to organize intermodule dependencies. We + hope to use this to reduce dependencies between modules that don't + need to be related, and to generally simplify our codebase. Closes + ticket 28226. + + o Major features (controller protocol): + - Controller commands are now parsed using a generalized parsing + subsystem. Previously, each controller command was responsible for + parsing its own input, which led to strange inconsistencies. + Closes ticket 30091. + + o Major features (flow control): + - Implement authenticated SENDMEs as detailed in proposal 289. A + SENDME cell now includes the digest of the traffic that it + acknowledges, so that once an end point receives the SENDME, it + can confirm the other side's knowledge of the previous cells that + were sent, and prevent certain types of denial-of-service attacks. + This behavior is controlled by two new consensus parameters: see + the proposal for more details. Fixes ticket 26288. + + o Major features (performance): + - Our node selection algorithm now excludes nodes in linear time. + Previously, the algorithm was quadratic, which could slow down + heavily used onion services. Closes ticket 30307. + + o Major features (performance, RNG): + - Tor now constructs a fast secure pseudorandom number generator for + each thread, to use when performance is critical. This PRNG is + based on AES-CTR, using a buffering construction similar to + libottery and the (newer) OpenBSD arc4random() code. It + outperforms OpenSSL 1.1.1a's CSPRNG by roughly a factor of 100 for + small outputs. Although we believe it to be cryptographically + strong, we are only using it when necessary for performance. + Implements tickets 29023 and 29536. + + o Major bugfixes (bridges): + - Consider our directory information to have changed when our list + of bridges changes. Previously, Tor would not re-compute the + status of its directory information when bridges changed, and + therefore would not realize that it was no longer able to build + circuits. Fixes part of bug 29875. + - Do not count previously configured working bridges towards our + total of working bridges. Previously, when Tor's list of bridges + changed, it would think that the old bridges were still usable, + and delay fetching router descriptors for the new ones. Fixes part + of bug 29875; bugfix on 0.3.0.1-alpha. + + o Major bugfixes (circuit build, guard): + - On relays, properly check that a padding machine is absent before + logging a warning about it being absent. Fixes bug 30649; bugfix + on 0.4.0.1-alpha. + - When considering upgrading circuits from "waiting for guard" to + "open", always ignore circuits that are marked for close. Otherwise, + we can end up in the situation where a subsystem is notified that + a closing circuit has just opened, leading to undesirable + behavior. Fixes bug 30871; bugfix on 0.3.0.1-alpha. + + o Major bugfixes (onion service reachability): + - Properly clean up the introduction point map when circuits change + purpose from onion service circuits to pathbias, measurement, or + other circuit types. This should fix some service-side instances + of introduction point failure. Fixes bug 29034; bugfix + on 0.3.2.1-alpha. + + o Major bugfixes (onion service v3): + - Fix an unreachable bug in which an introduction point could try to + send an INTRODUCE_ACK with a status code that Trunnel would refuse + to encode, leading the relay to assert(). We've consolidated the + ABI values into Trunnel now. Fixes bug 30454; bugfix + on 0.3.0.1-alpha. + - Clients can now handle unknown status codes from INTRODUCE_ACK + cells. (The NACK behavior will stay the same.) This will allow us + to extend status codes in the future without breaking the normal + client behavior. Fixes another part of bug 30454; bugfix + on 0.3.0.1-alpha. + + o Minor features (authenticated SENDME): + - Ensure that there is enough randomness on every circuit to prevent + an attacker from successfully predicting the hashes they will need + to include in authenticated SENDME cells. At a random interval, if + we have not sent randomness already, we now leave some extra space + at the end of a cell that we can fill with random bytes. Closes + ticket 26846. + + o Minor features (circuit padding logging): + - Demote noisy client-side warn logs about circuit padding to protocol + warnings. Add additional log messages and circuit ID fields to help + with bug 30992 and any other future issues. + + o Minor features (circuit padding): + - We now use a fast PRNG when scheduling circuit padding. Part of + ticket 28636. + - Allow the padding machine designer to pick the edges of their + histogram instead of trying to compute them automatically using an + exponential formula. Resolves some undefined behavior in the case + of small histograms and allows greater flexibility on machine + design. Closes ticket 29298; bugfix on 0.4.0.1-alpha. + - Allow circuit padding machines to hold a circuit open until they + are done padding it. Closes ticket 28780. + + o Minor features (compile-time modules): + - Add a "--list-modules" command to print a list of which compile- + time modules are enabled. Closes ticket 30452. + + o Minor features (continuous integration): + - Our Travis configuration now uses Chutney to run some network + integration tests automatically. Closes ticket 29280. + - When running coverage builds on Travis, we now set + TOR_TEST_RNG_SEED, to avoid RNG-based coverage differences. Part + of ticket 28878. + - Remove sudo configuration lines from .travis.yml as they are no + longer needed with current Travis build environment. Resolves + issue 30213. + - In Travis, show stem's tor log after failure. Closes ticket 30234. + + o Minor features (controller): + - Add onion service version 3 support to the HSFETCH command. + Previously, only version 2 onion services were supported. Closes + ticket 25417. Patch by Neel Chauhan. + + o Minor features (debugging): + - Introduce tor_assertf() and tor_assertf_nonfatal() to enable + logging of additional information during assert failure. Now we + can use format strings to include information for trouble + shooting. Resolves ticket 29662. + + o Minor features (defense in depth): + - In smartlist_remove_keeporder(), set unused pointers to NULL, in + case a bug causes them to be used later. Closes ticket 30176. + Patch from Tobias Stoeckmann. + - Tor now uses a cryptographically strong PRNG even for decisions + that we do not believe are security-sensitive. Previously, for + performance reasons, we had used a trivially predictable linear + congruential generator algorithm for certain load-balancing and + statistical sampling decisions. Now we use our fast RNG in those + cases. Closes ticket 29542. + + o Minor features (developer tools): + - Tor's "practracker" test script now checks for files and functions + that seem too long and complicated. Existing overlong functions + and files are accepted for now, but should eventually be + refactored. Closes ticket 29221. + - Add some scripts used for git maintenance to scripts/git. Closes + ticket 29391. + - Call practracker from pre-push and pre-commit git hooks to let + developers know if they made any code style violations. Closes + ticket 30051. + - Add a script to check that each header has a well-formed and + unique guard macro. Closes ticket 29756. + + o Minor features (fallback directory list): + - Replace the 157 fallbacks originally introduced in Tor 0.3.5.6-rc + in December 2018 (of which ~122 were still functional), with a + list of 148 fallbacks (70 new, 78 existing, 79 removed) generated + in June 2019. Closes ticket 28795. + + o Minor features (geoip): + - Update geoip and geoip6 to the June 10 2019 Maxmind GeoLite2 + Country database. Closes ticket 30852. + - Update geoip and geoip6 to the May 13 2019 Maxmind GeoLite2 + Country database. Closes ticket 30522. + + o Minor features (HTTP tunnel): + - Return an informative web page when the HTTPTunnelPort is used as + an HTTP proxy. Closes ticket 27821, patch by "eighthave". + + o Minor features (IPv6, v3 onion services): + - Make v3 onion services put IPv6 addresses in service descriptors. + Before this change, service descriptors only contained IPv4 + addresses. Implements 26992. + + o Minor features (logging): + - Give a more useful assertion failure message if we think we have + minherit() but we fail to make a region non-inheritable. Give a + compile-time warning if our support for minherit() is incomplete. + Closes ticket 30686. + + o Minor features (maintenance): + - Add a new "make autostyle" target that developers can use to apply + all automatic Tor style and consistency conversions to the + codebase. Closes ticket 30539. + + o Minor features (modularity): + - The "--disable-module-dirauth" compile-time option now disables + even more dirauth-only code. Closes ticket 30345. + + o Minor features (performance): + - Use OpenSSL's implementations of SHA3 when available (in OpenSSL + 1.1.1 and later), since they tend to be faster than tiny-keccak. + Closes ticket 28837. + + o Minor features (testing): + - The circuitpadding tests now use a reproducible RNG implementation, + so that if a test fails, we can learn why. Part of ticket 28878. + - Tor's tests now support an environment variable, TOR_TEST_RNG_SEED, + to set the RNG seed for tests that use a reproducible RNG. Part of + ticket 28878. + - When running tests in coverage mode, take additional care to make + our coverage deterministic, so that we can accurately track + changes in code coverage. Closes ticket 30519. + - Tor's unit test code now contains helper functions to replace the + PRNG with a deterministic or reproducible version for testing. + Previously, various tests implemented this in various ways. + Implements ticket 29732. + - We now have a script, cov-test-determinism.sh, to identify places + where our unit test coverage has become nondeterministic. Closes + ticket 29436. + - Check that representative subsets of values of `int` and `unsigned + int` can be represented by `void *`. Resolves issue 29537. + + o Minor bugfixes (bridge authority): + - Bridge authorities now set bridges as running or non-running when + about to dump their status to a file. Previously, they set bridges + as running in response to a GETINFO command, but those shouldn't + modify data structures. Fixes bug 24490; bugfix on 0.2.0.13-alpha. + Patch by Neel Chauhan. + + o Minor bugfixes (channel padding statistics): + - Channel padding write totals and padding-enabled totals are now + counted properly in relay extrainfo descriptors. Fixes bug 29231; + bugfix on 0.3.1.1-alpha. + + o Minor bugfixes (circuit isolation): + - Fix a logic error that prevented the SessionGroup sub-option from + being accepted. Fixes bug 22619; bugfix on 0.2.7.2-alpha. + + o Minor bugfixes (circuit padding): + - Add a "CircuitPadding" torrc option to disable circuit padding. + Fixes bug 28693; bugfix on 0.4.0.1-alpha. + - Allow circuit padding machines to specify that they do not + contribute much overhead, and provide consensus flags and torrc + options to force clients to only use these low overhead machines. + Fixes bug 29203; bugfix on 0.4.0.1-alpha. + - Provide a consensus parameter to fully disable circuit padding, to + be used in emergency network overload situations. Fixes bug 30173; + bugfix on 0.4.0.1-alpha. + - The circuit padding subsystem will no longer schedule padding if + dormant mode is enabled. Fixes bug 28636; bugfix on 0.4.0.1-alpha. + - Inspect a circuit-level cell queue before sending padding, to + avoid sending padding while too much data is already queued. Fixes + bug 29204; bugfix on 0.4.0.1-alpha. + - Avoid calling monotime_absolute_usec() in circuit padding machines + that do not use token removal or circuit RTT estimation. Fixes bug + 29085; bugfix on 0.4.0.1-alpha. + + o Minor bugfixes (clock skew detection): + - Don't believe clock skew results from NETINFO cells that appear to + arrive before we sent the VERSIONS cells they are responding to. + Previously, we would accept them up to 3 minutes "in the past". + Fixes bug 31343; bugfix on 0.2.4.4-alpha. + + o Minor bugfixes (compatibility, standards compliance): + - Fix a bug that would invoke undefined behavior on certain + operating systems when trying to asprintf() a string exactly + INT_MAX bytes long. We don't believe this is exploitable, but it's + better to fix it anyway. Fixes bug 31001; bugfix on 0.2.2.11-alpha. + Found and fixed by Tobias Stoeckmann. + + o Minor bugfixes (compilation warning): + - Fix a compilation warning on Windows about casting a function + pointer for GetTickCount64(). Fixes bug 31374; bugfix on + 0.2.9.1-alpha. + + o Minor bugfixes (compilation): + - Avoid using labs() on time_t, which can cause compilation warnings + on 64-bit Windows builds. Fixes bug 31343; bugfix on 0.2.4.4-alpha. + + o Minor bugfixes (compilation, unusual configurations): + - Avoid failures when building with the ALL_BUGS_ARE_FATAL option + due to missing declarations of abort(), and prevent other such + failures in the future. Fixes bug 30189; bugfix on 0.3.4.1-alpha. + + o Minor bugfixes (configuration, proxies): + - Fix a bug that prevented us from supporting SOCKS5 proxies that + want authentication along with configured (but unused!) + ClientTransportPlugins. Fixes bug 29670; bugfix on 0.2.6.1-alpha. + + o Minor bugfixes (continuous integration): + - Allow the test-stem job to fail in Travis, because it sometimes + hangs. Fixes bug 30744; bugfix on 0.3.5.4-alpha. + - Skip test_rebind on macOS in Travis, because it is unreliable on + macOS on Travis. Fixes bug 30713; bugfix on 0.3.5.1-alpha. + - Skip test_rebind when the TOR_SKIP_TEST_REBIND environment + variable is set. Fixes bug 30713; bugfix on 0.3.5.1-alpha. + + o Minor bugfixes (controller protocol): + - Teach the controller parser to distinguish an object preceded by + an argument list from one without. Previously, it couldn't + distinguish an argument list from the first line of a multiline + object. Fixes bug 29984; bugfix on 0.2.3.8-alpha. + + o Minor bugfixes (crash on exit): + - Avoid a set of possible code paths that could try to use freed + memory in routerlist_free() while Tor was exiting. Fixes bug + 31003; bugfix on 0.1.2.2-alpha. + + o Minor bugfixes (developer tooling): + - Fix pre-push hook to allow fixup and squash commits when pushing + to non-upstream git remote. Fixes bug 30286; bugfix + on 0.4.0.1-alpha. + + o Minor bugfixes (directory authorities): + - Stop crashing after parsing an unknown descriptor purpose + annotation. We think this bug can only be triggered by modifying a + local file. Fixes bug 30781; bugfix on 0.2.0.8-alpha. + - Move the "bandwidth-file-headers" line in directory authority + votes so that it conforms to dir-spec.txt. Fixes bug 30316; bugfix + on 0.3.5.1-alpha. + - Directory authorities with IPv6 support now always mark themselves + as reachable via IPv6. Fixes bug 24338; bugfix on 0.2.4.1-alpha. + Patch by Neel Chauhan. + + o Minor bugfixes (documentation): + - Improve the documentation for using MapAddress with ".exit". Fixes + bug 30109; bugfix on 0.1.0.1-rc. + - Improve the monotonic time module and function documentation to + explain what "monotonic" actually means, and document some results + that have surprised people. Fixes bug 29640; bugfix + on 0.2.9.1-alpha. + - Use proper formatting when providing an example on quoting options + that contain whitespace. Fixes bug 29635; bugfix on 0.2.3.18-rc. + + o Minor bugfixes (logging): + - Do not log a warning when running with an OpenSSL version other + than the one Tor was compiled with, if the two versions should be + compatible. Previously, we would warn whenever the version was + different. Fixes bug 30190; bugfix on 0.2.4.2-alpha. + - Warn operators when the MyFamily option is set but ContactInfo is + missing, as the latter should be set too. Fixes bug 25110; bugfix + on 0.3.3.1-alpha. + + o Minor bugfixes (memory leaks): + - Avoid a minor memory leak that could occur on relays when failing + to create a "keys" directory. Fixes bug 30148; bugfix + on 0.3.3.1-alpha. + - Fix a trivial memory leak when parsing an invalid value from a + download schedule in the configuration. Fixes bug 30894; bugfix + on 0.3.4.1-alpha. + + o Minor bugfixes (NetBSD): + - Fix usage of minherit() on NetBSD and other platforms that define + MAP_INHERIT_{ZERO,NONE} instead of INHERIT_{ZERO,NONE}. Fixes bug + 30614; bugfix on 0.4.0.2-alpha. Patch from Taylor Campbell. + + o Minor bugfixes (onion services): + - Avoid a GCC 9.1.1 warning (and possible crash depending on libc + implemenation) when failing to load an onion service client + authorization file. Fixes bug 30475; bugfix on 0.3.5.1-alpha. + - When refusing to launch a controller's HSFETCH request because of + rate-limiting, respond to the controller with a new response, + "QUERY_RATE_LIMITED". Previously, we would log QUERY_NO_HSDIR for + this case. Fixes bug 28269; bugfix on 0.3.1.1-alpha. Patch by + Neel Chauhan. + - When relaunching a circuit to a rendezvous service, mark the + circuit as needing high-uptime routers as appropriate. Fixes bug + 17357; bugfix on 0.1.0.1-rc. Patch by Neel Chauhan. + - Stop ignoring IPv6 link specifiers sent to v3 onion services. + (IPv6 support for v3 onion services is still incomplete: see + ticket 23493 for details.) Fixes bug 23588; bugfix on + 0.3.2.1-alpha. Patch by Neel Chauhan. + + o Minor bugfixes (onion services, performance): + - When building circuits to onion services, call tor_addr_parse() + less often. Previously, we called tor_addr_parse() in + circuit_is_acceptable() even if its output wasn't used. This + change should improve performance when building circuits. Fixes + bug 22210; bugfix on 0.2.8.12. Patch by Neel Chauhan. + + o Minor bugfixes (out-of-memory handler): + - When purging the DNS cache because of an out-of-memory condition, + try purging just the older entries at first. Previously, we would + always purge the whole thing. Fixes bug 29617; bugfix + on 0.3.5.1-alpha. + + o Minor bugfixes (performance): + - When checking whether a node is a bridge, use a fast check to make + sure that its identity is set. Previously, we used a constant-time + check, which is not necessary in this case. Fixes bug 30308; + bugfix on 0.3.5.1-alpha. + + o Minor bugfixes (pluggable transports): + - Tor now sets TOR_PT_EXIT_ON_STDIN_CLOSE=1 for client transports as + well as servers. Fixes bug 25614; bugfix on 0.2.7.1-alpha. + + o Minor bugfixes (portability): + - Avoid crashing in our tor_vasprintf() implementation on systems + that define neither vasprintf() nor _vscprintf(). (This bug has + been here long enough that we question whether people are running + Tor on such systems, but we're applying the fix out of caution.) + Fixes bug 30561; bugfix on 0.2.8.2-alpha. Found and fixed by + Tobias Stoeckmann. + + o Minor bugfixes (probability distributions): + - Refactor and improve parts of the probability distribution code + that made Coverity complain. Fixes bug 29805; bugfix + on 0.4.0.1-alpha. + + o Minor bugfixes (python): + - Stop assuming that /usr/bin/python3 exists. For scripts that work + with python2, use /usr/bin/python. Otherwise, use /usr/bin/env + python3. Fixes bug 29913; bugfix on 0.2.5.3-alpha. + + o Minor bugfixes (relay): + - When running as a relay, if IPv6Exit is set to 1 while ExitRelay + is auto, act as if ExitRelay is 1. Previously, we would ignore + IPv6Exit if ExitRelay was 0 or auto. Fixes bug 29613; bugfix on + 0.3.5.1-alpha. Patch by Neel Chauhan. + + o Minor bugfixes (static analysis): + - Fix several spurious Coverity warnings about the unit tests, to + lower our chances of missing real warnings in the future. Fixes + bug 30150; bugfix on 0.3.5.1-alpha and various other Tor versions. + + o Minor bugfixes (stats): + - When ExtraInfoStatistics is 0, stop including bandwidth usage + statistics, GeoIPFile hashes, ServerTransportPlugin lines, and + bridge statistics by country in extra-info documents. Fixes bug + 29018; bugfix on 0.2.4.1-alpha. + + o Minor bugfixes (testing): + - Call setrlimit() to disable core dumps in test_bt_cl.c. Previously + we used `ulimit -c` in test_bt.sh, which violates POSIX shell + compatibility. Fixes bug 29061; bugfix on 0.3.5.1-alpha. + - Fix some incorrect code in the v3 onion service unit tests. Fixes + bug 29243; bugfix on 0.3.2.1-alpha. + - In the "routerkeys/*" tests, check the return values of mkdir() + for possible failures. Fixes bug 29939; bugfix on 0.2.7.2-alpha. + Found by Coverity as CID 1444254. + - Split test_utils_general() into several smaller test functions. + This makes it easier to perform resource deallocation on assert + failure, and fixes Coverity warnings CID 1444117 and CID 1444118. + Fixes bug 29823; bugfix on 0.2.9.1-alpha. + + o Minor bugfixes (tor-resolve): + - Fix a memory leak in tor-resolve that could happen if Tor gave it + a malformed SOCKS response. (Memory leaks in tor-resolve don't + actually matter, but it's good to fix them anyway.) Fixes bug + 30151; bugfix on 0.4.0.1-alpha. + + o Code simplification and refactoring: + - Abstract out the low-level formatting of replies on the control + port. Implements ticket 30007. + - Add several assertions in an attempt to fix some Coverity + warnings. Closes ticket 30149. + - Introduce a connection_dir_buf_add() helper function that checks + for compress_state of dir_connection_t and automatically writes a + string to directory connection with or without compression. + Resolves issue 28816. + - Make the base32_decode() API return the number of bytes written, + for consistency with base64_decode(). Closes ticket 28913. + - Move most relay-only periodic events out of mainloop.c into the + relay subsystem. Closes ticket 30414. + - Refactor and encapsulate parts of the codebase that manipulate + crypt_path_t objects. Resolves issue 30236. + - Refactor several places in our code that Coverity incorrectly + believed might have memory leaks. Closes ticket 30147. + - Remove redundant return values in crypto_format, and the + associated return value checks elsewhere in the code. Make the + implementations in crypto_format consistent, and remove redundant + code. Resolves ticket 29660. + - Rename tor_mem_is_zero() to fast_mem_is_zero(), to emphasize that + it is not a constant-time function. Closes ticket 30309. + - Replace hs_desc_link_specifier_t with link_specifier_t, and remove + all hs_desc_link_specifier_t-specific code. Fixes bug 22781; + bugfix on 0.3.2.1-alpha. + - Simplify v3 onion service link specifier handling code. Fixes bug + 23576; bugfix on 0.3.2.1-alpha. + - Split crypto_digest.c into NSS code, OpenSSL code, and shared + code. Resolves ticket 29108. + - Split control.c into several submodules, in preparation for + distributing its current responsibilities throughout the codebase. + Closes ticket 29894. + - Start to move responsibility for knowing about periodic events to + the appropriate subsystems, so that the mainloop doesn't need to + know all the periodic events in the rest of the codebase. + Implements tickets 30293 and 30294. + + o Documentation: + - Mention URLs for Travis/Appveyor/Jenkins in ReleasingTor.md. + Closes ticket 30630. + - Document how to find git commits and tags for bug fixes in + CodingStandards.md. Update some file documentation. Closes + ticket 30261. + + o Removed features: + - Remove the linux-tor-prio.sh script from contrib/operator-tools + directory. Resolves issue 29434. + - Remove the obsolete OpenSUSE initscript. Resolves issue 30076. + - Remove the obsolete script at contrib/dist/tor.sh.in. Resolves + issue 30075. + + o Testing: + - Specify torrc paths (with empty files) when launching tor in + integration tests; refrain from reading user and system torrcs. + Resolves issue 29702. + + o Code simplification and refactoring (shell scripts): + - Clean up many of our shell scripts to fix shellcheck warnings. + These include autogen.sh (ticket 26069), test_keygen.sh (ticket + 29062), test_switch_id.sh (ticket 29065), test_rebind.sh (ticket + 29063), src/test/fuzz/minimize.sh (ticket 30079), test_rust.sh + (ticket 29064), torify (ticket 29070), asciidoc-helper.sh (29926), + fuzz_multi.sh (30077), fuzz_static_testcases.sh (ticket 29059), + nagios-check-tor-authority-cert (ticket 29071), + src/test/fuzz/fixup_filenames.sh (ticket 30078), test-network.sh + (ticket 29060), test_key_expiration.sh (ticket 30002), + zero_length_keys.sh (ticket 29068), and test_workqueue_*.sh + (ticket 29067). + + o Testing (chutney): + - In "make test-network-all", test IPv6-only v3 single onion + services, using the chutney network single-onion-v23-ipv6-md. + Closes ticket 27251. + + o Testing (continuous integration): + - In Travis, make stem log a controller trace to the console, and tail + stem's tor log after failure. Closes ticket 30591. + - In Travis, only run the stem tests that use a tor binary. + Closes ticket 30694. + + Changes in version 0.4.0.5 - 2019-05-02 This is the first stable release in the 0.4.0.x series. It contains improvements for power management and bootstrap reporting, as well as @@ -360,7 +993,7 @@ Changes in version 0.4.0.5 - 2019-05-02 we had added up the sum of all nodes with a descriptor, but that could cause us to build failing circuits when we had either too many bridges or not enough guard nodes. Fixes bug 25885; bugfix on - 0.3.6.1-alpha. Patch by Neel Chauhan. + 0.2.3.1-alpha. Patch by Neel Chauhan. o Minor bugfixes (IPv6): - Fix tor_ersatz_socketpair on IPv6-only systems. Previously, the @@ -2836,7 +3469,7 @@ Changes in version 0.3.4.8 - 2018-09-10 - Remove duplicate code in parse_{c,s}method_line and bootstrap their functionalities into a single function. Fixes bug 6236; bugfix on 0.2.3.6-alpha. - - We remove the PortForwsrding and PortForwardingHelper options, + - We remove the PortForwarding and PortForwardingHelper options, related functions, and the port_forwarding tests. These options were used by the now-deprecated Vidalia to help ordinary users become Tor relays or bridges. Closes ticket 25409. Patch by diff --git a/changes/bug12399 b/changes/bug12399 deleted file mode 100644 index 922c08c5e3..0000000000 --- a/changes/bug12399 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (logging): - - Change log level of message "Hash of session info was not as expected" - to LOG_PROTOCOL_WARN. Fixes bug 12399; bugfix on 0.1.1.10-alpha. diff --git a/changes/bug22619 b/changes/bug22619 deleted file mode 100644 index 9c71996f5b..0000000000 --- a/changes/bug22619 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (circuit isolation): - - Fix a logic error that prevented the SessionGroup sub-option from - being accepted. Fixes bug 22619; bugfix on 0.2.7.2-alpha. diff --git a/changes/bug23507 b/changes/bug23507 deleted file mode 100644 index de18273fdb..0000000000 --- a/changes/bug23507 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (v3 single onion services): - - Make v3 single onion services fall back to a 3-hop intro, when there - all intro points are unreachable via a 1-hop path. Previously, v3 - single onion services failed when all intro nodes were unreachable - via a 1-hop path. Fixes bug 23507; bugfix on 0.3.2.1-alpha. diff --git a/changes/bug23818_v2 b/changes/bug23818_v2 deleted file mode 100644 index 0219a20f49..0000000000 --- a/changes/bug23818_v2 +++ /dev/null @@ -1,6 +0,0 @@ - o Minor bugfixes (v2 single onion services): - - Always retry v2 single onion service intro and rend circuits with a - 3-hop path. Previously, v2 single onion services used a 3-hop path - when rend circuits were retried after a remote or delayed failure, - but a 1-hop path for immediate retries. Fixes bug 23818; - bugfix on 0.2.9.3-alpha. diff --git a/changes/bug23818_v3 b/changes/bug23818_v3 deleted file mode 100644 index c430144d81..0000000000 --- a/changes/bug23818_v3 +++ /dev/null @@ -1,6 +0,0 @@ - o Minor bugfixes (v3 single onion services): - - Always retry v3 single onion service intro and rend circuits with a - 3-hop path. Previously, v3 single onion services used a 3-hop path - when rend circuits were retried after a remote or delayed failure, - but a 1-hop path for immediate retries. Fixes bug 23818; - bugfix on 0.3.2.1-alpha. diff --git a/changes/bug29034 b/changes/bug29034 deleted file mode 100644 index e7aa9af00b..0000000000 --- a/changes/bug29034 +++ /dev/null @@ -1,5 +0,0 @@ - o Major bugfixes (Onion service reachability): - - Properly clean up the introduction point map when circuits change purpose - from onion service circuits to pathbias, measurement, or other circuit types. - This should fix some service-side instances of introduction point failure. - Fixes bug 29034; bugfix on 0.3.2.1-alpha. diff --git a/changes/bug30649 b/changes/bug30649 deleted file mode 100644 index 4b2c603171..0000000000 --- a/changes/bug30649 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (circuit padding): - - On relays, properly check that a padding machine is absent before - logging a warn about it being absent. Fixes bug 30649; - bugfix on 0.4.0.1-alpha. diff --git a/changes/bug30713 b/changes/bug30713 deleted file mode 100644 index e00b98da65..0000000000 --- a/changes/bug30713 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (testing): - - Skip test_rebind when the TOR_SKIP_TEST_REBIND environmental variable is - set. Fixes bug 30713; bugfix on 0.3.5.1-alpha. - - Skip test_rebind on macOS in Travis, because it is unreliable on - macOS on Travis. Fixes bug 30713; bugfix on 0.3.5.1-alpha. diff --git a/changes/bug30744 b/changes/bug30744 deleted file mode 100644 index 9f07d4855f..0000000000 --- a/changes/bug30744 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (continuous integration): - - Allow the test-stem job to fail in Travis, because it sometimes hangs. - Fixes bug 30744; bugfix on 0.3.5.4-alpha. diff --git a/changes/bug30781 b/changes/bug30781 deleted file mode 100644 index 7c7adf470e..0000000000 --- a/changes/bug30781 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (directory authorities): - - Stop crashing after parsing an unknown descriptor purpose annotation. - We think this bug can only be triggered by modifying a local file. - Fixes bug 30781; bugfix on 0.2.0.8-alpha. diff --git a/changes/bug30894 b/changes/bug30894 deleted file mode 100644 index 64c14c4e6d..0000000000 --- a/changes/bug30894 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (memory leaks): - - Fix a trivial memory leak when parsing an invalid value - from a download schedule in the configuration. Fixes bug - 30894; bugfix on 0.3.4.1-alpha. diff --git a/changes/bug30916 b/changes/bug30916 deleted file mode 100644 index b006bfc75d..0000000000 --- a/changes/bug30916 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (relay): - - Avoid crashing when starting with a corrupt keys directory where - the old ntor key and the new ntor key are identical. Fixes bug 30916; - bugfix on 0.2.4.8-alpha. diff --git a/changes/bug30942 b/changes/bug30942 deleted file mode 100644 index bd6b2ff581..0000000000 --- a/changes/bug30942 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (circuit padding): - - Ignore non-padding cells on padding circuits. This addresses various - warning messages from subsystems that were not expecting padding - circuits. Fixes bug 30942; bugfix on 0.4.1.1-alpha.
\ No newline at end of file diff --git a/changes/bug30956 b/changes/bug30956 deleted file mode 100644 index 8f52a81de3..0000000000 --- a/changes/bug30956 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (pluggable transports): - - Always publish bridge pluggable transport information in the extra info - descriptor, even if ExtraInfoStatistics is 0. This information is - needed by BridgeDB. Fixes bug 30956; bugfix on 0.4.1.1-alpha. diff --git a/changes/bug31003 b/changes/bug31003 deleted file mode 100644 index 6c75163380..0000000000 --- a/changes/bug31003 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (crash on exit): - - Avoid a set of possible code paths that could use try to use freed memory - in routerlist_free() while Tor was exiting. Fixes bug 31003; bugfix on - 0.1.2.2-alpha. diff --git a/changes/bug31024 b/changes/bug31024 deleted file mode 100644 index 888fb2a26b..0000000000 --- a/changes/bug31024 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (circuitpadding): - - Add two NULL checks in unreachable places to silence Coverity (CID 144729 - and 1447291) and better future proof ourselves. Fixes bug 31024; bugfix - on 0.4.1.1-alpha.
\ No newline at end of file diff --git a/changes/bug31027 b/changes/bug31027 deleted file mode 100644 index dd3ce20b60..0000000000 --- a/changes/bug31027 +++ /dev/null @@ -1,3 +0,0 @@ - o Code simplification and refactoring: - - Remove some dead code from circpad_machine_remove_token() to fix some - Coverity warnings (CID 1447298). Fixes bug 31027; bugfix on 0.4.1.1-alpha.
\ No newline at end of file diff --git a/changes/bug31080_041 b/changes/bug31080_041 deleted file mode 100644 index 1fe9ec508d..0000000000 --- a/changes/bug31080_041 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (logging): - - Fix a conflict between the flag used for messaging-domain - log messages, and the LD_NO_MOCK testing flag. Fixes bug 31080; - bugfix on 0.4.1.1-alpha. diff --git a/changes/bug31107 b/changes/bug31107 deleted file mode 100644 index 9652927c30..0000000000 --- a/changes/bug31107 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (logging, protocol violations): - - Do not log a nonfatal assertion failure when receiving a VERSIONS - cell on a connection using the obsolete v1 link protocol. Log a - protocol_warn instead. Fixes bug 31107; bugfix on 0.2.4.4-alpha. diff --git a/changes/bug31335 b/changes/bug31335 new file mode 100644 index 0000000000..f633cf8b24 --- /dev/null +++ b/changes/bug31335 @@ -0,0 +1,3 @@ + o Minor bugfixes (code quality): + - Fix "make check-includes" so it runs correctly on out-of-tree builds. + Fixes bug 31335; bugfix on 0.3.5.1-alpha. diff --git a/changes/bug31343 b/changes/bug31343 deleted file mode 100644 index 17a8057ead..0000000000 --- a/changes/bug31343 +++ /dev/null @@ -1,9 +0,0 @@ - o Minor bugfixes (compilation): - - Avoid using labs() on time_t, which can cause compilation warnings - on 64-bit Windows builds. Fixes bug 31343; bugfix on 0.2.4.4-alpha. - - o Minor bugfixes (clock skew detection): - - Don't believe clock skew results from NETINFO cells that appear to - arrive before the VERSIONS cells they are responding to were sent. - Previously, we would accept them up to 3 minutes "in the past". - Fixes bug 31343; bugfix on 0.2.4.4-alpha. diff --git a/changes/bug31356_and_logs b/changes/bug31356_and_logs deleted file mode 100644 index fb5307cb69..0000000000 --- a/changes/bug31356_and_logs +++ /dev/null @@ -1,11 +0,0 @@ - o Minor bugfixes (circuit padding negotiation): - - Bump circuit padding protover to explicitly signify that the hs setup - machine support is finalized in 0.4.1.x-stable. This also means that - 0.4.1.x-alpha clients will not negotiate padding with 0.4.1.x-stable - relays, and 0.4.1.x-stable clients will not negotiate padding with - 0.4.1.x-alpha relays (or 0.4.0.x relays). Fixes bug 31356; - bugfix on 0.4.1.1-alpha. - o Minor features (circuit padding logging): - - Demote noisy client-side warn log to a protocol warning. Add additional - log messages and circuit id fields to help with fixing bug 30992 and any - other future issues. diff --git a/changes/bug31408 b/changes/bug31408 deleted file mode 100644 index 3e4ffa927d..0000000000 --- a/changes/bug31408 +++ /dev/null @@ -1,5 +0,0 @@ - o Major bugfixes (torrc): - - Stop ignoring torrc options after an %include directive, when the - included directory ends with a file that does not contain any config - options. (But does contain comments or whitespace.) - Fixes bug 31408; bugfix on 0.3.1.1-alpha. diff --git a/changes/bug31463 b/changes/bug31463 deleted file mode 100644 index d85c0887c3..0000000000 --- a/changes/bug31463 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (rust): - - Correctly exclude a redundant rust build job in Travis. Fixes bug 31463; - bugfix on 0.3.5.4-alpha. diff --git a/changes/bug31552 b/changes/bug31552 deleted file mode 100644 index fb33e14429..0000000000 --- a/changes/bug31552 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (compilation): - - Add more stub functions to fix compilation on Android with LTO, when - --disable-module-dirauth is used. Previously, these compilation - settings would make the compiler look for functions that didn't exist. - Fixes bug 31552; bugfix on 0.4.1.1-alpha. diff --git a/changes/bug31570 b/changes/bug31570 deleted file mode 100644 index f70b577b4c..0000000000 --- a/changes/bug31570 +++ /dev/null @@ -1,5 +0,0 @@ - o Major bugfixes (crash, android): - - Tolerate systems (including some Android installations) where madvise - and MADV_DONTDUMP are available at build-time, but not at run time. - Previously, these systems would notice a failed syscall and abort. - Fixes bug 31570; bugfix on 0.4.1.1-alpha. diff --git a/changes/bug31571 b/changes/bug31571 deleted file mode 100644 index 86de3537ba..0000000000 --- a/changes/bug31571 +++ /dev/null @@ -1,7 +0,0 @@ - o Minor bugfixes (error handling): - - Report the tor version whenever an assertion fails. Previously, we only - reported the Tor version on some crashes, and some non-fatal assertions. - Fixes bug 31571; bugfix on 0.3.5.1-alpha. - - On abort, try harder to flush the output buffers of log messages. On - some platforms (macOS), log messages can be discarded when the process - terminates. Fixes bug 31571; bugfix on 0.3.5.1-alpha. diff --git a/changes/bug31594 b/changes/bug31594 deleted file mode 100644 index 75e6ec33cc..0000000000 --- a/changes/bug31594 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (error handling): - - When tor aborts due to an error, close log file descriptors before - aborting. Closing the logs makes some OSes flush log file buffers, - rather than deleting buffered log lines. Fixes bug 31594; - bugfix on 0.2.5.2-alpha. diff --git a/changes/bug31614 b/changes/bug31614 deleted file mode 100644 index c425a9fcd4..0000000000 --- a/changes/bug31614 +++ /dev/null @@ -1,9 +0,0 @@ - o Minor bugfixes (logging): - - Disable backtrace signal handlers when shutting down tor. - Fixes bug 31614; bugfix on 0.2.5.2-alpha. - - Add a missing check for HAVE_PTHREAD_H, because the backtrace code uses - mutexes. Fixes bug 31614; bugfix on 0.2.5.2-alpha. - o Documentation: - - Explain why we can't destroy the backtrace buffer mutex. Explain why - we don't need to destroy the log mutex. - Closes ticket 31736. diff --git a/changes/bug31652 b/changes/bug31652 new file mode 100644 index 0000000000..c4eca7994a --- /dev/null +++ b/changes/bug31652 @@ -0,0 +1,5 @@ + o Minor bugfixes (onion services): + - When we clean up intro circuits for a v3 onion service, don't remove + circuits that have an established or pending circuit even if ran out of + retries. This way, we don't cleanup the circuit of the last retry. Fixes + bug 31652; bugfix on 0.3.2.1-alpha. diff --git a/changes/bug31657 b/changes/bug31657 deleted file mode 100644 index 08e9d95fdf..0000000000 --- a/changes/bug31657 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (guards): - - When tor is missing descriptors for some primary entry guards, make the - log message less alarming. It's normal for descriptors to expire, as long - as tor fetches new ones soon after. Fixes bug 31657; - bugfix on 0.3.3.1-alpha. diff --git a/changes/bug31696 b/changes/bug31696 deleted file mode 100644 index b9d6c4130c..0000000000 --- a/changes/bug31696 +++ /dev/null @@ -1,5 +0,0 @@ - o Major bugfixes (crash, Linux): - - Tolerate systems (including some Linux installations) where madvise - and/or MADV_DONTFORK are available at build-time, but not at run time. - Previously, these systems would notice a failed syscall and abort. - Fixes bug 31696; bugfix on 0.4.1.1-alpha. diff --git a/changes/bug31734 b/changes/bug31734 new file mode 100644 index 0000000000..ce989ea5db --- /dev/null +++ b/changes/bug31734 @@ -0,0 +1,3 @@ + o Minor bugfixes (error handling): + - Always lock the backtrace buffer before it is used. + Fixes bug 31734; bugfix on 0.2.5.3-alpha. diff --git a/changes/bug31736 b/changes/bug31736 deleted file mode 100644 index beb09e5069..0000000000 --- a/changes/bug31736 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (multithreading): - - Avoid some undefined behaviour when freeing mutexes. - Fixes bug 31736; bugfix on 0.0.7. diff --git a/changes/bug31884 b/changes/bug31884 deleted file mode 100644 index ddb6c50d74..0000000000 --- a/changes/bug31884 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (Appveyor CI): - - Avoid spurious errors when Appveyor CI fails before the install step. - Fixes bug 31884; bugfix on 0.3.4.2-alpha. diff --git a/changes/bug31897 b/changes/bug31897 deleted file mode 100644 index 81c63e704e..0000000000 --- a/changes/bug31897 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (tests, SunOS): - - Avoid a map_anon_nofork test failure due to a signed/unsigned integer - comparison. Fixes bug 31897; bugfix on 0.4.1.1-alpha. diff --git a/changes/bug31898 b/changes/bug31898 deleted file mode 100644 index 6f3e0a5465..0000000000 --- a/changes/bug31898 +++ /dev/null @@ -1,4 +0,0 @@ - o Major bugfixes (embedded Tor): - - Avoid a possible crash when restarting Tor in embedded mode and - enabling a different set of publish/subscribe messages. Fixes bug - 31898; bugfix on 0.4.1.1-alpha. diff --git a/changes/bug31922 b/changes/bug31922 new file mode 100644 index 0000000000..e6f31ce66a --- /dev/null +++ b/changes/bug31922 @@ -0,0 +1,4 @@ + o Minor bugfixes (configuration): + - When pkg-config is not installed, or a library that depends on + pkg-config is not found, tell the user what to do to fix the + problem. Fixes bug 31922; bugfix on 0.3.1.1-alpha. diff --git a/changes/bug31995 b/changes/bug31995 new file mode 100644 index 0000000000..c7ddd437a6 --- /dev/null +++ b/changes/bug31995 @@ -0,0 +1,3 @@ + o Minor bugfixes (testing): + - Avoid intermittent test failures due to a test that had relied on + inconsistent timing sources. Fixes bug 31995; bugfix on 0.3.1.3-alpha. diff --git a/changes/bug32106 b/changes/bug32106 new file mode 100644 index 0000000000..c6e8e95860 --- /dev/null +++ b/changes/bug32106 @@ -0,0 +1,5 @@ + o Minor features (documentation): + - Make clear in the man page, in both the bandwidth section and the + accountingmax section, that Tor counts in powers of two, not + powers of ten: 1 GByte is 1024*1024*1024 bytes, not one billion + bytes. Resolves ticket 32106. diff --git a/changes/bug32124 b/changes/bug32124 new file mode 100644 index 0000000000..164b33c7e3 --- /dev/null +++ b/changes/bug32124 @@ -0,0 +1,7 @@ + o Minor bugfixes (build system): + - Stop failing when jemalloc is requested, but tcmalloc is not found. + Fixes bug 32124; bugfix on 0.3.5.1-alpha. + - Interpret --disable-module-dirauth=no correctly. + Fixes bug 32124; bugfix on 0.3.4.1-alpha. + - Interpret --with-tcmalloc=no correctly. + Fixes bug 32124; bugfix on 0.2.0.20-rc. diff --git a/changes/bug32295 b/changes/bug32295 new file mode 100644 index 0000000000..e5e5a4399d --- /dev/null +++ b/changes/bug32295 @@ -0,0 +1,3 @@ + o Minor bugfixes (configuration): + - Log the option name when skipping an obsolete option. + Fixes bug 32295; bugfix on 0.4.2.1-alpha. diff --git a/changes/bug32402 b/changes/bug32402 new file mode 100644 index 0000000000..0654389be3 --- /dev/null +++ b/changes/bug32402 @@ -0,0 +1,3 @@ + o Minor bugfixes (shellcheck): + - Start checking most scripts for shellcheck errors again. + Fixes bug 32402; bugfix on 0.4.2.1-alpha. diff --git a/changes/bug32402_git_scripts b/changes/bug32402_git_scripts new file mode 100644 index 0000000000..2b10a8998a --- /dev/null +++ b/changes/bug32402_git_scripts @@ -0,0 +1,3 @@ + o Minor bugfixes (shellcheck): + - Fix minor shellcheck errors in the git-*.sh scripts. + Fixes bug 32402; bugfix on 0.4.2.1-alpha. diff --git a/changes/bug32771 b/changes/bug32771 new file mode 100644 index 0000000000..606bcf4be4 --- /dev/null +++ b/changes/bug32771 @@ -0,0 +1,4 @@ + o Minor bugfixes (logging, crash): + - Avoid a possible crash when trying to log a (fatal) assertion failure + about mismatched magic numbers in configuration objects. Fixes bug 32771; + bugfix on 0.4.2.1-alpha. diff --git a/changes/chutney_ci b/changes/chutney_ci deleted file mode 100644 index b17d587329..0000000000 --- a/changes/chutney_ci +++ /dev/null @@ -1,3 +0,0 @@ - o Minor features (continuous integration): - - Our Travis configuration now uses Chutney to run some network - integration tests automatically. Closes ticket 29280. diff --git a/changes/doc30630 b/changes/doc30630 deleted file mode 100644 index 0fbd8d4dd4..0000000000 --- a/changes/doc30630 +++ /dev/null @@ -1,3 +0,0 @@ - o Documentation: - - Mention URLs for Travis/Appveyor/Jenkins in ReleasingTor.md. Closes - ticket 30630. diff --git a/changes/doc31089 b/changes/doc31089 deleted file mode 100644 index 2fc0ba4f7d..0000000000 --- a/changes/doc31089 +++ /dev/null @@ -1,4 +0,0 @@ - o Documentation: - - Use RFC 2397 data URL scheme to embed image into tor-exit-notice.html - so that operators would no longer have to host it themselves. - Closes ticket 31089. diff --git a/changes/geoip-2019-06-10 b/changes/geoip-2019-06-10 deleted file mode 100644 index 2d1e065649..0000000000 --- a/changes/geoip-2019-06-10 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (geoip): - - Update geoip and geoip6 to the June 10 2019 Maxmind GeoLite2 - Country database. Closes ticket 30852. - diff --git a/changes/geoip-2019-10-01 b/changes/geoip-2019-10-01 deleted file mode 100644 index c7ed17b5c4..0000000000 --- a/changes/geoip-2019-10-01 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (geoip): - - Update geoip and geoip6 to the October 1 2019 Maxmind GeoLite2 - Country database. Closes ticket 31931. - diff --git a/changes/ticket28795 b/changes/ticket28795 deleted file mode 100644 index 6ae72562bf..0000000000 --- a/changes/ticket28795 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor features (fallback directory list): - - Replace the 157 fallbacks originally introduced in Tor 0.3.5.6-rc - in December 2018 (of which ~122 were still functional), with a - list of 148 fallbacks (70 new, 78 existing, 79 removed) generated - in June 2019. Closes ticket 28795. diff --git a/changes/ticket29669 b/changes/ticket29669 new file mode 100644 index 0000000000..f7e98a16ce --- /dev/null +++ b/changes/ticket29669 @@ -0,0 +1,3 @@ + o Minor feature (hidden service, control port): + - The ADD_ONION key blob keyword "BEST" now defaults from RSA1024 (v2) to + ED25519-V3 (v3). Closes ticket 29669. diff --git a/changes/ticket30591 b/changes/ticket30591 deleted file mode 100644 index f97c024009..0000000000 --- a/changes/ticket30591 +++ /dev/null @@ -1,3 +0,0 @@ - o Testing (continuous integration): - - In Travis, make stem log a controller trace to the console. And tail - stem's tor log after failure. Closes ticket 30591. diff --git a/changes/ticket30686 b/changes/ticket30686 deleted file mode 100644 index 36473c1a02..0000000000 --- a/changes/ticket30686 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor features (logging): - - Give a more useful assertion failure message if we think we have - minherit() but we fail to make a region non-inheritable. Give a - compile-time warning if our support for minherit() is - incomplete. Closes ticket 30686. diff --git a/changes/ticket30694 b/changes/ticket30694 deleted file mode 100644 index 70dbf6481a..0000000000 --- a/changes/ticket30694 +++ /dev/null @@ -1,3 +0,0 @@ - o Testing (continuous integration): - - In Travis, only run the stem tests that use a tor binary. - Closes ticket 30694. diff --git a/changes/ticket30871 b/changes/ticket30871 deleted file mode 100644 index 81c076bb02..0000000000 --- a/changes/ticket30871 +++ /dev/null @@ -1,6 +0,0 @@ - o Major bugfixes (circuit build, guard): - - When considering upgrading circuits from "waiting for guard" to "open", - always ignore the ones that are mark for close. Else, we can end up in - the situation where a subsystem is notified of that circuit opening but - still marked for close leading to undesirable behavior. Fixes bug 30871; - bugfix on 0.3.0.1-alpha. diff --git a/changes/ticket31001 b/changes/ticket31001 deleted file mode 100644 index 2ce1cbdf34..0000000000 --- a/changes/ticket31001 +++ /dev/null @@ -1,6 +0,0 @@ - o Minor bugfixes (compatibility, standards compliance): - - Fix a bug that would invoke undefined behavior on certain operating - systems when trying to asprintf() a string exactly INT_MAX bytes - long. We don't believe this is exploitable, but it's better - to fix it anyway. Fixes bug 31001; bugfix on 0.2.2.11-alpha. - Found and fixed by Tobias Stoeckmann. diff --git a/changes/ticket31189 b/changes/ticket31189 new file mode 100644 index 0000000000..318941c794 --- /dev/null +++ b/changes/ticket31189 @@ -0,0 +1,3 @@ + o Documentation: + - Correct the description of "GuardLifetime". Fixes bug 31189; bugfix on + 0.3.0.1-alpha. diff --git a/changes/ticket31311 b/changes/ticket31311 deleted file mode 100644 index 88dfb85736..0000000000 --- a/changes/ticket31311 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (distribution): - - Do not ship any temporary files found in the scripts/maint/practracker - directory. Fixes bug 31311; bugfix on 0.4.1.1-alpha. diff --git a/changes/ticket31372_appveyor b/changes/ticket31372_appveyor deleted file mode 100644 index e7bb03182e..0000000000 --- a/changes/ticket31372_appveyor +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (continuous integration): - - When building on Appveyor, pass the "-k" flag to make, so that - we are informed of all compilation failures, not just the first - one or two. Closes part of ticket 31372. diff --git a/changes/ticket31372_travis b/changes/ticket31372_travis deleted file mode 100644 index 403869b2ed..0000000000 --- a/changes/ticket31372_travis +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (continuous integration): - - When building on Travis, pass the "-k" flag to make, so that - we are informed of all compilation failures, not just the first - one or two. Closes part of ticket 31372. diff --git a/changes/ticket31374 b/changes/ticket31374 deleted file mode 100644 index e8eef9cd49..0000000000 --- a/changes/ticket31374 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (compilation warning): - - Fix a compilation warning on Windows about casting a function - pointer for GetTickCount64(). Fixes bug 31374; bugfix on - 0.2.9.1-alpha. diff --git a/changes/ticket31406 b/changes/ticket31406 deleted file mode 100644 index 0ebe6f6c47..0000000000 --- a/changes/ticket31406 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor features (directory authority): - - A new IP address the directory authority "dizum" has been changed. Closes - ticket 31406; diff --git a/changes/ticket31466 b/changes/ticket31466 deleted file mode 100644 index e535b4502e..0000000000 --- a/changes/ticket31466 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (logging): - - Rate-limit our the logging message about the obsolete .exit notation. - Previously, there was no limit on this warning, which could potentially - be triggered many times by a hostile website. Fixes bug 31466; - bugfix on 0.2.2.1-alpha. diff --git a/changes/ticket31549 b/changes/ticket31549 deleted file mode 100644 index 2c27aca4fb..0000000000 --- a/changes/ticket31549 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (authority): - - Directory authorities now reject relays running all currently - deprecated release series. The currently supported release series - are: 0.2.9, 0.3.5, 0.4.0, 0.4.1, and 0.4.2. Closes ticket 31549. diff --git a/changes/ticket31554 b/changes/ticket31554 deleted file mode 100644 index 73f4159ff3..0000000000 --- a/changes/ticket31554 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (stem tests): - - Change "make test-stem" so it only runs the stem tests that use tor. - This change makes test-stem faster and more reliable. - Closes ticket 31554. diff --git a/changes/ticket31673 b/changes/ticket31673 deleted file mode 100644 index 3b2bb4a46e..0000000000 --- a/changes/ticket31673 +++ /dev/null @@ -1,3 +0,0 @@ - o New system requirements (build system): - - Do not include the deprecated <sys/sysctl.h> on Linux or Windows system. - Closes 31673; diff --git a/changes/ticket31682 b/changes/ticket31682 new file mode 100644 index 0000000000..9777dec1f3 --- /dev/null +++ b/changes/ticket31682 @@ -0,0 +1,3 @@ + o Minor bugfixes (hidden service v3, coverity): + - Fix an implicit conversion from ssize_t to size_t discovered by Coverity. + Fixes bug 31682; bugfix on 0.4.2.1-alpha. diff --git a/changes/ticket31687_1 b/changes/ticket31687_1 deleted file mode 100644 index 2f4d440974..0000000000 --- a/changes/ticket31687_1 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (compilation): - - Suppress spurious float-conversion warnings from GCC when calling - floating-point classifier functions on FreeBSD. Fixes part of bug - 31687; bugfix on 0.3.1.5-alpha. diff --git a/changes/ticket31687_2 b/changes/ticket31687_2 deleted file mode 100644 index eadc698275..0000000000 --- a/changes/ticket31687_2 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (FreeBSD, PF-based proxy, IPv6): - - When extracting an IPv6 address from a PF-based proxy, verify - that we are actually configured to receive an IPv6 address, - and log an internal error if not. Fixes part of bug 31687; - bugfix on 0.2.3.4-alpha. diff --git a/changes/ticket31772 b/changes/ticket31772 deleted file mode 100644 index 7847b3f746..0000000000 --- a/changes/ticket31772 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (controller protocol): - - Fix the MAPADDRESS controller command to accept one or more - arguments. Previously, it required two or more arguments, and ignored - the first. Fixes bug 31772; bugfix on 0.4.1.1-alpha. diff --git a/changes/ticket31841 b/changes/ticket31841 new file mode 100644 index 0000000000..6e7fbc1da1 --- /dev/null +++ b/changes/ticket31841 @@ -0,0 +1,5 @@ + o Minor features (testing): + - When running tests that attempt to look up hostname, replace the libc + name lookup functions with ones that do not actually touch the network. + This way, the tests complete more quickly in the presence of a slow or + missing DNS resolver. Closes ticket 31841. diff --git a/changes/ticket31958 b/changes/ticket31958 new file mode 100644 index 0000000000..8206064dfe --- /dev/null +++ b/changes/ticket31958 @@ -0,0 +1,5 @@ + o Minor bugfixes (directory): + - When checking if a directory connection is anonymous, test if the circuit + was marked for close before looking at its channel. This avoids a BUG() + stacktrace in case it was previously closed. Fixes bug 31958; bugfix on + 0.4.2.1-alpha. diff --git a/changes/ticket32063 b/changes/ticket32063 new file mode 100644 index 0000000000..2c0246917c --- /dev/null +++ b/changes/ticket32063 @@ -0,0 +1,3 @@ + o Minor bugfixes (hs-v3, memory leak): + - Fix memory leak in unlikely error code path when encoding HS DoS establish + intro extension cell. Fixes bug 32063; bugfix on 0.4.2.1-alpha. diff --git a/changes/ticket32191 b/changes/ticket32191 new file mode 100644 index 0000000000..6988328115 --- /dev/null +++ b/changes/ticket32191 @@ -0,0 +1,3 @@ + o Minor features (build system): + - Make pkg-config use --prefix when cross-compiling, if PKG_CONFIG_PATH + is not set. Closes ticket 32191. diff --git a/changes/ticket32705_disable b/changes/ticket32705_disable new file mode 100644 index 0000000000..6d5b0779ab --- /dev/null +++ b/changes/ticket32705_disable @@ -0,0 +1,7 @@ + o Minor bugfixes (testing): + - When TOR_DISABLE_PRACTRACKER is set, do not apply it to the + test_practracker.sh script. Doing so caused a test failure. + Fixes bug 32705; bugfix on 0.4.2.1-alpha. + - When TOR_DISABLE_PRACTRACKER is set, log a notice to stderr + when skipping practracker checks. + Fixes bug 32705; bugfix on 0.4.2.1-alpha. diff --git a/changes/ticket32765 b/changes/ticket32765 new file mode 100644 index 0000000000..a9663a5df3 --- /dev/null +++ b/changes/ticket32765 @@ -0,0 +1,4 @@ + o Minor bugfixes (correctness checks): + - Use GCC/Clang's printf-checking feature to make sure that + tor_assertf() arguments are correctly typed. Fixes bug 32765; + bugfix on 0.4.1.1-alpha. diff --git a/configure.ac b/configure.ac index 48f3dfc001..cc746407d3 100644 --- a/configure.ac +++ b/configure.ac @@ -4,7 +4,7 @@ dnl Copyright (c) 2007-2019, The Tor Project, Inc. dnl See LICENSE for licensing information AC_PREREQ([2.63]) -AC_INIT([tor],[0.4.1.8-dev]) +AC_INIT([tor],[0.4.2.6-dev]) AC_CONFIG_SRCDIR([src/app/main/tor_main.c]) AC_CONFIG_MACRO_DIR([m4]) @@ -14,7 +14,7 @@ AC_CONFIG_MACRO_DIR([m4]) # version number changes. Tor uses it to make sure that it # only shuts down for missing "required protocols" when those protocols # are listed as required by a consensus after this date. -AC_DEFINE(APPROX_RELEASE_DATE, ["2020-01-30"], # for 0.4.1.8-dev +AC_DEFINE(APPROX_RELEASE_DATE, ["2020-01-30"], # for 0.4.2.6-dev [Approximate date when this software was released. (Updated when the version changes.)]) # "foreign" means we don't follow GNU package layout standards @@ -29,6 +29,18 @@ AC_USE_SYSTEM_EXTENSIONS AC_CANONICAL_HOST PKG_PROG_PKG_CONFIG +if test "x$PKG_CONFIG" = "x" ; then + pkg_config_user_action="install pkg-config, and check the PKG_CONFIG_PATH environment variable" + AC_MSG_NOTICE([Some libraries need pkg-config, including systemd, nss, lzma, zstd, and custom mallocs.]) + AC_MSG_NOTICE([To use those libraries, $pkg_config_user_action.]) +else + pkg_config_user_action="check the PKG_CONFIG_PATH environment variable" +fi + +if test "x$PKG_CONFIG_PATH" = "x" && test "x$prefix" != "xNONE" && test "$host" != "$build"; then + export PKG_CONFIG_PATH=$prefix/lib/pkgconfig + AC_MSG_NOTICE([set PKG_CONFIG_PATH=$PKG_CONFIG_PATH to support cross-compiling]) +fi AC_ARG_ENABLE(openbsd-malloc, AS_HELP_STRING(--enable-openbsd-malloc, [use malloc code from OpenBSD. Linux only. Deprecated: see --with-malloc])) @@ -105,6 +117,12 @@ if test "$enable_memory_sentinels" = "no"; then [Defined if we're turning off memory safety code to look for bugs]) fi +AC_ARG_ENABLE(manpage, + AS_HELP_STRING(--disable-manpage, [Disable manpage generation.])) + +AC_ARG_ENABLE(html-manual, + AS_HELP_STRING(--disable-html-manual, [Disable HTML documentation.])) + AC_ARG_ENABLE(asciidoc, AS_HELP_STRING(--disable-asciidoc, [don't use asciidoc (disables building of manpages)]), [case "${enableval}" in @@ -154,13 +172,13 @@ if test "x$have_systemd" = "xyes"; then TOR_SYSTEMD_CFLAGS="${SYSTEMD_CFLAGS}" TOR_SYSTEMD_LIBS="${SYSTEMD_LIBS}" PKG_CHECK_MODULES(LIBSYSTEMD209, [libsystemd >= 209], - [AC_DEFINE(HAVE_SYSTEMD_209,1,[Have systemd v209 or more])], []) + [AC_DEFINE(HAVE_SYSTEMD_209,1,[Have systemd v209 or greater])], []) fi AC_SUBST(TOR_SYSTEMD_CFLAGS) AC_SUBST(TOR_SYSTEMD_LIBS) if test "x$enable_systemd" = "xyes" -a "x$have_systemd" != "xyes" ; then - AC_MSG_ERROR([Explicitly requested systemd support, but systemd not found]) + AC_MSG_ERROR([Explicitly requested systemd support, but systemd not found, $pkg_config_user_action, or set SYSTEMD_CFLAGS and SYSTEMD_LIBS.]) fi case "$host" in @@ -245,11 +263,11 @@ m4_define(MODULES, dirauth) dnl Directory Authority module. AC_ARG_ENABLE([module-dirauth], AS_HELP_STRING([--disable-module-dirauth], - [Do not build tor with the dirauth module]), - [], dnl Action if-given - AC_DEFINE([HAVE_MODULE_DIRAUTH], [1], - [Compile with Directory Authority feature support])) -AM_CONDITIONAL(BUILD_MODULE_DIRAUTH, [test "x$enable_module_dirauth" != "xno"]) + [Build tor without the Directory Authority module: tor can not run as a directory authority or bridge authority])) +AM_CONDITIONAL(BUILD_MODULE_DIRAUTH,[test "x$enable_module_dirauth" != "xno"]) +AM_COND_IF(BUILD_MODULE_DIRAUTH, + AC_DEFINE([HAVE_MODULE_DIRAUTH], [1], + [Compile with Directory Authority feature support])) dnl Helper variables. TOR_MODULES_ALL_ENABLED= @@ -299,6 +317,8 @@ AC_PATH_PROG([ASCIIDOC], [asciidoc], none) AC_PATH_PROGS([A2X], [a2x a2x.py], none) AM_CONDITIONAL(USE_ASCIIDOC, test "x$asciidoc" = "xtrue") +AM_CONDITIONAL(BUILD_MANPAGE, [test "x$enable_manpage" != "xno"]) +AM_CONDITIONAL(BUILD_HTML_DOCS, [test "x$enable_html_manual" != "xno"]) AM_PROG_CC_C_O AC_PROG_CC_C99 @@ -550,8 +570,8 @@ if test "x$enable_rust" = "xyes"; then if test "x$RUSTC_VERSION_MAJOR" = "x" -o "x$RUSTC_VERSION_MINOR" = "x"; then AC_MSG_ERROR([rustc version couldn't be identified]) fi - if test "$RUSTC_VERSION_MAJOR" -lt 2 -a "$RUSTC_VERSION_MINOR" -lt 14; then - AC_MSG_ERROR([rustc must be at least version 1.14]) + if test "$RUSTC_VERSION_MAJOR" -lt 2 -a "$RUSTC_VERSION_MINOR" -lt 31; then + AC_MSG_ERROR([rustc must be at least version 1.31.0]) fi AC_MSG_RESULT([$RUSTC_VERSION]) fi @@ -862,7 +882,7 @@ if test "x$enable_nss" = "xyes"; then PKG_CHECK_MODULES(NSS, [nss], [have_nss=yes], - [have_nss=no; AC_MSG_ERROR([You asked for NSS but I can't find it.])]) + [have_nss=no; AC_MSG_ERROR([You asked for NSS but I can't find it, $pkg_config_user_action, or set NSS_CFLAGS and NSS_LIBS.])]) AC_SUBST(NSS_CFLAGS) AC_SUBST(NSS_LIBS) fi @@ -1068,7 +1088,7 @@ else have_lzma=no) if test "x$have_lzma" = "xno" ; then - AC_MSG_WARN([Unable to find liblzma.]) + AC_MSG_WARN([Unable to find liblzma, $pkg_config_user_action, or set LZMA_CFLAGS and LZMA_LIBS.]) fi fi @@ -1100,7 +1120,7 @@ else have_zstd=no) if test "x$have_zstd" = "xno" ; then - AC_MSG_WARN([Unable to find libzstd.]) + AC_MSG_WARN([Unable to find libzstd, $pkg_config_user_action, or set ZSTD_CFLAGS and ZSTD_LIBS.]) fi fi @@ -1188,6 +1208,17 @@ m4_ifdef([AS_VAR_IF],[ TOR_CHECK_LDFLAGS(-pie, "$all_ldflags_for_check", "$all_libs_for_check") fi TOR_TRY_COMPILE_WITH_CFLAGS(-fwrapv, also_link, CFLAGS_FWRAPV="-fwrapv", true) + + AC_MSG_CHECKING([whether we can run hardened binaries]) + AC_RUN_IFELSE([AC_LANG_PROGRAM([], [return 0;])], + [AC_MSG_RESULT([yes])], + [AC_MSG_RESULT([no]) + AC_MSG_ERROR([dnl + We can link with compiler hardening options, but we can't run with them. + That's a bad sign! If you must, you can pass --disable-gcc-hardening to + configure, but it would be better to figure out what the underlying problem + is.])], + [AC_MSG_RESULT([cross])]) fi if test "$fragile_hardening" = "yes"; then @@ -1868,8 +1899,7 @@ if test "$tor_cv_uint8_uchar" = "no"; then fi AC_ARG_WITH(tcmalloc, -AS_HELP_STRING(--with-tcmalloc, [use tcmalloc memory allocation library. Deprecated; see --with-malloc]), -[ tcmalloc=yes ], [ tcmalloc=no ]) +AS_HELP_STRING(--with-tcmalloc, [use tcmalloc memory allocation library. Deprecated; see --with-malloc])) default_malloc=system @@ -1878,7 +1908,7 @@ if test "x$enable_openbsd_malloc" = "xyes" ; then default_malloc=openbsd fi -if test "x$tcmalloc" = "xyes"; then +if test "x$with_tcmalloc" = "xyes"; then AC_MSG_NOTICE([The --with-tcmalloc argument is deprecated; use --with-malloc=tcmalloc instead.]) default_malloc=tcmalloc fi @@ -1896,7 +1926,7 @@ AS_CASE([$malloc], have_tcmalloc=no) if test "x$have_tcmalloc" = "xno" ; then - AC_MSG_ERROR([Unable to find tcmalloc requested by --with-malloc.]) + AC_MSG_ERROR([Unable to find tcmalloc requested by --with-malloc, $pkg_config_user_action, or set TCMALLOC_CFLAGS and TCMALLOC_LIBS.]) fi CFLAGS="$CFLAGS $TCMALLOC_CFLAGS" @@ -1909,8 +1939,8 @@ AS_CASE([$malloc], have_jemalloc=yes, have_jemalloc=no) - if test "x$have_tcmalloc" = "xno" ; then - AC_MSG_ERROR([Unable to find jemalloc requested by --with-malloc.]) + if test "x$have_jemalloc" = "xno" ; then + AC_MSG_ERROR([Unable to find jemalloc requested by --with-malloc, $pkg_config_user_action, or set JEMALLOC_CFLAGS and JEMALLOC_LIBS.]) fi CFLAGS="$CFLAGS $JEMALLOC_CFLAGS" @@ -2458,7 +2488,6 @@ AC_CONFIG_FILES([ Makefile config.rust contrib/operator-tools/tor.logrotate - contrib/dist/torctl contrib/dist/tor.service src/config/torrc.sample src/config/torrc.minimal diff --git a/contrib/dist/torctl.in b/contrib/dist/torctl.in deleted file mode 100644 index 4cc137da46..0000000000 --- a/contrib/dist/torctl.in +++ /dev/null @@ -1,195 +0,0 @@ -#!/bin/sh -# -# TOR control script designed to allow an easy command line interface -# to controlling The Onion Router -# -# The exit codes returned are: -# 0 - operation completed successfully. For "status", tor running. -# 1 - For "status", tor not running. -# 2 - Command not supported -# 3 - Could not be started or reloaded -# 4 - Could not be stopped -# 5 - -# 6 - -# 7 - -# 8 - -# -# When multiple arguments are given, only the error from the _last_ -# one is reported. -# -# -# |||||||||||||||||||| START CONFIGURATION SECTION |||||||||||||||||||| -# -------------------- -------------------- -# Name of the executable -EXEC=tor -# -# the path to your binary, including options if necessary -TORBIN="@BINDIR@/$EXEC" -# -# the path to the configuration file -TORCONF="@CONFDIR@/torrc" -# -# the path to your PID file -PIDFILE="@LOCALSTATEDIR@/run/tor/tor.pid" -# -# The path to the log file -LOGFILE="@LOCALSTATEDIR@/log/tor/tor.log" -# -# The path to the datadirectory -TORDATA="@LOCALSTATEDIR@/lib/tor" -# -TORARGS="--pidfile $PIDFILE --log \"notice file $LOGFILE\" --runasdaemon 1" -TORARGS="$TORARGS --datadirectory $TORDATA" - -# If user name is set in the environment, then use it; -# otherwise run as the invoking user (or whatever user the config -# file says)... unless the invoking user is root. The idea here is to -# let an unprivileged user run tor for her own use using this script, -# while still providing for it to be used as a system daemon. -if [ "x`id -u`" = "x0" ]; then - TORUSER=@TORUSER@ -fi - -if [ "x$TORUSER" != "x" ]; then - TORARGS="$TORARGS --user $TORUSER" -fi - -# We no longer wrap the Tor daemon startup in an su when running as -# root, because it's too painful to make the use of su portable. -# Just let the daemon set the UID and GID. -START="$TORBIN -f $TORCONF $TORARGS" - -# -# -------------------- -------------------- -# |||||||||||||||||||| END CONFIGURATION SECTION |||||||||||||||||||| - -ERROR=0 -ARGV="$@" -if [ "x$ARGV" = "x" ] ; then - ARGS="help" -fi - -checkIfRunning ( ) { - # check for pidfile - PID=unknown - if [ -f $PIDFILE ] ; then - PID=`/bin/cat $PIDFILE` - if [ "x$PID" != "x" ] ; then - if kill -0 $PID 2>/dev/null ; then - STATUS="$EXEC (pid $PID) running" - RUNNING=1 - else - STATUS="PID file ($PIDFILE) present, but $EXEC ($PID) not running" - RUNNING=0 - fi - else - STATUS="$EXEC (pid $PID?) not running" - RUNNING=0 - fi - else - STATUS="$EXEC apparently not running (no pid file)" - RUNNING=0 - fi - return -} - -for ARG in $@ $ARGS -do - checkIfRunning - - case $ARG in - start) - if [ $RUNNING -eq 1 ]; then - echo "$0 $ARG: $EXEC (pid $PID) already running" - continue - fi - if eval "$START" ; then - echo "$0 $ARG: $EXEC started" - # Make sure it stayed up! - /bin/sleep 1 - checkIfRunning - if [ $RUNNING -eq 0 ]; then - echo "$0 $ARG: $EXEC (pid $PID) quit unexpectedly" - fi - else - echo "$0 $ARG: $EXEC could not be started" - ERROR=3 - fi - ;; - stop) - if [ $RUNNING -eq 0 ]; then - echo "$0 $ARG: $STATUS" - continue - fi - if kill -15 $PID ; then - echo "$0 $ARG: $EXEC stopped" - else - /bin/sleep 1 - if kill -9 $PID ; then - echo "$0 $ARG: $EXEC stopped" - else - echo "$0 $ARG: $EXEC could not be stopped" - ERROR=4 - fi - fi - # Make sure it really died! - /bin/sleep 1 - checkIfRunning - if [ $RUNNING -eq 1 ]; then - echo "$0 $ARG: $EXEC (pid $PID) unexpectedly still running" - ERROR=4 - fi - ;; - restart) - $0 stop start - ;; - reload) - if [ $RUNNING -eq 0 ]; then - echo "$0 $ARG: $STATUS" - continue - fi - if kill -1 $PID; then - /bin/sleep 1 - echo "$EXEC (PID $PID) reloaded" - else - echo "Can't reload $EXEC" - ERROR=3 - fi - ;; - status) - echo $STATUS - if [ $RUNNING -eq 1 ]; then - ERROR=0 - else - ERROR=1 - fi - ;; - log) - cat $LOGFILE - ;; - help) - echo "usage: $0 (start|stop|restart|status|help)" - /bin/cat <<EOF - -start - start $EXEC -stop - stop $EXEC -restart - stop and restart $EXEC if running or start if not running -reload - cause the running process to reinitialize itself -status - tell whether $EXEC is running or not -log - display the contents of the log file -help - this text - -EOF - ERROR=0 - ;; - *) - $0 help - ERROR=2 - ;; - - esac - -done - -exit $ERROR - diff --git a/contrib/include.am b/contrib/include.am index 8dd8593304..784f5427b8 100644 --- a/contrib/include.am +++ b/contrib/include.am @@ -3,7 +3,6 @@ EXTRA_DIST+= \ contrib/README \ contrib/client-tools/torify \ contrib/dist/rc.subr \ - contrib/dist/torctl \ contrib/dist/tor.service.in \ contrib/operator-tools/tor-exit-notice.html \ contrib/or-tools/exitlist \ diff --git a/contrib/win32build/tor-mingw.nsi.in b/contrib/win32build/tor-mingw.nsi.in index 40783299c8..2ccb1632bc 100644 --- a/contrib/win32build/tor-mingw.nsi.in +++ b/contrib/win32build/tor-mingw.nsi.in @@ -8,7 +8,7 @@ !include "LogicLib.nsh" !include "FileFunc.nsh" !insertmacro GetParameters -!define VERSION "0.4.1.8-dev" +!define VERSION "0.4.2.6-dev" !define INSTALLER "tor-${VERSION}-win32.exe" !define WEBSITE "https://www.torproject.org/" !define LICENSE "LICENSE" diff --git a/doc/HACKING/CodeStructure.md b/doc/HACKING/CodeStructure.md index 736d6cd484..fffafcaed1 100644 --- a/doc/HACKING/CodeStructure.md +++ b/doc/HACKING/CodeStructure.md @@ -2,128 +2,121 @@ TODO: revise this to talk about how things are, rather than how things have changed. -TODO: Make this into good markdown. - - - -For quite a while now, the program "tor" has been built from source -code in just two directories: src/common and src/or. +For quite a while now, the program *tor* has been built from source +code in just two directories: **src/common** and **src/or**. This has become more-or-less untenable, for a few reasons -- most notably of which is that it has led our code to become more spaghetti-ish than I can endorse with a clean conscience. So to fix that, we've gone and done a huge code movement in our git -master branch, which will land in a release once Tor 0.3.5.1-alpha is +master branch, which will land in a release once Tor `0.3.5.1-alpha` is out. Here's what we did: - * src/common has been turned into a set of static libraries. These -all live in the "src/lib/*" directories. The dependencies between + * **src/common** has been turned into a set of static libraries. These +all live in the **src/lib/*** directories. The dependencies between these libraries should have no cycles. The libraries are: - arch -- Headers to handle architectural differences - cc -- headers to handle differences among compilers - compress -- wraps zlib, zstd, lzma - container -- high-level container types - crypt_ops -- Cryptographic operations. Planning to split this into + - **arch** -- Headers to handle architectural differences + - **cc** -- headers to handle differences among compilers + - **compress** -- wraps zlib, zstd, lzma + - **container** -- high-level container types + - **crypt_ops** -- Cryptographic operations. Planning to split this into a higher and lower level library - ctime -- Operations that need to run in constant-time. (Properly, + - **ctime** -- Operations that need to run in constant-time. (Properly, data-invariant time) - defs -- miscelaneous definitions needed throughout Tor. - encoding -- transforming one data type into another, and various + - **defs** -- miscelaneous definitions needed throughout Tor. + - **encoding** -- transforming one data type into another, and various data types into strings. - err -- lowest-level error handling, in cases where we can't use + - **err** -- lowest-level error handling, in cases where we can't use the logs because something that the logging system needs has broken. - evloop -- Generic event-loop handling logic - fdio -- Low-level IO wrapper functions for file descriptors. - fs -- Operations on the filesystem - intmath -- low-level integer math and misc bit-twiddling hacks - lock -- low-level locking code - log -- Tor's logging module. This library sits roughly halfway up + - **evloop** -- Generic event-loop handling logic + - **fdio** -- Low-level IO wrapper functions for file descriptors. + - **fs** -- Operations on the filesystem + - **intmath** -- low-level integer math and misc bit-twiddling hacks + - **lock** -- low-level locking code + - **log** -- Tor's logging module. This library sits roughly halfway up the library dependency diagram, since everything it depends on has to be carefully crafted to *not* log. - malloc -- Low-level wrappers for the platform memory allocation functions. - math -- Higher-level mathematical functions, and floating-point math - memarea -- An arena allocator - meminfo -- Functions for querying the current process's memory + - **malloc** -- Low-level wrappers for the platform memory allocation functions. + - **math** -- Higher-level mathematical functions, and floating-point math + - **memarea** -- An arena allocator + - **meminfo** -- Functions for querying the current process's memory status and resources - net -- Networking compatibility and convenience code - osinfo -- Querying information about the operating system - process -- Launching and querying the status of other processes - sandbox -- Backend for the linux seccomp2 sandbox - smartlist_core -- The lowest-level of the smartlist_t data type. + - **net** -- Networking compatibility and convenience code + - **osinfo** -- Querying information about the operating system + - **process** -- Launching and querying the status of other processes + - **sandbox** -- Backend for the linux seccomp2 sandbox + - **smartlist_core** -- The lowest-level of the smartlist_t data type. Separated from the rest of the containers library because the logging subsystem depends on it. - string -- Compatibility and convenience functions for manipulating + - **string** -- Compatibility and convenience functions for manipulating C strings. - term -- Terminal-related functions (currently limited to a getpass + - **term** -- Terminal-related functions (currently limited to a getpass function). - testsupport -- Macros for mocking, unit tests, etc. - thread -- Higher-level thread compatibility code - time -- Higher-level time management code, including format + - **testsupport** -- Macros for mocking, unit tests, etc. + - **thread** -- Higher-level thread compatibility code + - **time** -- Higher-level time management code, including format conversions and monotonic time - tls -- Our wrapper around our TLS library - trace -- Formerly src/trace -- a generic event tracing API - wallclock -- Low-level time code, used by the log module. + - **tls** -- Our wrapper around our TLS library + - **trace** -- Formerly src/trace -- a generic event tracing API + - **wallclock** -- Low-level time code, used by the log module. - * To ensure that the dependency graph in src/common remains under -control, there is a tool that you can run called "make -check-includes". It verifies that each module in Tor only includes + * To ensure that the dependency graph in **src/common** remains under +control, there is a tool that you can run called `make +check-includes`. It verifies that each module in Tor only includes the headers that it is permitted to include, using a per-directory -".may_include" file. +*.may_include* file. - * The src/or/or.h header has been split into numerous smaller + * The **src/or/or.h** header has been split into numerous smaller headers. Notably, many important structures are now declared in a -header called foo_st.h, where "foo" is the name of the structure. +header called *foo_st.h*, where "foo" is the name of the structure. - * The src/or directory, which had most of Tor's code, had been split + * The **src/or** directory, which had most of Tor's code, had been split up into several directories. This is still a work in progress: This code has not itself been refactored, and its dependency graph is still a tangled web. I hope we'll be working on that over the coming releases, but it will take a while to do. - The new top-level source directories are: - - src/core -- Code necessary to actually perform or use onion routing. - src/feature -- Code used only by some onion routing + - The new top-level source directories are: + - **src/core** -- Code necessary to actually perform or use onion routing. + - **src/feature** -- Code used only by some onion routing configurations, or only for a special purpose. - src/app -- Top-level code to run, invoke, and configure the + - **src/app** -- Top-level code to run, invoke, and configure the lower-level code - The new second-level source directories are: - src/core/crypto -- High-level cryptographic protocols used in Tor - src/core/mainloop -- Tor's event loop, connection-handling, and + - The new second-level source directories are: + - **src/core/crypto** -- High-level cryptographic protocols used in Tor + - **src/core/mainloop** -- Tor's event loop, connection-handling, and traffic-routing code. - src/core/or -- Parts related to handling onion routing itself - src/core/proto -- support for encoding and decoding different + - **src/core/or** -- Parts related to handling onion routing itself + - **src/core/proto** -- support for encoding and decoding different wire protocols - - src/feature/api -- Support for making Tor embeddable - src/feature/client -- Functionality which only Tor clients need - src/feature/control -- Controller implementation - src/feature/dirauth -- Directory authority - src/feature/dircache -- Directory cache - src/feature/dirclient -- Directory client - src/feature/dircommon -- Shared code between the other directory modules - src/feature/hibernate -- Hibernating when Tor is out of bandwidth + - **src/feature/api** -- Support for making Tor embeddable + - **src/feature/client** -- Functionality which only Tor clients need + - **src/feature/control** -- Controller implementation + - **src/feature/dirauth** -- Directory authority + - **src/feature/dircache** -- Directory cache + - **src/feature/dirclient** -- Directory client + - **src/feature/dircommon** -- Shared code between the other directory modules + - **src/feature/hibernate** -- Hibernating when Tor is out of bandwidth or shutting down - src/feature/hs -- v3 onion service implementation - src/feature/hs_common -- shared code between both onion service + - **src/feature/hs** -- v3 onion service implementation + - **src/feature/hs_common** -- shared code between both onion service implementations - src/feature/nodelist -- storing and accessing the list of relays on + - **src/feature/nodelist** -- storing and accessing the list of relays on the network. - src/feature/relay -- code that only relay servers and exit servers need. - src/feature/rend -- v2 onion service implementation - src/feature/stats -- statistics and history - - src/app/config -- configuration and state for Tor - src/app/main -- Top-level functions to invoke the rest or Tor. + - **src/feature/relay** -- code that only relay servers and exit servers need. + - **src/feature/rend** -- v2 onion service implementation + - **src/feature/stats** -- statistics and history + - **src/app/config** -- configuration and state for Tor + - **src/app/main** -- Top-level functions to invoke the rest or Tor. - * The "tor" executable is now built in src/app/tor rather than src/or/tor. + * The `tor` executable is now built in **src/app/tor** rather than **src/or/tor**. * There are more static libraries than before that you need to build into your application if you want to embed Tor. Rather than -maintaining this list yourself, I recommend that you run "make -show-libs" to have Tor emit a list of what you need to link. +maintaining this list yourself, I recommend that you run `make +show-libs` to have Tor emit a list of what you need to link. diff --git a/doc/HACKING/CodingStandards.md b/doc/HACKING/CodingStandards.md index 74db2a39a3..2c273910d1 100644 --- a/doc/HACKING/CodingStandards.md +++ b/doc/HACKING/CodingStandards.md @@ -42,6 +42,7 @@ If you have changed build system components: - For example, if you have changed Makefiles, autoconf files, or anything else that affects the build system. + License issues ============== @@ -58,7 +59,6 @@ Some compatible licenses include: - CC0 Public Domain Dedication - How we use Git branches ======================= @@ -99,17 +99,28 @@ When you do a commit that needs a ChangeLog entry, add a new file to the `changes` toplevel subdirectory. It should have the format of a one-entry changelog section from the current ChangeLog file, as in -- Major bugfixes: + o Major bugfixes (security): - Fix a potential buffer overflow. Fixes bug 99999; bugfix on 0.3.1.4-beta. + o Minor features (performance): + - Make tor faster. Closes ticket 88888. To write a changes file, first categorize the change. Some common categories -are: Minor bugfixes, Major bugfixes, Minor features, Major features, Code -simplifications and refactoring. Then say what the change does. If -it's a bugfix, mention what bug it fixes and when the bug was -introduced. To find out which Git tag the change was introduced in, -you can use `git describe --contains <sha1 of commit>`. - +are: + o Minor bugfixes (subheading): + o Major bugfixes (subheading): + o Minor features (subheading): + o Major features (subheading): + o Code simplifications and refactoring: + o Testing: + o Documentation: + +The subheading is a particular area within Tor. See the ChangeLog for +examples. + +Then say what the change does. If it's a bugfix, mention what bug it fixes +and when the bug was introduced. To find out which Git tag the change was +introduced in, you can use `git describe --contains <sha1 of commit>`. If you don't know the commit, you can search the git diffs (-S) for the first instance of the feature (--reverse). @@ -147,10 +158,6 @@ that our CI passes. These checks are implemented in `scripts/maint/lintChanges.py`. Changes file style guide: - * Changes files begin with " o Header (subheading):". The header - should usually be "Minor/Major bugfixes/features". The subheading is a - particular area within Tor. See the ChangeLog for examples. - * Make everything terse. * Write from the user's point of view: describe the user-visible changes @@ -343,7 +350,6 @@ for more information about trunnel. For information on adding new trunnel code to Tor, see src/trunnel/README - Calling and naming conventions ------------------------------ @@ -451,7 +457,6 @@ to use it as a function callback), define it with a name like abc_free_(obj); } - Doxygen comment conventions --------------------------- diff --git a/doc/HACKING/EndOfLifeTor.md b/doc/HACKING/EndOfLifeTor.md new file mode 100644 index 0000000000..2fece2ca9d --- /dev/null +++ b/doc/HACKING/EndOfLifeTor.md @@ -0,0 +1,50 @@ + +End of Life on an old release series +------------------------------------ + +Here are the steps that the maintainer should take when an old Tor release +series reaches End of Life. Note that they are _only_ for entire series that +have reached their planned EOL: they do not apply to security-related +deprecations of individual versions. + +### 0. Preliminaries + +0. A few months before End of Life: + Write a deprecation announcement. + Send the announcement out with every new release announcement. + +1. A month before End of Life: + Send the announcement to tor-announce, tor-talk, tor-relays, and the + packagers. + +### 1. On the day + +1. Open tickets to remove the release from: + - the jenkins builds + - tor's Travis CI cron jobs + - chutney's Travis CI tests (#) + - stem's Travis CI tests (#) + +2. Close the milestone in Trac. To do this, go to Trac, log in, + select "Admin" near the top of the screen, then select "Milestones" from + the menu on the left. Click on the milestone for this version, and + select the "Completed" checkbox. By convention, we select the date as + the End of Life date. + +3. Replace NNN-backport with NNN-unreached-backport in all open trac tickets. + +4. If there are any remaining tickets in the milestone: + - merge_ready tickets are for backports: + - if there are no supported releases for the backport, close the ticket + - if there is an earlier (LTS) release for the backport, move the ticket + to that release + - other tickets should be closed (if we won't fix them) or moved to a + supported release (if we will fix them) + +5. Mail the end of life announcement to tor-announce, the packagers list, + and tor-relays. The current list of packagers is in ReleasingTor.md. + +6. Ask at least two of weasel/arma/Sebastian to remove the old version + number from their approved versions list. + +7. Update the CoreTorReleases wiki page. diff --git a/doc/HACKING/Fuzzing.md b/doc/HACKING/Fuzzing.md index 2039d6a4c0..c2db7e9853 100644 --- a/doc/HACKING/Fuzzing.md +++ b/doc/HACKING/Fuzzing.md @@ -1,6 +1,6 @@ -= Fuzzing Tor +# Fuzzing Tor -== The simple version (no fuzzing, only tests) +## The simple version (no fuzzing, only tests) Check out fuzzing-corpora, and set TOR_FUZZ_CORPORA to point to the place where you checked it out. @@ -12,7 +12,7 @@ This won't actually fuzz Tor! It will just run all the fuzz binaries on our existing set of testcases for the fuzzer. -== Different kinds of fuzzing +## Different kinds of fuzzing Right now we support three different kinds of fuzzer. @@ -37,7 +37,7 @@ In all cases, you'll need some starting examples to give the fuzzer when it starts out. There's a set in the "fuzzing-corpora" git repository. Try setting TOR_FUZZ_CORPORA to point to a checkout of that repository -== Writing Tor fuzzers +## Writing Tor fuzzers A tor fuzzing harness should have: * a fuzz_init() function to set up any necessary global state. @@ -52,7 +52,7 @@ bug, or accesses memory it shouldn't. This helps fuzzing frameworks detect "interesting" cases. -== Guided Fuzzing with AFL +## Guided Fuzzing with AFL There is no HTTPS, hash, or signature for American Fuzzy Lop's source code, so its integrity can't be verified. That said, you really shouldn't fuzz on a @@ -101,7 +101,7 @@ macOS (OS X) requires slightly more preparation, including: * using afl-clang (or afl-clang-fast from the llvm directory) * disabling external crash reporting (AFL will guide you through this step) -== Triaging Issues +## Triaging Issues Crashes are usually interesting, particularly if using AFL_HARDEN=1 and --enable-expensive-hardening. Sometimes crashes are due to bugs in the harness code. @@ -115,7 +115,7 @@ To see what fuzz-http is doing with a test case, call it like this: (Logging is disabled while fuzzing to increase fuzzing speed.) -== Reporting Issues +## Reporting Issues Please report any issues discovered using the process in Tor's security issue policy: diff --git a/doc/HACKING/HowToReview.md b/doc/HACKING/HowToReview.md index 2d1f3d1c9e..2325e70175 100644 --- a/doc/HACKING/HowToReview.md +++ b/doc/HACKING/HowToReview.md @@ -63,7 +63,7 @@ Let's look at the code! Let's look at the documentation! -------------------------------- -- Does the documentation confirm to CodingStandards.txt? +- Does the documentation conform to CodingStandards.txt? - Does it make sense? diff --git a/doc/HACKING/ReleasingTor.md b/doc/HACKING/ReleasingTor.md index a97ca08ce9..f40e2af573 100644 --- a/doc/HACKING/ReleasingTor.md +++ b/doc/HACKING/ReleasingTor.md @@ -5,7 +5,7 @@ Putting out a new release Here are the steps that the maintainer should take when putting out a new Tor release: -=== 0. Preliminaries +### 0. Preliminaries 1. Get at least two of weasel/arma/Sebastian to put the new version number in their approved versions list. Give them a few @@ -18,7 +18,7 @@ new Tor release: date of a TB that contains it. See note below in "commit, upload, announce". -=== I. Make sure it works +### I. Make sure it works 1. Make sure that CI passes: have a look at Travis (https://travis-ci.org/torproject/tor/branches), Appveyor @@ -52,7 +52,7 @@ new Tor release: memory leaks.) -=== II. Write a changelog +### II. Write a changelog 1a. (Alpha release variant) @@ -139,7 +139,7 @@ new Tor release: text of existing entries, though.) -=== III. Making the source release. +### III. Making the source release. 1. In `maint-0.?.x`, bump the version number in `configure.ac` and run `make update-versions` to update version numbers in other @@ -165,7 +165,7 @@ new Tor release: If it is not, you'll need to poke Roger, Weasel, and Sebastian again: see item 0.1 at the start of this document. -=== IV. Commit, upload, announce +### IV. Commit, upload, announce 1. Sign the tarball, then sign and push the git tag: @@ -241,15 +241,17 @@ new Tor release: For templates to use when announcing, see: https://trac.torproject.org/projects/tor/wiki/org/teams/NetworkTeam/AnnouncementTemplates -=== V. Aftermath and cleanup +### V. Aftermath and cleanup 1. If it's a stable release, bump the version number in the `maint-x.y.z` branch to "newversion-dev", and do a `merge -s ours` merge to avoid taking that change into master. -2. Forward-port the ChangeLog (and ReleaseNotes if appropriate) to the - master branch. - -3. Keep an eye on the blog post, to moderate comments and answer questions. +2. If there is a new `maint-x.y.z` branch, create a Travis CI cron job that + builds the release every week. (It's ok to skip the weekly build if the + branch was updated in the last 24 hours.) +3. Forward-port the ChangeLog (and ReleaseNotes if appropriate) to the + master branch. +4. Keep an eye on the blog post, to moderate comments and answer questions. diff --git a/doc/HACKING/design/00-overview.md b/doc/HACKING/design/00-overview.md new file mode 100644 index 0000000000..2103a9062a --- /dev/null +++ b/doc/HACKING/design/00-overview.md @@ -0,0 +1,124 @@ + +## Overview ## + +This document describes the general structure of the Tor codebase, how +it fits together, what functionality is available for extending Tor, +and gives some notes on how Tor got that way. + +Tor remains a work in progress: We've been working on it for more than a +decade, and we've learned a lot about good coding since we first +started. This means, however, that some of the older pieces of Tor will +have some "code smell" in them that could sure stand a brisk +refactoring. So when I describe a piece of code, I'll sometimes give a +note on how it got that way, and whether I still think that's a good +idea. + +The first drafts of this document were written in the Summer and Fall of +2015, when Tor 0.2.6 was the most recent stable version, and Tor 0.2.7 +was under development. If you're reading this far in the future, some +things may have changed. Caveat haxxor! + +This document is not an overview of the Tor protocol. For that, see the +design paper and the specifications at https://spec.torproject.org/ . + +For more information about Tor's coding standards and some helpful +development tools, see doc/HACKING in the Tor repository. + +For more information about writing tests, see doc/HACKING/WritingTests.txt +in the Tor repository. + +### The very high level ### + +Ultimately, Tor runs as an event-driven network daemon: it responds to +network events, signals, and timers by sending and receiving things over +the network. Clients, relays, and directory authorities all use the +same codebase: the Tor process will run as a client, relay, or authority +depending on its configuration. + +Tor has a few major dependencies, including Libevent (used to tell which +sockets are readable and writable), OpenSSL (used for many encryption +functions, and to implement the TLS protocol), and zlib (used to +compress and uncompress directory information). + +Most of Tor's work today is done in a single event-driven main thread. +Tor also spawns one or more worker threads to handle CPU-intensive +tasks. (Right now, this only includes circuit encryption.) + +On startup, Tor initializes its libraries, reads and responds to its +configuration files, and launches a main event loop. At first, the only +events that Tor listens for are a few signals (like TERM and HUP), and +one or more listener sockets (for different kinds of incoming +connections). Tor also configures a timer function to run once per +second to handle periodic events. As Tor runs over time, other events +will open, and new events will be scheduled. + +The codebase is divided into a few main subdirectories: + + src/common -- utility functions, not necessarily tor-specific. + + src/or -- implements the Tor protocols. + + src/test -- unit and regression tests + + src/ext -- Code maintained elsewhere that we include in the Tor + source distribution. + + src/trunnel -- automatically generated code (from the Trunnel) + tool: used to parse and encode binary formats. + +### Some key high-level abstractions ### + +The most important abstractions at Tor's high-level are Connections, +Channels, Circuits, and Nodes. + +A 'Connection' represents a stream-based information flow. Most +connections are TCP connections to remote Tor servers and clients. (But +as a shortcut, a relay will sometimes make a connection to itself +without actually using a TCP connection. More details later on.) +Connections exist in different varieties, depending on what +functionality they provide. The principle types of connection are +"edge" (eg a socks connection or a connection from an exit relay to a +destination), "OR" (a TLS stream connecting to a relay), "Directory" (an +HTTP connection to learn about the network), and "Control" (a connection +from a controller). + +A 'Circuit' is persistent tunnel through the Tor network, established +with public-key cryptography, and used to send cells one or more hops. +Clients keep track of multi-hop circuits, and the cryptography +associated with each hop. Relays, on the other hand, keep track only of +their hop of each circuit. + +A 'Channel' is an abstract view of sending cells to and from a Tor +relay. Currently, all channels are implemented using OR connections. +If we switch to other strategies in the future, we'll have more +connection types. + +A 'Node' is a view of a Tor instance's current knowledge and opinions +about a Tor relay orbridge. + +### The rest of this document. ### + +> **Note**: This section describes the eventual organization of this +> document, which is not yet complete. + +We'll begin with an overview of the various utility functions available +in Tor's 'common' directory. Knowing about these is key to writing +portable, simple code in Tor. + +Then we'll go on and talk about the main data-flow of the Tor network: +how Tor generates and responds to network traffic. This will occupy a +chapter for the main overview, with other chapters for special topics. + +After that, we'll mention the main modules in Tor, and describe the +function of each. + +We'll cover the directory subsystem next: how Tor learns about other +relays, and how relays advertise themselves. + +Then we'll cover a few specialized modules, such as hidden services, +sandboxing, hibernation, accounting, statistics, guards, path +generation, pluggable transports, and how they integrate with the rest of Tor. + +We'll close with a meandering overview of important pending issues in +the Tor codebase, and how they affect the future of the Tor software. + diff --git a/doc/HACKING/design/01-common-utils.md b/doc/HACKING/design/01-common-utils.md new file mode 100644 index 0000000000..79a6a7b7d3 --- /dev/null +++ b/doc/HACKING/design/01-common-utils.md @@ -0,0 +1,121 @@ + +## Utility code in Tor + +Most of Tor's utility code is in modules in the src/common subdirectory. + +These are divided, broadly, into _compatibility_ functions, _utility_ +functions, _containers_, and _cryptography_. (Someday in the future, it +would be great to split these modules into separate directories. Also, some +functions are probably put in the wrong modules) + +### Compatibility code + +These functions live in src/common/compat\*.c; some corresponding macros live +in src/common/compat\*.h. They serve as wrappers around platform-specific or +compiler-specific logic functionality. + +In general, the rest of the Tor code *should not* be calling platform-specific +or otherwise non-portable functions. Instead, they should call wrappers from +compat.c, which implement a common cross-platform API. (If you don't know +whether a function is portable, it's usually good enough to see whether it +exists on OSX, Linux, and Windows.) + +Other compatibility modules include backtrace.c, which generates stack traces +for crash reporting; sandbox.c, which implements the Linux seccomp2 sandbox; +and procmon.c, which handles monitoring a child process. + +Parts of address.c are compatibility code for handling network addressing +issues; other parts are in util.c. + +Notable compatibility areas are: + + * mmap support for mapping files into the address space (read-only) + + * Code to work around the intricacies + + * Workaround code for Windows's horrible winsock incompatibilities and + Linux's intricate socket extensions. + + * Helpful string functions like memmem, memstr, asprintf, strlcpy, and + strlcat that not all platforms have. + + * Locale-ignoring variants of the ctypes functions. + + * Time-manipulation functions + + * File locking function + + * IPv6 functions for platforms that don't have enough IPv6 support + + * Endianness functions + + * OS functions + + * Threading and locking functions. + +=== Utility functions + +General-purpose utilities are in util.c; they include higher-level wrappers +around many of the compatibility functions to provide things like +file-at-once access, memory management functions, math, string manipulation, +time manipulation, filesystem manipulation, etc. + +(Some functionality, like daemon-launching, would be better off in a +compatibility module.) + +In util_format.c, we have code to implement stuff like base-32 and base-64 +encoding. + +The address.c module interfaces with the system resolver and implements +address parsing and formatting functions. It converts sockaddrs to and from +a more compact tor_addr_t type. + +The di_ops.c module provides constant-time comparison and associative-array +operations, for side-channel avoidance. + +The logging subsystem in log.c supports logging to files, to controllers, to +stdout/stderr, or to the system log. + +The abstraction in memarea.c is used in cases when a large amount of +temporary objects need to be allocated, and they can all be freed at the same +time. + +The torgzip.c module wraps the zlib library to implement compression. + +Workqueue.c provides a simple multithreaded work-queue implementation. + +### Containers + +The container.c module defines these container types, used throughout the Tor +codebase. + +There is a dynamic array called **smartlist**, used as our general resizeable +array type. It supports sorting, searching, common set operations, and so +on. It has specialized functions for smartlists of strings, and for +heap-based priority queues. + +There's a bit-array type. + +A set of mapping types to map strings, 160-bit digests, and 256-bit digests +to void \*. These are what we generally use when we want O(1) lookup. + +Additionally, for containers, we use the ht.h and tor_queue.h headers, in +src/ext. These provide intrusive hashtable and linked-list macros. + +### Cryptography + +Once, we tried to keep our cryptography code in a single "crypto.c" file, +with an "aes.c" module containing an AES implementation for use with older +OpenSSLs. + +Now, our practice has become to introduce crypto_\*.c modules when adding new +cryptography backend code. We have modules for Ed25519, Curve25519, +secret-to-key algorithms, and password-based boxed encryption. + +Our various TLS compatibility code, wrappers, and hacks are kept in +tortls.c, which is probably too full of Tor-specific kludges. I'm +hoping we can eliminate most of those kludges when we finally remove +support for older versions of our TLS handshake. + + + diff --git a/doc/HACKING/design/01a-memory.md b/doc/HACKING/design/01a-memory.md new file mode 100644 index 0000000000..9a20782962 --- /dev/null +++ b/doc/HACKING/design/01a-memory.md @@ -0,0 +1,93 @@ + +## Memory management + +### Heap-allocation functions + +Tor imposes a few light wrappers over C's native malloc and free +functions, to improve convenience, and to allow wholescale replacement +of malloc and free as needed. + +You should never use 'malloc', 'calloc', 'realloc, or 'free' on their +own; always use the variants prefixed with 'tor_'. +They are the same as the standard C functions, with the following +exceptions: + + * tor_free(NULL) is a no-op. + * tor_free() is a macro that takes an lvalue as an argument and sets it to + NULL after freeing it. To avoid this behavior, you can use tor_free_() + instead. + * tor_malloc() and friends fail with an assertion if they are asked to + allocate a value so large that it is probably an underflow. + * It is always safe to tor_malloc(0), regardless of whether your libc + allows it. + * tor_malloc(), tor_realloc(), and friends are never allowed to fail. + Instead, Tor will die with an assertion. This means that you never + need to check their return values. See the next subsection for + information on why we think this is a good idea. + +We define additional general-purpose memory allocation functions as well: + + * tor_malloc_zero(x) behaves as calloc(1, x), except the it makes clear + the intent to allocate a single zeroed-out value. + * tor_reallocarray(x,y) behaves as the OpenBSD reallocarray function. + Use it for cases when you need to realloc() in a multiplication-safe + way. + +And specific-purpose functions as well: + + * tor_strdup() and tor_strndup() behaves as the underlying libc functions, + but use tor_malloc() instead of the underlying function. + * tor_memdup() copies a chunk of memory of a given size. + * tor_memdup_nulterm() copies a chunk of memory of a given size, then + NUL-terminates it just to be safe. + +#### Why assert on failure? + +Why don't we allow tor_malloc() and its allies to return NULL? + +First, it's error-prone. Many programmers forget to check for NULL return +values, and testing for malloc() failures is a major pain. + +Second, it's not necessarily a great way to handle OOM conditions. It's +probably better (we think) to have a memory target where we dynamically free +things ahead of time in order to stay under the target. Trying to respond to +an OOM at the point of tor_malloc() failure, on the other hand, would involve +a rare operation invoked from deep in the call stack. (Again, that's +error-prone and hard to debug.) + +Third, thanks to the rise of Linux and other operating systems that allow +memory to be overcommitted, you can't actually ever rely on getting a NULL +from malloc() when you're out of memory; instead you have to use an approach +closer to tracking the total memory usage. + +#### Conventions for your own allocation functions. + +Whenever you create a new type, the convention is to give it a pair of +x_new() and x_free() functions, named after the type. + +Calling x_free(NULL) should always be a no-op. + + +### Grow-only memory allocation: memarea.c + +It's often handy to allocate a large number of tiny objects, all of which +need to disappear at the same time. You can do this in tor using the +memarea.c abstraction, which uses a set of grow-only buffers for allocation, +and only supports a single "free" operation at the end. + +Using memareas also helps you avoid memory fragmentation. You see, some libc +malloc implementations perform badly on the case where a large number of +small temporary objects are allocated at the same time as a few long-lived +objects of similar size. But if you use tor_malloc() for the long-lived ones +and a memarea for the temporary object, the malloc implementation is likelier +to do better. + +To create a new memarea, use memarea_new(). To drop all the storage from a +memarea, and invalidate its pointers, use memarea_drop_all(). + +The allocation functions memarea_alloc(), memarea_alloc_zero(), +memarea_memdup(), memarea_strdup(), and memarea_strndup() are analogous to +the similarly-named malloc() functions. There is intentionally no +memarea_free() or memarea_realloc(). + + diff --git a/doc/HACKING/design/01b-collections.md b/doc/HACKING/design/01b-collections.md new file mode 100644 index 0000000000..def60b0f15 --- /dev/null +++ b/doc/HACKING/design/01b-collections.md @@ -0,0 +1,43 @@ + +## Collections in tor + +### Smartlists: Neither lists, nor especially smart. + +For historical reasons, we call our dynamic-allocated array type +"smartlist_t". It can grow or shrink as elements are added and removed. + +All smartlists hold an array of void \*. Whenever you expose a smartlist +in an API you *must* document which types its pointers actually hold. + +<!-- It would be neat to fix that, wouldn't it? -NM --> + +Smartlists are created empty with smartlist_new() and freed with +smartlist_free(). See the containers.h module documentation for more +information; there are many convenience functions for commonly needed +operations. + + +### Digest maps, string maps, and more. + +Tor makes frequent use of maps from 160-bit digests, 256-bit digests, +or nul-terminated strings to void \*. These types are digestmap_t, +digest256map_t, and strmap_t respectively. See the containers.h +module documentation for more information. + + +### Intrusive lists and hashtables + +For performance-sensitive cases, we sometimes want to use "intrusive" +collections: ones where the bookkeeping pointers are stuck inside the +structures that belong to the collection. If you've used the +BSD-style sys/queue.h macros, you'll be familiar with these. + +Unfortunately, the sys/queue.h macros vary significantly between the +platforms that have them, so we provide our own variants in +src/ext/tor_queue.h . + +We also provide an intrusive hashtable implementation in src/ext/ht.h +. When you're using it, you'll need to define your own hash +functions. If attacker-induced collisions are a worry here, use the +cryptographic siphash24g function to extract hashes. + diff --git a/doc/HACKING/design/01c-time.md b/doc/HACKING/design/01c-time.md new file mode 100644 index 0000000000..5cd0b354fd --- /dev/null +++ b/doc/HACKING/design/01c-time.md @@ -0,0 +1,75 @@ + +## Time in tor ## + +### What time is it? ### + +We have several notions of the current time in Tor. + +The *wallclock time* is available from time(NULL) with +second-granularity and tor_gettimeofday() with microsecond +granularity. It corresponds most closely to "the current time and date". + +The *monotonic time* is available with the set of monotime_\* +functions declared in compat_time.h. Unlike the wallclock time, it +can only move forward. It does not necessarily correspond to a real +world time, and it is not portable between systems. + +The *coarse monotonic time* is available from the set of +monotime_coarse_\* functions in compat_time.h. It is the same as +monotime_\* on some platforms. On others, it gives a monotonic timer +with less precision, but which it's more efficient to access. + +### Cached views of time. ### + +On some systems (like Linux), many time functions use a VDSO to avoid +the overhead of a system call. But on other systems, gettimeofday() +and time() can be costly enough that you wouldn't want to call them +tens of thousands of times. To get a recent, but not especially +accurate, view of the current time, see approx_time() and +tor_gettimeofday_cached(). + + +### Parsing and encoding time values ### + +Tor has functions to parse and format time in these formats: + + * RFC1123 format. ("Fri, 29 Sep 2006 15:54:20 GMT"). For this, + use format_rfc1123_time() and parse_rfc1123_time. + + * ISO8601 format. ("2006-10-29 10:57:20") For this, use + format_local_iso_time and format_iso_time. We also support the + variant format "2006-10-29T10:57:20" with format_iso_time_nospace, and + "2006-10-29T10:57:20.123456" with format_iso_time_nospace_usec. + + * HTTP format collections (preferably "Mon, 25 Jul 2016 04:01:11 + GMT" or possibly "Wed Jun 30 21:49:08 1993" or even "25-Jul-16 + 04:01:11 GMT"). For this, use parse_http_time. Don't generate anything + but the first format. + +Some of these functions use struct tm. You can use the standard +tor_localtime_r and tor_gmtime_r() to wrap these in a safe way. We +also have a tor_timegm() function. + +### Scheduling events ### + +The main way to schedule a not-too-frequent periodic event with +respect to the Tor mainloop is via the mechanism in periodic.c. +There's a big table of periodic_events in main.c, each of which gets +invoked on its own schedule. You should not expect more than about +one second of accuracy with these timers. + +You can create an independent timer using libevent directly, or using +the periodic_timer_new() function. But you should avoid doing this +for per-connection or per-circuit timers: Libevent's internal timer +implementation uses a min-heap, and those tend to start scaling poorly +once you have a few thousand entries. + +If you need to create a large number of fine-grained timers for some +purpose, you should consider the mechanism in src/common/timers.c, +which is optimized for the case where you have a large number of +timers with not-too-long duration, many of which will be deleted +before they actually expire. These timers should be reasonably +accurate within a handful of milliseconds -- possibly better on some +platforms. (The timers.c module uses William Ahern's timeout.c +implementation as its backend, which is based on a hierarchical timing +wheel algorithm. It's cool stuff; check it out.) diff --git a/doc/HACKING/design/01d-crypto.md b/doc/HACKING/design/01d-crypto.md new file mode 100644 index 0000000000..d4def947d1 --- /dev/null +++ b/doc/HACKING/design/01d-crypto.md @@ -0,0 +1,169 @@ + +## Lower-level cryptography functionality in Tor ## + +Generally speaking, Tor code shouldn't be calling OpenSSL (or any +other crypto library) directly. Instead, we should indirect through +one of the functions in src/common/crypto\*.c or src/common/tortls.c. + +Cryptography functionality that's available is described below. + +### RNG facilities ### + +The most basic RNG capability in Tor is the crypto_rand() family of +functions. These currently use OpenSSL's RAND_() backend, but may use +something faster in the future. + +In addition to crypto_rand(), which fills in a buffer with random +bytes, we also have functions to produce random integers in certain +ranges; to produce random hostnames; to produce random doubles, etc. + +When you're creating a long-term cryptographic secret, you might want +to use crypto_strongest_rand() instead of crypto_rand(). It takes the +operating system's entropy source and combines it with output from +crypto_rand(). This is a pure paranoia measure, but it might help us +someday. + +You can use smartlist_choose() to pick a random element from a smartlist +and smartlist_shuffle() to randomize the order of a smartlist. Both are +potentially a bit slow. + +### Cryptographic digests and related functions ### + +We treat digests as separate types based on the length of their +outputs. We support one 160-bit digest (SHA1), two 256-bit digests +(SHA256 and SHA3-256), and two 512-bit digests (SHA512 and SHA3-512). + +You should not use SHA1 for anything new. + +The crypto_digest\*() family of functions manipulates digests. You +can either compute a digest of a chunk of memory all at once using +crypto_digest(), crypto_digest256(), or crypto_digest512(). Or you +can create a crypto_digest_t object with +crypto_digest{,256,512}_new(), feed information to it in chunks using +crypto_digest_add_bytes(), and then extract the final digest using +crypto_digest_get_digest(). You can copy the state of one of these +objects using crypto_digest_dup() or crypto_digest_assign(). + +We support the HMAC hash-based message authentication code +instantiated using SHA256. See crypto_hmac_sha256. (You should not +add any HMAC users with SHA1, and HMAC is not necessary with SHA3.) + +We also support the SHA3 cousins, SHAKE128 and SHAKE256. Unlike +digests, these are extendable output functions (or XOFs) where you can +get any amount of output. Use the crypto_xof_\*() functions to access +these. + +We have several ways to derive keys from cryptographically strong secret +inputs (like diffie-hellman outputs). The old +crypto_expand_key_material-TAP() performs an ad-hoc KDF based on SHA1 -- you +shouldn't use it for implementing anything but old versions of the Tor +protocol. You can use HKDF-SHA256 (as defined in RFC5869) for more modern +protocols. Also consider SHAKE256. + +If your input is potentially weak, like a password or passphrase, use a salt +along with the secret_to_key() functions as defined in crypto_s2k.c. Prefer +scrypt over other hashing methods when possible. If you're using a password +to encrypt something, see the "boxed file storage" section below. + +Finally, in order to store objects in hash tables, Tor includes the +randomized SipHash 2-4 function. Call it via the siphash24g() function in +src/ext/siphash.h whenever you're creating a hashtable whose keys may be +manipulated by an attacker in order to DoS you with collisions. + + +### Stream ciphers ### + +You can create instances of a stream cipher using crypto_cipher_new(). +These are stateful objects of type crypto_cipher_t. Note that these +objects only support AES-128 right now; a future version should add +support for AES-128 and/or ChaCha20. + +You can encrypt/decrypt with crypto_cipher_encrypt or +crypto_cipher_decrypt. The crypto_cipher_crypt_inplace function performs +an encryption without a copy. + +Note that sensible people should not use raw stream ciphers; they should +probably be using some kind of AEAD. Sorry. + +### Public key functionality ### + +We support four public key algorithms: DH1024, RSA, Curve25519, and +Ed25519. + +We support DH1024 over two prime groups. You access these via the +crypto_dh_\*() family of functions. + +We support RSA in many bit sizes for signing and encryption. You access +it via the crypto_pk_*() family of functions. Note that a crypto_pk_t +may or may not include a private key. See the crypto_pk_* functions in +crypto.c for a full list of functions here. + +For Curve25519 functionality, see the functions and types in +crypto_curve25519.c. Curve25519 is generally suitable for when you need +a secure fast elliptic-curve diffie hellman implementation. When +designing new protocols, prefer it over DH in Z_p. + +For Ed25519 functionality, see the functions and types in +crypto_ed25519.c. Ed25519 is a generally suitable as a secure fast +elliptic curve signature method. For new protocols, prefer it over RSA +signatures. + +### Metaformats for storage ### + +When OpenSSL manages the storage of some object, we use whatever format +OpenSSL provides -- typically, some kind of PEM-wrapped base 64 encoding +that starts with "----- BEGIN CRYPTOGRAPHIC OBJECT ----". + +When we manage the storage of some cryptographic object, we prefix the +object with 32-byte NUL-padded prefix in order to avoid accidental +object confusion; see the crypto_read_tagged_contents_from_file() and +crypto_write_tagged_contents_to_file() functions for manipulating +these. The prefix is "== type: tag ==", where type describes the object +and its encoding, and tag indicates which one it is. + +### Boxed-file storage ### + +When managing keys, you frequently want to have some way to write a +secret object to disk, encrypted with a passphrase. The crypto_pwbox +and crypto_unpwbox functions do so in a way that's likely to be +readable by future versions of Tor. + +### Certificates ### + +We have, alas, several certificate types in Tor. + +The tor_x509_cert_t type represents an X.509 certificate. This document +won't explain X.509 to you -- possibly, no document can. (OTOH, Peter +Gutmann's "x.509 style guide", though severely dated, does a good job of +explaining how awful x.509 can be.) Do not introduce any new usages of +X.509. Right now we only use it in places where TLS forces us to do so. + +The authority_cert_t type is used only for directory authority keys. It +has a medium-term signing key (which the authorities actually keep +online) signed by a long-term identity key (which the authority operator +had really better be keeping offline). Don't use it for any new kind of +certificate. + +For new places where you need a certificate, consider tor_cert_t: it +represents a typed and dated _something_ signed by an Ed25519 key. The +format is described in tor-spec. Unlike x.509, you can write it on a +napkin. + +(Additionally, the Tor directory design uses a fairly wide variety of +documents that include keys and which are signed by keys. You can +consider these documents to be an additional kind of certificate if you +want.) + +### TLS ### + +Tor's TLS implementation is more tightly coupled to OpenSSL than we'd +prefer. You can read most of it in tortls.c. + +Unfortunately, TLS's state machine and our requirement for nonblocking +IO support means that using TLS in practice is a bit hairy, since +logical writes can block on a physical reads, and vice versa. + +If you are lucky, you will never have to look at the code here. + + + diff --git a/doc/HACKING/design/01e-os-compat.md b/doc/HACKING/design/01e-os-compat.md new file mode 100644 index 0000000000..072e95bc8a --- /dev/null +++ b/doc/HACKING/design/01e-os-compat.md @@ -0,0 +1,50 @@ + +## OS compatibility functions ## + +We've got a bunch of functions to wrap differences between various +operating systems where we run. + +### The filesystem ### + +We wrap the most important filesystem functions with load-file, +save-file, and map-file abstractions declared in util.c or compat.c. If +you're messing about with file descriptors yourself, you might be doing +things wrong. Most of the time, write_str_to_file() and +read_str_from_file() are all you need. + +Use the check_private_directory() function to create or verify the +presence of directories, and tor_listdir() to list the files in a +directory. + +Those modules also have functions for manipulating paths a bit. + +### Networking ### + +Nearly all the world is on a Berkeley sockets API, except for +windows, whose version of the Berkeley API was corrupted by late-90s +insistence on backward compatibility with the +sort-of-berkeley-sort-of-not add-on *thing* that was WinSocks. + +What's more, everybody who implemented sockets realized that select() +wasn't a very good way to do nonblocking IO... and then the various +implementations all decided to so something different. + +You can forget about most of these differences, fortunately: We use +libevent to hide most of the differences between the various networking +backends, and we add a few of our own functions to hide the differences +that Libevent doesn't. + +To create a network connection, the right level of abstraction to look +at is probably the connection_t system in connection.c. Most of the +lower level work has already been done for you. If you need to +instantiate something that doesn't fit well with connection_t, you +should see whether you can instantiate it with connection_t anyway -- or +you might need to refactor connection.c a little. + +Whenever possible, represent network addresses as tor_addr_t. + +### Process launch and monitoring ### + +Launching and/or monitoring a process is tricky business. You can use +the mechanisms in procmon.c and tor_spawn_background(), but they're both +a bit wonky. A refactoring would not be out of order. diff --git a/doc/HACKING/design/01f-threads.md b/doc/HACKING/design/01f-threads.md new file mode 100644 index 0000000000..a0dfa2d40e --- /dev/null +++ b/doc/HACKING/design/01f-threads.md @@ -0,0 +1,26 @@ + +## Threads in Tor ## + +Tor is based around a single main thread and one or more worker +threads. We aim (with middling success) to use worker threads for +CPU-intensive activities and the main thread for our networking. +Fortunately (?) we have enough cryptography that moving what we can of the +cryptographic processes to the workers should achieve good parallelism under most +loads. Unfortunately, we only have a small fraction of our +cryptography done in our worker threads right now. + +Our threads-and-workers abstraction is defined in workqueue.c, which +combines a work queue with a thread pool, and integrates the +signalling with libevent. Tor main instance of a work queue is +instantiated in cpuworker.c. It will probably need some refactoring +as more types of work are added. + +On a lower level, we provide locks with tor_mutex_t, conditions with +tor_cond_t, and thread-local storage with tor_threadlocal_t, all of +which are specified in compat_threads.h and implemented in an OS- +specific compat_\*threads.h module. + +Try to minimize sharing between threads: it is usually best to simply +make the worker "own" all the data it needs while the work is in +progress, and to give up ownership when it's complete. + diff --git a/doc/HACKING/design/01g-strings.md b/doc/HACKING/design/01g-strings.md new file mode 100644 index 0000000000..145a35cd6f --- /dev/null +++ b/doc/HACKING/design/01g-strings.md @@ -0,0 +1,95 @@ + +## String processing in Tor ## + +Since you're reading about a C program, you probably expected this +section: it's full of functions for manipulating the (notoriously +dubious) C string abstraction. I'll describe some often-missed +highlights here. + +### Comparing strings and memory chunks ### + +We provide strcmpstart() and strcmpend() to perform a strcmp with the start +or end of a string. + + tor_assert(!strcmpstart("Hello world","Hello")); + tor_assert(!strcmpend("Hello world","world")); + + tor_assert(!strcasecmpstart("HELLO WORLD","Hello")); + tor_assert(!strcasecmpend("HELLO WORLD","world")); + +To compare two string pointers, either of which might be NULL, use +strcmp_opt(). + +To search for a string or a chunk of memory within a non-null +terminated memory block, use tor_memstr or tor_memmem respectively. + +We avoid using memcmp() directly, since it tends to be used in cases +when having a constant-time operation would be better. Instead, we +recommend tor_memeq() and tor_memneq() for when you need a +constant-time operation. In cases when you need a fast comparison, +and timing leaks are not a danger, you can use fast_memeq() and +fast_memneq(). + +It's a common pattern to take a string representing one or more lines +of text, and search within it for some other string, at the start of a +line. You could search for "\\ntarget", but that would miss the first +line. Instead, use find_str_at_start_of_line. + +### Parsing text ### + +Over the years, we have accumulated lots of ways to parse text -- +probably too many. Refactoring them to be safer and saner could be a +good project! The one that seems most error-resistant is tokenizing +text with smartlist_split_strings(). This function takes a smartlist, +a string, and a separator, and splits the string along occurrences of +the separator, adding new strings for the sub-elements to the given +smartlist. + +To handle time, you can use one of the functions mentioned above in +"Parsing and encoding time values". + +For numbers in general, use the tor_parse_{long,ulong,double,uint64} +family of functions. Each of these can be called in a few ways. The +most general is as follows: + + const int BASE = 10; + const int MINVAL = 10, MAXVAL = 10000; + const char *next; + int ok; + long lng = tor_parse_long("100", BASE, MINVAL, MAXVAL, &ok, &next); + +The return value should be ignored if "ok" is set to false. The input +string needs to contain an entire number, or it's considered +invalid... unless the "next" pointer is available, in which case extra +characters at the end are allowed, and "next" is set to point to the +first such character. + +### Generating blocks of text ### + +For not-too-large blocks of text, we provide tor_asprintf(), which +behaves like other members of the sprintf() family, except that it +always allocates enough memory on the heap for its output. + +For larger blocks: Rather than using strlcat and strlcpy to build +text, or keeping pointers to the interior of a memory block, we +recommend that you use the smartlist_* functions to build a smartlist +full of substrings in order. Then you can concatenate them into a +single string with smartlist_join_strings(), which also takes optional +separator and terminator arguments. + +As a convenience, we provide smartlist_add_asprintf(), which combines +the two methods above together. Many of the cryptographic digest +functions also accept a not-yet-concatenated smartlist of strings. + +### Logging helpers ### + +Often we'd like to log a value that comes from an untrusted source. +To do this, use escaped() to escape the nonprintable characters and +other confusing elements in a string, and surround it by quotes. (Use +esc_for_log() if you need to allocate a new string.) + +It's also handy to put memory chunks into hexadecimal before logging; +you can use hex_str(memory, length) for that. + +The escaped() and hex_str() functions both provide outputs that are +only valid till they are next invoked; they are not threadsafe. diff --git a/doc/HACKING/design/02-dataflow.md b/doc/HACKING/design/02-dataflow.md new file mode 100644 index 0000000000..39f21a908c --- /dev/null +++ b/doc/HACKING/design/02-dataflow.md @@ -0,0 +1,236 @@ + +## Data flow in the Tor process ## + +We read bytes from the network, we write bytes to the network. For the +most part, the bytes we write correspond roughly to bytes we have read, +with bits of cryptography added in. + +The rest is a matter of details. + + + +### Connections and buffers: reading, writing, and interpreting. ### + +At a low level, Tor's networking code is based on "connections". Each +connection represents an object that can send or receive network-like +events. For the most part, each connection has a single underlying TCP +stream (I'll discuss counterexamples below). + +A connection that behaves like a TCP stream has an input buffer and an +output buffer. Incoming data is +written into the input buffer ("inbuf"); data to be written to the +network is queued on an output buffer ("outbuf"). + +Buffers are implemented in buffers.c. Each of these buffers is +implemented as a linked queue of memory extents, in the style of classic +BSD mbufs, or Linux skbufs. + +A connection's reading and writing can be enabled or disabled. Under +the hood, this functionality is implemented using libevent events: one +for reading, one for writing. These events are turned on/off in +main.c, in the functions connection_{start,stop}_{reading,writing}. + +When a read or write event is turned on, the main libevent loop polls +the kernel, asking which sockets are ready to read or write. (This +polling happens in the event_base_loop() call in run_main_loop_once() +in main.c.) When libevent finds a socket that's ready to read or write, +it invokes conn_{read,write}_callback(), also in main.c + +These callback functions delegate to connection_handle_read() and +connection_handle_write() in connection.c, which read or write on the +network as appropriate, possibly delegating to openssl. + +After data is read or written, or other event occurs, these +connection_handle_read_write() functions call logic functions whose job is +to respond to the information. Some examples included: + + * connection_flushed_some() -- called after a connection writes any + amount of data from its outbuf. + * connection_finished_flushing() -- called when a connection has + emptied its outbuf. + * connection_finished_connecting() -- called when an in-process connection + finishes making a remote connection. + * connection_reached_eof() -- called after receiving a FIN from the + remote server. + * connection_process_inbuf() -- called when more data arrives on + the inbuf. + +These functions then call into specific implementations depending on +the type of the connection. For example, if the connection is an +edge_connection_t, connection_reached_eof() will call +connection_edge_reached_eof(). + +> **Note:** "Also there are bufferevents!" We have vestigial +> code for an alternative low-level networking +> implementation, based on Libevent's evbuffer and bufferevent +> code. These two object types take on (most of) the roles of +> buffers and connections respectively. It isn't working in today's +> Tor, due to code rot and possible lingering libevent bugs. More +> work is needed; it would be good to get this working efficiently +> again, to have IOCP support on Windows. + + +#### Controlling connections #### + +A connection can have reading or writing enabled or disabled for a +wide variety of reasons, including: + + * Writing is disabled when there is no more data to write + * For some connection types, reading is disabled when the inbuf is + too full. + * Reading/writing is temporarily disabled on connections that have + recently read/written enough data up to their bandwidth + * Reading is disabled on connections when reading more data from them + would require that data to be buffered somewhere else that is + already full. + +Currently, these conditions are checked in a diffuse set of +increasingly complex conditional expressions. In the future, it could +be helpful to transition to a unified model for handling temporary +read/write suspensions. + +#### Kinds of connections #### + +Today Tor has the following connection and pseudoconnection types. +For the most part, each type of channel has an associated C module +that implements its underlying logic. + +**Edge connections** receive data from and deliver data to points +outside the onion routing network. See `connection_edge.c`. They fall into two types: + +**Entry connections** are a type of edge connection. They receive data +from the user running a Tor client, and deliver data to that user. +They are used to implement SOCKSPort, TransPort, NATDPort, and so on. +Sometimes they are called "AP" connections for historical reasons (it +used to stand for "Application Proxy"). + +**Exit connections** are a type of edge connection. They exist at an +exit node, and transmit traffic to and from the network. + +(Entry connections and exit connections are also used as placeholders +when performing a remote DNS request; they are not decoupled from the +notion of "stream" in the Tor protocol. This is implemented partially +in `connection_edge.c`, and partially in `dnsserv.c` and `dns.c`.) + +**OR connections** send and receive Tor cells over TLS, using some +version of the Tor link protocol. Their implementation is spread +across `connection_or.c`, with a bit of logic in `command.c`, +`relay.c`, and `channeltls.c`. + +**Extended OR connections** are a type of OR connection for use on +bridges using pluggable transports, so that the PT can tell the bridge +some information about the incoming connection before passing on its +data. They are implemented in `ext_orport.c`. + +**Directory connections** are server-side or client-side connections +that implement Tor's HTTP-based directory protocol. These are +instantiated using a socket when Tor is making an unencrypted HTTP +connection. When Tor is tunneling a directory request over a Tor +circuit, directory connections are implemented using a linked +connection pair (see below). Directory connections are implemented in +`directory.c`; some of the server-side logic is implemented in +`dirserver.c`. + +**Controller connections** are local connections to a controller +process implementing the controller protocol from +control-spec.txt. These are in `control.c`. + +**Listener connections** are not stream oriented! Rather, they wrap a +listening socket in order to detect new incoming connections. They +bypass most of stream logic. They don't have associated buffers. +They are implemented in `connection.c`. + + + +>**Note**: "History Time!" You might occasionally find reference to a couple types of connections +> which no longer exist in modern Tor. A *CPUWorker connection* +>connected the main Tor process to a thread or process used for +>computation. (Nowadays we use in-process communication.) Even more +>anciently, a *DNSWorker connection* connected the main tor process to +>a separate thread or process used for running `gethostbyname()` or +>`getaddrinfo()`. (Nowadays we use Libevent's evdns facility to +>perform DNS requests asynchronously.) + +#### Linked connections #### + +Sometimes two channels are joined together, such that data which the +Tor process sends on one should immediately be received by the same +Tor process on the other. (For example, when Tor makes a tunneled +directory connection, this is implemented on the client side as a +directory connection whose output goes, not to the network, but to a +local entry connection. And when a directory receives a tunnelled +directory connection, this is implemented as an exit connection whose +output goes, not to the network, but to a local directory connection.) + +The earliest versions of Tor to support linked connections used +socketpairs for the purpose. But using socketpairs forced us to copy +data through kernelspace, and wasted limited file descriptors. So +instead, a pair of connections can be linked in-process. Each linked +connection has a pointer to the other, such that data written on one +is immediately readable on the other, and vice versa. + +### From connections to channels ### + +There's an abstraction layer above OR connections (the ones that +handle cells) and below cells called **Channels**. A channel's +purpose is to transmit authenticated cells from one Tor instance +(relay or client) to another. + +Currently, only one implementation exists: Channel_tls, which sends +and receiveds cells over a TLS-based OR connection. + +Cells are sent on a channel using +`channel_write_{,packed_,var_}cell()`. Incoming cells arrive on a +channel from its backend using `channel_queue*_cell()`, and are +immediately processed using `channel_process_cells()`. + +Some cell types are handled below the channel layer, such as those +that affect handshaking only. And some others are passed up to the +generic cross-channel code in `command.c`: cells like `DESTROY` and +`CREATED` are all trivial to handle. But relay cells +require special handling... + +### From channels through circuits ### + +When a relay cell arrives on an existing circuit, it is handled in +`circuit_receive_relay_cell()` -- one of the innermost functions in +Tor. This function encrypts or decrypts the relay cell as +appropriate, and decides whether the cell is intended for the current +hop of the circuit. + +If the cell *is* intended for the current hop, we pass it to +`connection_edge_process_relay_cell()` in `relay.c`, which acts on it +based on its relay command, and (possibly) queues its data on an +`edge_connection_t`. + +If the cell *is not* intended for the current hop, we queue it for the +next channel in sequence with `append cell_to_circuit_queue()`. This +places the cell on a per-circuit queue for cells headed out on that +particular channel. + +### Sending cells on circuits: the complicated bit. ### + +Relay cells are queued onto circuits from one of two (main) sources: +reading data from edge connections, and receiving a cell to be relayed +on a circuit. Both of these sources place their cells on cell queue: +each circuit has one cell queue for each direction that it travels. + +A naive implementation would skip using cell queues, and instead write +each outgoing relay cell. (Tor did this in its earlier versions.) +But such an approach tends to give poor performance, because it allows +high-volume circuits to clog channels, and it forces the Tor server to +send data queued on a circuit even after that circuit has been closed. + +So by using queues on each circuit, we can add cells to each channel +on a just-in-time basis, choosing the cell at each moment based on +a performance-aware algorithm. + +This logic is implemented in two main modules: `scheduler.c` and +`circuitmux*.c`. The scheduler code is responsible for determining +globally, across all channels that could write cells, which one should +next receive queued cells. The circuitmux code determines, for all +of the circuits with queued cells for a channel, which one should +queue the next cell. + +(This logic applies to outgoing relay cells only; incoming relay cells +are processed as they arrive.) diff --git a/doc/HACKING/design/03-modules.md b/doc/HACKING/design/03-modules.md new file mode 100644 index 0000000000..93eb9d3089 --- /dev/null +++ b/doc/HACKING/design/03-modules.md @@ -0,0 +1,247 @@ + +## Tor's modules ## + +### Generic modules ### + +`buffers.c` +: Implements the `buf_t` buffered data type for connections, and several +low-level data handling functions to handle network protocols on it. + +`channel.c` +: Generic channel implementation. Channels handle sending and receiving cells +among tor nodes. + +`channeltls.c` +: Channel implementation for TLS-based OR connections. Uses `connection_or.c`. + +`circuitbuild.c` +: Code for constructing circuits and choosing their paths. (*Note*: +this module could plausibly be split into handling the client side, +the server side, and the path generation aspects of circuit building.) + +`circuitlist.c` +: Code for maintaining and navigating the global list of circuits. + +`circuitmux.c` +: Generic circuitmux implementation. A circuitmux handles deciding, for a +particular channel, which circuit should write next. + +`circuitmux_ewma.c` +: A circuitmux implementation based on the EWMA (exponentially +weighted moving average) algorithm. + +`circuituse.c` +: Code to actually send and receive data on circuits. + +`command.c` +: Handles incoming cells on channels. + +`config.c` +: Parses options from torrc, and uses them to configure the rest of Tor. + +`confparse.c` +: Generic torrc-style parser. Used to parse torrc and state files. + +`connection.c` +: Generic and common connection tools, and implementation for the simpler +connection types. + +`connection_edge.c` +: Implementation for entry and exit connections. + +`connection_or.c` +: Implementation for OR connections (the ones that send cells over TLS). + +`main.c` +: Principal entry point, main loops, scheduled events, and network +management for Tor. + +`ntmain.c` +: Implements Tor as a Windows service. (Not very well.) + +`onion.c` +: Generic code for generating and responding to CREATE and CREATED +cells, and performing the appropriate onion handshakes. Also contains +code to manage the server-side onion queue. + +`onion_fast.c` +: Implements the old SHA1-based CREATE_FAST/CREATED_FAST circuit +creation handshake. (Now deprecated.) + +`onion_ntor.c` +: Implements the Curve25519-based NTOR circuit creation handshake. + +`onion_tap.c` +: Implements the old RSA1024/DH1024-based TAP circuit creation handshake. (Now +deprecated.) + +`relay.c` +: Handles particular types of relay cells, and provides code to receive, +encrypt, route, and interpret relay cells. + +`scheduler.c` +: Decides which channel/circuit pair is ready to receive the next cell. + +`statefile.c` +: Handles loading and storing Tor's state file. + +`tor_main.c` +: Contains the actual `main()` function. (This is placed in a separate +file so that the unit tests can have their own `main()`.) + + +### Node-status modules ### + +`directory.c` +: Implements the HTTP-based directory protocol, including sending, +receiving, and handling most request types. (*Note*: The client parts +of this, and the generic-HTTP parts of this, could plausibly be split +off.) + +`microdesc.c` +: Implements the compact "microdescriptor" format for keeping track of +what we know about a router. + +`networkstatus.c` +: Code for fetching, storing, and interpreting consensus vote documents. + +`nodelist.c` +: Higher-level view of our knowledge of which Tor servers exist. Each +`node_t` corresponds to a router we know about. + +`routerlist.c` +: Code for storing and retrieving router descriptors and extrainfo +documents. + +`routerparse.c` +: Generic and specific code for parsing all Tor directory information +types. + +`routerset.c` +: Parses and interprets a specification for a set of routers (by IP +range, fingerprint, nickname (deprecated), or country). + + +### Client modules ### + +`addressmap.c` +: Handles client-side associations between one address and another. +These are used to implement client-side DNS caching (NOT RECOMMENDED), +MapAddress directives, Automapping, and more. + +`circpathbias.c` +: Path bias attack detection for circuits: tracks whether +connections made through a particular guard have an unusually high failure rate. + +`circuitstats.c` +: Code to track circuit performance statistics in order to adapt our behavior. +Notably includes an algorithm to track circuit build times. + +`dnsserv.c` +: Implements DNSPort for clients. (Note that in spite of the word +"server" in this module's name, it is used for Tor clients. It +implements a DNS server, not DNS for servers.) + +`entrynodes.c` +: Chooses, monitors, and remembers guard nodes. Also contains some +bridge-related code. + +`torcert.c` +: Code to interpret and generate Ed25519-based certificates. + +### Server modules ### + +`dns.c` +: Server-side DNS code. Handles sending and receiving DNS requests on +exit nodes, and implements the server-side DNS cache. + +`dirserv.c` +: Implements part of directory caches that handles responding to +client requests. + +`ext_orport.c` +: Implements the extended ORPort protocol for communication between +server-side pluggable transports and Tor servers. + +`hibernate.c` +: Performs bandwidth accounting, and puts Tor relays into hibernation +when their bandwidth is exhausted. + +`router.c` +: Management code for running a Tor server. In charge of RSA key +maintenance, descriptor generation and uploading. + +`routerkeys.c` +: Key handling code for a Tor server. (Currently handles only the +Ed25519 keys, but the RSA keys could be moved here too.) + + +### Onion service modules ### + +`rendcache.c` +: Stores onion service descriptors. + +`rendclient.c` +: Client-side implementation of the onion service protocol. + +`rendcommon.c` +: Parts of the onion service protocol that are shared by clients, +services, and/or Tor servers. + +`rendmid.c` +: Tor-server-side implementation of the onion service protocol. (Handles +acting as an introduction point or a rendezvous point.) + +`rendservice.c` +: Service-side implementation of the onion service protocol. + +`replaycache.c` +: Backend to check introduce2 requests for replay attempts. + + +### Authority modules ### + +`dircollate.c` +: Helper for `dirvote.c`: Given a set of votes, each containing a list +of Tor nodes, determines which entries across all the votes correspond +to the same nodes, and yields them in a useful order. + +`dirvote.c` +: Implements the directory voting algorithms that authorities use. + +`keypin.c` +: Implements a persistent key-pinning mechanism to tie RSA1024 +identities to ed25519 identities. + +### Miscellaneous modules ### + +`control.c` +: Implements the Tor controller protocol. + +`cpuworker.c` +: Implements the inner work queue function. We use this to move the +work of circuit creation (on server-side) to other CPUs. + +`fp_pair.c` +: Types for handling 2-tuples of 20-byte fingerprints. + +`geoip.c` +: Parses geoip files (which map IP addresses to country codes), and +performs lookups on the internal geoip table. Also stores some +geoip-related statistics. + +`policies.c` +: Parses and implements Tor exit policies. + +`reasons.c` +: Maps internal reason-codes to human-readable strings. + +`rephist.c` +: Tracks Tor servers' performance over time. + +`status.c` +: Writes periodic "heartbeat" status messages about the state of the Tor +process. + +`transports.c` +: Implements management for the pluggable transports subsystem. diff --git a/doc/HACKING/design/Makefile b/doc/HACKING/design/Makefile new file mode 100644 index 0000000000..e126130970 --- /dev/null +++ b/doc/HACKING/design/Makefile @@ -0,0 +1,34 @@ + + + +HTML= \ + 00-overview.html \ + 01-common-utils.html \ + 01a-memory.html \ + 01b-collections.html \ + 01c-time.html \ + 01d-crypto.html \ + 01e-os-compat.html \ + 01f-threads.html \ + 01g-strings.html \ + 02-dataflow.html \ + 03-modules.html \ + this-not-that.html + +PNG = \ + diagrams/02/02-dataflow.png \ + diagrams/02/02-connection-types.png + +all: generated + +generated: $(HTML) $(PNG) + +%.html: %.md + maruku $< -o $@ + +%.png: %.dia + dia $< --export=$@ + +clean: + rm -f $(HTML) + rm -f $(PNG) diff --git a/doc/HACKING/design/this-not-that.md b/doc/HACKING/design/this-not-that.md new file mode 100644 index 0000000000..815c7b2fbc --- /dev/null +++ b/doc/HACKING/design/this-not-that.md @@ -0,0 +1,51 @@ + +Don't use memcmp. Use {tor,fast}_{memeq,memneq,memcmp}. + +Don't use assert. Use tor_assert or tor_assert_nonfatal or BUG. Prefer +nonfatal assertions or BUG()s. + +Don't use sprintf or snprintf. Use tor_asprintf or tor_snprintf. + +Don't write hand-written binary parsers. Use trunnel. + +Don't use malloc, realloc, calloc, free, strdup, etc. Use tor_malloc, +tor_realloc, tor_calloc, tor_free, tor_strdup, etc. + +Don't use tor_realloc(x, y\*z). Use tor_reallocarray(x, y, z); + +Don't say "if (x) foo_free(x)". Just foo_free(x) and make sure that +foo_free(NULL) is a no-op. + +Don't use toupper or tolower; use TOR_TOUPPER and TOR_TOLOWER. + +Don't use isalpha, isalnum, etc. Instead use TOR_ISALPHA, TOR_ISALNUM, etc. + +Don't use strcat, strcpy, strncat, or strncpy. Use strlcat and strlcpy +instead. + +Don't use tor_asprintf then smartlist_add; use smartlist_add_asprintf. + +Don't use any of these functions: they aren't portable. Use the +version prefixed with `tor_` instead: strtok_r, memmem, memstr, +asprintf, localtime_r, gmtime_r, inet_aton, inet_ntop, inet_pton, +getpass, ntohll, htonll, strdup, (This list is incomplete.) + +Don't create or close sockets directly. Instead use the wrappers in +compat.h. + +When creating new APIs, only use 'char \*' to represent 'pointer to a +nul-terminated string'. Represent 'pointer to a chunk of memory' as +'uint8_t \*'. (Many older Tor APIs ignore this rule.) + +Don't encode/decode u32, u64, or u16 to byte arrays by casting +pointers. That can crash if the pointers aren't aligned, and can cause +endianness problems. Instead say something more like set_uint32(ptr, +htonl(foo)) to encode, and ntohl(get_uint32(ptr)) to decode. + +Don't declare a 0-argument function with "void foo()". That's C++ +syntax. In C you say "void foo(void)". + +When creating new APIs, use const everywhere you reasonably can. + +Sockets should have type tor_socket_t, not int. + diff --git a/doc/include.am b/doc/include.am index 0a123aae11..a9d3fa1c98 100644 --- a/doc/include.am +++ b/doc/include.am @@ -15,17 +15,32 @@ all_mans = doc/tor doc/tor-gencert doc/tor-resolve doc/torify doc/tor-print-ed-signing-cert if USE_ASCIIDOC -nodist_man1_MANS = $(all_mans:=.1) -doc_DATA = $(all_mans:=.html) +txt_in = $(all_mans:=.1.txt) + +if BUILD_HTML_DOCS html_in = $(all_mans:=.html.in) +doc_DATA = $(all_mans:=.html) +else +html_in = +doc_DATA = +endif + +if BUILD_MANPAGE +nodist_man1_MANS = $(all_mans:=.1) man_in = $(all_mans:=.1.in) -txt_in = $(all_mans:=.1.txt) else +nodist_man1_MANS = +man_in = +endif + +else + html_in = +doc_DATA = man_in = txt_in = nodist_man1_MANS = -doc_DATA = + endif EXTRA_DIST+= doc/asciidoc-helper.sh \ diff --git a/doc/tor-print-ed-signing-cert.1.txt b/doc/tor-print-ed-signing-cert.1.txt index 1a3109df95..48a3f095d5 100644 --- a/doc/tor-print-ed-signing-cert.1.txt +++ b/doc/tor-print-ed-signing-cert.1.txt @@ -21,6 +21,12 @@ DESCRIPTION **tor-print-ed-signing-cert** is utility program for Tor relay operators to check expiration date of ed25519 signing certificate. +Expiration date is printed in three formats: + +* Human-readable timestamp, localized to current timezone. +* RFC 1123 timestamp (at GMT timezone). +* Unix time value. + SEE ALSO -------- **tor**(1) + diff --git a/doc/tor.1.txt b/doc/tor.1.txt index 95d56c6dbd..1504223b89 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -37,7 +37,7 @@ Project's website. COMMAND-LINE OPTIONS -------------------- -[[opt-h]] **-h**, **-help**:: +[[opt-h]] **-h**, **--help**:: Display a short help message and exit. [[opt-f]] **-f** __FILE__:: @@ -211,10 +211,14 @@ GENERAL OPTIONS Note that this option, and other bandwidth-limiting options, apply to TCP data only: They do not count TCP headers or DNS traffic. + + + Tor uses powers of two, not powers of ten, so 1 GByte is + 1024*1024*1024 bytes as opposed to 1 billion bytes. + + + With this option, and in other options that take arguments in bytes, KBytes, and so on, other formats are also supported. Notably, "KBytes" can also be written as "kilobytes" or "kb"; "MBytes" can be written as "megabytes" or "MB"; "kbits" can be written as "kilobits"; and so forth. + Case doesn't matter. Tor also accepts "byte" and "bit" in the singular. The prefixes "tera" and "T" are also recognized. If no units are given, we default to bytes. @@ -447,13 +451,18 @@ GENERAL OPTIONS setting for DataDirectoryGroupReadable when the CacheDirectory is the same as the DataDirectory, and 0 otherwise. (Default: auto) -[[FallbackDir]] **FallbackDir** __ipv4address__:__port__ orport=__port__ id=__fingerprint__ [weight=__num__] [ipv6=**[**__ipv6address__**]**:__orport__]:: - When we're unable to connect to any directory cache for directory info - (usually because we don't know about any yet) we try a directory authority. - Clients also simultaneously try a FallbackDir, to avoid hangs on client - startup if a directory authority is down. Clients retry FallbackDirs more - often than directory authorities, to reduce the load on the directory - authorities. +[[FallbackDir]] **FallbackDir** __ipv4address__:__dirport__ orport=__orport__ id=__fingerprint__ [weight=__num__] [ipv6=**[**__ipv6address__**]**:__orport__]:: + When tor is unable to connect to any directory cache for directory info + (usually because it doesn't know about any yet) it tries a hard-coded + directory. Relays try one directory authority at a time. Clients try + multiple directory authorities and FallbackDirs, to avoid hangs on + startup if a hard-coded directory is down. Clients wait for a few seconds + between each attempt, and retry FallbackDirs more often than directory + authorities, to reduce the load on the directory authorities. + + + + FallbackDirs should be stable relays with stable IP addresses, ports, + and identity keys. They must have a DirPort. + + + By default, the directory authorities are also FallbackDirs. Specifying a FallbackDir replaces Tor's default hard-coded FallbackDirs (if any). (See the **DirAuthority** entry for an explanation of each flag.) @@ -463,30 +472,30 @@ GENERAL OPTIONS FallbackDir line is present, it replaces the hard-coded FallbackDirs, regardless of the value of UseDefaultFallbackDirs.) (Default: 1) -[[DirAuthority]] **DirAuthority** [__nickname__] [**flags**] __ipv4address__:__port__ __fingerprint__:: +[[DirAuthority]] **DirAuthority** [__nickname__] [**flags**] __ipv4address__:__dirport__ __fingerprint__:: Use a nonstandard authoritative directory server at the provided address and port, with the specified key fingerprint. This option can be repeated many times, for multiple authoritative directory servers. Flags are separated by spaces, and determine what kind of an authority this directory is. By default, an authority is not authoritative for any directory style - or version unless an appropriate flag is given. + or version unless an appropriate flag is given. + + + Tor will use this authority as a bridge authoritative directory if the - "bridge" flag is set. If a flag "orport=**port**" is given, Tor will use the - given port when opening encrypted tunnels to the dirserver. If a flag - "weight=**num**" is given, then the directory server is chosen randomly - with probability proportional to that weight (default 1.0). If a + "bridge" flag is set. If a flag "orport=**orport**" is given, Tor will + use the given port when opening encrypted tunnels to the dirserver. If a + flag "weight=**num**" is given, then the directory server is chosen + randomly with probability proportional to that weight (default 1.0). If a flag "v3ident=**fp**" is given, the dirserver is a v3 directory authority whose v3 long-term signing key has the fingerprint **fp**. Lastly, if an "ipv6=**[**__ipv6address__**]**:__orport__" flag is present, then - the directory - authority is listening for IPv6 connections on the indicated IPv6 address - and OR Port. + + the directory authority is listening for IPv6 connections on the + indicated IPv6 address and OR Port. + + Tor will contact the authority at __ipv4address__ to - download directory documents. The provided __port__ value is a dirport; - clients ignore this in favor of the specified "orport=" value. If an - IPv6 ORPort is supplied, Tor will - also download directory documents at the IPv6 ORPort. + + download directory documents. Clients always use the ORPort. Relays + usually use the DirPort, but will use the ORPort in some circumstances. + If an IPv6 ORPort is supplied, clients will also download directory + documents at the IPv6 ORPort, if they are configured to use IPv6. + + If no **DirAuthority** line is given, Tor will use the default directory authorities. NOTE: this option is intended for setting up a private Tor @@ -658,7 +667,16 @@ GENERAL OPTIONS debug, info, notice, warn, and err. We advise using "notice" in most cases, since anything more verbose may provide sensitive information to an attacker who obtains the logs. If only one severity level is given, all - messages of that level or higher will be sent to the listed destination. + messages of that level or higher will be sent to the listed destination. + + + + Some low-level logs may be sent from signal handlers, so their destination + logs must be signal-safe. These low-level logs include backtraces, + logging function errors, and errors in code called by logging functions. + Signal-safe logs are always sent to stderr or stdout. They are also sent to + a limited number of log files that are configured to log messages at error + severity from the bug or general domains. They are never sent as syslogs, + android logs, control port log events, or to any API-based log + destinations. [[Log2]] **Log** __minSeverity__[-__maxSeverity__] **file** __FILENAME__:: As above, but send log messages to the listed filename. The @@ -1128,7 +1146,7 @@ The following options are useful only for clients (that is, if information) to port 80. [[HidServAuth]] **HidServAuth** __onion-address__ __auth-cookie__ [__service-name__]:: - Client authorization for a hidden service. Valid onion addresses contain 16 + Client authorization for a v2 hidden service. Valid onion addresses contain 16 characters in a-z2-7 plus ".onion", and valid auth cookies contain 22 characters in A-Za-z0-9+/. The service name is only used for internal purposes, e.g., for Tor controllers. This option may be used multiple times @@ -1454,10 +1472,9 @@ The following options are useful only for clients (that is, if default to 3 if the consensus parameter isn't set. (Default: 0) [[GuardLifetime]] **GuardLifetime** __N__ **days**|**weeks**|**months**:: - If nonzero, and UseEntryGuards is set, minimum time to keep a guard before - picking a new one. If zero, we use the GuardLifetime parameter from the - consensus directory. No value here may be less than 1 month or greater - than 5 years; out-of-range values are clamped. (Default: 0) + If UseEntryGuards is set, minimum time to keep a guard on our guard list + before picking a new one. If less than one day, we use defaults from the + consensus directory. (Default: 0) [[SafeSocks]] **SafeSocks** **0**|**1**:: When this option is enabled, Tor will reject application connections that @@ -1939,10 +1956,6 @@ is non-zero): would like its bridge address to be given out. Set it to "none" if you want BridgeDB to avoid distributing your bridge address, or "any" to let BridgeDB decide. (Default: any) - + - Note: as of Oct 2017, the BridgeDB part of this option is not yet - implemented. Until BridgeDB is updated to obey this option, your - bridge will make this request, but it will not (yet) be obeyed. [[ContactInfo]] **ContactInfo** __email_address__:: Administrative contact information for this relay or bridge. This line @@ -2279,9 +2292,9 @@ is non-zero): using a given calculation rule (see: AccountingStart, AccountingRule). Useful if you need to stay under a specific bandwidth. By default, the number used for calculation is the max of either the bytes sent or - received. For example, with AccountingMax set to 1 GByte, a server - could send 900 MBytes and receive 800 MBytes and continue running. - It will only hibernate once one of the two reaches 1 GByte. This can + received. For example, with AccountingMax set to 1 TByte, a server + could send 900 GBytes and receive 800 GBytes and continue running. + It will only hibernate once one of the two reaches 1 TByte. This can be changed to use the sum of the both bytes received and sent by setting the AccountingRule option to "sum" (total bandwidth in/out). When the number of bytes remaining gets low, Tor will stop accepting new connections @@ -2292,7 +2305,12 @@ is non-zero): enabling hibernation is preferable to setting a low bandwidth, since it provides users with a collection of fast servers that are up some of the time, which is more useful than a set of slow servers that are - always "available". + always "available". + + + + Note that (as also described in the Bandwidth section) Tor uses + powers of two, not powers of ten: 1 GByte is 1024*1024*1024, not + one billion. Be careful: some internet service providers might count + GBytes differently. [[AccountingRule]] **AccountingRule** **sum**|**max**|**in**|**out**:: How we determine when our AccountingMax has been reached (when we @@ -2722,12 +2740,6 @@ on the public Tor network. multiple times: the values from multiple lines are spliced together. When this is set then **VersioningAuthoritativeDirectory** should be set too. -[[RecommendedPackages]] **RecommendedPackages** __PACKAGENAME__ __VERSION__ __URL__ __DIGESTTYPE__**=**__DIGEST__ :: - Adds "package" line to the directory authority's vote. This information - is used to vote on the correct URL and digest for the released versions - of different Tor-related packages, so that the consensus can certify - them. This line may appear any number of times. - [[RecommendedClientVersions]] **RecommendedClientVersions** __STRING__:: STRING is a comma-separated list of Tor versions currently believed to be safe for clients to use. This information is included in version 2 @@ -2916,7 +2928,13 @@ on the public Tor network. HIDDEN SERVICE OPTIONS ---------------------- -The following options are used to configure a hidden service. +The following options are used to configure a hidden service. Some options +apply per service and some apply for the whole tor instance. + +The next section describes the per service options that can only be set +**after** the **HiddenServiceDir** directive + +**PER SERVICE OPTIONS:** [[HiddenServiceDir]] **HiddenServiceDir** __DIRECTORY__:: Store data files for a hidden service in DIRECTORY. Every hidden service @@ -2942,18 +2960,12 @@ The following options are used to configure a hidden service. connects to that VIRTPORT, one of the TARGETs from those lines will be chosen at random. Note that address-port pairs have to be comma-separated. -[[PublishHidServDescriptors]] **PublishHidServDescriptors** **0**|**1**:: - If set to 0, Tor will run any hidden services you configure, but it won't - advertise them to the rendezvous directory. This option is only useful if - you're using a Tor controller that handles hidserv publishing for you. - (Default: 1) - [[HiddenServiceVersion]] **HiddenServiceVersion** **2**|**3**:: A list of rendezvous service descriptor versions to publish for the hidden service. Currently, versions 2 and 3 are supported. (Default: 3) [[HiddenServiceAuthorizeClient]] **HiddenServiceAuthorizeClient** __auth-type__ __client-name__,__client-name__,__...__:: - If configured, the hidden service is accessible for authorized clients + If configured, the v2 hidden service is accessible for authorized clients only. The auth-type can either be \'basic' for a general-purpose authorization protocol or \'stealth' for a less scalable protocol that also hides service activity from unauthorized clients. Only clients that are @@ -2974,7 +2986,7 @@ The following options are used to configure a hidden service. [[HiddenServiceExportCircuitID]] **HiddenServiceExportCircuitID** __protocol__:: The onion service will use the given protocol to expose the global circuit - identifier of each inbound client circuit via the selected protocol. The only + identifier of each inbound client circuit. The only protocol supported right now \'haproxy'. This option is only for v3 services. (Default: none) + + @@ -2991,12 +3003,12 @@ The following options are used to configure a hidden service. + global_circuit_id = (0xAA << 24) + (0xBB << 16) + (0xCC << 8) + 0xDD; + + - In the case above, where the last 32-bit is 0xffffffff, the global circuit + In the case above, where the last 32-bits are 0xffffffff, the global circuit identifier would be 4294967295. You can use this value together with Tor's - control port where it is possible to terminate a circuit given the global - circuit identifier. For more information about this see controls-spec.txt. + + control port to terminate particular circuits using their global + circuit identifiers. For more information about this see control-spec.txt. + + - The HAProxy version 1 proxy protocol is described in detail at + The HAProxy version 1 protocol is described in detail at https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt [[HiddenServiceMaxStreams]] **HiddenServiceMaxStreams** __N__:: @@ -3026,6 +3038,38 @@ The following options are used to configure a hidden service. Number of introduction points the hidden service will have. You can't have more than 10 for v2 service and 20 for v3. (Default: 3) +[[HiddenServiceEnableIntroDoSDefense]] **HiddenServiceEnableIntroDoSDefense** **0**|**1**:: + Enable DoS defense at the intropoint level. When this is enabled, the + rate and burst parameter (see below) will be sent to the intro point which + will then use them to apply rate limiting for introduction request to this + service. + + + The introduction point honors the consensus parameters except if this is + specifically set by the service operator using this option. The service + never looks at the consensus parameters in order to enable or disable this + defense. (Default: 0) + +[[HiddenServiceEnableIntroDoSRatePerSec]] **HiddenServiceEnableIntroDoSRatePerSec** __NUM__:: + The allowed client introduction rate per second at the introduction + point. If this option is 0, it is considered infinite and thus if + **HiddenServiceEnableIntroDoSDefense** is set, it then effectively + disables the defenses. (Default: 25) + +[[HiddenServiceEnableIntroDoSBurstPerSec]] **HiddenServiceEnableIntroDoSBurstPerSec** __NUM__:: + The allowed client introduction burst per second at the introduction + point. If this option is 0, it is considered infinite and thus if + **HiddenServiceEnableIntroDoSDefense** is set, it then effectively + disables the defenses. (Default: 200) + + +**PER INSTANCE OPTIONS:** + +[[PublishHidServDescriptors]] **PublishHidServDescriptors** **0**|**1**:: + If set to 0, Tor will run any hidden services you configure, but it won't + advertise them to the rendezvous directory. This option is only useful if + you're using a Tor controller that handles hidserv publishing for you. + (Default: 1) + [[HiddenServiceSingleHopMode]] **HiddenServiceSingleHopMode** **0**|**1**:: **Experimental - Non Anonymous** Hidden Services on a tor instance in HiddenServiceSingleHopMode make one-hop (direct) circuits between the onion @@ -3065,31 +3109,42 @@ Client Authorization (Version 3 only) -To configure client authorization on the service side, the -"<HiddenServiceDir>/authorized_clients/" directory needs to exist. Each file -in that directory should be suffixed with ".auth" (i.e. "alice.auth"; the -file name is irrelevant) and its content format MUST be: +Service side: + + To configure client authorization on the service side, the + "<HiddenServiceDir>/authorized_clients/" directory needs to exist. Each file + in that directory should be suffixed with ".auth" (i.e. "alice.auth"; the + file name is irrelevant) and its content format MUST be: + + <auth-type>:<key-type>:<base32-encoded-public-key> + + The supported <auth-type> are: "descriptor". The supported <key-type> are: + "x25519". The <base32-encoded-public-key> is the base32 representation of + the raw key bytes only (32 bytes for x25519). + + Each file MUST contain one line only. Any malformed file will be + ignored. Client authorization will only be enabled for the service if tor + successfully loads at least one authorization file. - <auth-type>:<key-type>:<base32-encoded-public-key> + Note that once you've configured client authorization, anyone else with the + address won't be able to access it from this point on. If no authorization is + configured, the service will be accessible to anyone with the onion address. -The supported <auth-type> are: "descriptor". The supported <key-type> are: -"x25519". The <base32-encoded-public-key> is the base32 representation of -the raw key bytes only (32 bytes for x25519). + Revoking a client can be done by removing their ".auth" file, however the + revocation will be in effect only after the tor process gets restarted even if + a SIGHUP takes place. -Each file MUST contain one line only. Any malformed file will be -ignored. Client authorization will only be enabled for the service if tor -successfully loads at least one authorization file. +Client side: -Note that once you've configured client authorization, anyone else with the -address won't be able to access it from this point on. If no authorization is -configured, the service will be accessible to anyone with the onion address. + To access a v3 onion service with client authorization as a client, make sure + you have ClientOnionAuthDir set in your torrc. Then, in the + <ClientOnionAuthDir> directory, create an .auth_private file for the onion + service corresponding to this key (i.e. 'bob_onion.auth_private'). The + contents of the <ClientOnionAuthDir>/<user>.auth_private file should look like: -Revoking a client can be done by removing their ".auth" file, however the -revocation will be in effect only after the tor process gets restarted even if -a SIGHUP takes place. + <56-char-onion-addr-without-.onion-part>:descriptor:x25519:<x25519 private key in base32> -See the Appendix G in the rend-spec-v3.txt file of -https://spec.torproject.org/[torspec] for more information. +For more information, please see https://2019.www.torproject.org/docs/tor-onion-service.html.en#ClientAuthorization . TESTING NETWORK OPTIONS ----------------------- diff --git a/scripts/coccinelle/debugmm.cocci b/scripts/coccinelle/debugmm.cocci new file mode 100644 index 0000000000..dbd308df33 --- /dev/null +++ b/scripts/coccinelle/debugmm.cocci @@ -0,0 +1,29 @@ +// Look for use of expressions with side-effects inside of debug logs. +// +// This script detects expressions like ++E, --E, E++, and E-- inside of +// calls to log_debug(). +// +// The log_debug() macro exits early if debug logging is not enabled, +// potentially causing problems if its arguments have side-effects. + +@@ +expression E; +@@ +*log_debug(... , <+... --E ...+>, ... ); + + +@@ +expression E; +@@ +*log_debug(... , <+... ++E ...+>, ... ); + +@@ +expression E; +@@ +*log_debug(... , <+... E-- ...+>, ... ); + + +@@ +expression E; +@@ +*log_debug(... , <+... E++ ...+>, ... ); diff --git a/scripts/git/git-merge-forward.sh b/scripts/git/git-merge-forward.sh index 67af7e98bf..bdd0da5b75 100755 --- a/scripts/git/git-merge-forward.sh +++ b/scripts/git/git-merge-forward.sh @@ -1,52 +1,110 @@ -#!/bin/bash +#!/usr/bin/env bash -############################## -# Configuration (change me!) # -############################## +SCRIPT_NAME=$(basename "$0") -# The general setup that is suggested here is: -# -# GIT_PATH = /home/<user>/git/ -# ... where the git repository directories resides. -# TOR_MASTER_NAME = "tor" -# ... which means that tor.git was cloned in /home/<user>/git/tor -# TOR_WKT_NAME = "tor-wkt" -# ... which means that the tor worktrees are in /home/<user>/git/tor-wkt +function usage() +{ + echo "$SCRIPT_NAME [-h] [-n] [-t <test-branch-prefix> [-u]]" + echo + echo " arguments:" + echo " -h: show this help text" + echo " -n: dry run mode" + echo " (default: run commands)" + echo " -t: test branch mode: create new branches from the commits checked" + echo " out in each maint directory. Call these branches prefix_029," + echo " prefix_035, ... , prefix_master." + echo " (default: merge forward maint-*, release-*, and master)" + echo " -u: in test branch mode, if a prefix_* branch already exists," + echo " skip creating that branch. Use after a merge error, to" + echo " restart the merge forward at the first unmerged branch." + echo " (default: if a prefix_* branch already exists, fail and exit)" + echo + echo " env vars:" + echo " required:" + echo " TOR_FULL_GIT_PATH: where the git repository directories reside." + echo " You must set this env var, we recommend \$HOME/git/" + echo " (default: fail if this env var is not set;" + echo " current: $GIT_PATH)" + echo + echo " optional:" + echo " TOR_MASTER: the name of the directory containing the tor.git clone" + echo " The tor master git directory is \$GIT_PATH/\$TOR_MASTER" + echo " (default: tor; current: $TOR_MASTER_NAME)" + echo " TOR_WKT_NAME: the name of the directory containing the tor" + echo " worktrees. The tor worktrees are:" + echo " \$GIT_PATH/\$TOR_WKT_NAME/{maint-*,release-*}" + echo " (default: tor-wkt; current: $TOR_WKT_NAME)" + echo " we recommend that you set these env vars in your ~/.profile" +} + +################# +# Configuration # +################# + +# Don't change this configuration - set the env vars in your .profile # Where are all those git repositories? -GIT_PATH="FULL_PATH_TO_GIT_REPOSITORY_DIRECTORY" +GIT_PATH=${TOR_FULL_GIT_PATH:-"FULL_PATH_TO_GIT_REPOSITORY_DIRECTORY"} # The tor master git repository directory from which all the worktree have # been created. -TOR_MASTER_NAME="tor" +TOR_MASTER_NAME=${TOR_MASTER_NAME:-"tor"} # The worktrees location (directory). -TOR_WKT_NAME="tor-wkt" +TOR_WKT_NAME=${TOR_WKT_NAME:-"tor-wkt"} -######################### -# End of configuration. # -######################### +########################## +# Git branches to manage # +########################## + +# The branches and worktrees need to be modified when there is a new branch, +# and when an old branch is no longer supported. # Configuration of the branches that needs merging. The values are in order: -# (1) Branch name that we merge onto. -# (2) Branch name to merge from. In other words, this is merge into (1) -# (3) Full path of the git worktree. +# (0) current maint/release branch name +# (1) previous maint/release name to merge into (0) +# (only used in merge forward mode) +# (2) Full path of the git worktree +# (3) current branch suffix +# (maint branches only, only used in test branch mode) +# (4) previous test branch suffix to merge into (3) +# (maint branches only, only used in test branch mode) +# +# Merge forward example: +# $ cd <PATH/TO/WORKTREE> (2) +# $ git checkout maint-0.3.5 (0) +# $ git pull +# $ git merge maint-0.3.4 (1) # -# As an example: -# $ cd <PATH/TO/WORKTREE> (3) -# $ git checkout maint-0.3.5 (1) +# Test branch example: +# $ cd <PATH/TO/WORKTREE> (2) +# $ git checkout -b ticket99999_035 (3) +# $ git checkout maint-0.3.5 (0) # $ git pull -# $ git merge maint-0.3.4 (2) +# $ git checkout ticket99999_035 +# $ git merge maint-0.3.5 +# $ git merge ticket99999_034 (4) # # First set of arrays are the maint-* branch and then the release-* branch. # New arrays need to be in the WORKTREE= array else they aren't considered. -MAINT_034=( "maint-0.3.4" "maint-0.2.9" "$GIT_PATH/$TOR_WKT_NAME/maint-0.3.4" ) -MAINT_035=( "maint-0.3.5" "maint-0.3.4" "$GIT_PATH/$TOR_WKT_NAME/maint-0.3.5" ) -MAINT_040=( "maint-0.4.0" "maint-0.3.5" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.0" ) -MAINT_MASTER=( "master" "maint-0.4.0" "$GIT_PATH/$TOR_MASTER_NAME" ) +# +# Only used in test branch mode +# There is no previous branch to merge forward, so the second and fifth items +# must be blank ("") +MAINT_029_TB=( "maint-0.2.9" "" "$GIT_PATH/$TOR_WKT_NAME/maint-0.2.9" \ + "_029" "") +# Used in maint/release merge and test branch modes +MAINT_035=( "maint-0.3.5" "maint-0.2.9" "$GIT_PATH/$TOR_WKT_NAME/maint-0.3.5" \ + "_035" "_029") +MAINT_040=( "maint-0.4.0" "maint-0.3.5" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.0" \ + "_040" "_035") +MAINT_041=( "maint-0.4.1" "maint-0.4.0" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.1" \ + "_041" "_040") +MAINT_MASTER=( "master" "maint-0.4.1" "$GIT_PATH/$TOR_MASTER_NAME" \ + "_master" "_041") RELEASE_029=( "release-0.2.9" "maint-0.2.9" "$GIT_PATH/$TOR_WKT_NAME/release-0.2.9" ) -RELEASE_034=( "release-0.3.4" "maint-0.3.4" "$GIT_PATH/$TOR_WKT_NAME/release-0.3.4" ) RELEASE_035=( "release-0.3.5" "maint-0.3.5" "$GIT_PATH/$TOR_WKT_NAME/release-0.3.5" ) RELEASE_040=( "release-0.4.0" "maint-0.4.0" "$GIT_PATH/$TOR_WKT_NAME/release-0.4.0" ) +RELEASE_041=( "release-0.4.1" "maint-0.4.1" "$GIT_PATH/$TOR_WKT_NAME/release-0.4.1" ) # The master branch path has to be the main repository thus contains the # origin that will be used to fetch the updates. All the worktrees are created @@ -55,41 +113,104 @@ ORIGIN_PATH="$GIT_PATH/$TOR_MASTER_NAME" # SC2034 -- shellcheck thinks that these are unused. We know better. ACTUALLY_THESE_ARE_USED=<<EOF -${MAINT_034[0]} +${MAINT_029_TB[0]} ${MAINT_035[0]} ${MAINT_040[0]} +${MAINT_041[0]} ${MAINT_MASTER[0]} ${RELEASE_029[0]} -${RELEASE_034[0]} ${RELEASE_035[0]} ${RELEASE_040[0]} +${RELEASE_041[0]} EOF -########################## -# Git Worktree to manage # -########################## +####################### +# Argument processing # +####################### + +# Controlled by the -n option. The dry run option will just output the command +# that would have been executed for each worktree. +DRY_RUN=0 + +# Controlled by the -t <test-branch-prefix> option. The test branch base +# name option makes git-merge-forward.sh create new test branches: +# <tbbn>_029, <tbbn>_035, ... , <tbbn>_master, and merge forward. +TEST_BRANCH_PREFIX= + +# Controlled by the -u option. The use existing option checks for existing +# branches with the <test-branch-prefix>, and checks them out, rather than +# creating a new branch. +USE_EXISTING=0 + +while getopts "hnt:u" opt; do + case "$opt" in + h) usage + exit 0 + ;; + n) DRY_RUN=1 + echo " *** DRY RUN MODE ***" + ;; + t) TEST_BRANCH_PREFIX="$OPTARG" + echo " *** CREATING TEST BRANCHES: ${TEST_BRANCH_PREFIX}_nnn ***" + ;; + u) USE_EXISTING=1 + echo " *** USE EXISTING TEST BRANCHES MODE ***" + ;; + *) + echo + usage + exit 1 + ;; + esac +done + +########################### +# Git worktrees to manage # +########################### + +if [ -z "$TEST_BRANCH_PREFIX" ]; then + + # maint/release merge mode + # + # List of all worktrees to work on. All defined above. Ordering is important. + # Always the maint-* branch BEFORE then the release-*. + WORKTREE=( + RELEASE_029[@] + + MAINT_035[@] + RELEASE_035[@] + + MAINT_040[@] + RELEASE_040[@] + + MAINT_041[@] + RELEASE_041[@] -# List of all worktrees to work on. All defined above. Ordering is important. -# Always the maint-* branch BEFORE then the release-*. -WORKTREE=( - RELEASE_029[@] + MAINT_MASTER[@] + ) - MAINT_034[@] - RELEASE_034[@] +else - MAINT_035[@] - RELEASE_035[@] + # Test branch mode: merge to maint only, and create a new branch for 0.2.9 + WORKTREE=( + MAINT_029_TB[@] - MAINT_040[@] - RELEASE_040[@] + MAINT_035[@] + + MAINT_040[@] + + MAINT_041[@] + + MAINT_MASTER[@] + ) + +fi - MAINT_MASTER[@] -) COUNT=${#WORKTREE[@]} -# Controlled by the -n option. The dry run option will just output the command -# that would have been executed for each worktree. -DRY_RUN=0 +############# +# Constants # +############# # Control characters CNRM=$'\x1b[0;0m' # Clear color @@ -127,7 +248,7 @@ function validate_ret # Switch to the given branch name. function switch_branch { - local cmd="git checkout $1" + local cmd="git checkout '$1'" printf " %s Switching branch to %s..." "$MARKER" "$1" if [ $DRY_RUN -eq 0 ]; then msg=$( eval "$cmd" 2>&1 ) @@ -137,6 +258,45 @@ function switch_branch fi } +# Checkout a new branch with the given branch name. +function new_branch +{ + local cmd="git checkout -b '$1'" + printf " %s Creating new branch %s..." "$MARKER" "$1" + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + +# Switch to an existing branch, or checkout a new branch with the given +# branch name. +function switch_or_new_branch +{ + local cmd="git rev-parse --verify '$1'" + if [ $DRY_RUN -eq 0 ]; then + # Call switch_branch if there is a branch, or new_branch if there is not + msg=$( eval "$cmd" 2>&1 ) + RET=$? + if [ $RET -eq 0 ]; then + # Branch: (commit id) + switch_branch "$1" + elif [ $RET -eq 128 ]; then + # Not a branch: "fatal: Needed a single revision" + new_branch "$1" + else + # Unexpected return value + validate_ret $RET "$msg" + fi + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}, then depending on the result:" + switch_branch "$1" + new_branch "$1" + fi +} + # Pull the given branch name. function pull_branch { @@ -150,10 +310,10 @@ function pull_branch fi } -# Merge the given branch name ($2) into the current branch ($1). +# Merge the given branch name ($1) into the current branch ($2). function merge_branch { - local cmd="git merge --no-edit $1" + local cmd="git merge --no-edit '$1'" printf " %s Merging branch %s into %s..." "$MARKER" "$1" "$2" if [ $DRY_RUN -eq 0 ]; then msg=$( eval "$cmd" 2>&1 ) @@ -166,7 +326,7 @@ function merge_branch # Pull the given branch name. function merge_branch_origin { - local cmd="git merge --ff-only origin/$1" + local cmd="git merge --ff-only 'origin/$1'" printf " %s Merging branch origin/%s..." "$MARKER" "$1" if [ $DRY_RUN -eq 0 ]; then msg=$( eval "$cmd" 2>&1 ) @@ -203,16 +363,6 @@ function fetch_origin # Entry point # ############### -while getopts "n" opt; do - case "$opt" in - n) DRY_RUN=1 - echo " *** DRY DRUN MODE ***" - ;; - *) - ;; - esac -done - # First, fetch the origin. goto_repo "$ORIGIN_PATH" fetch_origin @@ -222,15 +372,57 @@ for ((i=0; i<COUNT; i++)); do current=${!WORKTREE[$i]:0:1} previous=${!WORKTREE[$i]:1:1} repo_path=${!WORKTREE[$i]:2:1} + # default to merge forward mode + test_current= + test_previous= + target_current="$current" + target_previous="$previous" + if [ "$TEST_BRANCH_PREFIX" ]; then + test_current_suffix=${!WORKTREE[$i]:3:1} + test_current=${TEST_BRANCH_PREFIX}${test_current_suffix} + # the current test branch, if present, or maint/release branch, if not + target_current="$test_current" + test_previous_suffix=${!WORKTREE[$i]:4:1} + if [ "$test_previous_suffix" ]; then + test_previous=${TEST_BRANCH_PREFIX}${test_previous_suffix} + # the previous test branch, if present, or maint/release branch, if not + target_previous="$test_previous" + fi + fi - printf "%s Handling branch \\n" "$MARKER" "${BYEL}$current${CNRM}" + printf "%s Handling branch \\n" "$MARKER" "${BYEL}$target_current${CNRM}" # Go into the worktree to start merging. goto_repo "$repo_path" - # Checkout the current branch + if [ "$test_current" ]; then + if [ $USE_EXISTING -eq 0 ]; then + # Create a test branch from the currently checked-out branch/commit + # Fail if it already exists + new_branch "$test_current" + else + # Switch if it exists, or create if it does not + switch_or_new_branch "$test_current" + fi + fi + # Checkout the current maint/release branch switch_branch "$current" - # Update the current branch with an origin merge to get the latest. + # Update the current maint/release branch with an origin merge to get the + # latest updates merge_branch_origin "$current" - # Merge the previous branch. Ex: merge maint-0.2.5 into maint-0.2.9. - merge_branch "$previous" "$current" + if [ "$test_current" ]; then + # Checkout the test branch + switch_branch "$test_current" + # Merge the updated maint branch into the test branch + merge_branch "$current" "$test_current" + fi + # Merge the previous branch into the target branch + # Merge Forward Example: + # merge maint-0.2.9 into maint-0.3.5. + # Test Branch Example: + # merge bug99999_029 into bug99999_035. + # Skip the merge if the previous branch does not exist + # (there's nothing to merge forward into the oldest test branch) + if [ "$target_previous" ]; then + merge_branch "$target_previous" "$target_current" + fi done diff --git a/scripts/git/git-pull-all.sh b/scripts/git/git-pull-all.sh index 5d1d58e4bf..dc16066388 100755 --- a/scripts/git/git-pull-all.sh +++ b/scripts/git/git-pull-all.sh @@ -1,31 +1,53 @@ -#!/bin/bash +#!/usr/bin/env bash -################################## -# User configuration (change me) # -################################## +SCRIPT_NAME=$(basename "$0") -# The general setup that is suggested here is: -# -# GIT_PATH = /home/<user>/git/ -# ... where the git repository directories resides. -# TOR_MASTER_NAME = "tor" -# ... which means that tor.git was cloned in /home/<user>/git/tor -# TOR_WKT_NAME = "tor-wkt" -# ... which means that the tor worktrees are in /home/<user>/git/tor-wkt +function usage() +{ + echo "$SCRIPT_NAME [-h] [-n]" + echo + echo " arguments:" + echo " -h: show this help text" + echo " -n: dry run mode" + echo " (default: run commands)" + echo + echo " env vars:" + echo " required:" + echo " TOR_FULL_GIT_PATH: where the git repository directories reside." + echo " You must set this env var, we recommend \$HOME/git/" + echo " (default: fail if this env var is not set;" + echo " current: $GIT_PATH)" + echo + echo " optional:" + echo " TOR_MASTER: the name of the directory containing the tor.git clone" + echo " The tor master git directory is \$GIT_PATH/\$TOR_MASTER" + echo " (default: tor; current: $TOR_MASTER_NAME)" + echo " TOR_WKT_NAME: the name of the directory containing the tor" + echo " worktrees. The tor worktrees are:" + echo " \$GIT_PATH/\$TOR_WKT_NAME/{maint-*,release-*}" + echo " (default: tor-wkt; current: $TOR_WKT_NAME)" + echo " we recommend that you set these env vars in your ~/.profile" +} + +################# +# Configuration # +################# + +# Don't change this configuration - set the env vars in your .profile # Where are all those git repositories? -GIT_PATH="FULL_PATH_TO_GIT_REPOSITORY_DIRECTORY" +GIT_PATH=${TOR_FULL_GIT_PATH:-"FULL_PATH_TO_GIT_REPOSITORY_DIRECTORY"} # The tor master git repository directory from which all the worktree have # been created. -TOR_MASTER_NAME="tor" +TOR_MASTER_NAME=${TOR_MASTER_NAME:-"tor"} # The worktrees location (directory). -TOR_WKT_NAME="tor-wkt" +TOR_WKT_NAME=${TOR_WKT_NAME:-"tor-wkt"} -######################### -# End of configuration. # -######################### +########################## +# Git branches to manage # +########################## -# Configuration of the branches that needs merging. The values are in order: +# Configuration of the branches that need pulling. The values are in order: # (1) Branch name to pull (update). # (2) Full path of the git worktree. # @@ -37,15 +59,15 @@ TOR_WKT_NAME="tor-wkt" # First set of arrays are the maint-* branch and then the release-* branch. # New arrays need to be in the WORKTREE= array else they aren't considered. MAINT_029=( "maint-0.2.9" "$GIT_PATH/$TOR_WKT_NAME/maint-0.2.9" ) -MAINT_034=( "maint-0.3.4" "$GIT_PATH/$TOR_WKT_NAME/maint-0.3.4" ) MAINT_035=( "maint-0.3.5" "$GIT_PATH/$TOR_WKT_NAME/maint-0.3.5" ) MAINT_040=( "maint-0.4.0" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.0" ) +MAINT_041=( "maint-0.4.1" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.1" ) MAINT_MASTER=( "master" "$GIT_PATH/$TOR_MASTER_NAME" ) RELEASE_029=( "release-0.2.9" "$GIT_PATH/$TOR_WKT_NAME/release-0.2.9" ) -RELEASE_034=( "release-0.3.4" "$GIT_PATH/$TOR_WKT_NAME/release-0.3.4" ) RELEASE_035=( "release-0.3.5" "$GIT_PATH/$TOR_WKT_NAME/release-0.3.5" ) RELEASE_040=( "release-0.4.0" "$GIT_PATH/$TOR_WKT_NAME/release-0.4.0" ) +RELEASE_041=( "release-0.4.1" "$GIT_PATH/$TOR_WKT_NAME/release-0.4.1" ) # The master branch path has to be the main repository thus contains the # origin that will be used to fetch the updates. All the worktrees are created @@ -55,19 +77,19 @@ ORIGIN_PATH="$GIT_PATH/$TOR_MASTER_NAME" # SC2034 -- shellcheck thinks that these are unused. We know better. ACTUALLY_THESE_ARE_USED=<<EOF ${MAINT_029[0]} -${MAINT_034[0]} ${MAINT_035[0]} ${MAINT_040[0]} +${MAINT_041[0]} ${MAINT_MASTER[0]} ${RELEASE_029[0]} -${RELEASE_034[0]} ${RELEASE_035[0]} ${RELEASE_040[0]} +${RELEASE_041[0]} EOF -########################## -# Git Worktree to manage # -########################## +########################### +# Git worktrees to manage # +########################### # List of all worktrees to work on. All defined above. Ordering is important. # Always the maint-* branch first then the release-*. @@ -75,23 +97,47 @@ WORKTREE=( MAINT_029[@] RELEASE_029[@] - MAINT_034[@] - RELEASE_034[@] - MAINT_035[@] RELEASE_035[@] MAINT_040[@] RELEASE_040[@] + MAINT_041[@] + RELEASE_041[@] + MAINT_MASTER[@] ) COUNT=${#WORKTREE[@]} +####################### +# Argument processing # +####################### + # Controlled by the -n option. The dry run option will just output the command # that would have been executed for each worktree. DRY_RUN=0 +while getopts "hn" opt; do + case "$opt" in + h) usage + exit 0 + ;; + n) DRY_RUN=1 + echo " *** DRY DRUN MODE ***" + ;; + *) + echo + usage + exit 1 + ;; + esac +done + +############# +# Constants # +############# + # Control characters CNRM=$'\x1b[0;0m' # Clear color @@ -191,16 +237,6 @@ function fetch_tor_github # Entry point # ############### -while getopts "n" opt; do - case "$opt" in - n) DRY_RUN=1 - echo " *** DRY DRUN MODE ***" - ;; - *) - ;; - esac -done - # First, fetch tor-github. goto_repo "$ORIGIN_PATH" fetch_tor_github diff --git a/scripts/git/git-push-all.sh b/scripts/git/git-push-all.sh index 0ce951d4bd..7c43fe24d8 100755 --- a/scripts/git/git-push-all.sh +++ b/scripts/git/git-push-all.sh @@ -1,11 +1,292 @@ -#!/bin/bash +#!/usr/bin/env bash -# The remote upstream branch on which git.torproject.org/tor.git points to. -UPSTREAM_BRANCH="upstream" +SCRIPT_NAME=$(basename "$0") -git push $UPSTREAM_BRANCH \ - master \ - {release,maint}-0.4.0 \ - {release,maint}-0.3.5 \ - {release,maint}-0.3.4 \ - {release,maint}-0.2.9 +function usage() +{ + if [ "$TOR_PUSH_SAME" ]; then + CURRENT_PUSH_SAME="push" + else + CURRENT_PUSH_SAME="skip" + fi + + echo "$SCRIPT_NAME [-h] [-r <remote-name> [-t <test-branch-prefix>]] [-s]" + # The next line looks misaligned, but it lines up in the output + echo " [-- [-n] [--no-atomic] <git push options>]" + echo + echo " arguments:" + echo " -h: show this help text" + echo " -n: dry run mode" + echo " (default: run commands)" + echo " -r: push to remote-name, rather than the default upstream remote." + echo " (default: $DEFAULT_UPSTREAM_REMOTE, current: $UPSTREAM_REMOTE)" + echo " -t: test branch mode: push test branches to remote-name. Pushes" + echo " branches prefix_029, prefix_035, ... , prefix_master." + echo " (default: push maint-*, release-*, and master)" + echo " -s: push branches whose tips match upstream maint, release, or" + echo " master branches. The default is to skip these branches," + echo " because they do not contain any new code. Use -s to test for" + echo " CI environment failures, using code that previously passed CI." + echo " (default: skip; current: $CURRENT_PUSH_SAME matching branches)" + echo " --: pass further arguments to git push." + echo " (default: git push --atomic, current: $GIT_PUSH)" + echo + echo " env vars:" + echo " optional:" + echo " TOR_GIT_PUSH_PATH: change to this directory before pushing." + echo " (default: if \$TOR_FULL_GIT_PATH is set," + echo " use \$TOR_FULL_GIT_PATH/\$TOR_MASTER;" + echo " Otherwise, use the current directory for pushes;" + echo " current: $TOR_GIT_PUSH_PATH)" + echo " TOR_FULL_GIT_PATH: where the git repository directories reside." + echo " We recommend using \$HOME/git/." + echo " (default: use the current directory for pushes;" + echo " current: $TOR_FULL_GIT_PATH)" + echo " TOR_MASTER: the name of the directory containing the tor.git clone" + echo " The tor master git directory is \$GIT_PATH/\$TOR_MASTER" + echo " (default: tor; current: $TOR_MASTER_NAME)" + echo + echo " TOR_UPSTREAM_REMOTE_NAME: the default upstream remote." + echo " Overridden by -r." + echo " (default: upstream; current: $UPSTREAM_REMOTE)" + echo " TOR_GIT_PUSH: the git push command and default arguments." + echo " Overridden by <git push options> after --." + echo " (default: git push --atomic; current: $GIT_PUSH)" + echo " TOR_PUSH_SAME: push branches whose tips match upstream maint," + echo " release, or master branches. Inverted by -s." + echo " (default: skip; current: $CURRENT_PUSH_SAME matching branches)" + echo " TOR_PUSH_DELAY: pushes the master and maint branches separately," + echo " so that CI runs in a sensible order." + echo " (default: push all branches immediately; current: $PUSH_DELAY)" + echo " we recommend that you set these env vars in your ~/.profile" +} + +set -e + +################# +# Configuration # +################# + +# Don't change this configuration - set the env vars in your .profile +# +# The tor master git repository directory from which all the worktree have +# been created. +TOR_MASTER_NAME=${TOR_MASTER_NAME:-"tor"} +# Which directory do we push from? +if [ "$TOR_FULL_GIT_PATH" ]; then + TOR_GIT_PUSH_PATH=${TOR_GIT_PUSH_PATH:-"$TOR_FULL_GIT_PATH/$TOR_MASTER_NAME"} +fi +# git push command and default arguments +GIT_PUSH=${TOR_GIT_PUSH:-"git push --atomic"} +# The upstream remote which git.torproject.org/tor.git points to. +DEFAULT_UPSTREAM_REMOTE=${TOR_UPSTREAM_REMOTE_NAME:-"upstream"} +# Push to a different upstream remote using -r <remote-name> +UPSTREAM_REMOTE=${DEFAULT_UPSTREAM_REMOTE} +# Add a delay between pushes, so CI runs on the most important branches first +PUSH_DELAY=${TOR_PUSH_DELAY:-0} +# Push (1) or skip (0) test branches that are the same as an upstream +# maint/master branch. Push if you are testing that the CI environment still +# works on old code, skip if you are testing new code in the branch. +# Default: skip unchanged branches. +# Inverted by the -s option. +PUSH_SAME=${TOR_PUSH_SAME:-0} + +####################### +# Argument processing # +####################### + +# Controlled by the -t <test-branch-prefix> option. The test branch base +# name option makes git-merge-forward.sh create new test branches: +# <tbbn>_029, <tbbn>_035, ... , <tbbn>_master, and merge forward. +TEST_BRANCH_PREFIX= + +while getopts ":hr:st:" opt; do + case "$opt" in + h) usage + exit 0 + ;; + r) UPSTREAM_REMOTE="$OPTARG" + echo " *** PUSHING TO REMOTE: ${UPSTREAM_REMOTE} ***" + shift + shift + OPTIND=$((OPTIND - 2)) + ;; + s) PUSH_SAME=$((! PUSH_SAME)) + if [ "$PUSH_SAME" -eq 0 ]; then + echo " *** SKIPPING UNCHANGED TEST BRANCHES ***" + else + echo " *** PUSHING UNCHANGED TEST BRANCHES ***" + fi + shift + OPTIND=$((OPTIND - 1)) + ;; + t) TEST_BRANCH_PREFIX="$OPTARG" + echo " *** PUSHING TEST BRANCHES: ${TEST_BRANCH_PREFIX}_nnn ***" + shift + shift + OPTIND=$((OPTIND - 2)) + ;; + *) + # Assume we're done with script arguments, + # and git push will handle the option + break + ;; + esac +done + +# getopts doesn't allow "-" as an option character, +# so we have to handle -- manually +if [ "$1" = "--" ]; then + shift +fi + +if [ "$TEST_BRANCH_PREFIX" ]; then + if [ "$UPSTREAM_REMOTE" = "$DEFAULT_UPSTREAM_REMOTE" ]; then + echo "Pushing test branches ${TEST_BRANCH_PREFIX}_nnn to " \ + "the default remote $DEFAULT_UPSTREAM_REMOTE is not allowed." + echo + usage + exit 1 + fi +fi + +if [ "$TOR_GIT_PUSH_PATH" ]; then + echo "Changing to $GIT_PUSH_PATH before pushing" + cd "$TOR_GIT_PUSH_PATH" +else + echo "Pushing from the current directory" +fi + +echo "Calling $GIT_PUSH" "$@" "<branches>" + +################################ +# Git upstream remote branches # +################################ + +DEFAULT_UPSTREAM_BRANCHES= +if [ "$DEFAULT_UPSTREAM_REMOTE" != "$UPSTREAM_REMOTE" ]; then + DEFAULT_UPSTREAM_BRANCHES=$(echo \ + "$DEFAULT_UPSTREAM_REMOTE"/master \ + "$DEFAULT_UPSTREAM_REMOTE"/{release,maint}-0.4.1 \ + "$DEFAULT_UPSTREAM_REMOTE"/{release,maint}-0.4.0 \ + "$DEFAULT_UPSTREAM_REMOTE"/{release,maint}-0.3.5 \ + "$DEFAULT_UPSTREAM_REMOTE"/{release,maint}-0.2.9 \ + ) +fi + +UPSTREAM_BRANCHES=$(echo \ + "$UPSTREAM_REMOTE"/master \ + "$UPSTREAM_REMOTE"/{release,maint}-0.4.1 \ + "$UPSTREAM_REMOTE"/{release,maint}-0.4.0 \ + "$UPSTREAM_REMOTE"/{release,maint}-0.3.5 \ + "$UPSTREAM_REMOTE"/{release,maint}-0.2.9 \ + ) + +######################## +# Git branches to push # +######################## + +PUSH_BRANCHES=$(echo \ + master \ + {release,maint}-0.4.1 \ + {release,maint}-0.4.0 \ + {release,maint}-0.3.5 \ + {release,maint}-0.2.9 \ + ) + +if [ -z "$TEST_BRANCH_PREFIX" ]; then + + # maint/release push mode + # + # List of branches to push. Ordering is not important. + PUSH_BRANCHES=$(echo \ + master \ + {release,maint}-0.4.1 \ + {release,maint}-0.4.0 \ + {release,maint}-0.3.5 \ + {release,maint}-0.2.9 \ + ) +else + + # Test branch mode: merge to maint only, and create a new branch for 0.2.9 + # + # List of branches to push. Ordering is not important. + PUSH_BRANCHES=" \ + ${TEST_BRANCH_PREFIX}_master \ + ${TEST_BRANCH_PREFIX}_041 \ + ${TEST_BRANCH_PREFIX}_040 \ + ${TEST_BRANCH_PREFIX}_035 \ + ${TEST_BRANCH_PREFIX}_029 \ + " +fi + +############### +# Entry point # +############### + +# Skip the test branches that are the same as the upstream branches +if [ "$PUSH_SAME" -eq 0 ] && [ "$TEST_BRANCH_PREFIX" ]; then + NEW_PUSH_BRANCHES= + for b in $PUSH_BRANCHES; do + PUSH_COMMIT=$(git rev-parse "$b") + SKIP_UPSTREAM= + for u in $DEFAULT_UPSTREAM_BRANCHES $UPSTREAM_BRANCHES; do + UPSTREAM_COMMIT=$(git rev-parse "$u") + if [ "$PUSH_COMMIT" = "$UPSTREAM_COMMIT" ]; then + SKIP_UPSTREAM="$u" + fi + done + if [ "$SKIP_UPSTREAM" ]; then + printf "Skipping unchanged: %s remote: %s\\n" \ + "$b" "$SKIP_UPSTREAM" + else + if [ "$NEW_PUSH_BRANCHES" ]; then + NEW_PUSH_BRANCHES="${NEW_PUSH_BRANCHES} ${b}" + else + NEW_PUSH_BRANCHES="${b}" + fi + fi + done + PUSH_BRANCHES=${NEW_PUSH_BRANCHES} +fi + +if [ "$PUSH_DELAY" -le 0 ]; then + echo "Pushing $PUSH_BRANCHES" + # We know that there are no spaces in any branch within $PUSH_BRANCHES, so + # it is safe to use it unquoted. (This also applies to the other shellcheck + # exceptions below.) + # + # Push all the branches at the same time + # shellcheck disable=SC2086 + $GIT_PUSH "$@" "$UPSTREAM_REMOTE" $PUSH_BRANCHES +else + # Push the branches in optimal CI order, with a delay between each push + PUSH_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | sort -V) + MASTER_BRANCH=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep master) + if [ -z "$TEST_BRANCH_PREFIX" ]; then + MAINT_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep maint) + RELEASE_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep release | \ + tr "\\n" " ") + printf \ + "Pushing with %ss delays, so CI runs in this order:\\n%s\\n%s\\n%s\\n" \ + "$PUSH_DELAY" "$MASTER_BRANCH" "$MAINT_BRANCHES" "$RELEASE_BRANCHES" + else + # Actually test branches based on maint branches + MAINT_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep -v master) + printf "Pushing with %ss delays, so CI runs in this order:\\n%s\\n%s\\n" \ + "$PUSH_DELAY" "$MASTER_BRANCH" "$MAINT_BRANCHES" + # No release branches + RELEASE_BRANCHES= + fi + $GIT_PUSH "$@" "$UPSTREAM_REMOTE" "$MASTER_BRANCH" + sleep "$PUSH_DELAY" + # shellcheck disable=SC2086 + for b in $MAINT_BRANCHES; do + $GIT_PUSH "$@" "$UPSTREAM_REMOTE" "$b" + sleep "$PUSH_DELAY" + done + if [ "$RELEASE_BRANCHES" ]; then + # shellcheck disable=SC2086 + $GIT_PUSH "$@" "$UPSTREAM_REMOTE" $RELEASE_BRANCHES + fi +fi diff --git a/scripts/git/post-merge.git-hook b/scripts/git/post-merge.git-hook index 176b7c9bbd..eae4f999e7 100755 --- a/scripts/git/post-merge.git-hook +++ b/scripts/git/post-merge.git-hook @@ -35,6 +35,12 @@ check_for_script_update() { fi } +cur_branch=$(git rev-parse --abbrev-ref HEAD) +if [ "$cur_branch" != "master" ]; then + echo "post-merge: Not a master branch. Skipping." + exit 0 +fi + check_for_diffs "pre-push" check_for_diffs "pre-commit" check_for_diffs "post-merge" diff --git a/scripts/git/pre-commit.git-hook b/scripts/git/pre-commit.git-hook index b285776c04..1c381ec60a 100755 --- a/scripts/git/pre-commit.git-hook +++ b/scripts/git/pre-commit.git-hook @@ -1,10 +1,11 @@ -#!/bin/bash +#!/usr/bin/env bash # # To install this script, copy it to .git/hooks/pre-commit in local copy of # tor git repo and make sure it has permission to execute. # # This is pre-commit git hook script that prevents commiting your changeset if -# it fails our code formatting or changelog entry formatting checkers. +# it fails our code formatting, changelog entry formatting, module include +# rules, or best practices tracker. workdir=$(git rev-parse --show-toplevel) @@ -36,10 +37,23 @@ elif [ -d src/common ]; then src/tools/*.[ch] fi -if test -e scripts/maint/checkIncludes.py; then - python scripts/maint/checkIncludes.py +if test -e scripts/maint/practracker/includes.py; then + python scripts/maint/practracker/includes.py fi -if [ -e scripts/maint/practracker/practracker.py ]; then - python3 ./scripts/maint/practracker/practracker.py "$workdir" +# Only call practracker if ${PT_DIR}/.enable_practracker_in_hooks exists +# We do this check so that we can enable practracker in hooks in master, and +# disable it on maint branches +PT_DIR=scripts/maint/practracker + +if [ -e "${PT_DIR}/practracker.py" ]; then + if [ -e "${PT_DIR}/.enable_practracker_in_hooks" ]; then + if ! python3 "${PT_DIR}/practracker.py" "$workdir"; then + exit 1 + fi + fi +fi + +if [ -e scripts/maint/checkShellScripts.sh ]; then + scripts/maint/checkShellScripts.sh fi diff --git a/scripts/git/pre-push.git-hook b/scripts/git/pre-push.git-hook index c9e72a4d43..f4504c4215 100755 --- a/scripts/git/pre-push.git-hook +++ b/scripts/git/pre-push.git-hook @@ -1,10 +1,11 @@ -#!/bin/bash +#!/usr/bin/env bash # git pre-push hook script to: +# 0) Call the pre-commit hook, if it is available # 1) prevent "fixup!" and "squash!" commit from ending up in master, release-* # or maint-* # 2) Disallow pushing branches other than master, release-* -# and maint-* to origin (e.g. gitweb.torproject.org). +# and maint-* to origin (e.g. gitweb.torproject.org) # # To install this script, copy it into .git/hooks/pre-push path in your # local copy of git repository. Make sure it has permission to execute. @@ -21,6 +22,11 @@ z40=0000000000000000000000000000000000000000 upstream_name=${TOR_UPSTREAM_REMOTE_NAME:-"upstream"} +# Are you adding a new check to the git hooks? +# - Common checks belong in the pre-commit hook +# - Push-only checks belong in the pre-push hook +# +# Call the pre-commit hook for the common checks, if it is executable. workdir=$(git rev-parse --show-toplevel) if [ -x "$workdir/.git/hooks/pre-commit" ]; then if ! "$workdir"/.git/hooks/pre-commit; then @@ -28,14 +34,7 @@ if [ -x "$workdir/.git/hooks/pre-commit" ]; then fi fi -if [ -e scripts/maint/practracker/practracker.py ]; then - if ! python3 ./scripts/maint/practracker/practracker.py "$workdir"; then - exit 1 - fi -fi - remote="$1" -remote_loc="$2" remote_name=$(git remote --verbose | grep "$2" | awk '{print $1}' | head -n 1) @@ -105,4 +104,3 @@ do done exit 0 - diff --git a/scripts/maint/add_c_file.py b/scripts/maint/add_c_file.py index 499415974f..adf7ce79bb 100755 --- a/scripts/maint/add_c_file.py +++ b/scripts/maint/add_c_file.py @@ -125,8 +125,8 @@ class AutomakeChunk: Y \ Z """ - self.prespace = "\t" - self.postspace = "\t\t" + prespace = "\t" + postspace = "\t\t" for lineno, line in enumerate(self.lines): m = re.match(r'(\s+)(\S+)(\s+)\\', line) if not m: @@ -135,7 +135,7 @@ class AutomakeChunk: if fname > member: self.insert_before(lineno, member, prespace, postspace) return - self.insert_at_end(member) + self.insert_at_end(member, prespace, postspace) def insert_before(self, lineno, member, prespace, postspace): self.lines.insert(lineno, diff --git a/scripts/maint/annotate_ifdef_directives b/scripts/maint/annotate_ifdef_directives deleted file mode 100755 index ca267a865e..0000000000 --- a/scripts/maint/annotate_ifdef_directives +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/python -# Copyright (c) 2017-2019, The Tor Project, Inc. -# See LICENSE for licensing information - -import re - -LINE_OBVIOUSNESS_LIMIT = 4 - -class Problem(Exception): - pass - -def uncomment(s): - s = re.sub(r'//.*','',s) - s = re.sub(r'/\*.*','',s) - return s.strip() - -def translate(f_in, f_out): - whole_file = [] - stack = [] - cur_level = whole_file - lineno = 0 - for line in f_in: - lineno += 1 - m = re.match(r'\s*#\s*(if|ifdef|ifndef|else|endif|elif)\b\s*(.*)', - line) - if not m: - f_out.write(line) - continue - command,rest = m.groups() - if command in ("if", "ifdef", "ifndef"): - # The #if directive pushes us one level lower on the stack. - if command == 'ifdef': - rest = "defined(%s)"%uncomment(rest) - elif command == 'ifndef': - rest = "!defined(%s)"%uncomment(rest) - elif rest.endswith("\\"): - rest = rest[:-1]+"..." - - rest = uncomment(rest) - - new_level = [ (command, rest, lineno) ] - stack.append(cur_level) - cur_level = new_level - f_out.write(line) - elif command in ("else", "elif"): - if len(cur_level) == 0 or cur_level[-1][0] == 'else': - raise Problem("Unexpected #%s on %d"% (command,lineno)) - if (len(cur_level) == 1 and command == 'else' and - lineno > cur_level[0][2] + LINE_OBVIOUSNESS_LIMIT): - f_out.write("#else /* !(%s) */\n"%cur_level[0][1]) - else: - f_out.write(line) - cur_level.append((command, rest, lineno)) - else: - assert command == 'endif' - if len(stack) == 0: - raise Problem("Unmatched #%s on %s"% (command,lineno)) - if lineno <= cur_level[0][2] + LINE_OBVIOUSNESS_LIMIT: - f_out.write(line) - elif len(cur_level) == 1 or ( - len(cur_level) == 2 and cur_level[1][0] == 'else'): - f_out.write("#endif /* %s */\n"%cur_level[0][1]) - else: - f_out.write("#endif /* %s || ... */\n"%cur_level[0][1]) - cur_level = stack.pop() - if len(stack) or cur_level != whole_file: - raise Problem("Missing #endif") - -import sys,os -for fn in sys.argv[1:]: - with open(fn+"_OUT", 'w') as output_file: - translate(open(fn, 'r'), output_file) - os.rename(fn+"_OUT", fn) - diff --git a/scripts/maint/annotate_ifdef_directives.py b/scripts/maint/annotate_ifdef_directives.py new file mode 100755 index 0000000000..102128bfa0 --- /dev/null +++ b/scripts/maint/annotate_ifdef_directives.py @@ -0,0 +1,317 @@ +#!/usr/bin/python +# Copyright (c) 2017-2019, The Tor Project, Inc. +# See LICENSE for licensing information + +r""" +This script iterates over a list of C files. For each file, it looks at the +#if/#else C macros, and annotates them with comments explaining what they +match. + +For example, it replaces this kind of input... + +>>> INPUT = ''' +... #ifdef HAVE_OCELOT +... C code here +... #if MIMSY == BOROGROVE +... block 1 +... block 1 +... block 1 +... block 1 +... #else +... block 2 +... block 2 +... block 2 +... block 2 +... #endif +... #endif +... ''' + +With this kind of output: +>>> EXPECTED_OUTPUT = ''' +... #ifdef HAVE_OCELOT +... C code here +... #if MIMSY == BOROGROVE +... block 1 +... block 1 +... block 1 +... block 1 +... #else /* !(MIMSY == BOROGROVE) */ +... block 2 +... block 2 +... block 2 +... block 2 +... #endif /* MIMSY == BOROGROVE */ +... #endif /* defined(HAVE_OCELOT) */ +... ''' + +Here's how to use it: +>>> import sys +>>> if sys.version_info.major < 3: from cStringIO import StringIO +>>> if sys.version_info.major >= 3: from io import StringIO + +>>> OUTPUT = StringIO() +>>> translate(StringIO(INPUT), OUTPUT) +>>> assert OUTPUT.getvalue() == EXPECTED_OUTPUT + +Note that only #else and #endif lines are annotated. Existing comments +on those lines are removed. +""" + +import re + +# Any block with fewer than this many lines does not need annotations. +LINE_OBVIOUSNESS_LIMIT = 4 + +# Maximum line width. This includes a terminating newline character. +# +# (This is the maximum before encoding, so that if the the operating system +# uses multiple characers to encode newline, that's still okay.) +LINE_WIDTH=80 + +class Problem(Exception): + pass + +def close_parens_needed(expr): + """Return the number of left-parentheses needed to make 'expr' + balanced. + + >>> close_parens_needed("1+2") + 0 + >>> close_parens_needed("(1 + 2)") + 0 + >>> close_parens_needed("(1 + 2") + 1 + >>> close_parens_needed("(1 + (2 *") + 2 + >>> close_parens_needed("(1 + (2 * 3) + (4") + 2 + """ + return expr.count("(") - expr.count(")") + +def truncate_expression(expr, new_width): + """Given a parenthesized C expression in 'expr', try to return a new + expression that is similar to 'expr', but no more than 'new_width' + characters long. + + Try to return an expression with balanced parentheses. + + >>> truncate_expression("1+2+3", 8) + '1+2+3' + >>> truncate_expression("1+2+3+4+5", 8) + '1+2+3...' + >>> truncate_expression("(1+2+3+4)", 8) + '(1+2...)' + >>> truncate_expression("(1+(2+3+4))", 8) + '(1+...)' + >>> truncate_expression("(((((((((", 8) + '((...))' + """ + if len(expr) <= new_width: + # The expression is already short enough. + return expr + + ellipsis = "..." + + # Start this at the minimum that we might truncate. + n_to_remove = len(expr) + len(ellipsis) - new_width + + # Try removing characters, one by one, until we get something where + # re-balancing the parentheses still fits within the limit. + while n_to_remove < len(expr): + truncated = expr[:-n_to_remove] + ellipsis + truncated += ")" * close_parens_needed(truncated) + if len(truncated) <= new_width: + return truncated + n_to_remove += 1 + + return ellipsis + +def commented_line(fmt, argument, maxwidth=LINE_WIDTH): + # (This is a raw docstring so that our doctests can use \.) + r""" + Return fmt%argument, for use as a commented line. If the line would + be longer than maxwidth, truncate argument but try to keep its + parentheses balanced. + + Requires that fmt%"..." will fit into maxwidth characters. + + Requires that fmt ends with a newline. + + >>> commented_line("/* %s */\n", "hello world", 32) + '/* hello world */\n' + >>> commented_line("/* %s */\n", "hello world", 15) + '/* hello... */\n' + >>> commented_line("#endif /* %s */\n", "((1+2) && defined(FOO))", 32) + '#endif /* ((1+2) && defi...) */\n' + + + The default line limit is 80 characters including the newline: + + >>> long_argument = "long " * 100 + >>> long_line = commented_line("#endif /* %s */\n", long_argument) + >>> len(long_line) + 80 + + >>> long_line[:40] + '#endif /* long long long long long long ' + >>> long_line[40:] + 'long long long long long long lon... */\n' + + If a line works out to being 80 characters naturally, it isn't truncated, + and no ellipsis is added. + + >>> medium_argument = "a"*66 + >>> medium_line = commented_line("#endif /* %s */\n", medium_argument) + >>> len(medium_line) + 80 + >>> "..." in medium_line + False + >>> medium_line[:40] + '#endif /* aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + >>> medium_line[40:] + 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa */\n' + + + """ + assert fmt.endswith("\n") + result = fmt % argument + if len(result) <= maxwidth: + return result + else: + # How long can we let the argument be? Try filling in the + # format with an empty argument to find out. + max_arg_width = maxwidth - len(fmt % "") + result = fmt % truncate_expression(argument, max_arg_width) + assert len(result) <= maxwidth + return result + +def negate(expr): + """Return a negated version of expr; try to avoid double-negation. + + We usually wrap expressions in parentheses and add a "!". + >>> negate("A && B") + '!(A && B)' + + But if we recognize the expression as negated, we can restore it. + >>> negate(negate("A && B")) + 'A && B' + + The same applies for defined(FOO). + >>> negate("defined(FOO)") + '!defined(FOO)' + >>> negate(negate("defined(FOO)")) + 'defined(FOO)' + + Internal parentheses don't confuse us: + >>> negate("!(FOO) && !(BAR)") + '!(!(FOO) && !(BAR))' + + """ + expr = expr.strip() + # See whether we match !(...), with no intervening close-parens. + m = re.match(r'^!\s*\(([^\)]*)\)$', expr) + if m: + return m.group(1) + + + # See whether we match !?defined(...), with no intervening close-parens. + m = re.match(r'^(!?)\s*(defined\([^\)]*\))$', expr) + if m: + if m.group(1) == "!": + prefix = "" + else: + prefix = "!" + return prefix + m.group(2) + + return "!(%s)" % expr + +def uncomment(s): + """ + Remove existing trailing comments from an #else or #endif line. + """ + s = re.sub(r'//.*','',s) + s = re.sub(r'/\*.*','',s) + return s.strip() + +def translate(f_in, f_out): + """ + Read a file from f_in, and write its annotated version to f_out. + """ + # A stack listing our current if/else state. Each member of the stack + # is a list of directives. Each directive is a 3-tuple of + # (command, rest, lineno) + # where "command" is one of if/ifdef/ifndef/else/elif, and where + # "rest" is an expression in a format suitable for use with #if, and where + # lineno is the line number where the directive occurred. + stack = [] + # the stack element corresponding to the top level of the file. + whole_file = [] + cur_level = whole_file + lineno = 0 + for line in f_in: + lineno += 1 + m = re.match(r'\s*#\s*(if|ifdef|ifndef|else|endif|elif)\b\s*(.*)', + line) + if not m: + # no directive, so we can just write it out. + f_out.write(line) + continue + command,rest = m.groups() + if command in ("if", "ifdef", "ifndef"): + # The #if directive pushes us one level lower on the stack. + if command == 'ifdef': + rest = "defined(%s)"%uncomment(rest) + elif command == 'ifndef': + rest = "!defined(%s)"%uncomment(rest) + elif rest.endswith("\\"): + rest = rest[:-1]+"..." + + rest = uncomment(rest) + + new_level = [ (command, rest, lineno) ] + stack.append(cur_level) + cur_level = new_level + f_out.write(line) + elif command in ("else", "elif"): + # We stay at the same level on the stack. If we have an #else, + # we comment it. + if len(cur_level) == 0 or cur_level[-1][0] == 'else': + raise Problem("Unexpected #%s on %d"% (command,lineno)) + if (len(cur_level) == 1 and command == 'else' and + lineno > cur_level[0][2] + LINE_OBVIOUSNESS_LIMIT): + f_out.write(commented_line("#else /* %s */\n", + negate(cur_level[0][1]))) + else: + f_out.write(line) + cur_level.append((command, rest, lineno)) + else: + # We pop one element on the stack, and comment an endif. + assert command == 'endif' + if len(stack) == 0: + raise Problem("Unmatched #%s on %s"% (command,lineno)) + if lineno <= cur_level[0][2] + LINE_OBVIOUSNESS_LIMIT: + f_out.write(line) + elif len(cur_level) == 1 or ( + len(cur_level) == 2 and cur_level[1][0] == 'else'): + f_out.write(commented_line("#endif /* %s */\n", + cur_level[0][1])) + else: + f_out.write(commented_line("#endif /* %s || ... */\n", + cur_level[0][1])) + cur_level = stack.pop() + if len(stack) or cur_level != whole_file: + raise Problem("Missing #endif") + +if __name__ == '__main__': + + import sys,os + + if sys.argv[1] == "--self-test": + import doctest + doctest.testmod() + sys.exit(0) + + for fn in sys.argv[1:]: + with open(fn+"_OUT", 'w') as output_file: + translate(open(fn, 'r'), output_file) + os.rename(fn+"_OUT", fn) diff --git a/scripts/maint/checkIncludes.py b/scripts/maint/checkIncludes.py index ec9350b9b1..926b201b35 100755 --- a/scripts/maint/checkIncludes.py +++ b/scripts/maint/checkIncludes.py @@ -1,181 +1,14 @@ #!/usr/bin/python # Copyright 2018 The Tor Project, Inc. See LICENSE file for licensing info. -"""This script looks through all the directories for files matching *.c or - *.h, and checks their #include directives to make sure that only "permitted" - headers are included. +# This file is no longer here; see practracker/includes.py for this +# functionality. This is a stub file that exists so that older git +# hooks will know where to look. - Any #include directives with angle brackets (like #include <stdio.h>) are - ignored -- only directives with quotes (like #include "foo.h") are - considered. +import sys, os - To decide what includes are permitted, this script looks at a .may_include - file in each directory. This file contains empty lines, #-prefixed - comments, filenames (like "lib/foo/bar.h") and file globs (like lib/*/*.h) - for files that are permitted. -""" +dirname = os.path.split(sys.argv[0])[0] +new_location = os.path.join(dirname, "practracker", "includes.py") +python = sys.executable - -from __future__ import print_function - -import fnmatch -import os -import re -import sys - -# Global: Have there been any errors? -trouble = False - -if sys.version_info[0] <= 2: - def open_file(fname): - return open(fname, 'r') -else: - def open_file(fname): - return open(fname, 'r', encoding='utf-8') - -def warn(msg): - print(msg, file=sys.stderr) - -def err(msg): - """ Declare that an error has happened, and remember that there has - been an error. """ - global trouble - trouble = True - print(msg, file=sys.stderr) - -def fname_is_c(fname): - """ Return true iff 'fname' is the name of a file that we should - search for possibly disallowed #include directives. """ - return fname.endswith(".h") or fname.endswith(".c") - -INCLUDE_PATTERN = re.compile(r'\s*#\s*include\s+"([^"]*)"') -RULES_FNAME = ".may_include" - -ALLOWED_PATTERNS = [ - re.compile(r'^.*\*\.(h|inc)$'), - re.compile(r'^.*/.*\.h$'), - re.compile(r'^ext/.*\.c$'), - re.compile(r'^orconfig.h$'), - re.compile(r'^micro-revision.i$'), -] - -def pattern_is_normal(s): - for p in ALLOWED_PATTERNS: - if p.match(s): - return True - return False - -class Rules(object): - """ A 'Rules' object is the parsed version of a .may_include file. """ - def __init__(self, dirpath): - self.dirpath = dirpath - if dirpath.startswith("src/"): - self.incpath = dirpath[4:] - else: - self.incpath = dirpath - self.patterns = [] - self.usedPatterns = set() - - def addPattern(self, pattern): - if not pattern_is_normal(pattern): - warn("Unusual pattern {} in {}".format(pattern, self.dirpath)) - self.patterns.append(pattern) - - def includeOk(self, path): - for pattern in self.patterns: - if fnmatch.fnmatchcase(path, pattern): - self.usedPatterns.add(pattern) - return True - return False - - def applyToLines(self, lines, context=""): - lineno = 0 - for line in lines: - lineno += 1 - m = INCLUDE_PATTERN.match(line) - if m: - include = m.group(1) - if not self.includeOk(include): - err("Forbidden include of {} on line {}{}".format( - include, lineno, context)) - - def applyToFile(self, fname): - with open_file(fname) as f: - #print(fname) - self.applyToLines(iter(f), " of {}".format(fname)) - - def noteUnusedRules(self): - for p in self.patterns: - if p not in self.usedPatterns: - print("Pattern {} in {} was never used.".format(p, self.dirpath)) - - def getAllowedDirectories(self): - allowed = [] - for p in self.patterns: - m = re.match(r'^(.*)/\*\.(h|inc)$', p) - if m: - allowed.append(m.group(1)) - continue - m = re.match(r'^(.*)/[^/]*$', p) - if m: - allowed.append(m.group(1)) - continue - - return allowed - -def load_include_rules(fname): - """ Read a rules file from 'fname', and return it as a Rules object. """ - result = Rules(os.path.split(fname)[0]) - with open_file(fname) as f: - for line in f: - line = line.strip() - if line.startswith("#") or not line: - continue - result.addPattern(line) - return result - -list_unused = False -log_sorted_levels = False - -uses_dirs = { } - -for dirpath, dirnames, fnames in os.walk("src"): - if ".may_include" in fnames: - rules = load_include_rules(os.path.join(dirpath, RULES_FNAME)) - for fname in fnames: - if fname_is_c(fname): - rules.applyToFile(os.path.join(dirpath,fname)) - if list_unused: - rules.noteUnusedRules() - - uses_dirs[rules.incpath] = rules.getAllowedDirectories() - -if trouble: - err( -"""To change which includes are allowed in a C file, edit the {} -files in its enclosing directory.""".format(RULES_FNAME)) - sys.exit(1) - -all_levels = [] - -n = 0 -while uses_dirs: - n += 0 - cur_level = [] - for k in list(uses_dirs): - uses_dirs[k] = [ d for d in uses_dirs[k] - if (d in uses_dirs and d != k)] - if uses_dirs[k] == []: - cur_level.append(k) - for k in cur_level: - del uses_dirs[k] - n += 1 - if cur_level and log_sorted_levels: - print(n, cur_level) - if n > 100: - break - -if uses_dirs: - print("There are circular .may_include dependencies in here somewhere:", - uses_dirs) - sys.exit(1) +os.execl(python, python, new_location, *sys.argv[1:]) diff --git a/scripts/maint/checkShellScripts.sh b/scripts/maint/checkShellScripts.sh new file mode 100755 index 0000000000..4c872c7ee0 --- /dev/null +++ b/scripts/maint/checkShellScripts.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2019 The Tor Project, Inc. +# See LICENSE for license information +# +# checkShellScripts.sh +# -------------------- +# If shellcheck is installed, check all the shell scripts that we can fix. + +set -e + +# Only run this script if shellcheck is installed +# command echoes the path to shellcheck, which is a useful diagnostic log +if ! command -v shellcheck; then + printf "%s: Install shellcheck to check shell scripts.\\n" "$0" + exit 0 +fi + +# Some platforms don't have realpath +if command -v realpath ; then + HERE=$(dirname "$(realpath "$0")") +else + HERE=$(dirname "$0") + if [ ! -d "$HERE" ] || [ "$HERE" = "." ]; then + HERE=$(dirname "$PWD/$0") + fi +fi +TOPLEVEL=$(dirname "$(dirname "$HERE")") + +# Check we actually have a tor/src directory +if [ ! -d "$TOPLEVEL/src" ]; then + printf "Error: Couldn't find src directory in expected location: %s\\n" \ + "$TOPLEVEL/src" + exit 1 +fi + +# Check *.sh scripts, but ignore the ones that we can't fix +find "$TOPLEVEL/contrib" "$TOPLEVEL/doc" "$TOPLEVEL/scripts" "$TOPLEVEL/src" \ + -name "*.sh" \ + -not -path "$TOPLEVEL/src/ext/*" \ + -not -path "$TOPLEVEL/src/rust/registry/*" \ + -exec shellcheck {} + + +# Check scripts that aren't named *.sh +if [ -d "$TOPLEVEL/scripts/test" ]; then + shellcheck \ + "$TOPLEVEL/scripts/test/cov-diff" \ + "$TOPLEVEL/scripts/test/coverage" +fi +if [ -e \ + "$TOPLEVEL/contrib/dirauth-tools/nagios-check-tor-authority-cert" \ + ]; then + shellcheck \ + "$TOPLEVEL/contrib/dirauth-tools/nagios-check-tor-authority-cert" +fi +if [ -e "$TOPLEVEL/contrib/client-tools/torify" ]; then + shellcheck "$TOPLEVEL/contrib/client-tools/torify" +fi +if [ -d "$TOPLEVEL/scripts/git" ]; then + shellcheck "$TOPLEVEL/scripts/git/"*.git-hook +fi diff --git a/scripts/maint/checkSpace.pl b/scripts/maint/checkSpace.pl index 433ae62807..9c9b68ff9d 100755 --- a/scripts/maint/checkSpace.pl +++ b/scripts/maint/checkSpace.pl @@ -177,7 +177,7 @@ for my $fn (@ARGV) { $1 ne "elsif" and $1 ne "WINAPI" and $2 ne "WINAPI" and $1 ne "void" and $1 ne "__attribute__" and $1 ne "op" and $1 ne "size_t" and $1 ne "double" and $1 ne "uint64_t" and - $1 ne "workqueue_reply_t") { + $1 ne "workqueue_reply_t" and $1 ne "bool") { msg " fn ():$fn:$.\n"; } } diff --git a/scripts/maint/practracker/README b/scripts/maint/practracker/README new file mode 100644 index 0000000000..d978b39806 --- /dev/null +++ b/scripts/maint/practracker/README @@ -0,0 +1,21 @@ +Practracker is a simple python tool that keeps track of places where +our code is ugly, and tries to warn us about new ones or ones that +get worse. + +Right now, practracker looks for the following kinds of +best-practices violations: + + .c files greater than 3000 lines long + .h files greater than 500 lines long + .c files with more than 50 includes + .h files with more than 15 includes + + All files that include a local header not listed in a .may_include + file in the same directory, when that .may_include file has an + "!advisory" marker. + +The list of current violations is tracked in exceptions.txt; slight +deviations of the current exceptions cause warnings, whereas large +ones cause practracker to fail. + +For usage information, run "practracker.py --help". diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index 726dc9c3ef..7b15b37f8c 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -30,260 +30,306 @@ # Remember: It is better to fix the problem than to add a new exception! problem file-size /src/app/config/config.c 8518 -problem include-count /src/app/config/config.c 87 +problem include-count /src/app/config/config.c 89 problem function-size /src/app/config/config.c:options_act_reversible() 296 problem function-size /src/app/config/config.c:options_act() 589 -problem function-size /src/app/config/config.c:resolve_my_address() 192 -problem function-size /src/app/config/config.c:options_validate() 1217 +problem function-size /src/app/config/config.c:resolve_my_address() 190 +problem function-size /src/app/config/config.c:options_validate() 1209 problem function-size /src/app/config/config.c:options_init_from_torrc() 207 -problem function-size /src/app/config/config.c:options_init_from_string() 173 -problem function-size /src/app/config/config.c:options_init_logs() 146 +problem function-size /src/app/config/config.c:options_init_from_string() 171 +problem function-size /src/app/config/config.c:options_init_logs() 145 problem function-size /src/app/config/config.c:parse_bridge_line() 104 -problem function-size /src/app/config/config.c:parse_transport_line() 191 -problem function-size /src/app/config/config.c:parse_dir_authority_line() 151 -problem function-size /src/app/config/config.c:parse_dir_fallback_line() 102 -problem function-size /src/app/config/config.c:parse_port_config() 452 -problem function-size /src/app/config/config.c:parse_ports() 170 -problem function-size /src/app/config/config.c:getinfo_helper_config() 116 -problem function-size /src/app/config/confparse.c:config_assign_value() 205 -problem function-size /src/app/config/confparse.c:config_get_assigned_option() 129 -problem include-count /src/app/main/main.c 67 +problem function-size /src/app/config/config.c:parse_transport_line() 189 +problem function-size /src/app/config/config.c:parse_dir_authority_line() 150 +problem function-size /src/app/config/config.c:parse_dir_fallback_line() 101 +problem function-size /src/app/config/config.c:parse_port_config() 446 +problem function-size /src/app/config/config.c:parse_ports() 168 +problem file-size /src/app/config/or_options_st.h 1112 +problem include-count /src/app/main/main.c 68 problem function-size /src/app/main/main.c:dumpstats() 102 problem function-size /src/app/main/main.c:tor_init() 137 problem function-size /src/app/main/main.c:sandbox_init_filter() 291 problem function-size /src/app/main/main.c:run_tor_main_loop() 105 -problem function-size /src/app/main/ntmain.c:nt_service_install() 125 +problem function-size /src/app/main/ntmain.c:nt_service_install() 126 +problem dependency-violation /src/core/crypto/hs_ntor.c 1 +problem dependency-violation /src/core/crypto/onion_crypto.c 5 +problem dependency-violation /src/core/crypto/onion_fast.c 1 +problem dependency-violation /src/core/crypto/onion_tap.c 3 +problem dependency-violation /src/core/crypto/relay_crypto.c 9 problem file-size /src/core/mainloop/connection.c 5569 problem include-count /src/core/mainloop/connection.c 62 problem function-size /src/core/mainloop/connection.c:connection_free_minimal() 185 -problem function-size /src/core/mainloop/connection.c:connection_listener_new() 328 +problem function-size /src/core/mainloop/connection.c:connection_listener_new() 324 problem function-size /src/core/mainloop/connection.c:connection_handle_listener_read() 161 -problem function-size /src/core/mainloop/connection.c:connection_connect_sockaddr() 103 problem function-size /src/core/mainloop/connection.c:connection_proxy_connect() 148 problem function-size /src/core/mainloop/connection.c:connection_read_proxy_handshake() 153 -problem function-size /src/core/mainloop/connection.c:retry_listener_ports() 116 +problem function-size /src/core/mainloop/connection.c:retry_listener_ports() 112 problem function-size /src/core/mainloop/connection.c:connection_handle_read_impl() 111 -problem function-size /src/core/mainloop/connection.c:connection_buf_read_from_socket() 181 +problem function-size /src/core/mainloop/connection.c:connection_buf_read_from_socket() 180 problem function-size /src/core/mainloop/connection.c:connection_handle_write_impl() 241 problem function-size /src/core/mainloop/connection.c:assert_connection_ok() 143 +problem dependency-violation /src/core/mainloop/connection.c 44 +problem dependency-violation /src/core/mainloop/cpuworker.c 12 problem include-count /src/core/mainloop/mainloop.c 63 problem function-size /src/core/mainloop/mainloop.c:conn_close_if_marked() 108 problem function-size /src/core/mainloop/mainloop.c:run_connection_housekeeping() 123 +problem dependency-violation /src/core/mainloop/mainloop.c 49 +problem dependency-violation /src/core/mainloop/mainloop_pubsub.c 1 +problem dependency-violation /src/core/mainloop/mainloop_sys.c 1 +problem dependency-violation /src/core/mainloop/netstatus.c 4 +problem dependency-violation /src/core/mainloop/periodic.c 2 +problem dependency-violation /src/core/or/address_set.c 1 problem file-size /src/core/or/channel.c 3487 +problem dependency-violation /src/core/or/channel.c 9 +problem file-size /src/core/or/channel.h 780 +problem dependency-violation /src/core/or/channelpadding.c 6 problem function-size /src/core/or/channeltls.c:channel_tls_handle_var_cell() 160 problem function-size /src/core/or/channeltls.c:channel_tls_process_versions_cell() 170 problem function-size /src/core/or/channeltls.c:channel_tls_process_netinfo_cell() 214 problem function-size /src/core/or/channeltls.c:channel_tls_process_certs_cell() 246 problem function-size /src/core/or/channeltls.c:channel_tls_process_authenticate_cell() 202 +problem dependency-violation /src/core/or/channeltls.c 10 problem include-count /src/core/or/circuitbuild.c 54 problem function-size /src/core/or/circuitbuild.c:get_unique_circ_id_by_chan() 128 problem function-size /src/core/or/circuitbuild.c:circuit_extend() 147 problem function-size /src/core/or/circuitbuild.c:choose_good_exit_server_general() 206 +problem dependency-violation /src/core/or/circuitbuild.c 25 problem include-count /src/core/or/circuitlist.c 55 -problem function-size /src/core/or/circuitlist.c:HT_PROTOTYPE() 128 +problem function-size /src/core/or/circuitlist.c:HT_PROTOTYPE() 109 problem function-size /src/core/or/circuitlist.c:circuit_free_() 143 -problem function-size /src/core/or/circuitlist.c:circuit_find_to_cannibalize() 102 +problem function-size /src/core/or/circuitlist.c:circuit_find_to_cannibalize() 101 problem function-size /src/core/or/circuitlist.c:circuit_about_to_free() 120 problem function-size /src/core/or/circuitlist.c:circuits_handle_oom() 117 -problem function-size /src/core/or/circuitmux.c:circuitmux_set_policy() 110 -problem function-size /src/core/or/circuitmux.c:circuitmux_attach_circuit() 114 -problem file-size /src/core/or/circuitpadding.c 3040 -problem function-size /src/core/or/circuitpadding.c:circpad_machine_schedule_padding() 107 -problem function-size /src/core/or/circuitpadding.c:circpad_machine_schedule_padding() 113 -problem function-size /src/core/or/circuitpadding_machines.c:circpad_machine_relay_hide_intro_circuits() 104 +problem dependency-violation /src/core/or/circuitlist.c 19 +problem function-size /src/core/or/circuitmux.c:circuitmux_set_policy() 109 +problem function-size /src/core/or/circuitmux.c:circuitmux_attach_circuit() 113 +problem dependency-violation /src/core/or/circuitmux_ewma.c 2 +problem file-size /src/core/or/circuitpadding.c 3096 +problem function-size /src/core/or/circuitpadding.c:circpad_machine_schedule_padding() 113 +problem dependency-violation /src/core/or/circuitpadding.c 6 +problem file-size /src/core/or/circuitpadding.h 813 +problem function-size /src/core/or/circuitpadding_machines.c:circpad_machine_relay_hide_intro_circuits() 103 problem function-size /src/core/or/circuitpadding_machines.c:circpad_machine_client_hide_rend_circuits() 112 -problem function-size /src/core/or/circuitstats.c:circuit_build_times_parse_state() 124 +problem dependency-violation /src/core/or/circuitpadding_machines.c 1 +problem function-size /src/core/or/circuitstats.c:circuit_build_times_parse_state() 123 +problem dependency-violation /src/core/or/circuitstats.c 11 problem file-size /src/core/or/circuituse.c 3162 -problem function-size /src/core/or/circuituse.c:circuit_is_acceptable() 132 +problem function-size /src/core/or/circuituse.c:circuit_is_acceptable() 128 problem function-size /src/core/or/circuituse.c:circuit_expire_building() 394 problem function-size /src/core/or/circuituse.c:circuit_log_ancient_one_hop_circuits() 126 problem function-size /src/core/or/circuituse.c:circuit_build_failed() 149 -problem function-size /src/core/or/circuituse.c:circuit_launch_by_extend_info() 110 -problem function-size /src/core/or/circuituse.c:circuit_get_open_circ_or_launch() 354 +problem function-size /src/core/or/circuituse.c:circuit_launch_by_extend_info() 108 +problem function-size /src/core/or/circuituse.c:circuit_get_open_circ_or_launch() 352 problem function-size /src/core/or/circuituse.c:connection_ap_handshake_attach_circuit() 244 +problem dependency-violation /src/core/or/circuituse.c 23 problem function-size /src/core/or/command.c:command_process_create_cell() 156 problem function-size /src/core/or/command.c:command_process_relay_cell() 132 -problem file-size /src/core/or/connection_edge.c 4595 +problem dependency-violation /src/core/or/command.c 8 +problem file-size /src/core/or/connection_edge.c 4596 problem include-count /src/core/or/connection_edge.c 65 problem function-size /src/core/or/connection_edge.c:connection_ap_expire_beginning() 117 -problem function-size /src/core/or/connection_edge.c:connection_ap_handshake_rewrite() 192 -problem function-size /src/core/or/connection_edge.c:connection_ap_handle_onion() 188 -problem function-size /src/core/or/connection_edge.c:connection_ap_handshake_rewrite_and_attach() 423 +problem function-size /src/core/or/connection_edge.c:connection_ap_handshake_rewrite() 191 +problem function-size /src/core/or/connection_edge.c:connection_ap_handle_onion() 185 +problem function-size /src/core/or/connection_edge.c:connection_ap_handshake_rewrite_and_attach() 421 problem function-size /src/core/or/connection_edge.c:connection_ap_handshake_send_begin() 111 -problem function-size /src/core/or/connection_edge.c:connection_ap_handshake_socks_resolved() 106 -problem function-size /src/core/or/connection_edge.c:connection_exit_begin_conn() 184 +problem function-size /src/core/or/connection_edge.c:connection_ap_handshake_socks_resolved() 101 +problem function-size /src/core/or/connection_edge.c:connection_exit_begin_conn() 185 problem function-size /src/core/or/connection_edge.c:connection_exit_connect() 102 -problem file-size /src/core/or/connection_or.c 3124 +problem dependency-violation /src/core/or/connection_edge.c 27 +problem file-size /src/core/or/connection_or.c 3122 problem include-count /src/core/or/connection_or.c 51 problem function-size /src/core/or/connection_or.c:connection_or_group_set_badness_() 105 -problem function-size /src/core/or/connection_or.c:connection_or_client_learned_peer_id() 144 -problem function-size /src/core/or/connection_or.c:connection_or_compute_authenticate_cell_body() 235 +problem function-size /src/core/or/connection_or.c:connection_or_client_learned_peer_id() 142 +problem function-size /src/core/or/connection_or.c:connection_or_compute_authenticate_cell_body() 231 +problem dependency-violation /src/core/or/connection_or.c 20 +problem dependency-violation /src/core/or/dos.c 5 +problem dependency-violation /src/core/or/onion.c 2 +problem file-size /src/core/or/or.h 1107 +problem include-count /src/core/or/or.h 49 +problem dependency-violation /src/core/or/or_periodic.c 1 problem file-size /src/core/or/policies.c 3249 problem function-size /src/core/or/policies.c:policy_summarize() 107 +problem dependency-violation /src/core/or/policies.c 14 problem function-size /src/core/or/protover.c:protover_all_supported() 117 -problem file-size /src/core/or/relay.c 3244 +problem dependency-violation /src/core/or/reasons.c 2 +problem file-size /src/core/or/relay.c 3264 problem function-size /src/core/or/relay.c:circuit_receive_relay_cell() 127 -problem function-size /src/core/or/relay.c:relay_send_command_from_edge_() 112 -problem function-size /src/core/or/relay.c:connection_ap_process_end_not_open() 194 -problem function-size /src/core/or/relay.c:connection_edge_process_relay_cell_not_open() 139 -problem function-size /src/core/or/relay.c:connection_edge_process_relay_cell() 430 -problem function-size /src/core/or/relay.c:connection_edge_package_raw_inbuf() 129 -problem function-size /src/core/or/relay.c:circuit_resume_edge_reading_helper() 148 +problem function-size /src/core/or/relay.c:relay_send_command_from_edge_() 109 +problem function-size /src/core/or/relay.c:connection_ap_process_end_not_open() 192 +problem function-size /src/core/or/relay.c:connection_edge_process_relay_cell_not_open() 137 +problem function-size /src/core/or/relay.c:handle_relay_cell_command() 369 +problem function-size /src/core/or/relay.c:connection_edge_package_raw_inbuf() 128 +problem function-size /src/core/or/relay.c:circuit_resume_edge_reading_helper() 146 +problem dependency-violation /src/core/or/relay.c 16 +problem dependency-violation /src/core/or/scheduler.c 1 problem function-size /src/core/or/scheduler_kist.c:kist_scheduler_run() 171 +problem dependency-violation /src/core/or/scheduler_kist.c 2 problem function-size /src/core/or/scheduler_vanilla.c:vanilla_scheduler_run() 109 +problem dependency-violation /src/core/or/scheduler_vanilla.c 1 +problem dependency-violation /src/core/or/sendme.c 2 +problem dependency-violation /src/core/or/status.c 12 problem function-size /src/core/or/versions.c:tor_version_parse() 104 -problem function-size /src/core/proto/proto_socks.c:parse_socks_client() 112 -problem function-size /src/feature/client/addressmap.c:addressmap_rewrite() 112 +problem dependency-violation /src/core/proto/proto_cell.c 3 +problem dependency-violation /src/core/proto/proto_control0.c 1 +problem dependency-violation /src/core/proto/proto_ext_or.c 2 +problem dependency-violation /src/core/proto/proto_http.c 1 +problem function-size /src/core/proto/proto_socks.c:parse_socks_client() 110 +problem dependency-violation /src/core/proto/proto_socks.c 8 +problem function-size /src/feature/client/addressmap.c:addressmap_rewrite() 109 problem function-size /src/feature/client/bridges.c:rewrite_node_address_for_bridge() 126 problem function-size /src/feature/client/circpathbias.c:pathbias_measure_close_rate() 108 problem function-size /src/feature/client/dnsserv.c:evdns_server_callback() 153 problem file-size /src/feature/client/entrynodes.c 3824 -problem function-size /src/feature/client/entrynodes.c:entry_guards_upgrade_waiting_circuits() 157 +problem function-size /src/feature/client/entrynodes.c:entry_guards_upgrade_waiting_circuits() 155 problem function-size /src/feature/client/entrynodes.c:entry_guard_parse_from_state() 246 +problem file-size /src/feature/client/entrynodes.h 639 problem function-size /src/feature/client/transports.c:handle_proxy_line() 108 -problem function-size /src/feature/client/transports.c:parse_method_line_helper() 112 +problem function-size /src/feature/client/transports.c:parse_method_line_helper() 110 problem function-size /src/feature/client/transports.c:create_managed_proxy_environment() 109 -problem function-size /src/feature/control/control.c:connection_control_process_inbuf() 136 -problem function-size /src/feature/control/control_auth.c:handle_control_authchallenge() 103 -problem function-size /src/feature/control/control_auth.c:handle_control_authenticate() 187 -problem function-size /src/feature/control/control_cmd.c:handle_control_extendcircuit() 151 -problem function-size /src/feature/control/control_cmd.c:handle_control_add_onion() 269 -problem function-size /src/feature/control/control_cmd.c:add_onion_helper_keyarg() 125 -problem function-size /src/feature/control/control_events.c:control_event_stream_status() 119 +problem function-size /src/feature/control/control.c:connection_control_process_inbuf() 113 +problem function-size /src/feature/control/control_auth.c:handle_control_authenticate() 186 +problem function-size /src/feature/control/control_cmd.c:handle_control_extendcircuit() 150 +problem function-size /src/feature/control/control_cmd.c:handle_control_add_onion() 256 +problem function-size /src/feature/control/control_cmd.c:add_onion_helper_keyarg() 116 +problem function-size /src/feature/control/control_events.c:control_event_stream_status() 118 problem include-count /src/feature/control/control_getinfo.c 54 -problem function-size /src/feature/control/control_getinfo.c:getinfo_helper_misc() 109 -problem function-size /src/feature/control/control_getinfo.c:getinfo_helper_dir() 304 -problem function-size /src/feature/control/control_getinfo.c:getinfo_helper_events() 236 -problem function-size /src/feature/dirauth/bwauth.c:dirserv_read_measured_bandwidths() 124 -problem file-size /src/feature/dirauth/dirvote.c 4726 +problem function-size /src/feature/control/control_getinfo.c:getinfo_helper_misc() 108 +problem function-size /src/feature/control/control_getinfo.c:getinfo_helper_dir() 302 +problem function-size /src/feature/control/control_getinfo.c:getinfo_helper_events() 234 +problem function-size /src/feature/dirauth/bwauth.c:dirserv_read_measured_bandwidths() 121 +problem file-size /src/feature/dirauth/dirvote.c 4700 problem include-count /src/feature/dirauth/dirvote.c 53 -problem function-size /src/feature/dirauth/dirvote.c:format_networkstatus_vote() 249 -problem function-size /src/feature/dirauth/dirvote.c:networkstatus_compute_bw_weights_v10() 235 -problem function-size /src/feature/dirauth/dirvote.c:networkstatus_compute_consensus() 962 -problem function-size /src/feature/dirauth/dirvote.c:networkstatus_add_detached_signatures() 123 +problem function-size /src/feature/dirauth/dirvote.c:format_networkstatus_vote() 231 +problem function-size /src/feature/dirauth/dirvote.c:networkstatus_compute_bw_weights_v10() 233 +problem function-size /src/feature/dirauth/dirvote.c:networkstatus_compute_consensus() 956 +problem function-size /src/feature/dirauth/dirvote.c:networkstatus_add_detached_signatures() 119 problem function-size /src/feature/dirauth/dirvote.c:dirvote_add_vote() 162 problem function-size /src/feature/dirauth/dirvote.c:dirvote_compute_consensuses() 164 -problem function-size /src/feature/dirauth/dirvote.c:dirserv_generate_networkstatus_vote_obj() 293 +problem function-size /src/feature/dirauth/dirvote.c:dirserv_generate_networkstatus_vote_obj() 283 problem function-size /src/feature/dirauth/dsigs_parse.c:networkstatus_parse_detached_signatures() 196 -problem function-size /src/feature/dirauth/guardfraction.c:dirserv_read_guardfraction_file_from_str() 110 +problem function-size /src/feature/dirauth/guardfraction.c:dirserv_read_guardfraction_file_from_str() 109 problem function-size /src/feature/dirauth/process_descs.c:dirserv_add_descriptor() 125 -problem function-size /src/feature/dirauth/shared_random.c:should_keep_commit() 110 +problem function-size /src/feature/dirauth/shared_random.c:should_keep_commit() 109 problem function-size /src/feature/dirauth/voteflags.c:dirserv_compute_performance_thresholds() 172 problem function-size /src/feature/dircache/consdiffmgr.c:consdiffmgr_cleanup() 115 problem function-size /src/feature/dircache/consdiffmgr.c:consdiffmgr_rescan_flavor_() 111 problem function-size /src/feature/dircache/consdiffmgr.c:consensus_diff_worker_threadfn() 132 -problem function-size /src/feature/dircache/dircache.c:handle_get_current_consensus() 166 -problem function-size /src/feature/dircache/dircache.c:directory_handle_command_post() 120 +problem function-size /src/feature/dircache/dircache.c:handle_get_current_consensus() 165 +problem function-size /src/feature/dircache/dircache.c:directory_handle_command_post() 119 problem file-size /src/feature/dirclient/dirclient.c 3215 problem include-count /src/feature/dirclient/dirclient.c 51 -problem function-size /src/feature/dirclient/dirclient.c:directory_get_from_dirserver() 131 +problem function-size /src/feature/dirclient/dirclient.c:directory_get_from_dirserver() 126 problem function-size /src/feature/dirclient/dirclient.c:directory_initiate_request() 201 -problem function-size /src/feature/dirclient/dirclient.c:directory_send_command() 241 -problem function-size /src/feature/dirclient/dirclient.c:dir_client_decompress_response_body() 114 +problem function-size /src/feature/dirclient/dirclient.c:directory_send_command() 239 +problem function-size /src/feature/dirclient/dirclient.c:dir_client_decompress_response_body() 111 problem function-size /src/feature/dirclient/dirclient.c:connection_dir_client_reached_eof() 189 -problem function-size /src/feature/dirclient/dirclient.c:handle_response_fetch_consensus() 105 -problem function-size /src/feature/dircommon/consdiff.c:gen_ed_diff() 204 -problem function-size /src/feature/dircommon/consdiff.c:apply_ed_diff() 159 -problem function-size /src/feature/dirparse/authcert_parse.c:authority_cert_parse_from_string() 182 -problem function-size /src/feature/dirparse/microdesc_parse.c:microdescs_parse_from_string() 169 -problem function-size /src/feature/dirparse/ns_parse.c:routerstatus_parse_entry_from_string() 286 +problem function-size /src/feature/dirclient/dirclient.c:handle_response_fetch_consensus() 104 +problem function-size /src/feature/dircommon/consdiff.c:gen_ed_diff() 203 +problem function-size /src/feature/dircommon/consdiff.c:apply_ed_diff() 158 +problem function-size /src/feature/dirparse/authcert_parse.c:authority_cert_parse_from_string() 181 +problem function-size /src/feature/dirparse/ns_parse.c:routerstatus_parse_entry_from_string() 280 problem function-size /src/feature/dirparse/ns_parse.c:networkstatus_verify_bw_weights() 389 -problem function-size /src/feature/dirparse/ns_parse.c:networkstatus_parse_vote_from_string() 638 -problem function-size /src/feature/dirparse/parsecommon.c:tokenize_string() 103 -problem function-size /src/feature/dirparse/parsecommon.c:get_next_token() 159 -problem function-size /src/feature/dirparse/routerparse.c:router_parse_entry_from_string() 557 -problem function-size /src/feature/dirparse/routerparse.c:extrainfo_parse_entry_from_string() 210 +problem function-size /src/feature/dirparse/ns_parse.c:networkstatus_parse_vote_from_string() 635 +problem function-size /src/feature/dirparse/parsecommon.c:tokenize_string() 101 +problem function-size /src/feature/dirparse/parsecommon.c:get_next_token() 158 +problem function-size /src/feature/dirparse/routerparse.c:router_parse_entry_from_string() 554 +problem function-size /src/feature/dirparse/routerparse.c:extrainfo_parse_entry_from_string() 208 problem function-size /src/feature/hibernate/hibernate.c:accounting_parse_options() 109 problem function-size /src/feature/hs/hs_cell.c:hs_cell_build_establish_intro() 115 -problem function-size /src/feature/hs/hs_cell.c:hs_cell_parse_introduce2() 154 -problem function-size /src/feature/hs/hs_client.c:send_introduce1() 104 -problem function-size /src/feature/hs/hs_client.c:hs_config_client_authorization() 108 -problem function-size /src/feature/hs/hs_common.c:hs_get_responsible_hsdirs() 104 -problem function-size /src/feature/hs/hs_config.c:config_generic_service() 140 -problem function-size /src/feature/hs/hs_descriptor.c:desc_encode_v3() 104 -problem function-size /src/feature/hs/hs_descriptor.c:decrypt_desc_layer() 110 +problem function-size /src/feature/hs/hs_cell.c:hs_cell_parse_introduce2() 152 +problem function-size /src/feature/hs/hs_client.c:send_introduce1() 103 +problem function-size /src/feature/hs/hs_client.c:hs_config_client_authorization() 107 +problem function-size /src/feature/hs/hs_common.c:hs_get_responsible_hsdirs() 102 +problem function-size /src/feature/hs/hs_config.c:config_service_v3() 107 +problem function-size /src/feature/hs/hs_config.c:config_generic_service() 138 +problem function-size /src/feature/hs/hs_descriptor.c:desc_encode_v3() 101 +problem function-size /src/feature/hs/hs_descriptor.c:decrypt_desc_layer() 111 problem function-size /src/feature/hs/hs_descriptor.c:decode_introduction_point() 122 -problem function-size /src/feature/hs/hs_descriptor.c:desc_decode_superencrypted_v3() 109 -problem function-size /src/feature/hs/hs_descriptor.c:desc_decode_encrypted_v3() 109 -problem file-size /src/feature/hs/hs_service.c 4109 -problem function-size /src/feature/keymgt/loadkey.c:ed_key_init_from_file() 333 -problem function-size /src/feature/nodelist/authcert.c:trusted_dirs_load_certs_from_string() 124 -problem function-size /src/feature/nodelist/authcert.c:authority_certs_fetch_missing() 296 -problem function-size /src/feature/nodelist/fmt_routerstatus.c:routerstatus_format_entry() 166 +problem function-size /src/feature/hs/hs_descriptor.c:desc_decode_superencrypted_v3() 107 +problem function-size /src/feature/hs/hs_descriptor.c:desc_decode_encrypted_v3() 107 +problem file-size /src/feature/hs/hs_service.c 4116 +problem function-size /src/feature/keymgt/loadkey.c:ed_key_init_from_file() 326 +problem function-size /src/feature/nodelist/authcert.c:trusted_dirs_load_certs_from_string() 123 +problem function-size /src/feature/nodelist/authcert.c:authority_certs_fetch_missing() 295 +problem function-size /src/feature/nodelist/fmt_routerstatus.c:routerstatus_format_entry() 162 problem function-size /src/feature/nodelist/microdesc.c:microdesc_cache_rebuild() 134 -problem include-count /src/feature/nodelist/networkstatus.c 62 -problem function-size /src/feature/nodelist/networkstatus.c:networkstatus_check_consensus_signature() 176 -problem function-size /src/feature/nodelist/networkstatus.c:networkstatus_set_current_consensus() 293 -problem function-size /src/feature/nodelist/node_select.c:router_pick_directory_server_impl() 123 -problem function-size /src/feature/nodelist/node_select.c:compute_weighted_bandwidths() 206 -problem function-size /src/feature/nodelist/node_select.c:router_pick_trusteddirserver_impl() 114 -problem function-size /src/feature/nodelist/nodelist.c:compute_frac_paths_available() 193 -problem file-size /src/feature/nodelist/routerlist.c 3238 +problem include-count /src/feature/nodelist/networkstatus.c 63 +problem function-size /src/feature/nodelist/networkstatus.c:networkstatus_check_consensus_signature() 175 +problem function-size /src/feature/nodelist/networkstatus.c:networkstatus_set_current_consensus() 289 +problem function-size /src/feature/nodelist/node_select.c:router_pick_directory_server_impl() 122 +problem function-size /src/feature/nodelist/node_select.c:compute_weighted_bandwidths() 203 +problem function-size /src/feature/nodelist/node_select.c:router_pick_trusteddirserver_impl() 112 +problem function-size /src/feature/nodelist/nodelist.c:compute_frac_paths_available() 190 +problem file-size /src/feature/nodelist/routerlist.c 3241 problem function-size /src/feature/nodelist/routerlist.c:router_rebuild_store() 148 -problem function-size /src/feature/nodelist/routerlist.c:router_add_to_routerlist() 169 +problem function-size /src/feature/nodelist/routerlist.c:router_add_to_routerlist() 168 problem function-size /src/feature/nodelist/routerlist.c:routerlist_remove_old_routers() 121 -problem function-size /src/feature/nodelist/routerlist.c:update_consensus_router_descriptor_downloads() 136 +problem function-size /src/feature/nodelist/routerlist.c:update_consensus_router_descriptor_downloads() 135 problem function-size /src/feature/nodelist/routerlist.c:update_extrainfo_downloads() 103 -problem function-size /src/feature/relay/dns.c:dns_resolve_impl() 134 +problem function-size /src/feature/relay/dns.c:dns_resolve_impl() 131 problem function-size /src/feature/relay/dns.c:configure_nameservers() 161 -problem function-size /src/feature/relay/dns.c:evdns_callback() 109 -problem file-size /src/feature/relay/router.c 3407 +problem function-size /src/feature/relay/dns.c:evdns_callback() 108 +problem file-size /src/feature/relay/router.c 3522 problem include-count /src/feature/relay/router.c 56 problem function-size /src/feature/relay/router.c:init_keys() 252 problem function-size /src/feature/relay/router.c:get_my_declared_family() 114 problem function-size /src/feature/relay/router.c:router_build_fresh_unsigned_routerinfo() 136 -problem function-size /src/feature/relay/router.c:router_dump_router_to_string() 371 -problem function-size /src/feature/relay/router.c:extrainfo_dump_to_string() 206 +problem function-size /src/feature/relay/router.c:router_dump_router_to_string() 367 problem function-size /src/feature/relay/routerkeys.c:load_ed_keys() 294 -problem function-size /src/feature/rend/rendcache.c:rend_cache_store_v2_desc_as_client() 193 -problem function-size /src/feature/rend/rendclient.c:rend_client_send_introduction() 220 -problem function-size /src/feature/rend/rendcommon.c:rend_encode_v2_descriptors() 225 +problem function-size /src/feature/rend/rendcache.c:rend_cache_store_v2_desc_as_client() 190 +problem function-size /src/feature/rend/rendclient.c:rend_client_send_introduction() 219 +problem function-size /src/feature/rend/rendcommon.c:rend_encode_v2_descriptors() 221 problem function-size /src/feature/rend/rendmid.c:rend_mid_establish_intro_legacy() 104 -problem function-size /src/feature/rend/rendparse.c:rend_parse_v2_service_descriptor() 187 -problem function-size /src/feature/rend/rendparse.c:rend_decrypt_introduction_points() 104 -problem function-size /src/feature/rend/rendparse.c:rend_parse_introduction_points() 131 +problem function-size /src/feature/rend/rendparse.c:rend_parse_v2_service_descriptor() 181 +problem function-size /src/feature/rend/rendparse.c:rend_parse_introduction_points() 129 problem file-size /src/feature/rend/rendservice.c 4511 problem function-size /src/feature/rend/rendservice.c:rend_service_prune_list_impl_() 107 -problem function-size /src/feature/rend/rendservice.c:rend_config_service() 164 +problem function-size /src/feature/rend/rendservice.c:rend_config_service() 162 problem function-size /src/feature/rend/rendservice.c:rend_service_load_auth_keys() 178 -problem function-size /src/feature/rend/rendservice.c:rend_service_receive_introduction() 332 -problem function-size /src/feature/rend/rendservice.c:rend_service_parse_intro_for_v3() 115 -problem function-size /src/feature/rend/rendservice.c:rend_service_decrypt_intro() 115 +problem function-size /src/feature/rend/rendservice.c:rend_service_receive_introduction() 330 +problem function-size /src/feature/rend/rendservice.c:rend_service_parse_intro_for_v3() 111 +problem function-size /src/feature/rend/rendservice.c:rend_service_decrypt_intro() 112 problem function-size /src/feature/rend/rendservice.c:rend_service_intro_has_opened() 126 problem function-size /src/feature/rend/rendservice.c:rend_service_rendezvous_has_opened() 117 -problem function-size /src/feature/rend/rendservice.c:directory_post_to_hs_dir() 108 +problem function-size /src/feature/rend/rendservice.c:directory_post_to_hs_dir() 106 problem function-size /src/feature/rend/rendservice.c:upload_service_descriptor() 111 problem function-size /src/feature/rend/rendservice.c:rend_consider_services_intro_points() 170 problem function-size /src/feature/stats/rephist.c:rep_hist_load_mtbf_data() 185 problem function-size /src/feature/stats/rephist.c:rep_hist_format_exit_stats() 148 -problem function-size /src/lib/compress/compress.c:tor_compress_impl() 133 -problem function-size /src/lib/compress/compress_zstd.c:tor_zstd_compress_process() 126 -problem function-size /src/lib/container/smartlist.c:smartlist_bsearch_idx() 109 +problem function-size /src/lib/compress/compress.c:tor_compress_impl() 127 +problem function-size /src/lib/compress/compress_zstd.c:tor_zstd_compress_process() 123 +problem function-size /src/lib/container/smartlist.c:smartlist_bsearch_idx() 107 problem function-size /src/lib/crypt_ops/crypto_rand.c:crypto_strongest_rand_syscall() 102 -problem function-size /src/lib/encoding/binascii.c:base64_encode() 107 -problem function-size /src/lib/encoding/confline.c:parse_config_line_from_str_verbose() 119 +problem function-size /src/lib/encoding/binascii.c:base64_encode() 106 +problem function-size /src/lib/encoding/confline.c:parse_config_line_from_str_verbose() 117 problem function-size /src/lib/encoding/cstring.c:unescape_string() 108 -problem function-size /src/lib/fs/dir.c:check_private_dir() 231 -problem function-size /src/lib/log/log.c:parse_log_severity_config() 101 +problem function-size /src/lib/fs/dir.c:check_private_dir() 230 problem function-size /src/lib/math/prob_distr.c:sample_uniform_interval() 145 -problem function-size /src/lib/net/address.c:tor_addr_parse_mask_ports() 198 -problem function-size /src/lib/net/address.c:tor_addr_compare_masked() 111 +problem function-size /src/lib/net/address.c:tor_addr_parse_mask_ports() 194 +problem function-size /src/lib/net/address.c:tor_addr_compare_masked() 110 problem function-size /src/lib/net/inaddr.c:tor_inet_pton() 107 -problem function-size /src/lib/net/resolve.c:tor_addr_lookup() 110 problem function-size /src/lib/net/socketpair.c:tor_ersatz_socketpair() 102 problem function-size /src/lib/osinfo/uname.c:get_uname() 116 problem function-size /src/lib/process/process_unix.c:process_unix_exec() 220 -problem function-size /src/lib/process/process_win32.c:process_win32_exec() 133 -problem function-size /src/lib/process/process_win32.c:process_win32_create_pipe() 112 +problem function-size /src/lib/process/process_win32.c:process_win32_exec() 151 +problem function-size /src/lib/process/process_win32.c:process_win32_create_pipe() 109 problem function-size /src/lib/process/restrict.c:set_max_file_descriptors() 102 problem function-size /src/lib/process/setuid.c:switch_id() 156 problem function-size /src/lib/sandbox/sandbox.c:prot_strings() 104 problem function-size /src/lib/string/scanf.c:tor_vsscanf() 112 -problem function-size /src/lib/tls/tortls_nss.c:tor_tls_context_new() 153 -problem function-size /src/lib/tls/tortls_openssl.c:tor_tls_context_new() 171 -problem function-size /src/lib/tls/x509_nss.c:tor_tls_create_certificate_internal() 126 +problem function-size /src/lib/tls/tortls_nss.c:tor_tls_context_new() 152 +problem function-size /src/lib/tls/tortls_openssl.c:tor_tls_context_new() 170 +problem function-size /src/lib/tls/x509_nss.c:tor_tls_create_certificate_internal() 121 problem function-size /src/tools/tor-gencert.c:parse_commandline() 111 -problem function-size /src/tools/tor-resolve.c:build_socks5_resolve_request() 104 -problem function-size /src/tools/tor-resolve.c:do_resolve() 174 +problem function-size /src/tools/tor-resolve.c:build_socks5_resolve_request() 102 +problem function-size /src/tools/tor-resolve.c:do_resolve() 171 problem function-size /src/tools/tor-resolve.c:main() 112 +problem dependency-violation /scripts/maint/practracker/testdata/a.c 3 +problem dependency-violation /scripts/maint/practracker/testdata/header.h 3 +problem dependency-violation /src/core/crypto/hs_ntor.h 1 +problem dependency-violation /src/core/or/cell_queue_st.h 1 +problem dependency-violation /src/core/or/channel.h 1 +problem dependency-violation /src/core/or/circuitlist.h 1 +problem dependency-violation /src/core/or/connection_edge.h 1 +problem dependency-violation /src/core/or/or.h 1 diff --git a/scripts/maint/practracker/includes.py b/scripts/maint/practracker/includes.py new file mode 100755 index 0000000000..397439b4ef --- /dev/null +++ b/scripts/maint/practracker/includes.py @@ -0,0 +1,285 @@ +#!/usr/bin/python +# Copyright 2018 The Tor Project, Inc. See LICENSE file for licensing info. + +"""This script looks through all the directories for files matching *.c or + *.h, and checks their #include directives to make sure that only "permitted" + headers are included. + + Any #include directives with angle brackets (like #include <stdio.h>) are + ignored -- only directives with quotes (like #include "foo.h") are + considered. + + To decide what includes are permitted, this script looks at a .may_include + file in each directory. This file contains empty lines, #-prefixed + comments, filenames (like "lib/foo/bar.h") and file globs (like lib/*/*.h) + for files that are permitted. +""" + + +from __future__ import print_function + +import fnmatch +import os +import re +import sys + +if sys.version_info[0] <= 2: + def open_file(fname): + return open(fname, 'r') +else: + def open_file(fname): + return open(fname, 'r', encoding='utf-8') + +def warn(msg): + print(msg, file=sys.stderr) + +def fname_is_c(fname): + """ Return true iff 'fname' is the name of a file that we should + search for possibly disallowed #include directives. """ + return fname.endswith(".h") or fname.endswith(".c") + +INCLUDE_PATTERN = re.compile(r'\s*#\s*include\s+"([^"]*)"') +RULES_FNAME = ".may_include" + +ALLOWED_PATTERNS = [ + re.compile(r'^.*\*\.(h|inc)$'), + re.compile(r'^.*/.*\.h$'), + re.compile(r'^ext/.*\.c$'), + re.compile(r'^orconfig.h$'), + re.compile(r'^micro-revision.i$'), +] + +def pattern_is_normal(s): + for p in ALLOWED_PATTERNS: + if p.match(s): + return True + return False + +class Error(object): + def __init__(self, location, msg, is_advisory=False): + self.location = location + self.msg = msg + self.is_advisory = is_advisory + + def __str__(self): + return "{} at {}".format(self.msg, self.location) + +class Rules(object): + """ A 'Rules' object is the parsed version of a .may_include file. """ + def __init__(self, dirpath): + self.dirpath = dirpath + if dirpath.startswith("src/"): + self.incpath = dirpath[4:] + else: + self.incpath = dirpath + self.patterns = [] + self.usedPatterns = set() + self.is_advisory = False + + def addPattern(self, pattern): + if pattern == "!advisory": + self.is_advisory = True + return + if not pattern_is_normal(pattern): + warn("Unusual pattern {} in {}".format(pattern, self.dirpath)) + self.patterns.append(pattern) + + def includeOk(self, path): + for pattern in self.patterns: + if fnmatch.fnmatchcase(path, pattern): + self.usedPatterns.add(pattern) + return True + return False + + def applyToLines(self, lines, loc_prefix=""): + lineno = 0 + for line in lines: + lineno += 1 + m = INCLUDE_PATTERN.match(line) + if m: + include = m.group(1) + if not self.includeOk(include): + yield Error("{}{}".format(loc_prefix,str(lineno)), + "Forbidden include of {}".format(include), + is_advisory=self.is_advisory) + + def applyToFile(self, fname, f): + for error in self.applyToLines(iter(f), "{}:".format(fname)): + yield error + + def noteUnusedRules(self): + for p in self.patterns: + if p not in self.usedPatterns: + warn("Pattern {} in {} was never used.".format(p, self.dirpath)) + + def getAllowedDirectories(self): + allowed = [] + for p in self.patterns: + m = re.match(r'^(.*)/\*\.(h|inc)$', p) + if m: + allowed.append(m.group(1)) + continue + m = re.match(r'^(.*)/[^/]*$', p) + if m: + allowed.append(m.group(1)) + continue + + return allowed + +include_rules_cache = {} + +def load_include_rules(fname): + """ Read a rules file from 'fname', and return it as a Rules object. + Return 'None' if fname does not exist. + """ + if fname in include_rules_cache: + return include_rules_cache[fname] + if not os.path.exists(fname): + include_rules_cache[fname] = None + return None + result = Rules(os.path.split(fname)[0]) + with open_file(fname) as f: + for line in f: + line = line.strip() + if line.startswith("#") or not line: + continue + result.addPattern(line) + include_rules_cache[fname] = result + return result + +def get_all_include_rules(): + """Return a list of all the Rules objects we have loaded so far, + sorted by their directory names.""" + return [ rules for (fname,rules) in + sorted(include_rules_cache.items()) + if rules is not None ] + +def remove_self_edges(graph): + """Takes a directed graph in as an adjacency mapping (a mapping from + node to a list of the nodes to which it connects). + + Remove all edges from a node to itself.""" + + for k in list(graph): + graph[k] = [ d for d in graph[k] if d != k ] + +def toposort(graph, limit=100): + """Takes a directed graph in as an adjacency mapping (a mapping from + node to a list of the nodes to which it connects). Tries to + perform a topological sort on the graph, arranging the nodes into + "levels", such that every member of each level is only reachable + by members of later levels. + + Returns a list of the members of each level. + + Modifies the input graph, removing every member that could be + sorted. If the graph does not become empty, then it contains a + cycle. + + "limit" is the max depth of the graph after which we give up trying + to sort it and conclude we have a cycle. + """ + all_levels = [] + + n = 0 + while graph: + n += 0 + cur_level = [] + all_levels.append(cur_level) + for k in list(graph): + graph[k] = [ d for d in graph[k] if d in graph ] + if graph[k] == []: + cur_level.append(k) + for k in cur_level: + del graph[k] + n += 1 + if n > limit: + break + + return all_levels + +def consider_include_rules(fname, f): + dirpath = os.path.split(fname)[0] + rules_fname = os.path.join(dirpath, RULES_FNAME) + rules = load_include_rules(os.path.join(dirpath, RULES_FNAME)) + if rules is None: + return + + for err in rules.applyToFile(fname, f): + yield err + + list_unused = False + log_sorted_levels = False + +def walk_c_files(topdir="src"): + """Run through all .c and .h files under topdir, looking for + include-rule violations. Yield those violations.""" + + for dirpath, dirnames, fnames in os.walk(topdir): + for fname in fnames: + if fname_is_c(fname): + fullpath = os.path.join(dirpath,fname) + with open(fullpath) as f: + for err in consider_include_rules(fullpath, f): + yield err + +def run_check_includes(topdir, list_unused=False, log_sorted_levels=False, + list_advisories=False): + trouble = False + + for err in walk_c_files(topdir): + if err.is_advisory and not list_advisories: + continue + print(err, file=sys.stderr) + if not err.is_advisory: + trouble = True + + if trouble: + err( + """To change which includes are allowed in a C file, edit the {} + files in its enclosing directory.""".format(RULES_FNAME)) + sys.exit(1) + + if list_unused: + for rules in get_all_include_rules(): + rules.noteUnusedRules() + + uses_dirs = { } + for rules in get_all_include_rules(): + uses_dirs[rules.incpath] = rules.getAllowedDirectories() + + remove_self_edges(uses_dirs) + all_levels = toposort(uses_dirs) + + if log_sorted_levels: + for (n, cur_level) in enumerate(all_levels): + if cur_level: + print(n, cur_level) + + if uses_dirs: + print("There are circular .may_include dependencies in here somewhere:", + uses_dirs) + sys.exit(1) + +def main(argv): + import argparse + + progname = argv[0] + parser = argparse.ArgumentParser(prog=progname) + parser.add_argument("--toposort", action="store_true", + help="Print a topologically sorted list of modules") + parser.add_argument("--list-unused", action="store_true", + help="List unused lines in .may_include files.") + parser.add_argument("--list-advisories", action="store_true", + help="List advisories as well as forbidden includes") + parser.add_argument("topdir", default="src", nargs="?", + help="Top-level directory for the tor source") + args = parser.parse_args(argv[1:]) + + run_check_includes(topdir=args.topdir, + log_sorted_levels=args.toposort, + list_unused=args.list_unused, + list_advisories=args.list_advisories) + +if __name__ == '__main__': + main(sys.argv) diff --git a/scripts/maint/practracker/metrics.py b/scripts/maint/practracker/metrics.py index 5fa305a868..4c62bc2425 100644 --- a/scripts/maint/practracker/metrics.py +++ b/scripts/maint/practracker/metrics.py @@ -8,6 +8,7 @@ import re def get_file_len(f): """Get file length of file""" + i = -1 for i, l in enumerate(f): pass return i + 1 @@ -16,7 +17,7 @@ def get_include_count(f): """Get number of #include statements in the file""" include_count = 0 for line in f: - if re.match(r' *# *include', line): + if re.match(r'\s*#\s*include', line): include_count += 1 return include_count @@ -27,10 +28,13 @@ def get_function_lines(f): # Skip lines that look like they are defining functions with these # names: they aren't real function definitions. - REGEXP_CONFUSE_TERMS = {"MOCK_IMPL", "ENABLE_GCC_WARNINGS", "ENABLE_GCC_WARNING", "DUMMY_TYPECHECK_INSTANCE", + REGEXP_CONFUSE_TERMS = {"MOCK_IMPL", "MOCK_DECL", "HANDLE_DECL", + "ENABLE_GCC_WARNINGS", "ENABLE_GCC_WARNING", + "DUMMY_TYPECHECK_INSTANCE", "DISABLE_GCC_WARNING", "DISABLE_GCC_WARNINGS"} in_function = False + found_openbrace = False for lineno, line in enumerate(f): if not in_function: # find the start of a function @@ -41,10 +45,13 @@ def get_function_lines(f): continue func_start = lineno in_function = True - + elif not found_openbrace and line.startswith("{"): + found_openbrace = True + func_start = lineno else: # Find the end of a function if line.startswith("}"): - n_lines = lineno - func_start + n_lines = lineno - func_start + 1 in_function = False + found_openbrace = False yield (func_name, n_lines) diff --git a/scripts/maint/practracker/practracker.py b/scripts/maint/practracker/practracker.py index febb14639d..71741265f6 100755 --- a/scripts/maint/practracker/practracker.py +++ b/scripts/maint/practracker/practracker.py @@ -7,7 +7,8 @@ Go through the various .c files and collect metrics about them. If the metrics violate some of our best practices and they are not found in the optional exceptions file, then log a problem about them. -We currently do metrics about file size, function size and number of includes. +We currently do metrics about file size, function size and number of includes, +for C source files and headers. practracker.py should be run with its second argument pointing to the Tor top-level source directory like this: @@ -25,6 +26,7 @@ import os, sys import metrics import util import problem +import includes # The filename of the exceptions file (it should be placed in the practracker directory) EXCEPTIONS_FNAME = "./exceptions.txt" @@ -35,12 +37,23 @@ MAX_FILE_SIZE = 3000 # lines MAX_FUNCTION_SIZE = 100 # lines # Recommended number of #includes MAX_INCLUDE_COUNT = 50 +# Recommended file size for headers +MAX_H_FILE_SIZE = 500 +# Recommended include count for headers +MAX_H_INCLUDE_COUNT = 15 +# Recommended number of dependency violations +MAX_DEP_VIOLATIONS = 0 + +# Map from problem type to functions that adjust for tolerance +TOLERANCE_FNS = { + 'include-count': lambda n: int(n*1.1), + 'function-size': lambda n: int(n*1.1), + 'file-size': lambda n: int(n*1.02), + 'dependency-violation': lambda n: (n+2) +} ####################################################### -# ProblemVault singleton -ProblemVault = None - # The Tor source code topdir TOR_TOPDIR = None @@ -54,71 +67,73 @@ else: return open(fname, 'r', encoding='utf-8') def consider_file_size(fname, f): - """Consider file size issues for 'f' and return True if a new issue was found""" + """Consider the size of 'f' and yield an FileSizeItem for it. + """ file_size = metrics.get_file_len(f) - if file_size > MAX_FILE_SIZE: - p = problem.FileSizeProblem(fname, file_size) - return ProblemVault.register_problem(p) - return False + yield problem.FileSizeItem(fname, file_size) def consider_includes(fname, f): - """Consider #include issues for 'f' and return True if a new issue was found""" + """Consider the #include count in for 'f' and yield an IncludeCountItem + for it. + """ include_count = metrics.get_include_count(f) - if include_count > MAX_INCLUDE_COUNT: - p = problem.IncludeCountProblem(fname, include_count) - return ProblemVault.register_problem(p) - return False + yield problem.IncludeCountItem(fname, include_count) def consider_function_size(fname, f): - """Consider the function sizes for 'f' and return True if a new issue was found""" - found_new_issues = False + """yield a FunctionSizeItem for every function in f. + """ for name, lines in metrics.get_function_lines(f): - # Don't worry about functions within our limits - if lines <= MAX_FUNCTION_SIZE: - continue - - # That's a big function! Issue a problem! canonical_function_name = "%s:%s()" % (fname, name) - p = problem.FunctionSizeProblem(canonical_function_name, lines) - found_new_issues |= ProblemVault.register_problem(p) + yield problem.FunctionSizeItem(canonical_function_name, lines) + +def consider_include_violations(fname, real_fname, f): + n = 0 + for item in includes.consider_include_rules(real_fname, f): + n += 1 + if n: + yield problem.DependencyViolationItem(fname, n) - return found_new_issues ####################################################### def consider_all_metrics(files_list): - """Consider metrics for all files, and return True if new issues were found""" - found_new_issues = False + """Consider metrics for all files, and yield a sequence of problem.Item + object for those issues.""" for fname in files_list: with open_file(fname) as f: - found_new_issues |= consider_metrics_for_file(fname, f) - return found_new_issues + for item in consider_metrics_for_file(fname, f): + yield item def consider_metrics_for_file(fname, f): """ - Consider the various metrics for file with filename 'fname' and file descriptor 'f'. - Return True if we found new issues. + Yield a sequence of problem.Item objects for all of the metrics in + 'f'. """ + real_fname = fname # Strip the useless part of the path if fname.startswith(TOR_TOPDIR): fname = fname[len(TOR_TOPDIR):] - found_new_issues = False - # Get file length - found_new_issues |= consider_file_size(fname, f) + for item in consider_file_size(fname, f): + yield item # Consider number of #includes f.seek(0) - found_new_issues |= consider_includes(fname, f) + for item in consider_includes(fname, f): + yield item # Get function length f.seek(0) - found_new_issues |= consider_function_size(fname, f) + for item in consider_function_size(fname, f): + yield item - return found_new_issues + # Check for "upward" includes + f.seek(0) + for item in consider_include_violations(fname, real_fname, f): + yield item HEADER="""\ # Welcome to the exceptions file for Tor's best-practices tracker! @@ -129,8 +144,12 @@ HEADER="""\ # # There are three kinds of problems that we recognize right now: # function-size -- a function of more than {MAX_FUNCTION_SIZE} lines. -# file-size -- a file of more than {MAX_FILE_SIZE} lines. -# include-count -- a file with more than {MAX_INCLUDE_COUNT} #includes. +# file-size -- a .c file of more than {MAX_FILE_SIZE} lines, or a .h +# file with more than {MAX_H_FILE_SIZE} lines. +# include-count -- a .c file with more than {MAX_INCLUDE_COUNT} #includes, + or a .h file with more than {MAX_H_INCLUDE_COUNT} #includes. +# dependency-violation -- a file includes a header that it should +# not, according to an advisory .may_include file. # # Each line below represents a single exception that practracker should # _ignore_. Each line has four parts: @@ -161,8 +180,29 @@ def main(argv): parser = argparse.ArgumentParser(prog=progname) parser.add_argument("--regen", action="store_true", help="Regenerate the exceptions file") + parser.add_argument("--list-overbroad", action="store_true", + help="List over-broad exceptions") parser.add_argument("--exceptions", help="Override the location for the exceptions file") + parser.add_argument("--strict", action="store_true", + help="Make all warnings into errors") + parser.add_argument("--terse", action="store_true", + help="Do not emit helpful instructions.") + parser.add_argument("--max-h-file-size", default=MAX_H_FILE_SIZE, + help="Maximum lines per .h file") + parser.add_argument("--max-h-include-count", default=MAX_H_INCLUDE_COUNT, + help="Maximum includes per .h file") + parser.add_argument("--max-file-size", default=MAX_FILE_SIZE, + help="Maximum lines per .c file") + parser.add_argument("--max-include-count", default=MAX_INCLUDE_COUNT, + help="Maximum includes per .c file") + parser.add_argument("--max-function-size", default=MAX_FUNCTION_SIZE, + help="Maximum lines per function") + parser.add_argument("--max-dependency-violations", default=MAX_DEP_VIOLATIONS, + help="Maximum number of dependency violations to allow") + parser.add_argument("--include-dir", action="append", + default=["src"], + help="A directory (under topdir) to search for source") parser.add_argument("topdir", default=".", nargs="?", help="Top-level directory for the tor source") args = parser.parse_args(argv[1:]) @@ -174,24 +214,55 @@ def main(argv): else: exceptions_file = os.path.join(TOR_TOPDIR, "scripts/maint/practracker", EXCEPTIONS_FNAME) + # 0) Configure our thresholds of "what is a problem actually" + filt = problem.ProblemFilter() + filt.addThreshold(problem.FileSizeItem("*.c", int(args.max_file_size))) + filt.addThreshold(problem.IncludeCountItem("*.c", int(args.max_include_count))) + filt.addThreshold(problem.FileSizeItem("*.h", int(args.max_h_file_size))) + filt.addThreshold(problem.IncludeCountItem("*.h", int(args.max_h_include_count))) + filt.addThreshold(problem.FunctionSizeItem("*.c", int(args.max_function_size))) + filt.addThreshold(problem.DependencyViolationItem("*.c", int(args.max_dependency_violations))) + filt.addThreshold(problem.DependencyViolationItem("*.h", int(args.max_dependency_violations))) + + if args.list_overbroad and args.regen: + print("Cannot use --regen with --list-overbroad", + file=sys.stderr) + sys.exit(1) + # 1) Get all the .c files we care about - files_list = util.get_tor_c_files(TOR_TOPDIR) + files_list = util.get_tor_c_files(TOR_TOPDIR, args.include_dir) # 2) Initialize problem vault and load an optional exceptions file so that # we don't warn about the past - global ProblemVault - if args.regen: tmpname = exceptions_file + ".tmp" tmpfile = open(tmpname, "w") - sys.stdout = tmpfile - sys.stdout.write(HEADER) + problem_file = tmpfile + problem_file.write(HEADER) ProblemVault = problem.ProblemVault() else: ProblemVault = problem.ProblemVault(exceptions_file) + problem_file = sys.stdout + + if args.list_overbroad: + # If we're listing overbroad exceptions, don't list problems. + problem_file = util.NullFile() + + # 2.1) Adjust the exceptions so that we warn only about small problems, + # and produce errors on big ones. + if not (args.regen or args.list_overbroad or args.strict): + ProblemVault.set_tolerances(TOLERANCE_FNS) # 3) Go through all the files and report problems if they are not exceptions - found_new_issues = consider_all_metrics(files_list) + found_new_issues = 0 + for item in filt.filter(consider_all_metrics(files_list)): + status = ProblemVault.register_problem(item) + if status == problem.STATUS_ERR: + print(item, file=problem_file) + found_new_issues += 1 + elif status == problem.STATUS_WARN: + # warnings always go to stdout. + print("(warning) {}".format(item)) if args.regen: tmpfile.close() @@ -199,18 +270,34 @@ def main(argv): sys.exit(0) # If new issues were found, try to give out some advice to the developer on how to resolve it. - if found_new_issues and not args.regen: + if found_new_issues and not args.regen and not args.terse: new_issues_str = """\ -FAILURE: practracker found new problems in the code: see warnings above. +FAILURE: practracker found {} new problem(s) in the code: see warnings above. Please fix the problems if you can, and update the exceptions file ({}) if you can't. See doc/HACKING/HelpfulTools.md for more information on using practracker.\ -""".format(exceptions_file) + +You can disable this message by setting the TOR_DISABLE_PRACTRACKER environment +variable. +""".format(found_new_issues, exceptions_file) print(new_issues_str) + if args.list_overbroad: + def k_fn(tup): + return tup[0].key() + for (ex,p) in sorted(ProblemVault.list_overbroad_exceptions(), key=k_fn): + if p is None: + print(ex, "->", 0) + else: + print(ex, "->", p.metric_value) + sys.exit(found_new_issues) if __name__ == '__main__': + if os.environ.get("TOR_DISABLE_PRACTRACKER"): + print("TOR_DISABLE_PRACTRACKER is set, skipping practracker tests.", + file=sys.stderr) + sys.exit(0) main(sys.argv) diff --git a/scripts/maint/practracker/practracker_tests.py b/scripts/maint/practracker/practracker_tests.py index cdbab2908e..45719d6cb7 100755 --- a/scripts/maint/practracker/practracker_tests.py +++ b/scripts/maint/practracker/practracker_tests.py @@ -1,8 +1,15 @@ +#!/usr/bin/python + """Some simple tests for practracker metrics""" import unittest -import StringIO +try: + # python 2 names the module this way... + from StringIO import StringIO +except ImportError: + # python 3 names the module this way. + from io import StringIO import metrics @@ -36,15 +43,25 @@ fun,( class TestFunctionLength(unittest.TestCase): def test_function_length(self): - funcs = StringIO.StringIO(function_file) + funcs = StringIO(function_file) # All functions should have length 2 - for name, lines in metrics.function_lines(funcs): + for name, lines in metrics.get_function_lines(funcs): self.assertEqual(name, "fun") funcs.seek(0) - for name, lines in metrics.function_lines(funcs): - self.assertEqual(lines, 2) + for name, lines in metrics.get_function_lines(funcs): + self.assertEqual(lines, 4) + +class TestIncludeCount(unittest.TestCase): + def test_include_count(self): + f = StringIO(""" + # include <abc.h> + # include "def.h" +#include "ghi.h" +\t#\t include "jkl.h" +""") + self.assertEqual(metrics.get_include_count(f),4) if __name__ == '__main__': unittest.main() diff --git a/scripts/maint/practracker/problem.py b/scripts/maint/practracker/problem.py index c82c5db572..d21840a213 100644 --- a/scripts/maint/practracker/problem.py +++ b/scripts/maint/practracker/problem.py @@ -13,6 +13,10 @@ import os.path import re import sys +STATUS_ERR = 2 +STATUS_WARN = 1 +STATUS_OK = 0 + class ProblemVault(object): """ Singleton where we store the various new problems we @@ -22,6 +26,9 @@ class ProblemVault(object): def __init__(self, exception_fname=None): # Exception dictionary: { problem.key() : Problem object } self.exceptions = {} + # Exception dictionary: maps key to the problem it was used to + # suppress. + self.used_exception_for = {} if exception_fname == None: return @@ -57,42 +64,93 @@ class ProblemVault(object): def register_problem(self, problem): """ - Register this problem to the problem value. Return True if it was a new - problem or it worsens an already existing problem. + Register this problem to the problem value. Return true if it was a new + problem or it worsens an already existing problem. A true + value may be STATUS_ERR to indicate a hard violation, or STATUS_WARN + to indicate a warning. """ # This is a new problem, print it if problem.key() not in self.exceptions: - print(problem) - return True + return STATUS_ERR # If it's an old problem, we don't warn if the situation got better # (e.g. we went from 4k LoC to 3k LoC), but we do warn if the # situation worsened (e.g. we went from 60 includes to 80). - if problem.is_worse_than(self.exceptions[problem.key()]): - print(problem) - return True + status = problem.is_worse_than(self.exceptions[problem.key()]) + + # Remember that we used this exception, so that we can later + # determine whether the exception was overbroad. + self.used_exception_for[problem.key()] = problem - return False + return status -class Problem(object): + def list_overbroad_exceptions(self): + """Return an iterator of tuples containing (ex,prob) where ex is an + exceptions in this vault that are stricter than it needs to be, and + prob is the worst problem (if any) that it covered. + """ + for k in self.exceptions: + e = self.exceptions[k] + p = self.used_exception_for.get(k) + if p is None or e.is_worse_than(p): + yield (e, p) + + def set_tolerances(self, fns): + """Adjust the tolerances for the exceptions in this vault. Takes + a map of problem type to a function that adjusts the permitted + function to its new maximum value.""" + for k in self.exceptions: + ex = self.exceptions[k] + fn = fns.get(ex.problem_type) + if fn is not None: + ex.metric_value = fn(ex.metric_value) + +class ProblemFilter(object): + def __init__(self): + self.thresholds = dict() + + def addThreshold(self, item): + self.thresholds[(item.get_type(),item.get_file_type())] = item + + def matches(self, item): + key = (item.get_type(), item.get_file_type()) + filt = self.thresholds.get(key, None) + if filt is None: + return False + return item.is_worse_than(filt) + + def filter(self, sequence): + for item in iter(sequence): + if self.matches(item): + yield item + +class Item(object): """ - A generic problem in our source code. See the subclasses below for the - specific problems we are trying to tackle. + A generic measurement about some aspect of our source code. See + the subclasses below for the specific problems we are trying to tackle. """ def __init__(self, problem_type, problem_location, metric_value): self.problem_location = problem_location self.metric_value = int(metric_value) + self.warning_threshold = self.metric_value self.problem_type = problem_type def is_worse_than(self, other_problem): - """Return True if this is a worse problem than other_problem""" + """Return STATUS_ERR if this is a worse problem than other_problem. + Return STATUS_WARN if it is a little worse, but falls within the + warning threshold. Return STATUS_OK if this problem is not + at all worse than other_problem. + """ if self.metric_value > other_problem.metric_value: - return True - return False + return STATUS_ERR + elif self.metric_value > other_problem.warning_threshold: + return STATUS_WARN + else: + return STATUS_OK def key(self): """Generate a unique key that describes this problem that can be used as a dictionary key""" - # Problem location is a filesystem path, so we need to normalize this + # Item location is a filesystem path, so we need to normalize this # across platforms otherwise same paths are not gonna match. canonical_location = os.path.normcase(self.problem_location) return "%s:%s" % (canonical_location, self.problem_type) @@ -100,7 +158,16 @@ class Problem(object): def __str__(self): return "problem %s %s %s" % (self.problem_type, self.problem_location, self.metric_value) -class FileSizeProblem(Problem): + def get_type(self): + return self.problem_type + + def get_file_type(self): + if self.problem_location.endswith(".h"): + return "*.h" + else: + return "*.c" + +class FileSizeItem(Item): """ Denotes a problem with the size of a .c file. @@ -108,9 +175,9 @@ class FileSizeProblem(Problem): 'metric_value' is the number of lines in the .c file. """ def __init__(self, problem_location, metric_value): - super(FileSizeProblem, self).__init__("file-size", problem_location, metric_value) + super(FileSizeItem, self).__init__("file-size", problem_location, metric_value) -class IncludeCountProblem(Problem): +class IncludeCountItem(Item): """ Denotes a problem with the number of #includes in a .c file. @@ -118,9 +185,9 @@ class IncludeCountProblem(Problem): 'metric_value' is the number of #includes in the .c file. """ def __init__(self, problem_location, metric_value): - super(IncludeCountProblem, self).__init__("include-count", problem_location, metric_value) + super(IncludeCountItem, self).__init__("include-count", problem_location, metric_value) -class FunctionSizeProblem(Problem): +class FunctionSizeItem(Item): """ Denotes a problem with a size of a function in a .c file. @@ -131,7 +198,22 @@ class FunctionSizeProblem(Problem): The 'metric_value' is the size of the offending function in lines. """ def __init__(self, problem_location, metric_value): - super(FunctionSizeProblem, self).__init__("function-size", problem_location, metric_value) + super(FunctionSizeItem, self).__init__("function-size", problem_location, metric_value) + +class DependencyViolationItem(Item): + """ + Denotes a dependency violation in a .c or .h file. A dependency violation + occurs when a file includes a file from some module that is not listed + in its .may_include file. + + The 'problem_location' is the file that contains the problem. + + The 'metric_value' is the number of forbidden includes. + """ + def __init__(self, problem_location, metric_value): + super(DependencyViolationItem, self).__init__("dependency-violation", + problem_location, + metric_value) comment_re = re.compile(r'#.*$') @@ -149,10 +231,12 @@ def get_old_problem_from_exception_str(exception_str): raise ValueError("Misformatted line {!r}".format(orig_str)) if problem_type == "file-size": - return FileSizeProblem(problem_location, metric_value) + return FileSizeItem(problem_location, metric_value) elif problem_type == "include-count": - return IncludeCountProblem(problem_location, metric_value) + return IncludeCountItem(problem_location, metric_value) elif problem_type == "function-size": - return FunctionSizeProblem(problem_location, metric_value) + return FunctionSizeItem(problem_location, metric_value) + elif problem_type == "dependency-violation": + return DependencyViolationItem(problem_location, metric_value) else: raise ValueError("Unknown exception type {!r}".format(orig_str)) diff --git a/scripts/maint/practracker/test_practracker.sh b/scripts/maint/practracker/test_practracker.sh new file mode 100755 index 0000000000..207a5ceded --- /dev/null +++ b/scripts/maint/practracker/test_practracker.sh @@ -0,0 +1,70 @@ +#!/bin/sh + +umask 077 +unset TOR_DISABLE_PRACTRACKER + +TMPDIR="" +clean () { + if [ -n "$TMPDIR" ] && [ -d "$TMPDIR" ]; then + rm -rf "$TMPDIR" + fi +} +trap clean EXIT HUP INT TERM + +if test "${PRACTRACKER_DIR}" = "" || + test ! -e "${PRACTRACKER_DIR}/practracker.py" ; then + PRACTRACKER_DIR=$(dirname "$0") +fi + +TMPDIR="$(mktemp -d -t pracktracker.test.XXXXXX)" +if test -z "${TMPDIR}" || test ! -d "${TMPDIR}" ; then + echo >&2 "mktemp failed." + exit 1; +fi + +DATA="${PRACTRACKER_DIR}/testdata" + +run_practracker() { + "${PYTHON:-python}" "${PRACTRACKER_DIR}/practracker.py" \ + --include-dir "" \ + --max-file-size=0 \ + --max-function-size=0 \ + --max-h-file-size=0 \ + --max-h-include-count=0 \ + --max-include-count=0 \ + --terse \ + "${DATA}/" "$@"; +} +compare() { + # we can't use cmp because we need to use -b for windows + diff -b -u "$@" > "${TMPDIR}/test-diff" + if test -z "$(cat "${TMPDIR}"/test-diff)"; then + echo "OK" + else + cat "${TMPDIR}/test-diff" + echo "FAILED" + exit 1 + fi +} + +echo "unit tests:" + +"${PYTHON:-python}" "${PRACTRACKER_DIR}/practracker_tests.py" || exit 1 + +echo "ex0:" + +run_practracker --exceptions "${DATA}/ex0.txt" > "${TMPDIR}/ex0-received.txt" + +compare "${TMPDIR}/ex0-received.txt" "${DATA}/ex0-expected.txt" + +echo "ex1:" + +run_practracker --exceptions "${DATA}/ex1.txt" > "${TMPDIR}/ex1-received.txt" + +compare "${TMPDIR}/ex1-received.txt" "${DATA}/ex1-expected.txt" + +echo "ex1.overbroad:" + +run_practracker --exceptions "${DATA}/ex1.txt" --list-overbroad > "${TMPDIR}/ex1-overbroad-received.txt" + +compare "${TMPDIR}/ex1-overbroad-received.txt" "${DATA}/ex1-overbroad-expected.txt" diff --git a/scripts/maint/practracker/testdata/.may_include b/scripts/maint/practracker/testdata/.may_include new file mode 100644 index 0000000000..40bf8155d9 --- /dev/null +++ b/scripts/maint/practracker/testdata/.may_include @@ -0,0 +1,3 @@ +!advisory + +permitted.h diff --git a/scripts/maint/practracker/testdata/a.c b/scripts/maint/practracker/testdata/a.c new file mode 100644 index 0000000000..1939773f57 --- /dev/null +++ b/scripts/maint/practracker/testdata/a.c @@ -0,0 +1,38 @@ + +#include "one.h" +#include "two.h" +#incldue "three.h" + +# include "permitted.h" + +int +i_am_a_function(void) +{ + call(); + call(); + /* comment + + another */ + + return 3; +} + +# include "five.h" + +long +another_function(long x, + long y) +{ + int abcd; + + abcd = x+y; + abcd *= abcd; + + /* comment here */ + + return abcd + + abcd + + abcd; +} + +/* And a comment to grow! */ diff --git a/scripts/maint/practracker/testdata/b.c b/scripts/maint/practracker/testdata/b.c new file mode 100644 index 0000000000..bef277aaae --- /dev/null +++ b/scripts/maint/practracker/testdata/b.c @@ -0,0 +1,15 @@ + +MOCK_IMPL(int, +foo,(void)) +{ + // blah1 + return 0; +} + +MOCK_IMPL(int, +bar,( long z)) +{ + // blah2 + + return (int)(z+2); +} diff --git a/scripts/maint/practracker/testdata/ex.txt b/scripts/maint/practracker/testdata/ex.txt new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/scripts/maint/practracker/testdata/ex.txt diff --git a/scripts/maint/practracker/testdata/ex0-expected.txt b/scripts/maint/practracker/testdata/ex0-expected.txt new file mode 100644 index 0000000000..5f3d9e5aec --- /dev/null +++ b/scripts/maint/practracker/testdata/ex0-expected.txt @@ -0,0 +1,11 @@ +problem file-size a.c 38 +problem include-count a.c 4 +problem function-size a.c:i_am_a_function() 9 +problem function-size a.c:another_function() 12 +problem dependency-violation a.c 3 +problem file-size b.c 15 +problem function-size b.c:foo() 4 +problem function-size b.c:bar() 5 +problem file-size header.h 8 +problem include-count header.h 4 +problem dependency-violation header.h 3 diff --git a/scripts/maint/practracker/testdata/ex0.txt b/scripts/maint/practracker/testdata/ex0.txt new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/scripts/maint/practracker/testdata/ex0.txt diff --git a/scripts/maint/practracker/testdata/ex1-expected.txt b/scripts/maint/practracker/testdata/ex1-expected.txt new file mode 100644 index 0000000000..58140a4d9a --- /dev/null +++ b/scripts/maint/practracker/testdata/ex1-expected.txt @@ -0,0 +1,3 @@ +problem function-size a.c:i_am_a_function() 9 +(warning) problem function-size a.c:another_function() 12 +problem function-size b.c:foo() 4 diff --git a/scripts/maint/practracker/testdata/ex1-overbroad-expected.txt b/scripts/maint/practracker/testdata/ex1-overbroad-expected.txt new file mode 100644 index 0000000000..f69c608f40 --- /dev/null +++ b/scripts/maint/practracker/testdata/ex1-overbroad-expected.txt @@ -0,0 +1,2 @@ +problem file-size a.c 40 -> 38 +problem file-size z.c 100 -> 0 diff --git a/scripts/maint/practracker/testdata/ex1.txt b/scripts/maint/practracker/testdata/ex1.txt new file mode 100644 index 0000000000..c698005d07 --- /dev/null +++ b/scripts/maint/practracker/testdata/ex1.txt @@ -0,0 +1,18 @@ + +problem file-size a.c 40 +problem include-count a.c 4 +# this problem will produce an error +problem function-size a.c:i_am_a_function() 8 +# this problem will produce a warning +problem function-size a.c:another_function() 11 +problem file-size b.c 15 +# This is removed, and so will produce an error. +# problem function-size b.c:foo() 4 +# This exception isn't used. +problem file-size z.c 100 + +problem function-size b.c:bar() 5 +problem dependency-violation a.c 3 +problem dependency-violation header.h 3 +problem file-size header.h 8 +problem include-count header.h 4 diff --git a/scripts/maint/practracker/testdata/header.h b/scripts/maint/practracker/testdata/header.h new file mode 100644 index 0000000000..1183f5db9a --- /dev/null +++ b/scripts/maint/practracker/testdata/header.h @@ -0,0 +1,8 @@ + +// some forbidden includes +#include "foo.h" +#include "quux.h" +#include "quup.h" + +// a permitted include +#include "permitted.h" diff --git a/scripts/maint/practracker/testdata/not_c_file b/scripts/maint/practracker/testdata/not_c_file new file mode 100644 index 0000000000..e150962c02 --- /dev/null +++ b/scripts/maint/practracker/testdata/not_c_file @@ -0,0 +1,2 @@ + +This isn't a C file, so practracker shouldn't care about it. diff --git a/scripts/maint/practracker/util.py b/scripts/maint/practracker/util.py index b0ca73b997..df629110c2 100644 --- a/scripts/maint/practracker/util.py +++ b/scripts/maint/practracker/util.py @@ -2,27 +2,49 @@ import os # We don't want to run metrics for unittests, automatically-generated C files, # external libraries or git leftovers. -EXCLUDE_SOURCE_DIRS = {"/src/test/", "/src/trunnel/", "/src/ext/", "/.git/"} +EXCLUDE_SOURCE_DIRS = {"src/test/", "src/trunnel/", "src/rust/", + "src/ext/" } -def get_tor_c_files(tor_topdir): +EXCLUDE_FILES = {"orconfig.h"} + +def _norm(p): + return os.path.normcase(os.path.normpath(p)) + +def get_tor_c_files(tor_topdir, include_dirs=None): """ - Return a list with the .c filenames we want to get metrics of. + Return a list with the .c and .h filenames we want to get metrics of. """ files_list = [] + exclude_dirs = { _norm(os.path.join(tor_topdir, p)) for p in EXCLUDE_SOURCE_DIRS } - for root, directories, filenames in os.walk(tor_topdir): - directories.sort() - filenames.sort() - for filename in filenames: - # We only care about .c files - if not filename.endswith(".c"): - continue + if include_dirs is None: + topdirs = [ tor_topdir ] + else: + topdirs = [ os.path.join(tor_topdir, inc) for inc in include_dirs ] - # Exclude the excluded paths - full_path = os.path.join(root,filename) - if any(os.path.normcase(exclude_dir) in full_path for exclude_dir in EXCLUDE_SOURCE_DIRS): - continue + for topdir in topdirs: + for root, directories, filenames in os.walk(topdir): + # Remove all the directories that are excluded. + directories[:] = [ d for d in directories + if _norm(os.path.join(root,d)) not in exclude_dirs ] + directories.sort() + filenames.sort() + for filename in filenames: + # We only care about .c and .h files + if not (filename.endswith(".c") or filename.endswith(".h")): + continue + if filename in EXCLUDE_FILES: + continue - files_list.append(full_path) + full_path = os.path.join(root,filename) + + files_list.append(full_path) return files_list + +class NullFile: + """A file-like object that we can us to suppress output.""" + def __init__(self): + pass + def write(self, s): + pass diff --git a/src/app/config/config.c b/src/app/config/config.c index 8ccbac159a..deda2448b6 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -61,7 +61,7 @@ #define CONFIG_PRIVATE #include "core/or/or.h" #include "app/config/config.h" -#include "app/config/confparse.h" +#include "lib/confmgt/confparse.h" #include "app/config/statefile.h" #include "app/main/main.h" #include "app/main/subsysmgr.h" @@ -111,6 +111,7 @@ #include "feature/stats/predict_ports.h" #include "feature/stats/rephist.h" #include "lib/compress/compress.h" +#include "lib/confmgt/structvar.h" #include "lib/crypt_ops/crypto_init.h" #include "lib/crypt_ops/crypto_rand.h" #include "lib/crypt_ops/crypto_util.h" @@ -190,7 +191,7 @@ static const char unix_q_socket_prefix[] = "unix:\""; /** A list of abbreviations and aliases to map command-line options, obsolete * option names, or alternative option names, to their current values. */ -static config_abbrev_t option_abbrevs_[] = { +static const config_abbrev_t option_abbrevs_[] = { PLURAL(AuthDirBadDirCC), PLURAL(AuthDirBadExitCC), PLURAL(AuthDirInvalidCC), @@ -253,22 +254,33 @@ static config_abbrev_t option_abbrevs_[] = { * members with CONF_CHECK_VAR_TYPE. */ DUMMY_TYPECHECK_INSTANCE(or_options_t); -/** An entry for config_vars: "The option <b>name</b> has type +/** An entry for config_vars: "The option <b>varname</b> has type * CONFIG_TYPE_<b>conftype</b>, and corresponds to * or_options_t.<b>member</b>" */ -#define VAR(name,conftype,member,initvalue) \ - { name, CONFIG_TYPE_ ## conftype, offsetof(or_options_t, member), \ - initvalue CONF_TEST_MEMBERS(or_options_t, conftype, member) } -/** As VAR, but the option name and member name are the same. */ -#define V(member,conftype,initvalue) \ +#define VAR(varname,conftype,member,initvalue) \ + CONFIG_VAR_ETYPE(or_options_t, varname, conftype, member, 0, initvalue) + +/* As VAR, but uses a type definition in addition to a type enum. */ +#define VAR_D(varname,conftype,member,initvalue) \ + CONFIG_VAR_DEFN(or_options_t, varname, conftype, member, 0, initvalue) + +#define VAR_NODUMP(varname,conftype,member,initvalue) \ + CONFIG_VAR_ETYPE(or_options_t, varname, conftype, member, \ + CFLG_NODUMP, initvalue) +#define VAR_INVIS(varname,conftype,member,initvalue) \ + CONFIG_VAR_ETYPE(or_options_t, varname, conftype, member, \ + CFLG_NODUMP | CFLG_NOSET | CFLG_NOLIST, initvalue) + +#define V(member,conftype,initvalue) \ VAR(#member, conftype, member, initvalue) -/** An entry for config_vars: "The option <b>name</b> is obsolete." */ -#ifdef TOR_UNIT_TESTS -#define OBSOLETE(name) { name, CONFIG_TYPE_OBSOLETE, 0, NULL, {.INT=NULL} } -#else -#define OBSOLETE(name) { name, CONFIG_TYPE_OBSOLETE, 0, NULL } -#endif + +/** As V, but uses a type definition instead of a type enum */ +#define V_D(member,type,initvalue) \ + VAR_D(#member, type, member, initvalue) + +/** An entry for config_vars: "The option <b>varname</b> is obsolete." */ +#define OBSOLETE(varname) CONFIG_VAR_OBSOLETE(varname) /** * Macro to declare *Port options. Each one comes in three entries. @@ -280,7 +292,7 @@ DUMMY_TYPECHECK_INSTANCE(or_options_t); #define VPORT(member) \ VAR(#member "Lines", LINELIST_V, member ## _lines, NULL), \ VAR(#member, LINELIST_S, member ## _lines, NULL), \ - VAR("__" #member, LINELIST_S, member ## _lines, NULL) + VAR_NODUMP("__" #member, LINELIST_S, member ## _lines, NULL) /** UINT64_MAX as a decimal string */ #define UINT64_MAX_STRING "18446744073709551615" @@ -289,7 +301,7 @@ DUMMY_TYPECHECK_INSTANCE(or_options_t); * abbreviations, order is significant, since the first matching option will * be chosen first. */ -static config_var_t option_vars_[] = { +static const config_var_t option_vars_[] = { V(AccountingMax, MEMUNIT, "0 bytes"), VAR("AccountingRule", STRING, AccountingRule_option, "max"), V(AccountingStart, STRING, NULL), @@ -317,7 +329,7 @@ static config_var_t option_vars_[] = { OBSOLETE("AuthDirRejectUnlisted"), OBSOLETE("AuthDirListBadDirs"), V(AuthDirListBadExits, BOOL, "0"), - V(AuthDirMaxServersPerAddr, UINT, "2"), + V(AuthDirMaxServersPerAddr, POSINT, "2"), OBSOLETE("AuthDirMaxServersPerAuthAddr"), V(AuthDirHasIPv6Connectivity, BOOL, "0"), VAR("AuthoritativeDirectory", BOOL, AuthoritativeDir, "0"), @@ -352,7 +364,7 @@ static config_var_t option_vars_[] = { V(ClientUseIPv6, BOOL, "0"), V(ClientUseIPv4, BOOL, "1"), V(ConsensusParams, STRING, NULL), - V(ConnLimit, UINT, "1000"), + V(ConnLimit, POSINT, "1000"), V(ConnDirectionStatistics, BOOL, "0"), V(ConstrainedSockets, BOOL, "0"), V(ConstrainedSockSize, MEMUNIT, "8192"), @@ -402,14 +414,14 @@ static config_var_t option_vars_[] = { V(DormantCanceledByStartup, BOOL, "0"), /* DoS circuit creation options. */ V(DoSCircuitCreationEnabled, AUTOBOOL, "auto"), - V(DoSCircuitCreationMinConnections, UINT, "0"), - V(DoSCircuitCreationRate, UINT, "0"), - V(DoSCircuitCreationBurst, UINT, "0"), + V(DoSCircuitCreationMinConnections, POSINT, "0"), + V(DoSCircuitCreationRate, POSINT, "0"), + V(DoSCircuitCreationBurst, POSINT, "0"), V(DoSCircuitCreationDefenseType, INT, "0"), V(DoSCircuitCreationDefenseTimePeriod, INTERVAL, "0"), /* DoS connection options. */ V(DoSConnectionEnabled, AUTOBOOL, "auto"), - V(DoSConnectionMaxConcurrentCount, UINT, "0"), + V(DoSConnectionMaxConcurrentCount, POSINT, "0"), V(DoSConnectionDefenseType, INT, "0"), /* DoS single hop client options. */ V(DoSRefuseSingleHopClientRendezvous, AUTOBOOL, "auto"), @@ -418,17 +430,17 @@ static config_var_t option_vars_[] = { V(TestingEnableCellStatsEvent, BOOL, "0"), OBSOLETE("TestingEnableTbEmptyEvent"), V(EnforceDistinctSubnets, BOOL, "1"), - V(EntryNodes, ROUTERSET, NULL), + V_D(EntryNodes, ROUTERSET, NULL), V(EntryStatistics, BOOL, "0"), V(TestingEstimatedDescriptorPropagationTime, INTERVAL, "10 minutes"), - V(ExcludeNodes, ROUTERSET, NULL), - V(ExcludeExitNodes, ROUTERSET, NULL), + V_D(ExcludeNodes, ROUTERSET, NULL), + V_D(ExcludeExitNodes, ROUTERSET, NULL), OBSOLETE("ExcludeSingleHopRelays"), - V(ExitNodes, ROUTERSET, NULL), + V_D(ExitNodes, ROUTERSET, NULL), /* Researchers need a way to tell their clients to use specific * middles that they also control, to allow safe live-network * experimentation with new padding machines. */ - V(MiddleNodes, ROUTERSET, NULL), + V_D(MiddleNodes, ROUTERSET, NULL), V(ExitPolicy, LINELIST, NULL), V(ExitPolicyRejectPrivate, BOOL, "1"), V(ExitPolicyRejectLocalInterfaces, BOOL, "0"), @@ -484,6 +496,11 @@ static config_var_t option_vars_[] = { VAR("HiddenServiceMaxStreamsCloseCircuit",LINELIST_S, RendConfigLines, NULL), VAR("HiddenServiceNumIntroductionPoints", LINELIST_S, RendConfigLines, NULL), VAR("HiddenServiceExportCircuitID", LINELIST_S, RendConfigLines, NULL), + VAR("HiddenServiceEnableIntroDoSDefense", LINELIST_S, RendConfigLines, NULL), + VAR("HiddenServiceEnableIntroDoSRatePerSec", + LINELIST_S, RendConfigLines, NULL), + VAR("HiddenServiceEnableIntroDoSBurstPerSec", + LINELIST_S, RendConfigLines, NULL), VAR("HiddenServiceStatistics", BOOL, HiddenServiceStatistics_option, "1"), V(HidServAuth, LINELIST, NULL), V(ClientOnionAuthDir, FILENAME, NULL), @@ -507,8 +524,8 @@ static config_var_t option_vars_[] = { V(Socks5ProxyPassword, STRING, NULL), VAR("KeyDirectory", FILENAME, KeyDirectory_option, NULL), V(KeyDirectoryGroupReadable, BOOL, "0"), - VAR("HSLayer2Nodes", ROUTERSET, HSLayer2Nodes, NULL), - VAR("HSLayer3Nodes", ROUTERSET, HSLayer3Nodes, NULL), + VAR_D("HSLayer2Nodes", ROUTERSET, HSLayer2Nodes, NULL), + VAR_D("HSLayer3Nodes", ROUTERSET, HSLayer3Nodes, NULL), V(KeepalivePeriod, INTERVAL, "5 minutes"), V(KeepBindCapabilities, AUTOBOOL, "auto"), VAR("Log", LINELIST, Logs, NULL), @@ -522,7 +539,7 @@ static config_var_t option_vars_[] = { VAR("MapAddress", LINELIST, AddressMap, NULL), V(MaxAdvertisedBandwidth, MEMUNIT, "1 GB"), V(MaxCircuitDirtiness, INTERVAL, "10 minutes"), - V(MaxClientCircuitsPending, UINT, "32"), + V(MaxClientCircuitsPending, POSINT, "32"), V(MaxConsensusAgeForDiffs, INTERVAL, "0 seconds"), VAR("MaxMemInQueues", MEMUNIT, MaxMemInQueues_raw, "0"), OBSOLETE("MaxOnionsPending"), @@ -539,10 +556,10 @@ static config_var_t option_vars_[] = { OBSOLETE("WarnUnsafeSocks"), VAR("NodeFamily", LINELIST, NodeFamilies, NULL), V(NoExec, BOOL, "0"), - V(NumCPUs, UINT, "0"), - V(NumDirectoryGuards, UINT, "0"), - V(NumEntryGuards, UINT, "0"), - V(NumPrimaryGuards, UINT, "0"), + V(NumCPUs, POSINT, "0"), + V(NumDirectoryGuards, POSINT, "0"), + V(NumEntryGuards, POSINT, "0"), + V(NumPrimaryGuards, POSINT, "0"), V(OfflineMasterKey, BOOL, "0"), OBSOLETE("ORListenAddress"), VPORT(ORPort), @@ -593,7 +610,7 @@ static config_var_t option_vars_[] = { V(RecommendedVersions, LINELIST, NULL), V(RecommendedClientVersions, LINELIST, NULL), V(RecommendedServerVersions, LINELIST, NULL), - V(RecommendedPackages, LINELIST, NULL), + OBSOLETE("RecommendedPackages"), V(ReducedConnectionPadding, BOOL, "0"), V(ConnectionPadding, AUTOBOOL, "auto"), V(RefuseUnknownExits, AUTOBOOL, "auto"), @@ -666,7 +683,7 @@ static config_var_t option_vars_[] = { V(V3AuthVotingInterval, INTERVAL, "1 hour"), V(V3AuthVoteDelay, INTERVAL, "5 minutes"), V(V3AuthDistDelay, INTERVAL, "5 minutes"), - V(V3AuthNIntervalsValid, UINT, "3"), + V(V3AuthNIntervalsValid, POSINT, "3"), V(V3AuthUseLegacyKey, BOOL, "0"), V(V3BandwidthsFile, FILENAME, NULL), V(GuardfractionFile, FILENAME, NULL), @@ -677,15 +694,17 @@ static config_var_t option_vars_[] = { V(WarnPlaintextPorts, CSV, "23,109,110,143"), OBSOLETE("UseFilteringSSLBufferevents"), OBSOLETE("__UseFilteringSSLBufferevents"), - VAR("__ReloadTorrcOnSIGHUP", BOOL, ReloadTorrcOnSIGHUP, "1"), - VAR("__AllDirActionsPrivate", BOOL, AllDirActionsPrivate, "0"), - VAR("__DisablePredictedCircuits",BOOL,DisablePredictedCircuits, "0"), - VAR("__DisableSignalHandlers", BOOL, DisableSignalHandlers, "0"), - VAR("__LeaveStreamsUnattached",BOOL, LeaveStreamsUnattached, "0"), - VAR("__HashedControlSessionPassword", LINELIST, HashedControlSessionPassword, + VAR_NODUMP("__ReloadTorrcOnSIGHUP", BOOL, ReloadTorrcOnSIGHUP, "1"), + VAR_NODUMP("__AllDirActionsPrivate", BOOL, AllDirActionsPrivate, "0"), + VAR_NODUMP("__DisablePredictedCircuits",BOOL,DisablePredictedCircuits, "0"), + VAR_NODUMP("__DisableSignalHandlers", BOOL, DisableSignalHandlers, "0"), + VAR_NODUMP("__LeaveStreamsUnattached",BOOL, LeaveStreamsUnattached, "0"), + VAR_NODUMP("__HashedControlSessionPassword", LINELIST, + HashedControlSessionPassword, NULL), - VAR("__OwningControllerProcess",STRING,OwningControllerProcess, NULL), - VAR("__OwningControllerFD", UINT64, OwningControllerFD, UINT64_MAX_STRING), + VAR_NODUMP("__OwningControllerProcess",STRING,OwningControllerProcess, NULL), + VAR_NODUMP("__OwningControllerFD", UINT64, OwningControllerFD, + UINT64_MAX_STRING), V(MinUptimeHidServDirectoryV2, INTERVAL, "96 hours"), V(TestingServerDownloadInitialDelay, CSV_INTERVAL, "0"), V(TestingClientDownloadInitialDelay, CSV_INTERVAL, "0"), @@ -715,7 +734,7 @@ static config_var_t option_vars_[] = { * blocked), but we also don't want to fail if only some mirrors are * blackholed. Clients will try 3 directories simultaneously. * (Relays never use simultaneous connections.) */ - V(ClientBootstrapConsensusMaxInProgressTries, UINT, "3"), + V(ClientBootstrapConsensusMaxInProgressTries, POSINT, "3"), /* When a client has any running bridges, check each bridge occasionally, * whether or not that bridge is actually up. */ V(TestingBridgeDownloadInitialDelay, CSV_INTERVAL,"10800"), @@ -732,56 +751,40 @@ static config_var_t option_vars_[] = { OBSOLETE("TestingDescriptorMaxDownloadTries"), OBSOLETE("TestingMicrodescMaxDownloadTries"), OBSOLETE("TestingCertMaxDownloadTries"), - V(TestingDirAuthVoteExit, ROUTERSET, NULL), + V_D(TestingDirAuthVoteExit, ROUTERSET, NULL), V(TestingDirAuthVoteExitIsStrict, BOOL, "0"), - V(TestingDirAuthVoteGuard, ROUTERSET, NULL), + V_D(TestingDirAuthVoteGuard, ROUTERSET, NULL), V(TestingDirAuthVoteGuardIsStrict, BOOL, "0"), - V(TestingDirAuthVoteHSDir, ROUTERSET, NULL), + V_D(TestingDirAuthVoteHSDir, ROUTERSET, NULL), V(TestingDirAuthVoteHSDirIsStrict, BOOL, "0"), - VAR("___UsingTestNetworkDefaults", BOOL, UsingTestNetworkDefaults_, "0"), + VAR_INVIS("___UsingTestNetworkDefaults", BOOL, UsingTestNetworkDefaults_, + "0"), END_OF_CONFIG_VARS }; +/** List of default directory authorities */ +static const char *default_authorities[] = { +#include "auth_dirs.inc" + NULL +}; + +/** List of fallback directory authorities. The list is generated by opt-in of + * relays that meet certain stability criteria. + */ +static const char *default_fallbacks[] = { +#include "fallback_dirs.inc" + NULL +}; + /** Override default values with these if the user sets the TestingTorNetwork * option. */ -static const config_var_t testing_tor_network_defaults[] = { - V(DirAllowPrivateAddresses, BOOL, "1"), - V(EnforceDistinctSubnets, BOOL, "0"), - V(AssumeReachable, BOOL, "1"), - V(AuthDirMaxServersPerAddr, UINT, "0"), - V(ClientBootstrapConsensusAuthorityDownloadInitialDelay, CSV_INTERVAL, "0"), - V(ClientBootstrapConsensusFallbackDownloadInitialDelay, CSV_INTERVAL, "0"), - V(ClientBootstrapConsensusAuthorityOnlyDownloadInitialDelay, CSV_INTERVAL, - "0"), - V(ClientDNSRejectInternalAddresses, BOOL,"0"), - V(ClientRejectInternalAddresses, BOOL, "0"), - V(CountPrivateBandwidth, BOOL, "1"), - V(ExitPolicyRejectPrivate, BOOL, "0"), - V(ExtendAllowPrivateAddresses, BOOL, "1"), - V(V3AuthVotingInterval, INTERVAL, "5 minutes"), - V(V3AuthVoteDelay, INTERVAL, "20 seconds"), - V(V3AuthDistDelay, INTERVAL, "20 seconds"), - V(TestingV3AuthInitialVotingInterval, INTERVAL, "150 seconds"), - V(TestingV3AuthInitialVoteDelay, INTERVAL, "20 seconds"), - V(TestingV3AuthInitialDistDelay, INTERVAL, "20 seconds"), - V(TestingAuthDirTimeToLearnReachability, INTERVAL, "0 minutes"), - V(TestingEstimatedDescriptorPropagationTime, INTERVAL, "0 minutes"), - V(MinUptimeHidServDirectoryV2, INTERVAL, "0 minutes"), - V(TestingServerDownloadInitialDelay, CSV_INTERVAL, "0"), - V(TestingClientDownloadInitialDelay, CSV_INTERVAL, "0"), - V(TestingServerConsensusDownloadInitialDelay, CSV_INTERVAL, "0"), - V(TestingClientConsensusDownloadInitialDelay, CSV_INTERVAL, "0"), - V(TestingBridgeDownloadInitialDelay, CSV_INTERVAL, "10"), - V(TestingBridgeBootstrapDownloadInitialDelay, CSV_INTERVAL, "0"), - V(TestingClientMaxIntervalWithoutRequest, INTERVAL, "5 seconds"), - V(TestingDirConnectionMaxStall, INTERVAL, "30 seconds"), - V(TestingEnableConnBwEvent, BOOL, "1"), - V(TestingEnableCellStatsEvent, BOOL, "1"), - VAR("___UsingTestNetworkDefaults", BOOL, UsingTestNetworkDefaults_, "1"), - V(RendPostPeriod, INTERVAL, "2 minutes"), - - END_OF_CONFIG_VARS +static const struct { + const char *k; + const char *v; +} testing_tor_network_defaults[] = { +#include "testnet.inc" + { NULL, NULL } }; #undef VAR @@ -845,24 +848,28 @@ static void config_maybe_load_geoip_files_(const or_options_t *options, static int options_validate_cb(void *old_options, void *options, void *default_options, int from_setconf, char **msg); -static void options_free_cb(void *options); static void cleanup_protocol_warning_severity_level(void); static void set_protocol_warning_severity_level(int warning_severity); +static void options_clear_cb(const config_mgr_t *mgr, void *opts); /** Magic value for or_options_t. */ #define OR_OPTIONS_MAGIC 9090909 /** Configuration format for or_options_t. */ -STATIC config_format_t options_format = { +static const config_format_t options_format = { sizeof(or_options_t), - OR_OPTIONS_MAGIC, - offsetof(or_options_t, magic_), + { + "or_options_t", + OR_OPTIONS_MAGIC, + offsetof(or_options_t, magic_), + }, option_abbrevs_, option_deprecation_notes_, option_vars_, options_validate_cb, - options_free_cb, - NULL + options_clear_cb, + NULL, + offsetof(or_options_t, subconfigs_), }; /* @@ -894,6 +901,20 @@ static int in_option_validation = 0; /* True iff we've initialized libevent */ static int libevent_initialized = 0; +/* A global configuration manager to handle all configuration objects. */ +static config_mgr_t *options_mgr = NULL; + +/** Return the global configuration manager object for torrc options. */ +STATIC const config_mgr_t * +get_options_mgr(void) +{ + if (PREDICT_UNLIKELY(options_mgr == NULL)) { + options_mgr = config_mgr_new(&options_format); + config_mgr_freeze(options_mgr); + } + return options_mgr; +} + /** Return the contents of our frontpage string, or NULL if not configured. */ MOCK_IMPL(const char*, get_dirportfrontpage, (void)) @@ -917,6 +938,32 @@ get_options,(void)) return get_options_mutable(); } +/** + * True iff we have noticed that this is a testing tor network, and we + * should use the corresponding defaults. + **/ +static bool testing_network_configured = false; + +/** Return a set of lines for any default options that we want to override + * from those set in our config_var_t values. */ +static config_line_t * +get_options_defaults(void) +{ + int i; + config_line_t *result = NULL, **next = &result; + + if (testing_network_configured) { + for (i = 0; testing_tor_network_defaults[i].k; ++i) { + config_line_append(next, + testing_tor_network_defaults[i].k, + testing_tor_network_defaults[i].v); + next = &(*next)->next; + } + } + + return result; +} + /** Change the current global options to contain <b>new_val</b> instead of * their current value; take action based on the new value; free the old value * as necessary. Returns 0 on success, -1 on failure. @@ -924,9 +971,6 @@ get_options,(void)) int set_options(or_options_t *new_val, char **msg) { - int i; - smartlist_t *elements; - config_line_t *line; or_options_t *old_options = global_options; global_options = new_val; /* Note that we pass the *old* options below, for comparison. It @@ -948,35 +992,16 @@ set_options(or_options_t *new_val, char **msg) /* Issues a CONF_CHANGED event to notify controller of the change. If Tor is * just starting up then the old_options will be undefined. */ if (old_options && old_options != global_options) { - elements = smartlist_new(); - for (i=0; options_format.vars[i].name; ++i) { - const config_var_t *var = &options_format.vars[i]; - const char *var_name = var->name; - if (var->type == CONFIG_TYPE_LINELIST_S || - var->type == CONFIG_TYPE_OBSOLETE) { - continue; - } - if (!config_is_same(&options_format, new_val, old_options, var_name)) { - line = config_get_assigned_option(&options_format, new_val, - var_name, 1); - - if (line) { - config_line_t *next; - for (; line; line = next) { - next = line->next; - smartlist_add(elements, line->key); - smartlist_add(elements, line->value); - tor_free(line); - } - } else { - smartlist_add_strdup(elements, options_format.vars[i].name); - smartlist_add(elements, NULL); - } - } + smartlist_t *elements = smartlist_new(); + config_line_t *changes = + config_get_changes(get_options_mgr(), old_options, new_val); + for (config_line_t *line = changes; line; line = line->next) { + smartlist_add(elements, line->key); + smartlist_add(elements, line->value); } control_event_conf_changed(elements); - SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp)); smartlist_free(elements); + config_free_lines(changes); } if (old_options != global_options) { @@ -992,11 +1017,11 @@ set_options(or_options_t *new_val, char **msg) /** Release additional memory allocated in options */ -STATIC void -or_options_free_(or_options_t *options) +static void +options_clear_cb(const config_mgr_t *mgr, void *opts) { - if (!options) - return; + (void)mgr; + or_options_t *options = opts; routerset_free(options->ExcludeExitNodesUnion_); if (options->NodeFamilySets) { @@ -1019,7 +1044,14 @@ or_options_free_(or_options_t *options) tor_free(options->command_arg); tor_free(options->master_key_fname); config_free_lines(options->MyFamily); - config_free(&options_format, options); +} + +/** Release all memory allocated in options + */ +STATIC void +or_options_free_(or_options_t *options) +{ + config_free(get_options_mgr(), options); } /** Release all memory and resources held by global configuration structures. @@ -1053,6 +1085,8 @@ config_free_all(void) have_parsed_cmdline = 0; libevent_initialized = 0; + + config_mgr_free(options_mgr); } /** Make <b>address</b> -- a piece of information related to our operation as @@ -1170,21 +1204,6 @@ cleanup_protocol_warning_severity_level(void) atomic_counter_destroy(&protocol_warning_severity_level); } -/** List of default directory authorities */ - -static const char *default_authorities[] = { -#include "auth_dirs.inc" - NULL -}; - -/** List of fallback directory authorities. The list is generated by opt-in of - * relays that meet certain stability criteria. - */ -static const char *default_fallbacks[] = { -#include "fallback_dirs.inc" - NULL -}; - /** Add the default directory authorities directly into the trusted dir list, * but only add them insofar as they share bits with <b>type</b>. * Each authority's bits are restricted to the bits shared with <b>type</b>. @@ -1440,7 +1459,7 @@ options_act_reversible(const or_options_t *old_options, char **msg) "on this OS/with this build."); goto rollback; } -#else /* !(!defined(HAVE_SYS_UN_H)) */ +#else /* defined(HAVE_SYS_UN_H) */ if (options->ControlSocketsGroupWritable && !options->ControlSocket) { *msg = tor_strdup("Setting ControlSocketGroupWritable without setting" "a ControlSocket makes no sense."); @@ -2539,7 +2558,7 @@ config_parse_commandline(int argc, char **argv, int ignore_errors, param = tor_malloc_zero(sizeof(config_line_t)); param->key = is_cmdline ? tor_strdup(argv[i]) : - tor_strdup(config_expand_abbrev(&options_format, s, 1, 1)); + tor_strdup(config_expand_abbrev(get_options_mgr(), s, 1, 1)); param->value = arg; param->command = command; param->next = NULL; @@ -2565,8 +2584,7 @@ config_parse_commandline(int argc, char **argv, int ignore_errors, int option_is_recognized(const char *key) { - const config_var_t *var = config_find_option(&options_format, key); - return (var != NULL); + return config_find_option_name(get_options_mgr(), key) != NULL; } /** Return the canonical name of a configuration option, or NULL @@ -2574,8 +2592,7 @@ option_is_recognized(const char *key) const char * option_get_canonical_name(const char *key) { - const config_var_t *var = config_find_option(&options_format, key); - return var ? var->name : NULL; + return config_find_option_name(get_options_mgr(), key); } /** Return a canonical list of the options assigned for key. @@ -2583,7 +2600,7 @@ option_get_canonical_name(const char *key) config_line_t * option_get_assignment(const or_options_t *options, const char *key) { - return config_get_assigned_option(&options_format, options, key, 1); + return config_get_assigned_option(get_options_mgr(), options, key, 1); } /** Try assigning <b>list</b> to the global options. You do this by duping @@ -2599,9 +2616,9 @@ setopt_err_t options_trial_assign(config_line_t *list, unsigned flags, char **msg) { int r; - or_options_t *trial_options = config_dup(&options_format, get_options()); + or_options_t *trial_options = config_dup(get_options_mgr(), get_options()); - if ((r=config_assign(&options_format, trial_options, + if ((r=config_assign(get_options_mgr(), trial_options, list, flags, msg)) < 0) { or_options_free(trial_options); return r; @@ -2656,24 +2673,30 @@ print_usage(void) static void list_torrc_options(void) { - int i; - for (i = 0; option_vars_[i].name; ++i) { - const config_var_t *var = &option_vars_[i]; - if (var->type == CONFIG_TYPE_OBSOLETE || - var->type == CONFIG_TYPE_LINELIST_V) + smartlist_t *vars = config_mgr_list_vars(get_options_mgr()); + SMARTLIST_FOREACH_BEGIN(vars, const config_var_t *, var) { + /* Possibly this should check listable, rather than (or in addition to) + * settable. See ticket 31654. + */ + if (! config_var_is_settable(var)) { + /* This variable cannot be set, or cannot be set by this name. */ continue; - printf("%s\n", var->name); - } + } + printf("%s\n", var->member.name); + } SMARTLIST_FOREACH_END(var); + smartlist_free(vars); } /** Print all deprecated but non-obsolete torrc options. */ static void list_deprecated_options(void) { - const config_deprecation_t *d; - for (d = option_deprecation_notes_; d->name; ++d) { - printf("%s\n", d->name); - } + smartlist_t *deps = config_mgr_list_deprecated_vars(get_options_mgr()); + /* Possibly this should check whether the variables are listable, + * but currently it does not. See ticket 31654. */ + SMARTLIST_FOREACH(deps, const char *, name, + printf("%s\n", name)); + smartlist_free(deps); } /** Print all compile-time modules and their enabled/disabled status. */ @@ -2983,7 +3006,7 @@ is_local_addr, (const tor_addr_t *addr)) or_options_t * options_new(void) { - return config_new(&options_format); + return config_new(get_options_mgr()); } /** Set <b>options</b> to hold reasonable defaults for most options. @@ -2991,7 +3014,17 @@ options_new(void) void options_init(or_options_t *options) { - config_init(&options_format, options); + config_init(get_options_mgr(), options); + config_line_t *dflts = get_options_defaults(); + char *msg=NULL; + if (config_assign(get_options_mgr(), options, dflts, + CAL_WARN_DEPRECATIONS, &msg)<0) { + log_err(LD_BUG, "Unable to set default options: %s", msg); + tor_free(msg); + tor_assert_unreached(); + } + config_free_lines(dflts); + tor_free(msg); } /** Return a string containing a possible configuration file that would give @@ -3021,7 +3054,7 @@ options_dump(const or_options_t *options, int how_to_dump) return NULL; } - return config_dump(&options_format, use_defaults, options, minimal, 0); + return config_dump(get_options_mgr(), use_defaults, options, minimal, 0); } /** Return 0 if every element of sl is a string holding a decimal @@ -3153,13 +3186,6 @@ options_validate_cb(void *old_options, void *options, void *default_options, return rv; } -/** Callback to free an or_options_t */ -static void -options_free_cb(void *options) -{ - or_options_free_(options); -} - #define REJECT(arg) \ STMT_BEGIN *msg = tor_strdup(arg); return -1; STMT_END #if defined(__GNUC__) && __GNUC__ <= 3 @@ -3492,7 +3518,7 @@ options_validate(or_options_t *old_options, or_options_t *options, REJECT("Cannot use TransProxyType without any valid TransPort."); } } -#else /* !(defined(USE_TRANSPARENT)) */ +#else /* !defined(USE_TRANSPARENT) */ if (options->TransPort_set) REJECT("TransPort is disabled in this build."); #endif /* defined(USE_TRANSPARENT) */ @@ -3526,13 +3552,6 @@ options_validate(or_options_t *old_options, or_options_t *options, "features to be broken in unpredictable ways."); } - for (cl = options->RecommendedPackages; cl; cl = cl->next) { - if (! validate_recommended_package_line(cl->value)) { - log_warn(LD_CONFIG, "Invalid RecommendedPackage line %s will be ignored", - escaped(cl->value)); - } - } - if (options->AuthoritativeDir) { if (!options->ContactInfo && !options->TestingTorNetwork) REJECT("Authoritative directory servers must set ContactInfo"); @@ -4423,7 +4442,7 @@ options_validate(or_options_t *old_options, or_options_t *options, STMT_BEGIN \ if (!options->TestingTorNetwork && \ !options->UsingTestNetworkDefaults_ && \ - !config_is_same(&options_format,options, \ + !config_is_same(get_options_mgr(),options, \ default_options,#arg)) { \ REJECT(#arg " may only be changed in testing Tor " \ "networks!"); \ @@ -5086,7 +5105,7 @@ find_torrc_filename(config_line_t *cmd_arg, } else { fname = dflt ? tor_strdup(dflt) : NULL; } -#else /* !(!defined(_WIN32)) */ +#else /* defined(_WIN32) */ fname = dflt ? tor_strdup(dflt) : NULL; #endif /* !defined(_WIN32) */ } @@ -5388,6 +5407,7 @@ options_init_from_string(const char *cf_defaults, const char *cf, int command, const char *command_arg, char **msg) { + bool retry = false; or_options_t *oldoptions, *newoptions, *newdefaultoptions=NULL; config_line_t *cl; int retval; @@ -5398,8 +5418,7 @@ options_init_from_string(const char *cf_defaults, const char *cf, oldoptions = global_options; /* get_options unfortunately asserts if this is the first time we run*/ - newoptions = tor_malloc_zero(sizeof(or_options_t)); - newoptions->magic_ = OR_OPTIONS_MAGIC; + newoptions = options_new(); options_init(newoptions); newoptions->command = command; newoptions->command_arg = command_arg ? tor_strdup(command_arg) : NULL; @@ -5418,7 +5437,7 @@ options_init_from_string(const char *cf_defaults, const char *cf, err = SETOPT_ERR_PARSE; goto err; } - retval = config_assign(&options_format, newoptions, cl, + retval = config_assign(get_options_mgr(), newoptions, cl, CAL_WARN_DEPRECATIONS, msg); config_free_lines(cl); if (retval < 0) { @@ -5426,15 +5445,15 @@ options_init_from_string(const char *cf_defaults, const char *cf, goto err; } if (i==0) - newdefaultoptions = config_dup(&options_format, newoptions); + newdefaultoptions = config_dup(get_options_mgr(), newoptions); } if (newdefaultoptions == NULL) { - newdefaultoptions = config_dup(&options_format, global_default_options); + newdefaultoptions = config_dup(get_options_mgr(), global_default_options); } /* Go through command-line variables too */ - retval = config_assign(&options_format, newoptions, + retval = config_assign(get_options_mgr(), newoptions, global_cmdline_options, CAL_WARN_DEPRECATIONS, msg); if (retval < 0) { err = SETOPT_ERR_PARSE; @@ -5445,73 +5464,12 @@ options_init_from_string(const char *cf_defaults, const char *cf, newoptions->FilesOpenedByIncludes = opened_files; /* If this is a testing network configuration, change defaults - * for a list of dependent config options, re-initialize newoptions - * with the new defaults, and assign all options to it second time. */ - if (newoptions->TestingTorNetwork) { - /* XXXX this is a bit of a kludge. perhaps there's a better way to do - * this? We could, for example, make the parsing algorithm do two passes - * over the configuration. If it finds any "suite" options like - * TestingTorNetwork, it could change the defaults before its second pass. - * Not urgent so long as this seems to work, but at any sign of trouble, - * let's clean it up. -NM */ - - /* Change defaults. */ - for (int i = 0; testing_tor_network_defaults[i].name; ++i) { - const config_var_t *new_var = &testing_tor_network_defaults[i]; - config_var_t *old_var = - config_find_option_mutable(&options_format, new_var->name); - tor_assert(new_var); - tor_assert(old_var); - old_var->initvalue = new_var->initvalue; - - if ((config_find_deprecation(&options_format, new_var->name))) { - log_warn(LD_GENERAL, "Testing options override the deprecated " - "option %s. Is that intentional?", - new_var->name); - } - } - - /* Clear newoptions and re-initialize them with new defaults. */ - or_options_free(newoptions); - or_options_free(newdefaultoptions); - newdefaultoptions = NULL; - newoptions = tor_malloc_zero(sizeof(or_options_t)); - newoptions->magic_ = OR_OPTIONS_MAGIC; - options_init(newoptions); - newoptions->command = command; - newoptions->command_arg = command_arg ? tor_strdup(command_arg) : NULL; - - /* Assign all options a second time. */ - opened_files = smartlist_new(); - for (int i = 0; i < 2; ++i) { - const char *body = i==0 ? cf_defaults : cf; - if (!body) - continue; - - /* get config lines, assign them */ - retval = config_get_lines_include(body, &cl, 1, - body == cf ? &cf_has_include : NULL, - opened_files); - if (retval < 0) { - err = SETOPT_ERR_PARSE; - goto err; - } - retval = config_assign(&options_format, newoptions, cl, 0, msg); - config_free_lines(cl); - if (retval < 0) { - err = SETOPT_ERR_PARSE; - goto err; - } - if (i==0) - newdefaultoptions = config_dup(&options_format, newoptions); - } - /* Assign command-line variables a second time too */ - retval = config_assign(&options_format, newoptions, - global_cmdline_options, 0, msg); - if (retval < 0) { - err = SETOPT_ERR_PARSE; - goto err; - } + * for a list of dependent config options, and try this function again. */ + if (newoptions->TestingTorNetwork && ! testing_network_configured) { + // retry with the testing defaults. + testing_network_configured = true; + retry = true; + goto err; } newoptions->IncludeUsed = cf_has_include; @@ -5556,6 +5514,9 @@ options_init_from_string(const char *cf_defaults, const char *cf, tor_asprintf(msg, "Failed to parse/validate config: %s", old_msg); tor_free(old_msg); } + if (retry) + return options_init_from_string(cf_defaults, cf, command, command_arg, + msg); return err; } @@ -7834,7 +7795,7 @@ get_data_directory(const char *val) } else { return tor_strdup(get_windows_conf_root()); } -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ const char *d = val; if (!d) d = "~/.tor"; @@ -8178,72 +8139,48 @@ getinfo_helper_config(control_connection_t *conn, (void) errmsg; if (!strcmp(question, "config/names")) { smartlist_t *sl = smartlist_new(); - int i; - for (i = 0; option_vars_[i].name; ++i) { - const config_var_t *var = &option_vars_[i]; - const char *type; - /* don't tell controller about triple-underscore options */ - if (!strncmp(option_vars_[i].name, "___", 3)) + smartlist_t *vars = config_mgr_list_vars(get_options_mgr()); + SMARTLIST_FOREACH_BEGIN(vars, const config_var_t *, var) { + /* don't tell controller about invisible options */ + if (! config_var_is_listable(var)) continue; - switch (var->type) { - case CONFIG_TYPE_STRING: type = "String"; break; - case CONFIG_TYPE_FILENAME: type = "Filename"; break; - case CONFIG_TYPE_UINT: type = "Integer"; break; - case CONFIG_TYPE_UINT64: type = "Integer"; break; - case CONFIG_TYPE_INT: type = "SignedInteger"; break; - case CONFIG_TYPE_PORT: type = "Port"; break; - case CONFIG_TYPE_INTERVAL: type = "TimeInterval"; break; - case CONFIG_TYPE_MSEC_INTERVAL: type = "TimeMsecInterval"; break; - case CONFIG_TYPE_MEMUNIT: type = "DataSize"; break; - case CONFIG_TYPE_DOUBLE: type = "Float"; break; - case CONFIG_TYPE_BOOL: type = "Boolean"; break; - case CONFIG_TYPE_AUTOBOOL: type = "Boolean+Auto"; break; - case CONFIG_TYPE_ISOTIME: type = "Time"; break; - case CONFIG_TYPE_ROUTERSET: type = "RouterList"; break; - case CONFIG_TYPE_CSV: type = "CommaList"; break; - /* This type accepts more inputs than TimeInterval, but it ignores - * everything after the first entry, so we may as well pretend - * it's a TimeInterval. */ - case CONFIG_TYPE_CSV_INTERVAL: type = "TimeInterval"; break; - case CONFIG_TYPE_LINELIST: type = "LineList"; break; - case CONFIG_TYPE_LINELIST_S: type = "Dependent"; break; - case CONFIG_TYPE_LINELIST_V: type = "Virtual"; break; - default: - case CONFIG_TYPE_OBSOLETE: - type = NULL; break; - } + const char *type = struct_var_get_typename(&var->member); if (!type) continue; - smartlist_add_asprintf(sl, "%s %s\n",var->name,type); - } + smartlist_add_asprintf(sl, "%s %s\n",var->member.name,type); + } SMARTLIST_FOREACH_END(var); *answer = smartlist_join_strings(sl, "", 0, NULL); SMARTLIST_FOREACH(sl, char *, c, tor_free(c)); smartlist_free(sl); + smartlist_free(vars); } else if (!strcmp(question, "config/defaults")) { smartlist_t *sl = smartlist_new(); int dirauth_lines_seen = 0, fallback_lines_seen = 0; - for (int i = 0; option_vars_[i].name; ++i) { - const config_var_t *var = &option_vars_[i]; + /* Possibly this should check whether the variables are listable, + * but currently it does not. See ticket 31654. */ + smartlist_t *vars = config_mgr_list_vars(get_options_mgr()); + SMARTLIST_FOREACH_BEGIN(vars, const config_var_t *, var) { if (var->initvalue != NULL) { - if (strcmp(option_vars_[i].name, "DirAuthority") == 0) { + if (strcmp(var->member.name, "DirAuthority") == 0) { /* * Count dirauth lines we have a default for; we'll use the * count later to decide whether to add the defaults manually */ ++dirauth_lines_seen; } - if (strcmp(option_vars_[i].name, "FallbackDir") == 0) { + if (strcmp(var->member.name, "FallbackDir") == 0) { /* - * Similarly count fallback lines, so that we can decided later + * Similarly count fallback lines, so that we can decide later * to add the defaults manually. */ ++fallback_lines_seen; } char *val = esc_for_log(var->initvalue); - smartlist_add_asprintf(sl, "%s %s\n",var->name,val); + smartlist_add_asprintf(sl, "%s %s\n",var->member.name,val); tor_free(val); } - } + } SMARTLIST_FOREACH_END(var); + smartlist_free(vars); if (dirauth_lines_seen == 0) { /* @@ -8404,7 +8341,7 @@ config_load_geoip_file_(sa_family_t family, } r = geoip_load_file(family, fname, severity); tor_free(free_fname); -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ (void)default_fname; r = geoip_load_file(family, fname, severity); #endif /* defined(_WIN32) */ @@ -8492,7 +8429,7 @@ init_cookie_authentication(const char *fname, const char *header, log_warn(LD_FS,"Unable to make %s group-readable.", escaped(fname)); } } -#else /* !(!defined(_WIN32)) */ +#else /* defined(_WIN32) */ (void) group_readable; #endif /* !defined(_WIN32) */ diff --git a/src/app/config/config.h b/src/app/config/config.h index 46db02f944..44f09e5ee9 100644 --- a/src/app/config/config.h +++ b/src/app/config/config.h @@ -247,9 +247,8 @@ int options_any_client_port_set(const or_options_t *options); #define CL_PORT_DFLT_GROUP_WRITABLE (1u<<7) STATIC int options_act(const or_options_t *old_options); -#ifdef TOR_UNIT_TESTS -extern struct config_format_t options_format; -#endif +struct config_mgr_t; +STATIC const struct config_mgr_t *get_options_mgr(void); STATIC port_cfg_t *port_cfg_new(size_t namelen); #define port_cfg_free(port) \ diff --git a/src/app/config/confparse.c b/src/app/config/confparse.c deleted file mode 100644 index 729e7a4478..0000000000 --- a/src/app/config/confparse.c +++ /dev/null @@ -1,1207 +0,0 @@ -/* Copyright (c) 2001 Matej Pfajfar. - * Copyright (c) 2001-2004, Roger Dingledine. - * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ -/* See LICENSE for licensing information */ - -/** - * \file confparse.c - * - * \brief Back-end for parsing and generating key-value files, used to - * implement the torrc file format and the state file. - * - * This module is used by config.c to parse and encode torrc - * configuration files, and by statefile.c to parse and encode the - * $DATADIR/state file. - * - * To use this module, its callers provide an instance of - * config_format_t to describe the mappings from a set of configuration - * options to a number of fields in a C structure. With this mapping, - * the functions here can convert back and forth between the C structure - * specified, and a linked list of key-value pairs. - */ - -#include "core/or/or.h" -#include "app/config/confparse.h" -#include "feature/nodelist/routerset.h" - -#include "lib/container/bitarray.h" -#include "lib/encoding/confline.h" - -static uint64_t config_parse_memunit(const char *s, int *ok); -static int config_parse_msec_interval(const char *s, int *ok); -static int config_parse_interval(const char *s, int *ok); -static void config_reset(const config_format_t *fmt, void *options, - const config_var_t *var, int use_defaults); - -/** Allocate an empty configuration object of a given format type. */ -void * -config_new(const config_format_t *fmt) -{ - void *opts = tor_malloc_zero(fmt->size); - *(uint32_t*)STRUCT_VAR_P(opts, fmt->magic_offset) = fmt->magic; - CONFIG_CHECK(fmt, opts); - return opts; -} - -/* - * Functions to parse config options - */ - -/** If <b>option</b> is an official abbreviation for a longer option, - * return the longer option. Otherwise return <b>option</b>. - * If <b>command_line</b> is set, apply all abbreviations. Otherwise, only - * apply abbreviations that work for the config file and the command line. - * If <b>warn_obsolete</b> is set, warn about deprecated names. */ -const char * -config_expand_abbrev(const config_format_t *fmt, const char *option, - int command_line, int warn_obsolete) -{ - int i; - if (! fmt->abbrevs) - return option; - for (i=0; fmt->abbrevs[i].abbreviated; ++i) { - /* Abbreviations are case insensitive. */ - if (!strcasecmp(option,fmt->abbrevs[i].abbreviated) && - (command_line || !fmt->abbrevs[i].commandline_only)) { - if (warn_obsolete && fmt->abbrevs[i].warn) { - log_warn(LD_CONFIG, - "The configuration option '%s' is deprecated; " - "use '%s' instead.", - fmt->abbrevs[i].abbreviated, - fmt->abbrevs[i].full); - } - /* Keep going through the list in case we want to rewrite it more. - * (We could imagine recursing here, but I don't want to get the - * user into an infinite loop if we craft our list wrong.) */ - option = fmt->abbrevs[i].full; - } - } - return option; -} - -/** If <b>key</b> is a deprecated configuration option, return the message - * explaining why it is deprecated (which may be an empty string). Return NULL - * if it is not deprecated. The <b>key</b> field must be fully expanded. */ -const char * -config_find_deprecation(const config_format_t *fmt, const char *key) -{ - if (BUG(fmt == NULL) || BUG(key == NULL)) - return NULL; - if (fmt->deprecations == NULL) - return NULL; - - const config_deprecation_t *d; - for (d = fmt->deprecations; d->name; ++d) { - if (!strcasecmp(d->name, key)) { - return d->why_deprecated ? d->why_deprecated : ""; - } - } - return NULL; -} - -/** As config_find_option, but return a non-const pointer. */ -config_var_t * -config_find_option_mutable(config_format_t *fmt, const char *key) -{ - int i; - size_t keylen = strlen(key); - if (!keylen) - return NULL; /* if they say "--" on the command line, it's not an option */ - /* First, check for an exact (case-insensitive) match */ - for (i=0; fmt->vars[i].name; ++i) { - if (!strcasecmp(key, fmt->vars[i].name)) { - return &fmt->vars[i]; - } - } - /* If none, check for an abbreviated match */ - for (i=0; fmt->vars[i].name; ++i) { - if (!strncasecmp(key, fmt->vars[i].name, keylen)) { - log_warn(LD_CONFIG, "The abbreviation '%s' is deprecated. " - "Please use '%s' instead", - key, fmt->vars[i].name); - return &fmt->vars[i]; - } - } - /* Okay, unrecognized option */ - return NULL; -} - -/** If <b>key</b> is a configuration option, return the corresponding const - * config_var_t. Otherwise, if <b>key</b> is a non-standard abbreviation, - * warn, and return the corresponding const config_var_t. Otherwise return - * NULL. - */ -const config_var_t * -config_find_option(const config_format_t *fmt, const char *key) -{ - return config_find_option_mutable((config_format_t*)fmt, key); -} - -/** Return the number of option entries in <b>fmt</b>. */ -static int -config_count_options(const config_format_t *fmt) -{ - int i; - for (i=0; fmt->vars[i].name; ++i) - ; - return i; -} - -/* - * Functions to assign config options. - */ - -/** <b>c</b>-\>key is known to be a real key. Update <b>options</b> - * with <b>c</b>-\>value and return 0, or return -1 if bad value. - * - * Called from config_assign_line() and option_reset(). - */ -static int -config_assign_value(const config_format_t *fmt, void *options, - config_line_t *c, char **msg) -{ - int i, ok; - const config_var_t *var; - void *lvalue; - - CONFIG_CHECK(fmt, options); - - var = config_find_option(fmt, c->key); - tor_assert(var); - - lvalue = STRUCT_VAR_P(options, var->var_offset); - - switch (var->type) { - - case CONFIG_TYPE_PORT: - if (!strcasecmp(c->value, "auto")) { - *(int *)lvalue = CFG_AUTO_PORT; - break; - } - /* fall through */ - case CONFIG_TYPE_INT: - case CONFIG_TYPE_UINT: - i = (int)tor_parse_long(c->value, 10, - var->type==CONFIG_TYPE_INT ? INT_MIN : 0, - var->type==CONFIG_TYPE_PORT ? 65535 : INT_MAX, - &ok, NULL); - if (!ok) { - tor_asprintf(msg, - "Int keyword '%s %s' is malformed or out of bounds.", - c->key, c->value); - return -1; - } - *(int *)lvalue = i; - break; - - case CONFIG_TYPE_UINT64: { - uint64_t u64 = tor_parse_uint64(c->value, 10, - 0, UINT64_MAX, &ok, NULL); - if (!ok) { - tor_asprintf(msg, - "uint64 keyword '%s %s' is malformed or out of bounds.", - c->key, c->value); - return -1; - } - *(uint64_t *)lvalue = u64; - break; - } - - case CONFIG_TYPE_CSV_INTERVAL: { - /* We used to have entire smartlists here. But now that all of our - * download schedules use exponential backoff, only the first part - * matters. */ - const char *comma = strchr(c->value, ','); - const char *val = c->value; - char *tmp = NULL; - if (comma) { - tmp = tor_strndup(c->value, comma - c->value); - val = tmp; - } - - i = config_parse_interval(val, &ok); - if (!ok) { - tor_asprintf(msg, - "Interval '%s %s' is malformed or out of bounds.", - c->key, c->value); - tor_free(tmp); - return -1; - } - *(int *)lvalue = i; - tor_free(tmp); - break; - } - - case CONFIG_TYPE_INTERVAL: { - i = config_parse_interval(c->value, &ok); - if (!ok) { - tor_asprintf(msg, - "Interval '%s %s' is malformed or out of bounds.", - c->key, c->value); - return -1; - } - *(int *)lvalue = i; - break; - } - - case CONFIG_TYPE_MSEC_INTERVAL: { - i = config_parse_msec_interval(c->value, &ok); - if (!ok) { - tor_asprintf(msg, - "Msec interval '%s %s' is malformed or out of bounds.", - c->key, c->value); - return -1; - } - *(int *)lvalue = i; - break; - } - - case CONFIG_TYPE_MEMUNIT: { - uint64_t u64 = config_parse_memunit(c->value, &ok); - if (!ok) { - tor_asprintf(msg, - "Value '%s %s' is malformed or out of bounds.", - c->key, c->value); - return -1; - } - *(uint64_t *)lvalue = u64; - break; - } - - case CONFIG_TYPE_BOOL: - i = (int)tor_parse_long(c->value, 10, 0, 1, &ok, NULL); - if (!ok) { - tor_asprintf(msg, - "Boolean '%s %s' expects 0 or 1.", - c->key, c->value); - return -1; - } - *(int *)lvalue = i; - break; - - case CONFIG_TYPE_AUTOBOOL: - if (!strcasecmp(c->value, "auto")) - *(int *)lvalue = -1; - else if (!strcmp(c->value, "0")) - *(int *)lvalue = 0; - else if (!strcmp(c->value, "1")) - *(int *)lvalue = 1; - else { - tor_asprintf(msg, "Boolean '%s %s' expects 0, 1, or 'auto'.", - c->key, c->value); - return -1; - } - break; - - case CONFIG_TYPE_STRING: - case CONFIG_TYPE_FILENAME: - tor_free(*(char **)lvalue); - *(char **)lvalue = tor_strdup(c->value); - break; - - case CONFIG_TYPE_DOUBLE: - *(double *)lvalue = atof(c->value); - break; - - case CONFIG_TYPE_ISOTIME: - if (parse_iso_time(c->value, (time_t *)lvalue)) { - tor_asprintf(msg, - "Invalid time '%s' for keyword '%s'", c->value, c->key); - return -1; - } - break; - - case CONFIG_TYPE_ROUTERSET: - if (*(routerset_t**)lvalue) { - routerset_free(*(routerset_t**)lvalue); - } - *(routerset_t**)lvalue = routerset_new(); - if (routerset_parse(*(routerset_t**)lvalue, c->value, c->key)<0) { - tor_asprintf(msg, "Invalid exit list '%s' for option '%s'", - c->value, c->key); - return -1; - } - break; - - case CONFIG_TYPE_CSV: - if (*(smartlist_t**)lvalue) { - SMARTLIST_FOREACH(*(smartlist_t**)lvalue, char *, cp, tor_free(cp)); - smartlist_clear(*(smartlist_t**)lvalue); - } else { - *(smartlist_t**)lvalue = smartlist_new(); - } - - smartlist_split_string(*(smartlist_t**)lvalue, c->value, ",", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - break; - - case CONFIG_TYPE_LINELIST: - case CONFIG_TYPE_LINELIST_S: - { - config_line_t *lastval = *(config_line_t**)lvalue; - if (lastval && lastval->fragile) { - if (c->command != CONFIG_LINE_APPEND) { - config_free_lines(lastval); - *(config_line_t**)lvalue = NULL; - } else { - lastval->fragile = 0; - } - } - - config_line_append((config_line_t**)lvalue, c->key, c->value); - } - break; - case CONFIG_TYPE_OBSOLETE: - log_warn(LD_CONFIG, "Skipping obsolete configuration option '%s'", c->key); - break; - case CONFIG_TYPE_LINELIST_V: - tor_asprintf(msg, - "You may not provide a value for virtual option '%s'", c->key); - return -1; - default: - tor_assert(0); - break; - } - return 0; -} - -/** Mark every linelist in <b>options</b> "fragile", so that fresh assignments - * to it will replace old ones. */ -static void -config_mark_lists_fragile(const config_format_t *fmt, void *options) -{ - int i; - tor_assert(fmt); - tor_assert(options); - - for (i = 0; fmt->vars[i].name; ++i) { - const config_var_t *var = &fmt->vars[i]; - config_line_t *list; - if (var->type != CONFIG_TYPE_LINELIST && - var->type != CONFIG_TYPE_LINELIST_V) - continue; - - list = *(config_line_t **)STRUCT_VAR_P(options, var->var_offset); - if (list) - list->fragile = 1; - } -} - -void -warn_deprecated_option(const char *what, const char *why) -{ - const char *space = (why && strlen(why)) ? " " : ""; - log_warn(LD_CONFIG, "The %s option is deprecated, and will most likely " - "be removed in a future version of Tor.%s%s (If you think this is " - "a mistake, please let us know!)", - what, space, why); -} - -/** If <b>c</b> is a syntactically valid configuration line, update - * <b>options</b> with its value and return 0. Otherwise return -1 for bad - * key, -2 for bad value. - * - * If <b>clear_first</b> is set, clear the value first. Then if - * <b>use_defaults</b> is set, set the value to the default. - * - * Called from config_assign(). - */ -static int -config_assign_line(const config_format_t *fmt, void *options, - config_line_t *c, unsigned flags, - bitarray_t *options_seen, char **msg) -{ - const unsigned use_defaults = flags & CAL_USE_DEFAULTS; - const unsigned clear_first = flags & CAL_CLEAR_FIRST; - const unsigned warn_deprecations = flags & CAL_WARN_DEPRECATIONS; - const config_var_t *var; - - CONFIG_CHECK(fmt, options); - - var = config_find_option(fmt, c->key); - if (!var) { - if (fmt->extra) { - void *lvalue = STRUCT_VAR_P(options, fmt->extra->var_offset); - log_info(LD_CONFIG, - "Found unrecognized option '%s'; saving it.", c->key); - config_line_append((config_line_t**)lvalue, c->key, c->value); - return 0; - } else { - tor_asprintf(msg, - "Unknown option '%s'. Failing.", c->key); - return -1; - } - } - - /* Put keyword into canonical case. */ - if (strcmp(var->name, c->key)) { - tor_free(c->key); - c->key = tor_strdup(var->name); - } - - const char *deprecation_msg; - if (warn_deprecations && - (deprecation_msg = config_find_deprecation(fmt, var->name))) { - warn_deprecated_option(var->name, deprecation_msg); - } - - if (!strlen(c->value)) { - /* reset or clear it, then return */ - if (!clear_first) { - if ((var->type == CONFIG_TYPE_LINELIST || - var->type == CONFIG_TYPE_LINELIST_S) && - c->command != CONFIG_LINE_CLEAR) { - /* We got an empty linelist from the torrc or command line. - As a special case, call this an error. Warn and ignore. */ - log_warn(LD_CONFIG, - "Linelist option '%s' has no value. Skipping.", c->key); - } else { /* not already cleared */ - config_reset(fmt, options, var, use_defaults); - } - } - return 0; - } else if (c->command == CONFIG_LINE_CLEAR && !clear_first) { - config_reset(fmt, options, var, use_defaults); - } - - if (options_seen && (var->type != CONFIG_TYPE_LINELIST && - var->type != CONFIG_TYPE_LINELIST_S)) { - /* We're tracking which options we've seen, and this option is not - * supposed to occur more than once. */ - int var_index = (int)(var - fmt->vars); - if (bitarray_is_set(options_seen, var_index)) { - log_warn(LD_CONFIG, "Option '%s' used more than once; all but the last " - "value will be ignored.", var->name); - } - bitarray_set(options_seen, var_index); - } - - if (config_assign_value(fmt, options, c, msg) < 0) - return -2; - return 0; -} - -/** Restore the option named <b>key</b> in options to its default value. - * Called from config_assign(). */ -static void -config_reset_line(const config_format_t *fmt, void *options, - const char *key, int use_defaults) -{ - const config_var_t *var; - - CONFIG_CHECK(fmt, options); - - var = config_find_option(fmt, key); - if (!var) - return; /* give error on next pass. */ - - config_reset(fmt, options, var, use_defaults); -} - -/** Return true iff value needs to be quoted and escaped to be used in - * a configuration file. */ -static int -config_value_needs_escape(const char *value) -{ - if (*value == '\"') - return 1; - while (*value) { - switch (*value) - { - case '\r': - case '\n': - case '#': - /* Note: quotes and backspaces need special handling when we are using - * quotes, not otherwise, so they don't trigger escaping on their - * own. */ - return 1; - default: - if (!TOR_ISPRINT(*value)) - return 1; - } - ++value; - } - return 0; -} - -/** Return newly allocated line or lines corresponding to <b>key</b> in the - * configuration <b>options</b>. If <b>escape_val</b> is true and a - * value needs to be quoted before it's put in a config file, quote and - * escape that value. Return NULL if no such key exists. */ -config_line_t * -config_get_assigned_option(const config_format_t *fmt, const void *options, - const char *key, int escape_val) -{ - const config_var_t *var; - const void *value; - config_line_t *result; - tor_assert(options && key); - - CONFIG_CHECK(fmt, options); - - var = config_find_option(fmt, key); - if (!var) { - log_warn(LD_CONFIG, "Unknown option '%s'. Failing.", key); - return NULL; - } - value = STRUCT_VAR_P(options, var->var_offset); - - result = tor_malloc_zero(sizeof(config_line_t)); - result->key = tor_strdup(var->name); - switch (var->type) - { - case CONFIG_TYPE_STRING: - case CONFIG_TYPE_FILENAME: - if (*(char**)value) { - result->value = tor_strdup(*(char**)value); - } else { - tor_free(result->key); - tor_free(result); - return NULL; - } - break; - case CONFIG_TYPE_ISOTIME: - if (*(time_t*)value) { - result->value = tor_malloc(ISO_TIME_LEN+1); - format_iso_time(result->value, *(time_t*)value); - } else { - tor_free(result->key); - tor_free(result); - } - escape_val = 0; /* Can't need escape. */ - break; - case CONFIG_TYPE_PORT: - if (*(int*)value == CFG_AUTO_PORT) { - result->value = tor_strdup("auto"); - escape_val = 0; - break; - } - /* fall through */ - case CONFIG_TYPE_CSV_INTERVAL: - case CONFIG_TYPE_INTERVAL: - case CONFIG_TYPE_MSEC_INTERVAL: - case CONFIG_TYPE_UINT: - case CONFIG_TYPE_INT: - /* This means every or_options_t uint or bool element - * needs to be an int. Not, say, a uint16_t or char. */ - tor_asprintf(&result->value, "%d", *(int*)value); - escape_val = 0; /* Can't need escape. */ - break; - case CONFIG_TYPE_UINT64: /* Fall through */ - case CONFIG_TYPE_MEMUNIT: - tor_asprintf(&result->value, "%"PRIu64, - (*(uint64_t*)value)); - escape_val = 0; /* Can't need escape. */ - break; - case CONFIG_TYPE_DOUBLE: - tor_asprintf(&result->value, "%f", *(double*)value); - escape_val = 0; /* Can't need escape. */ - break; - - case CONFIG_TYPE_AUTOBOOL: - if (*(int*)value == -1) { - result->value = tor_strdup("auto"); - escape_val = 0; - break; - } - /* fall through */ - case CONFIG_TYPE_BOOL: - result->value = tor_strdup(*(int*)value ? "1" : "0"); - escape_val = 0; /* Can't need escape. */ - break; - case CONFIG_TYPE_ROUTERSET: - result->value = routerset_to_string(*(routerset_t**)value); - break; - case CONFIG_TYPE_CSV: - if (*(smartlist_t**)value) - result->value = - smartlist_join_strings(*(smartlist_t**)value, ",", 0, NULL); - else - result->value = tor_strdup(""); - break; - case CONFIG_TYPE_OBSOLETE: - log_fn(LOG_INFO, LD_CONFIG, - "You asked me for the value of an obsolete config option '%s'.", - key); - tor_free(result->key); - tor_free(result); - return NULL; - case CONFIG_TYPE_LINELIST_S: - tor_free(result->key); - tor_free(result); - result = config_lines_dup_and_filter(*(const config_line_t **)value, - key); - break; - case CONFIG_TYPE_LINELIST: - case CONFIG_TYPE_LINELIST_V: - tor_free(result->key); - tor_free(result); - result = config_lines_dup(*(const config_line_t**)value); - break; - default: - tor_free(result->key); - tor_free(result); - log_warn(LD_BUG,"Unknown type %d for known key '%s'", - var->type, key); - return NULL; - } - - if (escape_val) { - config_line_t *line; - for (line = result; line; line = line->next) { - if (line->value && config_value_needs_escape(line->value)) { - char *newval = esc_for_log(line->value); - tor_free(line->value); - line->value = newval; - } - } - } - - return result; -} -/** Iterate through the linked list of requested options <b>list</b>. - * For each item, convert as appropriate and assign to <b>options</b>. - * If an item is unrecognized, set *msg and return -1 immediately, - * else return 0 for success. - * - * If <b>clear_first</b>, interpret config options as replacing (not - * extending) their previous values. If <b>clear_first</b> is set, - * then <b>use_defaults</b> to decide if you set to defaults after - * clearing, or make the value 0 or NULL. - * - * Here are the use cases: - * 1. A non-empty AllowInvalid line in your torrc. Appends to current - * if linelist, replaces current if csv. - * 2. An empty AllowInvalid line in your torrc. Should clear it. - * 3. "RESETCONF AllowInvalid" sets it to default. - * 4. "SETCONF AllowInvalid" makes it NULL. - * 5. "SETCONF AllowInvalid=foo" clears it and sets it to "foo". - * - * Use_defaults Clear_first - * 0 0 "append" - * 1 0 undefined, don't use - * 0 1 "set to null first" - * 1 1 "set to defaults first" - * Return 0 on success, -1 on bad key, -2 on bad value. - * - * As an additional special case, if a LINELIST config option has - * no value and clear_first is 0, then warn and ignore it. - */ - -/* -There are three call cases for config_assign() currently. - -Case one: Torrc entry -options_init_from_torrc() calls config_assign(0, 0) - calls config_assign_line(0, 0). - if value is empty, calls config_reset(0) and returns. - calls config_assign_value(), appends. - -Case two: setconf -options_trial_assign() calls config_assign(0, 1) - calls config_reset_line(0) - calls config_reset(0) - calls option_clear(). - calls config_assign_line(0, 1). - if value is empty, returns. - calls config_assign_value(), appends. - -Case three: resetconf -options_trial_assign() calls config_assign(1, 1) - calls config_reset_line(1) - calls config_reset(1) - calls option_clear(). - calls config_assign_value(default) - calls config_assign_line(1, 1). - returns. -*/ -int -config_assign(const config_format_t *fmt, void *options, config_line_t *list, - unsigned config_assign_flags, char **msg) -{ - config_line_t *p; - bitarray_t *options_seen; - const int n_options = config_count_options(fmt); - const unsigned clear_first = config_assign_flags & CAL_CLEAR_FIRST; - const unsigned use_defaults = config_assign_flags & CAL_USE_DEFAULTS; - - CONFIG_CHECK(fmt, options); - - /* pass 1: normalize keys */ - for (p = list; p; p = p->next) { - const char *full = config_expand_abbrev(fmt, p->key, 0, 1); - if (strcmp(full,p->key)) { - tor_free(p->key); - p->key = tor_strdup(full); - } - } - - /* pass 2: if we're reading from a resetting source, clear all - * mentioned config options, and maybe set to their defaults. */ - if (clear_first) { - for (p = list; p; p = p->next) - config_reset_line(fmt, options, p->key, use_defaults); - } - - options_seen = bitarray_init_zero(n_options); - /* pass 3: assign. */ - while (list) { - int r; - if ((r=config_assign_line(fmt, options, list, config_assign_flags, - options_seen, msg))) { - bitarray_free(options_seen); - return r; - } - list = list->next; - } - bitarray_free(options_seen); - - /** Now we're done assigning a group of options to the configuration. - * Subsequent group assignments should _replace_ linelists, not extend - * them. */ - config_mark_lists_fragile(fmt, options); - - return 0; -} - -/** Reset config option <b>var</b> to 0, 0.0, NULL, or the equivalent. - * Called from config_reset() and config_free(). */ -static void -config_clear(const config_format_t *fmt, void *options, - const config_var_t *var) -{ - void *lvalue = STRUCT_VAR_P(options, var->var_offset); - (void)fmt; /* unused */ - switch (var->type) { - case CONFIG_TYPE_STRING: - case CONFIG_TYPE_FILENAME: - tor_free(*(char**)lvalue); - break; - case CONFIG_TYPE_DOUBLE: - *(double*)lvalue = 0.0; - break; - case CONFIG_TYPE_ISOTIME: - *(time_t*)lvalue = 0; - break; - case CONFIG_TYPE_CSV_INTERVAL: - case CONFIG_TYPE_INTERVAL: - case CONFIG_TYPE_MSEC_INTERVAL: - case CONFIG_TYPE_UINT: - case CONFIG_TYPE_INT: - case CONFIG_TYPE_PORT: - case CONFIG_TYPE_BOOL: - *(int*)lvalue = 0; - break; - case CONFIG_TYPE_AUTOBOOL: - *(int*)lvalue = -1; - break; - case CONFIG_TYPE_UINT64: - case CONFIG_TYPE_MEMUNIT: - *(uint64_t*)lvalue = 0; - break; - case CONFIG_TYPE_ROUTERSET: - if (*(routerset_t**)lvalue) { - routerset_free(*(routerset_t**)lvalue); - *(routerset_t**)lvalue = NULL; - } - break; - case CONFIG_TYPE_CSV: - if (*(smartlist_t**)lvalue) { - SMARTLIST_FOREACH(*(smartlist_t **)lvalue, char *, cp, tor_free(cp)); - smartlist_free(*(smartlist_t **)lvalue); - *(smartlist_t **)lvalue = NULL; - } - break; - case CONFIG_TYPE_LINELIST: - case CONFIG_TYPE_LINELIST_S: - config_free_lines(*(config_line_t **)lvalue); - *(config_line_t **)lvalue = NULL; - break; - case CONFIG_TYPE_LINELIST_V: - /* handled by linelist_s. */ - break; - case CONFIG_TYPE_OBSOLETE: - break; - } -} - -/** Clear the option indexed by <b>var</b> in <b>options</b>. Then if - * <b>use_defaults</b>, set it to its default value. - * Called by config_init() and option_reset_line() and option_assign_line(). */ -static void -config_reset(const config_format_t *fmt, void *options, - const config_var_t *var, int use_defaults) -{ - config_line_t *c; - char *msg = NULL; - CONFIG_CHECK(fmt, options); - config_clear(fmt, options, var); /* clear it first */ - if (!use_defaults) - return; /* all done */ - if (var->initvalue) { - c = tor_malloc_zero(sizeof(config_line_t)); - c->key = tor_strdup(var->name); - c->value = tor_strdup(var->initvalue); - if (config_assign_value(fmt, options, c, &msg) < 0) { - log_warn(LD_BUG, "Failed to assign default: %s", msg); - tor_free(msg); /* if this happens it's a bug */ - } - config_free_lines(c); - } -} - -/** Release storage held by <b>options</b>. */ -void -config_free_(const config_format_t *fmt, void *options) -{ - int i; - - if (!options) - return; - - tor_assert(fmt); - - for (i=0; fmt->vars[i].name; ++i) - config_clear(fmt, options, &(fmt->vars[i])); - if (fmt->extra) { - config_line_t **linep = STRUCT_VAR_P(options, fmt->extra->var_offset); - config_free_lines(*linep); - *linep = NULL; - } - tor_free(options); -} - -/** Return true iff the option <b>name</b> has the same value in <b>o1</b> - * and <b>o2</b>. Must not be called for LINELIST_S or OBSOLETE options. - */ -int -config_is_same(const config_format_t *fmt, - const void *o1, const void *o2, - const char *name) -{ - config_line_t *c1, *c2; - int r = 1; - CONFIG_CHECK(fmt, o1); - CONFIG_CHECK(fmt, o2); - - c1 = config_get_assigned_option(fmt, o1, name, 0); - c2 = config_get_assigned_option(fmt, o2, name, 0); - r = config_lines_eq(c1, c2); - config_free_lines(c1); - config_free_lines(c2); - return r; -} - -/** Copy storage held by <b>old</b> into a new or_options_t and return it. */ -void * -config_dup(const config_format_t *fmt, const void *old) -{ - void *newopts; - int i; - config_line_t *line; - - newopts = config_new(fmt); - for (i=0; fmt->vars[i].name; ++i) { - if (fmt->vars[i].type == CONFIG_TYPE_LINELIST_S) - continue; - if (fmt->vars[i].type == CONFIG_TYPE_OBSOLETE) - continue; - line = config_get_assigned_option(fmt, old, fmt->vars[i].name, 0); - if (line) { - char *msg = NULL; - if (config_assign(fmt, newopts, line, 0, &msg) < 0) { - log_err(LD_BUG, "config_get_assigned_option() generated " - "something we couldn't config_assign(): %s", msg); - tor_free(msg); - tor_assert(0); - } - } - config_free_lines(line); - } - return newopts; -} -/** Set all vars in the configuration object <b>options</b> to their default - * values. */ -void -config_init(const config_format_t *fmt, void *options) -{ - int i; - const config_var_t *var; - CONFIG_CHECK(fmt, options); - - for (i=0; fmt->vars[i].name; ++i) { - var = &fmt->vars[i]; - if (!var->initvalue) - continue; /* defaults to NULL or 0 */ - config_reset(fmt, options, var, 1); - } -} - -/** Allocate and return a new string holding the written-out values of the vars - * in 'options'. If 'minimal', do not write out any default-valued vars. - * Else, if comment_defaults, write default values as comments. - */ -char * -config_dump(const config_format_t *fmt, const void *default_options, - const void *options, int minimal, - int comment_defaults) -{ - smartlist_t *elements; - const void *defaults = default_options; - void *defaults_tmp = NULL; - config_line_t *line, *assigned; - char *result; - int i; - char *msg = NULL; - - if (defaults == NULL) { - defaults = defaults_tmp = config_new(fmt); - config_init(fmt, defaults_tmp); - } - - /* XXX use a 1 here so we don't add a new log line while dumping */ - if (default_options == NULL) { - if (fmt->validate_fn(NULL, defaults_tmp, defaults_tmp, 1, &msg) < 0) { - log_err(LD_BUG, "Failed to validate default config: %s", msg); - tor_free(msg); - tor_assert(0); - } - } - - elements = smartlist_new(); - for (i=0; fmt->vars[i].name; ++i) { - int comment_option = 0; - if (fmt->vars[i].type == CONFIG_TYPE_OBSOLETE || - fmt->vars[i].type == CONFIG_TYPE_LINELIST_S) - continue; - /* Don't save 'hidden' control variables. */ - if (!strcmpstart(fmt->vars[i].name, "__")) - continue; - if (minimal && config_is_same(fmt, options, defaults, fmt->vars[i].name)) - continue; - else if (comment_defaults && - config_is_same(fmt, options, defaults, fmt->vars[i].name)) - comment_option = 1; - - line = assigned = - config_get_assigned_option(fmt, options, fmt->vars[i].name, 1); - - for (; line; line = line->next) { - if (!strcmpstart(line->key, "__")) { - /* This check detects "hidden" variables inside LINELIST_V structures. - */ - continue; - } - smartlist_add_asprintf(elements, "%s%s %s\n", - comment_option ? "# " : "", - line->key, line->value); - } - config_free_lines(assigned); - } - - if (fmt->extra) { - line = *(config_line_t**)STRUCT_VAR_P(options, fmt->extra->var_offset); - for (; line; line = line->next) { - smartlist_add_asprintf(elements, "%s %s\n", line->key, line->value); - } - } - - result = smartlist_join_strings(elements, "", 0, NULL); - SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp)); - smartlist_free(elements); - if (defaults_tmp) { - fmt->free_fn(defaults_tmp); - } - return result; -} - -/** Mapping from a unit name to a multiplier for converting that unit into a - * base unit. Used by config_parse_unit. */ -struct unit_table_t { - const char *unit; /**< The name of the unit */ - uint64_t multiplier; /**< How many of the base unit appear in this unit */ -}; - -/** Table to map the names of memory units to the number of bytes they - * contain. */ -static struct unit_table_t memory_units[] = { - { "", 1 }, - { "b", 1<< 0 }, - { "byte", 1<< 0 }, - { "bytes", 1<< 0 }, - { "kb", 1<<10 }, - { "kbyte", 1<<10 }, - { "kbytes", 1<<10 }, - { "kilobyte", 1<<10 }, - { "kilobytes", 1<<10 }, - { "kilobits", 1<<7 }, - { "kilobit", 1<<7 }, - { "kbits", 1<<7 }, - { "kbit", 1<<7 }, - { "m", 1<<20 }, - { "mb", 1<<20 }, - { "mbyte", 1<<20 }, - { "mbytes", 1<<20 }, - { "megabyte", 1<<20 }, - { "megabytes", 1<<20 }, - { "megabits", 1<<17 }, - { "megabit", 1<<17 }, - { "mbits", 1<<17 }, - { "mbit", 1<<17 }, - { "gb", 1<<30 }, - { "gbyte", 1<<30 }, - { "gbytes", 1<<30 }, - { "gigabyte", 1<<30 }, - { "gigabytes", 1<<30 }, - { "gigabits", 1<<27 }, - { "gigabit", 1<<27 }, - { "gbits", 1<<27 }, - { "gbit", 1<<27 }, - { "tb", UINT64_C(1)<<40 }, - { "tbyte", UINT64_C(1)<<40 }, - { "tbytes", UINT64_C(1)<<40 }, - { "terabyte", UINT64_C(1)<<40 }, - { "terabytes", UINT64_C(1)<<40 }, - { "terabits", UINT64_C(1)<<37 }, - { "terabit", UINT64_C(1)<<37 }, - { "tbits", UINT64_C(1)<<37 }, - { "tbit", UINT64_C(1)<<37 }, - { NULL, 0 }, -}; - -/** Table to map the names of time units to the number of seconds they - * contain. */ -static struct unit_table_t time_units[] = { - { "", 1 }, - { "second", 1 }, - { "seconds", 1 }, - { "minute", 60 }, - { "minutes", 60 }, - { "hour", 60*60 }, - { "hours", 60*60 }, - { "day", 24*60*60 }, - { "days", 24*60*60 }, - { "week", 7*24*60*60 }, - { "weeks", 7*24*60*60 }, - { "month", 2629728, }, /* about 30.437 days */ - { "months", 2629728, }, - { NULL, 0 }, -}; - -/** Table to map the names of time units to the number of milliseconds - * they contain. */ -static struct unit_table_t time_msec_units[] = { - { "", 1 }, - { "msec", 1 }, - { "millisecond", 1 }, - { "milliseconds", 1 }, - { "second", 1000 }, - { "seconds", 1000 }, - { "minute", 60*1000 }, - { "minutes", 60*1000 }, - { "hour", 60*60*1000 }, - { "hours", 60*60*1000 }, - { "day", 24*60*60*1000 }, - { "days", 24*60*60*1000 }, - { "week", 7*24*60*60*1000 }, - { "weeks", 7*24*60*60*1000 }, - { NULL, 0 }, -}; - -/** Parse a string <b>val</b> containing a number, zero or more - * spaces, and an optional unit string. If the unit appears in the - * table <b>u</b>, then multiply the number by the unit multiplier. - * On success, set *<b>ok</b> to 1 and return this product. - * Otherwise, set *<b>ok</b> to 0. - */ -static uint64_t -config_parse_units(const char *val, struct unit_table_t *u, int *ok) -{ - uint64_t v = 0; - double d = 0; - int use_float = 0; - char *cp; - - tor_assert(ok); - - v = tor_parse_uint64(val, 10, 0, UINT64_MAX, ok, &cp); - if (!*ok || (cp && *cp == '.')) { - d = tor_parse_double(val, 0, (double)UINT64_MAX, ok, &cp); - if (!*ok) - goto done; - use_float = 1; - } - - if (!cp) { - *ok = 1; - v = use_float ? ((uint64_t)d) : v; - goto done; - } - - cp = (char*) eat_whitespace(cp); - - for ( ;u->unit;++u) { - if (!strcasecmp(u->unit, cp)) { - if (use_float) - v = (uint64_t)(u->multiplier * d); - else - v *= u->multiplier; - *ok = 1; - goto done; - } - } - log_warn(LD_CONFIG, "Unknown unit '%s'.", cp); - *ok = 0; - done: - - if (*ok) - return v; - else - return 0; -} - -/** Parse a string in the format "number unit", where unit is a unit of - * information (byte, KB, M, etc). On success, set *<b>ok</b> to true - * and return the number of bytes specified. Otherwise, set - * *<b>ok</b> to false and return 0. */ -static uint64_t -config_parse_memunit(const char *s, int *ok) -{ - uint64_t u = config_parse_units(s, memory_units, ok); - return u; -} - -/** Parse a string in the format "number unit", where unit is a unit of - * time in milliseconds. On success, set *<b>ok</b> to true and return - * the number of milliseconds in the provided interval. Otherwise, set - * *<b>ok</b> to 0 and return -1. */ -static int -config_parse_msec_interval(const char *s, int *ok) -{ - uint64_t r; - r = config_parse_units(s, time_msec_units, ok); - if (r > INT_MAX) { - log_warn(LD_CONFIG, "Msec interval '%s' is too long", s); - *ok = 0; - return -1; - } - return (int)r; -} - -/** Parse a string in the format "number unit", where unit is a unit of time. - * On success, set *<b>ok</b> to true and return the number of seconds in - * the provided interval. Otherwise, set *<b>ok</b> to 0 and return -1. - */ -static int -config_parse_interval(const char *s, int *ok) -{ - uint64_t r; - r = config_parse_units(s, time_units, ok); - if (r > INT_MAX) { - log_warn(LD_CONFIG, "Interval '%s' is too long", s); - *ok = 0; - return -1; - } - return (int)r; -} diff --git a/src/app/config/confparse.h b/src/app/config/confparse.h deleted file mode 100644 index 57f1ec1762..0000000000 --- a/src/app/config/confparse.h +++ /dev/null @@ -1,233 +0,0 @@ -/* Copyright (c) 2001 Matej Pfajfar. - * Copyright (c) 2001-2004, Roger Dingledine. - * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ -/* See LICENSE for licensing information */ - -/** - * \file confparse.h - * - * \brief Header for confparse.c. - */ - -#ifndef TOR_CONFPARSE_H -#define TOR_CONFPARSE_H - -/** Enumeration of types which option values can take */ -typedef enum config_type_t { - CONFIG_TYPE_STRING = 0, /**< An arbitrary string. */ - CONFIG_TYPE_FILENAME, /**< A filename: some prefixes get expanded. */ - CONFIG_TYPE_UINT, /**< A non-negative integer less than MAX_INT */ - CONFIG_TYPE_INT, /**< Any integer. */ - CONFIG_TYPE_UINT64, /**< A value in range 0..UINT64_MAX */ - CONFIG_TYPE_PORT, /**< A port from 1...65535, 0 for "not set", or - * "auto". */ - CONFIG_TYPE_INTERVAL, /**< A number of seconds, with optional units*/ - CONFIG_TYPE_MSEC_INTERVAL,/**< A number of milliseconds, with optional - * units */ - CONFIG_TYPE_MEMUNIT, /**< A number of bytes, with optional units*/ - CONFIG_TYPE_DOUBLE, /**< A floating-point value */ - CONFIG_TYPE_BOOL, /**< A boolean value, expressed as 0 or 1. */ - CONFIG_TYPE_AUTOBOOL, /**< A boolean+auto value, expressed 0 for false, - * 1 for true, and -1 for auto */ - CONFIG_TYPE_ISOTIME, /**< An ISO-formatted time relative to UTC. */ - CONFIG_TYPE_CSV, /**< A list of strings, separated by commas and - * optional whitespace. */ - CONFIG_TYPE_CSV_INTERVAL, /**< A list of strings, separated by commas and - * optional whitespace, representing intervals in - * seconds, with optional units. We allow - * multiple values here for legacy reasons, but - * ignore every value after the first. */ - CONFIG_TYPE_LINELIST, /**< Uninterpreted config lines */ - CONFIG_TYPE_LINELIST_S, /**< Uninterpreted, context-sensitive config lines, - * mixed with other keywords. */ - CONFIG_TYPE_LINELIST_V, /**< Catch-all "virtual" option to summarize - * context-sensitive config lines when fetching. - */ - CONFIG_TYPE_ROUTERSET, /**< A list of router names, addrs, and fps, - * parsed into a routerset_t. */ - CONFIG_TYPE_OBSOLETE, /**< Obsolete (ignored) option. */ -} config_type_t; - -#ifdef TOR_UNIT_TESTS -/** - * Union used when building in test mode typechecking the members of a type - * used with confparse.c. See CONF_CHECK_VAR_TYPE for a description of how - * it is used. */ -typedef union { - char **STRING; - char **FILENAME; - int *UINT; /* yes, really: Even though the confparse type is called - * "UINT", it still uses the C int type -- it just enforces that - * the values are in range [0,INT_MAX]. - */ - uint64_t *UINT64; - int *INT; - int *PORT; - int *INTERVAL; - int *MSEC_INTERVAL; - uint64_t *MEMUNIT; - double *DOUBLE; - int *BOOL; - int *AUTOBOOL; - time_t *ISOTIME; - smartlist_t **CSV; - int *CSV_INTERVAL; - struct config_line_t **LINELIST; - struct config_line_t **LINELIST_S; - struct config_line_t **LINELIST_V; - routerset_t **ROUTERSET; -} confparse_dummy_values_t; -#endif /* defined(TOR_UNIT_TESTS) */ - -/** An abbreviation for a configuration option allowed on the command line. */ -typedef struct config_abbrev_t { - const char *abbreviated; - const char *full; - int commandline_only; - int warn; -} config_abbrev_t; - -typedef struct config_deprecation_t { - const char *name; - const char *why_deprecated; -} config_deprecation_t; - -/* Handy macro for declaring "In the config file or on the command line, - * you can abbreviate <b>tok</b>s as <b>tok</b>". */ -#define PLURAL(tok) { #tok, #tok "s", 0, 0 } - -/** A variable allowed in the configuration file or on the command line. */ -typedef struct config_var_t { - const char *name; /**< The full keyword (case insensitive). */ - config_type_t type; /**< How to interpret the type and turn it into a - * value. */ - off_t var_offset; /**< Offset of the corresponding member of or_options_t. */ - const char *initvalue; /**< String (or null) describing initial value. */ - -#ifdef TOR_UNIT_TESTS - /** Used for compiler-magic to typecheck the corresponding field in the - * corresponding struct. Only used in unit test mode, at compile-time. */ - confparse_dummy_values_t var_ptr_dummy; -#endif -} config_var_t; - -/* Macros to define extra members inside config_var_t fields, and at the - * end of a list of them. - */ -#ifdef TOR_UNIT_TESTS -/* This is a somewhat magic type-checking macro for users of confparse.c. - * It initializes a union member "confparse_dummy_values_t.conftype" with - * the address of a static member "tp_dummy.member". This - * will give a compiler warning unless the member field is of the correct - * type. - * - * (This warning is mandatory, because a type mismatch here violates the type - * compatibility constraint for simple assignment, and requires a diagnostic, - * according to the C spec.) - * - * For example, suppose you say: - * "CONF_CHECK_VAR_TYPE(or_options_t, STRING, Address)". - * Then this macro will evaluate to: - * { .STRING = &or_options_t_dummy.Address } - * And since confparse_dummy_values_t.STRING has type "char **", that - * expression will create a warning unless or_options_t.Address also - * has type "char *". - */ -#define CONF_CHECK_VAR_TYPE(tp, conftype, member) \ - { . conftype = &tp ## _dummy . member } -#define CONF_TEST_MEMBERS(tp, conftype, member) \ - , CONF_CHECK_VAR_TYPE(tp, conftype, member) -#define END_OF_CONFIG_VARS \ - { NULL, CONFIG_TYPE_OBSOLETE, 0, NULL, { .INT=NULL } } -#define DUMMY_TYPECHECK_INSTANCE(tp) \ - static tp tp ## _dummy -#else /* !(defined(TOR_UNIT_TESTS)) */ -#define CONF_TEST_MEMBERS(tp, conftype, member) -#define END_OF_CONFIG_VARS { NULL, CONFIG_TYPE_OBSOLETE, 0, NULL } -/* Repeatedly declarable incomplete struct to absorb redundant semicolons */ -#define DUMMY_TYPECHECK_INSTANCE(tp) \ - struct tor_semicolon_eater -#endif /* defined(TOR_UNIT_TESTS) */ - -/** Type of a callback to validate whether a given configuration is - * well-formed and consistent. See options_trial_assign() for documentation - * of arguments. */ -typedef int (*validate_fn_t)(void*,void*,void*,int,char**); - -/** Callback to free a configuration object. */ -typedef void (*free_cfg_fn_t)(void*); - -/** Information on the keys, value types, key-to-struct-member mappings, - * variable descriptions, validation functions, and abbreviations for a - * configuration or storage format. */ -typedef struct config_format_t { - size_t size; /**< Size of the struct that everything gets parsed into. */ - uint32_t magic; /**< Required 'magic value' to make sure we have a struct - * of the right type. */ - off_t magic_offset; /**< Offset of the magic value within the struct. */ - config_abbrev_t *abbrevs; /**< List of abbreviations that we expand when - * parsing this format. */ - const config_deprecation_t *deprecations; /** List of deprecated options */ - config_var_t *vars; /**< List of variables we recognize, their default - * values, and where we stick them in the structure. */ - validate_fn_t validate_fn; /**< Function to validate config. */ - free_cfg_fn_t free_fn; /**< Function to free the configuration. */ - /** If present, extra is a LINELIST variable for unrecognized - * lines. Otherwise, unrecognized lines are an error. */ - config_var_t *extra; -} config_format_t; - -/** Macro: assert that <b>cfg</b> has the right magic field for format - * <b>fmt</b>. */ -#define CONFIG_CHECK(fmt, cfg) STMT_BEGIN \ - tor_assert(fmt && cfg); \ - tor_assert((fmt)->magic == \ - *(uint32_t*)STRUCT_VAR_P(cfg,fmt->magic_offset)); \ - STMT_END - -#define CAL_USE_DEFAULTS (1u<<0) -#define CAL_CLEAR_FIRST (1u<<1) -#define CAL_WARN_DEPRECATIONS (1u<<2) - -void *config_new(const config_format_t *fmt); -void config_free_(const config_format_t *fmt, void *options); -#define config_free(fmt, options) do { \ - config_free_((fmt), (options)); \ - (options) = NULL; \ - } while (0) - -struct config_line_t *config_get_assigned_option(const config_format_t *fmt, - const void *options, const char *key, - int escape_val); -int config_is_same(const config_format_t *fmt, - const void *o1, const void *o2, - const char *name); -void config_init(const config_format_t *fmt, void *options); -void *config_dup(const config_format_t *fmt, const void *old); -char *config_dump(const config_format_t *fmt, const void *default_options, - const void *options, int minimal, - int comment_defaults); -int config_assign(const config_format_t *fmt, void *options, - struct config_line_t *list, - unsigned flags, char **msg); -config_var_t *config_find_option_mutable(config_format_t *fmt, - const char *key); -const char *config_find_deprecation(const config_format_t *fmt, - const char *key); -const config_var_t *config_find_option(const config_format_t *fmt, - const char *key); -const char *config_expand_abbrev(const config_format_t *fmt, - const char *option, - int command_line, int warn_obsolete); -void warn_deprecated_option(const char *what, const char *why); - -/* Helper macros to compare an option across two configuration objects */ -#define CFG_EQ_BOOL(a,b,opt) ((a)->opt == (b)->opt) -#define CFG_EQ_INT(a,b,opt) ((a)->opt == (b)->opt) -#define CFG_EQ_STRING(a,b,opt) (!strcmp_opt((a)->opt, (b)->opt)) -#define CFG_EQ_SMARTLIST(a,b,opt) smartlist_strings_eq((a)->opt, (b)->opt) -#define CFG_EQ_LINELIST(a,b,opt) config_lines_eq((a)->opt, (b)->opt) -#define CFG_EQ_ROUTERSET(a,b,opt) routerset_equal((a)->opt, (b)->opt) - -#endif /* !defined(TOR_CONFPARSE_H) */ diff --git a/src/app/config/or_options_st.h b/src/app/config/or_options_st.h index 2ee2d15674..32dcd9fb18 100644 --- a/src/app/config/or_options_st.h +++ b/src/app/config/or_options_st.h @@ -18,6 +18,7 @@ struct smartlist_t; struct config_line_t; +struct config_suite_t; /** Enumeration of outbound address configuration types: * Exit-only, OR-only, or both */ @@ -121,7 +122,6 @@ struct or_options_t { struct config_line_t *RecommendedVersions; struct config_line_t *RecommendedClientVersions; struct config_line_t *RecommendedServerVersions; - struct config_line_t *RecommendedPackages; /** Whether dirservers allow router descriptors with private IPs. */ int DirAllowPrivateAddresses; /** Whether routers accept EXTEND cells to routers with private IPs. */ @@ -1108,6 +1108,14 @@ struct or_options_t { * a possible previous dormant state. **/ int DormantCanceledByStartup; + + /** + * Configuration objects for individual modules. + * + * Never access this field or its members directly: instead, use the module + * in question to get its relevant configuration object. + */ + struct config_suite_t *subconfigs_; }; #endif /* !defined(TOR_OR_OPTIONS_ST_H) */ diff --git a/src/app/config/or_state_st.h b/src/app/config/or_state_st.h index f45c6196cc..225003bb7e 100644 --- a/src/app/config/or_state_st.h +++ b/src/app/config/or_state_st.h @@ -15,6 +15,7 @@ #include "lib/cc/torint.h" struct smartlist_t; +struct config_suite_t; /** Persistent state for an onion router, as saved to disk. */ struct or_state_t { @@ -94,6 +95,14 @@ struct or_state_t { /** True if we were dormant when we last wrote the file; false if we * weren't. "auto" on initial startup. */ int Dormant; + + /** + * State objects for individual modules. + * + * Never access this field or its members directly: instead, use the module + * in question to get its relevant state object if you must. + */ + struct config_suite_t *substates_; }; #endif /* !defined(TOR_OR_STATE_ST_H) */ diff --git a/src/app/config/statefile.c b/src/app/config/statefile.c index fdfd68b244..552bd2c443 100644 --- a/src/app/config/statefile.c +++ b/src/app/config/statefile.c @@ -32,7 +32,7 @@ #include "core/or/or.h" #include "core/or/circuitstats.h" #include "app/config/config.h" -#include "app/config/confparse.h" +#include "lib/confmgt/confparse.h" #include "core/mainloop/mainloop.h" #include "core/mainloop/netstatus.h" #include "core/mainloop/connection.h" @@ -70,16 +70,13 @@ static config_abbrev_t state_abbrevs_[] = { * members with CONF_CHECK_VAR_TYPE. */ DUMMY_TYPECHECK_INSTANCE(or_state_t); -/*XXXX these next two are duplicates or near-duplicates from config.c */ -#define VAR(name,conftype,member,initvalue) \ - { name, CONFIG_TYPE_ ## conftype, offsetof(or_state_t, member), \ - initvalue CONF_TEST_MEMBERS(or_state_t, conftype, member) } -/** As VAR, but the option name and member name are the same. */ -#define V(member,conftype,initvalue) \ +#define VAR(varname,conftype,member,initvalue) \ + CONFIG_VAR_ETYPE(or_state_t, varname, conftype, member, 0, initvalue) +#define V(member,conftype,initvalue) \ VAR(#member, conftype, member, initvalue) /** Array of "state" variables saved to the ~/.tor/state file. */ -static config_var_t state_vars_[] = { +static const config_var_t state_vars_[] = { /* Remember to document these in state-contents.txt ! */ V(AccountingBytesReadInInterval, MEMUNIT, NULL), @@ -105,19 +102,19 @@ static config_var_t state_vars_[] = { V(HidServRevCounter, LINELIST, NULL), V(BWHistoryReadEnds, ISOTIME, NULL), - V(BWHistoryReadInterval, UINT, "900"), + V(BWHistoryReadInterval, POSINT, "900"), V(BWHistoryReadValues, CSV, ""), V(BWHistoryReadMaxima, CSV, ""), V(BWHistoryWriteEnds, ISOTIME, NULL), - V(BWHistoryWriteInterval, UINT, "900"), + V(BWHistoryWriteInterval, POSINT, "900"), V(BWHistoryWriteValues, CSV, ""), V(BWHistoryWriteMaxima, CSV, ""), V(BWHistoryDirReadEnds, ISOTIME, NULL), - V(BWHistoryDirReadInterval, UINT, "900"), + V(BWHistoryDirReadInterval, POSINT, "900"), V(BWHistoryDirReadValues, CSV, ""), V(BWHistoryDirReadMaxima, CSV, ""), V(BWHistoryDirWriteEnds, ISOTIME, NULL), - V(BWHistoryDirWriteInterval, UINT, "900"), + V(BWHistoryDirWriteInterval, POSINT, "900"), V(BWHistoryDirWriteValues, CSV, ""), V(BWHistoryDirWriteMaxima, CSV, ""), @@ -128,12 +125,12 @@ static config_var_t state_vars_[] = { V(LastRotatedOnionKey, ISOTIME, NULL), V(LastWritten, ISOTIME, NULL), - V(TotalBuildTimes, UINT, NULL), - V(CircuitBuildAbandonedCount, UINT, "0"), + V(TotalBuildTimes, POSINT, NULL), + V(CircuitBuildAbandonedCount, POSINT, "0"), VAR("CircuitBuildTimeBin", LINELIST_S, BuildtimeHistogram, NULL), VAR("BuildtimeHistogram", LINELIST_V, BuildtimeHistogram, NULL), - V(MinutesSinceUserActivity, UINT, NULL), + V(MinutesSinceUserActivity, POSINT, NULL), V(Dormant, AUTOBOOL, "auto"), END_OF_CONFIG_VARS @@ -148,31 +145,48 @@ static int or_state_validate_cb(void *old_options, void *options, void *default_options, int from_setconf, char **msg); -static void or_state_free_cb(void *state); - /** Magic value for or_state_t. */ #define OR_STATE_MAGIC 0x57A73f57 /** "Extra" variable in the state that receives lines we can't parse. This * lets us preserve options from versions of Tor newer than us. */ -static config_var_t state_extra_var = { - "__extra", CONFIG_TYPE_LINELIST, offsetof(or_state_t, ExtraLines), NULL - CONF_TEST_MEMBERS(or_state_t, LINELIST, ExtraLines) +static struct_member_t state_extra_var = { + .name = "__extra", + .type = CONFIG_TYPE_LINELIST, + .offset = offsetof(or_state_t, ExtraLines), }; /** Configuration format for or_state_t. */ static const config_format_t state_format = { sizeof(or_state_t), - OR_STATE_MAGIC, - offsetof(or_state_t, magic_), + { + "or_state_t", + OR_STATE_MAGIC, + offsetof(or_state_t, magic_), + }, state_abbrevs_, NULL, state_vars_, or_state_validate_cb, - or_state_free_cb, + NULL, &state_extra_var, + offsetof(or_state_t, substates_), }; +/* A global configuration manager for state-file objects */ +static config_mgr_t *state_mgr = NULL; + +/** Return the configuration manager for state-file objects. */ +static const config_mgr_t * +get_state_mgr(void) +{ + if (PREDICT_UNLIKELY(state_mgr == NULL)) { + state_mgr = config_mgr_new(&state_format); + config_mgr_freeze(state_mgr); + } + return state_mgr; +} + /** Persistent serialized state. */ static or_state_t *global_state = NULL; @@ -267,12 +281,6 @@ or_state_validate_cb(void *old_state, void *state, void *default_state, return or_state_validate(state, msg); } -static void -or_state_free_cb(void *state) -{ - or_state_free_(state); -} - /** Return 0 if every setting in <b>state</b> is reasonable, and a * permissible transition from <b>old_state</b>. Else warn and return -1. * Should have no side effects, except for normalizing the contents of @@ -297,7 +305,7 @@ or_state_set(or_state_t *new_state) char *err = NULL; int ret = 0; tor_assert(new_state); - config_free(&state_format, global_state); + config_free(get_state_mgr(), global_state); global_state = new_state; if (entry_guards_parse_state(global_state, 1, &err)<0) { log_warn(LD_GENERAL,"%s",err); @@ -360,9 +368,8 @@ or_state_save_broken(char *fname) STATIC or_state_t * or_state_new(void) { - or_state_t *new_state = tor_malloc_zero(sizeof(or_state_t)); - new_state->magic_ = OR_STATE_MAGIC; - config_init(&state_format, new_state); + or_state_t *new_state = config_new(get_state_mgr()); + config_init(get_state_mgr(), new_state); return new_state; } @@ -403,7 +410,7 @@ or_state_load(void) int assign_retval; if (config_get_lines(contents, &lines, 0)<0) goto done; - assign_retval = config_assign(&state_format, new_state, + assign_retval = config_assign(get_state_mgr(), new_state, lines, 0, &errmsg); config_free_lines(lines); if (assign_retval<0) @@ -430,7 +437,7 @@ or_state_load(void) or_state_save_broken(fname); tor_free(contents); - config_free(&state_format, new_state); + config_free(get_state_mgr(), new_state); new_state = or_state_new(); } else if (contents) { @@ -463,7 +470,7 @@ or_state_load(void) tor_free(fname); tor_free(contents); if (new_state) - config_free(&state_format, new_state); + config_free(get_state_mgr(), new_state); return r; } @@ -516,7 +523,7 @@ or_state_save(time_t now) tor_free(global_state->TorVersion); tor_asprintf(&global_state->TorVersion, "Tor %s", get_version()); - state = config_dump(&state_format, NULL, global_state, 1, 0); + state = config_dump(get_state_mgr(), NULL, global_state, 1, 0); format_local_iso_time(tbuf, now); tor_asprintf(&contents, "# Tor state file last generated on %s local time\n" @@ -726,7 +733,7 @@ or_state_free_(or_state_t *state) if (!state) return; - config_free(&state_format, state); + config_free(get_state_mgr(), state); } void @@ -734,4 +741,5 @@ or_state_free_all(void) { or_state_free(global_state); global_state = NULL; + config_mgr_free(state_mgr); } diff --git a/src/app/config/testnet.inc b/src/app/config/testnet.inc new file mode 100644 index 0000000000..0ed3c38627 --- /dev/null +++ b/src/app/config/testnet.inc @@ -0,0 +1,33 @@ +{ "DirAllowPrivateAddresses", "1" }, +{ "EnforceDistinctSubnets", "0" }, +{ "AssumeReachable", "1" }, +{ "AuthDirMaxServersPerAddr", "0" }, +{ "ClientBootstrapConsensusAuthorityDownloadInitialDelay", "0" }, +{ "ClientBootstrapConsensusFallbackDownloadInitialDelay", "0" }, +{ "ClientBootstrapConsensusAuthorityOnlyDownloadInitialDelay", "0" }, +{ "ClientDNSRejectInternalAddresses", "0" }, +{ "ClientRejectInternalAddresses", "0" }, +{ "CountPrivateBandwidth", "1" }, +{ "ExitPolicyRejectPrivate", "0" }, +{ "ExtendAllowPrivateAddresses", "1" }, +{ "V3AuthVotingInterval", "5 minutes" }, +{ "V3AuthVoteDelay", "20 seconds" }, +{ "V3AuthDistDelay", "20 seconds" }, +{ "TestingV3AuthInitialVotingInterval", "150 seconds" }, +{ "TestingV3AuthInitialVoteDelay", "20 seconds" }, +{ "TestingV3AuthInitialDistDelay", "20 seconds" }, +{ "TestingAuthDirTimeToLearnReachability", "0 minutes" }, +{ "TestingEstimatedDescriptorPropagationTime", "0 minutes" }, +{ "MinUptimeHidServDirectoryV2", "0 minutes" }, +{ "TestingServerDownloadInitialDelay", "0" }, +{ "TestingClientDownloadInitialDelay", "0" }, +{ "TestingServerConsensusDownloadInitialDelay", "0" }, +{ "TestingClientConsensusDownloadInitialDelay", "0" }, +{ "TestingBridgeDownloadInitialDelay", "10" }, +{ "TestingBridgeBootstrapDownloadInitialDelay", "0" }, +{ "TestingClientMaxIntervalWithoutRequest", "5 seconds" }, +{ "TestingDirConnectionMaxStall", "30 seconds" }, +{ "TestingEnableConnBwEvent", "1" }, +{ "TestingEnableCellStatsEvent", "1" }, +{ "RendPostPeriod", "2 minutes" }, +{ "___UsingTestNetworkDefaults", "1" }, diff --git a/src/app/main/main.c b/src/app/main/main.c index f3772f86e4..1901479eba 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -41,6 +41,7 @@ #include "feature/dircache/consdiffmgr.h" #include "feature/dirparse/routerparse.h" #include "feature/hibernate/hibernate.h" +#include "feature/hs/hs_dos.h" #include "feature/nodelist/authcert.h" #include "feature/nodelist/networkstatus.h" #include "feature/nodelist/routerlist.h" @@ -637,6 +638,10 @@ tor_init(int argc, char *argv[]) /* Initialize circuit padding to defaults+torrc until we get a consensus */ circpad_machines_init(); + /* Initialize hidden service DoS subsystem. We need to do this once the + * configuration object has been set because it can be accessed. */ + hs_dos_init(); + /* Initialize predicted ports list after loading options */ predicted_ports_init(); @@ -653,10 +658,6 @@ tor_init(int argc, char *argv[]) return -1; } - if (tor_init_libevent_rng() < 0) { - log_warn(LD_NET, "Problem initializing libevent RNG."); - } - /* Scan/clean unparseable descriptors; after reading config */ routerparse_init(); @@ -1256,6 +1257,8 @@ pubsub_connect(void) /* XXXX For each pubsub channel, its delivery strategy should be set at * this XXXX point, using tor_mainloop_set_delivery_strategy(). */ + tor_mainloop_set_delivery_strategy("orconn", DELIV_IMMEDIATE); + tor_mainloop_set_delivery_strategy("ocirc", DELIV_IMMEDIATE); } } diff --git a/src/app/main/ntmain.c b/src/app/main/ntmain.c index c5a8122030..de82eeb1de 100644 --- a/src/app/main/ntmain.c +++ b/src/app/main/ntmain.c @@ -615,6 +615,7 @@ nt_service_install(int argc, char **argv) &sidUse) == 0) { /* XXXX For some reason, the above test segfaults. Fix that. */ printf("User \"%s\" doesn't seem to exist.\n", user_acct); + tor_free(command); return -1; } else { printf("Will try to install service as user \"%s\".\n", user_acct); diff --git a/src/app/main/shutdown.c b/src/app/main/shutdown.c index cc0091a9ab..93d6351d1b 100644 --- a/src/app/main/shutdown.c +++ b/src/app/main/shutdown.c @@ -160,8 +160,6 @@ tor_free_all(int postfork) subsystems_shutdown(); - tor_libevent_free_all(); - /* Stuff in util.c and address.c*/ if (!postfork) { esc_router_info(NULL); diff --git a/src/app/main/subsystem_list.c b/src/app/main/subsystem_list.c index f595796232..1af9340c1a 100644 --- a/src/app/main/subsystem_list.c +++ b/src/app/main/subsystem_list.c @@ -25,6 +25,7 @@ #include "lib/time/time_sys.h" #include "lib/tls/tortls_sys.h" #include "lib/wallclock/wallclock_sys.h" +#include "lib/evloop/evloop_sys.h" #include "feature/dirauth/dirauth_sys.h" @@ -32,31 +33,38 @@ /** * Global list of the subsystems in Tor, in the order of their initialization. + * Want to know the exact level numbers? + * We'll implement a level dump command in #31614. **/ const subsys_fns_t *tor_subsystems[] = { - &sys_winprocess, /* -100 */ - &sys_torerr, /* -100 */ - &sys_wallclock, /* -99 */ - &sys_threads, /* -95 */ - &sys_logging, /* -90 */ - &sys_time, /* -90 */ - &sys_network, /* -90 */ - &sys_compress, /* -70 */ - &sys_crypto, /* -60 */ - &sys_tortls, /* -50 */ - &sys_process, /* -35 */ - - &sys_orconn_event, /* -33 */ - &sys_ocirc_event, /* -32 */ - &sys_btrack, /* -30 */ - - &sys_mainloop, /* 5 */ - &sys_or, /* 20 */ - - &sys_relay, /* 50 */ + &sys_winprocess, + &sys_torerr, + + &sys_wallclock, + &sys_threads, + &sys_logging, + + &sys_time, + &sys_network, + + &sys_compress, + &sys_crypto, + &sys_tortls, + &sys_process, + + &sys_orconn_event, + &sys_ocirc_event, + &sys_btrack, + + &sys_evloop, + + &sys_mainloop, + &sys_or, + + &sys_relay, #ifdef HAVE_MODULE_DIRAUTH - &sys_dirauth, /* 70 */ + &sys_dirauth, #endif }; diff --git a/src/config/torrc.minimal.in-staging b/src/config/torrc.minimal.in-staging index cb3adca35c..90bad7f7cc 100644 --- a/src/config/torrc.minimal.in-staging +++ b/src/config/torrc.minimal.in-staging @@ -88,6 +88,9 @@ ## yourself to make this work. #ORPort 443 NoListen #ORPort 127.0.0.1:9090 NoAdvertise +## If you want to listen on IPv6 your numeric address must be explictly +## between square brackets as follows. You must also listen on IPv4. +#ORPort [2001:DB8::1]:9050 ## The IP address or full DNS name for incoming connections to your ## relay. Leave commented out and Tor will guess. diff --git a/src/config/torrc.sample.in b/src/config/torrc.sample.in index 9d514e6bda..51e1c3af4b 100644 --- a/src/config/torrc.sample.in +++ b/src/config/torrc.sample.in @@ -88,6 +88,9 @@ ## yourself to make this work. #ORPort 443 NoListen #ORPort 127.0.0.1:9090 NoAdvertise +## If you want to listen on IPv6 your numeric address must be explictly +## between square brackets as follows. You must also listen on IPv4. +#ORPort [2001:DB8::1]:9050 ## The IP address or full DNS name for incoming connections to your ## relay. Leave commented out and Tor will guess. diff --git a/src/core/crypto/.may_include b/src/core/crypto/.may_include new file mode 100644 index 0000000000..5782a36797 --- /dev/null +++ b/src/core/crypto/.may_include @@ -0,0 +1,10 @@ +!advisory + +orconfig.h + +lib/crypt_ops/*.h +lib/ctime/*.h +lib/cc/*.h +lib/log/*.h + +core/crypto/*.h diff --git a/src/core/include.am b/src/core/include.am index 1a4b9fb8ab..9b4b251c81 100644 --- a/src/core/include.am +++ b/src/core/include.am @@ -9,7 +9,6 @@ endif # ADD_C_FILE: INSERT SOURCES HERE. LIBTOR_APP_A_SOURCES = \ src/app/config/config.c \ - src/app/config/confparse.c \ src/app/config/statefile.c \ src/app/main/main.c \ src/app/main/shutdown.c \ @@ -117,6 +116,7 @@ LIBTOR_APP_A_SOURCES = \ src/feature/hs/hs_config.c \ src/feature/hs/hs_control.c \ src/feature/hs/hs_descriptor.c \ + src/feature/hs/hs_dos.c \ src/feature/hs/hs_ident.c \ src/feature/hs/hs_intropoint.c \ src/feature/hs/hs_service.c \ @@ -213,7 +213,6 @@ src_core_libtor_app_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) # ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/app/config/config.h \ - src/app/config/confparse.h \ src/app/config/or_options_st.h \ src/app/config/or_state_st.h \ src/app/config/statefile.h \ @@ -374,6 +373,7 @@ noinst_HEADERS += \ src/feature/hs/hs_config.h \ src/feature/hs/hs_control.h \ src/feature/hs/hs_descriptor.h \ + src/feature/hs/hs_dos.h \ src/feature/hs/hs_ident.h \ src/feature/hs/hs_intropoint.h \ src/feature/hs/hs_service.h \ @@ -435,9 +435,10 @@ noinst_HEADERS += \ src/feature/stats/rephist.h \ src/feature/stats/predict_ports.h -noinst_HEADERS += \ - src/app/config/auth_dirs.inc \ - src/app/config/fallback_dirs.inc +noinst_HEADERS += \ + src/app/config/auth_dirs.inc \ + src/app/config/fallback_dirs.inc \ + src/app/config/testnet.inc # This may someday want to be an installed file? noinst_HEADERS += src/feature/api/tor_api.h diff --git a/src/core/mainloop/.may_include b/src/core/mainloop/.may_include new file mode 100644 index 0000000000..79d6a130a4 --- /dev/null +++ b/src/core/mainloop/.may_include @@ -0,0 +1,20 @@ +!advisory + +orconfig.h + +lib/container/*.h +lib/dispatch/*.h +lib/evloop/*.h +lib/pubsub/*.h +lib/subsys/*.h +lib/buf/*.h +lib/crypt_ops/*.h +lib/err/*.h +lib/tls/*.h +lib/net/*.h +lib/evloop/*.h +lib/geoip/*.h +lib/sandbox/*.h +lib/compress/*.h + +core/mainloop/*.h
\ No newline at end of file diff --git a/src/core/mainloop/connection.c b/src/core/mainloop/connection.c index 372494a577..6094f33e4d 100644 --- a/src/core/mainloop/connection.c +++ b/src/core/mainloop/connection.c @@ -1198,7 +1198,7 @@ make_win32_socket_exclusive(tor_socket_t sock) return -1; } return 0; -#else /* !(defined(SO_EXCLUSIVEADDRUSE)) */ +#else /* !defined(SO_EXCLUSIVEADDRUSE) */ (void) sock; return 0; #endif /* defined(SO_EXCLUSIVEADDRUSE) */ @@ -3963,7 +3963,7 @@ update_send_buffer_size(tor_socket_t sock) &isb, sizeof(isb), &bytesReturned, NULL, NULL)) { setsockopt(sock, SOL_SOCKET, SO_SNDBUF, (const char*)&isb, sizeof(isb)); } -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ (void) sock; #endif /* defined(_WIN32) */ } diff --git a/src/core/or/.may_include b/src/core/or/.may_include new file mode 100644 index 0000000000..5173e8a2b6 --- /dev/null +++ b/src/core/or/.may_include @@ -0,0 +1,38 @@ +!advisory + +orconfig.h + +lib/arch/*.h +lib/buf/*.h +lib/cc/*.h +lib/compress/*.h +lib/container/*.h +lib/crypt_ops/*.h +lib/ctime/*.h +lib/defs/*.h +lib/encoding/*.h +lib/err/*.h +lib/evloop/*.h +lib/fs/*.h +lib/geoip/*.h +lib/intmath/*.h +lib/log/*.h +lib/malloc/*.h +lib/math/*.h +lib/net/*.h +lib/pubsub/*.h +lib/string/*.h +lib/subsys/*.h +lib/test/*.h +lib/testsupport/*.h +lib/thread/*.h +lib/time/*.h +lib/tls/*.h +lib/wallclock/*.h + +trunnel/*.h + +core/mainloop/*.h +core/proto/*.h +core/crypto/*.h +core/or/*.h
\ No newline at end of file diff --git a/src/core/or/channeltls.c b/src/core/or/channeltls.c index 25f3baf924..e9497651ef 100644 --- a/src/core/or/channeltls.c +++ b/src/core/or/channeltls.c @@ -1027,6 +1027,16 @@ channel_tls_time_process_cell(cell_t *cell, channel_tls_t *chan, int *time, } #endif /* defined(KEEP_TIMING_STATS) */ +#ifdef KEEP_TIMING_STATS +#define PROCESS_CELL(tp, cl, cn) STMT_BEGIN { \ + ++num ## tp; \ + channel_tls_time_process_cell(cl, cn, & tp ## time , \ + channel_tls_process_ ## tp ## _cell); \ + } STMT_END +#else /* !defined(KEEP_TIMING_STATS) */ +#define PROCESS_CELL(tp, cl, cn) channel_tls_process_ ## tp ## _cell(cl, cn) +#endif /* defined(KEEP_TIMING_STATS) */ + /** * Handle an incoming cell on a channel_tls_t. * @@ -1046,16 +1056,6 @@ channel_tls_handle_cell(cell_t *cell, or_connection_t *conn) channel_tls_t *chan; int handshaking; -#ifdef KEEP_TIMING_STATS -#define PROCESS_CELL(tp, cl, cn) STMT_BEGIN { \ - ++num ## tp; \ - channel_tls_time_process_cell(cl, cn, & tp ## time , \ - channel_tls_process_ ## tp ## _cell); \ - } STMT_END -#else /* !(defined(KEEP_TIMING_STATS)) */ -#define PROCESS_CELL(tp, cl, cn) channel_tls_process_ ## tp ## _cell(cl, cn) -#endif /* defined(KEEP_TIMING_STATS) */ - tor_assert(cell); tor_assert(conn); @@ -1073,7 +1073,8 @@ channel_tls_handle_cell(cell_t *cell, or_connection_t *conn) return; /* Reject all but VERSIONS and NETINFO when handshaking. */ - /* (VERSIONS should actually be impossible; it's variable-length.) */ + /* (VERSIONS actually indicates a protocol warning: it's variable-length, + * so if it reaches this function, we're on a v1 connection.) */ if (handshaking && cell->command != CELL_VERSIONS && cell->command != CELL_NETINFO) { log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, @@ -1328,6 +1329,8 @@ channel_tls_handle_var_cell(var_cell_t *var_cell, or_connection_t *conn) } } +#undef PROCESS_CELL + /** * Update channel marks after connection_or.c has changed an address. * diff --git a/src/core/or/circuitbuild.c b/src/core/or/circuitbuild.c index 3a4e729429..1daf468715 100644 --- a/src/core/or/circuitbuild.c +++ b/src/core/or/circuitbuild.c @@ -30,7 +30,7 @@ #include "core/or/or.h" #include "app/config/config.h" -#include "app/config/confparse.h" +#include "lib/confmgt/confparse.h" #include "core/crypto/hs_ntor.h" #include "core/crypto/onion_crypto.h" #include "core/crypto/onion_fast.h" @@ -522,14 +522,13 @@ origin_circuit_get_guard_state(origin_circuit_t *circ) static void circuit_chan_publish(const origin_circuit_t *circ, const channel_t *chan) { - ocirc_event_msg_t msg; + ocirc_chan_msg_t *msg = tor_malloc(sizeof(*msg)); - msg.type = OCIRC_MSGTYPE_CHAN; - msg.u.chan.gid = circ->global_identifier; - msg.u.chan.chan = chan->global_identifier; - msg.u.chan.onehop = circ->build_state->onehop_tunnel; + msg->gid = circ->global_identifier; + msg->chan = chan->global_identifier; + msg->onehop = circ->build_state->onehop_tunnel; - ocirc_event_publish(&msg); + ocirc_chan_publish(msg); } /** Start establishing the first hop of our circuit. Figure out what diff --git a/src/core/or/circuitlist.c b/src/core/or/circuitlist.c index ebbe7f0824..9ee9f93c99 100644 --- a/src/core/or/circuitlist.c +++ b/src/core/or/circuitlist.c @@ -496,17 +496,16 @@ int circuit_event_status(origin_circuit_t *circ, circuit_status_event_t tp, int reason_code) { - ocirc_event_msg_t msg; + ocirc_cevent_msg_t *msg = tor_malloc(sizeof(*msg)); tor_assert(circ); - msg.type = OCIRC_MSGTYPE_CEVENT; - msg.u.cevent.gid = circ->global_identifier; - msg.u.cevent.evtype = tp; - msg.u.cevent.reason = reason_code; - msg.u.cevent.onehop = circ->build_state->onehop_tunnel; + msg->gid = circ->global_identifier; + msg->evtype = tp; + msg->reason = reason_code; + msg->onehop = circ->build_state->onehop_tunnel; - ocirc_event_publish(&msg); + ocirc_cevent_publish(msg); return control_event_circuit_status(circ, tp, reason_code); } @@ -514,26 +513,25 @@ circuit_event_status(origin_circuit_t *circ, circuit_status_event_t tp, * Helper function to publish a state change message * * circuit_set_state() calls this to notify subscribers about a change - * of the state of an origin circuit. + * of the state of an origin circuit. @a circ must be an origin + * circuit. **/ static void circuit_state_publish(const circuit_t *circ) { - ocirc_event_msg_t msg; + ocirc_state_msg_t *msg = tor_malloc(sizeof(*msg)); const origin_circuit_t *ocirc; - if (!CIRCUIT_IS_ORIGIN(circ)) - return; + tor_assert(CIRCUIT_IS_ORIGIN(circ)); ocirc = CONST_TO_ORIGIN_CIRCUIT(circ); /* Only inbound OR circuits can be in this state, not origin circuits. */ tor_assert(circ->state != CIRCUIT_STATE_ONIONSKIN_PENDING); - msg.type = OCIRC_MSGTYPE_STATE; - msg.u.state.gid = ocirc->global_identifier; - msg.u.state.state = circ->state; - msg.u.state.onehop = ocirc->build_state->onehop_tunnel; + msg->gid = ocirc->global_identifier; + msg->state = circ->state; + msg->onehop = ocirc->build_state->onehop_tunnel; - ocirc_event_publish(&msg); + ocirc_state_publish(msg); } /** Change the state of <b>circ</b> to <b>state</b>, adding it to or removing @@ -565,7 +563,8 @@ circuit_set_state(circuit_t *circ, uint8_t state) if (state == CIRCUIT_STATE_GUARD_WAIT || state == CIRCUIT_STATE_OPEN) tor_assert(!circ->n_chan_create_cell); circ->state = state; - circuit_state_publish(circ); + if (CIRCUIT_IS_ORIGIN(circ)) + circuit_state_publish(circ); } /** Append to <b>out</b> all circuits in state CHAN_WAIT waiting for diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index a62cdcf9e6..99c68d5f6b 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -17,7 +17,7 @@ * Each padding type is described by a state machine (circpad_machine_spec_t), * which is also referred as a "padding machine" in this file. Currently, * these state machines are hardcoded in the source code (e.g. see - * circpad_circ_client_machine_init()), but in the future we will be able to + * circpad_machines_init()), but in the future we will be able to * serialize them in the torrc or the consensus. * * As specified by prop#254, clients can negotiate padding with relays by using @@ -564,11 +564,12 @@ circpad_distribution_sample_iat_delay(const circpad_state_t *state, } /** - * Sample an expected time-until-next-packet delay from the histogram. + * Sample an expected time-until-next-packet delay from the histogram or + * probability distribution. * - * The bin is chosen with probability proportional to the number - * of tokens in each bin, and then a time value is chosen uniformly from - * that bin's [start,end) time range. + * A bin of the histogram is chosen with probability proportional to the number + * of tokens in each bin, and then a time value is chosen uniformly from that + * bin's [start,end) time range. */ STATIC circpad_delay_t circpad_machine_sample_delay(circpad_machine_runtime_t *mi) @@ -667,12 +668,7 @@ circpad_machine_sample_delay(circpad_machine_runtime_t *mi) /** * Sample a value from the specified probability distribution. * - * This performs inverse transform sampling - * (https://en.wikipedia.org/wiki/Inverse_transform_sampling). - * - * XXX: These formulas were taken verbatim. Need a floating wizard - * to check them for catastropic cancellation and other issues (teor?). - * Also: is 32bits of double from [0.0,1.0) enough? + * Uses functions from src/lib/math/prob_distr.c . */ static double circpad_distribution_sample(circpad_distribution_t dist) @@ -756,6 +752,8 @@ circpad_distribution_sample(circpad_distribution_t dist) /** * Find the index of the first bin whose upper bound is * greater than the target, and that has tokens remaining. + * + * Used for histograms with token removal. */ static circpad_hist_index_t circpad_machine_first_higher_index(const circpad_machine_runtime_t *mi, @@ -778,6 +776,8 @@ circpad_machine_first_higher_index(const circpad_machine_runtime_t *mi, /** * Find the index of the first bin whose lower bound is lower or equal to * <b>target_bin_usec</b>, and that still has tokens remaining. + * + * Used for histograms with token removal. */ static circpad_hist_index_t circpad_machine_first_lower_index(const circpad_machine_runtime_t *mi, @@ -799,6 +799,8 @@ circpad_machine_first_lower_index(const circpad_machine_runtime_t *mi, /** * Remove a token from the first non-empty bin whose upper bound is * greater than the target. + * + * Used for histograms with token removal. */ STATIC void circpad_machine_remove_higher_token(circpad_machine_runtime_t *mi, @@ -820,6 +822,8 @@ circpad_machine_remove_higher_token(circpad_machine_runtime_t *mi, /** * Remove a token from the first non-empty bin whose upper bound is * lower than the target. + * + * Used for histograms with token removal. */ STATIC void circpad_machine_remove_lower_token(circpad_machine_runtime_t *mi, @@ -849,6 +853,8 @@ circpad_machine_remove_lower_token(circpad_machine_runtime_t *mi, * midpoint. * * If it is false, use bin index distance only. + * + * Used for histograms with token removal. */ STATIC void circpad_machine_remove_closest_token(circpad_machine_runtime_t *mi, @@ -931,6 +937,8 @@ circpad_machine_remove_closest_token(circpad_machine_runtime_t *mi, * Remove a token from the exact bin corresponding to the target. * * If it is empty, do nothing. + * + * Used for histograms with token removal. */ static void circpad_machine_remove_exact(circpad_machine_runtime_t *mi, @@ -1365,7 +1373,7 @@ circpad_machine_reached_padding_limit(circpad_machine_runtime_t *mi) /* If circpad_max_global_padding_pct is non-zero, and we've * sent more than the global padding cell limit, then check our - * gloabl tor process percentage limit on padding. */ + * global tor process percentage limit on padding. */ if (circpad_global_max_padding_percent && circpad_global_padding_sent >= circpad_global_allowed_cells) { uint64_t total_cells = circpad_global_padding_sent + @@ -1511,7 +1519,7 @@ circpad_machine_schedule_padding,(circpad_machine_runtime_t *mi)) /** * If the machine transitioned to the END state, we need * to check to see if it wants us to shut it down immediately. - * If it does, then we need to send the appropate negotation commands + * If it does, then we need to send the appropiate negotiation commands * depending on which side it is. * * After this function is called, mi may point to freed memory. Do @@ -1534,7 +1542,7 @@ circpad_machine_spec_transitioned_to_end(circpad_machine_runtime_t *mi) * we can handle the case where this machine started while it was * the only machine that matched conditions, but *since* then more * "higher ranking" machines now match the conditions, and would - * be given a chance to take precidence over this one in + * be given a chance to take precedence over this one in * circpad_add_matching_machines(). * * Returning to START or waiting forever in END would not give those @@ -1662,7 +1670,7 @@ circpad_estimate_circ_rtt_on_received(circuit_t *circ, if (CIRCUIT_IS_ORIGIN(circ) || mi->stop_rtt_update) return; - /* If we already have a last receieved packet time, that means we + /* If we already have a last received packet time, that means we * did not get a response before this packet. The RTT estimate * only makes sense if we do not have multiple packets on the * wire, so stop estimating if this is the second packet @@ -1807,6 +1815,61 @@ circpad_cell_event_nonpadding_sent(circuit_t *on_circ) } FOR_EACH_ACTIVE_CIRCUIT_MACHINE_END; } +/** Check if this cell or circuit are related to circuit padding and handle + * them if so. Return 0 if the cell was handled in this subsystem and does + * not need any other consideration, otherwise return 1. + */ +int +circpad_check_received_cell(cell_t *cell, circuit_t *circ, + crypt_path_t *layer_hint, + const relay_header_t *rh) +{ + /* First handle the padding commands, since we want to ignore any other + * commands if this circuit is padding-specific. */ + switch (rh->command) { + case RELAY_COMMAND_DROP: + /* Already examined in circpad_deliver_recognized_relay_cell_events */ + return 0; + case RELAY_COMMAND_PADDING_NEGOTIATE: + circpad_handle_padding_negotiate(circ, cell); + return 0; + case RELAY_COMMAND_PADDING_NEGOTIATED: + if (circpad_handle_padding_negotiated(circ, cell, layer_hint) == 0) + circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh->length); + return 0; + } + + /* If this is a padding circuit we don't need to parse any other commands + * than the padding ones. Just drop them to the floor. + * + * Note: we deliberately do not call circuit_read_valid_data() here. The + * vanguards addon (specifically the 'bandguards' component's dropped cell + * detection) will thus close this circuit, as it would for any other + * unexpected cell. However, default tor will *not* close the circuit. + * + * This is intentional. We are not yet certain that is it optimal to keep + * padding circuits open in cases like these, rather than closing them. + * We suspect that continuing to pad is optimal against a passive classifier, + * but as soon as the adversary is active (even as a client adversary) this + * might change. + * + * So as a way forward, we log the cell command and circuit number, to + * help us enumerate the most common instances of this in testing with + * vanguards, to see which are common enough to verify and handle + * properly. + * - Mike + */ + if (circ->purpose == CIRCUIT_PURPOSE_C_CIRCUIT_PADDING) { + log_fn(LOG_PROTOCOL_WARN, LD_CIRC, + "Ignored cell (%d) that arrived in padding circuit " + " %u.", rh->command, CIRCUIT_IS_ORIGIN(circ) ? + TO_ORIGIN_CIRCUIT(circ)->global_identifier : 0); + return 0; + } + + return 1; +} + /** * A "non-padding" cell has been received by this endpoint. React * according to any padding state machines on the circuit. @@ -2332,7 +2395,7 @@ circpad_deliver_sent_relay_cell_events(circuit_t *circ, /* Optimization: The event for RELAY_COMMAND_DROP is sent directly * from circpad_send_padding_cell_for_callback(). This is to avoid * putting a cell_t and a relay_header_t on the stack repeatedly - * if we decide to send a long train of padidng cells back-to-back + * if we decide to send a long train of padding cells back-to-back * with 0 delay. So we do nothing here. */ return; } else { diff --git a/src/core/or/circuitpadding.h b/src/core/or/circuitpadding.h index 3cf40e11db..e9eb32c618 100644 --- a/src/core/or/circuitpadding.h +++ b/src/core/or/circuitpadding.h @@ -51,7 +51,7 @@ typedef enum { CIRCPAD_EVENT_INFINITY = 4, /* All histogram bins are empty (we are out of tokens) */ CIRCPAD_EVENT_BINS_EMPTY = 5, - /* just a counter of the events above */ + /* This state has used up its cell count */ CIRCPAD_EVENT_LENGTH_COUNT = 6 } circpad_event_t; #define CIRCPAD_NUM_EVENTS ((int)CIRCPAD_EVENT_LENGTH_COUNT+1) @@ -79,7 +79,7 @@ typedef uint32_t circpad_delay_t; * An infinite padding cell delay means don't schedule any padding -- * simply wait until a different event triggers a transition. * - * This means that the maximum delay we can scedule is UINT32_MAX-1 + * This means that the maximum delay we can schedule is UINT32_MAX-1 * microseconds, or about 4300 seconds (1.25 hours). * XXX: Is this enough if we want to simulate light, intermittent * activity on an onion service? @@ -106,8 +106,8 @@ typedef uint32_t circpad_delay_t; * * If any of these elements is set, then the circuit will be tested against * that specific condition. If an element is unset, then we don't test it. - * (E.g. If neither NO_STREAMS or STREAMS are set, then we will not care - * whether a circuit has streams attached when we apply a state machine) + * (E.g., if neither NO_STREAMS or STREAMS are set, then we will not care + * whether a circuit has streams attached when we apply a state machine.) * * The helper function circpad_circuit_state() converts circuit state * flags into this more compact representation. @@ -255,8 +255,9 @@ typedef struct circpad_distribution_t { typedef uint16_t circpad_statenum_t; #define CIRCPAD_STATENUM_MAX (UINT16_MAX) -/** A histogram is used to sample padding delays given a machine state. This - * constant defines the maximum histogram width (i.e. the max number of bins). +/** A histogram can be used to sample padding delays given a machine state. + * This constant defines the maximum histogram width (i.e. the max number of + * bins). * * The current limit is arbitrary and could be raised if there is a need, * however too many bins will be hard to serialize in the future. @@ -275,10 +276,10 @@ typedef uint16_t circpad_statenum_t; * happen. The mutable information that gets updated in runtime are carried in * a circpad_machine_runtime_t. * - * This struct describes the histograms and parameters of a single - * state in the adaptive padding machine. Instances of this struct - * exist in global circpad machine definitions that come from torrc - * or the consensus. + * This struct describes the histograms and/or probability distributions, as + * well as parameters of a single state in the adaptive padding machine. + * Instances of this struct exist in global circpad machine definitions that + * come from torrc or the consensus. */ typedef struct circpad_state_t { /** @@ -733,6 +734,10 @@ bool circpad_padding_negotiated(struct circuit_t *circ, circpad_purpose_mask_t circpad_circ_purpose_to_mask(uint8_t circ_purpose); +int circpad_check_received_cell(cell_t *cell, circuit_t *circ, + crypt_path_t *layer_hint, + const relay_header_t *rh); + MOCK_DECL(circpad_decision_t, circpad_machine_schedule_padding,(circpad_machine_runtime_t *)); diff --git a/src/core/or/circuitpadding_machines.c b/src/core/or/circuitpadding_machines.c index 75d2614aca..7220d657fc 100644 --- a/src/core/or/circuitpadding_machines.c +++ b/src/core/or/circuitpadding_machines.c @@ -155,7 +155,6 @@ circpad_machine_relay_hide_intro_circuits(smartlist_t *machines_sl) relay_machine->name = "relay_ip_circ"; relay_machine->conditions.state_mask = CIRCPAD_CIRC_OPENED; - relay_machine->target_hopnum = 2; /* This is a relay-side machine */ relay_machine->is_origin_side = 0; @@ -387,7 +386,6 @@ circpad_machine_relay_hide_rend_circuits(smartlist_t *machines_sl) /* Only pad after the circuit has been built and pad to the middle */ relay_machine->conditions.min_hops = 2; relay_machine->conditions.state_mask = CIRCPAD_CIRC_OPENED; - relay_machine->target_hopnum = 2; /* This is a relay-side machine */ relay_machine->is_origin_side = 0; @@ -408,7 +406,7 @@ circpad_machine_relay_hide_rend_circuits(smartlist_t *machines_sl) /* OBFUSCATE_CIRC_SETUP -> END transition when we send our first * padding packet and/or hit the state length (the state length is 1). */ relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP]. - next_state[CIRCPAD_EVENT_PADDING_RECV] = CIRCPAD_STATE_END; + next_state[CIRCPAD_EVENT_PADDING_SENT] = CIRCPAD_STATE_END; relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP]. next_state[CIRCPAD_EVENT_LENGTH_COUNT] = CIRCPAD_STATE_END; diff --git a/src/core/or/circuitstats.c b/src/core/or/circuitstats.c index 03eea1d779..7a7f3ca600 100644 --- a/src/core/or/circuitstats.c +++ b/src/core/or/circuitstats.c @@ -29,7 +29,7 @@ #include "core/or/circuitbuild.h" #include "core/or/circuitstats.h" #include "app/config/config.h" -#include "app/config/confparse.h" +#include "lib/confmgt/confparse.h" #include "feature/control/control_events.h" #include "lib/crypt_ops/crypto_rand.h" #include "core/mainloop/mainloop.h" diff --git a/src/core/or/circuituse.c b/src/core/or/circuituse.c index 18b419e99d..606c5e2dd2 100644 --- a/src/core/or/circuituse.c +++ b/src/core/or/circuituse.c @@ -2533,8 +2533,7 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn, circ->rend_data = rend_data_dup(edge_conn->rend_data); } else if (edge_conn->hs_ident) { circ->hs_ident = - hs_ident_circuit_new(&edge_conn->hs_ident->identity_pk, - HS_IDENT_CIRCUIT_INTRO); + hs_ident_circuit_new(&edge_conn->hs_ident->identity_pk); } if (circ->base_.purpose == CIRCUIT_PURPOSE_C_ESTABLISH_REND && circ->base_.state == CIRCUIT_STATE_OPEN) diff --git a/src/core/or/command.c b/src/core/or/command.c index 77e5447ce3..1c97437769 100644 --- a/src/core/or/command.c +++ b/src/core/or/command.c @@ -182,7 +182,7 @@ command_process_cell(channel_t *chan, cell_t *cell) command_time_process_cell(cl, cn, & tp ## time , \ command_process_ ## tp ## _cell); \ } STMT_END -#else /* !(defined(KEEP_TIMING_STATS)) */ +#else /* !defined(KEEP_TIMING_STATS) */ #define PROCESS_CELL(tp, cl, cn) command_process_ ## tp ## _cell(cl, cn) #endif /* defined(KEEP_TIMING_STATS) */ diff --git a/src/core/or/connection_edge.c b/src/core/or/connection_edge.c index 369a14e889..5f1664d286 100644 --- a/src/core/or/connection_edge.c +++ b/src/core/or/connection_edge.c @@ -1224,7 +1224,7 @@ connection_ap_rescan_and_attach_pending(void) entry_conn->marked_pending_circ_line = 0; \ entry_conn->marked_pending_circ_file = 0; \ } while (0) -#else /* !(defined(DEBUGGING_17659)) */ +#else /* !defined(DEBUGGING_17659) */ #define UNMARK() do { } while (0) #endif /* defined(DEBUGGING_17659) */ @@ -3838,6 +3838,7 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ) if (! bcell.is_begindir) { /* Steal reference */ + tor_assert(bcell.address); address = bcell.address; port = bcell.port; diff --git a/src/core/or/connection_or.c b/src/core/or/connection_or.c index 830e09fd54..4c93351e31 100644 --- a/src/core/or/connection_or.c +++ b/src/core/or/connection_or.c @@ -414,13 +414,12 @@ void connection_or_event_status(or_connection_t *conn, or_conn_status_event_t tp, int reason) { - orconn_event_msg_t msg; + orconn_status_msg_t *msg = tor_malloc(sizeof(*msg)); - msg.type = ORCONN_MSGTYPE_STATUS; - msg.u.status.gid = conn->base_.global_identifier; - msg.u.status.status = tp; - msg.u.status.reason = reason; - orconn_event_publish(&msg); + msg->gid = conn->base_.global_identifier; + msg->status = tp; + msg->reason = reason; + orconn_status_publish(msg); control_event_or_conn_status(conn, tp, reason); } @@ -433,26 +432,25 @@ connection_or_event_status(or_connection_t *conn, or_conn_status_event_t tp, static void connection_or_state_publish(const or_connection_t *conn, uint8_t state) { - orconn_event_msg_t msg; + orconn_state_msg_t *msg = tor_malloc(sizeof(*msg)); - msg.type = ORCONN_MSGTYPE_STATE; - msg.u.state.gid = conn->base_.global_identifier; + msg->gid = conn->base_.global_identifier; if (conn->is_pt) { /* Do extra decoding because conn->proxy_type indicates the proxy * protocol that tor uses to talk with the transport plugin, * instead of PROXY_PLUGGABLE. */ tor_assert_nonfatal(conn->proxy_type != PROXY_NONE); - msg.u.state.proxy_type = PROXY_PLUGGABLE; + msg->proxy_type = PROXY_PLUGGABLE; } else { - msg.u.state.proxy_type = conn->proxy_type; + msg->proxy_type = conn->proxy_type; } - msg.u.state.state = state; + msg->state = state; if (conn->chan) { - msg.u.state.chan = TLS_CHAN_TO_BASE(conn->chan)->global_identifier; + msg->chan = TLS_CHAN_TO_BASE(conn->chan)->global_identifier; } else { - msg.u.state.chan = 0; + msg->chan = 0; } - orconn_event_publish(&msg); + orconn_state_publish(msg); } /** Call this to change or_connection_t states, so the owning channel_tls_t can diff --git a/src/core/or/ocirc_event.c b/src/core/or/ocirc_event.c index 4a6fc748c9..3cb9147134 100644 --- a/src/core/or/ocirc_event.c +++ b/src/core/or/ocirc_event.c @@ -26,59 +26,103 @@ #include "core/or/origin_circuit_st.h" #include "lib/subsys/subsys.h" -/** List of subscribers */ -static smartlist_t *ocirc_event_rcvrs; +DECLARE_PUBLISH(ocirc_state); +DECLARE_PUBLISH(ocirc_chan); +DECLARE_PUBLISH(ocirc_cevent); + +static void +ocirc_event_free(msg_aux_data_t u) +{ + tor_free_(u.ptr); +} + +static char * +ocirc_state_fmt(msg_aux_data_t u) +{ + ocirc_state_msg_t *msg = (ocirc_state_msg_t *)u.ptr; + char *s = NULL; + + tor_asprintf(&s, "<gid=%"PRIu32" state=%d onehop=%d>", + msg->gid, msg->state, msg->onehop); + return s; +} + +static char * +ocirc_chan_fmt(msg_aux_data_t u) +{ + ocirc_chan_msg_t *msg = (ocirc_chan_msg_t *)u.ptr; + char *s = NULL; + + tor_asprintf(&s, "<gid=%"PRIu32" chan=%"PRIu64" onehop=%d>", + msg->gid, msg->chan, msg->onehop); + return s; +} + +static char * +ocirc_cevent_fmt(msg_aux_data_t u) +{ + ocirc_cevent_msg_t *msg = (ocirc_cevent_msg_t *)u.ptr; + char *s = NULL; + + tor_asprintf(&s, "<gid=%"PRIu32" evtype=%d reason=%d onehop=%d>", + msg->gid, msg->evtype, msg->reason, msg->onehop); + return s; +} + +static dispatch_typefns_t ocirc_state_fns = { + .free_fn = ocirc_event_free, + .fmt_fn = ocirc_state_fmt, +}; + +static dispatch_typefns_t ocirc_chan_fns = { + .free_fn = ocirc_event_free, + .fmt_fn = ocirc_chan_fmt, +}; + +static dispatch_typefns_t ocirc_cevent_fns = { + .free_fn = ocirc_event_free, + .fmt_fn = ocirc_cevent_fmt, +}; -/** Initialize subscriber list */ static int -ocirc_event_init(void) +ocirc_add_pubsub(struct pubsub_connector_t *connector) { - ocirc_event_rcvrs = smartlist_new(); + if (DISPATCH_REGISTER_TYPE(connector, ocirc_state, ô_state_fns)) + return -1; + if (DISPATCH_REGISTER_TYPE(connector, ocirc_chan, ô_chan_fns)) + return -1; + if (DISPATCH_REGISTER_TYPE(connector, ocirc_cevent, ô_cevent_fns)) + return -1; + if (DISPATCH_ADD_PUB(connector, ocirc, ocirc_state)) + return -1; + if (DISPATCH_ADD_PUB(connector, ocirc, ocirc_chan)) + return -1; + if (DISPATCH_ADD_PUB(connector, ocirc, ocirc_cevent)) + return -1; return 0; } -/** Free subscriber list */ -static void -ocirc_event_fini(void) +void +ocirc_state_publish(ocirc_state_msg_t *msg) { - smartlist_free(ocirc_event_rcvrs); + PUBLISH(ocirc_state, msg); } -/** - * Subscribe to messages about origin circuit events - * - * Register a callback function to receive messages about origin - * circuits. The publisher calls this function synchronously. - **/ void -ocirc_event_subscribe(ocirc_event_rcvr_t fn) +ocirc_chan_publish(ocirc_chan_msg_t *msg) { - tor_assert(fn); - /* Don't duplicate subscriptions. */ - if (smartlist_contains(ocirc_event_rcvrs, fn)) - return; - - smartlist_add(ocirc_event_rcvrs, fn); + PUBLISH(ocirc_chan, msg); } -/** - * Publish a message about OR connection events - * - * This calls the subscriber receiver function synchronously. - **/ void -ocirc_event_publish(const ocirc_event_msg_t *msg) +ocirc_cevent_publish(ocirc_cevent_msg_t *msg) { - SMARTLIST_FOREACH_BEGIN(ocirc_event_rcvrs, ocirc_event_rcvr_t, fn) { - tor_assert(fn); - (*fn)(msg); - } SMARTLIST_FOREACH_END(fn); + PUBLISH(ocirc_cevent, msg); } const subsys_fns_t sys_ocirc_event = { .name = "ocirc_event", .supported = true, .level = -32, - .initialize = ocirc_event_init, - .shutdown = ocirc_event_fini, + .add_pubsub = ocirc_add_pubsub, }; diff --git a/src/core/or/ocirc_event.h b/src/core/or/ocirc_event.h index 59ec9e27cb..8e9494874f 100644 --- a/src/core/or/ocirc_event.h +++ b/src/core/or/ocirc_event.h @@ -12,6 +12,7 @@ #include <stdbool.h> #include "lib/cc/torint.h" +#include "lib/pubsub/pubsub.h" /** Used to indicate the type of a circuit event passed to the controller. * The various types are defined in control-spec.txt */ @@ -30,6 +31,8 @@ typedef struct ocirc_state_msg_t { bool onehop; /**< one-hop circuit? */ } ocirc_state_msg_t; +DECLARE_MESSAGE(ocirc_state, ocirc_state, ocirc_state_msg_t *); + /** * Message when a channel gets associated to a circuit. * @@ -44,6 +47,8 @@ typedef struct ocirc_chan_msg_t { bool onehop; /**< one-hop circuit? */ } ocirc_chan_msg_t; +DECLARE_MESSAGE(ocirc_chan, ocirc_chan, ocirc_chan_msg_t *); + /** * Message for origin circuit status event * @@ -56,34 +61,12 @@ typedef struct ocirc_cevent_msg_t { bool onehop; /**< one-hop circuit? */ } ocirc_cevent_msg_t; -/** Discriminant values for origin circuit event message */ -typedef enum ocirc_msgtype_t { - OCIRC_MSGTYPE_STATE, - OCIRC_MSGTYPE_CHAN, - OCIRC_MSGTYPE_CEVENT, -} ocirc_msgtype_t; - -/** Discriminated union for the actual message */ -typedef struct ocirc_event_msg_t { - int type; - union { - ocirc_state_msg_t state; - ocirc_chan_msg_t chan; - ocirc_cevent_msg_t cevent; - } u; -} ocirc_event_msg_t; - -/** - * Receiver function pointer for origin circuit subscribers - * - * This function gets called synchronously by the publisher. - **/ -typedef void (*ocirc_event_rcvr_t)(const ocirc_event_msg_t *); - -void ocirc_event_subscribe(ocirc_event_rcvr_t fn); +DECLARE_MESSAGE(ocirc_cevent, ocirc_cevent, ocirc_cevent_msg_t *); #ifdef OCIRC_EVENT_PRIVATE -void ocirc_event_publish(const ocirc_event_msg_t *msg); +void ocirc_state_publish(ocirc_state_msg_t *msg); +void ocirc_chan_publish(ocirc_chan_msg_t *msg); +void ocirc_cevent_publish(ocirc_cevent_msg_t *msg); #endif #endif /* !defined(TOR_OCIRC_EVENT_H) */ diff --git a/src/core/or/or.h b/src/core/or/or.h index ab258629a6..990cfacbc0 100644 --- a/src/core/or/or.h +++ b/src/core/or/or.h @@ -843,6 +843,10 @@ typedef struct protover_summary_flags_t { /** True iff this router has a protocol list that allows clients to * negotiate hs circuit setup padding. Requires Padding>=2. */ unsigned int supports_hs_setup_padding : 1; + + /** True iff this router has a protocol list that allows it to support the + * ESTABLISH_INTRO DoS cell extension. Requires HSIntro>=5. */ + unsigned int supports_establish_intro_dos_extension : 1; } protover_summary_flags_t; typedef struct routerinfo_t routerinfo_t; diff --git a/src/core/or/or_circuit_st.h b/src/core/or/or_circuit_st.h index 6789668224..f3eb861613 100644 --- a/src/core/or/or_circuit_st.h +++ b/src/core/or/or_circuit_st.h @@ -12,6 +12,8 @@ #include "core/or/circuit_st.h" #include "core/or/crypt_path_st.h" +#include "lib/evloop/token_bucket.h" + struct onion_queue_t; /** An or_circuit_t holds information needed to implement a circuit at an @@ -69,6 +71,15 @@ struct or_circuit_t { * exit-ward queues of this circuit; reset every time when writing * buffer stats to disk. */ uint64_t total_cell_waiting_time; + + /** If set, the DoS defenses are enabled on this circuit meaning that the + * introduce2_bucket is initialized and used. */ + unsigned int introduce2_dos_defense_enabled : 1; + + /** INTRODUCE2 cell bucket controlling how much can go on this circuit. Only + * used if this is a service introduction circuit at the intro point + * (purpose = CIRCUIT_PURPOSE_INTRO_POINT). */ + token_bucket_ctr_t introduce2_bucket; }; #endif /* !defined(OR_CIRCUIT_ST_H) */ diff --git a/src/core/or/orconn_event.c b/src/core/or/orconn_event.c index 9fb34bd1ff..86f112fc09 100644 --- a/src/core/or/orconn_event.c +++ b/src/core/or/orconn_event.c @@ -17,65 +17,83 @@ **/ #include "core/or/or.h" +#include "lib/pubsub/pubsub.h" #include "lib/subsys/subsys.h" #define ORCONN_EVENT_PRIVATE #include "core/or/orconn_event.h" #include "core/or/orconn_event_sys.h" -/** List of subscribers */ -static smartlist_t *orconn_event_rcvrs; +DECLARE_PUBLISH(orconn_state); +DECLARE_PUBLISH(orconn_status); -/** Initialize subscriber list */ -static int -orconn_event_init(void) +static void +orconn_event_free(msg_aux_data_t u) { - orconn_event_rcvrs = smartlist_new(); - return 0; + tor_free_(u.ptr); } -/** Free subscriber list */ -static void -orconn_event_fini(void) +static char * +orconn_state_fmt(msg_aux_data_t u) { - smartlist_free(orconn_event_rcvrs); + orconn_state_msg_t *msg = (orconn_state_msg_t *)u.ptr; + char *s = NULL; + + tor_asprintf(&s, "<gid=%"PRIu64" chan=%"PRIu64" proxy_type=%d state=%d>", + msg->gid, msg->chan, msg->proxy_type, msg->state); + return s; } -/** - * Subscribe to messages about OR connection events - * - * Register a callback function to receive messages about ORCONNs. - * The publisher calls this function synchronously. - **/ -void -orconn_event_subscribe(orconn_event_rcvr_t fn) +static char * +orconn_status_fmt(msg_aux_data_t u) { - tor_assert(fn); - /* Don't duplicate subscriptions. */ - if (smartlist_contains(orconn_event_rcvrs, fn)) - return; + orconn_status_msg_t *msg = (orconn_status_msg_t *)u.ptr; + char *s = NULL; - smartlist_add(orconn_event_rcvrs, fn); + tor_asprintf(&s, "<gid=%"PRIu64" status=%d reason=%d>", + msg->gid, msg->status, msg->reason); + return s; +} + +static dispatch_typefns_t orconn_state_fns = { + .free_fn = orconn_event_free, + .fmt_fn = orconn_state_fmt, +}; + +static dispatch_typefns_t orconn_status_fns = { + .free_fn = orconn_event_free, + .fmt_fn = orconn_status_fmt, +}; + +static int +orconn_add_pubsub(struct pubsub_connector_t *connector) +{ + if (DISPATCH_REGISTER_TYPE(connector, orconn_state, &orconn_state_fns)) + return -1; + if (DISPATCH_REGISTER_TYPE(connector, orconn_status, &orconn_status_fns)) + return -1; + if (DISPATCH_ADD_PUB(connector, orconn, orconn_state) != 0) + return -1; + if (DISPATCH_ADD_PUB(connector, orconn, orconn_status) != 0) + return -1; + return 0; +} + +void +orconn_state_publish(orconn_state_msg_t *msg) +{ + PUBLISH(orconn_state, msg); } -/** - * Publish a message about OR connection events - * - * This calls the subscriber receiver function synchronously. - **/ void -orconn_event_publish(const orconn_event_msg_t *msg) +orconn_status_publish(orconn_status_msg_t *msg) { - SMARTLIST_FOREACH_BEGIN(orconn_event_rcvrs, orconn_event_rcvr_t, fn) { - tor_assert(fn); - (*fn)(msg); - } SMARTLIST_FOREACH_END(fn); + PUBLISH(orconn_status, msg); } const subsys_fns_t sys_orconn_event = { .name = "orconn_event", .supported = true, .level = -33, - .initialize = orconn_event_init, - .shutdown = orconn_event_fini, + .add_pubsub = orconn_add_pubsub, }; diff --git a/src/core/or/orconn_event.h b/src/core/or/orconn_event.h index d6635793db..fb67a7d183 100644 --- a/src/core/or/orconn_event.h +++ b/src/core/or/orconn_event.h @@ -16,6 +16,8 @@ #ifndef TOR_ORCONN_EVENT_H #define TOR_ORCONN_EVENT_H +#include "lib/pubsub/pubsub.h" + /** * @name States of OR connections * @@ -62,12 +64,6 @@ typedef enum or_conn_status_event_t { OR_CONN_EVENT_NEW = 4, } or_conn_status_event_t; -/** Discriminant values for orconn event message */ -typedef enum orconn_msgtype_t { - ORCONN_MSGTYPE_STATE, - ORCONN_MSGTYPE_STATUS, -} orconn_msgtype_t; - /** * Message for orconn state update * @@ -83,6 +79,8 @@ typedef struct orconn_state_msg_t { uint8_t state; /**< new connection state */ } orconn_state_msg_t; +DECLARE_MESSAGE(orconn_state, orconn_state, orconn_state_msg_t *); + /** * Message for orconn status event * @@ -95,26 +93,11 @@ typedef struct orconn_status_msg_t { int reason; /**< reason */ } orconn_status_msg_t; -/** Discriminated union for the actual message */ -typedef struct orconn_event_msg_t { - int type; - union { - orconn_state_msg_t state; - orconn_status_msg_t status; - } u; -} orconn_event_msg_t; - -/** - * Receiver function pointer for OR subscribers - * - * This function gets called synchronously by the publisher. - **/ -typedef void (*orconn_event_rcvr_t)(const orconn_event_msg_t *); - -void orconn_event_subscribe(orconn_event_rcvr_t); +DECLARE_MESSAGE(orconn_status, orconn_status, orconn_status_msg_t *); #ifdef ORCONN_EVENT_PRIVATE -void orconn_event_publish(const orconn_event_msg_t *); +void orconn_state_publish(orconn_state_msg_t *); +void orconn_status_publish(orconn_status_msg_t *); #endif #endif /* !defined(TOR_ORCONN_EVENT_H) */ diff --git a/src/core/or/protover.c b/src/core/or/protover.c index ccd33fabf7..905c5e9ed3 100644 --- a/src/core/or/protover.c +++ b/src/core/or/protover.c @@ -392,7 +392,7 @@ protover_get_supported_protocols(void) "Desc=1-2 " "DirCache=1-2 " "HSDir=1-2 " - "HSIntro=3-4 " + "HSIntro=3-5 " "HSRend=1-2 " "Link=1-5 " #ifdef HAVE_WORKING_TOR_TLS_GET_TLSSECRETS diff --git a/src/core/or/relay.c b/src/core/or/relay.c index 9e691a02b5..a437b54792 100644 --- a/src/core/or/relay.c +++ b/src/core/or/relay.c @@ -265,8 +265,8 @@ circuit_receive_relay_cell(cell_t *cell, circuit_t *circ, if (cell_direction == CELL_DIRECTION_OUT) { ++stats_n_relay_cells_delivered; log_debug(LD_OR,"Sending away from origin."); - if ((reason=connection_edge_process_relay_cell(cell, circ, conn, NULL)) - < 0) { + reason = connection_edge_process_relay_cell(cell, circ, conn, NULL); + if (reason < 0) { log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "connection_edge_process_relay_cell (away from origin) " "failed."); @@ -276,8 +276,9 @@ circuit_receive_relay_cell(cell_t *cell, circuit_t *circ, if (cell_direction == CELL_DIRECTION_IN) { ++stats_n_relay_cells_delivered; log_debug(LD_OR,"Sending to origin."); - if ((reason = connection_edge_process_relay_cell(cell, circ, conn, - layer_hint)) < 0) { + reason = connection_edge_process_relay_cell(cell, circ, conn, + layer_hint); + if (reason < 0) { /* If a client is trying to connect to unknown hidden service port, * END_CIRC_AT_ORIGIN is sent back so we can then close the circuit. * Do not log warn as this is an expected behavior for a service. */ @@ -1576,104 +1577,33 @@ process_sendme_cell(const relay_header_t *rh, const cell_t *cell, return 0; } -/** An incoming relay cell has arrived on circuit <b>circ</b>. If - * <b>conn</b> is NULL this is a control cell, else <b>cell</b> is - * destined for <b>conn</b>. - * - * If <b>layer_hint</b> is defined, then we're the origin of the - * circuit, and it specifies the hop that packaged <b>cell</b>. +/** A helper for connection_edge_process_relay_cell(): Actually handles the + * cell that we received on the connection. * - * Return -reason if you want to warn and tear down the circuit, else 0. + * The arguments are the same as in the parent function + * connection_edge_process_relay_cell(), plus the relay header <b>rh</b> as + * unpacked by the parent function, and <b>optimistic_data</b> as set by the + * parent function. */ STATIC int -connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, - edge_connection_t *conn, - crypt_path_t *layer_hint) +handle_relay_cell_command(cell_t *cell, circuit_t *circ, + edge_connection_t *conn, crypt_path_t *layer_hint, + relay_header_t *rh, int optimistic_data) { - static int num_seen=0; - relay_header_t rh; unsigned domain = layer_hint?LD_APP:LD_EXIT; int reason; - int optimistic_data = 0; /* Set to 1 if we receive data on a stream - * that's in the EXIT_CONN_STATE_RESOLVING - * or EXIT_CONN_STATE_CONNECTING states. */ - - tor_assert(cell); - tor_assert(circ); - - relay_header_unpack(&rh, cell->payload); -// log_fn(LOG_DEBUG,"command %d stream %d", rh.command, rh.stream_id); - num_seen++; - log_debug(domain, "Now seen %d relay cells here (command %d, stream %d).", - num_seen, rh.command, rh.stream_id); - - if (rh.length > RELAY_PAYLOAD_SIZE) { - log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, - "Relay cell length field too long. Closing circuit."); - return - END_CIRC_REASON_TORPROTOCOL; - } - - if (rh.stream_id == 0) { - switch (rh.command) { - case RELAY_COMMAND_BEGIN: - case RELAY_COMMAND_CONNECTED: - case RELAY_COMMAND_END: - case RELAY_COMMAND_RESOLVE: - case RELAY_COMMAND_RESOLVED: - case RELAY_COMMAND_BEGIN_DIR: - log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "Relay command %d with zero " - "stream_id. Dropping.", (int)rh.command); - return 0; - default: - ; - } - } - - /* Tell circpad that we've received a recognized cell */ - circpad_deliver_recognized_relay_cell_events(circ, rh.command, layer_hint); - - /* either conn is NULL, in which case we've got a control cell, or else - * conn points to the recognized stream. */ - if (conn && !connection_state_is_open(TO_CONN(conn))) { - if (conn->base_.type == CONN_TYPE_EXIT && - (conn->base_.state == EXIT_CONN_STATE_CONNECTING || - conn->base_.state == EXIT_CONN_STATE_RESOLVING) && - rh.command == RELAY_COMMAND_DATA) { - /* Allow DATA cells to be delivered to an exit node in state - * EXIT_CONN_STATE_CONNECTING or EXIT_CONN_STATE_RESOLVING. - * This speeds up HTTP, for example. */ - optimistic_data = 1; - } else if (rh.stream_id == 0 && rh.command == RELAY_COMMAND_DATA) { - log_warn(LD_BUG, "Somehow I had a connection that matched a " - "data cell with stream ID 0."); - } else { - return connection_edge_process_relay_cell_not_open( - &rh, cell, circ, conn, layer_hint); - } - } - switch (rh.command) { - case RELAY_COMMAND_DROP: - /* Already examined in circpad_deliver_recognized_relay_cell_events */ - return 0; - case RELAY_COMMAND_PADDING_NEGOTIATE: - circpad_handle_padding_negotiate(circ, cell); - return 0; - case RELAY_COMMAND_PADDING_NEGOTIATED: - if (circpad_handle_padding_negotiated(circ, cell, layer_hint) == 0) - circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh.length); - return 0; - } + tor_assert(rh); - /* If this is a padding circuit we don't need to parse any other commands - * than the padding ones. Just drop them to the floor. */ - if (circ->purpose == CIRCUIT_PURPOSE_C_CIRCUIT_PADDING) { - log_info(domain, "Ignored cell (%d) that arrived in padding circuit.", - rh.command); + /* First pass the cell to the circuit padding subsystem, in case it's a + * padding cell or circuit that should be handled there. */ + if (circpad_check_received_cell(cell, circ, layer_hint, rh) == 0) { + log_debug(domain, "Cell handled as circuit padding"); return 0; } - switch (rh.command) { + /* Now handle all the other commands */ + switch (rh->command) { case RELAY_COMMAND_BEGIN: case RELAY_COMMAND_BEGIN_DIR: if (layer_hint && @@ -1694,7 +1624,7 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, "Begin cell for known stream. Dropping."); return 0; } - if (rh.command == RELAY_COMMAND_BEGIN_DIR && + if (rh->command == RELAY_COMMAND_BEGIN_DIR && circ->purpose != CIRCUIT_PURPOSE_S_REND_JOINED) { /* Assign this circuit and its app-ward OR connection a unique ID, * so that we can measure download times. The local edge and dir @@ -1721,7 +1651,7 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, /* Consider sending a circuit-level SENDME cell. */ sendme_circuit_consider_sending(circ, layer_hint); - if (rh.stream_id == 0) { + if (rh->stream_id == 0) { log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "Relay data cell with zero " "stream_id. Dropping."); return 0; @@ -1729,16 +1659,16 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, if (CIRCUIT_IS_ORIGIN(circ)) { origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); if (connection_half_edge_is_valid_data(ocirc->half_streams, - rh.stream_id)) { - circuit_read_valid_data(ocirc, rh.length); + rh->stream_id)) { + circuit_read_valid_data(ocirc, rh->length); log_info(domain, "data cell on circ %u valid on half-closed " - "stream id %d", ocirc->global_identifier, rh.stream_id); + "stream id %d", ocirc->global_identifier, rh->stream_id); } } log_info(domain,"data cell dropped, unknown stream (streamid %d).", - rh.stream_id); + rh->stream_id); return 0; } @@ -1753,13 +1683,13 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, return -END_CIRC_REASON_TORPROTOCOL; } /* Total all valid application bytes delivered */ - if (CIRCUIT_IS_ORIGIN(circ) && rh.length > 0) { - circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh.length); + if (CIRCUIT_IS_ORIGIN(circ) && rh->length > 0) { + circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh->length); } - stats_n_data_bytes_received += rh.length; + stats_n_data_bytes_received += rh->length; connection_buf_add((char*)(cell->payload + RELAY_HEADER_SIZE), - rh.length, TO_CONN(conn)); + rh->length, TO_CONN(conn)); #ifdef MEASUREMENTS_21206 /* Count number of RELAY_DATA cells received on a linked directory @@ -1780,20 +1710,20 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, return 0; case RELAY_COMMAND_END: - reason = rh.length > 0 ? + reason = rh->length > 0 ? get_uint8(cell->payload+RELAY_HEADER_SIZE) : END_STREAM_REASON_MISC; if (!conn) { if (CIRCUIT_IS_ORIGIN(circ)) { origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); if (connection_half_edge_is_valid_end(ocirc->half_streams, - rh.stream_id)) { + rh->stream_id)) { - circuit_read_valid_data(ocirc, rh.length); + circuit_read_valid_data(ocirc, rh->length); log_info(domain, "end cell (%s) on circ %u valid on half-closed " "stream id %d", stream_end_reason_to_string(reason), - ocirc->global_identifier, rh.stream_id); + ocirc->global_identifier, rh->stream_id); return 0; } } @@ -1825,7 +1755,7 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, /* Total all valid application bytes delivered */ if (CIRCUIT_IS_ORIGIN(circ)) { - circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh.length); + circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh->length); } } return 0; @@ -1833,7 +1763,7 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, case RELAY_COMMAND_EXTEND2: { static uint64_t total_n_extend=0, total_nonearly=0; total_n_extend++; - if (rh.stream_id) { + if (rh->stream_id) { log_fn(LOG_PROTOCOL_WARN, domain, "'extend' cell received for non-zero stream. Dropping."); return 0; @@ -1874,9 +1804,9 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, log_debug(domain,"Got an extended cell! Yay."); { extended_cell_t extended_cell; - if (extended_cell_parse(&extended_cell, rh.command, + if (extended_cell_parse(&extended_cell, rh->command, (const uint8_t*)cell->payload+RELAY_HEADER_SIZE, - rh.length)<0) { + rh->length)<0) { log_warn(LD_PROTOCOL, "Can't parse EXTENDED cell; killing circuit."); return -END_CIRC_REASON_TORPROTOCOL; @@ -1894,7 +1824,7 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, } /* Total all valid bytes delivered. */ if (CIRCUIT_IS_ORIGIN(circ)) { - circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh.length); + circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh->length); } return 0; case RELAY_COMMAND_TRUNCATE: @@ -1938,7 +1868,7 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, * circuit is being torn down anyway, though. */ if (CIRCUIT_IS_ORIGIN(circ)) { circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), - rh.length); + rh->length); } circuit_truncated(TO_ORIGIN_CIRCUIT(circ), get_uint8(cell->payload + RELAY_HEADER_SIZE)); @@ -1953,11 +1883,11 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, if (CIRCUIT_IS_ORIGIN(circ)) { origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); if (connection_half_edge_is_valid_connected(ocirc->half_streams, - rh.stream_id)) { - circuit_read_valid_data(ocirc, rh.length); + rh->stream_id)) { + circuit_read_valid_data(ocirc, rh->length); log_info(domain, "connected cell on circ %u valid on half-closed " - "stream id %d", ocirc->global_identifier, rh.stream_id); + "stream id %d", ocirc->global_identifier, rh->stream_id); return 0; } } @@ -1965,10 +1895,10 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, log_info(domain, "'connected' received on circid %u for streamid %d, " "no conn attached anymore. Ignoring.", - (unsigned)circ->n_circ_id, rh.stream_id); + (unsigned)circ->n_circ_id, rh->stream_id); return 0; case RELAY_COMMAND_SENDME: - return process_sendme_cell(&rh, cell, circ, conn, layer_hint, domain); + return process_sendme_cell(rh, cell, circ, conn, layer_hint, domain); case RELAY_COMMAND_RESOLVE: if (layer_hint) { log_fn(LOG_PROTOCOL_WARN, LD_APP, @@ -1996,11 +1926,11 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, if (CIRCUIT_IS_ORIGIN(circ)) { origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); if (connection_half_edge_is_valid_resolved(ocirc->half_streams, - rh.stream_id)) { - circuit_read_valid_data(ocirc, rh.length); + rh->stream_id)) { + circuit_read_valid_data(ocirc, rh->length); log_info(domain, "resolved cell on circ %u valid on half-closed " - "stream id %d", ocirc->global_identifier, rh.stream_id); + "stream id %d", ocirc->global_identifier, rh->stream_id); return 0; } } @@ -2018,17 +1948,96 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, case RELAY_COMMAND_INTRO_ESTABLISHED: case RELAY_COMMAND_RENDEZVOUS_ESTABLISHED: rend_process_relay_cell(circ, layer_hint, - rh.command, rh.length, + rh->command, rh->length, cell->payload+RELAY_HEADER_SIZE); return 0; } log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "Received unknown relay command %d. Perhaps the other side is using " "a newer version of Tor? Dropping.", - rh.command); + rh->command); return 0; /* for forward compatibility, don't kill the circuit */ } +/** An incoming relay cell has arrived on circuit <b>circ</b>. If + * <b>conn</b> is NULL this is a control cell, else <b>cell</b> is + * destined for <b>conn</b>. + * + * If <b>layer_hint</b> is defined, then we're the origin of the + * circuit, and it specifies the hop that packaged <b>cell</b>. + * + * Return -reason if you want to warn and tear down the circuit, else 0. + */ +STATIC int +connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, + edge_connection_t *conn, + crypt_path_t *layer_hint) +{ + static int num_seen=0; + relay_header_t rh; + unsigned domain = layer_hint?LD_APP:LD_EXIT; + int optimistic_data = 0; /* Set to 1 if we receive data on a stream + * that's in the EXIT_CONN_STATE_RESOLVING + * or EXIT_CONN_STATE_CONNECTING states. */ + + tor_assert(cell); + tor_assert(circ); + + relay_header_unpack(&rh, cell->payload); +// log_fn(LOG_DEBUG,"command %d stream %d", rh.command, rh.stream_id); + num_seen++; + log_debug(domain, "Now seen %d relay cells here (command %d, stream %d).", + num_seen, rh.command, rh.stream_id); + + if (rh.length > RELAY_PAYLOAD_SIZE) { + log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, + "Relay cell length field too long. Closing circuit."); + return - END_CIRC_REASON_TORPROTOCOL; + } + + if (rh.stream_id == 0) { + switch (rh.command) { + case RELAY_COMMAND_BEGIN: + case RELAY_COMMAND_CONNECTED: + case RELAY_COMMAND_END: + case RELAY_COMMAND_RESOLVE: + case RELAY_COMMAND_RESOLVED: + case RELAY_COMMAND_BEGIN_DIR: + log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "Relay command %d with zero " + "stream_id. Dropping.", (int)rh.command); + return 0; + default: + ; + } + } + + /* Tell circpad that we've received a recognized cell */ + circpad_deliver_recognized_relay_cell_events(circ, rh.command, layer_hint); + + /* either conn is NULL, in which case we've got a control cell, or else + * conn points to the recognized stream. */ + if (conn && !connection_state_is_open(TO_CONN(conn))) { + if (conn->base_.type == CONN_TYPE_EXIT && + (conn->base_.state == EXIT_CONN_STATE_CONNECTING || + conn->base_.state == EXIT_CONN_STATE_RESOLVING) && + rh.command == RELAY_COMMAND_DATA) { + /* Allow DATA cells to be delivered to an exit node in state + * EXIT_CONN_STATE_CONNECTING or EXIT_CONN_STATE_RESOLVING. + * This speeds up HTTP, for example. */ + optimistic_data = 1; + } else if (rh.stream_id == 0 && rh.command == RELAY_COMMAND_DATA) { + log_warn(LD_BUG, "Somehow I had a connection that matched a " + "data cell with stream ID 0."); + } else { + return connection_edge_process_relay_cell_not_open( + &rh, cell, circ, conn, layer_hint); + } + } + + return handle_relay_cell_command(cell, circ, conn, layer_hint, + &rh, optimistic_data); +} + /** How many relay_data cells have we built, ever? */ uint64_t stats_n_data_cells_packaged = 0; /** How many bytes of data have we put in relay_data cells have we built, diff --git a/src/core/or/relay.h b/src/core/or/relay.h index 79036f97bd..99f7553013 100644 --- a/src/core/or/relay.h +++ b/src/core/or/relay.h @@ -99,6 +99,11 @@ circid_t packed_cell_get_circid(const packed_cell_t *cell, int wide_circ_ids); uint8_t packed_cell_get_command(const packed_cell_t *cell, int wide_circ_ids); #ifdef RELAY_PRIVATE +STATIC int +handle_relay_cell_command(cell_t *cell, circuit_t *circ, + edge_connection_t *conn, crypt_path_t *layer_hint, + relay_header_t *rh, int optimistic_data); + STATIC int connected_cell_parse(const relay_header_t *rh, const cell_t *cell, tor_addr_t *addr_out, int *ttl_out); /** An address-and-ttl tuple as yielded by resolved_cell_parse */ diff --git a/src/core/or/scheduler.c b/src/core/or/scheduler.c index ee22a38142..063342480a 100644 --- a/src/core/or/scheduler.c +++ b/src/core/or/scheduler.c @@ -267,7 +267,7 @@ select_scheduler(void) log_notice(LD_SCHED, "Scheduler type KIST has been disabled by " "the consensus or no kernel support."); } -#else /* !(defined(HAVE_KIST_SUPPORT)) */ +#else /* !defined(HAVE_KIST_SUPPORT) */ log_info(LD_SCHED, "Scheduler type KIST not built in"); #endif /* defined(HAVE_KIST_SUPPORT) */ continue; diff --git a/src/core/or/scheduler_kist.c b/src/core/or/scheduler_kist.c index 1ec1c49c73..35b613cb8a 100644 --- a/src/core/or/scheduler_kist.c +++ b/src/core/or/scheduler_kist.c @@ -104,7 +104,7 @@ static unsigned int kist_lite_mode = 0; * changed and it doesn't recognized the values passed to the syscalls needed * by KIST. In that case, fallback to the naive approach. */ static unsigned int kist_no_kernel_support = 0; -#else /* !(defined(HAVE_KIST_SUPPORT)) */ +#else /* !defined(HAVE_KIST_SUPPORT) */ static unsigned int kist_lite_mode = 1; #endif /* defined(HAVE_KIST_SUPPORT) */ @@ -298,7 +298,7 @@ update_socket_info_impl, (socket_table_ent_t *ent)) } return; -#else /* !(defined(HAVE_KIST_SUPPORT)) */ +#else /* !defined(HAVE_KIST_SUPPORT) */ goto fallback; #endif /* defined(HAVE_KIST_SUPPORT) */ @@ -833,7 +833,7 @@ scheduler_can_use_kist(void) return run_interval > 0; } -#else /* !(defined(HAVE_KIST_SUPPORT)) */ +#else /* !defined(HAVE_KIST_SUPPORT) */ int scheduler_can_use_kist(void) diff --git a/src/core/or/sendme.c b/src/core/or/sendme.c index 47ac95f3cf..0757ce3d52 100644 --- a/src/core/or/sendme.c +++ b/src/core/or/sendme.c @@ -23,7 +23,7 @@ #include "core/or/sendme.h" #include "feature/nodelist/networkstatus.h" #include "lib/ctime/di_ops.h" -#include "trunnel/sendme.h" +#include "trunnel/sendme_cell.h" /* Return the minimum version given by the consensus (if any) that should be * used when emitting a SENDME cell. */ diff --git a/src/core/or/versions.c b/src/core/or/versions.c index 06417bb4eb..2c32b529f7 100644 --- a/src/core/or/versions.c +++ b/src/core/or/versions.c @@ -450,7 +450,9 @@ memoize_protover_summary(protover_summary_flags_t *out, PROTOVER_HS_RENDEZVOUS_POINT_V3); out->supports_hs_setup_padding = protocol_list_supports_protocol(protocols, PRT_PADDING, - PROTOVER_HS_SETUP_PADDING); + PROTOVER_HS_SETUP_PADDING); + out->supports_establish_intro_dos_extension = + protocol_list_supports_protocol(protocols, PRT_HSINTRO, 5); protover_summary_flags_t *new_cached = tor_memdup(out, sizeof(*out)); cached = strmap_set(protover_summary_map, protocols, new_cached); diff --git a/src/core/proto/.may_include b/src/core/proto/.may_include new file mode 100644 index 0000000000..c1647a5cf9 --- /dev/null +++ b/src/core/proto/.may_include @@ -0,0 +1,10 @@ +!advisory + +orconfig.h + +lib/crypt_ops/*.h +lib/buf/*.h + +trunnel/*.h + +core/proto/*.h
\ No newline at end of file diff --git a/src/ext/csiphash.c b/src/ext/csiphash.c index af8559a476..faa52ae4e1 100644 --- a/src/ext/csiphash.c +++ b/src/ext/csiphash.c @@ -87,6 +87,13 @@ uint64_t siphash24(const void *src, unsigned long src_sz, const struct sipkey *k v0 ^= mi; } +#ifdef __COVERITY__ + { + uint64_t mi = 0; + memcpy(&mi, m+i, (src_sz-blocks)); + last7 = _le64toh(mi) | (uint64_t)(src_sz & 0xff) << 56; + } +#else switch (src_sz - blocks) { case 7: last7 |= (uint64_t)m[i + 6] << 48; /* Falls through. */ case 6: last7 |= (uint64_t)m[i + 5] << 40; /* Falls through. */ @@ -98,6 +105,7 @@ uint64_t siphash24(const void *src, unsigned long src_sz, const struct sipkey *k case 0: default:; } +#endif v3 ^= last7; DOUBLE_ROUND(v0,v1,v2,v3); v0 ^= last7; diff --git a/src/ext/trunnel/trunnel-impl.h b/src/ext/trunnel/trunnel-impl.h index 15d1c8633e..52afa9ccd4 100644 --- a/src/ext/trunnel/trunnel-impl.h +++ b/src/ext/trunnel/trunnel-impl.h @@ -1,4 +1,4 @@ -/* trunnel-impl.h -- copied from Trunnel v1.5.2 +/* trunnel-impl.h -- copied from Trunnel v1.5.3 * https://gitweb.torproject.org/trunnel.git * You probably shouldn't edit this file. */ diff --git a/src/ext/trunnel/trunnel.c b/src/ext/trunnel/trunnel.c index 3ae3fe02c8..01a55c5bec 100644 --- a/src/ext/trunnel/trunnel.c +++ b/src/ext/trunnel/trunnel.c @@ -1,4 +1,4 @@ -/* trunnel.c -- copied from Trunnel v1.5.2 +/* trunnel.c -- copied from Trunnel v1.5.3 * https://gitweb.torproject.org/trunnel.git * You probably shouldn't edit this file. */ diff --git a/src/ext/trunnel/trunnel.h b/src/ext/trunnel/trunnel.h index 9b708437b8..87c75f4ec3 100644 --- a/src/ext/trunnel/trunnel.h +++ b/src/ext/trunnel/trunnel.h @@ -1,4 +1,4 @@ -/* trunnel.h -- copied from Trunnel v1.5.2 +/* trunnel.h -- copied from Trunnel v1.5.3 * https://gitweb.torproject.org/trunnel.git * You probably shouldn't edit this file. */ diff --git a/src/feature/api/tor_api.c b/src/feature/api/tor_api.c index fd9d241353..e270c51ac9 100644 --- a/src/feature/api/tor_api.c +++ b/src/feature/api/tor_api.c @@ -40,7 +40,7 @@ #define raw_socketpair tor_ersatz_socketpair #define raw_closesocket closesocket #define snprintf _snprintf -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ #define raw_socketpair socketpair #define raw_closesocket close #endif /* defined(_WIN32) */ diff --git a/src/feature/client/circpathbias.c b/src/feature/client/circpathbias.c index 3544dbb859..60a52664f1 100644 --- a/src/feature/client/circpathbias.c +++ b/src/feature/client/circpathbias.c @@ -303,7 +303,7 @@ pathbias_is_new_circ_attempt(origin_circuit_t *circ) return circ->cpath && circ->cpath->next != circ->cpath && circ->cpath->next->state == CPATH_STATE_AWAITING_KEYS; -#else /* !(defined(N2N_TAGGING_IS_POSSIBLE)) */ +#else /* !defined(N2N_TAGGING_IS_POSSIBLE) */ /* If tagging attacks are no longer possible, we probably want to * count bias from the first hop. However, one could argue that * timing-based tagging is still more useful than per-hop failure. diff --git a/src/feature/client/dnsserv.c b/src/feature/client/dnsserv.c index 7fb3fff6c1..f9e436e88f 100644 --- a/src/feature/client/dnsserv.c +++ b/src/feature/client/dnsserv.c @@ -238,7 +238,7 @@ dnsserv_launch_request(const char *name, int reverse, TO_CONN(conn)->port = control_conn->base_.port; TO_CONN(conn)->address = tor_addr_to_str_dup(&control_conn->base_.addr); } -#else /* !(defined(AF_UNIX)) */ +#else /* !defined(AF_UNIX) */ TO_CONN(conn)->port = control_conn->base_.port; TO_CONN(conn)->address = tor_addr_to_str_dup(&control_conn->base_.addr); #endif /* defined(AF_UNIX) */ diff --git a/src/feature/client/entrynodes.c b/src/feature/client/entrynodes.c index 5b6216f483..36b575ef20 100644 --- a/src/feature/client/entrynodes.c +++ b/src/feature/client/entrynodes.c @@ -114,7 +114,7 @@ #include "core/or/or.h" #include "app/config/config.h" -#include "app/config/confparse.h" +#include "lib/confmgt/confparse.h" #include "app/config/statefile.h" #include "core/mainloop/connection.h" #include "core/mainloop/mainloop.h" diff --git a/src/feature/control/btrack.c b/src/feature/control/btrack.c index d3d12cb2b7..3ce97dc855 100644 --- a/src/feature/control/btrack.c +++ b/src/feature/control/btrack.c @@ -24,6 +24,7 @@ #include "feature/control/btrack_circuit.h" #include "feature/control/btrack_orconn.h" #include "feature/control/btrack_sys.h" +#include "lib/pubsub/pubsub.h" #include "lib/subsys/subsys.h" static int @@ -31,8 +32,6 @@ btrack_init(void) { if (btrack_orconn_init()) return -1; - if (btrack_circ_init()) - return -1; return 0; } @@ -44,10 +43,22 @@ btrack_fini(void) btrack_circ_fini(); } +static int +btrack_add_pubsub(pubsub_connector_t *connector) +{ + if (btrack_orconn_add_pubsub(connector)) + return -1; + if (btrack_circ_add_pubsub(connector)) + return -1; + + return 0; +} + const subsys_fns_t sys_btrack = { .name = "btrack", .supported = true, .level = -30, .initialize = btrack_init, .shutdown = btrack_fini, + .add_pubsub = btrack_add_pubsub, }; diff --git a/src/feature/control/btrack_circuit.c b/src/feature/control/btrack_circuit.c index dcee9e460e..2980c77ddc 100644 --- a/src/feature/control/btrack_circuit.c +++ b/src/feature/control/btrack_circuit.c @@ -109,51 +109,53 @@ btc_update_evtype(const ocirc_cevent_msg_t *msg, btc_best_t *best, return false; } +DECLARE_SUBSCRIBE(ocirc_state, btc_state_rcvr); +DECLARE_SUBSCRIBE(ocirc_cevent, btc_cevent_rcvr); +DECLARE_SUBSCRIBE(ocirc_chan, btc_chan_rcvr); + static void -btc_state_rcvr(const ocirc_state_msg_t *msg) +btc_state_rcvr(const msg_t *msg, const ocirc_state_msg_t *arg) { + (void)msg; log_debug(LD_BTRACK, "CIRC gid=%"PRIu32" state=%d onehop=%d", - msg->gid, msg->state, msg->onehop); + arg->gid, arg->state, arg->onehop); - btc_update_state(msg, &best_any_state, "ANY"); - if (msg->onehop) + btc_update_state(arg, &best_any_state, "ANY"); + if (arg->onehop) return; - btc_update_state(msg, &best_ap_state, "AP"); + btc_update_state(arg, &best_ap_state, "AP"); } static void -btc_cevent_rcvr(const ocirc_cevent_msg_t *msg) +btc_cevent_rcvr(const msg_t *msg, const ocirc_cevent_msg_t *arg) { + (void)msg; log_debug(LD_BTRACK, "CIRC gid=%"PRIu32" evtype=%d reason=%d onehop=%d", - msg->gid, msg->evtype, msg->reason, msg->onehop); + arg->gid, arg->evtype, arg->reason, arg->onehop); - btc_update_evtype(msg, &best_any_evtype, "ANY"); - if (msg->onehop) + btc_update_evtype(arg, &best_any_evtype, "ANY"); + if (arg->onehop) return; - btc_update_evtype(msg, &best_ap_evtype, "AP"); + btc_update_evtype(arg, &best_ap_evtype, "AP"); } static void -btc_event_rcvr(const ocirc_event_msg_t *msg) +btc_chan_rcvr(const msg_t *msg, const ocirc_chan_msg_t *arg) { - switch (msg->type) { - case OCIRC_MSGTYPE_STATE: - return btc_state_rcvr(&msg->u.state); - case OCIRC_MSGTYPE_CHAN: - log_debug(LD_BTRACK, "CIRC gid=%"PRIu32" chan=%"PRIu64" onehop=%d", - msg->u.chan.gid, msg->u.chan.chan, msg->u.chan.onehop); - break; - case OCIRC_MSGTYPE_CEVENT: - return btc_cevent_rcvr(&msg->u.cevent); - default: - break; - } + (void)msg; + log_debug(LD_BTRACK, "CIRC gid=%"PRIu32" chan=%"PRIu64" onehop=%d", + arg->gid, arg->chan, arg->onehop); } int -btrack_circ_init(void) +btrack_circ_add_pubsub(pubsub_connector_t *connector) { - ocirc_event_subscribe(btc_event_rcvr); + if (DISPATCH_ADD_SUB(connector, ocirc, ocirc_chan)) + return -1; + if (DISPATCH_ADD_SUB(connector, ocirc, ocirc_cevent)) + return -1; + if (DISPATCH_ADD_SUB(connector, ocirc, ocirc_state)) + return -1; return 0; } diff --git a/src/feature/control/btrack_circuit.h b/src/feature/control/btrack_circuit.h index 9e06fefb07..b326c22ccf 100644 --- a/src/feature/control/btrack_circuit.h +++ b/src/feature/control/btrack_circuit.h @@ -9,7 +9,10 @@ #ifndef TOR_BTRACK_CIRCUIT_H #define TOR_BTRACK_CIRCUIT_H +#include "lib/pubsub/pubsub.h" + int btrack_circ_init(void); void btrack_circ_fini(void); +int btrack_circ_add_pubsub(pubsub_connector_t *); #endif /* !defined(TOR_BTRACK_CIRCUIT_H) */ diff --git a/src/feature/control/btrack_orconn.c b/src/feature/control/btrack_orconn.c index 93ebe8d9cc..922b542a0c 100644 --- a/src/feature/control/btrack_orconn.c +++ b/src/feature/control/btrack_orconn.c @@ -45,6 +45,11 @@ #include "feature/control/btrack_orconn_cevent.h" #include "feature/control/btrack_orconn_maps.h" #include "lib/log/log.h" +#include "lib/pubsub/pubsub.h" + +DECLARE_SUBSCRIBE(orconn_state, bto_state_rcvr); +DECLARE_SUBSCRIBE(orconn_status, bto_status_rcvr); +DECLARE_SUBSCRIBE(ocirc_chan, bto_chan_rcvr); /** Pair of a best ORCONN GID and with its state */ typedef struct bto_best_t { @@ -110,16 +115,17 @@ bto_reset_bests(void) * message comes from code in connection_or.c. **/ static void -bto_state_rcvr(const orconn_state_msg_t *msg) +bto_state_rcvr(const msg_t *msg, const orconn_state_msg_t *arg) { bt_orconn_t *bto; - bto = bto_find_or_new(msg->gid, msg->chan); + (void)msg; + bto = bto_find_or_new(arg->gid, arg->chan); log_debug(LD_BTRACK, "ORCONN gid=%"PRIu64" chan=%"PRIu64 " proxy_type=%d state=%d", - msg->gid, msg->chan, msg->proxy_type, msg->state); - bto->proxy_type = msg->proxy_type; - bto->state = msg->state; + arg->gid, arg->chan, arg->proxy_type, arg->state); + bto->proxy_type = arg->proxy_type; + bto->state = arg->state; if (bto->is_orig) bto_update_bests(bto); } @@ -130,54 +136,38 @@ bto_state_rcvr(const orconn_state_msg_t *msg) * control.c. **/ static void -bto_status_rcvr(const orconn_status_msg_t *msg) +bto_status_rcvr(const msg_t *msg, const orconn_status_msg_t *arg) { - switch (msg->status) { + (void)msg; + switch (arg->status) { case OR_CONN_EVENT_FAILED: case OR_CONN_EVENT_CLOSED: log_info(LD_BTRACK, "ORCONN DELETE gid=%"PRIu64" status=%d reason=%d", - msg->gid, msg->status, msg->reason); - return bto_delete(msg->gid); + arg->gid, arg->status, arg->reason); + return bto_delete(arg->gid); default: break; } } -/** Dispatch to individual ORCONN message handlers */ -static void -bto_event_rcvr(const orconn_event_msg_t *msg) -{ - switch (msg->type) { - case ORCONN_MSGTYPE_STATE: - return bto_state_rcvr(&msg->u.state); - case ORCONN_MSGTYPE_STATUS: - return bto_status_rcvr(&msg->u.status); - default: - tor_assert(false); - } -} - /** * Create or update a cached ORCONN state for a newly launched * connection, including whether it's launched by an origin circuit * and whether it's a one-hop circuit. **/ static void -bto_chan_rcvr(const ocirc_event_msg_t *msg) +bto_chan_rcvr(const msg_t *msg, const ocirc_chan_msg_t *arg) { bt_orconn_t *bto; - /* Ignore other kinds of origin circuit events; we don't need them */ - if (msg->type != OCIRC_MSGTYPE_CHAN) - return; - - bto = bto_find_or_new(0, msg->u.chan.chan); - if (!bto->is_orig || (bto->is_onehop && !msg->u.chan.onehop)) { + (void)msg; + bto = bto_find_or_new(0, arg->chan); + if (!bto->is_orig || (bto->is_onehop && !arg->onehop)) { log_debug(LD_BTRACK, "ORCONN LAUNCH chan=%"PRIu64" onehop=%d", - msg->u.chan.chan, msg->u.chan.onehop); + arg->chan, arg->onehop); } bto->is_orig = true; - if (!msg->u.chan.onehop) + if (!arg->onehop) bto->is_onehop = false; bto_update_bests(bto); } @@ -190,12 +180,22 @@ int btrack_orconn_init(void) { bto_init_maps(); - orconn_event_subscribe(bto_event_rcvr); - ocirc_event_subscribe(bto_chan_rcvr); return 0; } +int +btrack_orconn_add_pubsub(pubsub_connector_t *connector) +{ + if (DISPATCH_ADD_SUB(connector, orconn, orconn_state)) + return -1; + if (DISPATCH_ADD_SUB(connector, orconn, orconn_status)) + return -1; + if (DISPATCH_ADD_SUB(connector, ocirc, ocirc_chan)) + return -1; + return 0; +} + /** Clear the hash maps and reset the "best" states */ void btrack_orconn_fini(void) diff --git a/src/feature/control/btrack_orconn.h b/src/feature/control/btrack_orconn.h index f8f5c1096c..07b1b755f3 100644 --- a/src/feature/control/btrack_orconn.h +++ b/src/feature/control/btrack_orconn.h @@ -9,6 +9,8 @@ #ifndef TOR_BTRACK_ORCONN_H #define TOR_BTRACK_ORCONN_H +#include "lib/pubsub/pubsub.h" + #ifdef BTRACK_ORCONN_PRIVATE #include "ht.h" @@ -33,6 +35,7 @@ typedef struct bt_orconn_t { #endif /* defined(BTRACK_ORCONN_PRIVATE) */ int btrack_orconn_init(void); +int btrack_orconn_add_pubsub(pubsub_connector_t *); void btrack_orconn_fini(void); #endif /* !defined(TOR_BTRACK_ORCONN_H) */ diff --git a/src/feature/control/control.c b/src/feature/control/control.c index 436bf423cf..d6581808c0 100644 --- a/src/feature/control/control.c +++ b/src/feature/control/control.c @@ -339,6 +339,60 @@ static const char CONTROLPORT_IS_NOT_AN_HTTP_PROXY_MSG[] = "</body>\n" "</html>\n"; +/** Return an error on a control connection that tried to use the v0 protocol. + */ +static void +control_send_v0_reject(control_connection_t *conn) +{ + size_t body_len; + char buf[128]; + set_uint16(buf+2, htons(0x0000)); /* type == error */ + set_uint16(buf+4, htons(0x0001)); /* code == internal error */ + strlcpy(buf+6, "The v0 control protocol is not supported by Tor 0.1.2.17 " + "and later; upgrade your controller.", + sizeof(buf)-6); + body_len = 2+strlen(buf+6)+2; /* code, msg, nul. */ + set_uint16(buf+0, htons(body_len)); + connection_buf_add(buf, 4+body_len, TO_CONN(conn)); + + connection_mark_and_flush(TO_CONN(conn)); +} + +/** Return an error on a control connection that tried to use HTTP. + */ +static void +control_send_http_reject(control_connection_t *conn) +{ + connection_write_str_to_buf(CONTROLPORT_IS_NOT_AN_HTTP_PROXY_MSG, conn); + log_notice(LD_CONTROL, "Received HTTP request on ControlPort"); + connection_mark_and_flush(TO_CONN(conn)); +} + +/** Check if a control connection has tried to use a known invalid protocol. + * If it has, then: + * - send a reject response, + * - log a notice-level message, and + * - return false. */ +static bool +control_protocol_is_valid(control_connection_t *conn) +{ + /* Detect v0 commands and send a "no more v0" message. */ + if (conn->base_.state == CONTROL_CONN_STATE_NEEDAUTH && + peek_connection_has_control0_command(TO_CONN(conn))) { + control_send_v0_reject(conn); + return 0; + } + + /* If the user has the HTTP proxy port and the control port confused. */ + if (conn->base_.state == CONTROL_CONN_STATE_NEEDAUTH && + peek_connection_has_http_command(TO_CONN(conn))) { + control_send_http_reject(conn); + return 0; + } + + return 1; +} + /** Called when data has arrived on a v1 control connection: Try to fetch * commands from conn->inbuf, and execute them. */ @@ -359,30 +413,7 @@ connection_control_process_inbuf(control_connection_t *conn) conn->incoming_cmd_cur_len = 0; } - if (conn->base_.state == CONTROL_CONN_STATE_NEEDAUTH && - peek_connection_has_control0_command(TO_CONN(conn))) { - /* Detect v0 commands and send a "no more v0" message. */ - size_t body_len; - char buf[128]; - set_uint16(buf+2, htons(0x0000)); /* type == error */ - set_uint16(buf+4, htons(0x0001)); /* code == internal error */ - strlcpy(buf+6, "The v0 control protocol is not supported by Tor 0.1.2.17 " - "and later; upgrade your controller.", - sizeof(buf)-6); - body_len = 2+strlen(buf+6)+2; /* code, msg, nul. */ - set_uint16(buf+0, htons(body_len)); - connection_buf_add(buf, 4+body_len, TO_CONN(conn)); - - connection_mark_and_flush(TO_CONN(conn)); - return 0; - } - - /* If the user has the HTTP proxy port and the control port confused. */ - if (conn->base_.state == CONTROL_CONN_STATE_NEEDAUTH && - peek_connection_has_http_command(TO_CONN(conn))) { - connection_write_str_to_buf(CONTROLPORT_IS_NOT_AN_HTTP_PROXY_MSG, conn); - log_notice(LD_CONTROL, "Received HTTP request on ControlPort"); - connection_mark_and_flush(TO_CONN(conn)); + if (!control_protocol_is_valid(conn)) { return 0; } diff --git a/src/feature/control/control_auth.c b/src/feature/control/control_auth.c index 49d4d415c6..a574d07b33 100644 --- a/src/feature/control/control_auth.c +++ b/src/feature/control/control_auth.c @@ -151,12 +151,8 @@ handle_control_authchallenge(control_connection_t *conn, goto fail; } if (args->kwargs == NULL || args->kwargs->next != NULL) { - /* connection_write_str_to_buf("512 AUTHCHALLENGE requires exactly " - "2 arguments.\r\n", conn); - */ - control_printf_endreply(conn, 512, - "AUTHCHALLENGE dislikes argument list %s", - escaped(args->raw_body)); + control_write_endreply(conn, 512, + "Wrong number of arguments for AUTHCHALLENGE"); goto fail; } if (strcmp(args->kwargs->key, "")) { diff --git a/src/feature/control/control_cmd.c b/src/feature/control/control_cmd.c index e0706ee4c8..de1bef7e59 100644 --- a/src/feature/control/control_cmd.c +++ b/src/feature/control/control_cmd.c @@ -13,7 +13,7 @@ #include "core/or/or.h" #include "app/config/config.h" -#include "app/config/confparse.h" +#include "lib/confmgt/confparse.h" #include "app/main/main.h" #include "core/mainloop/connection.h" #include "core/or/circuitbuild.h" @@ -705,9 +705,8 @@ handle_control_mapaddress(control_connection_t *conn, connection_buf_add(r, sz, TO_CONN(conn)); tor_free(r); } else { - const char *response = - "512 syntax error: not enough arguments to mapaddress.\r\n"; - connection_buf_add(response, strlen(response), TO_CONN(conn)); + control_write_endreply(conn, 512, "syntax error: " + "not enough arguments to mapaddress."); } SMARTLIST_FOREACH(reply, char *, cp, tor_free(cp)); @@ -847,7 +846,7 @@ handle_control_extendcircuit(control_connection_t *conn, "addresses that are allowed by the firewall configuration; " "circuit marked for closing."); circuit_mark_for_close(TO_CIRCUIT(circ), -END_CIRC_REASON_CONNECTFAILED); - connection_write_str_to_buf("551 Couldn't start circuit\r\n", conn); + control_write_endreply(conn, 551, "Couldn't start circuit"); goto done; } circuit_append_new_exit(circ, info); @@ -1746,16 +1745,10 @@ handle_control_add_onion(control_connection_t *conn, goto out; } else if (!strcasecmp(arg->key, "ClientAuth")) { - char *err_msg = NULL; int created = 0; rend_authorized_client_t *client = - add_onion_helper_clientauth(arg->value, - &created, &err_msg); + add_onion_helper_clientauth(arg->value, &created, conn); if (!client) { - if (err_msg) { - connection_write_str_to_buf(err_msg, conn); - tor_free(err_msg); - } goto out; } @@ -1820,19 +1813,13 @@ handle_control_add_onion(control_connection_t *conn, add_onion_secret_key_t pk = { NULL }; const char *key_new_alg = NULL; char *key_new_blob = NULL; - char *err_msg = NULL; const char *onionkey = smartlist_get(args->args, 0); if (add_onion_helper_keyarg(onionkey, discard_pk, &key_new_alg, &key_new_blob, &pk, &hs_version, - &err_msg) < 0) { - if (err_msg) { - connection_write_str_to_buf(err_msg, conn); - tor_free(err_msg); - } + conn) < 0) { goto out; } - tor_assert(!err_msg); /* Hidden service version 3 don't have client authentication support so if * ClientAuth was given, send back an error. */ @@ -1878,8 +1865,8 @@ handle_control_add_onion(control_connection_t *conn, char *encoded = rend_auth_encode_cookie(ac->descriptor_cookie, auth_type); tor_assert(encoded); - connection_printf_to_buf(conn, "250-ClientAuth=%s:%s\r\n", - ac->client_name, encoded); + control_printf_midreply(conn, 250, "ClientAuth=%s:%s", + ac->client_name, encoded); memwipe(encoded, 0, strlen(encoded)); tor_free(encoded); }); @@ -1932,27 +1919,30 @@ handle_control_add_onion(control_connection_t *conn, * ADD_ONION command. Return a new crypto_pk_t and if a new key was generated * and the private key not discarded, the algorithm and serialized private key, * or NULL and an optional control protocol error message on failure. The - * caller is responsible for freeing the returned key_new_blob and err_msg. + * caller is responsible for freeing the returned key_new_blob. * * Note: The error messages returned are deliberately vague to avoid echoing * key material. + * + * Note: conn is only used for writing control replies. For testing + * purposes, it can be NULL if control_write_reply() is appropriately + * mocked. */ STATIC int add_onion_helper_keyarg(const char *arg, int discard_pk, const char **key_new_alg_out, char **key_new_blob_out, add_onion_secret_key_t *decoded_key, int *hs_version, - char **err_msg_out) + control_connection_t *conn) { smartlist_t *key_args = smartlist_new(); crypto_pk_t *pk = NULL; const char *key_new_alg = NULL; char *key_new_blob = NULL; - char *err_msg = NULL; int ret = -1; smartlist_split_string(key_args, arg, ":", SPLIT_IGNORE_BLANK, 0); if (smartlist_len(key_args) != 2) { - err_msg = tor_strdup("512 Invalid key type/blob\r\n"); + control_write_endreply(conn, 512, "Invalid key type/blob"); goto err; } @@ -1969,12 +1959,12 @@ add_onion_helper_keyarg(const char *arg, int discard_pk, /* "RSA:<Base64 Blob>" - Loading a pre-existing RSA1024 key. */ pk = crypto_pk_base64_decode_private(key_blob, strlen(key_blob)); if (!pk) { - err_msg = tor_strdup("512 Failed to decode RSA key\r\n"); + control_write_endreply(conn, 512, "Failed to decode RSA key"); goto err; } if (crypto_pk_num_bits(pk) != PK_BYTES*8) { crypto_pk_free(pk); - err_msg = tor_strdup("512 Invalid RSA key size\r\n"); + control_write_endreply(conn, 512, "Invalid RSA key size"); goto err; } decoded_key->v2 = pk; @@ -1985,39 +1975,40 @@ add_onion_helper_keyarg(const char *arg, int discard_pk, if (base64_decode((char *) sk->seckey, sizeof(sk->seckey), key_blob, strlen(key_blob)) != sizeof(sk->seckey)) { tor_free(sk); - err_msg = tor_strdup("512 Failed to decode ED25519-V3 key\r\n"); + control_write_endreply(conn, 512, "Failed to decode ED25519-V3 key"); goto err; } decoded_key->v3 = sk; *hs_version = HS_VERSION_THREE; } else if (!strcasecmp(key_type_new, key_type)) { /* "NEW:<Algorithm>" - Generating a new key, blob as algorithm. */ - if (!strcasecmp(key_type_rsa1024, key_blob) || - !strcasecmp(key_type_best, key_blob)) { + if (!strcasecmp(key_type_rsa1024, key_blob)) { /* "RSA1024", RSA 1024 bit, also currently "BEST" by default. */ pk = crypto_pk_new(); if (crypto_pk_generate_key(pk)) { - tor_asprintf(&err_msg, "551 Failed to generate %s key\r\n", - key_type_rsa1024); + control_printf_endreply(conn, 551, "Failed to generate %s key", + key_type_rsa1024); goto err; } if (!discard_pk) { if (crypto_pk_base64_encode_private(pk, &key_new_blob)) { crypto_pk_free(pk); - tor_asprintf(&err_msg, "551 Failed to encode %s key\r\n", - key_type_rsa1024); + control_printf_endreply(conn, 551, "Failed to encode %s key", + key_type_rsa1024); goto err; } key_new_alg = key_type_rsa1024; } decoded_key->v2 = pk; *hs_version = HS_VERSION_TWO; - } else if (!strcasecmp(key_type_ed25519_v3, key_blob)) { + } else if (!strcasecmp(key_type_ed25519_v3, key_blob) || + !strcasecmp(key_type_best, key_blob)) { + /* "ED25519-V3", ed25519 key, also currently "BEST" by default. */ ed25519_secret_key_t *sk = tor_malloc_zero(sizeof(*sk)); if (ed25519_secret_key_generate(sk, 1) < 0) { tor_free(sk); - tor_asprintf(&err_msg, "551 Failed to generate %s key\r\n", - key_type_ed25519_v3); + control_printf_endreply(conn, 551, "Failed to generate %s key", + key_type_ed25519_v3); goto err; } if (!discard_pk) { @@ -2027,8 +2018,8 @@ add_onion_helper_keyarg(const char *arg, int discard_pk, sizeof(sk->seckey), 0) != (len - 1)) { tor_free(sk); tor_free(key_new_blob); - tor_asprintf(&err_msg, "551 Failed to encode %s key\r\n", - key_type_ed25519_v3); + control_printf_endreply(conn, 551, "Failed to encode %s key", + key_type_ed25519_v3); goto err; } key_new_alg = key_type_ed25519_v3; @@ -2036,11 +2027,11 @@ add_onion_helper_keyarg(const char *arg, int discard_pk, decoded_key->v3 = sk; *hs_version = HS_VERSION_THREE; } else { - err_msg = tor_strdup("513 Invalid key type\r\n"); + control_write_endreply(conn, 513, "Invalid key type"); goto err; } } else { - err_msg = tor_strdup("513 Invalid key type\r\n"); + control_write_endreply(conn, 513, "Invalid key type"); goto err; } @@ -2054,11 +2045,6 @@ add_onion_helper_keyarg(const char *arg, int discard_pk, }); smartlist_free(key_args); - if (err_msg_out) { - *err_msg_out = err_msg; - } else { - tor_free(err_msg); - } *key_new_alg_out = key_new_alg; *key_new_blob_out = key_new_blob; @@ -2068,27 +2054,30 @@ add_onion_helper_keyarg(const char *arg, int discard_pk, /** Helper function to handle parsing a ClientAuth argument to the * ADD_ONION command. Return a new rend_authorized_client_t, or NULL * and an optional control protocol error message on failure. The - * caller is responsible for freeing the returned auth_client and err_msg. + * caller is responsible for freeing the returned auth_client. * * If 'created' is specified, it will be set to 1 when a new cookie has * been generated. + * + * Note: conn is only used for writing control replies. For testing + * purposes, it can be NULL if control_write_reply() is appropriately + * mocked. */ STATIC rend_authorized_client_t * -add_onion_helper_clientauth(const char *arg, int *created, char **err_msg) +add_onion_helper_clientauth(const char *arg, int *created, + control_connection_t *conn) { int ok = 0; tor_assert(arg); tor_assert(created); - tor_assert(err_msg); - *err_msg = NULL; smartlist_t *auth_args = smartlist_new(); rend_authorized_client_t *client = tor_malloc_zero(sizeof(rend_authorized_client_t)); smartlist_split_string(auth_args, arg, ":", 0, 0); if (smartlist_len(auth_args) < 1 || smartlist_len(auth_args) > 2) { - *err_msg = tor_strdup("512 Invalid ClientAuth syntax\r\n"); + control_write_endreply(conn, 512, "Invalid ClientAuth syntax"); goto err; } client->client_name = tor_strdup(smartlist_get(auth_args, 0)); @@ -2098,7 +2087,7 @@ add_onion_helper_clientauth(const char *arg, int *created, char **err_msg) client->descriptor_cookie, NULL, &decode_err_msg) < 0) { tor_assert(decode_err_msg); - tor_asprintf(err_msg, "512 %s\r\n", decode_err_msg); + control_write_endreply(conn, 512, decode_err_msg); tor_free(decode_err_msg); goto err; } @@ -2109,7 +2098,7 @@ add_onion_helper_clientauth(const char *arg, int *created, char **err_msg) } if (!rend_valid_client_name(client->client_name)) { - *err_msg = tor_strdup("512 Invalid name in ClientAuth\r\n"); + control_write_endreply(conn, 512, "Invalid name in ClientAuth"); goto err; } diff --git a/src/feature/control/control_cmd.h b/src/feature/control/control_cmd.h index 5c3d1a1cec..4b6d54abe7 100644 --- a/src/feature/control/control_cmd.h +++ b/src/feature/control/control_cmd.h @@ -91,10 +91,11 @@ STATIC int add_onion_helper_keyarg(const char *arg, int discard_pk, const char **key_new_alg_out, char **key_new_blob_out, add_onion_secret_key_t *decoded_key, - int *hs_version, char **err_msg_out); + int *hs_version, + control_connection_t *conn); STATIC rend_authorized_client_t *add_onion_helper_clientauth(const char *arg, - int *created, char **err_msg_out); + int *created, control_connection_t *conn); STATIC control_cmd_args_t *control_cmd_parse_args( const char *command, diff --git a/src/feature/control/control_events.c b/src/feature/control/control_events.c index 9e0966ca54..82ea943999 100644 --- a/src/feature/control/control_events.c +++ b/src/feature/control/control_events.c @@ -26,9 +26,9 @@ #include "feature/control/control_fmt.h" #include "feature/control/control_proto.h" #include "feature/dircommon/directory.h" +#include "feature/nodelist/describe.h" #include "feature/nodelist/networkstatus.h" #include "feature/nodelist/nodelist.h" -#include "feature/nodelist/routerinfo.h" #include "feature/control/control_connection_st.h" #include "core/or/entry_connection_st.h" diff --git a/src/feature/control/control_proto.c b/src/feature/control/control_proto.c index 1dd62da2be..5dec87491d 100644 --- a/src/feature/control/control_proto.c +++ b/src/feature/control/control_proto.c @@ -174,10 +174,11 @@ send_control_done(control_connection_t *conn) * @param conn control connection * @param code numeric result code * @param c separator character, usually ' ', '-', or '+' - * @param s string + * @param s string reply content */ -void -control_write_reply(control_connection_t *conn, int code, int c, const char *s) +MOCK_IMPL(void, +control_write_reply, (control_connection_t *conn, int code, int c, + const char *s)) { connection_printf_to_buf(conn, "%03d%c%s\r\n", code, c, s); } diff --git a/src/feature/control/control_proto.h b/src/feature/control/control_proto.h index 101b808d88..3182f3d415 100644 --- a/src/feature/control/control_proto.h +++ b/src/feature/control/control_proto.h @@ -21,8 +21,8 @@ size_t write_escaped_data(const char *data, size_t len, char **out); size_t read_escaped_data(const char *data, size_t len, char **out); void send_control_done(control_connection_t *conn); -void control_write_reply(control_connection_t *conn, int code, int c, - const char *s); +MOCK_DECL(void, control_write_reply, (control_connection_t *conn, int code, + int c, const char *s)); void control_vprintf_reply(control_connection_t *conn, int code, int c, const char *fmt, va_list ap) CHECK_PRINTF(4, 0); diff --git a/src/feature/control/fmt_serverstatus.c b/src/feature/control/fmt_serverstatus.c index a80bf50ad9..33c5ba1336 100644 --- a/src/feature/control/fmt_serverstatus.c +++ b/src/feature/control/fmt_serverstatus.c @@ -9,8 +9,8 @@ #include "app/config/config.h" #include "feature/dirauth/authmode.h" #include "feature/dirauth/voteflags.h"// XXXX remove +#include "feature/nodelist/describe.h" #include "feature/nodelist/nodelist.h" -#include "feature/nodelist/routerinfo.h" #include "feature/nodelist/node_st.h" #include "feature/nodelist/routerinfo_st.h" diff --git a/src/feature/dirauth/authmode.h b/src/feature/dirauth/authmode.h index 48afc3cdb4..bfd5f4dc04 100644 --- a/src/feature/dirauth/authmode.h +++ b/src/feature/dirauth/authmode.h @@ -29,7 +29,7 @@ authdir_mode_v3(const or_options_t *options) #define have_module_dirauth() (1) -#else /* !(defined(HAVE_MODULE_DIRAUTH)) */ +#else /* !defined(HAVE_MODULE_DIRAUTH) */ #define authdir_mode(options) (((void)(options)),0) #define authdir_mode_handles_descs(options,purpose) \ diff --git a/src/feature/dirauth/dirauth_periodic.h b/src/feature/dirauth/dirauth_periodic.h index 1124fae952..866fbd35de 100644 --- a/src/feature/dirauth/dirauth_periodic.h +++ b/src/feature/dirauth/dirauth_periodic.h @@ -12,7 +12,7 @@ void dirauth_register_periodic_events(void); void reschedule_dirvote(const or_options_t *options); -#else /* !(defined(HAVE_MODULE_DIRAUTH)) */ +#else /* !defined(HAVE_MODULE_DIRAUTH) */ static inline void reschedule_dirvote(const or_options_t *options) diff --git a/src/feature/dirauth/dirvote.c b/src/feature/dirauth/dirvote.c index cdbdf5a216..043bbfc227 100644 --- a/src/feature/dirauth/dirvote.c +++ b/src/feature/dirauth/dirvote.c @@ -220,7 +220,6 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key, networkstatus_t *v3_ns) { smartlist_t *chunks = smartlist_new(); - char *packages = NULL; char fingerprint[FINGERPRINT_LEN+1]; char digest[DIGEST_LEN]; uint32_t addr; @@ -246,19 +245,6 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key, v3_ns->server_versions); protocols_lines = format_protocols_lines_for_vote(v3_ns); - if (v3_ns->package_lines) { - smartlist_t *tmp = smartlist_new(); - SMARTLIST_FOREACH(v3_ns->package_lines, const char *, p, - if (validate_recommended_package_line(p)) - smartlist_add_asprintf(tmp, "package %s\n", p)); - smartlist_sort_strings(tmp); - packages = smartlist_join_strings(tmp, "", 0, NULL); - SMARTLIST_FOREACH(tmp, char *, cp, tor_free(cp)); - smartlist_free(tmp); - } else { - packages = tor_strdup(""); - } - /* Get shared random commitments/reveals line(s). */ shared_random_vote_str = sr_get_string_for_vote(); @@ -344,7 +330,6 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key, "voting-delay %d %d\n" "%s%s" /* versions */ "%s" /* protocols */ - "%s" /* packages */ "known-flags %s\n" "flag-thresholds %s\n" "params %s\n" @@ -361,7 +346,6 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key, client_versions_line, server_versions_line, protocols_lines, - packages, flags, flag_thresholds, params, @@ -460,7 +444,6 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key, tor_free(client_versions_line); tor_free(server_versions_line); tor_free(protocols_lines); - tor_free(packages); SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp)); smartlist_free(chunks); @@ -4668,15 +4651,6 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, tor_assert_nonfatal(protover_all_supported( v3_out->recommended_client_protocols, NULL)); - v3_out->package_lines = smartlist_new(); - { - config_line_t *cl; - for (cl = get_options()->RecommendedPackages; cl; cl = cl->next) { - if (validate_recommended_package_line(cl->value)) - smartlist_add_strdup(v3_out->package_lines, cl->value); - } - } - v3_out->known_flags = smartlist_new(); smartlist_split_string(v3_out->known_flags, DIRVOTE_UNIVERSAL_FLAGS, diff --git a/src/feature/dirauth/dirvote.h b/src/feature/dirauth/dirvote.h index a0cfe0a34c..b7df33a3a9 100644 --- a/src/feature/dirauth/dirvote.h +++ b/src/feature/dirauth/dirvote.h @@ -128,7 +128,7 @@ struct config_line_t; char *format_recommended_version_list(const struct config_line_t *line, int warn); -#else /* !(defined(HAVE_MODULE_DIRAUTH)) */ +#else /* !defined(HAVE_MODULE_DIRAUTH) */ static inline time_t dirvote_act(const or_options_t *options, time_t now) diff --git a/src/feature/dirauth/keypin.c b/src/feature/dirauth/keypin.c index 667feb2c03..3ca2c3ef91 100644 --- a/src/feature/dirauth/keypin.c +++ b/src/feature/dirauth/keypin.c @@ -438,7 +438,7 @@ keypin_load_journal_impl(const char *data, size_t size) tor_log(severity, LD_DIRSERV, "Loaded %d entries from keypin journal. " "Found %d corrupt lines (ignored), %d duplicates (harmless), " - "and %d conflicts (resolved in favor or more recent entry).", + "and %d conflicts (resolved in favor of more recent entry).", n_entries, n_corrupt_lines, n_duplicates, n_conflicts); return 0; diff --git a/src/feature/dirauth/keypin.h b/src/feature/dirauth/keypin.h index ab2362b3f8..1de84f6d4a 100644 --- a/src/feature/dirauth/keypin.h +++ b/src/feature/dirauth/keypin.h @@ -29,7 +29,7 @@ keypin_load_journal(const char *fname) (void)fname; return 0; } -#endif +#endif /* defined(HAVE_MODULE_DIRAUTH) */ void keypin_clear(void); int keypin_check_lone_rsa(const uint8_t *rsa_id_digest); diff --git a/src/feature/dirauth/process_descs.c b/src/feature/dirauth/process_descs.c index 760560a5d9..71e3195c01 100644 --- a/src/feature/dirauth/process_descs.c +++ b/src/feature/dirauth/process_descs.c @@ -216,9 +216,14 @@ dirserv_load_fingerprint_file(void) #define DISABLE_DISABLING_ED25519 -/** Check whether <b>router</b> has a nickname/identity key combination that - * we recognize from the fingerprint list, or an IP we automatically act on - * according to our configuration. Return the appropriate router status. +/** Check whether <b>router</b> has: + * - a nickname/identity key combination that we recognize from the fingerprint + * list, + * - an IP we automatically act on according to our configuration, + * - an appropriate version, and + * - matching pinned keys. + * + * Return the appropriate router status. * * If the status is 'FP_REJECT' and <b>msg</b> is provided, set * *<b>msg</b> to an explanation of why. */ @@ -236,7 +241,7 @@ dirserv_router_get_status(const routerinfo_t *router, const char **msg, return FP_REJECT; } - /* Check for the more usual versions to reject a router first. */ + /* Check for the more common reasons to reject a router first. */ const uint32_t r = dirserv_get_status_impl(d, router->nickname, router->addr, router->or_port, router->platform, msg, severity); @@ -316,7 +321,7 @@ dirserv_would_reject_router(const routerstatus_t *rs) * true, and set *<b>msg</b> (if present) to a rejection message. Otherwise * return false. */ -static bool +STATIC bool dirserv_rejects_tor_version(const char *platform, const char **msg) { @@ -450,20 +455,32 @@ dirserv_free_fingerprint_list(void) /** Return -1 if <b>ri</b> has a private or otherwise bad address, * unless we're configured to not care. Return 0 if all ok. */ -static int +STATIC int dirserv_router_has_valid_address(routerinfo_t *ri) { tor_addr_t addr; + if (get_options()->DirAllowPrivateAddresses) return 0; /* whatever it is, we're fine with it */ + tor_addr_from_ipv4h(&addr, ri->addr); + if (tor_addr_is_null(&addr) || tor_addr_is_internal(&addr, 0)) { + log_info(LD_DIRSERV, + "Router %s published internal IPv4 address. Refusing.", + router_describe(ri)); + return -1; /* it's a private IP, we should reject it */ + } - if (tor_addr_is_internal(&addr, 0)) { + /* We only check internal v6 on non-null addresses because we do not require + * IPv6 and null IPv6 is normal. */ + if (!tor_addr_is_null(&ri->ipv6_addr) && + tor_addr_is_internal(&ri->ipv6_addr, 0)) { log_info(LD_DIRSERV, - "Router %s published internal IP address. Refusing.", + "Router %s published internal IPv6 address. Refusing.", router_describe(ri)); return -1; /* it's a private IP, we should reject it */ } + return 0; } @@ -562,7 +579,7 @@ dirserv_add_multiple_descriptors(const char *desc, size_t desclen, int general = purpose == ROUTER_PURPOSE_GENERAL; tor_assert(msg); - r=ROUTER_ADDED_SUCCESSFULLY; /*Least severe return value. */ + r=ROUTER_ADDED_SUCCESSFULLY; /* Least severe return value. */ if (!string_is_utf8_no_bom(desc, desclen)) { *msg = "descriptor(s) or extrainfo(s) not valid UTF-8 or had BOM."; @@ -578,9 +595,7 @@ dirserv_add_multiple_descriptors(const char *desc, size_t desclen, !general ? router_purpose_to_string(purpose) : "", !general ? "\n" : "")<0) { *msg = "Couldn't format annotations"; - /* XXX Not cool: we return -1 below, but (was_router_added_t)-1 is - * ROUTER_BAD_EI, which isn't what's gone wrong here. :( */ - return -1; + return ROUTER_AUTHDIR_BUG_ANNOTATIONS; } s = desc; diff --git a/src/feature/dirauth/process_descs.h b/src/feature/dirauth/process_descs.h index a8a1dcca1e..e504daa7b7 100644 --- a/src/feature/dirauth/process_descs.h +++ b/src/feature/dirauth/process_descs.h @@ -38,7 +38,7 @@ uint32_t dirserv_router_get_status(const routerinfo_t *router, int severity); void dirserv_set_node_flags_from_authoritative_status(node_t *node, uint32_t authstatus); -#else +#else /* !defined(HAVE_MODULE_DIRAUTH) */ static inline int dirserv_load_fingerprint_file(void) { @@ -107,6 +107,12 @@ dirserv_set_node_flags_from_authoritative_status(node_t *node, (void)node; (void)authstatus; } -#endif +#endif /* defined(HAVE_MODULE_DIRAUTH) */ + +#ifdef TOR_UNIT_TESTS +STATIC int dirserv_router_has_valid_address(routerinfo_t *ri); +STATIC bool dirserv_rejects_tor_version(const char *platform, + const char **msg); +#endif /* defined(TOR_UNIT_TESTS) */ #endif /* !defined(TOR_RECV_UPLOADS_H) */ diff --git a/src/feature/dirauth/reachability.h b/src/feature/dirauth/reachability.h index 8a83f0c493..46d0e7ee2e 100644 --- a/src/feature/dirauth/reachability.h +++ b/src/feature/dirauth/reachability.h @@ -34,7 +34,7 @@ void dirserv_orconn_tls_done(const tor_addr_t *addr, uint16_t or_port, const char *digest_rcvd, const struct ed25519_public_key_t *ed_id_rcvd); -#else +#else /* !defined(HAVE_MODULE_DIRAUTH) */ static inline int dirserv_should_launch_reachability_test(const routerinfo_t *ri, const routerinfo_t *ri_old) @@ -54,6 +54,6 @@ dirserv_orconn_tls_done(const tor_addr_t *addr, (void)digest_rcvd; (void)ed_id_rcvd; } -#endif +#endif /* defined(HAVE_MODULE_DIRAUTH) */ #endif /* !defined(TOR_REACHABILITY_H) */ diff --git a/src/feature/dirauth/shared_random.c b/src/feature/dirauth/shared_random.c index 5ccf1a95e5..a45f0a29c3 100644 --- a/src/feature/dirauth/shared_random.c +++ b/src/feature/dirauth/shared_random.c @@ -90,7 +90,7 @@ #include "core/or/or.h" #include "feature/dirauth/shared_random.h" #include "app/config/config.h" -#include "app/config/confparse.h" +#include "lib/confmgt/confparse.h" #include "lib/crypt_ops/crypto_rand.h" #include "lib/crypt_ops/crypto_util.h" #include "feature/nodelist/networkstatus.h" diff --git a/src/feature/dirauth/shared_random.h b/src/feature/dirauth/shared_random.h index 1d8fa89b0f..7ff9f15512 100644 --- a/src/feature/dirauth/shared_random.h +++ b/src/feature/dirauth/shared_random.h @@ -110,7 +110,7 @@ int sr_init(int save_to_disk); void sr_save_and_cleanup(void); void sr_act_post_consensus(const networkstatus_t *consensus); -#else /* !(defined(HAVE_MODULE_DIRAUTH)) */ +#else /* !defined(HAVE_MODULE_DIRAUTH) */ static inline int sr_init(int save_to_disk) diff --git a/src/feature/dirauth/shared_random_state.c b/src/feature/dirauth/shared_random_state.c index b669e3836e..76befb0f5f 100644 --- a/src/feature/dirauth/shared_random_state.c +++ b/src/feature/dirauth/shared_random_state.c @@ -12,7 +12,7 @@ #include "core/or/or.h" #include "app/config/config.h" -#include "app/config/confparse.h" +#include "lib/confmgt/confparse.h" #include "lib/crypt_ops/crypto_util.h" #include "feature/dirauth/dirvote.h" #include "feature/nodelist/networkstatus.h" @@ -51,24 +51,21 @@ static const char dstate_cur_srv_key[] = "SharedRandCurrentValue"; * members with CONF_CHECK_VAR_TYPE. */ DUMMY_TYPECHECK_INSTANCE(sr_disk_state_t); -/* These next two are duplicates or near-duplicates from config.c */ -#define VAR(name, conftype, member, initvalue) \ - { name, CONFIG_TYPE_ ## conftype, offsetof(sr_disk_state_t, member), \ - initvalue CONF_TEST_MEMBERS(sr_disk_state_t, conftype, member) } -/* As VAR, but the option name and member name are the same. */ -#define V(member, conftype, initvalue) \ +#define VAR(varname,conftype,member,initvalue) \ + CONFIG_VAR_ETYPE(sr_disk_state_t, varname, conftype, member, 0, initvalue) +#define V(member,conftype,initvalue) \ VAR(#member, conftype, member, initvalue) + /* Our persistent state magic number. */ #define SR_DISK_STATE_MAGIC 0x98AB1254 static int disk_state_validate_cb(void *old_state, void *state, void *default_state, int from_setconf, char **msg); -static void disk_state_free_cb(void *); /* Array of variables that are saved to disk as a persistent state. */ -static config_var_t state_vars[] = { - V(Version, UINT, "0"), +static const config_var_t state_vars[] = { + V(Version, POSINT, "0"), V(TorVersion, STRING, NULL), V(ValidAfter, ISOTIME, NULL), V(ValidUntil, ISOTIME, NULL), @@ -83,25 +80,43 @@ static config_var_t state_vars[] = { /* "Extra" variable in the state that receives lines we can't parse. This * lets us preserve options from versions of Tor newer than us. */ -static config_var_t state_extra_var = { - "__extra", CONFIG_TYPE_LINELIST, - offsetof(sr_disk_state_t, ExtraLines), NULL - CONF_TEST_MEMBERS(sr_disk_state_t, LINELIST, ExtraLines) +static const struct_member_t state_extra_var = { + .name = "__extra", + .type = CONFIG_TYPE_LINELIST, + .offset = offsetof(sr_disk_state_t, ExtraLines), }; /* Configuration format of sr_disk_state_t. */ static const config_format_t state_format = { sizeof(sr_disk_state_t), - SR_DISK_STATE_MAGIC, - offsetof(sr_disk_state_t, magic_), + { + "sr_disk_state_t", + SR_DISK_STATE_MAGIC, + offsetof(sr_disk_state_t, magic_), + }, NULL, NULL, state_vars, disk_state_validate_cb, - disk_state_free_cb, + NULL, &state_extra_var, + -1, }; +/* Global configuration manager for the shared-random state file */ +static config_mgr_t *shared_random_state_mgr = NULL; + +/** Return the configuration manager for the shared-random state file. */ +static const config_mgr_t * +get_srs_mgr(void) +{ + if (PREDICT_UNLIKELY(shared_random_state_mgr == NULL)) { + shared_random_state_mgr = config_mgr_new(&state_format); + config_mgr_freeze(shared_random_state_mgr); + } + return shared_random_state_mgr; +} + static void state_query_del_(sr_state_object_t obj_type, void *data); /* Return a string representation of a protocol phase. */ @@ -263,23 +278,22 @@ disk_state_free_(sr_disk_state_t *state) if (state == NULL) { return; } - config_free(&state_format, state); + config_free(get_srs_mgr(), state); } /* Allocate a new disk state, initialize it and return it. */ static sr_disk_state_t * disk_state_new(time_t now) { - sr_disk_state_t *new_state = tor_malloc_zero(sizeof(*new_state)); + sr_disk_state_t *new_state = config_new(get_srs_mgr()); - new_state->magic_ = SR_DISK_STATE_MAGIC; new_state->Version = SR_PROTO_VERSION; new_state->TorVersion = tor_strdup(get_version()); new_state->ValidUntil = get_state_valid_until_time(now); new_state->ValidAfter = now; /* Init config format. */ - config_init(&state_format, new_state); + config_init(get_srs_mgr(), new_state); return new_state; } @@ -347,12 +361,6 @@ disk_state_validate_cb(void *old_state, void *state, void *default_state, return 0; } -static void -disk_state_free_cb(void *state) -{ - disk_state_free_(state); -} - /* Parse the Commit line(s) in the disk state and translate them to the * the memory state. Return 0 on success else -1 on error. */ static int @@ -583,11 +591,12 @@ disk_state_reset(void) config_free_lines(sr_disk_state->ExtraLines); tor_free(sr_disk_state->TorVersion); - /* Clean up the struct */ - memset(sr_disk_state, 0, sizeof(*sr_disk_state)); + /* Clear other fields. */ + sr_disk_state->ValidAfter = 0; + sr_disk_state->ValidUntil = 0; + sr_disk_state->Version = 0; /* Reset it with useful data */ - sr_disk_state->magic_ = SR_DISK_STATE_MAGIC; sr_disk_state->TorVersion = tor_strdup(get_version()); } @@ -682,7 +691,7 @@ disk_state_load_from_disk_impl(const char *fname) } disk_state = disk_state_new(time(NULL)); - config_assign(&state_format, disk_state, lines, 0, &errmsg); + config_assign(get_srs_mgr(), disk_state, lines, 0, &errmsg); config_free_lines(lines); if (errmsg) { log_warn(LD_DIR, "SR: Reading state error: %s", errmsg); @@ -735,7 +744,7 @@ disk_state_save_to_disk(void) /* Make sure that our disk state is up to date with our memory state * before saving it to disk. */ disk_state_update(); - state = config_dump(&state_format, NULL, sr_disk_state, 0, 0); + state = config_dump(get_srs_mgr(), NULL, sr_disk_state, 0, 0); format_local_iso_time(tbuf, now); tor_asprintf(&content, "# Tor shared random state file last generated on %s " @@ -1277,6 +1286,7 @@ sr_state_free_all(void) /* Nullify our global state. */ sr_state = NULL; sr_disk_state = NULL; + config_mgr_free(shared_random_state_mgr); } /* Save our current state in memory to disk. */ diff --git a/src/feature/dircache/conscache.c b/src/feature/dircache/conscache.c index cf4fe8701d..2ec9981c03 100644 --- a/src/feature/dircache/conscache.c +++ b/src/feature/dircache/conscache.c @@ -92,7 +92,7 @@ consensus_cache_open(const char *subdir, int max_entries) */ #define VERY_LARGE_STORAGEDIR_LIMIT (1000*1000) storagedir_max_entries = VERY_LARGE_STORAGEDIR_LIMIT; -#else /* !(defined(MUST_UNMAP_TO_UNLINK)) */ +#else /* !defined(MUST_UNMAP_TO_UNLINK) */ /* Otherwise, we can just tell the storagedir to use the same limits * as this cache. */ storagedir_max_entries = max_entries; diff --git a/src/feature/dircache/dircache.c b/src/feature/dircache/dircache.c index 1b36f716f4..d4d0ad9939 100644 --- a/src/feature/dircache/dircache.c +++ b/src/feature/dircache/dircache.c @@ -1390,9 +1390,11 @@ handle_get_hs_descriptor_v3(dir_connection_t *conn, const char *pubkey_str = NULL; const char *url = args->url; - /* Reject unencrypted dir connections */ - if (!connection_dir_is_encrypted(conn)) { - write_short_http_response(conn, 404, "Not found"); + /* Reject non anonymous dir connections (which also tests if encrypted). We + * do not allow single hop clients to query an HSDir. */ + if (!connection_dir_is_anonymous(conn)) { + write_short_http_response(conn, 503, + "Rejecting single hop HS v3 descriptor request"); goto done; } @@ -1632,10 +1634,15 @@ directory_handle_command_post,(dir_connection_t *conn, const char *headers, goto done; } - /* Handle HS descriptor publish request. */ - /* XXX: This should be disabled with a consensus param until we want to - * the prop224 be deployed and thus use. */ - if (connection_dir_is_encrypted(conn) && !strcmpstart(url, "/tor/hs/")) { + /* Handle HS descriptor publish request. We force an anonymous connection + * (which also tests for encrypted). We do not allow single-hop client to + * post a descriptor onto an HSDir. */ + if (!strcmpstart(url, "/tor/hs/")) { + if (!connection_dir_is_anonymous(conn)) { + write_short_http_response(conn, 503, + "Rejecting single hop HS descriptor post"); + goto done; + } const char *msg = "HS descriptor stored successfully."; /* We most probably have a publish request for an HS descriptor. */ diff --git a/src/feature/dircommon/directory.c b/src/feature/dircommon/directory.c index 9e6f72e9ac..8e5b413326 100644 --- a/src/feature/dircommon/directory.c +++ b/src/feature/dircommon/directory.c @@ -7,6 +7,10 @@ #include "app/config/config.h" #include "core/mainloop/connection.h" +#include "core/or/circuitlist.h" +#include "core/or/connection_edge.h" +#include "core/or/connection_or.h" +#include "core/or/channeltls.h" #include "feature/dircache/dircache.h" #include "feature/dircache/dirserv.h" #include "feature/dirclient/dirclient.h" @@ -15,6 +19,10 @@ #include "feature/stats/geoip_stats.h" #include "lib/compress/compress.h" +#include "core/or/circuit_st.h" +#include "core/or/or_circuit_st.h" +#include "core/or/edge_connection_st.h" +#include "core/or/or_connection_st.h" #include "feature/dircommon/dir_connection_st.h" #include "feature/nodelist/routerinfo_st.h" @@ -167,6 +175,82 @@ connection_dir_is_encrypted(const dir_connection_t *conn) return TO_CONN(conn)->linked; } +/** Return true iff the given directory connection <b>dir_conn</b> is + * anonymous, that is, it is on a circuit via a public relay and not directly + * from a client or bridge. + * + * For client circuits via relays: true for 2-hop+ paths. + * For client circuits via bridges: true for 3-hop+ paths. + * + * This first test if the connection is encrypted since it is a strong + * requirement for anonymity. */ +bool +connection_dir_is_anonymous(const dir_connection_t *dir_conn) +{ + const connection_t *conn, *linked_conn; + const edge_connection_t *edge_conn; + const circuit_t *circ; + + tor_assert(dir_conn); + + if (!connection_dir_is_encrypted(dir_conn)) { + return false; + } + + /* + * Buckle up, we'll do a deep dive into the connection in order to get the + * final connection channel of that connection in order to figure out if + * this is a client or relay link. + * + * We go: dir_conn -> linked_conn -> edge_conn -> on_circuit -> p_chan. + */ + + conn = TO_CONN(dir_conn); + linked_conn = conn->linked_conn; + + /* The dir connection should be connected to an edge connection. It can not + * be closed or marked for close. */ + if (linked_conn == NULL || linked_conn->magic != EDGE_CONNECTION_MAGIC || + conn->linked_conn_is_closed || conn->linked_conn->marked_for_close) { + log_debug(LD_DIR, "Directory connection is not anonymous: " + "not linked to edge"); + return false; + } + + edge_conn = TO_EDGE_CONN((connection_t *) linked_conn); + circ = edge_conn->on_circuit; + + /* Can't be a circuit we initiated and without a circuit, no channel. */ + if (circ == NULL || CIRCUIT_IS_ORIGIN(circ)) { + log_debug(LD_DIR, "Directory connection is not anonymous: " + "not on OR circuit"); + return false; + } + + /* It is possible that the circuit was closed because one of the channel was + * closed or a DESTROY cell was received. Either way, this connection can + * not continue so return that it is not anonymous since we can not know for + * sure if it is. */ + if (circ->marked_for_close) { + log_debug(LD_DIR, "Directory connection is not anonymous: " + "circuit marked for close"); + return false; + } + + /* Get the previous channel to learn if it is a client or relay link. We + * BUG() because if the circuit is not mark for close, we ought to have a + * p_chan else we have a code flow issue. */ + if (BUG(CONST_TO_OR_CIRCUIT(circ)->p_chan == NULL)) { + log_debug(LD_DIR, "Directory connection is not anonymous: " + "no p_chan on circuit"); + return false; + } + + /* Will be true if the channel is an unauthenticated peer which is only true + * for clients and bridges. */ + return !channel_is_client(CONST_TO_OR_CIRCUIT(circ)->p_chan); +} + /** Parse an HTTP request line at the start of a headers string. On failure, * return -1. On success, set *<b>command_out</b> to a copy of the HTTP * command ("get", "post", etc), set *<b>url_out</b> to a copy of the URL, and diff --git a/src/feature/dircommon/directory.h b/src/feature/dircommon/directory.h index ba3f8c1b0e..4fc743ad3d 100644 --- a/src/feature/dircommon/directory.h +++ b/src/feature/dircommon/directory.h @@ -94,6 +94,7 @@ int parse_http_command(const char *headers, char *http_get_header(const char *headers, const char *which); int connection_dir_is_encrypted(const dir_connection_t *conn); +bool connection_dir_is_anonymous(const dir_connection_t *conn); int connection_dir_reached_eof(dir_connection_t *conn); int connection_dir_process_inbuf(dir_connection_t *conn); int connection_dir_finished_flushing(dir_connection_t *conn); diff --git a/src/feature/dirparse/microdesc_parse.c b/src/feature/dirparse/microdesc_parse.c index 22cc1e272e..4bb4db7821 100644 --- a/src/feature/dirparse/microdesc_parse.c +++ b/src/feature/dirparse/microdesc_parse.c @@ -92,6 +92,190 @@ find_start_of_next_microdesc(const char *s, const char *eos) #undef NEXT_LINE } +static inline int +policy_is_reject_star_or_null(struct short_policy_t *policy) +{ + return !policy || short_policy_is_reject_star(policy); +} + +/** + * Return a human-readable description of a given saved_location_t. + * Never returns NULL. + **/ +static const char * +saved_location_to_string(saved_location_t where) +{ + const char *location; + switch (where) { + case SAVED_NOWHERE: + location = "download or generated string"; + break; + case SAVED_IN_CACHE: + location = "cache"; + break; + case SAVED_IN_JOURNAL: + location = "journal"; + break; + default: + location = "unknown location"; + break; + } + return location; +} + +/** + * Given a microdescriptor stored in <b>where</b> which starts at <b>s</b>, + * which ends at <b>start_of_next_microdescriptor</b>, and which is located + * within a larger document beginning at <b>start</b>: Fill in the body, + * bodylen, bodylen, saved_location, off, and digest fields of <b>md</b> as + * appropriate. + * + * The body field will be an alias within <b>s</b> if <b>saved_location</b> + * is SAVED_IN_CACHE, and will be copied into body and nul-terminated + * otherwise. + **/ +static int +microdesc_extract_body(microdesc_t *md, + const char *start, + const char *s, const char *start_of_next_microdesc, + saved_location_t where) +{ + const bool copy_body = (where != SAVED_IN_CACHE); + + const char *cp = tor_memstr(s, start_of_next_microdesc-s, "onion-key"); + + const bool no_onion_key = (cp == NULL); + if (no_onion_key) { + cp = s; /* So that we have *some* junk to put in the body */ + } + + md->bodylen = start_of_next_microdesc - cp; + md->saved_location = where; + if (copy_body) + md->body = tor_memdup_nulterm(cp, md->bodylen); + else + md->body = (char*)cp; + md->off = cp - start; + + crypto_digest256(md->digest, md->body, md->bodylen, DIGEST_SHA256); + + return no_onion_key ? -1 : 0; +} + +/** + * Parse a microdescriptor which begins at <b>s</b> and ends at + * <b>start_of_next_microdesc. Store its fields into <b>md</b>. Use + * <b>where</b> for generating log information. If <b>allow_annotations</b> + * is true, then one or more annotations may precede the microdescriptor body + * proper. Use <b>area</b> for memory management, clearing it when done. + * + * On success, return 0; otherwise return -1. + **/ +static int +microdesc_parse_fields(microdesc_t *md, + memarea_t *area, + const char *s, const char *start_of_next_microdesc, + int allow_annotations, + saved_location_t where) +{ + smartlist_t *tokens = smartlist_new(); + int rv = -1; + int flags = allow_annotations ? TS_ANNOTATIONS_OK : 0; + directory_token_t *tok; + + if (tokenize_string(area, s, start_of_next_microdesc, tokens, + microdesc_token_table, flags)) { + log_warn(LD_DIR, "Unparseable microdescriptor found in %s", + saved_location_to_string(where)); + goto err; + } + + if ((tok = find_opt_by_keyword(tokens, A_LAST_LISTED))) { + if (parse_iso_time(tok->args[0], &md->last_listed)) { + log_warn(LD_DIR, "Bad last-listed time in microdescriptor"); + goto err; + } + } + + tok = find_by_keyword(tokens, K_ONION_KEY); + if (!crypto_pk_public_exponent_ok(tok->key)) { + log_warn(LD_DIR, + "Relay's onion key had invalid exponent."); + goto err; + } + md->onion_pkey = tor_memdup(tok->object_body, tok->object_size); + md->onion_pkey_len = tok->object_size; + crypto_pk_free(tok->key); + + if ((tok = find_opt_by_keyword(tokens, K_ONION_KEY_NTOR))) { + curve25519_public_key_t k; + tor_assert(tok->n_args >= 1); + if (curve25519_public_from_base64(&k, tok->args[0]) < 0) { + log_warn(LD_DIR, "Bogus ntor-onion-key in microdesc"); + goto err; + } + md->onion_curve25519_pkey = + tor_memdup(&k, sizeof(curve25519_public_key_t)); + } + + smartlist_t *id_lines = find_all_by_keyword(tokens, K_ID); + if (id_lines) { + SMARTLIST_FOREACH_BEGIN(id_lines, directory_token_t *, t) { + tor_assert(t->n_args >= 2); + if (!strcmp(t->args[0], "ed25519")) { + if (md->ed25519_identity_pkey) { + log_warn(LD_DIR, "Extra ed25519 key in microdesc"); + smartlist_free(id_lines); + goto err; + } + ed25519_public_key_t k; + if (ed25519_public_from_base64(&k, t->args[1])<0) { + log_warn(LD_DIR, "Bogus ed25519 key in microdesc"); + smartlist_free(id_lines); + goto err; + } + md->ed25519_identity_pkey = tor_memdup(&k, sizeof(k)); + } + } SMARTLIST_FOREACH_END(t); + smartlist_free(id_lines); + } + + { + smartlist_t *a_lines = find_all_by_keyword(tokens, K_A); + if (a_lines) { + find_single_ipv6_orport(a_lines, &md->ipv6_addr, &md->ipv6_orport); + smartlist_free(a_lines); + } + } + + if ((tok = find_opt_by_keyword(tokens, K_FAMILY))) { + md->family = nodefamily_parse(tok->args[0], + NULL, + NF_WARN_MALFORMED); + } + + if ((tok = find_opt_by_keyword(tokens, K_P))) { + md->exit_policy = parse_short_policy(tok->args[0]); + } + if ((tok = find_opt_by_keyword(tokens, K_P6))) { + md->ipv6_exit_policy = parse_short_policy(tok->args[0]); + } + + if (policy_is_reject_star_or_null(md->exit_policy) && + policy_is_reject_star_or_null(md->ipv6_exit_policy)) { + md->policy_is_reject_star = 1; + } + + rv = 0; + err: + + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); + memarea_clear(area); + smartlist_free(tokens); + + return rv; +} + /** Parse as many microdescriptors as are found from the string starting at * <b>s</b> and ending at <b>eos</b>. If allow_annotations is set, read any * annotations we recognize and ignore ones we don't. @@ -109,16 +293,11 @@ microdescs_parse_from_string(const char *s, const char *eos, saved_location_t where, smartlist_t *invalid_digests_out) { - smartlist_t *tokens; smartlist_t *result; microdesc_t *md = NULL; memarea_t *area; const char *start = s; const char *start_of_next_microdesc; - int flags = allow_annotations ? TS_ANNOTATIONS_OK : 0; - const int copy_body = (where != SAVED_IN_CACHE); - - directory_token_t *tok; if (!eos) eos = s + strlen(s); @@ -126,151 +305,47 @@ microdescs_parse_from_string(const char *s, const char *eos, s = eat_whitespace_eos(s, eos); area = memarea_new(); result = smartlist_new(); - tokens = smartlist_new(); while (s < eos) { - int okay = 0; + bool okay = false; start_of_next_microdesc = find_start_of_next_microdesc(s, eos); if (!start_of_next_microdesc) start_of_next_microdesc = eos; md = tor_malloc_zero(sizeof(microdesc_t)); + uint8_t md_digest[DIGEST256_LEN]; { - const char *cp = tor_memstr(s, start_of_next_microdesc-s, - "onion-key"); - const int no_onion_key = (cp == NULL); - if (no_onion_key) { - cp = s; /* So that we have *some* junk to put in the body */ - } + const bool body_not_found = + microdesc_extract_body(md, start, s, + start_of_next_microdesc, + where) < 0; - md->bodylen = start_of_next_microdesc - cp; - md->saved_location = where; - if (copy_body) - md->body = tor_memdup_nulterm(cp, md->bodylen); - else - md->body = (char*)cp; - md->off = cp - start; - crypto_digest256(md->digest, md->body, md->bodylen, DIGEST_SHA256); - if (no_onion_key) { + memcpy(md_digest, md->digest, DIGEST256_LEN); + if (body_not_found) { log_fn(LOG_PROTOCOL_WARN, LD_DIR, "Malformed or truncated descriptor"); goto next; } } - if (tokenize_string(area, s, start_of_next_microdesc, tokens, - microdesc_token_table, flags)) { - const char *location; - switch (where) { - case SAVED_NOWHERE: - location = "download or generated string"; - break; - case SAVED_IN_CACHE: - location = "cache"; - break; - case SAVED_IN_JOURNAL: - location = "journal"; - break; - default: - location = "unknown location"; - break; - } - log_warn(LD_DIR, "Unparseable microdescriptor found in %s", location); - goto next; - } - - if ((tok = find_opt_by_keyword(tokens, A_LAST_LISTED))) { - if (parse_iso_time(tok->args[0], &md->last_listed)) { - log_warn(LD_DIR, "Bad last-listed time in microdescriptor"); - goto next; - } + if (microdesc_parse_fields(md, area, s, start_of_next_microdesc, + allow_annotations, where) == 0) { + smartlist_add(result, md); + md = NULL; // prevent free + okay = true; } - tok = find_by_keyword(tokens, K_ONION_KEY); - if (!crypto_pk_public_exponent_ok(tok->key)) { - log_warn(LD_DIR, - "Relay's onion key had invalid exponent."); - goto next; - } - md->onion_pkey = tor_memdup(tok->object_body, tok->object_size); - md->onion_pkey_len = tok->object_size; - crypto_pk_free(tok->key); - - if ((tok = find_opt_by_keyword(tokens, K_ONION_KEY_NTOR))) { - curve25519_public_key_t k; - tor_assert(tok->n_args >= 1); - if (curve25519_public_from_base64(&k, tok->args[0]) < 0) { - log_warn(LD_DIR, "Bogus ntor-onion-key in microdesc"); - goto next; - } - md->onion_curve25519_pkey = - tor_memdup(&k, sizeof(curve25519_public_key_t)); - } - - smartlist_t *id_lines = find_all_by_keyword(tokens, K_ID); - if (id_lines) { - SMARTLIST_FOREACH_BEGIN(id_lines, directory_token_t *, t) { - tor_assert(t->n_args >= 2); - if (!strcmp(t->args[0], "ed25519")) { - if (md->ed25519_identity_pkey) { - log_warn(LD_DIR, "Extra ed25519 key in microdesc"); - smartlist_free(id_lines); - goto next; - } - ed25519_public_key_t k; - if (ed25519_public_from_base64(&k, t->args[1])<0) { - log_warn(LD_DIR, "Bogus ed25519 key in microdesc"); - smartlist_free(id_lines); - goto next; - } - md->ed25519_identity_pkey = tor_memdup(&k, sizeof(k)); - } - } SMARTLIST_FOREACH_END(t); - smartlist_free(id_lines); - } - - { - smartlist_t *a_lines = find_all_by_keyword(tokens, K_A); - if (a_lines) { - find_single_ipv6_orport(a_lines, &md->ipv6_addr, &md->ipv6_orport); - smartlist_free(a_lines); - } - } - - if ((tok = find_opt_by_keyword(tokens, K_FAMILY))) { - md->family = nodefamily_parse(tok->args[0], - NULL, - NF_WARN_MALFORMED); - } - - if ((tok = find_opt_by_keyword(tokens, K_P))) { - md->exit_policy = parse_short_policy(tok->args[0]); - } - if ((tok = find_opt_by_keyword(tokens, K_P6))) { - md->ipv6_exit_policy = parse_short_policy(tok->args[0]); - } - - smartlist_add(result, md); - okay = 1; - - md = NULL; next: if (! okay && invalid_digests_out) { smartlist_add(invalid_digests_out, - tor_memdup(md->digest, DIGEST256_LEN)); + tor_memdup(md_digest, DIGEST256_LEN)); } microdesc_free(md); md = NULL; - - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); - memarea_clear(area); - smartlist_clear(tokens); s = start_of_next_microdesc; } - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); memarea_drop_all(area); - smartlist_free(tokens); return result; } diff --git a/src/feature/dirparse/unparseable.h b/src/feature/dirparse/unparseable.h index 49e047961f..36c6b5a1ec 100644 --- a/src/feature/dirparse/unparseable.h +++ b/src/feature/dirparse/unparseable.h @@ -26,7 +26,7 @@ void dump_desc_init(void); log_debug(LD_MM, "Area for %s has %lu allocated; using %lu.", \ name, (unsigned long)alloc, (unsigned long)used); \ STMT_END -#else /* !(defined(DEBUG_AREA_ALLOC)) */ +#else /* !defined(DEBUG_AREA_ALLOC) */ #define DUMP_AREA(a,name) STMT_NIL #endif /* defined(DEBUG_AREA_ALLOC) */ diff --git a/src/feature/hibernate/hibernate.c b/src/feature/hibernate/hibernate.c index 674fe3c813..f7847d9a16 100644 --- a/src/feature/hibernate/hibernate.c +++ b/src/feature/hibernate/hibernate.c @@ -67,7 +67,7 @@ static hibernate_state_t hibernate_state = HIBERNATE_STATE_INITIAL; /** If are hibernating, when do we plan to wake up? Set to 0 if we * aren't hibernating. */ static time_t hibernate_end_time = 0; -/** If we are shutting down, when do we plan finally exit? Set to 0 if we +/** If we are shutting down, when do we plan to finally exit? Set to 0 if we * aren't shutting down. (This is obsolete; scheduled shutdowns are supposed * to happen from mainloop_schedule_shutdown() now.) */ static time_t shutdown_time = 0; @@ -562,7 +562,7 @@ time_to_record_bandwidth_usage(time_t now) /* Note every 600 sec */ #define NOTE_INTERVAL (600) /* Or every 20 megabytes */ -#define NOTE_BYTES 20*(1024*1024) +#define NOTE_BYTES (20*1024*1024) static uint64_t last_read_bytes_noted = 0; static uint64_t last_written_bytes_noted = 0; static time_t last_time_noted = 0; @@ -815,7 +815,7 @@ hibernate_soft_limit_reached(void) * We want to stop accepting connections when ALL of the following are true: * - We expect to use up the remaining bytes in under 3 hours * - We have used up 95% of our bytes. - * - We have less than 500MB of bytes left. + * - We have less than 500MBytes left. */ uint64_t soft_limit = (uint64_t) (acct_max * SOFT_LIM_PCT); if (acct_max > SOFT_LIM_BYTES && acct_max - SOFT_LIM_BYTES > soft_limit) { diff --git a/src/feature/hs/hs_cache.c b/src/feature/hs/hs_cache.c index 05f9940ae6..9817113b23 100644 --- a/src/feature/hs/hs_cache.c +++ b/src/feature/hs/hs_cache.c @@ -710,6 +710,11 @@ cache_clean_v3_as_client(time_t now) MAP_DEL_CURRENT(key); entry_size = cache_get_client_entry_size(entry); bytes_removed += entry_size; + /* We just removed an old descriptor. We need to close all intro circuits + * so we don't have leftovers that can be selected while lacking a + * descriptor. We leave the rendezvous circuits opened because they could + * be in use. */ + hs_client_close_intro_circuits_from_desc(entry->desc); /* Entry is not in the cache anymore, destroy it. */ cache_client_desc_free(entry); /* Update our OOM. We didn't use the remove() function because we are in diff --git a/src/feature/hs/hs_cell.c b/src/feature/hs/hs_cell.c index 69f1ccbef4..df59f73c1b 100644 --- a/src/feature/hs/hs_cell.c +++ b/src/feature/hs/hs_cell.c @@ -473,10 +473,132 @@ introduce1_set_legacy_id(trn_cell_introduce1_t *cell, } } +/* Build and add to the given DoS cell extension the given parameter type and + * value. */ +static void +build_establish_intro_dos_param(trn_cell_extension_dos_t *dos_ext, + uint8_t param_type, uint64_t param_value) +{ + trn_cell_extension_dos_param_t *dos_param = + trn_cell_extension_dos_param_new(); + + /* Extra safety. We should never send an unknown parameter type. */ + tor_assert(param_type == TRUNNEL_DOS_PARAM_TYPE_INTRO2_RATE_PER_SEC || + param_type == TRUNNEL_DOS_PARAM_TYPE_INTRO2_BURST_PER_SEC); + + trn_cell_extension_dos_param_set_type(dos_param, param_type); + trn_cell_extension_dos_param_set_value(dos_param, param_value); + trn_cell_extension_dos_add_params(dos_ext, dos_param); + + /* Not freeing the trunnel object because it is now owned by dos_ext. */ +} + +/* Build the DoS defense cell extension and put it in the given extensions + * object. Return 0 on success, -1 on failure. (Right now, failure is only + * possible if there is a bug.) */ +static int +build_establish_intro_dos_extension(const hs_service_config_t *service_config, + trn_cell_extension_t *extensions) +{ + ssize_t ret; + size_t dos_ext_encoded_len; + uint8_t *field_array; + trn_cell_extension_field_t *field = NULL; + trn_cell_extension_dos_t *dos_ext = NULL; + + tor_assert(service_config); + tor_assert(extensions); + + /* We are creating a cell extension field of the type DoS. */ + field = trn_cell_extension_field_new(); + trn_cell_extension_field_set_field_type(field, + TRUNNEL_CELL_EXTENSION_TYPE_DOS); + + /* Build DoS extension field. We will put in two parameters. */ + dos_ext = trn_cell_extension_dos_new(); + trn_cell_extension_dos_set_n_params(dos_ext, 2); + + /* Build DoS parameter INTRO2 rate per second. */ + build_establish_intro_dos_param(dos_ext, + TRUNNEL_DOS_PARAM_TYPE_INTRO2_RATE_PER_SEC, + service_config->intro_dos_rate_per_sec); + /* Build DoS parameter INTRO2 burst per second. */ + build_establish_intro_dos_param(dos_ext, + TRUNNEL_DOS_PARAM_TYPE_INTRO2_BURST_PER_SEC, + service_config->intro_dos_burst_per_sec); + + /* Set the field with the encoded DoS extension. */ + ret = trn_cell_extension_dos_encoded_len(dos_ext); + if (BUG(ret <= 0)) { + goto err; + } + dos_ext_encoded_len = ret; + /* Set length field and the field array size length. */ + trn_cell_extension_field_set_field_len(field, dos_ext_encoded_len); + trn_cell_extension_field_setlen_field(field, dos_ext_encoded_len); + /* Encode the DoS extension into the cell extension field. */ + field_array = trn_cell_extension_field_getarray_field(field); + ret = trn_cell_extension_dos_encode(field_array, + trn_cell_extension_field_getlen_field(field), dos_ext); + if (BUG(ret <= 0)) { + goto err; + } + tor_assert(ret == (ssize_t) dos_ext_encoded_len); + + /* Finally, encode field into the cell extension. */ + trn_cell_extension_add_fields(extensions, field); + + /* We've just add an extension field to the cell extensions so increment the + * total number. */ + trn_cell_extension_set_num(extensions, + trn_cell_extension_get_num(extensions) + 1); + + /* Cleanup. DoS extension has been encoded at this point. */ + trn_cell_extension_dos_free(dos_ext); + + return 0; + + err: + trn_cell_extension_field_free(field); + trn_cell_extension_dos_free(dos_ext); + return -1; +} + /* ========== */ /* Public API */ /* ========== */ +/* Allocate and build all the ESTABLISH_INTRO cell extension. The given + * extensions pointer is always set to a valid cell extension object. */ +STATIC trn_cell_extension_t * +build_establish_intro_extensions(const hs_service_config_t *service_config, + const hs_service_intro_point_t *ip) +{ + int ret; + trn_cell_extension_t *extensions; + + tor_assert(service_config); + tor_assert(ip); + + extensions = trn_cell_extension_new(); + trn_cell_extension_set_num(extensions, 0); + + /* If the defense has been enabled service side (by the operator with a + * torrc option) and the intro point does support it. */ + if (service_config->has_dos_defense_enabled && + ip->support_intro2_dos_defense) { + /* This function takes care to increment the number of extensions. */ + ret = build_establish_intro_dos_extension(service_config, extensions); + if (ret < 0) { + /* Return no extensions on error. */ + goto end; + } + } + + end: + return extensions; +} + /* Build an ESTABLISH_INTRO cell with the given circuit nonce and intro point * object. The encoded cell is put in cell_out that MUST at least be of the * size of RELAY_PAYLOAD_SIZE. Return the encoded cell length on success else @@ -484,15 +606,17 @@ introduce1_set_legacy_id(trn_cell_introduce1_t *cell, * legacy cell creation. */ ssize_t hs_cell_build_establish_intro(const char *circ_nonce, + const hs_service_config_t *service_config, const hs_service_intro_point_t *ip, uint8_t *cell_out) { ssize_t cell_len = -1; uint16_t sig_len = ED25519_SIG_LEN; - trn_cell_extension_t *ext; trn_cell_establish_intro_t *cell = NULL; + trn_cell_extension_t *extensions; tor_assert(circ_nonce); + tor_assert(service_config); tor_assert(ip); /* Quickly handle the legacy IP. */ @@ -505,11 +629,12 @@ hs_cell_build_establish_intro(const char *circ_nonce, goto done; } + /* Build the extensions, if any. */ + extensions = build_establish_intro_extensions(service_config, ip); + /* Set extension data. None used here. */ - ext = trn_cell_extension_new(); - trn_cell_extension_set_num(ext, 0); cell = trn_cell_establish_intro_new(); - trn_cell_establish_intro_set_extensions(cell, ext); + trn_cell_establish_intro_set_extensions(cell, extensions); /* Set signature size. Array is then allocated in the cell. We need to do * this early so we can use trunnel API to get the signature length. */ trn_cell_establish_intro_set_sig_len(cell, sig_len); @@ -956,4 +1081,3 @@ hs_cell_introduce1_data_clear(hs_cell_introduce1_data_t *data) /* The data object has no ownership of any members. */ memwipe(data, 0, sizeof(hs_cell_introduce1_data_t)); } - diff --git a/src/feature/hs/hs_cell.h b/src/feature/hs/hs_cell.h index 9569de535e..864b6fda5f 100644 --- a/src/feature/hs/hs_cell.h +++ b/src/feature/hs/hs_cell.h @@ -79,6 +79,7 @@ typedef struct hs_cell_introduce2_data_t { /* Build cell API. */ ssize_t hs_cell_build_establish_intro(const char *circ_nonce, + const hs_service_config_t *config, const hs_service_intro_point_t *ip, uint8_t *cell_out); ssize_t hs_cell_build_rendezvous1(const uint8_t *rendezvous_cookie, @@ -105,5 +106,15 @@ int hs_cell_parse_rendezvous2(const uint8_t *payload, size_t payload_len, /* Util API. */ void hs_cell_introduce1_data_clear(hs_cell_introduce1_data_t *data); +#ifdef TOR_UNIT_TESTS + +#include "trunnel/hs/cell_common.h" + +STATIC trn_cell_extension_t * +build_establish_intro_extensions(const hs_service_config_t *service_config, + const hs_service_intro_point_t *ip); + +#endif /* defined(TOR_UNIT_TESTS) */ + #endif /* !defined(TOR_HS_CELL_H) */ diff --git a/src/feature/hs/hs_circuit.c b/src/feature/hs/hs_circuit.c index 716c4b1f17..5e213b5aba 100644 --- a/src/feature/hs/hs_circuit.c +++ b/src/feature/hs/hs_circuit.c @@ -259,8 +259,7 @@ create_rp_circuit_identifier(const hs_service_t *service, tor_assert(server_pk); tor_assert(keys); - ident = hs_ident_circuit_new(&service->keys.identity_pk, - HS_IDENT_CIRCUIT_RENDEZVOUS); + ident = hs_ident_circuit_new(&service->keys.identity_pk); /* Copy the RENDEZVOUS_COOKIE which is the unique identifier. */ memcpy(ident->rendezvous_cookie, rendezvous_cookie, sizeof(ident->rendezvous_cookie)); @@ -294,8 +293,7 @@ create_intro_circuit_identifier(const hs_service_t *service, tor_assert(service); tor_assert(ip); - ident = hs_ident_circuit_new(&service->keys.identity_pk, - HS_IDENT_CIRCUIT_INTRO); + ident = hs_ident_circuit_new(&service->keys.identity_pk); ed25519_pubkey_copy(&ident->intro_auth_pk, &ip->auth_key_kp.pubkey); return ident; @@ -319,7 +317,7 @@ send_establish_intro(const hs_service_t *service, /* Encode establish intro cell. */ cell_len = hs_cell_build_establish_intro(circ->cpath->prev->rend_circ_nonce, - ip, payload); + &service->config, ip, payload); if (cell_len < 0) { log_warn(LD_REND, "Unable to encode ESTABLISH_INTRO cell for service %s " "on circuit %u. Closing circuit.", @@ -389,10 +387,7 @@ launch_rendezvous_point_circuit(const hs_service_t *service, &data->onion_pk, service->config.is_single_onion); if (info == NULL) { - /* We are done here, we can't extend to the rendezvous point. - * If you're running an IPv6-only v3 single onion service on 0.3.2 or with - * 0.3.2 clients, and somehow disable the option check, it will fail here. - */ + /* We are done here, we can't extend to the rendezvous point. */ log_fn(LOG_PROTOCOL_WARN, LD_REND, "Not enough info to open a circuit to a rendezvous point for " "%s service %s.", diff --git a/src/feature/hs/hs_circuitmap.c b/src/feature/hs/hs_circuitmap.c index 5480d5eb84..e34f564fb4 100644 --- a/src/feature/hs/hs_circuitmap.c +++ b/src/feature/hs/hs_circuitmap.c @@ -272,6 +272,33 @@ hs_circuitmap_get_or_circuit(hs_token_type_t type, /**** Public relay-side getters: */ +/* Public function: Return v2 and v3 introduction circuit to this relay. + * Always return a newly allocated list for which it is the caller's + * responsability to free it. */ +smartlist_t * +hs_circuitmap_get_all_intro_circ_relay_side(void) +{ + circuit_t **iter; + smartlist_t *circuit_list = smartlist_new(); + + HT_FOREACH(iter, hs_circuitmap_ht, the_hs_circuitmap) { + circuit_t *circ = *iter; + + /* An origin circuit or purpose is wrong or the hs token is not set to be + * a v2 or v3 intro relay side type, we ignore the circuit. Else, we have + * a match so add it to our list. */ + if (CIRCUIT_IS_ORIGIN(circ) || + circ->purpose != CIRCUIT_PURPOSE_INTRO_POINT || + (circ->hs_token->type != HS_TOKEN_INTRO_V3_RELAY_SIDE && + circ->hs_token->type != HS_TOKEN_INTRO_V2_RELAY_SIDE)) { + continue; + } + smartlist_add(circuit_list, circ); + } + + return circuit_list; +} + /* Public function: Return a v3 introduction circuit to this relay with * <b>auth_key</b>. Return NULL if no such circuit is found in the * circuitmap. */ diff --git a/src/feature/hs/hs_circuitmap.h b/src/feature/hs/hs_circuitmap.h index c1bbb1ff1c..eac8230bbf 100644 --- a/src/feature/hs/hs_circuitmap.h +++ b/src/feature/hs/hs_circuitmap.h @@ -34,6 +34,8 @@ void hs_circuitmap_register_intro_circ_v2_relay_side(struct or_circuit_t *circ, void hs_circuitmap_register_intro_circ_v3_relay_side(struct or_circuit_t *circ, const ed25519_public_key_t *auth_key); +smartlist_t *hs_circuitmap_get_all_intro_circ_relay_side(void); + /** Public service-side API: */ struct origin_circuit_t * diff --git a/src/feature/hs/hs_common.c b/src/feature/hs/hs_common.c index a5747fe170..036d23a6b0 100644 --- a/src/feature/hs/hs_common.c +++ b/src/feature/hs/hs_common.c @@ -21,6 +21,7 @@ #include "feature/hs/hs_circuitmap.h" #include "feature/hs/hs_client.h" #include "feature/hs/hs_common.h" +#include "feature/hs/hs_dos.h" #include "feature/hs/hs_ident.h" #include "feature/hs/hs_service.h" #include "feature/hs_common/shared_random_client.h" @@ -30,6 +31,7 @@ #include "feature/nodelist/routerset.h" #include "feature/rend/rendcommon.h" #include "feature/rend/rendservice.h" +#include "feature/relay/routermode.h" #include "lib/crypt_ops/crypto_rand.h" #include "lib/crypt_ops/crypto_util.h" @@ -84,7 +86,7 @@ set_unix_port(edge_connection_t *conn, rend_service_port_config_t *p) return 0; } -#else /* !(defined(HAVE_SYS_UN_H)) */ +#else /* !defined(HAVE_SYS_UN_H) */ static int set_unix_port(edge_connection_t *conn, rend_service_port_config_t *p) diff --git a/src/feature/hs/hs_config.c b/src/feature/hs/hs_config.c index 87f6257591..3b6caaec6a 100644 --- a/src/feature/hs/hs_config.c +++ b/src/feature/hs/hs_config.c @@ -218,6 +218,9 @@ config_has_invalid_options(const config_line_t *line_, const char *opts_exclude_v2[] = { "HiddenServiceExportCircuitID", + "HiddenServiceEnableIntroDoSDefense", + "HiddenServiceEnableIntroDoSRatePerSec", + "HiddenServiceEnableIntroDoSBurstPerSec", NULL /* End marker. */ }; @@ -250,6 +253,16 @@ config_has_invalid_options(const config_line_t *line_, "version %" PRIu32 " of service in %s", opt, service->config.version, service->config.directory_path); + + if (!strcasecmp(line->key, "HiddenServiceAuthorizeClient")) { + /* Special case this v2 option so that we can offer alternatives. + * If more such special cases appear, it would be good to + * generalize the exception mechanism here. */ + log_warn(LD_CONFIG, "For v3 onion service client authorization, " + "please read the 'CLIENT AUTHORIZATION' section in the " + "manual."); + } + ret = 1; /* Continue the loop so we can find all possible options. */ continue; @@ -276,6 +289,15 @@ config_validate_service(const hs_service_config_t *config) goto invalid; } + /* DoS validation values. */ + if (config->has_dos_defense_enabled && + (config->intro_dos_burst_per_sec < config->intro_dos_rate_per_sec)) { + log_warn(LD_CONFIG, "Hidden service DoS defenses burst (%" PRIu32 ") can " + "not be smaller than the rate value (%" PRIu32 ").", + config->intro_dos_burst_per_sec, config->intro_dos_rate_per_sec); + goto invalid; + } + /* Valid. */ return 0; invalid: @@ -296,6 +318,8 @@ config_service_v3(const config_line_t *line_, { int have_num_ip = 0; bool export_circuit_id = false; /* just to detect duplicate options */ + bool dos_enabled = false, dos_rate_per_sec = false; + bool dos_burst_per_sec = false; const char *dup_opt_seen = NULL; const config_line_t *line; @@ -334,6 +358,52 @@ config_service_v3(const config_line_t *line_, export_circuit_id = true; continue; } + if (!strcasecmp(line->key, "HiddenServiceEnableIntroDoSDefense")) { + config->has_dos_defense_enabled = + (unsigned int) helper_parse_uint64(line->key, line->value, + HS_CONFIG_V3_DOS_DEFENSE_DEFAULT, + 1, &ok); + if (!ok || dos_enabled) { + if (dos_enabled) { + dup_opt_seen = line->key; + } + goto err; + } + dos_enabled = true; + continue; + } + if (!strcasecmp(line->key, "HiddenServiceEnableIntroDoSRatePerSec")) { + config->intro_dos_rate_per_sec = + (unsigned int) helper_parse_uint64(line->key, line->value, + HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_MIN, + HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_MAX, &ok); + if (!ok || dos_rate_per_sec) { + if (dos_rate_per_sec) { + dup_opt_seen = line->key; + } + goto err; + } + dos_rate_per_sec = true; + log_info(LD_REND, "Service INTRO2 DoS defenses rate set to: %" PRIu32, + config->intro_dos_rate_per_sec); + continue; + } + if (!strcasecmp(line->key, "HiddenServiceEnableIntroDoSBurstPerSec")) { + config->intro_dos_burst_per_sec = + (unsigned int) helper_parse_uint64(line->key, line->value, + HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_MIN, + HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_MAX, &ok); + if (!ok || dos_burst_per_sec) { + if (dos_burst_per_sec) { + dup_opt_seen = line->key; + } + goto err; + } + dos_burst_per_sec = true; + log_info(LD_REND, "Service INTRO2 DoS defenses burst set to: %" PRIu32, + config->intro_dos_burst_per_sec); + continue; + } } /* We do not load the key material for the service at this stage. This is diff --git a/src/feature/hs/hs_config.h b/src/feature/hs/hs_config.h index 040e451f13..beefc7a613 100644 --- a/src/feature/hs/hs_config.h +++ b/src/feature/hs/hs_config.h @@ -15,6 +15,15 @@ #define HS_CONFIG_MAX_STREAMS_PER_RDV_CIRCUIT 65535 /* Maximum number of intro points per version 3 services. */ #define HS_CONFIG_V3_MAX_INTRO_POINTS 20 +/* Default value for the introduction DoS defenses. The MIN/MAX are inclusive + * meaning they can be used as valid values. */ +#define HS_CONFIG_V3_DOS_DEFENSE_DEFAULT 0 +#define HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_DEFAULT 25 +#define HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_MIN 0 +#define HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_MAX INT32_MAX +#define HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_DEFAULT 200 +#define HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_MIN 0 +#define HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_MAX INT32_MAX /* API */ diff --git a/src/feature/hs/hs_descriptor.c b/src/feature/hs/hs_descriptor.c index a8796c0029..924ab3115e 100644 --- a/src/feature/hs/hs_descriptor.c +++ b/src/feature/hs/hs_descriptor.c @@ -1477,10 +1477,8 @@ decrypt_descriptor_cookie(const hs_descriptor_t *desc, */ MOCK_IMPL(STATIC size_t, decrypt_desc_layer,(const hs_descriptor_t *desc, - const uint8_t *encrypted_blob, - size_t encrypted_blob_size, const uint8_t *descriptor_cookie, - int is_superencrypted_layer, + bool is_superencrypted_layer, char **decrypted_out)) { uint8_t *decrypted = NULL; @@ -1490,6 +1488,12 @@ decrypt_desc_layer,(const hs_descriptor_t *desc, uint8_t mac_key[DIGEST256_LEN], our_mac[DIGEST256_LEN]; const uint8_t *salt, *encrypted, *desc_mac; size_t encrypted_len, result_len = 0; + const uint8_t *encrypted_blob = (is_superencrypted_layer) + ? desc->plaintext_data.superencrypted_blob + : desc->superencrypted_data.encrypted_blob; + size_t encrypted_blob_size = (is_superencrypted_layer) + ? desc->plaintext_data.superencrypted_blob_size + : desc->superencrypted_data.encrypted_blob_size; tor_assert(decrypted_out); tor_assert(desc); @@ -1603,9 +1607,8 @@ desc_decrypt_superencrypted(const hs_descriptor_t *desc, char **decrypted_out) tor_assert(decrypted_out); superencrypted_len = decrypt_desc_layer(desc, - desc->plaintext_data.superencrypted_blob, - desc->plaintext_data.superencrypted_blob_size, - NULL, 1, &superencrypted_plaintext); + NULL, + true, &superencrypted_plaintext); if (!superencrypted_len) { log_warn(LD_REND, "Decrypting superencrypted desc failed."); @@ -1654,9 +1657,9 @@ desc_decrypt_encrypted(const hs_descriptor_t *desc, } encrypted_len = decrypt_desc_layer(desc, - desc->superencrypted_data.encrypted_blob, - desc->superencrypted_data.encrypted_blob_size, - descriptor_cookie, 0, &encrypted_plaintext); + descriptor_cookie, + false, &encrypted_plaintext); + if (!encrypted_len) { goto err; } diff --git a/src/feature/hs/hs_descriptor.h b/src/feature/hs/hs_descriptor.h index dbe0cb1c94..0a843f4f3c 100644 --- a/src/feature/hs/hs_descriptor.h +++ b/src/feature/hs/hs_descriptor.h @@ -276,6 +276,7 @@ void hs_desc_authorized_client_free_(hs_desc_authorized_client_t *client); hs_desc_authorized_client_free_, (client)) hs_desc_authorized_client_t *hs_desc_build_fake_authorized_client(void); + void hs_desc_build_authorized_client(const uint8_t *subcredential, const curve25519_public_key_t * client_auth_pk, @@ -308,10 +309,8 @@ STATIC int desc_sig_is_valid(const char *b64_sig, const char *encoded_desc, size_t encoded_len); MOCK_DECL(STATIC size_t, decrypt_desc_layer,(const hs_descriptor_t *desc, - const uint8_t *encrypted_blob, - size_t encrypted_blob_size, const uint8_t *descriptor_cookie, - int is_superencrypted_layer, + bool is_superencrypted_layer, char **decrypted_out)); #endif /* defined(HS_DESCRIPTOR_PRIVATE) */ diff --git a/src/feature/hs/hs_dos.c b/src/feature/hs/hs_dos.c new file mode 100644 index 0000000000..19794e09d3 --- /dev/null +++ b/src/feature/hs/hs_dos.c @@ -0,0 +1,200 @@ +/* Copyright (c) 2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file hs_dos.c + * \brief Implement denial of service mitigation for the onion service + * subsystem. + * + * This module defenses: + * + * - Introduction Rate Limiting: If enabled by the consensus, an introduction + * point will rate limit client introduction towards the service (INTRODUCE2 + * cells). It uses a token bucket model with a rate and burst per second. + * + * Proposal 305 will expand this module by allowing an operator to define + * these values into the ESTABLISH_INTRO cell. Not yet implemented. + **/ + +#define HS_DOS_PRIVATE + +#include "core/or/or.h" +#include "app/config/config.h" + +#include "core/or/circuitlist.h" + +#include "feature/hs/hs_circuitmap.h" +#include "feature/nodelist/networkstatus.h" +#include "feature/relay/routermode.h" + +#include "lib/evloop/token_bucket.h" + +#include "feature/hs/hs_dos.h" + +/* Default value of the allowed INTRODUCE2 cell rate per second. Above that + * value per second, the introduction is denied. */ +#define HS_DOS_INTRODUCE_DEFAULT_CELL_RATE_PER_SEC 25 + +/* Default value of the allowed INTRODUCE2 cell burst per second. This is the + * maximum value a token bucket has per second. We thus allow up to this value + * of INTRODUCE2 cell per second but the bucket is refilled by the rate value + * but never goes above that burst value. */ +#define HS_DOS_INTRODUCE_DEFAULT_CELL_BURST_PER_SEC 200 + +/* Default value of the consensus parameter enabling or disabling the + * introduction DoS defense. Disabled by default. */ +#define HS_DOS_INTRODUCE_ENABLED_DEFAULT 0 + +/* Consensus parameters. The ESTABLISH_INTRO DoS cell extension have higher + * priority than these values. If no extension is sent, these are used only by + * the introduction point. */ +static uint32_t consensus_param_introduce_rate_per_sec = + HS_DOS_INTRODUCE_DEFAULT_CELL_RATE_PER_SEC; +static uint32_t consensus_param_introduce_burst_per_sec = + HS_DOS_INTRODUCE_DEFAULT_CELL_BURST_PER_SEC; +static uint32_t consensus_param_introduce_defense_enabled = + HS_DOS_INTRODUCE_ENABLED_DEFAULT; + +STATIC uint32_t +get_intro2_enable_consensus_param(const networkstatus_t *ns) +{ + return networkstatus_get_param(ns, "HiddenServiceEnableIntroDoSDefense", + HS_DOS_INTRODUCE_ENABLED_DEFAULT, 0, 1); +} + +/* Return the parameter for the introduction rate per sec. */ +STATIC uint32_t +get_intro2_rate_consensus_param(const networkstatus_t *ns) +{ + return networkstatus_get_param(ns, "HiddenServiceEnableIntroDoSRatePerSec", + HS_DOS_INTRODUCE_DEFAULT_CELL_RATE_PER_SEC, + 0, INT32_MAX); +} + +/* Return the parameter for the introduction burst per sec. */ +STATIC uint32_t +get_intro2_burst_consensus_param(const networkstatus_t *ns) +{ + return networkstatus_get_param(ns, "HiddenServiceEnableIntroDoSBurstPerSec", + HS_DOS_INTRODUCE_DEFAULT_CELL_BURST_PER_SEC, + 0, INT32_MAX); +} + +/* Go over all introduction circuit relay side and adjust their rate/burst + * values using the global parameters. This is called right after the + * consensus parameters might have changed. */ +static void +update_intro_circuits(void) +{ + /* Returns all HS version intro circuits. */ + smartlist_t *intro_circs = hs_circuitmap_get_all_intro_circ_relay_side(); + + SMARTLIST_FOREACH_BEGIN(intro_circs, circuit_t *, circ) { + /* Defenses might have been enabled or disabled. */ + TO_OR_CIRCUIT(circ)->introduce2_dos_defense_enabled = + consensus_param_introduce_defense_enabled; + /* Adjust the rate/burst value that might have changed. */ + token_bucket_ctr_adjust(&TO_OR_CIRCUIT(circ)->introduce2_bucket, + consensus_param_introduce_rate_per_sec, + consensus_param_introduce_burst_per_sec); + } SMARTLIST_FOREACH_END(circ); + + smartlist_free(intro_circs); +} + +/* Set consensus parameters. */ +static void +set_consensus_parameters(const networkstatus_t *ns) +{ + consensus_param_introduce_rate_per_sec = + get_intro2_rate_consensus_param(ns); + consensus_param_introduce_burst_per_sec = + get_intro2_burst_consensus_param(ns); + consensus_param_introduce_defense_enabled = + get_intro2_enable_consensus_param(ns); + + /* The above might have changed which means we need to go through all + * introduction circuits (relay side) and update the token buckets. */ + update_intro_circuits(); +} + +/* + * Public API. + */ + +/* Initialize the INTRODUCE2 token bucket for the DoS defenses using the + * consensus/default values. We might get a cell extension that changes those + * later but if we don't, the default or consensus parameters are used. */ +void +hs_dos_setup_default_intro2_defenses(or_circuit_t *circ) +{ + tor_assert(circ); + + circ->introduce2_dos_defense_enabled = + consensus_param_introduce_defense_enabled; + token_bucket_ctr_init(&circ->introduce2_bucket, + consensus_param_introduce_rate_per_sec, + consensus_param_introduce_burst_per_sec, + (uint32_t) approx_time()); +} + +/* Called when the consensus has changed. We might have new consensus + * parameters to look at. */ +void +hs_dos_consensus_has_changed(const networkstatus_t *ns) +{ + /* No point on updating these values if we are not a public relay that can + * be picked to be an introduction point. */ + if (!public_server_mode(get_options())) { + return; + } + + set_consensus_parameters(ns); +} + +/* Return true iff an INTRODUCE2 cell can be sent on the given service + * introduction circuit. */ +bool +hs_dos_can_send_intro2(or_circuit_t *s_intro_circ) +{ + tor_assert(s_intro_circ); + + /* Allow to send the cell if the DoS defenses are disabled on the circuit. + * This can be set by the consensus, the ESTABLISH_INTRO cell extension or + * the hardcoded values in tor code. */ + if (!s_intro_circ->introduce2_dos_defense_enabled) { + return true; + } + + /* Should not happen but if so, scream loudly. */ + if (BUG(TO_CIRCUIT(s_intro_circ)->purpose != CIRCUIT_PURPOSE_INTRO_POINT)) { + return false; + } + + /* This is called just after we got a valid and parsed INTRODUCE1 cell. The + * service has been found and we have its introduction circuit. + * + * First, the INTRODUCE2 bucket will be refilled (if any). Then, decremented + * because we are about to send or not the cell we just got. Finally, + * evaluate if we can send it based on our token bucket state. */ + + /* Refill INTRODUCE2 bucket. */ + token_bucket_ctr_refill(&s_intro_circ->introduce2_bucket, + (uint32_t) approx_time()); + + /* Decrement the bucket for this valid INTRODUCE1 cell we just got. Don't + * underflow else we end up with a too big of a bucket. */ + if (token_bucket_ctr_get(&s_intro_circ->introduce2_bucket) > 0) { + token_bucket_ctr_dec(&s_intro_circ->introduce2_bucket, 1); + } + + /* Finally, we can send a new INTRODUCE2 if there are still tokens. */ + return token_bucket_ctr_get(&s_intro_circ->introduce2_bucket) > 0; +} + +/* Initialize the onion service Denial of Service subsystem. */ +void +hs_dos_init(void) +{ + set_consensus_parameters(NULL); +} diff --git a/src/feature/hs/hs_dos.h b/src/feature/hs/hs_dos.h new file mode 100644 index 0000000000..ccf4e27179 --- /dev/null +++ b/src/feature/hs/hs_dos.h @@ -0,0 +1,39 @@ +/* Copyright (c) 2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file hs_dos.h + * \brief Header file containing denial of service defenses for the HS + * subsystem for all versions. + **/ + +#ifndef TOR_HS_DOS_H +#define TOR_HS_DOS_H + +#include "core/or/or_circuit_st.h" + +#include "feature/nodelist/networkstatus_st.h" + +/* Init */ +void hs_dos_init(void); + +/* Consensus. */ +void hs_dos_consensus_has_changed(const networkstatus_t *ns); + +/* Introduction Point. */ +bool hs_dos_can_send_intro2(or_circuit_t *s_intro_circ); +void hs_dos_setup_default_intro2_defenses(or_circuit_t *circ); + +#ifdef HS_DOS_PRIVATE + +#ifdef TOR_UNIT_TESTS + +STATIC uint32_t get_intro2_enable_consensus_param(const networkstatus_t *ns); +STATIC uint32_t get_intro2_rate_consensus_param(const networkstatus_t *ns); +STATIC uint32_t get_intro2_burst_consensus_param(const networkstatus_t *ns); + +#endif /* defined(TOR_UNIT_TESTS) */ + +#endif /* defined(HS_DOS_PRIVATE) */ + +#endif /* !defined(TOR_HS_DOS_H) */ diff --git a/src/feature/hs/hs_ident.c b/src/feature/hs/hs_ident.c index 8fd0013941..a00e55ec23 100644 --- a/src/feature/hs/hs_ident.c +++ b/src/feature/hs/hs_ident.c @@ -13,14 +13,10 @@ /* Return a newly allocated circuit identifier. The given public key is copied * identity_pk into the identifier. */ hs_ident_circuit_t * -hs_ident_circuit_new(const ed25519_public_key_t *identity_pk, - hs_ident_circuit_type_t circuit_type) +hs_ident_circuit_new(const ed25519_public_key_t *identity_pk) { - tor_assert(circuit_type == HS_IDENT_CIRCUIT_INTRO || - circuit_type == HS_IDENT_CIRCUIT_RENDEZVOUS); hs_ident_circuit_t *ident = tor_malloc_zero(sizeof(*ident)); ed25519_pubkey_copy(&ident->identity_pk, identity_pk); - ident->circuit_type = circuit_type; return ident; } diff --git a/src/feature/hs/hs_ident.h b/src/feature/hs/hs_ident.h index 8c46936a1e..82ca50f6b5 100644 --- a/src/feature/hs/hs_ident.h +++ b/src/feature/hs/hs_ident.h @@ -44,13 +44,6 @@ typedef struct hs_ident_circuit_t { * the one found in the onion address. */ ed25519_public_key_t identity_pk; - /* (All circuit) The type of circuit this identifier is attached to. - * Accessors of the fields in this object assert non fatal on this circuit - * type. In other words, if a rendezvous field is being accessed, the - * circuit type MUST BE of type HS_IDENT_CIRCUIT_RENDEZVOUS. This value is - * set when an object is initialized in its constructor. */ - hs_ident_circuit_type_t circuit_type; - /* (All circuit) Introduction point authentication key. It's also needed on * the rendezvous circuit for the ntor handshake. It's used as the unique key * of the introduction point so it should not be shared between multiple @@ -120,8 +113,7 @@ typedef struct hs_ident_edge_conn_t { /* Circuit identifier API. */ hs_ident_circuit_t *hs_ident_circuit_new( - const ed25519_public_key_t *identity_pk, - hs_ident_circuit_type_t circuit_type); + const ed25519_public_key_t *identity_pk); void hs_ident_circuit_free_(hs_ident_circuit_t *ident); #define hs_ident_circuit_free(id) \ FREE_AND_NULL(hs_ident_circuit_t, hs_ident_circuit_free_, (id)) diff --git a/src/feature/hs/hs_intropoint.c b/src/feature/hs/hs_intropoint.c index 9333060e7e..fe8486b1a6 100644 --- a/src/feature/hs/hs_intropoint.c +++ b/src/feature/hs/hs_intropoint.c @@ -10,6 +10,7 @@ #include "core/or/or.h" #include "app/config/config.h" +#include "core/or/channel.h" #include "core/or/circuitlist.h" #include "core/or/circuituse.h" #include "core/or/relay.h" @@ -24,9 +25,11 @@ #include "trunnel/hs/cell_introduce1.h" #include "feature/hs/hs_circuitmap.h" +#include "feature/hs/hs_common.h" +#include "feature/hs/hs_config.h" #include "feature/hs/hs_descriptor.h" +#include "feature/hs/hs_dos.h" #include "feature/hs/hs_intropoint.h" -#include "feature/hs/hs_common.h" #include "core/or/or_circuit_st.h" @@ -179,6 +182,185 @@ hs_intro_send_intro_established_cell,(or_circuit_t *circ)) return ret; } +/* Validate the cell DoS extension parameters. Return true iff they've been + * bound check and can be used. Else return false. See proposal 305 for + * details and reasons about this validation. */ +STATIC bool +cell_dos_extension_parameters_are_valid(uint64_t intro2_rate_per_sec, + uint64_t intro2_burst_per_sec) +{ + bool ret = false; + + /* Check that received value is not below the minimum. Don't check if minimum + is set to 0, since the param is a positive value and gcc will complain. */ +#if HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_MIN > 0 + if (intro2_rate_per_sec < HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_MIN) { + log_fn(LOG_PROTOCOL_WARN, LD_REND, + "Intro point DoS defenses rate per second is " + "too small. Received value: %" PRIu64, intro2_rate_per_sec); + goto end; + } +#endif /* HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_MIN > 0 */ + + /* Check that received value is not above maximum */ + if (intro2_rate_per_sec > HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_MAX) { + log_fn(LOG_PROTOCOL_WARN, LD_REND, + "Intro point DoS defenses rate per second is " + "too big. Received value: %" PRIu64, intro2_rate_per_sec); + goto end; + } + + /* Check that received value is not below the minimum */ +#if HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_MIN > 0 + if (intro2_burst_per_sec < HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_MIN) { + log_fn(LOG_PROTOCOL_WARN, LD_REND, + "Intro point DoS defenses burst per second is " + "too small. Received value: %" PRIu64, intro2_burst_per_sec); + goto end; + } +#endif /* HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_MIN > 0 */ + + /* Check that received value is not above maximum */ + if (intro2_burst_per_sec > HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_MAX) { + log_fn(LOG_PROTOCOL_WARN, LD_REND, + "Intro point DoS defenses burst per second is " + "too big. Received value: %" PRIu64, intro2_burst_per_sec); + goto end; + } + + /* In a rate limiting scenario, burst can never be smaller than the rate. At + * best it can be equal. */ + if (intro2_burst_per_sec < intro2_rate_per_sec) { + log_info(LD_REND, "Intro point DoS defenses burst is smaller than rate. " + "Rate: %" PRIu64 " vs Burst: %" PRIu64, + intro2_rate_per_sec, intro2_burst_per_sec); + goto end; + } + + /* Passing validation. */ + ret = true; + + end: + return ret; +} + +/* Parse the cell DoS extension and apply defenses on the given circuit if + * validation passes. If the cell extension is malformed or contains unusable + * values, the DoS defenses is disabled on the circuit. */ +static void +handle_establish_intro_cell_dos_extension( + const trn_cell_extension_field_t *field, + or_circuit_t *circ) +{ + ssize_t ret; + uint64_t intro2_rate_per_sec = 0, intro2_burst_per_sec = 0; + trn_cell_extension_dos_t *dos = NULL; + + tor_assert(field); + tor_assert(circ); + + ret = trn_cell_extension_dos_parse(&dos, + trn_cell_extension_field_getconstarray_field(field), + trn_cell_extension_field_getlen_field(field)); + if (ret < 0) { + goto end; + } + + for (size_t i = 0; i < trn_cell_extension_dos_get_n_params(dos); i++) { + const trn_cell_extension_dos_param_t *param = + trn_cell_extension_dos_getconst_params(dos, i); + if (BUG(param == NULL)) { + goto end; + } + + switch (trn_cell_extension_dos_param_get_type(param)) { + case TRUNNEL_DOS_PARAM_TYPE_INTRO2_RATE_PER_SEC: + intro2_rate_per_sec = trn_cell_extension_dos_param_get_value(param); + break; + case TRUNNEL_DOS_PARAM_TYPE_INTRO2_BURST_PER_SEC: + intro2_burst_per_sec = trn_cell_extension_dos_param_get_value(param); + break; + default: + goto end; + } + } + + /* A value of 0 is valid in the sense that we accept it but we still disable + * the defenses so return false. */ + if (intro2_rate_per_sec == 0 || intro2_burst_per_sec == 0) { + log_info(LD_REND, "Intro point DoS defenses parameter set to 0. " + "Disabling INTRO2 DoS defenses on circuit id %u", + circ->p_circ_id); + circ->introduce2_dos_defense_enabled = 0; + goto end; + } + + /* If invalid, we disable the defense on the circuit. */ + if (!cell_dos_extension_parameters_are_valid(intro2_rate_per_sec, + intro2_burst_per_sec)) { + circ->introduce2_dos_defense_enabled = 0; + log_info(LD_REND, "Disabling INTRO2 DoS defenses on circuit id %u", + circ->p_circ_id); + goto end; + } + + /* We passed validation, enable defenses and apply rate/burst. */ + circ->introduce2_dos_defense_enabled = 1; + + /* Initialize the INTRODUCE2 token bucket for the rate limiting. */ + token_bucket_ctr_init(&circ->introduce2_bucket, + (uint32_t) intro2_rate_per_sec, + (uint32_t) intro2_burst_per_sec, + (uint32_t) approx_time()); + log_info(LD_REND, "Intro point DoS defenses enabled. Rate is %" PRIu64 + " and Burst is %" PRIu64, + intro2_rate_per_sec, intro2_burst_per_sec); + + end: + trn_cell_extension_dos_free(dos); + return; +} + +/* Parse every cell extension in the given ESTABLISH_INTRO cell. */ +static void +handle_establish_intro_cell_extensions( + const trn_cell_establish_intro_t *parsed_cell, + or_circuit_t *circ) +{ + const trn_cell_extension_t *extensions; + + tor_assert(parsed_cell); + tor_assert(circ); + + extensions = trn_cell_establish_intro_getconst_extensions(parsed_cell); + if (extensions == NULL) { + goto end; + } + + /* Go over all extensions. */ + for (size_t idx = 0; idx < trn_cell_extension_get_num(extensions); idx++) { + const trn_cell_extension_field_t *field = + trn_cell_extension_getconst_fields(extensions, idx); + if (BUG(field == NULL)) { + /* The number of extensions should match the number of fields. */ + break; + } + + switch (trn_cell_extension_field_get_field_type(field)) { + case TRUNNEL_CELL_EXTENSION_TYPE_DOS: + /* After this, the circuit should be set for DoS defenses. */ + handle_establish_intro_cell_dos_extension(field, circ); + break; + default: + /* Unknown extension. Skip over. */ + break; + } + } + + end: + return; +} + /** We received an ESTABLISH_INTRO <b>parsed_cell</b> on <b>circ</b>. It's * well-formed and passed our verifications. Perform appropriate actions to * establish an intro point. */ @@ -191,6 +373,13 @@ handle_verified_establish_intro_cell(or_circuit_t *circ, get_auth_key_from_cell(&auth_key, RELAY_COMMAND_ESTABLISH_INTRO, parsed_cell); + /* Setup INTRODUCE2 defenses on the circuit. Must be done before parsing the + * cell extension that can possibly change the defenses' values. */ + hs_dos_setup_default_intro2_defenses(circ); + + /* Handle cell extension if any. */ + handle_establish_intro_cell_extensions(parsed_cell, circ); + /* Then notify the hidden service that the intro point is established by sending an INTRO_ESTABLISHED cell */ if (hs_intro_send_intro_established_cell(circ)) { @@ -480,6 +669,20 @@ handle_introduce1(or_circuit_t *client_circ, const uint8_t *request, } } + /* Before sending, lets make sure this cell can be sent on the service + * circuit asking the DoS defenses. */ + if (!hs_dos_can_send_intro2(service_circ)) { + char *msg; + static ratelim_t rlimit = RATELIM_INIT(5 * 60); + if ((msg = rate_limit_log(&rlimit, approx_time()))) { + log_info(LD_PROTOCOL, "Can't relay INTRODUCE1 v3 cell due to DoS " + "limitations. Sending NACK to client."); + tor_free(msg); + } + status = TRUNNEL_HS_INTRO_ACK_STATUS_UNKNOWN_ID; + goto send_ack; + } + /* Relay the cell to the service on its intro circuit with an INTRODUCE2 * cell which is the same exact payload. */ if (relay_send_command_from_edge(CONTROL_CELL_ID, TO_CIRCUIT(service_circ), @@ -546,6 +749,14 @@ circuit_is_suitable_for_introduce1(const or_circuit_t *circ) return 0; } + /* Disallow single hop client circuit. */ + if (circ->p_chan && channel_is_client(circ->p_chan)) { + log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, + "Single hop client was rejected while trying to introduce. " + "Closing circuit."); + return 0; + } + return 1; } diff --git a/src/feature/hs/hs_intropoint.h b/src/feature/hs/hs_intropoint.h index e82575f052..94ebf021e4 100644 --- a/src/feature/hs/hs_intropoint.h +++ b/src/feature/hs/hs_intropoint.h @@ -57,6 +57,9 @@ STATIC int handle_introduce1(or_circuit_t *client_circ, const uint8_t *request, size_t request_len); STATIC int validate_introduce1_parsed_cell(const trn_cell_introduce1_t *cell); STATIC int circuit_is_suitable_for_introduce1(const or_circuit_t *circ); +STATIC bool cell_dos_extension_parameters_are_valid( + uint64_t intro2_rate_per_sec, + uint64_t intro2_burst_per_sec); #endif /* defined(HS_INTROPOINT_PRIVATE) */ diff --git a/src/feature/hs/hs_service.c b/src/feature/hs/hs_service.c index d1ca33b12e..18c38ebc0a 100644 --- a/src/feature/hs/hs_service.c +++ b/src/feature/hs/hs_service.c @@ -242,6 +242,9 @@ set_service_default_config(hs_service_config_t *c, c->is_single_onion = 0; c->dir_group_readable = 0; c->is_ephemeral = 0; + c->has_dos_defense_enabled = HS_CONFIG_V3_DOS_DEFENSE_DEFAULT; + c->intro_dos_rate_per_sec = HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_DEFAULT; + c->intro_dos_burst_per_sec = HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_DEFAULT; } /* From a service configuration object config, clear everything from it @@ -489,6 +492,10 @@ service_intro_point_new(const node_t *node) } } + /* Flag if this intro point supports the INTRO2 dos defenses. */ + ip->support_intro2_dos_defense = + node_supports_establish_intro_dos_extension(node); + /* Finally, copy onion key from the node. */ memcpy(&ip->onion_key, node_get_curve25519_onion_key(node), sizeof(ip->onion_key)); @@ -2326,15 +2333,70 @@ intro_point_should_expire(const hs_service_intro_point_t *ip, return 1; } -/* Go over the given set of intro points for each service and remove any - * invalid ones. The conditions for removal are: +/* Return true iff we should remove the intro point ip from its service. + * + * We remove an intro point from the service descriptor list if one of + * these criteria is met: + * - It has expired (either in INTRO2 count or in time). + * - No node was found (fell off the consensus). + * - We are over the maximum amount of retries. * - * - The node doesn't exists anymore (not in consensus) - * OR - * - The intro point maximum circuit retry count has been reached and no - * circuit can be found associated with it. - * OR - * - The intro point has expired and we should pick a new one. + * If an established or pending circuit is found for the given ip object, this + * return false indicating it should not be removed. */ +static bool +should_remove_intro_point(hs_service_intro_point_t *ip, time_t now) +{ + bool ret = false; + + tor_assert(ip); + + /* Any one of the following needs to be True to furfill the criteria to + * remove an intro point. */ + bool has_no_retries = (ip->circuit_retries > + MAX_INTRO_POINT_CIRCUIT_RETRIES); + bool has_no_node = (get_node_from_intro_point(ip) == NULL); + bool has_expired = intro_point_should_expire(ip, now); + + /* If the node fell off the consensus or the IP has expired, we have to + * remove it now. */ + if (has_no_node || has_expired) { + ret = true; + goto end; + } + + /* Pass this point, even though we might be over the retry limit, we check + * if a circuit (established or pending) exists. In that case, we should not + * remove it because it might simply be valid and opened at the previous + * scheduled event for the last retry. */ + + /* Did we established already? */ + if (ip->circuit_established) { + goto end; + } + /* Do we simply have an existing circuit regardless of its state? */ + if (hs_circ_service_get_intro_circ(ip)) { + goto end; + } + + /* Getting here means we have _no_ circuits so then return if we have any + * remaining retries. */ + ret = has_no_retries; + + end: + /* Meaningful log in case we are about to remove the IP. */ + if (ret) { + log_info(LD_REND, "Intro point %s%s (retried: %u times). " + "Removing it.", + describe_intro_point(ip), + has_expired ? " has expired" : + (has_no_node) ? " fell off the consensus" : "", + ip->circuit_retries); + } + return ret; +} + +/* Go over the given set of intro points for each service and remove any + * invalid ones. * * If an intro point is removed, the circuit (if any) is immediately close. * If a circuit can't be found, the intro point is kept if it hasn't reached @@ -2359,21 +2421,7 @@ cleanup_intro_points(hs_service_t *service, time_t now) * valid and remove any of them that aren't. */ DIGEST256MAP_FOREACH_MODIFY(desc->intro_points.map, key, hs_service_intro_point_t *, ip) { - const node_t *node = get_node_from_intro_point(ip); - int has_expired = intro_point_should_expire(ip, now); - - /* We cleanup an intro point if it has expired or if we do not know the - * node_t anymore (removed from our latest consensus) or if we've - * reached the maximum number of retry with a non existing circuit. */ - if (has_expired || node == NULL || - ip->circuit_retries > MAX_INTRO_POINT_CIRCUIT_RETRIES) { - log_info(LD_REND, "Intro point %s%s (retried: %u times). " - "Removing it.", - describe_intro_point(ip), - has_expired ? " has expired" : - (node == NULL) ? " fell off the consensus" : "", - ip->circuit_retries); - + if (should_remove_intro_point(ip, now)) { /* We've retried too many times, remember it as a failed intro point * so we don't pick it up again for INTRO_CIRC_RETRY_PERIOD sec. */ if (ip->circuit_retries > MAX_INTRO_POINT_CIRCUIT_RETRIES) { diff --git a/src/feature/hs/hs_service.h b/src/feature/hs/hs_service.h index 22aa00b2d7..c4bbb293bb 100644 --- a/src/feature/hs/hs_service.h +++ b/src/feature/hs/hs_service.h @@ -76,6 +76,10 @@ typedef struct hs_service_intro_point_t { * circuit associated with this intro point has received. This is used to * prevent replay attacks. */ replaycache_t *replay_cache; + + /* Support the INTRO2 DoS defense. If set, the DoS extension described by + * proposal 305 is sent. */ + unsigned int support_intro2_dos_defense : 1; } hs_service_intro_point_t; /* Object handling introduction points of a service. */ @@ -241,6 +245,11 @@ typedef struct hs_service_config_t { /* Does this service export the circuit ID of its clients? */ hs_circuit_id_protocol_t circuit_id_protocol; + + /* DoS defenses. For the ESTABLISH_INTRO cell extension. */ + unsigned int has_dos_defense_enabled : 1; + uint32_t intro_dos_rate_per_sec; + uint32_t intro_dos_burst_per_sec; } hs_service_config_t; /* Service state. */ diff --git a/src/feature/nodelist/describe.c b/src/feature/nodelist/describe.c index 5c376408c0..1e46837685 100644 --- a/src/feature/nodelist/describe.c +++ b/src/feature/nodelist/describe.c @@ -9,66 +9,108 @@ * \brief Format short descriptions of relays. */ +#define DESCRIBE_PRIVATE + #include "core/or/or.h" #include "feature/nodelist/describe.h" -#include "feature/nodelist/routerinfo.h" #include "core/or/extend_info_st.h" #include "feature/nodelist/node_st.h" #include "feature/nodelist/routerinfo_st.h" #include "feature/nodelist/routerstatus_st.h" - -/** - * Longest allowed output of format_node_description, plus 1 character for - * NUL. This allows space for: - * "$FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF~xxxxxxxxxxxxxxxxxxx at" - * " [ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255]" - * plus a terminating NUL. - */ -#define NODE_DESC_BUF_LEN (MAX_VERBOSE_NICKNAME_LEN+4+TOR_ADDR_BUF_LEN) +#include "feature/nodelist/microdesc_st.h" /** Use <b>buf</b> (which must be at least NODE_DESC_BUF_LEN bytes long) to * hold a human-readable description of a node with identity digest - * <b>id_digest</b>, named-status <b>is_named</b>, nickname <b>nickname</b>, - * and address <b>addr</b> or <b>addr32h</b>. + * <b>id_digest</b>, nickname <b>nickname</b>, and addresses <b>addr32h</b> and + * <b>addr</b>. * * The <b>nickname</b> and <b>addr</b> fields are optional and may be set to - * NULL. The <b>addr32h</b> field is optional and may be set to 0. + * NULL or the null address. The <b>addr32h</b> field is optional and may be + * set to 0. * * Return a pointer to the front of <b>buf</b>. + * If buf is NULL, return a string constant describing the error. */ -static const char * +STATIC const char * format_node_description(char *buf, const char *id_digest, - int is_named, const char *nickname, const tor_addr_t *addr, uint32_t addr32h) { - char *cp; + size_t rv = 0; + bool has_addr = addr && !tor_addr_is_null(addr); if (!buf) return "<NULL BUFFER>"; - buf[0] = '$'; - base16_encode(buf+1, HEX_DIGEST_LEN+1, id_digest, DIGEST_LEN); - cp = buf+1+HEX_DIGEST_LEN; + memset(buf, 0, NODE_DESC_BUF_LEN); + + if (!id_digest) { + /* strlcpy() returns the length of the source string it attempted to copy, + * ignoring any required truncation due to the buffer length. */ + rv = strlcpy(buf, "<NULL ID DIGEST>", NODE_DESC_BUF_LEN); + tor_assert_nonfatal(rv < NODE_DESC_BUF_LEN); + return buf; + } + + /* strlcat() returns the length of the concatenated string it attempted to + * create, ignoring any required truncation due to the buffer length. */ + rv = strlcat(buf, "$", NODE_DESC_BUF_LEN); + tor_assert_nonfatal(rv < NODE_DESC_BUF_LEN); + + { + char hex_digest[HEX_DIGEST_LEN+1]; + memset(hex_digest, 0, sizeof(hex_digest)); + + base16_encode(hex_digest, sizeof(hex_digest), + id_digest, DIGEST_LEN); + rv = strlcat(buf, hex_digest, NODE_DESC_BUF_LEN); + tor_assert_nonfatal(rv < NODE_DESC_BUF_LEN); + } + if (nickname) { - buf[1+HEX_DIGEST_LEN] = is_named ? '=' : '~'; - strlcpy(buf+1+HEX_DIGEST_LEN+1, nickname, MAX_NICKNAME_LEN+1); - cp += strlen(cp); + rv = strlcat(buf, "~", NODE_DESC_BUF_LEN); + tor_assert_nonfatal(rv < NODE_DESC_BUF_LEN); + rv = strlcat(buf, nickname, NODE_DESC_BUF_LEN); + tor_assert_nonfatal(rv < NODE_DESC_BUF_LEN); } - if (addr32h || addr) { - memcpy(cp, " at ", 4); - cp += 4; - if (addr) { - tor_addr_to_str(cp, addr, TOR_ADDR_BUF_LEN, 0); - } else { - struct in_addr in; - in.s_addr = htonl(addr32h); - tor_inet_ntoa(&in, cp, INET_NTOA_BUF_LEN); - } + if (addr32h || has_addr) { + rv = strlcat(buf, " at ", NODE_DESC_BUF_LEN); + tor_assert_nonfatal(rv < NODE_DESC_BUF_LEN); } + if (addr32h) { + int ntoa_rv = 0; + char ipv4_addr_str[INET_NTOA_BUF_LEN]; + memset(ipv4_addr_str, 0, sizeof(ipv4_addr_str)); + struct in_addr in; + memset(&in, 0, sizeof(in)); + + in.s_addr = htonl(addr32h); + ntoa_rv = tor_inet_ntoa(&in, ipv4_addr_str, sizeof(ipv4_addr_str)); + tor_assert_nonfatal(ntoa_rv >= 0); + + rv = strlcat(buf, ipv4_addr_str, NODE_DESC_BUF_LEN); + tor_assert_nonfatal(rv < NODE_DESC_BUF_LEN); + } + /* Both addresses are valid */ + if (addr32h && has_addr) { + rv = strlcat(buf, " and ", NODE_DESC_BUF_LEN); + tor_assert_nonfatal(rv < NODE_DESC_BUF_LEN); + } + if (has_addr) { + const char *str_rv = NULL; + char addr_str[TOR_ADDR_BUF_LEN]; + memset(addr_str, 0, sizeof(addr_str)); + + str_rv = tor_addr_to_str(addr_str, addr, sizeof(addr_str), 1); + tor_assert_nonfatal(str_rv == addr_str); + + rv = strlcat(buf, addr_str, NODE_DESC_BUF_LEN); + tor_assert_nonfatal(rv < NODE_DESC_BUF_LEN); + } + return buf; } @@ -84,11 +126,11 @@ router_describe(const routerinfo_t *ri) if (!ri) return "<null>"; + return format_node_description(buf, ri->cache_info.identity_digest, - 0, ri->nickname, - NULL, + &ri->ipv6_addr, ri->addr); } @@ -103,25 +145,33 @@ node_describe(const node_t *node) static char buf[NODE_DESC_BUF_LEN]; const char *nickname = NULL; uint32_t addr32h = 0; - int is_named = 0; + const tor_addr_t *ipv6_addr = NULL; if (!node) return "<null>"; if (node->rs) { nickname = node->rs->nickname; - is_named = node->rs->is_named; addr32h = node->rs->addr; + ipv6_addr = &node->rs->ipv6_addr; + /* Support consensus versions less than 28, when IPv6 addresses were in + * microdescs. This code can be removed when 0.2.9 is no longer supported, + * and the MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC macro is removed. */ + if (node->md && tor_addr_is_null(ipv6_addr)) { + ipv6_addr = &node->md->ipv6_addr; + } } else if (node->ri) { nickname = node->ri->nickname; addr32h = node->ri->addr; + ipv6_addr = &node->ri->ipv6_addr; + } else { + return "<null rs and ri>"; } return format_node_description(buf, node->identity, - is_named, nickname, - NULL, + ipv6_addr, addr32h); } @@ -137,11 +187,11 @@ routerstatus_describe(const routerstatus_t *rs) if (!rs) return "<null>"; + return format_node_description(buf, rs->identity_digest, - rs->is_named, rs->nickname, - NULL, + &rs->ipv6_addr, rs->addr); } @@ -157,9 +207,9 @@ extend_info_describe(const extend_info_t *ei) if (!ei) return "<null>"; + return format_node_description(buf, ei->identity_digest, - 0, ei->nickname, &ei->addr, 0); @@ -175,9 +225,39 @@ extend_info_describe(const extend_info_t *ei) void router_get_verbose_nickname(char *buf, const routerinfo_t *router) { - buf[0] = '$'; - base16_encode(buf+1, HEX_DIGEST_LEN+1, router->cache_info.identity_digest, - DIGEST_LEN); - buf[1+HEX_DIGEST_LEN] = '~'; - strlcpy(buf+1+HEX_DIGEST_LEN+1, router->nickname, MAX_NICKNAME_LEN+1); + size_t rv = 0; + + if (!buf) + return; + + memset(buf, 0, MAX_VERBOSE_NICKNAME_LEN+1); + + if (!router) { + /* strlcpy() returns the length of the source string it attempted to copy, + * ignoring any required truncation due to the buffer length. */ + rv = strlcpy(buf, "<null>", MAX_VERBOSE_NICKNAME_LEN+1); + tor_assert_nonfatal(rv < MAX_VERBOSE_NICKNAME_LEN+1); + return; + } + + /* strlcat() returns the length of the concatenated string it attempted to + * create, ignoring any required truncation due to the buffer length. */ + rv = strlcat(buf, "$", MAX_VERBOSE_NICKNAME_LEN+1); + tor_assert_nonfatal(rv < MAX_VERBOSE_NICKNAME_LEN+1); + + { + char hex_digest[HEX_DIGEST_LEN+1]; + memset(hex_digest, 0, sizeof(hex_digest)); + + base16_encode(hex_digest, sizeof(hex_digest), + router->cache_info.identity_digest, DIGEST_LEN); + rv = strlcat(buf, hex_digest, MAX_VERBOSE_NICKNAME_LEN+1); + tor_assert_nonfatal(rv < MAX_VERBOSE_NICKNAME_LEN+1); + } + + rv = strlcat(buf, "~", MAX_VERBOSE_NICKNAME_LEN+1); + tor_assert_nonfatal(rv < MAX_VERBOSE_NICKNAME_LEN+1); + + rv = strlcat(buf, router->nickname, MAX_VERBOSE_NICKNAME_LEN+1); + tor_assert_nonfatal(rv < MAX_VERBOSE_NICKNAME_LEN+1); } diff --git a/src/feature/nodelist/describe.h b/src/feature/nodelist/describe.h index d29192200e..6c0d6dc48d 100644 --- a/src/feature/nodelist/describe.h +++ b/src/feature/nodelist/describe.h @@ -22,4 +22,36 @@ const char *node_describe(const struct node_t *node); const char *router_describe(const struct routerinfo_t *ri); const char *routerstatus_describe(const struct routerstatus_t *ri); +void router_get_verbose_nickname(char *buf, const routerinfo_t *router); + +#if defined(DESCRIBE_PRIVATE) || defined(TOR_UNIT_TESTS) + +/** + * Longest allowed output for an IPv4 address "255.255.255.255", with NO + * terminating NUL. + */ +#define IPV4_BUF_LEN_NO_NUL 15 + +/** + * Longest allowed output of format_node_description, plus 1 character for + * NUL. This allows space for: + * "$FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF~xxxxxxxxxxxxxxxxxxx at" + * " 255.255.255.255 and [ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255]" + * plus a terminating NUL. + */ +#define NODE_DESC_BUF_LEN \ + (MAX_VERBOSE_NICKNAME_LEN+4+IPV4_BUF_LEN_NO_NUL+5+TOR_ADDR_BUF_LEN) + +#endif /* defined(DESCRIBE_PRIVATE) || defined(TOR_UNIT_TESTS) */ + +#ifdef TOR_UNIT_TESTS + +STATIC const char *format_node_description(char *buf, + const char *id_digest, + const char *nickname, + const tor_addr_t *addr, + uint32_t addr32h); + +#endif /* defined(TOR_UNIT_TESTS) */ + #endif /* !defined(TOR_DESCRIBE_H) */ diff --git a/src/feature/nodelist/microdesc_st.h b/src/feature/nodelist/microdesc_st.h index c8265cb778..e017c46c79 100644 --- a/src/feature/nodelist/microdesc_st.h +++ b/src/feature/nodelist/microdesc_st.h @@ -33,6 +33,8 @@ struct microdesc_t { unsigned int no_save : 1; /** If true, this microdesc has an entry in the microdesc_map */ unsigned int held_in_map : 1; + /** True iff the exit policy for this router rejects everything. */ + unsigned int policy_is_reject_star : 1; /** Reference count: how many node_ts have a reference to this microdesc? */ unsigned int held_by_nodes; diff --git a/src/feature/nodelist/networkstatus.c b/src/feature/nodelist/networkstatus.c index 2db293a8af..496bafb865 100644 --- a/src/feature/nodelist/networkstatus.c +++ b/src/feature/nodelist/networkstatus.c @@ -68,6 +68,7 @@ #include "feature/dircommon/voting_schedule.h" #include "feature/dirparse/ns_parse.h" #include "feature/hibernate/hibernate.h" +#include "feature/hs/hs_dos.h" #include "feature/nodelist/authcert.h" #include "feature/nodelist/dirlist.h" #include "feature/nodelist/fmt_routerstatus.h" @@ -1674,6 +1675,7 @@ notify_before_networkstatus_changes(const networkstatus_t *old_c, notify_control_networkstatus_changed(old_c, new_c); dos_consensus_has_changed(new_c); relay_consensus_has_changed(new_c); + hs_dos_consensus_has_changed(new_c); } /* Called after a new consensus has been put in the global state. It is safe diff --git a/src/feature/nodelist/nodelist.c b/src/feature/nodelist/nodelist.c index 21914c6c6d..bd80fc1b4b 100644 --- a/src/feature/nodelist/nodelist.c +++ b/src/feature/nodelist/nodelist.c @@ -1106,7 +1106,7 @@ node_ed25519_id_matches(const node_t *node, const ed25519_public_key_t *id) /** Dummy object that should be unreturnable. Used to ensure that * node_get_protover_summary_flags() always returns non-NULL. */ static const protover_summary_flags_t zero_protover_flags = { - 0,0,0,0,0,0,0,0 + 0,0,0,0,0,0,0,0,0 }; /** Return the protover_summary_flags for a given node. */ @@ -1166,6 +1166,17 @@ node_supports_ed25519_hs_intro(const node_t *node) return node_get_protover_summary_flags(node)->supports_ed25519_hs_intro; } +/** Return true iff <b>node</b> supports the DoS ESTABLISH_INTRO cell + * extenstion. */ +int +node_supports_establish_intro_dos_extension(const node_t *node) +{ + tor_assert(node); + + return node_get_protover_summary_flags(node)-> + supports_establish_intro_dos_extension; +} + /** Return true iff <b>node</b> supports to be a rendezvous point for hidden * service version 3 (HSRend=2). */ int @@ -1424,8 +1435,7 @@ node_exit_policy_rejects_all(const node_t *node) if (node->ri) return node->ri->policy_is_reject_star; else if (node->md) - return node->md->exit_policy == NULL || - short_policy_is_reject_star(node->md->exit_policy); + return node->md->policy_is_reject_star; else return 1; } diff --git a/src/feature/nodelist/nodelist.h b/src/feature/nodelist/nodelist.h index 84ab5f7a54..af144c197f 100644 --- a/src/feature/nodelist/nodelist.h +++ b/src/feature/nodelist/nodelist.h @@ -76,6 +76,7 @@ int node_supports_ed25519_link_authentication(const node_t *node, int node_supports_v3_hsdir(const node_t *node); int node_supports_ed25519_hs_intro(const node_t *node); int node_supports_v3_rendezvous_point(const node_t *node); +int node_supports_establish_intro_dos_extension(const node_t *node); const uint8_t *node_get_rsa_id_digest(const node_t *node); smartlist_t *node_get_link_specifier_smartlist(const node_t *node, bool direct_conn); diff --git a/src/feature/nodelist/routerinfo.h b/src/feature/nodelist/routerinfo.h index ca66e660b3..8465060f93 100644 --- a/src/feature/nodelist/routerinfo.h +++ b/src/feature/nodelist/routerinfo.h @@ -17,8 +17,6 @@ void router_get_prim_orport(const routerinfo_t *router, int router_has_orport(const routerinfo_t *router, const tor_addr_port_t *orport); -void router_get_verbose_nickname(char *buf, const routerinfo_t *router); - smartlist_t *router_get_all_orports(const routerinfo_t *ri); const char *router_purpose_to_string(uint8_t p); diff --git a/src/feature/nodelist/routerlist.c b/src/feature/nodelist/routerlist.c index c56b714cb0..0cd7a76a9a 100644 --- a/src/feature/nodelist/routerlist.c +++ b/src/feature/nodelist/routerlist.c @@ -1459,12 +1459,13 @@ router_descriptor_is_older_than,(const routerinfo_t *router, int seconds)) } /** Add <b>router</b> to the routerlist, if we don't already have it. Replace - * older entries (if any) with the same key. Note: Callers should not hold - * their pointers to <b>router</b> if this function fails; <b>router</b> - * will either be inserted into the routerlist or freed. Similarly, even - * if this call succeeds, they should not hold their pointers to - * <b>router</b> after subsequent calls with other routerinfo's -- they - * might cause the original routerinfo to get freed. + * older entries (if any) with the same key. + * + * Note: Callers should not hold their pointers to <b>router</b> if this + * function fails; <b>router</b> will either be inserted into the routerlist or + * freed. Similarly, even if this call succeeds, they should not hold their + * pointers to <b>router</b> after subsequent calls with other routerinfo's -- + * they might cause the original routerinfo to get freed. * * Returns the status for the operation. Might set *<b>msg</b> if it wants * the poster of the router to know something. diff --git a/src/feature/nodelist/routerlist.h b/src/feature/nodelist/routerlist.h index 5771ebb1ab..dc9203e015 100644 --- a/src/feature/nodelist/routerlist.h +++ b/src/feature/nodelist/routerlist.h @@ -37,9 +37,12 @@ typedef enum was_router_added_t { ROUTER_WAS_NOT_WANTED = -6, /* Router descriptor was rejected because it was older than * OLD_ROUTER_DESC_MAX_AGE. */ - ROUTER_WAS_TOO_OLD = -7, /* note contrast with 'NOT_NEW' */ - /* DOCDOC */ - ROUTER_CERTS_EXPIRED = -8 + ROUTER_WAS_TOO_OLD = -7, /* note contrast with 'ROUTER_IS_ALREADY_KNOWN' */ + /* Some certs on this router are expired. */ + ROUTER_CERTS_EXPIRED = -8, + /* We couldn't format the annotations for this router. This is a directory + * authority bug. */ + ROUTER_AUTHDIR_BUG_ANNOTATIONS = -10 } was_router_added_t; /** How long do we avoid using a directory server after it's given us a 503? */ diff --git a/src/feature/nodelist/routerset.c b/src/feature/nodelist/routerset.c index e801fd81b1..9a205d39b7 100644 --- a/src/feature/nodelist/routerset.c +++ b/src/feature/nodelist/routerset.c @@ -1,5 +1,5 @@ /* Copyright (c) 2001 Matej Pfajfar. -n * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. * Copyright (c) 2007-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ @@ -34,6 +34,9 @@ n * Copyright (c) 2001-2004, Roger Dingledine. #include "feature/nodelist/nickname.h" #include "feature/nodelist/nodelist.h" #include "feature/nodelist/routerset.h" +#include "lib/conf/conftypes.h" +#include "lib/confmgt/typedvar.h" +#include "lib/encoding/confline.h" #include "lib/geoip/geoip.h" #include "core/or/addr_policy_st.h" @@ -41,6 +44,7 @@ n * Copyright (c) 2001-2004, Roger Dingledine. #include "feature/nodelist/node_st.h" #include "feature/nodelist/routerinfo_st.h" #include "feature/nodelist/routerstatus_st.h" +#include "lib/confmgt/var_type_def_st.h" /** Return a new empty routerset. */ routerset_t * @@ -461,3 +465,111 @@ routerset_free_(routerset_t *routerset) bitarray_free(routerset->countries); tor_free(routerset); } + +/** + * config helper: parse a routerset-typed variable. + * + * Takes as input as a single line in <b>line</b>; writes its results into a + * routerset_t** passed as <b>target</b>. On success return 0; on failure + * return -1 and store an error message into *<b>errmsg</b>. + **/ +/* + * Warning: For this type, the default value (NULL) and "" are sometimes + * considered different values. That is generally risky, and best avoided for + * other types in the future. For cases where we want the default to be "all + * routers" (like EntryNodes) we should add a new routerset value indicating + * "all routers" (see #31908) + */ +static int +routerset_kv_parse(void *target, const config_line_t *line, char **errmsg, + const void *params) +{ + (void)params; + routerset_t **p = (routerset_t**)target; + routerset_free(*p); // clear the old value, if any. + routerset_t *rs = routerset_new(); + if (routerset_parse(rs, line->value, line->key) < 0) { + routerset_free(rs); + *errmsg = tor_strdup("Invalid router list."); + return -1; + } else { + if (routerset_is_empty(rs)) { + /* Represent empty sets as NULL. */ + routerset_free(rs); + } + *p = rs; + return 0; + } +} + +/** + * config helper: encode a routerset-typed variable. + * + * Return a newly allocated string containing the value of the + * routerset_t** passed as <b>value</b>. + */ +static char * +routerset_encode(const void *value, const void *params) +{ + (void)params; + const routerset_t **p = (const routerset_t**)value; + return routerset_to_string(*p); +} + +/** + * config helper: free and clear a routerset-typed variable. + * + * Clear the routerset_t** passed as <b>value</b>. + */ +static void +routerset_clear(void *value, const void *params) +{ + (void)params; + routerset_t **p = (routerset_t**)value; + routerset_free(*p); // sets *p to NULL. +} + +/** + * config helper: copy a routerset-typed variable. + * + * Takes it input from a routerset_t** in <b>src</b>; writes its output to a + * routerset_t** in <b>dest</b>. Returns 0 on success, -1 on (impossible) + * failure. + **/ +static int +routerset_copy(void *dest, const void *src, const void *params) +{ + (void)params; + routerset_t **output = (routerset_t**)dest; + const routerset_t *input = *(routerset_t**)src; + routerset_free(*output); // sets *output to NULL + if (! routerset_is_empty(input)) { + *output = routerset_new(); + routerset_union(*output, input); + } + return 0; +} + +/** + * Function table to implement a routerset_t-based configuration type. + **/ +static const var_type_fns_t routerset_type_fns = { + .kv_parse = routerset_kv_parse, + .encode = routerset_encode, + .clear = routerset_clear, + .copy = routerset_copy +}; + +/** + * Definition of a routerset_t-based configuration type. + * + * Values are mapped to and from strings using the format defined in + * routerset_parse(): nicknames, IP address patterns, and fingerprints--with + * optional space, separated by commas. + * + * Empty sets are represented as NULL. + **/ +const var_type_def_t ROUTERSET_type_defn = { + .name = "RouterList", + .fns = &routerset_type_fns +}; diff --git a/src/feature/nodelist/routerset.h b/src/feature/nodelist/routerset.h index ca8b6fed93..f3bf4a1f7c 100644 --- a/src/feature/nodelist/routerset.h +++ b/src/feature/nodelist/routerset.h @@ -44,6 +44,9 @@ void routerset_free_(routerset_t *routerset); #define routerset_free(rs) FREE_AND_NULL(routerset_t, routerset_free_, (rs)) int routerset_len(const routerset_t *set); +struct var_type_def_t; +extern const struct var_type_def_t ROUTERSET_type_defn; + #ifdef ROUTERSET_PRIVATE #include "lib/container/bitarray.h" diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c index 29a8ffaa39..ab0762e17e 100644 --- a/src/feature/relay/router.c +++ b/src/feature/relay/router.c @@ -3111,33 +3111,22 @@ load_stats_file(const char *filename, const char *end_line, time_t now, return r; } -/** Write the contents of <b>extrainfo</b>, to * *<b>s_out</b>, signing them - * with <b>ident_key</b>. - * - * If ExtraInfoStatistics is 1, also write aggregated statistics and related - * configuration data before signing. Most statistics also have an option that - * enables or disables that particular statistic. - * - * Return 0 on success, negative on failure. */ -int -extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo, - crypto_pk_t *ident_key, - const ed25519_keypair_t *signing_keypair) +/** Add header strings to chunks, based on the extrainfo object extrainfo, + * and ed25519 keypair signing_keypair, if emit_ed_sigs is true. + * Helper for extrainfo_dump_to_string(). + * Returns 0 on success, negative on failure. */ +static int +extrainfo_dump_to_string_header_helper( + smartlist_t *chunks, + const extrainfo_t *extrainfo, + const ed25519_keypair_t *signing_keypair, + int emit_ed_sigs) { - const or_options_t *options = get_options(); char identity[HEX_DIGEST_LEN+1]; char published[ISO_TIME_LEN+1]; - char digest[DIGEST_LEN]; - int result; - static int write_stats_to_extrainfo = 1; - char sig[DIROBJ_MAX_SIG_LEN+1]; - char *s = NULL, *pre, *contents, *cp, *s_dup = NULL; - time_t now = time(NULL); - smartlist_t *chunks = smartlist_new(); - extrainfo_t *ei_tmp = NULL; - const int emit_ed_sigs = signing_keypair && - extrainfo->cache_info.signing_key_cert; char *ed_cert_line = NULL; + char *pre = NULL; + int rv = -1; base16_encode(identity, sizeof(identity), extrainfo->cache_info.identity_digest, DIGEST_LEN); @@ -3167,12 +3156,41 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo, ed_cert_line = tor_strdup(""); } + /* This is the first chunk in the file. If the file is too big, other chunks + * are removed. So we must only add one chunk here. */ tor_asprintf(&pre, "extra-info %s %s\n%spublished %s\n", extrainfo->nickname, identity, ed_cert_line, published); smartlist_add(chunks, pre); + rv = 0; + goto done; + + err: + rv = -1; + + done: + tor_free(ed_cert_line); + return rv; +} + +/** Add pluggable transport and statistics strings to chunks, skipping + * statistics if write_stats_to_extrainfo is false. + * Helper for extrainfo_dump_to_string(). + * Can not fail. */ +static void +extrainfo_dump_to_string_stats_helper(smartlist_t *chunks, + int write_stats_to_extrainfo) +{ + const or_options_t *options = get_options(); + char *contents = NULL; + time_t now = time(NULL); + + /* If the file is too big, these chunks are removed, starting with the last + * chunk. So each chunk must be a complete line, and the file must be valid + * after each chunk. */ + /* Add information about the pluggable transports we support, even if we * are not publishing statistics. This information is needed by BridgeDB * to distribute bridges. */ @@ -3239,34 +3257,132 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo, } } } +} + +/** Add an ed25519 signature of chunks to chunks, using the ed25519 keypair + * signing_keypair. + * Helper for extrainfo_dump_to_string(). + * Returns 0 on success, negative on failure. */ +static int +extrainfo_dump_to_string_ed_sig_helper( + smartlist_t *chunks, + const ed25519_keypair_t *signing_keypair) +{ + char sha256_digest[DIGEST256_LEN]; + ed25519_signature_t ed_sig; + char buf[ED25519_SIG_BASE64_LEN+1]; + int rv = -1; + + /* These are two of the three final chunks in the file. If the file is too + * big, other chunks are removed. So we must only add two chunks here. */ + smartlist_add_strdup(chunks, "router-sig-ed25519 "); + crypto_digest_smartlist_prefix(sha256_digest, DIGEST256_LEN, + ED_DESC_SIGNATURE_PREFIX, + chunks, "", DIGEST_SHA256); + if (ed25519_sign(&ed_sig, (const uint8_t*)sha256_digest, DIGEST256_LEN, + signing_keypair) < 0) + goto err; + ed25519_signature_to_base64(buf, &ed_sig); + + smartlist_add_asprintf(chunks, "%s\n", buf); + + rv = 0; + goto done; + + err: + rv = -1; + + done: + return rv; +} + +/** Add an RSA signature of extrainfo_string to chunks, using the RSA key + * ident_key. + * Helper for extrainfo_dump_to_string(). + * Returns 0 on success, negative on failure. */ +static int +extrainfo_dump_to_string_rsa_sig_helper(smartlist_t *chunks, + crypto_pk_t *ident_key, + const char *extrainfo_string) +{ + char sig[DIROBJ_MAX_SIG_LEN+1]; + char digest[DIGEST_LEN]; + int rv = -1; + + memset(sig, 0, sizeof(sig)); + if (router_get_extrainfo_hash(extrainfo_string, strlen(extrainfo_string), + digest) < 0 || + router_append_dirobj_signature(sig, sizeof(sig), digest, DIGEST_LEN, + ident_key) < 0) { + log_warn(LD_BUG, "Could not append signature to extra-info " + "descriptor."); + goto err; + } + smartlist_add_strdup(chunks, sig); + + rv = 0; + goto done; + + err: + rv = -1; + + done: + return rv; +} + +/** Write the contents of <b>extrainfo</b>, to * *<b>s_out</b>, signing them + * with <b>ident_key</b>. + * + * If ExtraInfoStatistics is 1, also write aggregated statistics and related + * configuration data before signing. Most statistics also have an option that + * enables or disables that particular statistic. + * + * Always write pluggable transport lines. + * + * Return 0 on success, negative on failure. */ +int +extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo, + crypto_pk_t *ident_key, + const ed25519_keypair_t *signing_keypair) +{ + int result; + static int write_stats_to_extrainfo = 1; + char *s = NULL, *cp, *s_dup = NULL; + smartlist_t *chunks = smartlist_new(); + extrainfo_t *ei_tmp = NULL; + const int emit_ed_sigs = signing_keypair && + extrainfo->cache_info.signing_key_cert; + int rv = 0; + + rv = extrainfo_dump_to_string_header_helper(chunks, extrainfo, + signing_keypair, + emit_ed_sigs); + if (rv < 0) + goto err; + + extrainfo_dump_to_string_stats_helper(chunks, write_stats_to_extrainfo); if (emit_ed_sigs) { - char sha256_digest[DIGEST256_LEN]; - smartlist_add_strdup(chunks, "router-sig-ed25519 "); - crypto_digest_smartlist_prefix(sha256_digest, DIGEST256_LEN, - ED_DESC_SIGNATURE_PREFIX, - chunks, "", DIGEST_SHA256); - ed25519_signature_t ed_sig; - char buf[ED25519_SIG_BASE64_LEN+1]; - if (ed25519_sign(&ed_sig, (const uint8_t*)sha256_digest, DIGEST256_LEN, - signing_keypair) < 0) + rv = extrainfo_dump_to_string_ed_sig_helper(chunks, signing_keypair); + if (rv < 0) goto err; - ed25519_signature_to_base64(buf, &ed_sig); - - smartlist_add_asprintf(chunks, "%s\n", buf); } + /* This is one of the three final chunks in the file. If the file is too big, + * other chunks are removed. So we must only add one chunk here. */ smartlist_add_strdup(chunks, "router-signature\n"); s = smartlist_join_strings(chunks, "", 0, NULL); while (strlen(s) > MAX_EXTRAINFO_UPLOAD_SIZE - DIROBJ_MAX_SIG_LEN) { /* So long as there are at least two chunks (one for the initial * extra-info line and one for the router-signature), we can keep removing - * things. */ - if (smartlist_len(chunks) > 2) { - /* We remove the next-to-last element (remember, len-1 is the last - element), since we need to keep the router-signature element. */ - int idx = smartlist_len(chunks) - 2; + * things. If emit_ed_sigs is true, we also keep 2 additional chunks at the + * end for the ed25519 signature. */ + const int required_chunks = emit_ed_sigs ? 4 : 2; + if (smartlist_len(chunks) > required_chunks) { + /* We remove the next-to-last or 4th-last element (remember, len-1 is the + * last element), since we need to keep the router-signature elements. */ + int idx = smartlist_len(chunks) - required_chunks; char *e = smartlist_get(chunks, idx); smartlist_del_keeporder(chunks, idx); log_warn(LD_GENERAL, "We just generated an extra-info descriptor " @@ -3283,15 +3399,10 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo, } } - memset(sig, 0, sizeof(sig)); - if (router_get_extrainfo_hash(s, strlen(s), digest) < 0 || - router_append_dirobj_signature(sig, sizeof(sig), digest, DIGEST_LEN, - ident_key) < 0) { - log_warn(LD_BUG, "Could not append signature to extra-info " - "descriptor."); + rv = extrainfo_dump_to_string_rsa_sig_helper(chunks, ident_key, s); + if (rv < 0) goto err; - } - smartlist_add_strdup(chunks, sig); + tor_free(s); s = smartlist_join_strings(chunks, "", 0, NULL); @@ -3327,7 +3438,6 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo, SMARTLIST_FOREACH(chunks, char *, chunk, tor_free(chunk)); smartlist_free(chunks); tor_free(s_dup); - tor_free(ed_cert_line); extrainfo_free(ei_tmp); return result; diff --git a/src/feature/rend/rendclient.c b/src/feature/rend/rendclient.c index 5bdd4d453e..2540066dfc 100644 --- a/src/feature/rend/rendclient.c +++ b/src/feature/rend/rendclient.c @@ -119,7 +119,7 @@ rend_client_send_introduction(origin_circuit_t *introcirc, char tmp[RELAY_PAYLOAD_SIZE]; rend_cache_entry_t *entry = NULL; crypt_path_t *cpath; - off_t dh_offset; + ptrdiff_t dh_offset; crypto_pk_t *intro_key = NULL; int status = 0; const char *onion_address; diff --git a/src/feature/rend/rendcommon.c b/src/feature/rend/rendcommon.c index 777de2984c..0a606a9f02 100644 --- a/src/feature/rend/rendcommon.c +++ b/src/feature/rend/rendcommon.c @@ -786,39 +786,39 @@ rend_process_relay_cell(circuit_t *circ, const crypt_path_t *layer_hint, switch (command) { case RELAY_COMMAND_ESTABLISH_INTRO: if (or_circ) - r = hs_intro_received_establish_intro(or_circ,payload,length); + r = hs_intro_received_establish_intro(or_circ, payload, length); break; case RELAY_COMMAND_ESTABLISH_RENDEZVOUS: if (or_circ) - r = rend_mid_establish_rendezvous(or_circ,payload,length); + r = rend_mid_establish_rendezvous(or_circ, payload, length); break; case RELAY_COMMAND_INTRODUCE1: if (or_circ) - r = hs_intro_received_introduce1(or_circ,payload,length); + r = hs_intro_received_introduce1(or_circ, payload, length); break; case RELAY_COMMAND_INTRODUCE2: if (origin_circ) - r = hs_service_receive_introduce2(origin_circ,payload,length); + r = hs_service_receive_introduce2(origin_circ, payload, length); break; case RELAY_COMMAND_INTRODUCE_ACK: if (origin_circ) - r = hs_client_receive_introduce_ack(origin_circ,payload,length); + r = hs_client_receive_introduce_ack(origin_circ, payload, length); break; case RELAY_COMMAND_RENDEZVOUS1: if (or_circ) - r = rend_mid_rendezvous(or_circ,payload,length); + r = rend_mid_rendezvous(or_circ, payload, length); break; case RELAY_COMMAND_RENDEZVOUS2: if (origin_circ) - r = hs_client_receive_rendezvous2(origin_circ,payload,length); + r = hs_client_receive_rendezvous2(origin_circ, payload, length); break; case RELAY_COMMAND_INTRO_ESTABLISHED: if (origin_circ) - r = hs_service_receive_intro_established(origin_circ,payload,length); + r = hs_service_receive_intro_established(origin_circ, payload, length); break; case RELAY_COMMAND_RENDEZVOUS_ESTABLISHED: if (origin_circ) - r = hs_client_receive_rendezvous_acked(origin_circ,payload,length); + r = hs_client_receive_rendezvous_acked(origin_circ, payload, length); break; default: tor_fragile_assert(); diff --git a/src/feature/rend/rendmid.c b/src/feature/rend/rendmid.c index 421e0f2139..06471b2a7f 100644 --- a/src/feature/rend/rendmid.c +++ b/src/feature/rend/rendmid.c @@ -18,6 +18,7 @@ #include "feature/rend/rendmid.h" #include "feature/stats/rephist.h" #include "feature/hs/hs_circuitmap.h" +#include "feature/hs/hs_dos.h" #include "feature/hs/hs_intropoint.h" #include "core/or/or_circuit_st.h" @@ -117,6 +118,7 @@ rend_mid_establish_intro_legacy(or_circuit_t *circ, const uint8_t *request, /* Now, set up this circuit. */ circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_INTRO_POINT); hs_circuitmap_register_intro_circ_v2_relay_side(circ, (uint8_t *)pk_digest); + hs_dos_setup_default_intro2_defenses(circ); log_info(LD_REND, "Established introduction point on circuit %u for service %s", @@ -181,6 +183,14 @@ rend_mid_introduce_legacy(or_circuit_t *circ, const uint8_t *request, goto err; } + /* Before sending, lets make sure this cell can be sent on the service + * circuit asking the DoS defenses. */ + if (!hs_dos_can_send_intro2(intro_circ)) { + log_info(LD_PROTOCOL, "Can't relay INTRODUCE1 v2 cell due to DoS " + "limitations. Sending NACK to client."); + goto err; + } + log_info(LD_REND, "Sending introduction request for service %s " "from circ %u to circ %u", diff --git a/src/feature/rend/rendservice.c b/src/feature/rend/rendservice.c index 119a6f9c89..e0cf06b9df 100644 --- a/src/feature/rend/rendservice.c +++ b/src/feature/rend/rendservice.c @@ -2127,7 +2127,7 @@ rend_service_receive_introduction(origin_circuit_t *circuit, * * We only use a one-hop path on the first attempt. If the first attempt * fails, we use a 3-hop path for reachability / reliability. - * See the comment in rend_service_relauch_rendezvous() for details. */ + * See the comment in rend_service_relaunch_rendezvous() for details. */ if (rend_service_use_direct_connection(options, rp) && i == 0) { flags = flags | CIRCLAUNCH_ONEHOP_TUNNEL; } @@ -3999,7 +3999,7 @@ remove_invalid_intro_points(rend_service_t *service, * accounted for when considiring uploading a descriptor. */ intro->circuit_established = 0; - /* Node is gone or we've reached our maximum circuit creationg retry + /* Node is gone or we've reached our maximum circuit creation retry * count, clean up everything, we'll find a new one. */ if (node == NULL || intro->circuit_retries >= MAX_INTRO_POINT_CIRCUIT_RETRIES) { diff --git a/src/include.am b/src/include.am index 77c126ba45..065bdc31cb 100644 --- a/src/include.am +++ b/src/include.am @@ -5,6 +5,8 @@ include src/lib/err/include.am include src/lib/cc/include.am include src/lib/ctime/include.am include src/lib/compress/include.am +include src/lib/conf/include.am +include src/lib/confmgt/include.am include src/lib/container/include.am include src/lib/crypt_ops/include.am include src/lib/defs/include.am diff --git a/src/lib/arch/bytes.h b/src/lib/arch/bytes.h index b8b6288139..4756ca2beb 100644 --- a/src/lib/arch/bytes.h +++ b/src/lib/arch/bytes.h @@ -129,7 +129,7 @@ tor_ntohll(uint64_t a) { return a; } -#else /* !(defined(WORDS_BIGENDIAN)) */ +#else /* !defined(WORDS_BIGENDIAN) */ static inline uint16_t tor_htons(uint16_t a) { diff --git a/src/lib/buf/buffers.c b/src/lib/buf/buffers.c index 88a25b8470..4d026bd37d 100644 --- a/src/lib/buf/buffers.c +++ b/src/lib/buf/buffers.c @@ -158,7 +158,7 @@ chunk_new_with_alloc_size(size_t alloc) static inline chunk_t * chunk_grow(chunk_t *chunk, size_t sz) { - off_t offset; + ptrdiff_t offset; const size_t memlen_orig = chunk->memlen; const size_t orig_alloc = CHUNK_ALLOC_SIZE(memlen_orig); const size_t new_alloc = CHUNK_ALLOC_SIZE(sz); @@ -440,7 +440,7 @@ chunk_copy(const chunk_t *in_chunk) #endif newch->next = NULL; if (in_chunk->data) { - off_t offset = in_chunk->data - in_chunk->mem; + ptrdiff_t offset = in_chunk->data - in_chunk->mem; newch->data = newch->mem + offset; } return newch; @@ -710,7 +710,8 @@ buf_move_all(buf_t *buf_out, buf_t *buf_in) /** Internal structure: represents a position in a buffer. */ typedef struct buf_pos_t { const chunk_t *chunk; /**< Which chunk are we pointing to? */ - int pos;/**< Which character inside the chunk's data are we pointing to? */ + ptrdiff_t pos;/**< Which character inside the chunk's data are we pointing + * to? */ size_t chunk_pos; /**< Total length of all previous chunks. */ } buf_pos_t; @@ -726,15 +727,15 @@ buf_pos_init(const buf_t *buf, buf_pos_t *out) /** Advance <b>out</b> to the first appearance of <b>ch</b> at the current * position of <b>out</b>, or later. Return -1 if no instances are found; * otherwise returns the absolute position of the character. */ -static off_t +static ptrdiff_t buf_find_pos_of_char(char ch, buf_pos_t *out) { const chunk_t *chunk; - int pos; + ptrdiff_t pos; tor_assert(out); if (out->chunk) { if (out->chunk->datalen) { - tor_assert(out->pos < (off_t)out->chunk->datalen); + tor_assert(out->pos < (ptrdiff_t)out->chunk->datalen); } else { tor_assert(out->pos == 0); } @@ -762,7 +763,7 @@ buf_pos_inc(buf_pos_t *pos) { tor_assert(pos->pos < INT_MAX - 1); ++pos->pos; - if (pos->pos == (off_t)pos->chunk->datalen) { + if (pos->pos == (ptrdiff_t)pos->chunk->datalen) { if (!pos->chunk->next) return -1; pos->chunk_pos += pos->chunk->datalen; @@ -836,11 +837,11 @@ buf_peek_startswith(const buf_t *buf, const char *cmd) /** Return the index within <b>buf</b> at which <b>ch</b> first appears, * or -1 if <b>ch</b> does not appear on buf. */ -static off_t +static ptrdiff_t buf_find_offset_of_char(buf_t *buf, char ch) { chunk_t *chunk; - off_t offset = 0; + ptrdiff_t offset = 0; tor_assert(buf->datalen < INT_MAX); for (chunk = buf->head; chunk; chunk = chunk->next) { char *cp = memchr(chunk->data, ch, chunk->datalen); @@ -863,7 +864,7 @@ int buf_get_line(buf_t *buf, char *data_out, size_t *data_len) { size_t sz; - off_t offset; + ptrdiff_t offset; if (!buf->head) return 0; diff --git a/src/lib/cc/compat_compiler.h b/src/lib/cc/compat_compiler.h index a8d1593214..3ef866ecce 100644 --- a/src/lib/cc/compat_compiler.h +++ b/src/lib/cc/compat_compiler.h @@ -82,7 +82,7 @@ # define ENABLE_GCC_WARNING(warningopt) \ PRAGMA_DIAGNOSTIC_(warning PRAGMA_JOIN_STRINGIFY_(-W,warningopt)) #endif /* defined(__clang__) || GCC_VERSION >= 406 */ -#else /* !(defined(__GNUC__)) */ +#else /* !defined(__GNUC__) */ /* not gcc at all */ # define DISABLE_GCC_WARNING(warning) # define ENABLE_GCC_WARNING(warning) @@ -195,7 +195,7 @@ * structure <b>st</b>. Example: * <pre> * struct a { int foo; int bar; } x; - * off_t bar_offset = offsetof(struct a, bar); + * ptrdiff_t bar_offset = offsetof(struct a, bar); * int *bar_p = STRUCT_VAR_P(&x, bar_offset); * *bar_p = 3; * </pre> diff --git a/src/lib/cc/torint.h b/src/lib/cc/torint.h index 523f378ed7..94b79d30a1 100644 --- a/src/lib/cc/torint.h +++ b/src/lib/cc/torint.h @@ -96,7 +96,7 @@ typedef int32_t ssize_t; # else # define TOR_PRIuSZ PRIu32 # endif -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ # define TOR_PRIuSZ "zu" #endif /* defined(_WIN32) */ @@ -106,7 +106,7 @@ typedef int32_t ssize_t; # else # define TOR_PRIdSZ PRId32 # endif -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ # define TOR_PRIdSZ "zd" #endif /* defined(_WIN32) */ diff --git a/src/lib/compress/compress_lzma.c b/src/lib/compress/compress_lzma.c index 2dab37e433..915f4949ae 100644 --- a/src/lib/compress/compress_lzma.c +++ b/src/lib/compress/compress_lzma.c @@ -221,7 +221,7 @@ tor_lzma_compress_new(int compress, tor_free(result); return NULL; /* LCOV_EXCL_STOP */ -#else /* !(defined(HAVE_LZMA)) */ +#else /* !defined(HAVE_LZMA) */ (void)compress; (void)method; (void)level; @@ -312,7 +312,7 @@ tor_lzma_compress_process(tor_lzma_compress_state_t *state, lzma_error_str(retval)); return TOR_COMPRESS_ERROR; } -#else /* !(defined(HAVE_LZMA)) */ +#else /* !defined(HAVE_LZMA) */ (void)state; (void)out; (void)out_len; diff --git a/src/lib/compress/compress_zstd.c b/src/lib/compress/compress_zstd.c index a99ea67e0b..9076665295 100644 --- a/src/lib/compress/compress_zstd.c +++ b/src/lib/compress/compress_zstd.c @@ -93,7 +93,7 @@ tor_zstd_get_version_str(void) ZSTD_versionNumber()); return version_str; -#else /* !(defined(HAVE_ZSTD)) */ +#else /* !defined(HAVE_ZSTD) */ return NULL; #endif /* defined(HAVE_ZSTD) */ } @@ -317,7 +317,7 @@ tor_zstd_compress_new(int compress, tor_free(result); return NULL; // LCOV_EXCL_STOP -#else /* !(defined(HAVE_ZSTD)) */ +#else /* !defined(HAVE_ZSTD) */ (void)compress; (void)method; (void)level; @@ -454,7 +454,7 @@ tor_zstd_compress_process(tor_zstd_compress_state_t *state, return TOR_COMPRESS_OK; } -#else /* !(defined(HAVE_ZSTD)) */ +#else /* !defined(HAVE_ZSTD) */ (void)state; (void)out; (void)out_len; diff --git a/src/lib/conf/.may_include b/src/lib/conf/.may_include new file mode 100644 index 0000000000..629e2f897d --- /dev/null +++ b/src/lib/conf/.may_include @@ -0,0 +1,3 @@ +orconfig.h +lib/cc/*.h +lib/conf/*.h diff --git a/src/lib/conf/confmacros.h b/src/lib/conf/confmacros.h new file mode 100644 index 0000000000..68121891f1 --- /dev/null +++ b/src/lib/conf/confmacros.h @@ -0,0 +1,67 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file confmacros.h + * @brief Macro definitions for declaring configuration variables + **/ + +#ifndef TOR_LIB_CONF_CONFMACROS_H +#define TOR_LIB_CONF_CONFMACROS_H + +#include "orconfig.h" +#include "lib/conf/conftesting.h" + +/** + * Used to indicate the end of an array of configuration variables. + **/ +#define END_OF_CONFIG_VARS \ + { .member = { .name = NULL } DUMMY_CONF_TEST_MEMBERS } + +/** + * Declare a config_var_t as a member named <b>membername</b> of the structure + * <b>structtype</b>, whose user-visible name is <b>varname</b>, whose + * type corresponds to the config_type_t member CONFIG_TYPE_<b>vartype</b>, + * and whose initial value is <b>intval</b>. + * + * Most modules that use this macro should wrap it in a local macro that + * sets structtype to the local configuration type. + **/ +#define CONFIG_VAR_ETYPE(structtype, varname, vartype, membername, \ + varflags, initval) \ + { .member = \ + { .name = varname, \ + .type = CONFIG_TYPE_ ## vartype, \ + .offset = offsetof(structtype, membername), \ + }, \ + .flags = varflags, \ + .initvalue = initval \ + CONF_TEST_MEMBERS(structtype, vartype, membername) \ + } + +/** + * As CONFIG_VAR_XTYPE, but declares a value using an extension type whose + * type definition is <b>vartype</b>_type_defn. + **/ +#define CONFIG_VAR_DEFN(structtype, varname, vartype, membername, \ + varflags, initval) \ + { .member = \ + { .name = varname, \ + .type = CONFIG_TYPE_EXTENDED, \ + .type_def = &vartype ## _type_defn, \ + .offset = offsetof(structtype, membername), \ + }, \ + .flags = varflags, \ + .initvalue = initval \ + CONF_TEST_MEMBERS(structtype, vartype, membername) \ + } + +#define CONFIG_VAR_OBSOLETE(varname) \ + { .member = { .name = varname, .type = CONFIG_TYPE_OBSOLETE }, \ + .flags = CFLG_GROUP_OBSOLETE \ + } + +#endif /* !defined(TOR_LIB_CONF_CONFMACROS_H) */ diff --git a/src/lib/conf/conftesting.h b/src/lib/conf/conftesting.h new file mode 100644 index 0000000000..f01f52d59e --- /dev/null +++ b/src/lib/conf/conftesting.h @@ -0,0 +1,86 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file conftesting.h + * @brief Macro and type declarations for testing + **/ + +#ifndef TOR_LIB_CONF_CONFTESTING_H +#define TOR_LIB_CONF_CONFTESTING_H + +#ifdef TOR_UNIT_TESTS +/** + * Union used when building in test mode typechecking the members of a type + * used with confparse.c. See CONF_CHECK_VAR_TYPE for a description of how + * it is used. */ +typedef union { + char **STRING; + char **FILENAME; + int *POSINT; /* yes, this is really an int, and not an unsigned int. For + * historical reasons, many configuration values are restricted + * to the range [0,INT_MAX], and stored in signed ints. + */ + uint64_t *UINT64; + int *INT; + int *INTERVAL; + int *MSEC_INTERVAL; + uint64_t *MEMUNIT; + double *DOUBLE; + int *BOOL; + int *AUTOBOOL; + time_t *ISOTIME; + struct smartlist_t **CSV; + int *CSV_INTERVAL; + struct config_line_t **LINELIST; + struct config_line_t **LINELIST_S; + struct config_line_t **LINELIST_V; + // XXXX this doesn't belong at this level of abstraction. + struct routerset_t **ROUTERSET; +} confparse_dummy_values_t; +#endif /* defined(TOR_UNIT_TESTS) */ + +/* Macros to define extra members inside config_var_t fields, and at the + * end of a list of them. + */ +#ifdef TOR_UNIT_TESTS +/* This is a somewhat magic type-checking macro for users of confparse.c. + * It initializes a union member "confparse_dummy_values_t.conftype" with + * the address of a static member "tp_dummy.member". This + * will give a compiler warning unless the member field is of the correct + * type. + * + * (This warning is mandatory, because a type mismatch here violates the type + * compatibility constraint for simple assignment, and requires a diagnostic, + * according to the C spec.) + * + * For example, suppose you say: + * "CONF_CHECK_VAR_TYPE(or_options_t, STRING, Address)". + * Then this macro will evaluate to: + * { .STRING = &or_options_t_dummy.Address } + * And since confparse_dummy_values_t.STRING has type "char **", that + * expression will create a warning unless or_options_t.Address also + * has type "char *". + */ +#define CONF_CHECK_VAR_TYPE(tp, conftype, member) \ + { . conftype = &tp ## _dummy . member } +#define CONF_TEST_MEMBERS(tp, conftype, member) \ + , .var_ptr_dummy=CONF_CHECK_VAR_TYPE(tp, conftype, member) +#define DUMMY_CONF_TEST_MEMBERS , .var_ptr_dummy={ .INT=NULL } +#define DUMMY_TYPECHECK_INSTANCE(tp) \ + static tp tp ## _dummy + +#else /* !defined(TOR_UNIT_TESTS) */ + +#define CONF_TEST_MEMBERS(tp, conftype, member) +/* Repeatedly declarable incomplete struct to absorb redundant semicolons */ +#define DUMMY_TYPECHECK_INSTANCE(tp) \ + struct tor_semicolon_eater +#define DUMMY_CONF_TEST_MEMBERS + +#endif /* defined(TOR_UNIT_TESTS) */ + +#endif /* !defined(TOR_LIB_CONF_CONFTESTING_H) */ diff --git a/src/lib/conf/conftypes.h b/src/lib/conf/conftypes.h new file mode 100644 index 0000000000..274065cff2 --- /dev/null +++ b/src/lib/conf/conftypes.h @@ -0,0 +1,202 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file conftypes.h + * @brief Types used to specify configurable options. + * + * This header defines the types that different modules will use in order to + * declare their configuration and state variables, and tell the configuration + * management code about those variables. From the individual module's point + * of view, its configuration and state are simply data structures. + * + * For defining new variable types, see var_type_def_st.h. + * + * For the code that manipulates variables defined via this module, see + * lib/confmgt/, especially typedvar.h and (later) structvar.h. The + * configuration manager is responsible for encoding, decoding, and + * maintaining the configuration structures used by the various modules. + * + * STATUS NOTE: This is a work in process refactoring. It is not yet possible + * for modules to define their own variables, and much of the configuration + * management code is still in src/app/config/. + **/ + +#ifndef TOR_SRC_LIB_CONF_CONFTYPES_H +#define TOR_SRC_LIB_CONF_CONFTYPES_H + +#include "lib/cc/torint.h" +#ifdef TOR_UNIT_TESTS +#include "lib/conf/conftesting.h" +#endif + +#include <stddef.h> + +/** Enumeration of types which option values can take */ +typedef enum config_type_t { + CONFIG_TYPE_STRING = 0, /**< An arbitrary string. */ + CONFIG_TYPE_FILENAME, /**< A filename: some prefixes get expanded. */ + CONFIG_TYPE_POSINT, /**< A non-negative integer less than MAX_INT */ + CONFIG_TYPE_INT, /**< Any integer. */ + CONFIG_TYPE_UINT64, /**< A value in range 0..UINT64_MAX */ + CONFIG_TYPE_INTERVAL, /**< A number of seconds, with optional units*/ + CONFIG_TYPE_MSEC_INTERVAL,/**< A number of milliseconds, with optional + * units */ + CONFIG_TYPE_MEMUNIT, /**< A number of bytes, with optional units*/ + CONFIG_TYPE_DOUBLE, /**< A floating-point value */ + CONFIG_TYPE_BOOL, /**< A boolean value, expressed as 0 or 1. */ + CONFIG_TYPE_AUTOBOOL, /**< A boolean+auto value, expressed 0 for false, + * 1 for true, and -1 for auto */ + CONFIG_TYPE_ISOTIME, /**< An ISO-formatted time relative to UTC. */ + CONFIG_TYPE_CSV, /**< A list of strings, separated by commas and + * optional whitespace. */ + CONFIG_TYPE_CSV_INTERVAL, /**< A list of strings, separated by commas and + * optional whitespace, representing intervals in + * seconds, with optional units. We allow + * multiple values here for legacy reasons, but + * ignore every value after the first. */ + CONFIG_TYPE_LINELIST, /**< Uninterpreted config lines */ + CONFIG_TYPE_LINELIST_S, /**< Uninterpreted, context-sensitive config lines, + * mixed with other keywords. */ + CONFIG_TYPE_LINELIST_V, /**< Catch-all "virtual" option to summarize + * context-sensitive config lines when fetching. + */ + CONFIG_TYPE_OBSOLETE, /**< Obsolete (ignored) option. */ + /** + * Extended type: definition appears in the <b>type_def</b> pointer + * of the corresponding struct_member_t. + * + * For some types, we cannot define them as particular values of this + * enumeration, since those types are abstractions defined at a higher level + * than this module. (For example, parsing a routerset_t is higher-level + * than this module.) To handle this, we use CONFIG_TYPE_EXTENDED for those + * types, and give a definition for them in the struct_member_t.type_def. + **/ + CONFIG_TYPE_EXTENDED, +} config_type_t; + +/* Forward delcaration for var_type_def_t, for extended types. */ +struct var_type_def_t; + +/** Structure to specify a named, typed member within a structure. */ +typedef struct struct_member_t { + /** Name of the field. */ + const char *name; + /** + * Type of the field, according to the config_type_t enumeration. + * + * For any type not otherwise listed in config_type_t, this field's value + * should be CONFIG_TYPE_EXTENDED. When it is, the <b>type_def</b> pointer + * must be set. + **/ + /* + * NOTE: In future refactoring, we might remove this field entirely, along + * with its corresponding enumeration. In that case, we will require that + * type_def be set in all cases. If we do, we will also need a new mechanism + * to enforce consistency between configuration variable types and their + * corresponding structures, since our current design in + * lib/conf/conftesting.h won't work any more. + */ + config_type_t type; + /** + * Pointer to a type definition for the type of this field. Overrides + * <b>type</b> if it is not NULL. Must be set when <b>type</b> is + * CONFIG_TYPE_EXTENDED. + **/ + const struct var_type_def_t *type_def; + /** + * Offset of this field within the structure. Compute this with + * offsetof(structure, fieldname). + **/ + ptrdiff_t offset; +} struct_member_t; + +/** + * Structure to describe the location and preferred value of a "magic number" + * field within a structure. + * + * These 'magic numbers' are 32-bit values used to tag objects to make sure + * that they have the correct type. + */ +typedef struct struct_magic_decl_t { + /** The name of the structure */ + const char *typename; + /** A value used to recognize instances of this structure. */ + uint32_t magic_val; + /** The location within the structure at which we expect to find + * <b>magic_val</b>. */ + ptrdiff_t magic_offset; +} struct_magic_decl_t; + +/** + * Flag to indicate that an option or type is "undumpable". An + * undumpable option is never saved to disk. + * + * For historical reasons its name is usually is prefixed with __. + **/ +#define CFLG_NODUMP (1u<<0) +/** + * Flag to indicate that an option or type is "unlisted". + * + * We don't tell the controller about unlisted options when it asks for a + * list of them. + **/ +#define CFLG_NOLIST (1u<<1) +/** + * Flag to indicate that an option or type is "unsettable". + * + * An unsettable option can never be set directly by name. + **/ +#define CFLG_NOSET (1u<<2) +/** + * Flag to indicate that an option or type does not need to be copied when + * copying the structure that contains it. + * + * (Usually, if an option does not need to be copied, then either it contains + * no data, or the data that it does contain is completely contained within + * another option.) + **/ +#define CFLG_NOCOPY (1u<<3) +/** + * Flag to indicate that an option or type does not need to be compared + * when telling the controller about the differences between two + * configurations. + * + * (Usually, if an option does not need to be compared, then either it + * contains no data, or the data that it does contain is completely contained + * within another option.) + **/ +#define CFLG_NOCMP (1u<<4) +/** + * Flag to indicate that an option or type should not be replaced when setting + * it. + * + * For most options, setting them replaces their old value. For some options, + * however, setting them appends to their old value. + */ +#define CFLG_NOREPLACE (1u<<5) + +/** + * A group of flags that should be set on all obsolete options and types. + **/ +#define CFLG_GROUP_OBSOLETE \ + (CFLG_NOCOPY|CFLG_NOCMP|CFLG_NODUMP|CFLG_NOSET|CFLG_NOLIST) + +/** A variable allowed in the configuration file or on the command line. */ +typedef struct config_var_t { + struct_member_t member; /** A struct member corresponding to this + * variable. */ + const char *initvalue; /**< String (or null) describing initial value. */ + uint32_t flags; /**< One or more flags describing special handling for this + * variable */ +#ifdef TOR_UNIT_TESTS + /** Used for compiler-magic to typecheck the corresponding field in the + * corresponding struct. Only used in unit test mode, at compile-time. */ + confparse_dummy_values_t var_ptr_dummy; +#endif +} config_var_t; + +#endif /* !defined(TOR_SRC_LIB_CONF_CONFTYPES_H) */ diff --git a/src/lib/conf/include.am b/src/lib/conf/include.am new file mode 100644 index 0000000000..cb7126184d --- /dev/null +++ b/src/lib/conf/include.am @@ -0,0 +1,6 @@ + +# ADD_C_FILE: INSERT HEADERS HERE. +noinst_HEADERS += \ + src/lib/conf/conftesting.h \ + src/lib/conf/conftypes.h \ + src/lib/conf/confmacros.h diff --git a/src/lib/confmgt/.may_include b/src/lib/confmgt/.may_include new file mode 100644 index 0000000000..2564133917 --- /dev/null +++ b/src/lib/confmgt/.may_include @@ -0,0 +1,11 @@ +orconfig.h +lib/cc/*.h +lib/conf/*.h +lib/confmgt/*.h +lib/container/*.h +lib/encoding/*.h +lib/log/*.h +lib/malloc/*.h +lib/string/*.h +lib/testsupport/*.h +ext/*.h diff --git a/src/lib/confmgt/confparse.c b/src/lib/confmgt/confparse.c new file mode 100644 index 0000000000..08e562f654 --- /dev/null +++ b/src/lib/confmgt/confparse.c @@ -0,0 +1,1239 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file confparse.c + * + * \brief Back-end for parsing and generating key-value files, used to + * implement the torrc file format and the state file. + * + * This module is used by config.c to parse and encode torrc + * configuration files, and by statefile.c to parse and encode the + * $DATADIR/state file. + * + * To use this module, its callers provide an instance of + * config_format_t to describe the mappings from a set of configuration + * options to a number of fields in a C structure. With this mapping, + * the functions here can convert back and forth between the C structure + * specified, and a linked list of key-value pairs. + */ + +#define CONFPARSE_PRIVATE +#include "orconfig.h" +#include "lib/confmgt/confparse.h" + +#include "lib/confmgt/structvar.h" +#include "lib/confmgt/unitparse.h" +#include "lib/container/bitarray.h" +#include "lib/container/smartlist.h" +#include "lib/encoding/confline.h" +#include "lib/log/escape.h" +#include "lib/log/log.h" +#include "lib/log/util_bug.h" +#include "lib/string/compat_ctype.h" +#include "lib/string/printf.h" +#include "lib/string/util_string.h" + +#include "ext/siphash.h" + +/** + * A managed_var_t is an internal wrapper around a config_var_t in + * a config_format_t structure. It is used by config_mgr_t to + * keep track of which option goes with which structure. */ +typedef struct managed_var_t { + /** + * A pointer to the config_var_t for this option. + */ + const config_var_t *cvar; + /** + * The index of the object in which this option is stored. It is + * IDX_TOPLEVEL to indicate that the object is the top-level object. + **/ + int object_idx; +} managed_var_t; + +static void config_reset(const config_mgr_t *fmt, void *options, + const managed_var_t *var, int use_defaults); +static void config_mgr_register_fmt(config_mgr_t *mgr, + const config_format_t *fmt, + int object_idx); + +/** Release all storage held in a managed_var_t. */ +static void +managed_var_free_(managed_var_t *mv) +{ + if (!mv) + return; + tor_free(mv); +} +#define managed_var_free(mv) \ + FREE_AND_NULL(managed_var_t, managed_var_free_, (mv)) + +struct config_suite_t { + /** A list of configuration objects managed by a given configuration + * manager. They are stored in the same order as the config_format_t + * objects in the manager's list of subformats. */ + smartlist_t *configs; +}; + +/** + * Allocate a new empty config_suite_t. + **/ +static config_suite_t * +config_suite_new(void) +{ + config_suite_t *suite = tor_malloc_zero(sizeof(config_suite_t)); + suite->configs = smartlist_new(); + return suite; +} + +/** Release all storage held by a config_suite_t. (Does not free + * any configuration objects it holds; the caller must do that first.) */ +static void +config_suite_free_(config_suite_t *suite) +{ + if (!suite) + return; + smartlist_free(suite->configs); + tor_free(suite); +} + +#define config_suite_free(suite) \ + FREE_AND_NULL(config_suite_t, config_suite_free_, (suite)) + +struct config_mgr_t { + /** The 'top-level' configuration format. This one is used for legacy + * options that have not yet been assigned to different sub-modules. + * + * (NOTE: for now, this is the only config_format_t that a config_mgr_t + * contains. A subsequent commit will add more. XXXX) + */ + const config_format_t *toplevel; + /** + * List of second-level configuration format objects that this manager + * also knows about. + */ + smartlist_t *subconfigs; + /** A smartlist of managed_var_t objects for all configuration formats. */ + smartlist_t *all_vars; + /** A smartlist of config_abbrev_t objects for all configuration + * formats. These objects are used to track synonyms and abbreviations for + * different configuration options. */ + smartlist_t *all_abbrevs; + /** A smartlist of config_deprecation_t for all configuration formats. */ + smartlist_t *all_deprecations; + /** True if this manager has been frozen and cannot have any more formats + * added to it. A manager must be frozen before it can be used to construct + * or manipulate objects. */ + bool frozen; + /** A replacement for the magic number of the toplevel object. We override + * that number to make it unique for this particular config_mgr_t, so that + * an object constructed with one mgr can't be used with another, even if + * those managers' contents are equal. + */ + struct_magic_decl_t toplevel_magic; +}; + +#define IDX_TOPLEVEL (-1) + +/** Create a new config_mgr_t to manage a set of configuration objects to be + * wrapped under <b>toplevel_fmt</b>. */ +config_mgr_t * +config_mgr_new(const config_format_t *toplevel_fmt) +{ + config_mgr_t *mgr = tor_malloc_zero(sizeof(config_mgr_t)); + mgr->subconfigs = smartlist_new(); + mgr->all_vars = smartlist_new(); + mgr->all_abbrevs = smartlist_new(); + mgr->all_deprecations = smartlist_new(); + + config_mgr_register_fmt(mgr, toplevel_fmt, IDX_TOPLEVEL); + mgr->toplevel = toplevel_fmt; + + return mgr; +} + +/** Add a config_format_t to a manager, with a specified (unique) index. */ +static void +config_mgr_register_fmt(config_mgr_t *mgr, + const config_format_t *fmt, + int object_idx) +{ + int i; + + tor_assertf(!mgr->frozen, + "Tried to add a format to a configuration manager after " + "it had been frozen."); + + if (object_idx != IDX_TOPLEVEL) { + tor_assertf(fmt->config_suite_offset < 0, + "Tried to register a toplevel format in a non-toplevel position"); + } + tor_assertf(fmt != mgr->toplevel && + ! smartlist_contains(mgr->subconfigs, fmt), + "Tried to register an already-registered format."); + + /* register variables */ + for (i = 0; fmt->vars[i].member.name; ++i) { + managed_var_t *mv = tor_malloc_zero(sizeof(managed_var_t)); + mv->cvar = &fmt->vars[i]; + mv->object_idx = object_idx; + smartlist_add(mgr->all_vars, mv); + } + + /* register abbrevs */ + if (fmt->abbrevs) { + for (i = 0; fmt->abbrevs[i].abbreviated; ++i) { + smartlist_add(mgr->all_abbrevs, (void*)&fmt->abbrevs[i]); + } + } + + /* register deprecations. */ + if (fmt->deprecations) { + const config_deprecation_t *d; + for (d = fmt->deprecations; d->name; ++d) { + smartlist_add(mgr->all_deprecations, (void*)d); + } + } +} + +/** + * Add a new format to this configuration object. Asserts on failure. + * + * Returns an internal "index" value used to identify this format within + * all of those formats contained in <b>mgr</b>. This index value + * should not generally be used outside of this module. + **/ +int +config_mgr_add_format(config_mgr_t *mgr, + const config_format_t *fmt) +{ + tor_assert(mgr); + int idx = smartlist_len(mgr->subconfigs); + config_mgr_register_fmt(mgr, fmt, idx); + smartlist_add(mgr->subconfigs, (void *)fmt); + return idx; +} + +/** Return a pointer to the config_suite_t * pointer inside a + * configuration object; returns NULL if there is no such member. */ +static inline config_suite_t ** +config_mgr_get_suite_ptr(const config_mgr_t *mgr, void *toplevel) +{ + if (mgr->toplevel->config_suite_offset < 0) + return NULL; + return STRUCT_VAR_P(toplevel, mgr->toplevel->config_suite_offset); +} + +/** + * Return a pointer to the configuration object within <b>toplevel</b> whose + * index is <b>idx</b>. + * + * NOTE: XXXX Eventually, there will be multiple objects supported within the + * toplevel object. For example, the or_options_t will contain pointers + * to configuration objects for other modules. This function gets + * the sub-object for a particular module. + */ +STATIC void * +config_mgr_get_obj_mutable(const config_mgr_t *mgr, void *toplevel, int idx) +{ + tor_assert(mgr); + tor_assert(toplevel); + if (idx == IDX_TOPLEVEL) + return toplevel; + + tor_assertf(idx >= 0 && idx < smartlist_len(mgr->subconfigs), + "Index %d is out of range.", idx); + config_suite_t **suite = config_mgr_get_suite_ptr(mgr, toplevel); + tor_assert(suite); + tor_assert(smartlist_len(mgr->subconfigs) == + smartlist_len((*suite)->configs)); + + return smartlist_get((*suite)->configs, idx); +} + +/** As config_mgr_get_obj_mutable(), but return a const pointer. */ +STATIC const void * +config_mgr_get_obj(const config_mgr_t *mgr, const void *toplevel, int idx) +{ + return config_mgr_get_obj_mutable(mgr, (void*)toplevel, idx); +} + +/** Sorting helper for smartlist of managed_var_t */ +static int +managed_var_cmp(const void **a, const void **b) +{ + const managed_var_t *mv1 = *(const managed_var_t**)a; + const managed_var_t *mv2 = *(const managed_var_t**)b; + + return strcasecmp(mv1->cvar->member.name, mv2->cvar->member.name); +} + +/** + * Mark a configuration manager as "frozen", so that no more formats can be + * added, and so that it can be used for manipulating configuration objects. + **/ +void +config_mgr_freeze(config_mgr_t *mgr) +{ + static uint64_t mgr_count = 0; + + smartlist_sort(mgr->all_vars, managed_var_cmp); + memcpy(&mgr->toplevel_magic, &mgr->toplevel->magic, + sizeof(struct_magic_decl_t)); + uint64_t magic_input[3] = { mgr->toplevel_magic.magic_val, + (uint64_t) (uintptr_t) mgr, + ++mgr_count }; + mgr->toplevel_magic.magic_val = + (uint32_t)siphash24g(magic_input, sizeof(magic_input)); + mgr->frozen = true; +} + +/** Release all storage held in <b>mgr</b> */ +void +config_mgr_free_(config_mgr_t *mgr) +{ + if (!mgr) + return; + SMARTLIST_FOREACH(mgr->all_vars, managed_var_t *, mv, managed_var_free(mv)); + smartlist_free(mgr->all_vars); + smartlist_free(mgr->all_abbrevs); + smartlist_free(mgr->all_deprecations); + smartlist_free(mgr->subconfigs); + memset(mgr, 0, sizeof(*mgr)); + tor_free(mgr); +} + +/** Return a new smartlist_t containing a config_var_t for every variable that + * <b>mgr</b> knows about. The elements of this smartlist do not need + * to be freed; they have the same lifespan as <b>mgr</b>. */ +smartlist_t * +config_mgr_list_vars(const config_mgr_t *mgr) +{ + smartlist_t *result = smartlist_new(); + tor_assert(mgr); + SMARTLIST_FOREACH(mgr->all_vars, managed_var_t *, mv, + smartlist_add(result, (void*) mv->cvar)); + return result; +} + +/** Return a new smartlist_t containing the names of all deprecated variables. + * The elements of this smartlist do not need to be freed; they have the same + * lifespan as <b>mgr</b>. + */ +smartlist_t * +config_mgr_list_deprecated_vars(const config_mgr_t *mgr) +{ + smartlist_t *result = smartlist_new(); + tor_assert(mgr); + SMARTLIST_FOREACH(mgr->all_deprecations, config_deprecation_t *, d, + smartlist_add(result, (char*)d->name)); + return result; +} + +/** Assert that the magic fields in <b>options</b> and its subsidiary + * objects are all okay. */ +static void +config_mgr_assert_magic_ok(const config_mgr_t *mgr, + const void *options) +{ + tor_assert(mgr); + tor_assert(options); + tor_assert(mgr->frozen); + struct_check_magic(options, &mgr->toplevel_magic); + + config_suite_t **suitep = config_mgr_get_suite_ptr(mgr, (void*)options); + if (suitep == NULL) { + tor_assert(smartlist_len(mgr->subconfigs) == 0); + return; + } + + tor_assert(smartlist_len((*suitep)->configs) == + smartlist_len(mgr->subconfigs)); + SMARTLIST_FOREACH_BEGIN(mgr->subconfigs, const config_format_t *, fmt) { + void *obj = smartlist_get((*suitep)->configs, fmt_sl_idx); + tor_assert(obj); + struct_check_magic(obj, &fmt->magic); + } SMARTLIST_FOREACH_END(fmt); +} + +/** Macro: assert that <b>cfg</b> has the right magic field for + * <b>mgr</b>. */ +#define CONFIG_CHECK(mgr, cfg) STMT_BEGIN \ + config_mgr_assert_magic_ok((mgr), (cfg)); \ + STMT_END + +/** Allocate an empty configuration object of a given format type. */ +void * +config_new(const config_mgr_t *mgr) +{ + tor_assert(mgr->frozen); + void *opts = tor_malloc_zero(mgr->toplevel->size); + struct_set_magic(opts, &mgr->toplevel_magic); + config_suite_t **suitep = config_mgr_get_suite_ptr(mgr, opts); + if (suitep) { + *suitep = config_suite_new(); + SMARTLIST_FOREACH_BEGIN(mgr->subconfigs, const config_format_t *, fmt) { + void *obj = tor_malloc_zero(fmt->size); + struct_set_magic(obj, &fmt->magic); + smartlist_add((*suitep)->configs, obj); + } SMARTLIST_FOREACH_END(fmt); + } + CONFIG_CHECK(mgr, opts); + return opts; +} + +/* + * Functions to parse config options + */ + +/** If <b>option</b> is an official abbreviation for a longer option, + * return the longer option. Otherwise return <b>option</b>. + * If <b>command_line</b> is set, apply all abbreviations. Otherwise, only + * apply abbreviations that work for the config file and the command line. + * If <b>warn_obsolete</b> is set, warn about deprecated names. */ +const char * +config_expand_abbrev(const config_mgr_t *mgr, const char *option, + int command_line, int warn_obsolete) +{ + SMARTLIST_FOREACH_BEGIN(mgr->all_abbrevs, const config_abbrev_t *, abbrev) { + /* Abbreviations are case insensitive. */ + if (!strcasecmp(option, abbrev->abbreviated) && + (command_line || !abbrev->commandline_only)) { + if (warn_obsolete && abbrev->warn) { + log_warn(LD_CONFIG, + "The configuration option '%s' is deprecated; " + "use '%s' instead.", + abbrev->abbreviated, + abbrev->full); + } + /* Keep going through the list in case we want to rewrite it more. + * (We could imagine recursing here, but I don't want to get the + * user into an infinite loop if we craft our list wrong.) */ + option = abbrev->full; + } + } SMARTLIST_FOREACH_END(abbrev); + return option; +} + +/** If <b>key</b> is a deprecated configuration option, return the message + * explaining why it is deprecated (which may be an empty string). Return NULL + * if it is not deprecated. The <b>key</b> field must be fully expanded. */ +const char * +config_find_deprecation(const config_mgr_t *mgr, const char *key) +{ + if (BUG(mgr == NULL) || BUG(key == NULL)) + return NULL; // LCOV_EXCL_LINE + + SMARTLIST_FOREACH_BEGIN(mgr->all_deprecations, const config_deprecation_t *, + d) { + if (!strcasecmp(d->name, key)) { + return d->why_deprecated ? d->why_deprecated : ""; + } + } SMARTLIST_FOREACH_END(d); + return NULL; +} + +/** + * Find the managed_var_t object for a variable whose name is <b>name</b> + * according to <b>mgr</b>. Return that object, or NULL if none exists. + * + * If <b>allow_truncated</b> is true, then accept any variable whose + * name begins with <b>name</b>. + * + * If <b>idx_out</b> is not NULL, set *<b>idx_out</b> to the position of + * that variable within mgr->all_vars, or to -1 if the variable is + * not found. + */ +static const managed_var_t * +config_mgr_find_var(const config_mgr_t *mgr, + const char *key, + bool allow_truncated, int *idx_out) +{ + const size_t keylen = strlen(key); + if (idx_out) + *idx_out = -1; + + if (!keylen) + return NULL; /* if they say "--" on the command line, it's not an option */ + + /* First, check for an exact (case-insensitive) match */ + SMARTLIST_FOREACH_BEGIN(mgr->all_vars, const managed_var_t *, mv) { + if (!strcasecmp(mv->cvar->member.name, key)) { + if (idx_out) + *idx_out = mv_sl_idx; + return mv; + } + } SMARTLIST_FOREACH_END(mv); + + if (!allow_truncated) + return NULL; + + /* If none, check for an abbreviated match */ + SMARTLIST_FOREACH_BEGIN(mgr->all_vars, const managed_var_t *, mv) { + if (!strncasecmp(key, mv->cvar->member.name, keylen)) { + log_warn(LD_CONFIG, "The abbreviation '%s' is deprecated. " + "Please use '%s' instead", + key, mv->cvar->member.name); + if (idx_out) + *idx_out = mv_sl_idx; + return mv; + } + } SMARTLIST_FOREACH_END(mv); + + /* Okay, unrecognized option */ + return NULL; +} + +/** + * If <b>key</b> is a name or an abbreviation configuration option, return + * the corresponding canonical name for it. Warn if the abbreviation is + * non-standard. Return NULL if the option does not exist. + */ +const char * +config_find_option_name(const config_mgr_t *mgr, const char *key) +{ + key = config_expand_abbrev(mgr, key, 0, 0); + const managed_var_t *mv = config_mgr_find_var(mgr, key, true, NULL); + if (mv) + return mv->cvar->member.name; + else + return NULL; +} + +/** Return the number of option entries in <b>fmt</b>. */ +static int +config_count_options(const config_mgr_t *mgr) +{ + return smartlist_len(mgr->all_vars); +} + +/** + * Return true iff at least one bit from <b>flag</b> is set on <b>var</b>, + * either in <b>var</b>'s flags, or on the flags of its type. + **/ +static bool +config_var_has_flag(const config_var_t *var, uint32_t flag) +{ + uint32_t have_flags = var->flags | struct_var_get_flags(&var->member); + + return (have_flags & flag) != 0; +} + +/** + * Return true if assigning a value to <b>var</b> replaces the previous + * value. Return false if assigning a value to <b>var</b> appends + * to the previous value. + **/ +static bool +config_var_is_replaced_on_set(const config_var_t *var) +{ + return ! config_var_has_flag(var, CFLG_NOREPLACE); +} + +/** + * Return true iff <b>var</b> may be assigned by name (e.g., via the + * CLI, the configuration files, or the controller API). + **/ +bool +config_var_is_settable(const config_var_t *var) +{ + return ! config_var_has_flag(var, CFLG_NOSET); +} + +/** + * Return true iff the controller is allowed to fetch the value of + * <b>var</b>. + **/ +static bool +config_var_is_gettable(const config_var_t *var) +{ + /* Arguably, invisible or obsolete options should not be gettable. However, + * they have been gettable for a long time, and making them ungettable could + * have compatibility effects. For now, let's leave them alone. + */ + + // return ! config_var_has_flag(var, CVFLAG_OBSOLETE|CFGLAGS_INVISIBLE); + (void)var; + return true; +} + +/** + * Return true iff we need to check <b>var</b> for changes when we are + * comparing config options for changes. + * + * A false result might mean that the variable is a derived variable, and that + * comparing the variable it derives from compares this one too-- or it might + * mean that there is no data to compare. + **/ +static bool +config_var_should_list_changes(const config_var_t *var) +{ + return ! config_var_has_flag(var, CFLG_NOCMP); +} + +/** + * Return true iff we need to copy the data for <b>var</b> when we are + * copying a config option. + * + * A false option might mean that the variable is a derived variable, and that + * copying the variable it derives from copies it-- or it might mean that + * there is no data to copy. + **/ +static bool +config_var_needs_copy(const config_var_t *var) +{ + return ! config_var_has_flag(var, CFLG_NOCOPY); +} + +/** + * Return true iff variable <b>var</b> should appear on list of variable + * names given to the controller or the CLI. + * + * (Note that this option is imperfectly obeyed. The + * --list-torrc-options command looks at the "settable" flag, whereas + * "GETINFO config/defaults" and "list_deprecated_*()" do not filter + * their results. It would be good for consistency to try to converge + * these behaviors in the future.) + **/ +bool +config_var_is_listable(const config_var_t *var) +{ + return ! config_var_has_flag(var, CFLG_NOLIST); +} + +/** + * Return true iff variable <b>var</b> should be written out when we + * are writing our configuration to disk, to a controller, or via the + * --dump-config command. + * + * This option may be set because a variable is hidden, or because it is + * derived from another variable which will already be written out. + **/ +static bool +config_var_is_dumpable(const config_var_t *var) +{ + return ! config_var_has_flag(var, CFLG_NODUMP); +} + +/* + * Functions to assign config options. + */ + +/** <b>c</b>-\>key is known to be a real key. Update <b>options</b> + * with <b>c</b>-\>value and return 0, or return -1 if bad value. + * + * Called from config_assign_line() and option_reset(). + */ +static int +config_assign_value(const config_mgr_t *mgr, void *options, + config_line_t *c, char **msg) +{ + const managed_var_t *var; + + CONFIG_CHECK(mgr, options); + + var = config_mgr_find_var(mgr, c->key, true, NULL); + tor_assert(var); + tor_assert(!strcmp(c->key, var->cvar->member.name)); + void *object = config_mgr_get_obj_mutable(mgr, options, var->object_idx); + + return struct_var_kvassign(object, c, msg, &var->cvar->member); +} + +/** Mark every linelist in <b>options</b> "fragile", so that fresh assignments + * to it will replace old ones. */ +static void +config_mark_lists_fragile(const config_mgr_t *mgr, void *options) +{ + tor_assert(mgr); + tor_assert(options); + + SMARTLIST_FOREACH_BEGIN(mgr->all_vars, const managed_var_t *, mv) { + void *object = config_mgr_get_obj_mutable(mgr, options, mv->object_idx); + struct_var_mark_fragile(object, &mv->cvar->member); + } SMARTLIST_FOREACH_END(mv); +} + +/** + * Log a warning that declaring that the option called <b>what</b> + * is deprecated because of the reason in <b>why</b>. + * + * (Both arguments must be non-NULL.) + **/ +void +warn_deprecated_option(const char *what, const char *why) +{ + const char *space = (why && strlen(why)) ? " " : ""; + log_warn(LD_CONFIG, "The %s option is deprecated, and will most likely " + "be removed in a future version of Tor.%s%s (If you think this is " + "a mistake, please let us know!)", + what, space, why); +} + +/** If <b>c</b> is a syntactically valid configuration line, update + * <b>options</b> with its value and return 0. Otherwise return -1 for bad + * key, -2 for bad value. + * + * If <b>clear_first</b> is set, clear the value first. Then if + * <b>use_defaults</b> is set, set the value to the default. + * + * Called from config_assign(). + */ +static int +config_assign_line(const config_mgr_t *mgr, void *options, + config_line_t *c, unsigned flags, + bitarray_t *options_seen, char **msg) +{ + const unsigned use_defaults = flags & CAL_USE_DEFAULTS; + const unsigned clear_first = flags & CAL_CLEAR_FIRST; + const unsigned warn_deprecations = flags & CAL_WARN_DEPRECATIONS; + const managed_var_t *mvar; + + CONFIG_CHECK(mgr, options); + + int var_index = -1; + mvar = config_mgr_find_var(mgr, c->key, true, &var_index); + if (!mvar) { + const config_format_t *fmt = mgr->toplevel; + if (fmt->extra) { + void *lvalue = STRUCT_VAR_P(options, fmt->extra->offset); + log_info(LD_CONFIG, + "Found unrecognized option '%s'; saving it.", c->key); + config_line_append((config_line_t**)lvalue, c->key, c->value); + return 0; + } else { + tor_asprintf(msg, + "Unknown option '%s'. Failing.", c->key); + return -1; + } + } + + const config_var_t *cvar = mvar->cvar; + tor_assert(cvar); + + /* Put keyword into canonical case. */ + if (strcmp(cvar->member.name, c->key)) { + tor_free(c->key); + c->key = tor_strdup(cvar->member.name); + } + + const char *deprecation_msg; + if (warn_deprecations && + (deprecation_msg = config_find_deprecation(mgr, cvar->member.name))) { + warn_deprecated_option(cvar->member.name, deprecation_msg); + } + + if (!strlen(c->value)) { + /* reset or clear it, then return */ + if (!clear_first) { + if (! config_var_is_replaced_on_set(cvar) && + c->command != CONFIG_LINE_CLEAR) { + /* We got an empty linelist from the torrc or command line. + As a special case, call this an error. Warn and ignore. */ + log_warn(LD_CONFIG, + "Linelist option '%s' has no value. Skipping.", c->key); + } else { /* not already cleared */ + config_reset(mgr, options, mvar, use_defaults); + } + } + return 0; + } else if (c->command == CONFIG_LINE_CLEAR && !clear_first) { + // This block is unreachable, since a CLEAR line always has an + // empty value, and so will trigger be handled by the previous + // "if (!strlen(c->value))" block. + + // LCOV_EXCL_START + tor_assert_nonfatal_unreached(); + config_reset(mgr, options, mvar, use_defaults); + // LCOV_EXCL_STOP + } + + if (options_seen && config_var_is_replaced_on_set(cvar)) { + /* We're tracking which options we've seen, and this option is not + * supposed to occur more than once. */ + tor_assert(var_index >= 0); + if (bitarray_is_set(options_seen, var_index)) { + log_warn(LD_CONFIG, "Option '%s' used more than once; all but the last " + "value will be ignored.", cvar->member.name); + } + bitarray_set(options_seen, var_index); + } + + if (config_assign_value(mgr, options, c, msg) < 0) + return -2; + return 0; +} + +/** Restore the option named <b>key</b> in options to its default value. + * Called from config_assign(). */ +STATIC void +config_reset_line(const config_mgr_t *mgr, void *options, + const char *key, int use_defaults) +{ + const managed_var_t *var; + + CONFIG_CHECK(mgr, options); + + var = config_mgr_find_var(mgr, key, true, NULL); + if (!var) + return; /* give error on next pass. */ + + config_reset(mgr, options, var, use_defaults); +} + +/** Return true iff value needs to be quoted and escaped to be used in + * a configuration file. */ +static int +config_value_needs_escape(const char *value) +{ + if (*value == '\"') + return 1; + while (*value) { + switch (*value) + { + case '\r': + case '\n': + case '#': + /* Note: quotes and backspaces need special handling when we are using + * quotes, not otherwise, so they don't trigger escaping on their + * own. */ + return 1; + default: + if (!TOR_ISPRINT(*value)) + return 1; + } + ++value; + } + return 0; +} + +/** Return newly allocated line or lines corresponding to <b>key</b> in the + * configuration <b>options</b>. If <b>escape_val</b> is true and a + * value needs to be quoted before it's put in a config file, quote and + * escape that value. Return NULL if no such key exists. */ +config_line_t * +config_get_assigned_option(const config_mgr_t *mgr, const void *options, + const char *key, int escape_val) +{ + const managed_var_t *var; + config_line_t *result; + + tor_assert(options && key); + + CONFIG_CHECK(mgr, options); + + var = config_mgr_find_var(mgr, key, true, NULL); + if (!var) { + log_warn(LD_CONFIG, "Unknown option '%s'. Failing.", key); + return NULL; + } + if (! config_var_is_gettable(var->cvar)) { + log_warn(LD_CONFIG, "Option '%s' is obsolete or unfetchable. Failing.", + key); + return NULL; + } + const void *object = config_mgr_get_obj(mgr, options, var->object_idx); + + result = struct_var_kvencode(object, &var->cvar->member); + + if (escape_val) { + config_line_t *line; + for (line = result; line; line = line->next) { + if (line->value && config_value_needs_escape(line->value)) { + char *newval = esc_for_log(line->value); + tor_free(line->value); + line->value = newval; + } + } + } + + return result; +} +/** Iterate through the linked list of requested options <b>list</b>. + * For each item, convert as appropriate and assign to <b>options</b>. + * If an item is unrecognized, set *msg and return -1 immediately, + * else return 0 for success. + * + * If <b>clear_first</b>, interpret config options as replacing (not + * extending) their previous values. If <b>clear_first</b> is set, + * then <b>use_defaults</b> to decide if you set to defaults after + * clearing, or make the value 0 or NULL. + * + * Here are the use cases: + * 1. A non-empty AllowInvalid line in your torrc. Appends to current + * if linelist, replaces current if csv. + * 2. An empty AllowInvalid line in your torrc. Should clear it. + * 3. "RESETCONF AllowInvalid" sets it to default. + * 4. "SETCONF AllowInvalid" makes it NULL. + * 5. "SETCONF AllowInvalid=foo" clears it and sets it to "foo". + * + * Use_defaults Clear_first + * 0 0 "append" + * 1 0 undefined, don't use + * 0 1 "set to null first" + * 1 1 "set to defaults first" + * Return 0 on success, -1 on bad key, -2 on bad value. + * + * As an additional special case, if a LINELIST config option has + * no value and clear_first is 0, then warn and ignore it. + */ + +/* +There are three call cases for config_assign() currently. + +Case one: Torrc entry +options_init_from_torrc() calls config_assign(0, 0) + calls config_assign_line(0, 0). + if value is empty, calls config_reset(0) and returns. + calls config_assign_value(), appends. + +Case two: setconf +options_trial_assign() calls config_assign(0, 1) + calls config_reset_line(0) + calls config_reset(0) + calls option_clear(). + calls config_assign_line(0, 1). + if value is empty, returns. + calls config_assign_value(), appends. + +Case three: resetconf +options_trial_assign() calls config_assign(1, 1) + calls config_reset_line(1) + calls config_reset(1) + calls option_clear(). + calls config_assign_value(default) + calls config_assign_line(1, 1). + returns. +*/ +int +config_assign(const config_mgr_t *mgr, void *options, config_line_t *list, + unsigned config_assign_flags, char **msg) +{ + config_line_t *p; + bitarray_t *options_seen; + const int n_options = config_count_options(mgr); + const unsigned clear_first = config_assign_flags & CAL_CLEAR_FIRST; + const unsigned use_defaults = config_assign_flags & CAL_USE_DEFAULTS; + + CONFIG_CHECK(mgr, options); + + /* pass 1: normalize keys */ + for (p = list; p; p = p->next) { + const char *full = config_expand_abbrev(mgr, p->key, 0, 1); + if (strcmp(full,p->key)) { + tor_free(p->key); + p->key = tor_strdup(full); + } + } + + /* pass 2: if we're reading from a resetting source, clear all + * mentioned config options, and maybe set to their defaults. */ + if (clear_first) { + for (p = list; p; p = p->next) + config_reset_line(mgr, options, p->key, use_defaults); + } + + options_seen = bitarray_init_zero(n_options); + /* pass 3: assign. */ + while (list) { + int r; + if ((r=config_assign_line(mgr, options, list, config_assign_flags, + options_seen, msg))) { + bitarray_free(options_seen); + return r; + } + list = list->next; + } + bitarray_free(options_seen); + + /** Now we're done assigning a group of options to the configuration. + * Subsequent group assignments should _replace_ linelists, not extend + * them. */ + config_mark_lists_fragile(mgr, options); + + return 0; +} + +/** Reset config option <b>var</b> to 0, 0.0, NULL, or the equivalent. + * Called from config_reset() and config_free(). */ +static void +config_clear(const config_mgr_t *mgr, void *options, const managed_var_t *var) +{ + void *object = config_mgr_get_obj_mutable(mgr, options, var->object_idx); + struct_var_free(object, &var->cvar->member); +} + +/** Clear the option indexed by <b>var</b> in <b>options</b>. Then if + * <b>use_defaults</b>, set it to its default value. + * Called by config_init() and option_reset_line() and option_assign_line(). */ +static void +config_reset(const config_mgr_t *mgr, void *options, + const managed_var_t *var, int use_defaults) +{ + config_line_t *c; + char *msg = NULL; + CONFIG_CHECK(mgr, options); + config_clear(mgr, options, var); /* clear it first */ + + if (!use_defaults) + return; /* all done */ + + if (var->cvar->initvalue) { + c = tor_malloc_zero(sizeof(config_line_t)); + c->key = tor_strdup(var->cvar->member.name); + c->value = tor_strdup(var->cvar->initvalue); + if (config_assign_value(mgr, options, c, &msg) < 0) { + // LCOV_EXCL_START + log_warn(LD_BUG, "Failed to assign default: %s", msg); + tor_free(msg); /* if this happens it's a bug */ + // LCOV_EXCL_STOP + } + config_free_lines(c); + } +} + +/** Release storage held by <b>options</b>. */ +void +config_free_(const config_mgr_t *mgr, void *options) +{ + if (!options) + return; + + tor_assert(mgr); + + if (mgr->toplevel->clear_fn) { + mgr->toplevel->clear_fn(mgr, options); + } + config_suite_t **suitep = config_mgr_get_suite_ptr(mgr, options); + if (suitep) { + tor_assert(smartlist_len((*suitep)->configs) == + smartlist_len(mgr->subconfigs)); + SMARTLIST_FOREACH_BEGIN(mgr->subconfigs, const config_format_t *, fmt) { + void *obj = smartlist_get((*suitep)->configs, fmt_sl_idx); + if (fmt->clear_fn) { + fmt->clear_fn(mgr, obj); + } + } SMARTLIST_FOREACH_END(fmt); + } + + SMARTLIST_FOREACH_BEGIN(mgr->all_vars, const managed_var_t *, mv) { + config_clear(mgr, options, mv); + } SMARTLIST_FOREACH_END(mv); + + if (mgr->toplevel->extra) { + config_line_t **linep = STRUCT_VAR_P(options, + mgr->toplevel->extra->offset); + config_free_lines(*linep); + *linep = NULL; + } + + if (suitep) { + SMARTLIST_FOREACH((*suitep)->configs, void *, obj, tor_free(obj)); + config_suite_free(*suitep); + } + + tor_free(options); +} + +/** Return true iff the option <b>name</b> has the same value in <b>o1</b> + * and <b>o2</b>. Must not be called for LINELIST_S or OBSOLETE options. + */ +int +config_is_same(const config_mgr_t *mgr, + const void *o1, const void *o2, + const char *name) +{ + CONFIG_CHECK(mgr, o1); + CONFIG_CHECK(mgr, o2); + + const managed_var_t *var = config_mgr_find_var(mgr, name, true, NULL); + if (!var) { + return true; + } + const void *obj1 = config_mgr_get_obj(mgr, o1, var->object_idx); + const void *obj2 = config_mgr_get_obj(mgr, o2, var->object_idx); + + return struct_var_eq(obj1, obj2, &var->cvar->member); +} + +/** + * Return a list of the options which have changed between <b>options1</b> and + * <b>options2</b>. If an option has reverted to its default value, it has a + * value entry of NULL. + * + * <b>options1</b> and <b>options2</b> must be top-level configuration objects + * of the type managed by <b>mgr</b>. + **/ +config_line_t * +config_get_changes(const config_mgr_t *mgr, + const void *options1, const void *options2) +{ + config_line_t *result = NULL; + config_line_t **next = &result; + SMARTLIST_FOREACH_BEGIN(mgr->all_vars, managed_var_t *, mv) { + if (! config_var_should_list_changes(mv->cvar)) { + /* something else will check this var, or it doesn't need checking */ + continue; + } + const void *obj1 = config_mgr_get_obj(mgr, options1, mv->object_idx); + const void *obj2 = config_mgr_get_obj(mgr, options2, mv->object_idx); + + if (struct_var_eq(obj1, obj2, &mv->cvar->member)) { + continue; + } + + const char *varname = mv->cvar->member.name; + config_line_t *line = + config_get_assigned_option(mgr, options2, varname, 1); + + if (line) { + *next = line; + } else { + *next = tor_malloc_zero(sizeof(config_line_t)); + (*next)->key = tor_strdup(varname); + } + while (*next) + next = &(*next)->next; + } SMARTLIST_FOREACH_END(mv); + + return result; +} + +/** Copy storage held by <b>old</b> into a new or_options_t and return it. */ +void * +config_dup(const config_mgr_t *mgr, const void *old) +{ + void *newopts; + + newopts = config_new(mgr); + SMARTLIST_FOREACH_BEGIN(mgr->all_vars, managed_var_t *, mv) { + if (! config_var_needs_copy(mv->cvar)) { + // Something else will copy this option, or it doesn't need copying. + continue; + } + const void *oldobj = config_mgr_get_obj(mgr, old, mv->object_idx); + void *newobj = config_mgr_get_obj_mutable(mgr, newopts, mv->object_idx); + if (struct_var_copy(newobj, oldobj, &mv->cvar->member) < 0) { + // LCOV_EXCL_START + log_err(LD_BUG, "Unable to copy value for %s.", + mv->cvar->member.name); + tor_assert_unreached(); + // LCOV_EXCL_STOP + } + } SMARTLIST_FOREACH_END(mv); + + return newopts; +} +/** Set all vars in the configuration object <b>options</b> to their default + * values. */ +void +config_init(const config_mgr_t *mgr, void *options) +{ + CONFIG_CHECK(mgr, options); + + SMARTLIST_FOREACH_BEGIN(mgr->all_vars, const managed_var_t *, mv) { + if (!mv->cvar->initvalue) + continue; /* defaults to NULL or 0 */ + config_reset(mgr, options, mv, 1); + } SMARTLIST_FOREACH_END(mv); +} + +/** Allocate and return a new string holding the written-out values of the vars + * in 'options'. If 'minimal', do not write out any default-valued vars. + * Else, if comment_defaults, write default values as comments. + */ +char * +config_dump(const config_mgr_t *mgr, const void *default_options, + const void *options, int minimal, + int comment_defaults) +{ + const config_format_t *fmt = mgr->toplevel; + smartlist_t *elements; + const void *defaults = default_options; + void *defaults_tmp = NULL; + config_line_t *line, *assigned; + char *result; + char *msg = NULL; + + if (defaults == NULL) { + defaults = defaults_tmp = config_new(mgr); + config_init(mgr, defaults_tmp); + } + + /* XXX use a 1 here so we don't add a new log line while dumping */ + if (default_options == NULL) { + if (fmt->validate_fn(NULL, defaults_tmp, defaults_tmp, 1, &msg) < 0) { + // LCOV_EXCL_START + log_err(LD_BUG, "Failed to validate default config: %s", msg); + tor_free(msg); + tor_assert(0); + // LCOV_EXCL_STOP + } + } + + elements = smartlist_new(); + SMARTLIST_FOREACH_BEGIN(mgr->all_vars, managed_var_t *, mv) { + int comment_option = 0; + /* Don't save 'hidden' control variables. */ + if (! config_var_is_dumpable(mv->cvar)) + continue; + const char *name = mv->cvar->member.name; + if (minimal && config_is_same(mgr, options, defaults, name)) + continue; + else if (comment_defaults && + config_is_same(mgr, options, defaults, name)) + comment_option = 1; + + line = assigned = + config_get_assigned_option(mgr, options, name, 1); + + for (; line; line = line->next) { + if (!strcmpstart(line->key, "__")) { + /* This check detects "hidden" variables inside LINELIST_V structures. + */ + continue; + } + smartlist_add_asprintf(elements, "%s%s %s\n", + comment_option ? "# " : "", + line->key, line->value); + } + config_free_lines(assigned); + } SMARTLIST_FOREACH_END(mv); + + if (fmt->extra) { + line = *(config_line_t**)STRUCT_VAR_P(options, fmt->extra->offset); + for (; line; line = line->next) { + smartlist_add_asprintf(elements, "%s %s\n", line->key, line->value); + } + } + + result = smartlist_join_strings(elements, "", 0, NULL); + SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp)); + smartlist_free(elements); + config_free(mgr, defaults_tmp); + return result; +} + +/** + * Return true if every member of <b>options</b> is in-range and well-formed. + * Return false otherwise. Log errors at level <b>severity</b>. + */ +bool +config_check_ok(const config_mgr_t *mgr, const void *options, int severity) +{ + bool all_ok = true; + + SMARTLIST_FOREACH_BEGIN(mgr->all_vars, const managed_var_t *, mv) { + if (!struct_var_ok(options, &mv->cvar->member)) { + log_fn(severity, LD_BUG, "Invalid value for %s", + mv->cvar->member.name); + all_ok = false; + } + } SMARTLIST_FOREACH_END(mv); + + return all_ok; +} diff --git a/src/lib/confmgt/confparse.h b/src/lib/confmgt/confparse.h new file mode 100644 index 0000000000..2332f69790 --- /dev/null +++ b/src/lib/confmgt/confparse.h @@ -0,0 +1,212 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file confparse.h + * + * \brief Header for confparse.c. + */ + +#ifndef TOR_CONFPARSE_H +#define TOR_CONFPARSE_H + +#include "lib/conf/conftypes.h" +#include "lib/conf/confmacros.h" +#include "lib/testsupport/testsupport.h" + +/** + * An abbreviation or alias for a configuration option. + **/ +typedef struct config_abbrev_t { + /** The option name as abbreviated. Not case-sensitive. */ + const char *abbreviated; + /** The full name of the option. Not case-sensitive. */ + const char *full; + /** True if this abbreviation should only be allowed on the command line. */ + int commandline_only; + /** True if we should warn whenever this abbreviation is used. */ + int warn; +} config_abbrev_t; + +/** + * A note that a configuration option is deprecated, with an explanation why. + */ +typedef struct config_deprecation_t { + /** The option that is deprecated. */ + const char *name; + /** A user-facing string explaining why the option is deprecated. */ + const char *why_deprecated; +} config_deprecation_t; + +/** + * Handy macro for declaring "In the config file or on the command line, you + * can abbreviate <b>tok</b>s as <b>tok</b>". Used inside an array of + * config_abbrev_t. + * + * For example, to declare "NumCpu" as an abbreviation for "NumCPUs", + * you can say PLURAL(NumCpu). + **/ +#define PLURAL(tok) { #tok, #tok "s", 0, 0 } + +/** + * Type of a callback to validate whether a given configuration is + * well-formed and consistent. + * + * The configuration to validate is passed as <b>newval</b>. The previous + * configuration, if any, is provided in <b>oldval</b>. The + * <b>default_val</b> argument receives a configuration object initialized + * with default values for all its fields. The <b>from_setconf</b> argument + * is true iff the input comes from a SETCONF controller command. + * + * On success, return 0. On failure, set *<b>msg_out</b> to a newly allocated + * error message, and return -1. + * + * REFACTORING NOTE: Currently, this callback type is only used from inside + * config_dump(); later in our refactoring, it will be cleaned up and used + * more generally. + */ +typedef int (*validate_fn_t)(void *oldval, + void *newval, + void *default_val, + int from_setconf, + char **msg_out); + +struct config_mgr_t; + +/** + * Callback to clear all non-managed fields of a configuration object. + * + * <b>obj</b> is the configuration object whose non-managed fields should be + * cleared. + * + * (Regular fields get cleared by config_reset(), but you might have fields + * in the object that do not correspond to configuration variables. If those + * fields need to be cleared or freed, this is where to do it.) + */ +typedef void (*clear_cfg_fn_t)(const struct config_mgr_t *mgr, void *obj); + +/** Information on the keys, value types, key-to-struct-member mappings, + * variable descriptions, validation functions, and abbreviations for a + * configuration or storage format. */ +typedef struct config_format_t { + size_t size; /**< Size of the struct that everything gets parsed into. */ + struct_magic_decl_t magic; /**< Magic number info for this struct. */ + const config_abbrev_t *abbrevs; /**< List of abbreviations that we expand + * when parsing this format. */ + const config_deprecation_t *deprecations; /** List of deprecated options */ + const config_var_t *vars; /**< List of variables we recognize, their default + * values, and where we stick them in the + * structure. */ + validate_fn_t validate_fn; /**< Function to validate config. */ + clear_cfg_fn_t clear_fn; /**< Function to clear the configuration. */ + /** If present, extra denotes a LINELIST variable for unrecognized + * lines. Otherwise, unrecognized lines are an error. */ + const struct_member_t *extra; + /** The position of a config_suite_t pointer within the toplevel object, + * or -1 if there is no such pointer. */ + ptrdiff_t config_suite_offset; +} config_format_t; + +/** + * A collection of config_format_t objects to describe several objects + * that are all configured with the same configuration file. + * + * (NOTE: for now, this only handles a single config_format_t.) + **/ +typedef struct config_mgr_t config_mgr_t; + +config_mgr_t *config_mgr_new(const config_format_t *toplevel_fmt); +void config_mgr_free_(config_mgr_t *mgr); +int config_mgr_add_format(config_mgr_t *mgr, + const config_format_t *fmt); +void config_mgr_freeze(config_mgr_t *mgr); +#define config_mgr_free(mgr) \ + FREE_AND_NULL(config_mgr_t, config_mgr_free_, (mgr)) +struct smartlist_t *config_mgr_list_vars(const config_mgr_t *mgr); +struct smartlist_t *config_mgr_list_deprecated_vars(const config_mgr_t *mgr); + +/** A collection of managed configuration objects. */ +typedef struct config_suite_t config_suite_t; + +/** + * Flag for config_assign: if set, then "resetting" an option changes it to + * its default value, as specified in the config_var_t. Otherwise, + * "resetting" an option changes it to a type-dependent null value -- + * typically 0 or NULL. + * + * (An option is "reset" when it is set to an empty value, or as described in + * CAL_CLEAR_FIRST). + **/ +#define CAL_USE_DEFAULTS (1u<<0) +/** + * Flag for config_assign: if set, then we reset every provided config + * option before we set it. + * + * For example, if this flag is not set, then passing a multi-line option to + * config_assign will cause any previous value to be extended. But if this + * flag is set, then a multi-line option will replace any previous value. + **/ +#define CAL_CLEAR_FIRST (1u<<1) +/** + * Flag for config_assign: if set, we warn about deprecated options. + **/ +#define CAL_WARN_DEPRECATIONS (1u<<2) + +void *config_new(const config_mgr_t *fmt); +void config_free_(const config_mgr_t *fmt, void *options); +#define config_free(mgr, options) do { \ + config_free_((mgr), (options)); \ + (options) = NULL; \ + } while (0) + +struct config_line_t *config_get_assigned_option(const config_mgr_t *mgr, + const void *options, const char *key, + int escape_val); +int config_is_same(const config_mgr_t *fmt, + const void *o1, const void *o2, + const char *name); +struct config_line_t *config_get_changes(const config_mgr_t *mgr, + const void *options1, const void *options2); +void config_init(const config_mgr_t *mgr, void *options); +void *config_dup(const config_mgr_t *mgr, const void *old); +char *config_dump(const config_mgr_t *mgr, const void *default_options, + const void *options, int minimal, + int comment_defaults); +bool config_check_ok(const config_mgr_t *mgr, const void *options, + int severity); +int config_assign(const config_mgr_t *mgr, void *options, + struct config_line_t *list, + unsigned flags, char **msg); +const char *config_find_deprecation(const config_mgr_t *mgr, + const char *key); +const char *config_find_option_name(const config_mgr_t *mgr, + const char *key); +const char *config_expand_abbrev(const config_mgr_t *mgr, + const char *option, + int command_line, int warn_obsolete); +void warn_deprecated_option(const char *what, const char *why); + +bool config_var_is_settable(const config_var_t *var); +bool config_var_is_listable(const config_var_t *var); + +/* Helper macros to compare an option across two configuration objects */ +#define CFG_EQ_BOOL(a,b,opt) ((a)->opt == (b)->opt) +#define CFG_EQ_INT(a,b,opt) ((a)->opt == (b)->opt) +#define CFG_EQ_STRING(a,b,opt) (!strcmp_opt((a)->opt, (b)->opt)) +#define CFG_EQ_SMARTLIST(a,b,opt) smartlist_strings_eq((a)->opt, (b)->opt) +#define CFG_EQ_LINELIST(a,b,opt) config_lines_eq((a)->opt, (b)->opt) +#define CFG_EQ_ROUTERSET(a,b,opt) routerset_equal((a)->opt, (b)->opt) + +#ifdef CONFPARSE_PRIVATE +STATIC void config_reset_line(const config_mgr_t *mgr, void *options, + const char *key, int use_defaults); +STATIC void *config_mgr_get_obj_mutable(const config_mgr_t *mgr, + void *toplevel, int idx); +STATIC const void *config_mgr_get_obj(const config_mgr_t *mgr, + const void *toplevel, int idx); +#endif /* defined(CONFPARSE_PRIVATE) */ + +#endif /* !defined(TOR_CONFPARSE_H) */ diff --git a/src/lib/confmgt/include.am b/src/lib/confmgt/include.am new file mode 100644 index 0000000000..81cd868e5e --- /dev/null +++ b/src/lib/confmgt/include.am @@ -0,0 +1,27 @@ +noinst_LIBRARIES += src/lib/libtor-confmgt.a + +if UNITTESTS_ENABLED +noinst_LIBRARIES += src/lib/libtor-confmgt-testing.a +endif + +# ADD_C_FILE: INSERT SOURCES HERE. +src_lib_libtor_confmgt_a_SOURCES = \ + src/lib/confmgt/confparse.c \ + src/lib/confmgt/structvar.c \ + src/lib/confmgt/type_defs.c \ + src/lib/confmgt/typedvar.c \ + src/lib/confmgt/unitparse.c + +src_lib_libtor_confmgt_testing_a_SOURCES = \ + $(src_lib_libtor_confmgt_a_SOURCES) +src_lib_libtor_confmgt_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) +src_lib_libtor_confmgt_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) + +# ADD_C_FILE: INSERT HEADERS HERE. +noinst_HEADERS += \ + src/lib/confmgt/confparse.h \ + src/lib/confmgt/structvar.h \ + src/lib/confmgt/type_defs.h \ + src/lib/confmgt/typedvar.h \ + src/lib/confmgt/unitparse.h \ + src/lib/confmgt/var_type_def_st.h diff --git a/src/lib/confmgt/structvar.c b/src/lib/confmgt/structvar.c new file mode 100644 index 0000000000..7a3b8c7df2 --- /dev/null +++ b/src/lib/confmgt/structvar.c @@ -0,0 +1,221 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file structvar.c + * @brief Functions to manipulate named and typed elements of + * a structure. + * + * These functions represent a low-level API for accessing a member of a + * structure. They use typedvar.c to work, and they are used in turn by the + * configuration system to examine and set fields in configuration objects + * used by individual modules. + * + * Almost no code should call these directly. + **/ + +#include "orconfig.h" +#include "lib/confmgt/structvar.h" +#include "lib/cc/compat_compiler.h" +#include "lib/conf/conftypes.h" +#include "lib/confmgt/type_defs.h" +#include "lib/confmgt/typedvar.h" +#include "lib/log/util_bug.h" + +#include "lib/confmgt/var_type_def_st.h" + +#include <stddef.h> + +/** + * Set the 'magic number' on <b>object</b> to correspond to decl. + **/ +void +struct_set_magic(void *object, const struct_magic_decl_t *decl) +{ + tor_assert(object); + tor_assert(decl); + uint32_t *ptr = STRUCT_VAR_P(object, decl->magic_offset); + *ptr = decl->magic_val; +} + +/** + * Assert that the 'magic number' on <b>object</b> to corresponds to decl. + **/ +void +struct_check_magic(const void *object, const struct_magic_decl_t *decl) +{ + tor_assert(object); + tor_assert(decl); + + const uint32_t *ptr = STRUCT_VAR_P(object, decl->magic_offset); + tor_assertf(*ptr == decl->magic_val, + "Bad magic number on purported %s object. " + "Expected %"PRIu32"x but got %"PRIu32"x.", + decl->typename, decl->magic_val, *ptr); +} + +/** + * Return a mutable pointer to the member of <b>object</b> described + * by <b>member</b>. + **/ +void * +struct_get_mptr(void *object, const struct_member_t *member) +{ + tor_assert(object); + return STRUCT_VAR_P(object, member->offset); +} + +/** + * Return a const pointer to the member of <b>object</b> described + * by <b>member</b>. + **/ +const void * +struct_get_ptr(const void *object, const struct_member_t *member) +{ + tor_assert(object); + return STRUCT_VAR_P(object, member->offset); +} + +/** + * Helper: given a struct_member_t, look up the type definition for its + * variable. + */ +static const var_type_def_t * +get_type_def(const struct_member_t *member) +{ + if (member->type_def) + return member->type_def; + + return lookup_type_def(member->type); +} + +/** + * (As typed_var_free, but free and clear the member of <b>object</b> defined + * by <b>member</b>.) + **/ +void +struct_var_free(void *object, const struct_member_t *member) +{ + void *p = struct_get_mptr(object, member); + const var_type_def_t *def = get_type_def(member); + + typed_var_free(p, def); +} + +/** + * (As typed_var_copy, but copy from <b>src</b> to <b>dest</b> the member + * defined by <b>member</b>.) + **/ +int +struct_var_copy(void *dest, const void *src, const struct_member_t *member) +{ + void *p_dest = struct_get_mptr(dest, member); + const void *p_src = struct_get_ptr(src, member); + const var_type_def_t *def = get_type_def(member); + + return typed_var_copy(p_dest, p_src, def); +} + +/** + * (As typed_var_eq, but compare the members of <b>a</b> and <b>b</b> + * defined by <b>member</b>.) + **/ +bool +struct_var_eq(const void *a, const void *b, const struct_member_t *member) +{ + const void *p_a = struct_get_ptr(a, member); + const void *p_b = struct_get_ptr(b, member); + const var_type_def_t *def = get_type_def(member); + + return typed_var_eq(p_a, p_b, def); +} + +/** + * (As typed_var_ok, but validate the member of <b>object</b> defined by + * <b>member</b>.) + **/ +bool +struct_var_ok(const void *object, const struct_member_t *member) +{ + const void *p = struct_get_ptr(object, member); + const var_type_def_t *def = get_type_def(member); + + return typed_var_ok(p, def); +} + +/** + * (As typed_var_kvassign, but assign a value to the member of <b>object</b> + * defined by <b>member</b>.) + **/ +int +struct_var_kvassign(void *object, const struct config_line_t *line, + char **errmsg, + const struct_member_t *member) +{ + void *p = struct_get_mptr(object, member); + const var_type_def_t *def = get_type_def(member); + + return typed_var_kvassign(p, line, errmsg, def); +} + +/** + * (As typed_var_kvencode, but encode the value of the member of <b>object</b> + * defined by <b>member</b>.) + **/ +struct config_line_t * +struct_var_kvencode(const void *object, const struct_member_t *member) +{ + const void *p = struct_get_ptr(object, member); + const var_type_def_t *def = get_type_def(member); + + return typed_var_kvencode(member->name, p, def); +} + +/** + * Mark the field in <b>object</b> determined by <b>member</b> -- a variable + * that ordinarily would be extended by assignment -- as "fragile", so that it + * will get replaced by the next assignment instead. + */ +void +struct_var_mark_fragile(void *object, const struct_member_t *member) +{ + void *p = struct_get_mptr(object, member); + const var_type_def_t *def = get_type_def(member); + return typed_var_mark_fragile(p, def); +} + +/** + * Return the official name of this struct member. + **/ +const char * +struct_var_get_name(const struct_member_t *member) +{ + return member->name; +} + +/** + * Return the type name for this struct member. + * + * Do not use the output of this function to inspect a type within Tor. It is + * suitable for debugging, informing the controller or user of a variable's + * type, etc. + **/ +const char * +struct_var_get_typename(const struct_member_t *member) +{ + const var_type_def_t *def = get_type_def(member); + + return def ? def->name : NULL; +} + +/** Return all of the flags set for this struct member. */ +uint32_t +struct_var_get_flags(const struct_member_t *member) +{ + const var_type_def_t *def = get_type_def(member); + + return def ? def->flags : 0; +} diff --git a/src/lib/confmgt/structvar.h b/src/lib/confmgt/structvar.h new file mode 100644 index 0000000000..bcb4b58c3f --- /dev/null +++ b/src/lib/confmgt/structvar.h @@ -0,0 +1,54 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file structvar.h + * @brief Header for lib/confmgt/structvar.c + **/ + +#ifndef TOR_LIB_CONFMGT_STRUCTVAR_H +#define TOR_LIB_CONFMGT_STRUCTVAR_H + +struct struct_magic_decl_t; +struct struct_member_t; +struct config_line_t; + +#include <stdbool.h> +#include "lib/cc/torint.h" + +void struct_set_magic(void *object, + const struct struct_magic_decl_t *decl); +void struct_check_magic(const void *object, + const struct struct_magic_decl_t *decl); + +void *struct_get_mptr(void *object, + const struct struct_member_t *member); +const void *struct_get_ptr(const void *object, + const struct struct_member_t *member); + +void struct_var_free(void *object, + const struct struct_member_t *member); +int struct_var_copy(void *dest, const void *src, + const struct struct_member_t *member); +bool struct_var_eq(const void *a, const void *b, + const struct struct_member_t *member); +bool struct_var_ok(const void *object, + const struct struct_member_t *member); +void struct_var_mark_fragile(void *object, + const struct struct_member_t *member); + +const char *struct_var_get_name(const struct struct_member_t *member); +const char *struct_var_get_typename(const struct struct_member_t *member); +uint32_t struct_var_get_flags(const struct struct_member_t *member); + +int struct_var_kvassign(void *object, const struct config_line_t *line, + char **errmsg, + const struct struct_member_t *member); +struct config_line_t *struct_var_kvencode( + const void *object, + const struct struct_member_t *member); + +#endif /* !defined(TOR_LIB_CONFMGT_STRUCTVAR_H) */ diff --git a/src/lib/confmgt/type_defs.c b/src/lib/confmgt/type_defs.c new file mode 100644 index 0000000000..5066e12265 --- /dev/null +++ b/src/lib/confmgt/type_defs.c @@ -0,0 +1,791 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file type_defs.c + * @brief Definitions for various low-level configuration types. + * + * This module creates a number of var_type_def_t objects, to be used by + * typedvar.c in manipulating variables. + * + * The types here are common types that can be implemented with Tor's + * low-level functionality. To define new types, see var_type_def_st.h. + **/ + +#include "orconfig.h" +#include "lib/conf/conftypes.h" +#include "lib/confmgt/typedvar.h" +#include "lib/confmgt/type_defs.h" +#include "lib/confmgt/unitparse.h" + +#include "lib/cc/compat_compiler.h" +#include "lib/conf/conftypes.h" +#include "lib/container/smartlist.h" +#include "lib/encoding/confline.h" +#include "lib/encoding/time_fmt.h" +#include "lib/log/escape.h" +#include "lib/log/log.h" +#include "lib/log/util_bug.h" +#include "lib/malloc/malloc.h" +#include "lib/string/parse_int.h" +#include "lib/string/printf.h" + +#include "lib/confmgt/var_type_def_st.h" + +#include <stddef.h> +#include <string.h> +#include <errno.h> + +////// +// CONFIG_TYPE_STRING +// CONFIG_TYPE_FILENAME +// +// These two types are the same for now, but they have different names. +// +// Warning: For this type, the default value (NULL) and "" are considered +// different values. That is generally risky, and best avoided for other +// types in the future. +////// + +static int +string_parse(void *target, const char *value, char **errmsg, + const void *params, const char *key) +{ + (void)params; + (void)errmsg; + (void)key; + char **p = (char**)target; + *p = tor_strdup(value); + return 0; +} + +static char * +string_encode(const void *value, const void *params) +{ + (void)params; + const char **p = (const char**)value; + return *p ? tor_strdup(*p) : NULL; +} + +static void +string_clear(void *value, const void *params) +{ + (void)params; + char **p = (char**)value; + tor_free(*p); // sets *p to NULL. +} + +static const var_type_fns_t string_fns = { + .parse = string_parse, + .encode = string_encode, + .clear = string_clear, +}; + +///// +// CONFIG_TYPE_INT +// CONFIG_TYPE_POSINT +// +// These types are implemented as int, possibly with a restricted range. +///// + +typedef struct int_type_params_t { + int minval; + int maxval; +} int_parse_params_t; + +static const int_parse_params_t INT_PARSE_UNRESTRICTED = { + .minval = INT_MIN, + .maxval = INT_MAX, +}; + +static const int_parse_params_t INT_PARSE_POSINT = { + .minval = 0, + .maxval = INT_MAX, +}; + +static int +int_parse(void *target, const char *value, char **errmsg, const void *params, + const char *key) +{ + (void)key; + const int_parse_params_t *pp; + if (params) { + pp = params; + } else { + pp = &INT_PARSE_UNRESTRICTED; + } + int *p = target; + int ok=0; + *p = (int)tor_parse_long(value, 10, pp->minval, pp->maxval, &ok, NULL); + if (!ok) { + tor_asprintf(errmsg, "Integer %s is malformed or out of bounds.", + value); + return -1; + } + return 0; +} + +static char * +int_encode(const void *value, const void *params) +{ + (void)params; + int v = *(int*)value; + char *result; + tor_asprintf(&result, "%d", v); + return result; +} + +static void +int_clear(void *value, const void *params) +{ + (void)params; + *(int*)value = 0; +} + +static bool +int_ok(const void *value, const void *params) +{ + const int_parse_params_t *pp = params; + if (pp) { + int v = *(int*)value; + return pp->minval <= v && v <= pp->maxval; + } else { + return true; + } +} + +static const var_type_fns_t int_fns = { + .parse = int_parse, + .encode = int_encode, + .clear = int_clear, + .ok = int_ok, +}; + +///// +// CONFIG_TYPE_UINT64 +// +// This type is an unrestricted u64. +///// + +static int +uint64_parse(void *target, const char *value, char **errmsg, + const void *params, const char *key) +{ + (void)params; + (void)errmsg; + (void)key; + uint64_t *p = target; + int ok=0; + *p = tor_parse_uint64(value, 10, 0, UINT64_MAX, &ok, NULL); + if (!ok) { + tor_asprintf(errmsg, "Integer %s is malformed or out of bounds.", + value); + return -1; + } + return 0; +} + +static char * +uint64_encode(const void *value, const void *params) +{ + (void)params; + uint64_t v = *(uint64_t*)value; + char *result; + tor_asprintf(&result, "%"PRIu64, v); + return result; +} + +static void +uint64_clear(void *value, const void *params) +{ + (void)params; + *(uint64_t*)value = 0; +} + +static const var_type_fns_t uint64_fns = { + .parse = uint64_parse, + .encode = uint64_encode, + .clear = uint64_clear, +}; + +///// +// CONFIG_TYPE_INTERVAL +// CONFIG_TYPE_MSEC_INTERVAL +// CONFIG_TYPE_MEMUNIT +// +// These types are implemented using the config_parse_units() function. +// The intervals are stored as ints, whereas memory units are stored as +// uint64_ts. +///// + +static int +units_parse_u64(void *target, const char *value, char **errmsg, + const void *params, const char *key) +{ + (void)key; + const unit_table_t *table = params; + tor_assert(table); + uint64_t *v = (uint64_t*)target; + int ok=1; + *v = config_parse_units(value, table, &ok); + if (!ok) { + *errmsg = tor_strdup("Provided value is malformed or out of bounds."); + return -1; + } + return 0; +} + +static int +units_parse_int(void *target, const char *value, char **errmsg, + const void *params, const char *key) +{ + (void)key; + const unit_table_t *table = params; + tor_assert(table); + int *v = (int*)target; + int ok=1; + uint64_t u64 = config_parse_units(value, table, &ok); + if (!ok) { + *errmsg = tor_strdup("Provided value is malformed or out of bounds."); + return -1; + } + if (u64 > INT_MAX) { + tor_asprintf(errmsg, "Provided value %s is too large", value); + return -1; + } + *v = (int) u64; + return 0; +} + +static bool +units_ok_int(const void *value, const void *params) +{ + (void)params; + int v = *(int*)value; + return v >= 0; +} + +static const var_type_fns_t memunit_fns = { + .parse = units_parse_u64, + .encode = uint64_encode, // doesn't use params + .clear = uint64_clear, // doesn't use params +}; + +static const var_type_fns_t interval_fns = { + .parse = units_parse_int, + .encode = int_encode, // doesn't use params + .clear = int_clear, // doesn't use params, + .ok = units_ok_int // can't use int_ok, since that expects int params. +}; + +///// +// CONFIG_TYPE_DOUBLE +// +// This is a nice simple double. +///// + +static int +double_parse(void *target, const char *value, char **errmsg, + const void *params, const char *key) +{ + (void)params; + (void)errmsg; + (void)key; + double *v = (double*)target; + char *endptr=NULL; + errno = 0; + *v = strtod(value, &endptr); + if (endptr == value || *endptr != '\0') { + // Either there are no converted characters, or there were some characters + // that didn't get converted. + tor_asprintf(errmsg, "Could not convert %s to a number.", escaped(value)); + return -1; + } + if (errno == ERANGE) { + // strtod will set errno to ERANGE on underflow or overflow. + bool underflow = -.00001 < *v && *v < .00001; + tor_asprintf(errmsg, + "%s is too %s to express as a floating-point number.", + escaped(value), underflow ? "small" : "large"); + return -1; + } + return 0; +} + +static char * +double_encode(const void *value, const void *params) +{ + (void)params; + double v = *(double*)value; + char *result; + tor_asprintf(&result, "%f", v); + return result; +} + +static void +double_clear(void *value, const void *params) +{ + (void)params; + double *v = (double *)value; + *v = 0.0; +} + +static const var_type_fns_t double_fns = { + .parse = double_parse, + .encode = double_encode, + .clear = double_clear, +}; + +///// +// CONFIG_TYPE_BOOL +// CONFIG_TYPE_AUTOBOOL +// +// These types are implemented as a case-insensitive string-to-integer +// mapping. +///// + +typedef struct enumeration_table_t { + const char *name; + int value; +} enumeration_table_t; + +static int +enum_parse(void *target, const char *value, char **errmsg, + const void *params, const char *key) +{ + (void)key; + const enumeration_table_t *table = params; + int *p = (int *)target; + for (; table->name; ++table) { + if (!strcasecmp(value, table->name)) { + *p = table->value; + return 0; + } + } + tor_asprintf(errmsg, "Unrecognized value %s.", value); + return -1; +} + +static char * +enum_encode(const void *value, const void *params) +{ + int v = *(const int*)value; + const enumeration_table_t *table = params; + for (; table->name; ++table) { + if (v == table->value) + return tor_strdup(table->name); + } + return NULL; // error. +} + +static void +enum_clear(void *value, const void *params) +{ + int *p = (int*)value; + const enumeration_table_t *table = params; + tor_assert(table->name); + *p = table->value; +} + +static bool +enum_ok(const void *value, const void *params) +{ + int v = *(const int*)value; + const enumeration_table_t *table = params; + for (; table->name; ++table) { + if (v == table->value) + return true; + } + return false; +} + +static const enumeration_table_t enum_table_bool[] = { + { "0", 0 }, + { "1", 1 }, + { NULL, 0 }, +}; + +static const enumeration_table_t enum_table_autobool[] = { + { "0", 0 }, + { "1", 1 }, + { "auto", -1 }, + { NULL, 0 }, +}; + +static const var_type_fns_t enum_fns = { + .parse = enum_parse, + .encode = enum_encode, + .clear = enum_clear, + .ok = enum_ok, +}; + +///// +// CONFIG_TYPE_ISOTIME +// +// This is a time_t, encoded in ISO8601 format. +///// + +static int +time_parse(void *target, const char *value, char **errmsg, + const void *params, const char *key) +{ + (void) params; + (void) key; + time_t *p = target; + if (parse_iso_time(value, p) < 0) { + tor_asprintf(errmsg, "Invalid time %s", escaped(value)); + return -1; + } + return 0; +} + +static char * +time_encode(const void *value, const void *params) +{ + (void)params; + time_t v = *(const time_t *)value; + char *result = tor_malloc(ISO_TIME_LEN+1); + format_iso_time(result, v); + return result; +} + +static void +time_clear(void *value, const void *params) +{ + (void)params; + time_t *t = value; + *t = 0; +} + +static const var_type_fns_t time_fns = { + .parse = time_parse, + .encode = time_encode, + .clear = time_clear, +}; + +///// +// CONFIG_TYPE_CSV +// +// This type is a comma-separated list of strings, stored in a smartlist_t. +// An empty list may be encoded either as an empty smartlist, or as NULL. +///// + +static int +csv_parse(void *target, const char *value, char **errmsg, + const void *params, const char *key) +{ + (void)params; + (void)errmsg; + (void)key; + smartlist_t **sl = (smartlist_t**)target; + *sl = smartlist_new(); + smartlist_split_string(*sl, value, ",", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + return 0; +} + +static char * +csv_encode(const void *value, const void *params) +{ + (void)params; + const smartlist_t *sl = *(const smartlist_t **)value; + if (! sl) + return tor_strdup(""); + + return smartlist_join_strings(*(smartlist_t**)value, ",", 0, NULL); +} + +static void +csv_clear(void *value, const void *params) +{ + (void)params; + smartlist_t **sl = (smartlist_t**)value; + if (!*sl) + return; + SMARTLIST_FOREACH(*sl, char *, cp, tor_free(cp)); + smartlist_free(*sl); // clears pointer. +} + +static const var_type_fns_t csv_fns = { + .parse = csv_parse, + .encode = csv_encode, + .clear = csv_clear, +}; + +///// +// CONFIG_TYPE_CSV_INTERVAL +// +// This type used to be a list of time intervals, used to determine a download +// schedule. Now, only the first interval counts: everything after the first +// comma is discarded. +///// + +static int +legacy_csv_interval_parse(void *target, const char *value, char **errmsg, + const void *params, const char *key) +{ + (void)params; + /* We used to have entire smartlists here. But now that all of our + * download schedules use exponential backoff, only the first part + * matters. */ + const char *comma = strchr(value, ','); + const char *val = value; + char *tmp = NULL; + if (comma) { + tmp = tor_strndup(val, comma - val); + val = tmp; + } + + int rv = units_parse_int(target, val, errmsg, &time_units, key); + tor_free(tmp); + return rv; +} + +static const var_type_fns_t legacy_csv_interval_fns = { + .parse = legacy_csv_interval_parse, + .encode = int_encode, + .clear = int_clear, +}; + +///// +// CONFIG_TYPE_LINELIST +// CONFIG_TYPE_LINELIST_S +// CONFIG_TYPE_LINELIST_V +// +// A linelist is a raw config_line_t list. Order is preserved. +// +// The LINELIST type is used for homogeneous lists, where all the lines +// have the same key. +// +// The LINELIST_S and LINELIST_V types are used for the case where multiple +// lines of different keys are kept in a single list, to preserve their +// relative order. The unified list is stored as a "virtual" variable whose +// type is LINELIST_V; the individual sublists are treated as variables of +// type LINELIST_S. +// +// A linelist may be fragile or non-fragile. Assigning a line to a fragile +// linelist replaces the list with the line. If the line has the "APPEND" +// command set on it, or if the list is non-fragile, the line is appended. +// Either way, the new list is non-fragile. +///// + +static int +linelist_kv_parse(void *target, const struct config_line_t *line, + char **errmsg, const void *params) +{ + (void)params; + (void)errmsg; + config_line_t **lines = target; + + if (*lines && (*lines)->fragile) { + if (line->command == CONFIG_LINE_APPEND) { + (*lines)->fragile = 0; + } else { + config_free_lines(*lines); // sets it to NULL + } + } + + config_line_append(lines, line->key, line->value); + return 0; +} + +static int +linelist_kv_virt_noparse(void *target, const struct config_line_t *line, + char **errmsg, const void *params) +{ + (void)target; + (void)line; + (void)params; + *errmsg = tor_strdup("Cannot assign directly to virtual option."); + return -1; +} + +static struct config_line_t * +linelist_kv_encode(const char *key, const void *value, + const void *params) +{ + (void)key; + (void)params; + config_line_t *lines = *(config_line_t **)value; + return config_lines_dup(lines); +} + +static struct config_line_t * +linelist_s_kv_encode(const char *key, const void *value, + const void *params) +{ + (void)params; + config_line_t *lines = *(config_line_t **)value; + return config_lines_dup_and_filter(lines, key); +} + +static void +linelist_clear(void *target, const void *params) +{ + (void)params; + config_line_t **lines = target; + config_free_lines(*lines); // sets it to NULL +} + +static bool +linelist_eq(const void *a, const void *b, const void *params) +{ + (void)params; + const config_line_t *lines_a = *(const config_line_t **)a; + const config_line_t *lines_b = *(const config_line_t **)b; + return config_lines_eq(lines_a, lines_b); +} + +static int +linelist_copy(void *target, const void *value, const void *params) +{ + (void)params; + config_line_t **ptr = (config_line_t **)target; + const config_line_t *val = *(const config_line_t **)value; + config_free_lines(*ptr); + *ptr = config_lines_dup(val); + return 0; +} + +static void +linelist_mark_fragile(void *target, const void *params) +{ + (void)params; + config_line_t **ptr = (config_line_t **)target; + if (*ptr) + (*ptr)->fragile = 1; +} + +static const var_type_fns_t linelist_fns = { + .kv_parse = linelist_kv_parse, + .kv_encode = linelist_kv_encode, + .clear = linelist_clear, + .eq = linelist_eq, + .copy = linelist_copy, + .mark_fragile = linelist_mark_fragile, +}; + +static const var_type_fns_t linelist_v_fns = { + .kv_parse = linelist_kv_virt_noparse, + .kv_encode = linelist_kv_encode, + .clear = linelist_clear, + .eq = linelist_eq, + .copy = linelist_copy, + .mark_fragile = linelist_mark_fragile, +}; + +static const var_type_fns_t linelist_s_fns = { + .kv_parse = linelist_kv_parse, + .kv_encode = linelist_s_kv_encode, + .clear = linelist_clear, + .eq = linelist_eq, + .copy = linelist_copy, +}; + +///// +// CONFIG_TYPE_ROUTERSET +// +// XXXX This type is not implemented here, since routerset_t is not available +// XXXX to this module. +///// + +///// +// CONFIG_TYPE_OBSOLETE +// +// Used to indicate an obsolete option. +// +// XXXX This is not a type, and should be handled at a higher level of +// XXXX abstraction. +///// + +static int +ignore_parse(void *target, const char *value, char **errmsg, + const void *params, const char *key) +{ + (void)target; + (void)value; + (void)errmsg; + (void)params; + // XXXX move this to a higher level, once such a level exists. + log_warn(LD_GENERAL, "Skipping obsolete configuration option%s%s%s", + key && *key ? " \"" : "", + key && *key ? key : "", + key && *key ? "\"." : "."); + return 0; +} + +static char * +ignore_encode(const void *value, const void *params) +{ + (void)value; + (void)params; + return NULL; +} + +static const var_type_fns_t ignore_fns = { + .parse = ignore_parse, + .encode = ignore_encode, +}; + +/** + * Table mapping conf_type_t values to var_type_def_t objects. + **/ +static const var_type_def_t type_definitions_table[] = { + [CONFIG_TYPE_STRING] = { .name="String", .fns=&string_fns }, + [CONFIG_TYPE_FILENAME] = { .name="Filename", .fns=&string_fns }, + [CONFIG_TYPE_INT] = { .name="SignedInteger", .fns=&int_fns, + .params=&INT_PARSE_UNRESTRICTED }, + [CONFIG_TYPE_POSINT] = { .name="Integer", .fns=&int_fns, + .params=&INT_PARSE_POSINT }, + [CONFIG_TYPE_UINT64] = { .name="Integer", .fns=&uint64_fns, }, + [CONFIG_TYPE_MEMUNIT] = { .name="DataSize", .fns=&memunit_fns, + .params=&memory_units }, + [CONFIG_TYPE_INTERVAL] = { .name="TimeInterval", .fns=&interval_fns, + .params=&time_units }, + [CONFIG_TYPE_MSEC_INTERVAL] = { .name="TimeMsecInterval", + .fns=&interval_fns, + .params=&time_msec_units }, + [CONFIG_TYPE_DOUBLE] = { .name="Float", .fns=&double_fns, }, + [CONFIG_TYPE_BOOL] = { .name="Boolean", .fns=&enum_fns, + .params=&enum_table_bool }, + [CONFIG_TYPE_AUTOBOOL] = { .name="Boolean+Auto", .fns=&enum_fns, + .params=&enum_table_autobool }, + [CONFIG_TYPE_ISOTIME] = { .name="Time", .fns=&time_fns, }, + [CONFIG_TYPE_CSV] = { .name="CommaList", .fns=&csv_fns, }, + [CONFIG_TYPE_CSV_INTERVAL] = { .name="TimeInterval", + .fns=&legacy_csv_interval_fns, }, + [CONFIG_TYPE_LINELIST] = { .name="LineList", .fns=&linelist_fns, + .flags=CFLG_NOREPLACE }, + /* + * A "linelist_s" is a derived view of a linelist_v: inspecting + * it gets part of a linelist_v, and setting it adds to the linelist_v. + */ + [CONFIG_TYPE_LINELIST_S] = { .name="Dependent", .fns=&linelist_s_fns, + .flags=CFLG_NOREPLACE| + /* The operations we disable here are + * handled by the linelist_v. */ + CFLG_NOCOPY|CFLG_NOCMP|CFLG_NODUMP }, + [CONFIG_TYPE_LINELIST_V] = { .name="Virtual", .fns=&linelist_v_fns, + .flags=CFLG_NOREPLACE|CFLG_NOSET }, + [CONFIG_TYPE_OBSOLETE] = { + .name="Obsolete", .fns=&ignore_fns, + .flags=CFLG_GROUP_OBSOLETE, + } +}; + +/** + * Return a pointer to the var_type_def_t object for the given + * config_type_t value, or NULL if no such type definition exists. + **/ +const var_type_def_t * +lookup_type_def(config_type_t type) +{ + int t = type; + tor_assert(t >= 0); + if (t >= (int)ARRAY_LENGTH(type_definitions_table)) + return NULL; + return &type_definitions_table[t]; +} diff --git a/src/lib/confmgt/type_defs.h b/src/lib/confmgt/type_defs.h new file mode 100644 index 0000000000..ecf040529e --- /dev/null +++ b/src/lib/confmgt/type_defs.h @@ -0,0 +1,17 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file type_defs.h + * @brief Header for lib/confmgt/type_defs.c + **/ + +#ifndef TOR_LIB_CONFMGT_TYPE_DEFS_H +#define TOR_LIB_CONFMGT_TYPE_DEFS_H + +const struct var_type_def_t *lookup_type_def(config_type_t type); + +#endif /* !defined(TOR_LIB_CONFMGT_TYPE_DEFS_H) */ diff --git a/src/lib/confmgt/typedvar.c b/src/lib/confmgt/typedvar.c new file mode 100644 index 0000000000..ce11a69379 --- /dev/null +++ b/src/lib/confmgt/typedvar.c @@ -0,0 +1,228 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file typedvar.c + * @brief Functions for accessing a pointer as an object of a given type. + * + * These functions represent a low-level API for accessing a typed variable. + * They are used in the configuration system to examine and set fields in + * configuration objects used by individual modules. + * + * Almost no code should call these directly. + **/ + +#include "orconfig.h" +#include "lib/conf/conftypes.h" +#include "lib/confmgt/type_defs.h" +#include "lib/confmgt/typedvar.h" +#include "lib/encoding/confline.h" +#include "lib/log/escape.h" +#include "lib/log/log.h" +#include "lib/log/util_bug.h" +#include "lib/malloc/malloc.h" +#include "lib/string/util_string.h" + +#include "lib/confmgt/var_type_def_st.h" + +#include <stddef.h> +#include <string.h> + +/** + * Try to parse a string in <b>value</b> that encodes an object of the type + * defined by <b>def</b>. If not NULL, <b>key</b> is the name of the option, + * which may be used for logging. + * + * On success, adjust the lvalue pointed to by <b>target</b> to hold that + * value, and return 0. On failure, set *<b>errmsg</b> to a newly allocated + * string holding an error message, and return -1. + **/ +int +typed_var_assign(void *target, const char *value, char **errmsg, + const var_type_def_t *def, const char *key) +{ + if (BUG(!def)) + return -1; // LCOV_EXCL_LINE + // clear old value if needed. + typed_var_free(target, def); + + tor_assert(def->fns->parse); + return def->fns->parse(target, value, errmsg, def->params, key); +} + +/** + * Try to parse a single line from the head of<b>line</b> that encodes an + * object of the type defined in <b>def</b>. On success and failure, behave as + * typed_var_assign(). + * + * All types for which keys are significant should use this function. + * + * Note that although multiple lines may be provided in <b>line</b>, + * only the first one is handled by this function. + **/ +int +typed_var_kvassign(void *target, const config_line_t *line, + char **errmsg, const var_type_def_t *def) +{ + if (BUG(!def)) + return -1; // LCOV_EXCL_LINE + + if (def->fns->kv_parse) { + // We do _not_ free the old value here, since linelist options + // sometimes have append semantics. + return def->fns->kv_parse(target, line, errmsg, def->params); + } + + return typed_var_assign(target, line->value, errmsg, def, line->key); +} + +/** + * Release storage held by a variable in <b>target</b> of type defined by + * <b>def</b>, and set <b>target</b> to a reasonable default. + **/ +void +typed_var_free(void *target, const var_type_def_t *def) +{ + if (BUG(!def)) + return; // LCOV_EXCL_LINE + if (def->fns->clear) { + def->fns->clear(target, def->params); + } +} + +/** + * Encode a value of type <b>def</b> pointed to by <b>value</b>, and return + * its result in a newly allocated string. The string may need to be escaped. + * + * Returns NULL if this option has a NULL value, or on internal error. + **/ +char * +typed_var_encode(const void *value, const var_type_def_t *def) +{ + if (BUG(!def)) + return NULL; // LCOV_EXCL_LINE + tor_assert(def->fns->encode); + return def->fns->encode(value, def->params); +} + +/** + * As typed_var_encode(), but returns a newly allocated config_line_t + * object. The provided <b>key</b> is used as the key of the lines, unless + * the type is one (line a linelist) that encodes its own keys. + * + * This function may return a list of multiple lines. + * + * Returns NULL if there are no lines to encode, or on internal error. + */ +config_line_t * +typed_var_kvencode(const char *key, const void *value, + const var_type_def_t *def) +{ + if (BUG(!def)) + return NULL; // LCOV_EXCL_LINE + if (def->fns->kv_encode) { + return def->fns->kv_encode(key, value, def->params); + } + char *encoded_value = typed_var_encode(value, def); + if (!encoded_value) + return NULL; + + config_line_t *result = tor_malloc_zero(sizeof(config_line_t)); + result->key = tor_strdup(key); + result->value = encoded_value; + return result; +} + +/** + * Set <b>dest</b> to contain the same value as <b>src</b>. Both types + * must be as defined by <b>def</b>. + * + * Return 0 on success, and -1 on failure. + **/ +int +typed_var_copy(void *dest, const void *src, const var_type_def_t *def) +{ + if (BUG(!def)) + return -1; // LCOV_EXCL_LINE + if (def->fns->copy) { + // If we have been provided a copy fuction, use it. + return def->fns->copy(dest, src, def); + } + + // Otherwise, encode 'src' and parse the result into 'def'. + char *enc = typed_var_encode(src, def); + if (!enc) { + typed_var_free(dest, def); + return 0; + } + char *err = NULL; + int rv = typed_var_assign(dest, enc, &err, def, NULL); + if (BUG(rv < 0)) { + // LCOV_EXCL_START + log_warn(LD_BUG, "Encoded value %s was not parseable as a %s: %s", + escaped(enc), def->name, err?err:""); + // LCOV_EXCL_STOP + } + tor_free(err); + tor_free(enc); + return rv; +} + +/** + * Return true if <b>a</b> and <b>b</b> are semantically equivalent. + * Both types must be as defined by <b>def</b>. + **/ +bool +typed_var_eq(const void *a, const void *b, const var_type_def_t *def) +{ + if (BUG(!def)) + return false; // LCOV_EXCL_LINE + + if (def->fns->eq) { + // Use a provided eq function if we got one. + return def->fns->eq(a, b, def->params); + } + + // Otherwise, encode the values and compare them. + char *enc_a = typed_var_encode(a, def); + char *enc_b = typed_var_encode(b, def); + bool eq = !strcmp_opt(enc_a,enc_b); + tor_free(enc_a); + tor_free(enc_b); + return eq; +} + +/** + * Check whether <b>value</b> encodes a valid value according to the + * type definition in <b>def</b>. + */ +bool +typed_var_ok(const void *value, const var_type_def_t *def) +{ + if (BUG(!def)) + return false; // LCOV_EXCL_LINE + + if (def->fns->ok) + return def->fns->ok(value, def->params); + + return true; +} + +/** + * Mark <b>value</b> -- a variable that ordinarily would be extended by + * assignment -- as "fragile", so that it will get replaced by the next + * assignment instead. + **/ +void +typed_var_mark_fragile(void *value, const var_type_def_t *def) +{ + if (BUG(!def)) { + return; // LCOV_EXCL_LINE + } + + if (def->fns->mark_fragile) + def->fns->mark_fragile(value, def->params); +} diff --git a/src/lib/confmgt/typedvar.h b/src/lib/confmgt/typedvar.h new file mode 100644 index 0000000000..4382613833 --- /dev/null +++ b/src/lib/confmgt/typedvar.h @@ -0,0 +1,38 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file typedvar.h + * @brief Header for lib/confmgt/typedvar.c + **/ + +#ifndef TOR_LIB_CONFMGT_TYPEDVAR_H +#define TOR_LIB_CONFMGT_TYPEDVAR_H + +#include <stdbool.h> + +enum config_type_t; +struct config_line_t; + +typedef struct var_type_fns_t var_type_fns_t; +typedef struct var_type_def_t var_type_def_t; + +int typed_var_assign(void *target, const char *value, char **errmsg, + const var_type_def_t *def, const char *key); +void typed_var_free(void *target, const var_type_def_t *def); +char *typed_var_encode(const void *value, const var_type_def_t *def); +int typed_var_copy(void *dest, const void *src, const var_type_def_t *def); +bool typed_var_eq(const void *a, const void *b, const var_type_def_t *def); +bool typed_var_ok(const void *value, const var_type_def_t *def); + +int typed_var_kvassign(void *target, const struct config_line_t *line, + char **errmsg, const var_type_def_t *def); +struct config_line_t *typed_var_kvencode(const char *key, const void *value, + const var_type_def_t *def); + +void typed_var_mark_fragile(void *value, const var_type_def_t *def); + +#endif /* !defined(TOR_LIB_CONFMGT_TYPEDVAR_H) */ diff --git a/src/lib/confmgt/unitparse.c b/src/lib/confmgt/unitparse.c new file mode 100644 index 0000000000..c3ed8285a4 --- /dev/null +++ b/src/lib/confmgt/unitparse.c @@ -0,0 +1,206 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file unitparse.c + * @brief Functions for parsing values with units from a configuration file. + **/ + +#include "orconfig.h" +#include "lib/confmgt/unitparse.h" +#include "lib/log/log.h" +#include "lib/log/util_bug.h" +#include "lib/string/parse_int.h" +#include "lib/string/util_string.h" + +#include <string.h> + +/** Table to map the names of memory units to the number of bytes they + * contain. */ +const struct unit_table_t memory_units[] = { + { "", 1 }, + { "b", 1<< 0 }, + { "byte", 1<< 0 }, + { "bytes", 1<< 0 }, + { "kb", 1<<10 }, + { "kbyte", 1<<10 }, + { "kbytes", 1<<10 }, + { "kilobyte", 1<<10 }, + { "kilobytes", 1<<10 }, + { "kilobits", 1<<7 }, + { "kilobit", 1<<7 }, + { "kbits", 1<<7 }, + { "kbit", 1<<7 }, + { "m", 1<<20 }, + { "mb", 1<<20 }, + { "mbyte", 1<<20 }, + { "mbytes", 1<<20 }, + { "megabyte", 1<<20 }, + { "megabytes", 1<<20 }, + { "megabits", 1<<17 }, + { "megabit", 1<<17 }, + { "mbits", 1<<17 }, + { "mbit", 1<<17 }, + { "gb", 1<<30 }, + { "gbyte", 1<<30 }, + { "gbytes", 1<<30 }, + { "gigabyte", 1<<30 }, + { "gigabytes", 1<<30 }, + { "gigabits", 1<<27 }, + { "gigabit", 1<<27 }, + { "gbits", 1<<27 }, + { "gbit", 1<<27 }, + { "tb", UINT64_C(1)<<40 }, + { "tbyte", UINT64_C(1)<<40 }, + { "tbytes", UINT64_C(1)<<40 }, + { "terabyte", UINT64_C(1)<<40 }, + { "terabytes", UINT64_C(1)<<40 }, + { "terabits", UINT64_C(1)<<37 }, + { "terabit", UINT64_C(1)<<37 }, + { "tbits", UINT64_C(1)<<37 }, + { "tbit", UINT64_C(1)<<37 }, + { NULL, 0 }, +}; + +/** Table to map the names of time units to the number of seconds they + * contain. */ +const struct unit_table_t time_units[] = { + { "", 1 }, + { "second", 1 }, + { "seconds", 1 }, + { "minute", 60 }, + { "minutes", 60 }, + { "hour", 60*60 }, + { "hours", 60*60 }, + { "day", 24*60*60 }, + { "days", 24*60*60 }, + { "week", 7*24*60*60 }, + { "weeks", 7*24*60*60 }, + { "month", 2629728, }, /* about 30.437 days */ + { "months", 2629728, }, + { NULL, 0 }, +}; + +/** Table to map the names of time units to the number of milliseconds + * they contain. */ +const struct unit_table_t time_msec_units[] = { + { "", 1 }, + { "msec", 1 }, + { "millisecond", 1 }, + { "milliseconds", 1 }, + { "second", 1000 }, + { "seconds", 1000 }, + { "minute", 60*1000 }, + { "minutes", 60*1000 }, + { "hour", 60*60*1000 }, + { "hours", 60*60*1000 }, + { "day", 24*60*60*1000 }, + { "days", 24*60*60*1000 }, + { "week", 7*24*60*60*1000 }, + { "weeks", 7*24*60*60*1000 }, + { NULL, 0 }, +}; + +/** Parse a string <b>val</b> containing a number, zero or more + * spaces, and an optional unit string. If the unit appears in the + * table <b>u</b>, then multiply the number by the unit multiplier. + * On success, set *<b>ok</b> to 1 and return this product. + * Otherwise, set *<b>ok</b> to 0. + */ +uint64_t +config_parse_units(const char *val, const unit_table_t *u, int *ok) +{ + uint64_t v = 0; + double d = 0; + int use_float = 0; + char *cp; + + tor_assert(ok); + + v = tor_parse_uint64(val, 10, 0, UINT64_MAX, ok, &cp); + if (!*ok || (cp && *cp == '.')) { + d = tor_parse_double(val, 0, (double)UINT64_MAX, ok, &cp); + if (!*ok) + goto done; + use_float = 1; + } + + if (BUG(!cp)) { + // cp should always be non-NULL if the parse operation succeeds. + + // LCOV_EXCL_START + *ok = 1; + v = use_float ? ((uint64_t)d) : v; + goto done; + // LCOV_EXCL_STOP + } + + cp = (char*) eat_whitespace(cp); + + for ( ;u->unit;++u) { + if (!strcasecmp(u->unit, cp)) { + if (use_float) + v = (uint64_t)(u->multiplier * d); + else + v *= u->multiplier; + *ok = 1; + goto done; + } + } + log_warn(LD_CONFIG, "Unknown unit '%s'.", cp); + *ok = 0; + done: + + if (*ok) + return v; + else + return 0; +} + +/** Parse a string in the format "number unit", where unit is a unit of + * information (byte, KB, M, etc). On success, set *<b>ok</b> to true + * and return the number of bytes specified. Otherwise, set + * *<b>ok</b> to false and return 0. */ +uint64_t +config_parse_memunit(const char *s, int *ok) +{ + uint64_t u = config_parse_units(s, memory_units, ok); + return u; +} + +/** Parse a string in the format "number unit", where unit is a unit of + * time in milliseconds. On success, set *<b>ok</b> to true and return + * the number of milliseconds in the provided interval. Otherwise, set + * *<b>ok</b> to 0 and return -1. */ +int +config_parse_msec_interval(const char *s, int *ok) +{ + uint64_t r; + r = config_parse_units(s, time_msec_units, ok); + if (r > INT_MAX) { + log_warn(LD_CONFIG, "Msec interval '%s' is too long", s); + *ok = 0; + return -1; + } + return (int)r; +} + +/** Parse a string in the format "number unit", where unit is a unit of time. + * On success, set *<b>ok</b> to true and return the number of seconds in + * the provided interval. Otherwise, set *<b>ok</b> to 0 and return -1. + */ +int +config_parse_interval(const char *s, int *ok) +{ + uint64_t r; + r = config_parse_units(s, time_units, ok); + if (r > INT_MAX) { + log_warn(LD_CONFIG, "Interval '%s' is too long", s); + *ok = 0; + return -1; + } + return (int)r; +} diff --git a/src/lib/confmgt/unitparse.h b/src/lib/confmgt/unitparse.h new file mode 100644 index 0000000000..216361a7d4 --- /dev/null +++ b/src/lib/confmgt/unitparse.h @@ -0,0 +1,34 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file unitparse.h + * @brief Header for lib/confmgt/unitparse.c + **/ + +#ifndef TOR_LIB_CONFMGT_UNITPARSE_H +#define TOR_LIB_CONFMGT_UNITPARSE_H + +#include <lib/cc/torint.h> + +/** Mapping from a unit name to a multiplier for converting that unit into a + * base unit. Used by config_parse_unit. */ +typedef struct unit_table_t { + const char *unit; /**< The name of the unit */ + uint64_t multiplier; /**< How many of the base unit appear in this unit */ +} unit_table_t; + +extern const unit_table_t memory_units[]; +extern const unit_table_t time_units[]; +extern const struct unit_table_t time_msec_units[]; + +uint64_t config_parse_units(const char *val, const unit_table_t *u, int *ok); + +uint64_t config_parse_memunit(const char *s, int *ok); +int config_parse_msec_interval(const char *s, int *ok); +int config_parse_interval(const char *s, int *ok); + +#endif /* !defined(TOR_LIB_CONFMGT_UNITPARSE_H) */ diff --git a/src/lib/confmgt/var_type_def_st.h b/src/lib/confmgt/var_type_def_st.h new file mode 100644 index 0000000000..aa9ded39e9 --- /dev/null +++ b/src/lib/confmgt/var_type_def_st.h @@ -0,0 +1,170 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file var_type_def_st.h + * @brief Structure declarations for typedvar type definitions. + * + * This structure is used for defining new variable types. If you are not + * defining a new variable type for use by the configuration management + * system, you don't need this structure. + * + * For defining new variables, see the types in conftypes.h. + * + * For data-driven access to configuration variables, see the other members of + * lib/confmgt/. + * + * STATUS NOTE: It is not yet possible to actually define new variables + * outside of config.c, and many of the types that will eventually be used + * to do so are not yet moved. This will change as more of #29211 is + * completed. + **/ + +#ifndef TOR_LIB_CONFMGT_VAR_TYPE_DEF_ST_H +#define TOR_LIB_CONFMGT_VAR_TYPE_DEF_ST_H + +#include <stdbool.h> + +struct config_line_t; + +/** + * A structure full of functions pointers to implement a variable type. + * + * Every type MUST implement parse or kv_parse and encode or kv_encode; + * the other functions pointers MAY be NULL. + * + * All functions here take a <b>params</b> argument, whose value + * is determined by the type definition. Two types may have the + * same functions, but differ only in parameters. + * + * Implementation considerations: If "" encodes a valid value for a type, try + * to make sure that it encodes the same thing as the default value for the + * type (that is, the value that is set by config_clear() or memset(0)). If + * this is not the case, you need to make extra certain that your parse/encode + * implementations preserve the NULL/"" distinction. + **/ +struct var_type_fns_t { + /** + * Try to parse a string in <b>value</b> that encodes an object of this + * type. On success, adjust the lvalue pointed to by <b>target</b> to hold + * that value, and return 0. On failure, set *<b>errmsg</b> to a newly + * allocated string holding an error message, and return -1. + * + * If not NULL, <b>key</b> is the name of the option, which may be used for + * logging. + **/ + int (*parse)(void *target, const char *value, char **errmsg, + const void *params, const char *key); + /** + * Try to parse a single line from the head of<b>line</b> that encodes + * an object of this type. On success and failure, behave as in the parse() + * function. + * + * If this function is absent, it is implemented in terms of parse(). + * + * All types for which keys are significant should use this method. For + * example, a "linelist" type records the actual keys that are given + * for each line, and so should use this method. + * + * Note that although multiple lines may be provided in <b>line</b>, + * only the first one should be handled by this function. + **/ + int (*kv_parse)(void *target, const struct config_line_t *line, + char **errmsg, const void *params); + /** + * Encode a value pointed to by <b>value</b> and return its result + * in a newly allocated string. The string may need to be escaped. + * + * If this function is absent, it is implemented in terms of kv_encode(). + * + * Returns NULL if this option has a NULL value, or on internal error. + * + * Requirement: all strings generated by encode() should produce a + * semantically equivalent value when given to parse(). + **/ + char *(*encode)(const void *value, const void *params); + /** + * As encode(), but returns a newly allocated config_line_t object. The + * provided <b>key</b> is used as the key of the lines, unless the type is + * one that encodes its own keys. + * + * Unlike kv_parse(), this function will return a list of multiple lines, + * if <b>value</b> is such that it must be encoded by multiple lines. + * + * Returns NULL if there are no lines to encode, or on internal error. + * + * If this function is absent, it is implemented in terms of encode(). + **/ + struct config_line_t *(*kv_encode)(const char *key, const void *value, + const void *params); + /** + * Free all storage held in <b>arg</b>, and set <b>arg</b> to a default + * value -- usually zero or NULL. + * + * If this function is absent, the default implementation does nothing. + **/ + void (*clear)(void *arg, const void *params); + /** + * Return true if <b>a</b> and <b>b</b> hold the same value, and false + * otherwise. + * + * If this function is absent, it is implemented by encoding both a and + * b and comparing their encoded strings for equality. + **/ + bool (*eq)(const void *a, const void *b, const void *params); + /** + * Try to copy the value from <b>value</b> into <b>target</b>. + * On success return 0; on failure return -1. + * + * If this function is absent, it is implemented by encoding the value + * into a string, and then parsing it into the target. + **/ + int (*copy)(void *target, const void *value, const void *params); + /** + * Check whether <b>value</b> holds a valid value according to the + * rules of this type; return true if it does and false if it doesn't. + * + * The default implementation for this function assumes that all + * values are valid. + **/ + bool (*ok)(const void *value, const void *params); + /** + * Mark a value of this variable as "fragile", so that future attempts to + * assign to this variable will replace rather than extending it. + * + * The default implementation for this function does nothing. + * + * Only meaningful for types with is_cumulative set. + **/ + void (*mark_fragile)(void *value, const void *params); +}; + +/** + * A structure describing a type that can be manipulated with the typedvar_* + * functions. + **/ +struct var_type_def_t { + /** + * The name of this type. Should not include spaces. Used for + * debugging, log messages, and the controller API. */ + const char *name; + /** + * A function table for this type. + */ + const struct var_type_fns_t *fns; + /** + * A pointer to a value that should be passed as the 'params' argument when + * calling the functions in this type's function table. + */ + const void *params; + /** + * A bitwise OR of one or more VTFLAG_* values, describing properties + * for all values of this type. + **/ + uint32_t flags; +}; + +#endif /* !defined(TOR_LIB_CONFMGT_VAR_TYPE_DEF_ST_H) */ diff --git a/src/lib/container/smartlist.c b/src/lib/container/smartlist.c index 3ab2797d68..2b71c11287 100644 --- a/src/lib/container/smartlist.c +++ b/src/lib/container/smartlist.c @@ -678,7 +678,7 @@ smartlist_sort_pointers(smartlist_t *sl) static inline void smartlist_heapify(smartlist_t *sl, int (*compare)(const void *a, const void *b), - int idx_field_offset, + ptrdiff_t idx_field_offset, int idx) { while (1) { @@ -725,7 +725,7 @@ smartlist_heapify(smartlist_t *sl, void smartlist_pqueue_add(smartlist_t *sl, int (*compare)(const void *a, const void *b), - int idx_field_offset, + ptrdiff_t idx_field_offset, void *item) { int idx; @@ -754,7 +754,7 @@ smartlist_pqueue_add(smartlist_t *sl, void * smartlist_pqueue_pop(smartlist_t *sl, int (*compare)(const void *a, const void *b), - int idx_field_offset) + ptrdiff_t idx_field_offset) { void *top; tor_assert(sl->num_used); @@ -778,7 +778,7 @@ smartlist_pqueue_pop(smartlist_t *sl, void smartlist_pqueue_remove(smartlist_t *sl, int (*compare)(const void *a, const void *b), - int idx_field_offset, + ptrdiff_t idx_field_offset, void *item) { int idx = IDX_OF_ITEM(item); @@ -802,7 +802,7 @@ smartlist_pqueue_remove(smartlist_t *sl, void smartlist_pqueue_assert_ok(smartlist_t *sl, int (*compare)(const void *a, const void *b), - int idx_field_offset) + ptrdiff_t idx_field_offset) { int i; for (i = sl->num_used - 1; i >= 0; --i) { diff --git a/src/lib/container/smartlist.h b/src/lib/container/smartlist.h index 81b0151433..25638e4b22 100644 --- a/src/lib/container/smartlist.h +++ b/src/lib/container/smartlist.h @@ -13,6 +13,7 @@ **/ #include <stdarg.h> +#include <stddef.h> #include "lib/smartlist_core/smartlist_core.h" #include "lib/smartlist_core/smartlist_foreach.h" @@ -72,18 +73,18 @@ int smartlist_bsearch_idx(const smartlist_t *sl, const void *key, void smartlist_pqueue_add(smartlist_t *sl, int (*compare)(const void *a, const void *b), - int idx_field_offset, + ptrdiff_t idx_field_offset, void *item); void *smartlist_pqueue_pop(smartlist_t *sl, int (*compare)(const void *a, const void *b), - int idx_field_offset); + ptrdiff_t idx_field_offset); void smartlist_pqueue_remove(smartlist_t *sl, int (*compare)(const void *a, const void *b), - int idx_field_offset, + ptrdiff_t idx_field_offset, void *item); void smartlist_pqueue_assert_ok(smartlist_t *sl, int (*compare)(const void *a, const void *b), - int idx_field_offset); + ptrdiff_t idx_field_offset); char *smartlist_join_strings(smartlist_t *sl, const char *join, int terminate, size_t *len_out) ATTR_MALLOC; diff --git a/src/lib/crypt_ops/aes_openssl.c b/src/lib/crypt_ops/aes_openssl.c index 2f985d4512..64564892ad 100644 --- a/src/lib/crypt_ops/aes_openssl.c +++ b/src/lib/crypt_ops/aes_openssl.c @@ -148,7 +148,7 @@ evaluate_ctr_for_aes(void) { return 0; } -#else /* !(defined(USE_EVP_AES_CTR)) */ +#else /* !defined(USE_EVP_AES_CTR) */ /*======================================================================*/ /* Interface to AES code, and counter implementation */ diff --git a/src/lib/crypt_ops/compat_openssl.h b/src/lib/crypt_ops/compat_openssl.h index 9c10386c34..61ca51315f 100644 --- a/src/lib/crypt_ops/compat_openssl.h +++ b/src/lib/crypt_ops/compat_openssl.h @@ -45,7 +45,7 @@ ((st) == SSL3_ST_SW_SRVR_HELLO_B)) #define OSSL_HANDSHAKE_STATE int #define CONST_IF_OPENSSL_1_1_API -#else /* !(!defined(OPENSSL_1_1_API)) */ +#else /* defined(OPENSSL_1_1_API) */ #define STATE_IS_SW_SERVER_HELLO(st) \ ((st) == TLS_ST_SW_SRVR_HELLO) #define CONST_IF_OPENSSL_1_1_API const diff --git a/src/lib/crypt_ops/crypto_dh_openssl.c b/src/lib/crypt_ops/crypto_dh_openssl.c index 75cee1b596..8ae97373e8 100644 --- a/src/lib/crypt_ops/crypto_dh_openssl.c +++ b/src/lib/crypt_ops/crypto_dh_openssl.c @@ -68,7 +68,7 @@ crypto_validate_dh_params(const BIGNUM *p, const BIGNUM *g) goto out; if (!DH_set0_pqg(dh, dh_p, NULL, dh_g)) goto out; -#else /* !(defined(OPENSSL_1_1_API)) */ +#else /* !defined(OPENSSL_1_1_API) */ if (!(dh->p = BN_dup(p))) goto out; if (!(dh->g = BN_dup(g))) @@ -231,7 +231,7 @@ new_openssl_dh_from_params(BIGNUM *p, BIGNUM *g) if (!DH_set_length(res_dh, DH_PRIVATE_KEY_BITS)) goto err; -#else /* !(defined(OPENSSL_1_1_API)) */ +#else /* !defined(OPENSSL_1_1_API) */ res_dh->p = dh_p; res_dh->g = dh_g; res_dh->length = DH_PRIVATE_KEY_BITS; @@ -298,7 +298,7 @@ crypto_dh_generate_public(crypto_dh_t *dh) "the-universe chances really do happen. Treating as a failure."); return -1; } -#else /* !(defined(OPENSSL_1_1_API)) */ +#else /* !defined(OPENSSL_1_1_API) */ if (tor_check_dh_key(LOG_WARN, dh->dh->pub_key)<0) { /* LCOV_EXCL_START * If this happens, then openssl's DH implementation is busted. */ diff --git a/src/lib/crypt_ops/crypto_digest.c b/src/lib/crypt_ops/crypto_digest.c index 64a7d2d52c..ba226f8756 100644 --- a/src/lib/crypt_ops/crypto_digest.c +++ b/src/lib/crypt_ops/crypto_digest.c @@ -149,7 +149,7 @@ struct crypto_xof_t { * outside the tests yet. */ EVP_MD_CTX *ctx; -#else /* !(defined(OPENSSL_HAS_SHAKE3_EVP)) */ +#else /* !defined(OPENSSL_HAS_SHAKE3_EVP) */ keccak_state s; #endif /* defined(OPENSSL_HAS_SHAKE3_EVP) */ }; @@ -169,7 +169,7 @@ crypto_xof_new(void) tor_assert(xof->ctx); int r = EVP_DigestInit(xof->ctx, EVP_shake256()); tor_assert(r == 1); -#else /* !(defined(OPENSSL_HAS_SHAKE256)) */ +#else /* !defined(OPENSSL_HAS_SHAKE256) */ keccak_xof_init(&xof->s, 256); #endif /* defined(OPENSSL_HAS_SHAKE256) */ return xof; @@ -236,7 +236,7 @@ crypto_xof(uint8_t *output, size_t output_len, r = EVP_DigestFinalXOF(ctx, output, output_len); tor_assert(r == 1); EVP_MD_CTX_free(ctx); -#else /* !(defined(OPENSSL_HAS_SHA3)) */ +#else /* !defined(OPENSSL_HAS_SHA3) */ crypto_xof_t *xof = crypto_xof_new(); crypto_xof_add_bytes(xof, input, input_len); crypto_xof_squeeze_bytes(xof, output, output_len); diff --git a/src/lib/crypt_ops/crypto_digest_openssl.c b/src/lib/crypt_ops/crypto_digest_openssl.c index c631b0eac0..b0d8b6aee9 100644 --- a/src/lib/crypt_ops/crypto_digest_openssl.c +++ b/src/lib/crypt_ops/crypto_digest_openssl.c @@ -212,7 +212,7 @@ crypto_digest_new_internal(digest_algorithm_t algorithm) return NULL; } break; -#else /* !(defined(OPENSSL_HAS_SHA3)) */ +#else /* !defined(OPENSSL_HAS_SHA3) */ case DIGEST_SHA3_256: keccak_digest_init(&r->d.sha3, 256); break; @@ -310,7 +310,7 @@ crypto_digest_add_bytes(crypto_digest_t *digest, const char *data, tor_assert(r); } break; -#else /* !(defined(OPENSSL_HAS_SHA3)) */ +#else /* !defined(OPENSSL_HAS_SHA3) */ case DIGEST_SHA3_256: /* FALLSTHROUGH */ case DIGEST_SHA3_512: keccak_digest_update(&digest->d.sha3, (const uint8_t *)data, len); @@ -354,7 +354,7 @@ crypto_digest_get_digest(crypto_digest_t *digest, EVP_MD_CTX_free(tmp); tor_assert(res == 1); goto done; -#else /* !(defined(OPENSSL_HAS_SHA3)) */ +#else /* !defined(OPENSSL_HAS_SHA3) */ /* Tiny-Keccak handles copying into a temporary ctx, and also can handle * short output buffers by truncating appropriately. */ keccak_digest_sum(&digest->d.sha3, (uint8_t *)out, out_len); diff --git a/src/lib/crypt_ops/crypto_hkdf.c b/src/lib/crypt_ops/crypto_hkdf.c index e0f3d65ad1..e0d241d4ea 100644 --- a/src/lib/crypt_ops/crypto_hkdf.c +++ b/src/lib/crypt_ops/crypto_hkdf.c @@ -109,7 +109,7 @@ crypto_expand_key_material_rfc5869_sha256_openssl( return 0; } -#else /* !(defined(HAVE_OPENSSL_HKDF)) */ +#else /* !defined(HAVE_OPENSSL_HKDF) */ /** * Perform RFC5869 HKDF computation using our own legacy implementation. @@ -191,7 +191,7 @@ crypto_expand_key_material_rfc5869_sha256( salt_in_len, info_in, info_in_len, key_out, key_out_len); -#else /* !(defined(HAVE_OPENSSL_HKDF)) */ +#else /* !defined(HAVE_OPENSSL_HKDF) */ return crypto_expand_key_material_rfc5869_sha256_legacy(key_in, key_in_len, salt_in, salt_in_len, info_in, diff --git a/src/lib/crypt_ops/crypto_ope.c b/src/lib/crypt_ops/crypto_ope.c index 4bd4b35706..ed832d852e 100644 --- a/src/lib/crypt_ops/crypto_ope.c +++ b/src/lib/crypt_ops/crypto_ope.c @@ -57,7 +57,7 @@ ope_val_from_le(ope_val_t x) ((x) >> 8) | (((x)&0xff) << 8); } -#else /* !(defined(WORDS_BIGENDIAN)) */ +#else /* !defined(WORDS_BIGENDIAN) */ #define ope_val_from_le(x) (x) #endif /* defined(WORDS_BIGENDIAN) */ diff --git a/src/lib/crypt_ops/crypto_openssl_mgt.c b/src/lib/crypt_ops/crypto_openssl_mgt.c index 6d8364ebf8..f51309219a 100644 --- a/src/lib/crypt_ops/crypto_openssl_mgt.c +++ b/src/lib/crypt_ops/crypto_openssl_mgt.c @@ -204,7 +204,7 @@ crypto_openssl_early_init(void) OPENSSL_INIT_LOAD_CRYPTO_STRINGS | OPENSSL_INIT_ADD_ALL_CIPHERS | OPENSSL_INIT_ADD_ALL_DIGESTS, NULL); -#else /* !(defined(OPENSSL_1_1_API)) */ +#else /* !defined(OPENSSL_1_1_API) */ ERR_load_crypto_strings(); OpenSSL_add_all_algorithms(); #endif /* defined(OPENSSL_1_1_API) */ diff --git a/src/lib/crypt_ops/crypto_rand.c b/src/lib/crypt_ops/crypto_rand.c index a80a98f267..afbafbfa35 100644 --- a/src/lib/crypt_ops/crypto_rand.c +++ b/src/lib/crypt_ops/crypto_rand.c @@ -248,7 +248,7 @@ crypto_strongest_rand_fallback(uint8_t *out, size_t out_len) (void)out; (void)out_len; return -1; -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ static const char *filenames[] = { "/dev/srandom", "/dev/urandom", "/dev/random", NULL }; @@ -520,7 +520,7 @@ crypto_rand_unmocked(char *to, size_t n) #undef BUFLEN } -#else /* !(defined(ENABLE_NSS)) */ +#else /* !defined(ENABLE_NSS) */ int r = RAND_bytes((unsigned char*)to, (int)n); /* We consider a PRNG failure non-survivable. Let's assert so that we get a * stack trace about where it happened. diff --git a/src/lib/crypt_ops/crypto_rsa_openssl.c b/src/lib/crypt_ops/crypto_rsa_openssl.c index fbdc76ccd6..05d7c26b25 100644 --- a/src/lib/crypt_ops/crypto_rsa_openssl.c +++ b/src/lib/crypt_ops/crypto_rsa_openssl.c @@ -53,7 +53,7 @@ crypto_pk_key_is_private(const crypto_pk_t *k) const BIGNUM *p, *q; RSA_get0_factors(k->key, &p, &q); return p != NULL; /* XXX/yawning: Should we check q? */ -#else /* !(defined(OPENSSL_1_1_API)) */ +#else /* !defined(OPENSSL_1_1_API) */ return k && k->key && k->key->p; #endif /* defined(OPENSSL_1_1_API) */ } @@ -287,7 +287,7 @@ crypto_pk_num_bits(crypto_pk_t *env) tor_assert(n != NULL); return RSA_bits(env->key); -#else /* !(defined(OPENSSL_1_1_API)) */ +#else /* !defined(OPENSSL_1_1_API) */ tor_assert(env->key->n); return BN_num_bits(env->key->n); #endif /* defined(OPENSSL_1_1_API) */ diff --git a/src/lib/crypt_ops/crypto_s2k.c b/src/lib/crypt_ops/crypto_s2k.c index 5cf98e3e64..361db18927 100644 --- a/src/lib/crypt_ops/crypto_s2k.c +++ b/src/lib/crypt_ops/crypto_s2k.c @@ -285,7 +285,7 @@ secret_to_key_compute_key(uint8_t *key_out, size_t key_out_len, if (rv < 0) return S2K_FAILED; return (int)key_out_len; -#else /* !(defined(ENABLE_OPENSSL)) */ +#else /* !defined(ENABLE_OPENSSL) */ SECItem passItem = { .type = siBuffer, .data = (unsigned char *) secret, .len = (int)secret_len }; @@ -348,7 +348,7 @@ secret_to_key_compute_key(uint8_t *key_out, size_t key_out_len, if (rv != 0) return S2K_FAILED; return (int)key_out_len; -#else /* !(defined(HAVE_SCRYPT)) */ +#else /* !defined(HAVE_SCRYPT) */ return S2K_NO_SCRYPT_SUPPORT; #endif /* defined(HAVE_SCRYPT) */ } diff --git a/src/lib/defs/logging_types.h b/src/lib/defs/logging_types.h index 57db818007..d3eacde464 100644 --- a/src/lib/defs/logging_types.h +++ b/src/lib/defs/logging_types.h @@ -20,4 +20,4 @@ /** Mask of zero or more log domains, OR'd together. */ typedef uint64_t log_domain_mask_t; -#endif +#endif /* !defined(TOR_LOGGING_TYPES_H) */ diff --git a/src/lib/encoding/confline.c b/src/lib/encoding/confline.c index fdb575e03f..0d8384db13 100644 --- a/src/lib/encoding/confline.c +++ b/src/lib/encoding/confline.c @@ -256,7 +256,7 @@ config_lines_dup_and_filter(const config_line_t *inp, /** Return true iff a and b contain identical keys and values in identical * order. */ int -config_lines_eq(config_line_t *a, config_line_t *b) +config_lines_eq(const config_line_t *a, const config_line_t *b) { while (a && b) { if (strcasecmp(a->key, b->key) || strcmp(a->value, b->value)) diff --git a/src/lib/encoding/confline.h b/src/lib/encoding/confline.h index 56ea36bf61..12c554c6e7 100644 --- a/src/lib/encoding/confline.h +++ b/src/lib/encoding/confline.h @@ -50,7 +50,7 @@ const config_line_t *config_line_find(const config_line_t *lines, const char *key); const config_line_t *config_line_find_case(const config_line_t *lines, const char *key); -int config_lines_eq(config_line_t *a, config_line_t *b); +int config_lines_eq(const config_line_t *a, const config_line_t *b); int config_count_key(const config_line_t *a, const char *key); void config_free_lines_(config_line_t *front); #define config_free_lines(front) \ diff --git a/src/lib/err/backtrace.c b/src/lib/err/backtrace.c index 8bc7e6965c..ce8ddcd7c0 100644 --- a/src/lib/err/backtrace.c +++ b/src/lib/err/backtrace.c @@ -52,9 +52,10 @@ #include <pthread.h> #endif +#include "lib/cc/ctassert.h" + #define EXPOSE_CLEAN_BACKTRACE #include "lib/err/backtrace.h" -#include "lib/err/torerr.h" #if defined(HAVE_EXECINFO_H) && defined(HAVE_BACKTRACE) && \ defined(HAVE_BACKTRACE_SYMBOLS_FD) && defined(HAVE_SIGACTION) && \ @@ -73,15 +74,40 @@ static char bt_version[128] = ""; #ifdef USE_BACKTRACE + /** Largest stack depth to try to dump. */ #define MAX_DEPTH 256 -/** Static allocation of stack to dump. This is static so we avoid stack - * pressure. */ -static void *cb_buf[MAX_DEPTH]; +/** The size of the callback buffer, so we can clear it in unlock_cb_buf(). */ +#define SIZEOF_CB_BUF (MAX_DEPTH * sizeof(void *)) /** Protects cb_buf from concurrent access. Pthreads, since this code * is Unix-only, and since this code needs to be lowest-level. */ static pthread_mutex_t cb_buf_mutex = PTHREAD_MUTEX_INITIALIZER; +/** Lock and return a static stack pointer buffer that can hold up to + * MAX_DEPTH function pointers. */ +static void ** +lock_cb_buf(void) +{ + /* Lock the mutex first, before even declaring the buffer. */ + pthread_mutex_lock(&cb_buf_mutex); + + /** Static allocation of stack to dump. This is static so we avoid stack + * pressure. */ + static void *cb_buf[MAX_DEPTH]; + CTASSERT(SIZEOF_CB_BUF == sizeof(cb_buf)); + memset(cb_buf, 0, SIZEOF_CB_BUF); + + return cb_buf; +} + +/** Unlock the static stack pointer buffer. */ +static void +unlock_cb_buf(void **cb_buf) +{ + memset(cb_buf, 0, SIZEOF_CB_BUF); + pthread_mutex_unlock(&cb_buf_mutex); +} + /** Change a stacktrace in <b>stack</b> of depth <b>depth</b> so that it will * log the correct function from which a signal was received with context * <b>ctx</b>. (When we get a signal, the current function will not have @@ -105,7 +131,7 @@ clean_backtrace(void **stack, size_t depth, const ucontext_t *ctx) return; stack[n] = (void*) ctx->PC_FROM_UCONTEXT; -#else /* !(defined(PC_FROM_UCONTEXT)) */ +#else /* !defined(PC_FROM_UCONTEXT) */ (void) depth; (void) ctx; (void) stack; @@ -123,7 +149,7 @@ log_backtrace_impl(int severity, log_domain_mask_t domain, const char *msg, char **symbols; size_t i; - pthread_mutex_lock(&cb_buf_mutex); + void **cb_buf = lock_cb_buf(); depth = backtrace(cb_buf, MAX_DEPTH); symbols = backtrace_symbols(cb_buf, (int)depth); @@ -141,7 +167,7 @@ log_backtrace_impl(int severity, log_domain_mask_t domain, const char *msg, raw_free(symbols); done: - pthread_mutex_unlock(&cb_buf_mutex); + unlock_cb_buf(cb_buf); } static void crash_handler(int sig, siginfo_t *si, void *ctx_) @@ -157,6 +183,8 @@ crash_handler(int sig, siginfo_t *si, void *ctx_) int n_fds, i; const int *fds = NULL; + void **cb_buf = lock_cb_buf(); + (void) si; depth = backtrace(cb_buf, MAX_DEPTH); @@ -173,6 +201,8 @@ crash_handler(int sig, siginfo_t *si, void *ctx_) for (i=0; i < n_fds; ++i) backtrace_symbols_fd(cb_buf, (int)depth, fds[i]); + unlock_cb_buf(cb_buf); + tor_raw_abort_(); } @@ -184,11 +214,15 @@ dump_stack_symbols_to_error_fds(void) const int *fds = NULL; size_t depth; + void **cb_buf = lock_cb_buf(); + depth = backtrace(cb_buf, MAX_DEPTH); n_fds = tor_log_get_sigsafe_err_fds(&fds); for (i=0; i < n_fds; ++i) backtrace_symbols_fd(cb_buf, (int)depth, fds[i]); + + unlock_cb_buf(cb_buf); } /* The signals that we want our backtrace handler to trap */ @@ -222,10 +256,12 @@ install_bt_handler(void) * libc has pre-loaded the symbols we need to dump things, so that later * reads won't be denied by the sandbox code */ char **symbols; + void **cb_buf = lock_cb_buf(); size_t depth = backtrace(cb_buf, MAX_DEPTH); symbols = backtrace_symbols(cb_buf, (int) depth); if (symbols) raw_free(symbols); + unlock_cb_buf(cb_buf); } return rv; diff --git a/src/lib/err/torerr.c b/src/lib/err/torerr.c index b7e32a3e20..0a4ee5d417 100644 --- a/src/lib/err/torerr.c +++ b/src/lib/err/torerr.c @@ -290,8 +290,7 @@ format_number_sigsafe(unsigned long x, char *buf, int buf_len, * does not guarantee that an int is wider than a char (an int must be at * least 16 bits but it is permitted for a char to be that wide as well), we * can't assume a signed int is sufficient to accommodate an unsigned char. - * Thus, format_helper_exit_status() will still need to emit any require '-' - * on its own. + * Thus, callers will still need to add any required '-' to the final string. * * For most purposes, you'd want to use tor_snprintf("%x") instead of this * function; it's designed to be used in code paths where you can't call diff --git a/src/lib/err/torerr_sys.c b/src/lib/err/torerr_sys.c index a14c46f945..eb818004fb 100644 --- a/src/lib/err/torerr_sys.c +++ b/src/lib/err/torerr_sys.c @@ -36,7 +36,10 @@ subsys_torerr_shutdown(void) const subsys_fns_t sys_torerr = { .name = "err", - .level = -100, + /* Low-level error handling is a diagnostic feature, we want it to init + * right after windows process security, and shutdown last. + * (Security never shuts down.) */ + .level = -99, .supported = true, .initialize = subsys_torerr_initialize, .shutdown = subsys_torerr_shutdown diff --git a/src/lib/evloop/.may_include b/src/lib/evloop/.may_include index 273de7bb94..54aa75fbff 100644 --- a/src/lib/evloop/.may_include +++ b/src/lib/evloop/.may_include @@ -8,6 +8,7 @@ lib/log/*.h lib/malloc/*.h lib/net/*.h lib/string/*.h +lib/subsys/*.h lib/testsupport/*.h lib/thread/*.h lib/time/*.h diff --git a/src/lib/evloop/evloop_sys.c b/src/lib/evloop/evloop_sys.c new file mode 100644 index 0000000000..56641a3175 --- /dev/null +++ b/src/lib/evloop/evloop_sys.c @@ -0,0 +1,49 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file evloop_sys.c + * @brief Subsystem definition for the event loop module + **/ + +#include "orconfig.h" +#include "lib/subsys/subsys.h" +#include "lib/evloop/compat_libevent.h" +#include "lib/evloop/evloop_sys.h" +#include "lib/log/log.h" + +static int +subsys_evloop_initialize(void) +{ + if (tor_init_libevent_rng() < 0) { + log_warn(LD_NET, "Problem initializing libevent RNG."); + return -1; + } + return 0; +} + +static void +subsys_evloop_postfork(void) +{ +#ifdef TOR_UNIT_TESTS + tor_libevent_postfork(); +#endif +} + +static void +subsys_evloop_shutdown(void) +{ + tor_libevent_free_all(); +} + +const struct subsys_fns_t sys_evloop = { + .name = "evloop", + .supported = true, + .level = -20, + .initialize = subsys_evloop_initialize, + .shutdown = subsys_evloop_shutdown, + .postfork = subsys_evloop_postfork, +}; diff --git a/src/lib/evloop/evloop_sys.h b/src/lib/evloop/evloop_sys.h new file mode 100644 index 0000000000..e6155c25b0 --- /dev/null +++ b/src/lib/evloop/evloop_sys.h @@ -0,0 +1,17 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file evloop_sys.h + * @brief Declare subsystem object for the event loop module. + **/ + +#ifndef TOR_LIB_EVLOOP_EVLOOP_SYS_H +#define TOR_LIB_EVLOOP_EVLOOP_SYS_H + +extern const struct subsys_fns_t sys_evloop; + +#endif /* !defined(TOR_LIB_EVLOOP_EVLOOP_SYS_H) */ diff --git a/src/lib/evloop/include.am b/src/lib/evloop/include.am index 6595b3a34b..41cd2f45c5 100644 --- a/src/lib/evloop/include.am +++ b/src/lib/evloop/include.am @@ -8,6 +8,7 @@ endif # ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_evloop_a_SOURCES = \ src/lib/evloop/compat_libevent.c \ + src/lib/evloop/evloop_sys.c \ src/lib/evloop/procmon.c \ src/lib/evloop/timers.c \ src/lib/evloop/token_bucket.c \ @@ -21,6 +22,7 @@ src_lib_libtor_evloop_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) # ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/evloop/compat_libevent.h \ + src/lib/evloop/evloop_sys.h \ src/lib/evloop/procmon.h \ src/lib/evloop/timers.h \ src/lib/evloop/token_bucket.h \ diff --git a/src/lib/evloop/procmon.c b/src/lib/evloop/procmon.c index 52469fa5fc..b2d81fc14b 100644 --- a/src/lib/evloop/procmon.c +++ b/src/lib/evloop/procmon.c @@ -303,7 +303,7 @@ tor_process_monitor_poll_cb(periodic_timer_t *event, void *procmon_) tor_free(errmsg); } } -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ /* Unix makes this part easy, if a bit racy. */ its_dead_jim = kill(procmon->pid, 0); its_dead_jim = its_dead_jim && (errno == ESRCH); diff --git a/src/lib/evloop/token_bucket.c b/src/lib/evloop/token_bucket.c index ee6d631e3b..ec62d1b018 100644 --- a/src/lib/evloop/token_bucket.c +++ b/src/lib/evloop/token_bucket.c @@ -256,3 +256,55 @@ token_bucket_rw_dec(token_bucket_rw_t *bucket, flags |= TB_WRITE; return flags; } + +/** Initialize a token bucket in <b>bucket</b>, set up to allow <b>rate</b> + * per second, with a maximum burst of <b>burst</b>. The bucket is created + * such that <b>now_ts</b> is the current timestamp. The bucket starts out + * full. */ +void +token_bucket_ctr_init(token_bucket_ctr_t *bucket, uint32_t rate, + uint32_t burst, uint32_t now_ts) +{ + memset(bucket, 0, sizeof(token_bucket_ctr_t)); + token_bucket_ctr_adjust(bucket, rate, burst); + token_bucket_ctr_reset(bucket, now_ts); +} + +/** Change the configured rate and burst of the given token bucket object in + * <b>bucket</b>. */ +void +token_bucket_ctr_adjust(token_bucket_ctr_t *bucket, uint32_t rate, + uint32_t burst) +{ + token_bucket_cfg_init(&bucket->cfg, rate, burst); + token_bucket_raw_adjust(&bucket->counter, &bucket->cfg); +} + +/** Reset <b>bucket</b> to be full, as of timestamp <b>now_ts</b>. */ +void +token_bucket_ctr_reset(token_bucket_ctr_t *bucket, uint32_t now_ts) +{ + token_bucket_raw_reset(&bucket->counter, &bucket->cfg); + bucket->last_refilled_at_timestamp = now_ts; +} + +/** Refill <b>bucket</b> as appropriate, given that the current timestamp is + * <b>now_ts</b>. */ +void +token_bucket_ctr_refill(token_bucket_ctr_t *bucket, uint32_t now_ts) +{ + const uint32_t elapsed_ticks = + (now_ts - bucket->last_refilled_at_timestamp); + if (elapsed_ticks > UINT32_MAX-(300*1000)) { + /* Either about 48 days have passed since the last refill, or the + * monotonic clock has somehow moved backwards. (We're looking at you, + * Windows.). We accept up to a 5 minute jump backwards as + * "unremarkable". + */ + return; + } + + token_bucket_raw_refill_steps(&bucket->counter, &bucket->cfg, + elapsed_ticks); + bucket->last_refilled_at_timestamp = now_ts; +} diff --git a/src/lib/evloop/token_bucket.h b/src/lib/evloop/token_bucket.h index 1ce6f1bf94..dde9bd65a4 100644 --- a/src/lib/evloop/token_bucket.h +++ b/src/lib/evloop/token_bucket.h @@ -103,6 +103,35 @@ token_bucket_rw_get_write(const token_bucket_rw_t *bucket) return token_bucket_raw_get(&bucket->write_bucket); } +/** + * A specialized bucket containing a single counter. + */ + +typedef struct token_bucket_ctr_t { + token_bucket_cfg_t cfg; + token_bucket_raw_t counter; + uint32_t last_refilled_at_timestamp; +} token_bucket_ctr_t; + +void token_bucket_ctr_init(token_bucket_ctr_t *bucket, uint32_t rate, + uint32_t burst, uint32_t now_ts); +void token_bucket_ctr_adjust(token_bucket_ctr_t *bucket, uint32_t rate, + uint32_t burst); +void token_bucket_ctr_reset(token_bucket_ctr_t *bucket, uint32_t now_ts); +void token_bucket_ctr_refill(token_bucket_ctr_t *bucket, uint32_t now_ts); + +static inline bool +token_bucket_ctr_dec(token_bucket_ctr_t *bucket, ssize_t n) +{ + return token_bucket_raw_dec(&bucket->counter, n); +} + +static inline size_t +token_bucket_ctr_get(const token_bucket_ctr_t *bucket) +{ + return token_bucket_raw_get(&bucket->counter); +} + #ifdef TOKEN_BUCKET_PRIVATE /* To avoid making the rates too small, we consider units of "steps", diff --git a/src/lib/fs/dir.c b/src/lib/fs/dir.c index 3c31e00d99..390836b048 100644 --- a/src/lib/fs/dir.c +++ b/src/lib/fs/dir.c @@ -262,7 +262,7 @@ check_private_dir,(const char *dirname, cpd_check_t check, } } close(fd); -#else /* !(!defined(_WIN32)) */ +#else /* defined(_WIN32) */ /* Win32 case: we can't open() a directory. */ (void)effective_user; @@ -347,7 +347,7 @@ tor_listdir, (const char *dirname)) } FindClose(handle); tor_free(pattern); -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ const char *prot_dname = sandbox_intern_string(dirname); DIR *d; struct dirent *de; diff --git a/src/lib/fs/files.h b/src/lib/fs/files.h index 81dba8c140..ed983f3b3c 100644 --- a/src/lib/fs/files.h +++ b/src/lib/fs/files.h @@ -123,7 +123,7 @@ ssize_t compat_getdelim_(char **lineptr, size_t *n, int delim, FILE *stream); */ #define tor_getdelim(lineptr, n, delim, stream) \ getdelim((lineptr), (n), (delim), (stream)) -#else /* !(defined(HAVE_GETDELIM)) */ +#else /* !defined(HAVE_GETDELIM) */ #define tor_getdelim(lineptr, n, delim, stream) \ compat_getdelim_((lineptr), (n), (delim), (stream)) #endif /* defined(HAVE_GETDELIM) */ @@ -137,7 +137,7 @@ ssize_t compat_getdelim_(char **lineptr, size_t *n, int delim, FILE *stream); */ #define tor_getline(lineptr, n, stream) \ getline((lineptr), (n), (stream)) -#else /* !(defined(HAVE_GETLINE)) */ +#else /* !defined(HAVE_GETLINE) */ #define tor_getline(lineptr, n, stream) \ tor_getdelim((lineptr), (n), '\n', (stream)) #endif /* defined(HAVE_GETLINE) */ diff --git a/src/lib/fs/path.c b/src/lib/fs/path.c index b3ef61979d..28dde62aea 100644 --- a/src/lib/fs/path.c +++ b/src/lib/fs/path.c @@ -72,7 +72,7 @@ expand_filename(const char *filename) * Chapter+3.+Input+Validation/3.7+Validating+Filenames+and+Paths/ */ return tor_strdup(filename); -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ if (*filename == '~') { char *home, *result=NULL; const char *rest; @@ -102,7 +102,7 @@ expand_filename(const char *filename) } tor_free(username); rest = slash ? (slash+1) : ""; -#else /* !(defined(HAVE_PWD_H)) */ +#else /* !defined(HAVE_PWD_H) */ log_warn(LD_CONFIG, "Couldn't expand homedir on system without pwd.h"); return tor_strdup(filename); #endif /* defined(HAVE_PWD_H) */ @@ -153,7 +153,7 @@ clean_fname_for_stat(char *name) return; name[len-1]='\0'; } -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ (void)name; #endif /* defined(_WIN32) */ } @@ -233,7 +233,7 @@ alloc_getcwd(void) raw_free(cwd); // alias for free to avoid tripping check-spaces. } return result; -#else /* !(defined(HAVE_GET_CURRENT_DIR_NAME)) */ +#else /* !defined(HAVE_GET_CURRENT_DIR_NAME) */ size_t size = 1024; char *buf = NULL; char *ptr = NULL; @@ -268,7 +268,7 @@ make_path_absolute(char *fname) if (absfname_malloced) raw_free(absfname_malloced); return absfname; -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ char *absfname = NULL, *path = NULL; tor_assert(fname); diff --git a/src/lib/log/log.c b/src/lib/log/log.c index ec7c2fa24e..83f04a3467 100644 --- a/src/lib/log/log.c +++ b/src/lib/log/log.c @@ -55,10 +55,6 @@ #include <android/log.h> #endif // HAVE_ANDROID_LOG_H. -/** Given a severity, yields an index into log_severity_list_t.masks to use - * for that severity. */ -#define SEVERITY_MASK_IDX(sev) ((sev) - LOG_ERR) - /** @{ */ /** The string we stick at the end of a log message when it is too long, * and its length. */ @@ -536,7 +532,7 @@ logfile_deliver(logfile_t *lf, const char *buf, size_t msg_len, if (m != msg_after_prefix) { tor_free(m); } -#else /* !(defined(MAXLINE)) */ +#else /* !defined(MAXLINE) */ /* We have syslog but not MAXLINE. That's promising! */ syslog(severity, "%s", msg_after_prefix); #endif /* defined(MAXLINE) */ @@ -588,8 +584,7 @@ logv,(int severity, log_domain_mask_t domain, const char *funcname, /* check that severity is sane. Overrunning the masks array leads to * interesting and hard to diagnose effects */ raw_assert(severity >= LOG_ERR && severity <= LOG_DEBUG); - /* check that we've initialised the log mutex before we try to lock it */ - raw_assert(log_mutex_initialized); + LOCK_LOGS(); if ((! (domain & LD_NOCB)) && pending_cb_messages @@ -633,6 +628,10 @@ void tor_log(int severity, log_domain_mask_t domain, const char *format, ...) { va_list ap; + + /* check that domain is composed of known domains and flags */ + raw_assert((domain & (LD_ALL_DOMAINS|LD_ALL_FLAGS)) == domain); + if (severity > log_global_min_severity_) return; va_start(ap,format); @@ -687,8 +686,9 @@ tor_log_update_sigsafe_err_fds(void) n_fds = 1; for (lf = logfiles; lf; lf = lf->next) { - /* Don't try callback to the control port, or syslogs: We can't - * do them from a signal handler. Don't try stdout: we always do stderr. + /* Don't try callback to the control port, syslogs, android logs, or any + * other non-file descriptor log: We can't call arbitrary functions from a + * signal handler. */ if (lf->is_temporary || logfile_is_external(lf) || lf->seems_dead || lf->fd < 0) @@ -720,7 +720,10 @@ tor_log_update_sigsafe_err_fds(void) if (!found_real_stderr && int_array_contains(log_fds, n_fds, STDOUT_FILENO)) { - /* Don't use a virtual stderr when we're also logging to stdout. */ + /* Don't use a virtual stderr when we're also logging to stdout. + * If we reached max_fds logs, we'll now have (max_fds - 1) logs. + * That's ok, max_fds is large enough that most tor instances don't exceed + * it. */ raw_assert(n_fds >= 2); /* Don't tor_assert inside log fns */ --n_fds; log_fds[0] = log_fds[n_fds]; @@ -755,7 +758,7 @@ tor_log_get_logfile_names(smartlist_t *out) /** Implementation of the log_fn backend, used when we have * variadic macros. All arguments are as for log_fn, except for - * <b>fn</b>, which is the name of the calling functions. */ + * <b>fn</b>, which is the name of the calling function. */ void log_fn_(int severity, log_domain_mask_t domain, const char *fn, const char *format, ...) @@ -866,9 +869,6 @@ logs_close_sigsafe(void) * logfiles (it is probably present, but it might not be due to thread * racing issues). After this function is called, the caller shouldn't * refer to <b>victim</b> anymore. - * - * Long-term, we need to do something about races in the log subsystem - * in general. See bug 222 for more details. */ static void delete_log(logfile_t *victim) @@ -931,7 +931,7 @@ set_log_severity_config(int loglevelMin, int loglevelMax, raw_assert(loglevelMax >= LOG_ERR && loglevelMax <= LOG_DEBUG); memset(severity_out, 0, sizeof(log_severity_list_t)); for (i = loglevelMin; i >= loglevelMax; --i) { - severity_out->masks[SEVERITY_MASK_IDX(i)] = ~0u; + severity_out->masks[SEVERITY_MASK_IDX(i)] = LD_ALL_DOMAINS; } } @@ -1341,7 +1341,9 @@ static const char *domain_list[] = { CTASSERT(ARRAY_LENGTH(domain_list) == N_LOGGING_DOMAINS + 1); -CTASSERT((UINT64_C(1)<<(N_LOGGING_DOMAINS-1)) < LOWEST_RESERVED_LD_FLAG_); +CTASSERT(HIGHEST_RESERVED_LD_DOMAIN_ < LD_ALL_DOMAINS); +CTASSERT(LD_ALL_DOMAINS < LOWEST_RESERVED_LD_FLAG_); +CTASSERT(LOWEST_RESERVED_LD_FLAG_ < LD_ALL_FLAGS); /** Return a bitmask for the log domain for which <b>domain</b> is the name, * or 0 if there is no such name. */ @@ -1351,7 +1353,7 @@ parse_log_domain(const char *domain) int i; for (i=0; domain_list[i]; ++i) { if (!strcasecmp(domain, domain_list[i])) - return (1u<<i); + return (UINT64_C(1)<<i); } return 0; } @@ -1423,7 +1425,7 @@ parse_log_severity_config(const char **cfg_ptr, const char *dash, *space; char *sev_lo, *sev_hi; int low, high, i; - log_domain_mask_t domains = ~0u; + log_domain_mask_t domains = LD_ALL_DOMAINS; if (*cfg == '[') { int err = 0; @@ -1441,7 +1443,7 @@ parse_log_severity_config(const char **cfg_ptr, tor_free(domains_str); SMARTLIST_FOREACH_BEGIN(domains_list, const char *, domain) { if (!strcmp(domain, "*")) { - domains = ~0u; + domains = LD_ALL_DOMAINS; } else { log_domain_mask_t d; int negate=0; @@ -1537,7 +1539,7 @@ switch_logs_debug(void) LOCK_LOGS(); for (lf = logfiles; lf; lf=lf->next) { for (i = LOG_DEBUG; i >= LOG_ERR; --i) - lf->severities->masks[SEVERITY_MASK_IDX(i)] = ~0u; + lf->severities->masks[SEVERITY_MASK_IDX(i)] = LD_ALL_DOMAINS; } log_global_min_severity_ = get_min_log_level(); UNLOCK_LOGS(); diff --git a/src/lib/log/log.h b/src/lib/log/log.h index 4291418eb6..8e36012616 100644 --- a/src/lib/log/log.h +++ b/src/lib/log/log.h @@ -26,7 +26,7 @@ #error "Your syslog.h thinks high numbers are more important. " \ "We aren't prepared to deal with that." #endif -#else /* !(defined(HAVE_SYSLOG_H)) */ +#else /* !defined(HAVE_SYSLOG_H) */ /* Note: Syslog's logging code refers to priorities, with 0 being the most * important. Thus, all our comparisons needed to be reversed when we added * syslog support. @@ -117,10 +117,21 @@ #define LD_BTRACK (UINT64_C(1)<<28) /** Message-passing backend. */ #define LD_MESG (UINT64_C(1)<<29) + +/** The number of log domains. */ #define N_LOGGING_DOMAINS 30 +/** The highest log domain */ +#define HIGHEST_RESERVED_LD_DOMAIN_ (UINT64_C(1)<<(N_LOGGING_DOMAINS - 1)) +/** All log domains. */ +#define LD_ALL_DOMAINS ((~(UINT64_C(0)))>>(64 - N_LOGGING_DOMAINS)) +/** The number of log flags. */ +#define N_LOGGING_FLAGS 3 /** First bit that is reserved in log_domain_mask_t for non-domain flags. */ -#define LOWEST_RESERVED_LD_FLAG_ (UINT64_C(1)<<61) +#define LOWEST_RESERVED_LD_FLAG_ (UINT64_C(1)<<(64 - N_LOGGING_FLAGS)) +/** All log flags. */ +#define LD_ALL_FLAGS ((~(UINT64_C(0)))<<(64 - N_LOGGING_FLAGS)) + #ifdef TOR_UNIT_TESTS /** This log message should not be intercepted by mock_saving_logv */ #define LD_NO_MOCK (UINT64_C(1)<<61) @@ -297,4 +308,10 @@ MOCK_DECL(STATIC void, logv, (int severity, log_domain_mask_t domain, va_list ap) CHECK_PRINTF(5,0)); #endif +#if defined(LOG_PRIVATE) || defined(TOR_UNIT_TESTS) +/** Given a severity, yields an index into log_severity_list_t.masks to use + * for that severity. */ +#define SEVERITY_MASK_IDX(sev) ((sev) - LOG_ERR) +#endif + #endif /* !defined(TOR_TORLOG_H) */ diff --git a/src/lib/log/log_sys.c b/src/lib/log/log_sys.c index d1080f2264..826358546a 100644 --- a/src/lib/log/log_sys.c +++ b/src/lib/log/log_sys.c @@ -29,6 +29,8 @@ subsys_logging_shutdown(void) const subsys_fns_t sys_logging = { .name = "log", .supported = true, + /* Logging depends on threads, approx time, raw logging, and security. + * Most other lib modules depend on logging. */ .level = -90, .initialize = subsys_logging_initialize, .shutdown = subsys_logging_shutdown, diff --git a/src/lib/log/util_bug.c b/src/lib/log/util_bug.c index 0e99be35a4..72c614a3b2 100644 --- a/src/lib/log/util_bug.c +++ b/src/lib/log/util_bug.c @@ -64,7 +64,7 @@ tor_set_failed_assertion_callback(void (*fn)(void)) { failed_assertion_cb = fn; } -#else /* !(defined(TOR_UNIT_TESTS)) */ +#else /* !defined(TOR_UNIT_TESTS) */ #define capturing_bugs() (0) #define add_captured_bug(s) do { } while (0) #endif /* defined(TOR_UNIT_TESTS) */ diff --git a/src/lib/log/util_bug.h b/src/lib/log/util_bug.h index 546ae1e3ef..030bfa2cc9 100644 --- a/src/lib/log/util_bug.h +++ b/src/lib/log/util_bug.h @@ -96,7 +96,7 @@ (void)(a); \ (void)(fmt); \ STMT_END -#else /* !(defined(TOR_UNIT_TESTS) && ... */ +#else /* !(defined(TOR_UNIT_TESTS) && defined(DISABLE_ASSERTS_IN_UNIT_T...)) */ /** Like assert(3), but send assertion failures to the log as well as to * stderr. */ #define tor_assert(expr) tor_assertf(expr, NULL) @@ -211,7 +211,7 @@ "!("#cond")", 1, NULL); \ } \ bool_result; } )) -#else /* !(defined(__GNUC__)) */ +#else /* !defined(__GNUC__) */ #define IF_BUG_ONCE__(cond,var) \ static int var = 0; \ if ((cond) ? \ @@ -242,10 +242,12 @@ void tor_assertion_failed_(const char *fname, unsigned int line, const char *func, const char *expr, - const char *fmt, ...); + const char *fmt, ...) + CHECK_PRINTF(5,6); void tor_bug_occurred_(const char *fname, unsigned int line, const char *func, const char *expr, - int once, const char *fmt, ...); + int once, const char *fmt, ...) + CHECK_PRINTF(6,7); void tor_abort_(void) ATTR_NORETURN; diff --git a/src/lib/log/win32err.c b/src/lib/log/win32err.c index dc45cb4c3d..03d5c9fad2 100644 --- a/src/lib/log/win32err.c +++ b/src/lib/log/win32err.c @@ -47,7 +47,7 @@ format_win32_error(DWORD err) result = tor_malloc(len); wcstombs(result,str,len); result[len-1] = '\0'; -#else /* !(defined(UNICODE)) */ +#else /* !defined(UNICODE) */ result = tor_strdup(str); #endif /* defined(UNICODE) */ } else { diff --git a/src/lib/malloc/malloc.h b/src/lib/malloc/malloc.h index 8c81d30dd5..39a45901a1 100644 --- a/src/lib/malloc/malloc.h +++ b/src/lib/malloc/malloc.h @@ -48,7 +48,7 @@ void tor_free_(void *mem); raw_free(*tor_free__tmpvar); \ *tor_free__tmpvar=NULL; \ STMT_END -#else /* !(defined(__GNUC__)) */ +#else /* !defined(__GNUC__) */ #define tor_free(p) STMT_BEGIN \ raw_free(p); \ (p)=NULL; \ diff --git a/src/lib/malloc/map_anon.c b/src/lib/malloc/map_anon.c index ae4edff769..9559cbe2d4 100644 --- a/src/lib/malloc/map_anon.c +++ b/src/lib/malloc/map_anon.c @@ -122,7 +122,7 @@ nodump_mem(void *mem, size_t sz) NULL); return -1; } -#else +#else /* !defined(MADV_DONTDUMP) */ (void) mem; (void) sz; return 0; @@ -170,12 +170,12 @@ noinherit_mem(void *mem, size_t sz, inherit_res_t *inherit_result_out) NULL); return -1; } -#else +#else /* !(defined(FLAG_ZERO) || defined(FLAG_NOINHERIT)) */ (void)inherit_result_out; (void)mem; (void)sz; return 0; -#endif +#endif /* defined(FLAG_ZERO) || defined(FLAG_NOINHERIT) */ } /** diff --git a/src/lib/math/fp.c b/src/lib/math/fp.c index 616e4f15c0..49a2a6a2ca 100644 --- a/src/lib/math/fp.c +++ b/src/lib/math/fp.c @@ -75,7 +75,7 @@ clamp_double_to_int64(double number) */ #define PROBLEMATIC_FLOAT_CONVERSION_WARNING DISABLE_GCC_WARNING(float-conversion) -#endif /* defined(MINGW_ANY) && GCC_VERSION >= 409 */ +#endif /* (defined(MINGW_ANY)||defined(__FreeBSD__)) && GCC_VERSION >= 409 */ /* With clang 4.0 we apparently run into "double promotion" warnings here, diff --git a/src/lib/memarea/memarea.c b/src/lib/memarea/memarea.c index 84c73b0b95..f3bb79a1e2 100644 --- a/src/lib/memarea/memarea.c +++ b/src/lib/memarea/memarea.c @@ -68,7 +68,7 @@ uint32_t sent_val = get_uint32(&(chunk)->U_MEM[chunk->mem_size]); \ tor_assert(sent_val == SENTINEL_VAL); \ STMT_END -#else /* !(defined(USE_SENTINELS)) */ +#else /* !defined(USE_SENTINELS) */ #define SENTINEL_LEN 0 #define SET_SENTINEL(chunk) STMT_NIL #define CHECK_SENTINEL(chunk) STMT_NIL @@ -315,7 +315,7 @@ memarea_assert_ok(memarea_t *area) } } -#else /* !(!defined(DISABLE_MEMORY_SENTINELS)) */ +#else /* defined(DISABLE_MEMORY_SENTINELS) */ struct memarea_t { smartlist_t *pieces; diff --git a/src/lib/meminfo/meminfo.c b/src/lib/meminfo/meminfo.c index f4fa45167e..bff71c2f05 100644 --- a/src/lib/meminfo/meminfo.c +++ b/src/lib/meminfo/meminfo.c @@ -54,7 +54,7 @@ tor_log_mallinfo(int severity) mi.arena, mi.ordblks, mi.smblks, mi.hblks, mi.hblkhd, mi.usmblks, mi.fsmblks, mi.uordblks, mi.fordblks, mi.keepcost); -#else /* !(defined(HAVE_MALLINFO)) */ +#else /* !defined(HAVE_MALLINFO) */ (void)severity; #endif /* defined(HAVE_MALLINFO) */ } diff --git a/src/lib/net/address.c b/src/lib/net/address.c index 546af800a9..0a2c84caf2 100644 --- a/src/lib/net/address.c +++ b/src/lib/net/address.c @@ -373,7 +373,8 @@ tor_addr_to_str(char *dest, const tor_addr_t *addr, size_t len, int decorate) * * If <b>accept_regular</b> is set and the address is in neither recognized * reverse lookup hostname format, try parsing the address as a regular - * IPv4 or IPv6 address too. + * IPv4 or IPv6 address too. This mode will accept IPv6 addresses with or + * without square brackets. */ int tor_addr_parse_PTR_name(tor_addr_t *result, const char *address, @@ -1187,17 +1188,22 @@ fmt_addr32(uint32_t addr) } /** Convert the string in <b>src</b> to a tor_addr_t <b>addr</b>. The string - * may be an IPv4 address, an IPv6 address, or an IPv6 address surrounded by - * square brackets. + * may be an IPv4 address, or an IPv6 address surrounded by square brackets. * - * Return an address family on success, or -1 if an invalid address string is - * provided. */ -int -tor_addr_parse(tor_addr_t *addr, const char *src) + * If <b>allow_ipv6_without_brackets</b> is true, also allow IPv6 addresses + * without brackets. + * + * Always rejects IPv4 addresses with brackets. + * + * Returns an address family on success, or -1 if an invalid address string is + * provided. */ +static int +tor_addr_parse_impl(tor_addr_t *addr, const char *src, + bool allow_ipv6_without_brackets) { /* Holds substring of IPv6 address after removing square brackets */ char *tmp = NULL; - int result; + int result = -1; struct in_addr in_tmp; struct in6_addr in6_tmp; int brackets_detected = 0; @@ -1211,21 +1217,46 @@ tor_addr_parse(tor_addr_t *addr, const char *src) src = tmp = tor_strndup(src+1, strlen(src)-2); } - if (tor_inet_pton(AF_INET6, src, &in6_tmp) > 0) { - result = AF_INET6; - tor_addr_from_in6(addr, &in6_tmp); - } else if (!brackets_detected && - tor_inet_pton(AF_INET, src, &in_tmp) > 0) { - result = AF_INET; - tor_addr_from_in(addr, &in_tmp); - } else { - result = -1; + /* Try to parse an IPv6 address if it has brackets, or if IPv6 addresses + * without brackets are allowed */ + if (brackets_detected || allow_ipv6_without_brackets) { + if (tor_inet_pton(AF_INET6, src, &in6_tmp) > 0) { + result = AF_INET6; + tor_addr_from_in6(addr, &in6_tmp); + } + } + + /* Try to parse an IPv4 address without brackets */ + if (!brackets_detected) { + if (tor_inet_pton(AF_INET, src, &in_tmp) > 0) { + result = AF_INET; + tor_addr_from_in(addr, &in_tmp); + } + } + + /* Clear the address on error, to avoid returning uninitialised or partly + * parsed data. + */ + if (result == -1) { + memset(addr, 0, sizeof(tor_addr_t)); } tor_free(tmp); return result; } +/** Convert the string in <b>src</b> to a tor_addr_t <b>addr</b>. The string + * may be an IPv4 address, an IPv6 address, or an IPv6 address surrounded by + * square brackets. + * + * Returns an address family on success, or -1 if an invalid address string is + * provided. */ +int +tor_addr_parse(tor_addr_t *addr, const char *src) +{ + return tor_addr_parse_impl(addr, src, 1); +} + #ifdef HAVE_IFADDRS_TO_SMARTLIST /* * Convert a linked list consisting of <b>ifaddrs</b> structures @@ -1718,6 +1749,11 @@ get_interface_address6_list,(int severity, * form "ip" or "ip:0". Otherwise, accept those forms, and set * *<b>port_out</b> to <b>default_port</b>. * + * This function accepts: + * - IPv6 address and port, when the IPv6 address is in square brackets, + * - IPv6 address with square brackets, + * - IPv6 address without square brackets. + * * Return 0 on success, -1 on failure. */ int tor_addr_port_parse(int severity, const char *addrport, @@ -1727,6 +1763,7 @@ tor_addr_port_parse(int severity, const char *addrport, int retval = -1; int r; char *addr_tmp = NULL; + bool has_port; tor_assert(addrport); tor_assert(address_out); @@ -1736,28 +1773,47 @@ tor_addr_port_parse(int severity, const char *addrport, if (r < 0) goto done; - if (!*port_out) { + has_port = !! *port_out; + /* If there's no port, use the default port, or fail if there is no default + */ + if (!has_port) { if (default_port >= 0) *port_out = default_port; else goto done; } - /* make sure that address_out is an IP address */ - if (tor_addr_parse(address_out, addr_tmp) < 0) + /* Make sure that address_out is an IP address. + * If there is no port in addrport, allow IPv6 addresses without brackets. */ + if (tor_addr_parse_impl(address_out, addr_tmp, !has_port) < 0) goto done; retval = 0; done: + /* Clear the address and port on error, to avoid returning uninitialised or + * partly parsed data. + */ + if (retval == -1) { + memset(address_out, 0, sizeof(tor_addr_t)); + *port_out = 0; + } tor_free(addr_tmp); return retval; } /** Given an address of the form "host[:port]", try to divide it into its host - * and port portions, setting *<b>address_out</b> to a newly allocated string - * holding the address portion and *<b>port_out</b> to the port (or 0 if no - * port is given). Return 0 on success, -1 on failure. */ + * and port portions. + * + * Like tor_addr_port_parse(), this function accepts: + * - IPv6 address and port, when the IPv6 address is in square brackets, + * - IPv6 address with square brackets, + * - IPv6 address without square brackets. + * + * Sets *<b>address_out</b> to a newly allocated string holding the address + * portion, and *<b>port_out</b> to the port (or 0 if no port is given). + * + * Return 0 on success, -1 on failure. */ int tor_addr_port_split(int severity, const char *addrport, char **address_out, uint16_t *port_out) @@ -1766,8 +1822,11 @@ tor_addr_port_split(int severity, const char *addrport, tor_assert(addrport); tor_assert(address_out); tor_assert(port_out); + /* We need to check for IPv6 manually because the logic below doesn't - * do a good job on IPv6 addresses that lack a port. */ + * do a good job on IPv6 addresses that lack a port. + * If an IPv6 address without square brackets is ambiguous, it gets parsed + * here as an address, rather than address:port. */ if (tor_addr_parse(&a_tmp, addrport) == AF_INET6) { *port_out = 0; *address_out = tor_strdup(addrport); @@ -1807,8 +1866,7 @@ tor_addr_port_split(int severity, const char *addrport, tor_free(address_); } - if (port_out) - *port_out = ok ? ((uint16_t) port_) : 0; + *port_out = ok ? ((uint16_t) port_) : 0; return ok ? 0 : -1; } diff --git a/src/lib/net/nettypes.h b/src/lib/net/nettypes.h index 0eb352c657..60039bac09 100644 --- a/src/lib/net/nettypes.h +++ b/src/lib/net/nettypes.h @@ -31,7 +31,7 @@ typedef int socklen_t; #define TOR_SOCKET_T_FORMAT "%"PRIuPTR #define SOCKET_OK(s) ((SOCKET)(s) != INVALID_SOCKET) #define TOR_INVALID_SOCKET INVALID_SOCKET -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ /** Type used for a network socket. */ #define tor_socket_t int #define TOR_SOCKET_T_FORMAT "%d" diff --git a/src/lib/net/network_sys.c b/src/lib/net/network_sys.c index 9dfdb2b45a..e0a2625d73 100644 --- a/src/lib/net/network_sys.c +++ b/src/lib/net/network_sys.c @@ -37,7 +37,9 @@ subsys_network_shutdown(void) const subsys_fns_t sys_network = { .name = "network", - .level = -90, + /* Network depends on logging, and a lot of other modules depend on network. + */ + .level = -80, .supported = true, .initialize = subsys_network_initialize, .shutdown = subsys_network_shutdown, diff --git a/src/lib/net/resolve.c b/src/lib/net/resolve.c index 2dda491d14..442bc4a6b3 100644 --- a/src/lib/net/resolve.c +++ b/src/lib/net/resolve.c @@ -8,6 +8,7 @@ * \brief Use the libc DNS resolver to convert hostnames into addresses. **/ +#define RESOLVE_PRIVATE #include "lib/net/resolve.h" #include "lib/net/address.h" @@ -35,6 +36,8 @@ * *<b>addr</b> to the proper IP address, in host byte order. Returns 0 * on success, -1 on failure; 1 on transient failure. * + * This function only accepts IPv4 addresses. + * * (This function exists because standard windows gethostbyname * doesn't treat raw IP addresses properly.) */ @@ -45,6 +48,11 @@ tor_lookup_hostname,(const char *name, uint32_t *addr)) tor_addr_t myaddr; int ret; + if (BUG(!addr)) + return -1; + + *addr = 0; + if ((ret = tor_addr_lookup(name, AF_INET, &myaddr))) return ret; @@ -56,183 +64,257 @@ tor_lookup_hostname,(const char *name, uint32_t *addr)) return -1; } -/** Similar behavior to Unix gethostbyname: resolve <b>name</b>, and set - * *<b>addr</b> to the proper IP address and family. The <b>family</b> - * argument (which must be AF_INET, AF_INET6, or AF_UNSPEC) declares a - * <i>preferred</i> family, though another one may be returned if only one - * family is implemented for this address. +#ifdef HAVE_GETADDRINFO + +/* Host lookup helper for tor_addr_lookup(), when getaddrinfo() is + * available on this system. * - * Return 0 on success, -1 on failure; 1 on transient failure. + * See tor_addr_lookup() for details. */ -MOCK_IMPL(int, -tor_addr_lookup,(const char *name, uint16_t family, tor_addr_t *addr)) +MOCK_IMPL(STATIC int, +tor_addr_lookup_host_impl,(const char *name, + uint16_t family, + tor_addr_t *addr)) { - /* Perhaps eventually this should be replaced by a tor_getaddrinfo or - * something. - */ - struct in_addr iaddr; - struct in6_addr iaddr6; - tor_assert(name); - tor_assert(addr); - tor_assert(family == AF_INET || family == AF_INET6 || family == AF_UNSPEC); - if (!*name) { - /* Empty address is an error. */ - return -1; - } else if (tor_inet_pton(AF_INET, name, &iaddr)) { - /* It's an IPv4 IP. */ - if (family == AF_INET6) - return -1; - tor_addr_from_in(addr, &iaddr); - return 0; - } else if (tor_inet_pton(AF_INET6, name, &iaddr6)) { - if (family == AF_INET) - return -1; - tor_addr_from_in6(addr, &iaddr6); - return 0; - } else { -#ifdef HAVE_GETADDRINFO - int err; - struct addrinfo *res=NULL, *res_p; - struct addrinfo *best=NULL; - struct addrinfo hints; - int result = -1; - memset(&hints, 0, sizeof(hints)); - hints.ai_family = family; - hints.ai_socktype = SOCK_STREAM; - err = tor_getaddrinfo(name, NULL, &hints, &res); - /* The check for 'res' here shouldn't be necessary, but it makes static - * analysis tools happy. */ - if (!err && res) { - best = NULL; - for (res_p = res; res_p; res_p = res_p->ai_next) { - if (family == AF_UNSPEC) { - if (res_p->ai_family == AF_INET) { - best = res_p; - break; - } else if (res_p->ai_family == AF_INET6 && !best) { - best = res_p; - } - } else if (family == res_p->ai_family) { + int err; + struct addrinfo *res=NULL, *res_p; + struct addrinfo *best=NULL; + struct addrinfo hints; + int result = -1; + memset(&hints, 0, sizeof(hints)); + hints.ai_family = family; + hints.ai_socktype = SOCK_STREAM; + err = tor_getaddrinfo(name, NULL, &hints, &res); + /* The check for 'res' here shouldn't be necessary, but it makes static + * analysis tools happy. */ + if (!err && res) { + best = NULL; + for (res_p = res; res_p; res_p = res_p->ai_next) { + if (family == AF_UNSPEC) { + if (res_p->ai_family == AF_INET) { best = res_p; break; + } else if (res_p->ai_family == AF_INET6 && !best) { + best = res_p; } + } else if (family == res_p->ai_family) { + best = res_p; + break; } - if (!best) - best = res; - if (best->ai_family == AF_INET) { - tor_addr_from_in(addr, - &((struct sockaddr_in*)best->ai_addr)->sin_addr); - result = 0; - } else if (best->ai_family == AF_INET6) { - tor_addr_from_in6(addr, - &((struct sockaddr_in6*)best->ai_addr)->sin6_addr); - result = 0; - } - tor_freeaddrinfo(res); - return result; } - return (err == EAI_AGAIN) ? 1 : -1; -#else /* !(defined(HAVE_GETADDRINFO)) */ - struct hostent *ent; - int err; + if (!best) + best = res; + if (best->ai_family == AF_INET) { + tor_addr_from_in(addr, + &((struct sockaddr_in*)best->ai_addr)->sin_addr); + result = 0; + } else if (best->ai_family == AF_INET6) { + tor_addr_from_in6(addr, + &((struct sockaddr_in6*)best->ai_addr)->sin6_addr); + result = 0; + } + tor_freeaddrinfo(res); + return result; + } + return (err == EAI_AGAIN) ? 1 : -1; +} + +#else /* !defined(HAVE_GETADDRINFO) */ + +/* Host lookup helper for tor_addr_lookup(), which calls gethostbyname(). + * Used when getaddrinfo() is not available on this system. + * + * See tor_addr_lookup() for details. + */ +MOCK_IMPL(STATIC int, +tor_addr_lookup_host_impl,(const char *name, + uint16_t family, + tor_addr_t *addr)) +{ + (void) family; + struct hostent *ent; + int err; #ifdef HAVE_GETHOSTBYNAME_R_6_ARG - char buf[2048]; - struct hostent hostent; - int r; - r = gethostbyname_r(name, &hostent, buf, sizeof(buf), &ent, &err); + char buf[2048]; + struct hostent hostent; + int r; + r = gethostbyname_r(name, &hostent, buf, sizeof(buf), &ent, &err); #elif defined(HAVE_GETHOSTBYNAME_R_5_ARG) - char buf[2048]; - struct hostent hostent; - ent = gethostbyname_r(name, &hostent, buf, sizeof(buf), &err); + char buf[2048]; + struct hostent hostent; + ent = gethostbyname_r(name, &hostent, buf, sizeof(buf), &err); #elif defined(HAVE_GETHOSTBYNAME_R_3_ARG) - struct hostent_data data; - struct hostent hent; - memset(&data, 0, sizeof(data)); - err = gethostbyname_r(name, &hent, &data); - ent = err ? NULL : &hent; + struct hostent_data data; + struct hostent hent; + memset(&data, 0, sizeof(data)); + err = gethostbyname_r(name, &hent, &data); + ent = err ? NULL : &hent; #else - ent = gethostbyname(name); + ent = gethostbyname(name); #ifdef _WIN32 - err = WSAGetLastError(); + err = WSAGetLastError(); #else - err = h_errno; -#endif + err = h_errno; +#endif /* defined(_WIN32) */ #endif /* defined(HAVE_GETHOSTBYNAME_R_6_ARG) || ... */ - if (ent) { - if (ent->h_addrtype == AF_INET) { - tor_addr_from_in(addr, (struct in_addr*) ent->h_addr); - } else if (ent->h_addrtype == AF_INET6) { - tor_addr_from_in6(addr, (struct in6_addr*) ent->h_addr); - } else { - tor_assert(0); // LCOV_EXCL_LINE: gethostbyname() returned bizarre type - } - return 0; + if (ent) { + if (ent->h_addrtype == AF_INET) { + tor_addr_from_in(addr, (struct in_addr*) ent->h_addr); + } else if (ent->h_addrtype == AF_INET6) { + tor_addr_from_in6(addr, (struct in6_addr*) ent->h_addr); + } else { + tor_assert(0); // LCOV_EXCL_LINE: gethostbyname() returned bizarre type } + return 0; + } #ifdef _WIN32 - return (err == WSATRY_AGAIN) ? 1 : -1; + return (err == WSATRY_AGAIN) ? 1 : -1; #else - return (err == TRY_AGAIN) ? 1 : -1; + return (err == TRY_AGAIN) ? 1 : -1; #endif +} #endif /* defined(HAVE_GETADDRINFO) */ + +/** Similar behavior to Unix gethostbyname: resolve <b>name</b>, and set + * *<b>addr</b> to the proper IP address and family. The <b>family</b> + * argument (which must be AF_INET, AF_INET6, or AF_UNSPEC) declares a + * <i>preferred</i> family, though another one may be returned if only one + * family is implemented for this address. + * + * Like tor_addr_parse(), this function accepts IPv6 addresses with or without + * square brackets. + * + * Return 0 on success, -1 on failure; 1 on transient failure. + */ +MOCK_IMPL(int, +tor_addr_lookup,(const char *name, uint16_t family, tor_addr_t *addr)) +{ + /* Perhaps eventually this should be replaced by a tor_getaddrinfo or + * something. + */ + int parsed_family = 0; + int result = -1; + + tor_assert(name); + tor_assert(addr); + tor_assert(family == AF_INET || family == AF_INET6 || family == AF_UNSPEC); + + if (!*name) { + /* Empty address is an error. */ + goto permfail; + } + + /* Is it an IP address? */ + parsed_family = tor_addr_parse(addr, name); + + if (parsed_family >= 0) { + /* If the IP address family matches, or was unspecified */ + if (parsed_family == family || family == AF_UNSPEC) { + goto success; + } else { + goto permfail; + } + } else { + /* Clear the address after a failed tor_addr_parse(). */ + memset(addr, 0, sizeof(tor_addr_t)); + result = tor_addr_lookup_host_impl(name, family, addr); + goto done; + } + + /* If we weren't successful, and haven't already set the result, + * assume it's a permanent failure */ + permfail: + result = -1; + goto done; + success: + result = 0; + + /* We have set the result, now it's time to clean up */ + done: + if (result) { + /* Clear the address on error */ + memset(addr, 0, sizeof(tor_addr_t)); } + return result; } /** Parse an address or address-port combination from <b>s</b>, resolve the * address as needed, and put the result in <b>addr_out</b> and (optionally) - * <b>port_out</b>. Return 0 on success, negative on failure. */ + * <b>port_out</b>. + * + * Like tor_addr_port_parse(), this function accepts: + * - IPv6 address and port, when the IPv6 address is in square brackets, + * - IPv6 address with square brackets, + * - IPv6 address without square brackets. + * + * Return 0 on success, negative on failure. */ int tor_addr_port_lookup(const char *s, tor_addr_t *addr_out, uint16_t *port_out) { - const char *port; tor_addr_t addr; - uint16_t portval; + uint16_t portval = 0; char *tmp = NULL; + int rv = 0; + int result; tor_assert(s); tor_assert(addr_out); s = eat_whitespace(s); - if (*s == '[') { - port = strstr(s, "]"); - if (!port) - goto err; - tmp = tor_strndup(s+1, port-(s+1)); - port = port+1; - if (*port == ':') - port++; - else - port = NULL; - } else { - port = strchr(s, ':'); - if (port) - tmp = tor_strndup(s, port-s); - else - tmp = tor_strdup(s); - if (port) - ++port; + /* Try parsing s as an address:port first, so we don't have to duplicate + * the logic that rejects IPv6:Port with no square brackets. */ + rv = tor_addr_port_parse(LOG_WARN, s, &addr, &portval, 0); + /* That was easy, no DNS required. */ + if (rv == 0) + goto success; + + /* Now let's check for malformed IPv6 addresses and ports: + * tor_addr_port_parse() requires squared brackes if there is a port, + * and we want tor_addr_port_lookup() to have the same requirement. + * But we strip the port using tor_addr_port_split(), so tor_addr_lookup() + * only sees the address, and will accept it without square brackets. */ + int family = tor_addr_parse(&addr, s); + /* If tor_addr_parse() succeeds where tor_addr_port_parse() failed, we need + * to reject this address as malformed. */ + if (family >= 0) { + /* Double-check it's an IPv6 address. If not, we have a parsing bug. + */ + tor_assertf_nonfatal(family == AF_INET6, + "Wrong family: %d (should be IPv6: %d) which " + "failed IP:port parsing, but passed IP parsing. " + "input string: '%s'; parsed address: '%s'.", + family, AF_INET6, s, fmt_addr(&addr)); + goto err; } - if (tor_addr_lookup(tmp, AF_UNSPEC, &addr) != 0) + /* Now we have a hostname. Let's split off the port, if any. */ + rv = tor_addr_port_split(LOG_WARN, s, &tmp, &portval); + if (rv < 0) goto err; - tor_free(tmp); - if (port) { - portval = (int) tor_parse_long(port, 10, 1, 65535, NULL, NULL); - if (!portval) - goto err; - } else { - portval = 0; - } + /* And feed the hostname to the lookup function. */ + if (tor_addr_lookup(tmp, AF_UNSPEC, &addr) != 0) + goto err; + success: if (port_out) *port_out = portval; tor_addr_copy(addr_out, &addr); + result = 0; + goto done; - return 0; err: + /* Clear the address and port on error */ + memset(addr_out, 0, sizeof(tor_addr_t)); + if (port_out) + *port_out = 0; + result = -1; + + /* We have set the result, now it's time to clean up */ + done: tor_free(tmp); - return -1; + return result; } #ifdef USE_SANDBOX_GETADDRINFO @@ -421,7 +503,7 @@ tor_make_getaddrinfo_cache_active(void) { sandbox_getaddrinfo_is_active = 1; } -#else /* !(defined(USE_SANDBOX_GETADDRINFO)) */ +#else /* !defined(USE_SANDBOX_GETADDRINFO) */ void sandbox_disable_getaddrinfo_cache(void) { diff --git a/src/lib/net/resolve.h b/src/lib/net/resolve.h index d11c902a91..b979b2fb41 100644 --- a/src/lib/net/resolve.h +++ b/src/lib/net/resolve.h @@ -24,12 +24,18 @@ struct tor_addr_t; +/* + * Primary lookup functions. + */ MOCK_DECL(int, tor_lookup_hostname,(const char *name, uint32_t *addr)); MOCK_DECL(int, tor_addr_lookup,(const char *name, uint16_t family, struct tor_addr_t *addr_out)); int tor_addr_port_lookup(const char *s, struct tor_addr_t *addr_out, uint16_t *port_out); +/* + * Sandbox helpers + */ struct addrinfo; #ifdef USE_SANDBOX_GETADDRINFO /** Pre-calls getaddrinfo in order to pre-record result. */ @@ -42,7 +48,7 @@ int tor_getaddrinfo(const char *name, const char *servname, struct addrinfo **res); void tor_freeaddrinfo(struct addrinfo *addrinfo); void tor_free_getaddrinfo_cache(void); -#else /* !(defined(USE_SANDBOX_GETADDRINFO)) */ +#else /* !defined(USE_SANDBOX_GETADDRINFO) */ #define tor_getaddrinfo(name, servname, hints, res) \ getaddrinfo((name),(servname), (hints),(res)) #define tor_add_addrinfo(name) \ @@ -55,4 +61,13 @@ void tor_free_getaddrinfo_cache(void); void sandbox_disable_getaddrinfo_cache(void); void tor_make_getaddrinfo_cache_active(void); +/* + * Internal resolver wrapper; exposed for mocking. + */ +#ifdef RESOLVE_PRIVATE +MOCK_DECL(STATIC int, tor_addr_lookup_host_impl, (const char *name, + uint16_t family, + struct tor_addr_t *addr)); +#endif + #endif /* !defined(TOR_RESOLVE_H) */ diff --git a/src/lib/net/socket.c b/src/lib/net/socket.c index e824a05045..e1b82251ed 100644 --- a/src/lib/net/socket.c +++ b/src/lib/net/socket.c @@ -84,7 +84,7 @@ check_network_configuration(bool server_mode) "so your relay makes it harder to figure out how busy it is."); } } -#else /* !(defined(__FreeBSD__)) */ +#else /* !defined(__FreeBSD__) */ (void) server_mode; #endif /* defined(__FreeBSD__) */ } @@ -206,7 +206,7 @@ mark_socket_closed(tor_socket_t s) bitarray_clear(open_sockets, s); } } -#else /* !(defined(DEBUG_SOCKET_COUNTING)) */ +#else /* !defined(DEBUG_SOCKET_COUNTING) */ #define mark_socket_open(s) ((void) (s)) #define mark_socket_closed(s) ((void) (s)) #endif /* defined(DEBUG_SOCKET_COUNTING) */ @@ -308,7 +308,7 @@ tor_open_socket_with_extensions(int domain, int type, int protocol, return TOR_INVALID_SOCKET; } } -#else /* !(defined(FD_CLOEXEC)) */ +#else /* !defined(FD_CLOEXEC) */ (void)cloexec; #endif /* defined(FD_CLOEXEC) */ @@ -418,7 +418,7 @@ tor_accept_socket_with_extensions(tor_socket_t sockfd, struct sockaddr *addr, return TOR_INVALID_SOCKET; } } -#else /* !(defined(FD_CLOEXEC)) */ +#else /* !defined(FD_CLOEXEC) */ (void)cloexec; #endif /* defined(FD_CLOEXEC) */ diff --git a/src/lib/net/socket.h b/src/lib/net/socket.h index 193ad91e4c..53a9f1bb92 100644 --- a/src/lib/net/socket.h +++ b/src/lib/net/socket.h @@ -92,7 +92,7 @@ ssize_t read_all_from_socket(tor_socket_t fd, char *buf, size_t count); #define ERRNO_IS_EINTR(e) ((e) == WSAEINTR || 0) int tor_socket_errno(tor_socket_t sock); const char *tor_socket_strerror(int e); -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ #define SOCK_ERRNO(e) e #if EAGAIN == EWOULDBLOCK /* || 0 is for -Wparentheses-equality (-Wall?) appeasement under clang */ diff --git a/src/lib/net/socketpair.c b/src/lib/net/socketpair.c index 3be7b26f7f..f3a0c3770a 100644 --- a/src/lib/net/socketpair.c +++ b/src/lib/net/socketpair.c @@ -22,7 +22,7 @@ #include <windows.h> #define socket_errno() (WSAGetLastError()) #define SOCKET_EPROTONOSUPPORT WSAEPROTONOSUPPORT -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ #define closesocket(x) close(x) #define socket_errno() (errno) #define SOCKET_EPROTONOSUPPORT EPROTONOSUPPORT diff --git a/src/lib/osinfo/uname.c b/src/lib/osinfo/uname.c index 2b37ff136c..34860c407a 100644 --- a/src/lib/osinfo/uname.c +++ b/src/lib/osinfo/uname.c @@ -137,7 +137,7 @@ get_uname,(void)) if (!is_server && !is_client) { strlcat(uname_result, " [client or server]", sizeof(uname_result)); } -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ /* LCOV_EXCL_START -- can't provoke uname failure */ strlcpy(uname_result, "Unknown platform", sizeof(uname_result)); /* LCOV_EXCL_STOP */ diff --git a/src/lib/process/daemon.c b/src/lib/process/daemon.c index 3b90bef671..ae34b5bcb8 100644 --- a/src/lib/process/daemon.c +++ b/src/lib/process/daemon.c @@ -165,7 +165,7 @@ finish_daemon(const char *desired_cwd) return 0; } -#else /* !(!defined(_WIN32)) */ +#else /* defined(_WIN32) */ /* defined(_WIN32) */ int start_daemon(void) diff --git a/src/lib/process/env.c b/src/lib/process/env.c index 0060200ba1..3912ade197 100644 --- a/src/lib/process/env.c +++ b/src/lib/process/env.c @@ -47,7 +47,7 @@ get_environment(void) * when we do a mostly-static build on OSX 10.7, the resulting binary won't * work on OSX 10.6. */ return *_NSGetEnviron(); -#else /* !(defined(HAVE__NSGETENVIRON)) */ +#else /* !defined(HAVE__NSGETENVIRON) */ return environ; #endif /* defined(HAVE__NSGETENVIRON) */ } diff --git a/src/lib/process/process.c b/src/lib/process/process.c index 631c7169f1..2194a603ff 100644 --- a/src/lib/process/process.c +++ b/src/lib/process/process.c @@ -513,7 +513,7 @@ process_get_unix_process(const process_t *process) tor_assert(process->unix_process); return process->unix_process; } -#else /* !(!defined(_WIN32)) */ +#else /* defined(_WIN32) */ /** Get the internal handle for Windows backend. */ process_win32_t * process_get_win32_process(const process_t *process) diff --git a/src/lib/process/process_unix.c b/src/lib/process/process_unix.c index ef81ec9bf9..8191bdc1f0 100644 --- a/src/lib/process/process_unix.c +++ b/src/lib/process/process_unix.c @@ -199,7 +199,7 @@ process_unix_exec(process_t *process) "Cannot find maximum file descriptor, assuming: %d", max_fd); } } -#else /* !(defined(_SC_OPEN_MAX)) */ +#else /* !defined(_SC_OPEN_MAX) */ max_fd = DEFAULT_MAX_FD; #endif /* defined(_SC_OPEN_MAX) */ diff --git a/src/lib/process/restrict.c b/src/lib/process/restrict.c index 534b39d101..fda284f3d9 100644 --- a/src/lib/process/restrict.c +++ b/src/lib/process/restrict.c @@ -152,7 +152,7 @@ tor_mlockall(void) "pages: %s", strerror(errno)); return -1; } -#else /* !(defined(HAVE_UNIX_MLOCKALL)) */ +#else /* !defined(HAVE_UNIX_MLOCKALL) */ log_warn(LD_GENERAL, "Unable to lock memory pages. mlockall() unsupported?"); return -1; #endif /* defined(HAVE_UNIX_MLOCKALL) */ @@ -214,7 +214,7 @@ set_max_file_descriptors(rlim_t limit, int *max_out) return -1; } limit = MAX_CONNECTIONS; -#else /* !(!defined(HAVE_GETRLIMIT)) */ +#else /* defined(HAVE_GETRLIMIT) */ struct rlimit rlim; if (getrlimit(RLIMIT_NOFILE, &rlim) != 0) { diff --git a/src/lib/process/setuid.c b/src/lib/process/setuid.c index 6e8258f279..3c94ce4bac 100644 --- a/src/lib/process/setuid.c +++ b/src/lib/process/setuid.c @@ -72,7 +72,7 @@ log_credential_status(void) "UID is %u (real), %u (effective), %u (saved)", (unsigned)ruid, (unsigned)euid, (unsigned)suid); } -#else /* !(defined(HAVE_GETRESUID)) */ +#else /* !defined(HAVE_GETRESUID) */ /* getresuid is not present on MacOS X, so we can't get the saved (E)UID */ ruid = getuid(); euid = geteuid(); @@ -93,7 +93,7 @@ log_credential_status(void) "GID is %u (real), %u (effective), %u (saved)", (unsigned)rgid, (unsigned)egid, (unsigned)sgid); } -#else /* !(defined(HAVE_GETRESGID)) */ +#else /* !defined(HAVE_GETRESGID) */ /* getresgid is not present on MacOS X, so we can't get the saved (E)GID */ rgid = getgid(); egid = getegid(); @@ -154,7 +154,7 @@ have_capability_support(void) return 0; cap_free(caps); return 1; -#else /* !(defined(HAVE_LINUX_CAPABILITIES)) */ +#else /* !defined(HAVE_LINUX_CAPABILITIES) */ return 0; #endif /* defined(HAVE_LINUX_CAPABILITIES) */ } @@ -265,7 +265,7 @@ switch_id(const char *user, const unsigned flags) if (drop_capabilities(1)) return -1; } -#else /* !(defined(HAVE_LINUX_CAPABILITIES)) */ +#else /* !defined(HAVE_LINUX_CAPABILITIES) */ (void) keep_bindlow; if (warn_if_no_caps) { log_warn(LD_CONFIG, "KeepBindCapabilities set, but no capability support " @@ -376,7 +376,7 @@ switch_id(const char *user, const unsigned flags) #endif /* defined(__linux__) && defined(HAVE_SYS_PRCTL_H) && ... */ return 0; -#else /* !(!defined(_WIN32)) */ +#else /* defined(_WIN32) */ (void)user; (void)flags; diff --git a/src/lib/process/winprocess_sys.c b/src/lib/process/winprocess_sys.c index 48c0888658..ad65886422 100644 --- a/src/lib/process/winprocess_sys.c +++ b/src/lib/process/winprocess_sys.c @@ -51,13 +51,15 @@ subsys_winprocess_initialize(void) return 0; } -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ #define WINPROCESS_SYS_ENABLED false #define subsys_winprocess_initialize NULL #endif /* defined(_WIN32) */ const subsys_fns_t sys_winprocess = { .name = "winprocess", + /* HeapEnableTerminationOnCorruption and setdeppolicy() are security + * features, we want them to run first. */ .level = -100, .supported = WINPROCESS_SYS_ENABLED, .initialize = subsys_winprocess_initialize, diff --git a/src/lib/pubsub/pubsub_check.c b/src/lib/pubsub/pubsub_check.c index a3c22d4f25..bf1196df2c 100644 --- a/src/lib/pubsub/pubsub_check.c +++ b/src/lib/pubsub/pubsub_check.c @@ -172,34 +172,20 @@ pubsub_cfg_dump(const pubsub_cfg_t *cfg, int severity, const char *prefix) /** * Helper: fill a bitarray <b>out</b> with entries corresponding to the - * subsystems listed in <b>items</b>. If any subsystem is listed more than - * once, log a warning. Return 0 on success, -1 on failure. + * subsystems listed in <b>items</b>. **/ -static int +static void get_message_bitarray(const pubsub_adjmap_t *map, - message_id_t msg, const smartlist_t *items, - const char *operation, bitarray_t **out) { - bool ok = true; *out = bitarray_init_zero((unsigned)map->n_subsystems); if (! items) - return 0; + return; SMARTLIST_FOREACH_BEGIN(items, const pubsub_cfg_t *, cfg) { - if (bitarray_is_set(*out, cfg->subsys)) { - log_warn(LD_MESG|LD_BUG, - "Message \"%s\" is configured to be %s by subsystem " - "\"%s\" more than once.", - get_message_id_name(msg), operation, - get_subsys_id_name(cfg->subsys)); - ok = false; - } bitarray_set(*out, cfg->subsys); } SMARTLIST_FOREACH_END(cfg); - - return ok ? 0 : -1; } /** @@ -222,10 +208,8 @@ lint_message_graph(const pubsub_adjmap_t *map, bitarray_t *subscribed_by = NULL; bool ok = true; - if (get_message_bitarray(map, msg, pub, "published", &published_by) < 0) - ok = false; - if (get_message_bitarray(map, msg, sub, "subscribed", &subscribed_by) < 0) - ok = false; + get_message_bitarray(map, pub, &published_by); + get_message_bitarray(map, sub, &subscribed_by); /* Check whether any subsystem is publishing and subscribing the same * message. [??] diff --git a/src/lib/sandbox/sandbox.c b/src/lib/sandbox/sandbox.c index de37989fad..3515e8854e 100644 --- a/src/lib/sandbox/sandbox.c +++ b/src/lib/sandbox/sandbox.c @@ -445,7 +445,7 @@ libc_uses_openat_for_everything(void) return 1; else return 0; -#else /* !(defined(CHECK_LIBC_VERSION)) */ +#else /* !defined(CHECK_LIBC_VERSION) */ return 0; #endif /* defined(CHECK_LIBC_VERSION) */ } diff --git a/src/lib/sandbox/sandbox.h b/src/lib/sandbox/sandbox.h index 5bec09a36a..b4ae6e5c07 100644 --- a/src/lib/sandbox/sandbox.h +++ b/src/lib/sandbox/sandbox.h @@ -108,7 +108,7 @@ typedef struct { * it matches the parameter. */ const char* sandbox_intern_string(const char *param); -#else /* !(defined(USE_LIBSECCOMP)) */ +#else /* !defined(USE_LIBSECCOMP) */ #define sandbox_intern_string(s) (s) #endif /* defined(USE_LIBSECCOMP) */ diff --git a/src/lib/smartlist_core/smartlist_core.h b/src/lib/smartlist_core/smartlist_core.h index 795741c447..36f23e2009 100644 --- a/src/lib/smartlist_core/smartlist_core.h +++ b/src/lib/smartlist_core/smartlist_core.h @@ -77,7 +77,7 @@ static inline void smartlist_set(smartlist_t *sl, int idx, void *val) { raw_assert(sl->num_used > idx); sl->list[idx] = val; } -#else /* !(defined(DEBUG_SMARTLIST)) */ +#else /* !defined(DEBUG_SMARTLIST) */ #define smartlist_len(sl) ((sl)->num_used) #define smartlist_get(sl, idx) ((sl)->list[idx]) #define smartlist_set(sl, idx, val) ((sl)->list[idx] = (val)) diff --git a/src/lib/string/compat_string.h b/src/lib/string/compat_string.h index 4f30bf5392..ffc892c3e5 100644 --- a/src/lib/string/compat_string.h +++ b/src/lib/string/compat_string.h @@ -39,6 +39,9 @@ static inline int strcasecmp(const char *a, const char *b) { * appear to have a severe bug that can sometimes cause aborts in Tor. * Instead, use the non-checking variants. This is sad. * + * (If --enable-fragile-hardening is passed to configure, we use the hardened + * variants, which do not suffer from this issue.) + * * See https://trac.torproject.org/projects/tor/ticket/15205 */ #undef strlcat diff --git a/src/lib/testsupport/testsupport.h b/src/lib/testsupport/testsupport.h index 631ec0228c..90b7c43b19 100644 --- a/src/lib/testsupport/testsupport.h +++ b/src/lib/testsupport/testsupport.h @@ -21,7 +21,7 @@ * tests. */ #define STATIC #define EXTERN(type, name) extern type name; -#else /* !(defined(TOR_UNIT_TESTS)) */ +#else /* !defined(TOR_UNIT_TESTS) */ #define STATIC static #define EXTERN(type, name) #endif /* defined(TOR_UNIT_TESTS) */ @@ -90,7 +90,7 @@ do { \ func = func ##__real; \ } while (0) -#else /* !(defined(TOR_UNIT_TESTS)) */ +#else /* !defined(TOR_UNIT_TESTS) */ #define MOCK_DECL(rv, funcname, arglist) \ rv funcname arglist #define MOCK_DECL_ATTR(rv, funcname, arglist, attr) \ diff --git a/src/lib/thread/compat_threads.c b/src/lib/thread/compat_threads.c index 8cf1967e53..5c8ffa55c6 100644 --- a/src/lib/thread/compat_threads.c +++ b/src/lib/thread/compat_threads.c @@ -130,6 +130,8 @@ subsys_threads_initialize(void) const subsys_fns_t sys_threads = { .name = "threads", .supported = true, + /* Threads is used by logging, which is a diagnostic feature, we want it to + * init right after low-level error handling and approx time. */ .level = -95, .initialize = subsys_threads_initialize, }; diff --git a/src/lib/thread/threads.h b/src/lib/thread/threads.h index de3da6a585..4b42b9abd9 100644 --- a/src/lib/thread/threads.h +++ b/src/lib/thread/threads.h @@ -107,7 +107,7 @@ typedef struct atomic_counter_t { atomic_size_t val; } atomic_counter_t; #define ATOMIC_LINKAGE static -#else /* !(defined(HAVE_WORKING_STDATOMIC)) */ +#else /* !defined(HAVE_WORKING_STDATOMIC) */ typedef struct atomic_counter_t { tor_mutex_t mutex; size_t val; @@ -172,7 +172,7 @@ atomic_counter_exchange(atomic_counter_t *counter, size_t newval) return atomic_exchange(&counter->val, newval); } -#else /* !(defined(HAVE_WORKING_STDATOMIC)) */ +#else /* !defined(HAVE_WORKING_STDATOMIC) */ #endif /* defined(HAVE_WORKING_STDATOMIC) */ #endif /* !defined(TOR_COMPAT_THREADS_H) */ diff --git a/src/lib/time/compat_time.c b/src/lib/time/compat_time.c index 3f41500f3a..ab45224a7f 100644 --- a/src/lib/time/compat_time.c +++ b/src/lib/time/compat_time.c @@ -833,7 +833,7 @@ monotime_coarse_absolute_msec(void) { return monotime_coarse_absolute_nsec() / ONE_MILLION; } -#else /* !(defined(MONOTIME_COARSE_FN_IS_DIFFERENT)) */ +#else /* !defined(MONOTIME_COARSE_FN_IS_DIFFERENT) */ #define initialized_at_coarse initialized_at #endif /* defined(MONOTIME_COARSE_FN_IS_DIFFERENT) */ @@ -865,7 +865,7 @@ monotime_msec_to_approx_coarse_stamp_units(uint64_t msec) mach_time_info.numer; return abstime_val >> monotime_shift; } -#else /* !(defined(__APPLE__)) */ +#else /* !defined(__APPLE__) */ uint64_t monotime_coarse_stamp_units_to_approx_msec(uint64_t units) { diff --git a/src/lib/time/compat_time.h b/src/lib/time/compat_time.h index 8c7661d7cb..4d16effd29 100644 --- a/src/lib/time/compat_time.h +++ b/src/lib/time/compat_time.h @@ -259,7 +259,7 @@ void monotime_coarse_get(monotime_coarse_t *out); uint64_t monotime_coarse_absolute_nsec(void); uint64_t monotime_coarse_absolute_usec(void); uint64_t monotime_coarse_absolute_msec(void); -#else /* !(defined(MONOTIME_COARSE_FN_IS_DIFFERENT)) */ +#else /* !defined(MONOTIME_COARSE_FN_IS_DIFFERENT) */ #define monotime_coarse_get monotime_get #define monotime_coarse_absolute_nsec monotime_absolute_nsec #define monotime_coarse_absolute_usec monotime_absolute_usec @@ -304,7 +304,7 @@ void monotime_coarse_zero(monotime_coarse_t *out); int monotime_coarse_is_zero(const monotime_coarse_t *val); void monotime_coarse_add_msec(monotime_coarse_t *out, const monotime_coarse_t *val, uint32_t msec); -#else /* !(defined(MONOTIME_COARSE_TYPE_IS_DIFFERENT)) */ +#else /* !defined(MONOTIME_COARSE_TYPE_IS_DIFFERENT) */ #define monotime_coarse_diff_nsec monotime_diff_nsec #define monotime_coarse_diff_usec monotime_diff_usec #define monotime_coarse_diff_msec monotime_diff_msec diff --git a/src/lib/time/time_sys.c b/src/lib/time/time_sys.c index b3feb7b46a..8b9aa2856c 100644 --- a/src/lib/time/time_sys.c +++ b/src/lib/time/time_sys.c @@ -20,7 +20,9 @@ subsys_time_initialize(void) const subsys_fns_t sys_time = { .name = "time", - .level = -90, + /* Monotonic time depends on logging, and a lot of other modules depend on + * monotonic time. */ + .level = -80, .supported = true, .initialize = subsys_time_initialize, }; diff --git a/src/lib/tls/tortls.h b/src/lib/tls/tortls.h index 9e195c6af2..799bd6aaeb 100644 --- a/src/lib/tls/tortls.h +++ b/src/lib/tls/tortls.h @@ -25,7 +25,7 @@ struct ssl_ctx_st; struct ssl_session_st; typedef struct ssl_ctx_st tor_tls_context_impl_t; typedef struct ssl_st tor_tls_impl_t; -#else /* !(defined(ENABLE_OPENSSL)) */ +#else /* !defined(ENABLE_OPENSSL) */ struct PRFileDesc; typedef struct PRFileDesc tor_tls_context_impl_t; typedef struct PRFileDesc tor_tls_impl_t; @@ -144,7 +144,7 @@ void check_no_tls_errors_(const char *fname, int line); void tor_tls_log_one_error(tor_tls_t *tls, unsigned long err, int severity, int domain, const char *doing); -#else /* !(defined(ENABLE_OPENSSL)) */ +#else /* !defined(ENABLE_OPENSSL) */ #define check_no_tls_errors() STMT_NIL #endif /* defined(ENABLE_OPENSSL) */ diff --git a/src/lib/tls/tortls_openssl.c b/src/lib/tls/tortls_openssl.c index 86f0ac42cc..5bafcf676d 100644 --- a/src/lib/tls/tortls_openssl.c +++ b/src/lib/tls/tortls_openssl.c @@ -657,7 +657,7 @@ tor_tls_context_new(crypto_pk_t *identity, unsigned int key_lifetime, if (r < 0) goto error; } -#else /* !(defined(SSL_CTX_set1_groups_list) || ...) */ +#else /* !(defined(SSL_CTX_set1_groups_list) || defined(HAVE_SSL_CTX_SE...)) */ if (! is_client) { int nid; EC_KEY *ec_key; @@ -673,7 +673,7 @@ tor_tls_context_new(crypto_pk_t *identity, unsigned int key_lifetime, SSL_CTX_set_tmp_ecdh(result->ctx, ec_key); EC_KEY_free(ec_key); } -#endif /* defined(SSL_CTX_set1_groups_list) || ...) */ +#endif /* defined(SSL_CTX_set1_groups_list) || defined(HAVE_SSL_CTX_SET1...) */ SSL_CTX_set_verify(result->ctx, SSL_VERIFY_PEER, always_accept_verify_cb); /* let us realloc bufs that we're writing from */ @@ -764,7 +764,7 @@ find_cipher_by_id(const SSL *ssl, const SSL_METHOD *m, uint16_t cipher) tor_assert((SSL_CIPHER_get_id(c) & 0xffff) == cipher); return c != NULL; } -#else /* !(defined(HAVE_SSL_CIPHER_FIND)) */ +#else /* !defined(HAVE_SSL_CIPHER_FIND) */ # if defined(HAVE_STRUCT_SSL_METHOD_ST_GET_CIPHER_BY_CHAR) if (m && m->get_cipher_by_char) { diff --git a/src/lib/tls/x509_openssl.c b/src/lib/tls/x509_openssl.c index 03f65049cf..7724288279 100644 --- a/src/lib/tls/x509_openssl.c +++ b/src/lib/tls/x509_openssl.c @@ -59,7 +59,7 @@ ENABLE_GCC_WARNING(redundant-decls) #define X509_get_notAfter(cert) \ X509_getm_notAfter(cert) #endif -#else /* !(defined(OPENSSL_1_1_API)) */ +#else /* !defined(OPENSSL_1_1_API) */ #define X509_get_notBefore_const(cert) \ ((const ASN1_TIME*) X509_get_notBefore((X509 *)cert)) #define X509_get_notAfter_const(cert) \ diff --git a/src/lib/trace/events.h b/src/lib/trace/events.h index 0674f7d501..9de86d63f2 100644 --- a/src/lib/trace/events.h +++ b/src/lib/trace/events.h @@ -34,7 +34,7 @@ #include "lib/trace/debug.h" #endif -#else /* !(defined(TOR_EVENT_TRACING_ENABLED)) */ +#else /* !defined(TOR_EVENT_TRACING_ENABLED) */ /* Reaching this point, we NOP every event declaration because event tracing * is not been enabled at compile time. */ diff --git a/src/lib/wallclock/approx_time.c b/src/lib/wallclock/approx_time.c index 7b32804026..77eeddaf56 100644 --- a/src/lib/wallclock/approx_time.c +++ b/src/lib/wallclock/approx_time.c @@ -54,6 +54,8 @@ subsys_wallclock_initialize(void) const subsys_fns_t sys_wallclock = { .name = "wallclock", .supported = true, - .level = -99, + /* Approximate time is a diagnostic feature, we want it to init right after + * low-level error handling. */ + .level = -98, .initialize = subsys_wallclock_initialize, }; diff --git a/src/test/conf_examples/badnick_1/error b/src/test/conf_examples/badnick_1/error new file mode 100644 index 0000000000..3e92ddc832 --- /dev/null +++ b/src/test/conf_examples/badnick_1/error @@ -0,0 +1 @@ +nicknames must be between 1 and 19 characters inclusive diff --git a/src/test/conf_examples/badnick_1/torrc b/src/test/conf_examples/badnick_1/torrc new file mode 100644 index 0000000000..41ee4894f1 --- /dev/null +++ b/src/test/conf_examples/badnick_1/torrc @@ -0,0 +1,2 @@ +# This nickname is too long; we won't accept it. +Nickname TooManyCharactersInThisNickname diff --git a/src/test/conf_examples/badnick_2/error b/src/test/conf_examples/badnick_2/error new file mode 100644 index 0000000000..ceac99f012 --- /dev/null +++ b/src/test/conf_examples/badnick_2/error @@ -0,0 +1 @@ +must contain only the characters \[a-zA-Z0-9\] diff --git a/src/test/conf_examples/badnick_2/torrc b/src/test/conf_examples/badnick_2/torrc new file mode 100644 index 0000000000..07acc61698 --- /dev/null +++ b/src/test/conf_examples/badnick_2/torrc @@ -0,0 +1,2 @@ +# this nickname has spaces in it and won't work. +Nickname has a space diff --git a/src/test/conf_examples/contactinfo_notutf8/error b/src/test/conf_examples/contactinfo_notutf8/error new file mode 100644 index 0000000000..6d165152ce --- /dev/null +++ b/src/test/conf_examples/contactinfo_notutf8/error @@ -0,0 +1 @@ +ContactInfo config option must be UTF-8 diff --git a/src/test/conf_examples/contactinfo_notutf8/torrc b/src/test/conf_examples/contactinfo_notutf8/torrc new file mode 100644 index 0000000000..2ee4d093c5 --- /dev/null +++ b/src/test/conf_examples/contactinfo_notutf8/torrc @@ -0,0 +1 @@ +ContactInfo ÄëÄëÄë@example.com diff --git a/src/test/conf_examples/example_1/expected b/src/test/conf_examples/example_1/expected new file mode 100644 index 0000000000..9d6688a565 --- /dev/null +++ b/src/test/conf_examples/example_1/expected @@ -0,0 +1,2 @@ +ContactInfo tor_tellini@example.com +SocksPort 80 diff --git a/src/test/conf_examples/example_1/torrc b/src/test/conf_examples/example_1/torrc new file mode 100644 index 0000000000..bff7fa0aa2 --- /dev/null +++ b/src/test/conf_examples/example_1/torrc @@ -0,0 +1,5 @@ + +# Here is a simple example torrc. + SocksPort 80 + +ContactInfo "tor_tellini@example.com" diff --git a/src/test/conf_examples/example_2/error b/src/test/conf_examples/example_2/error new file mode 100644 index 0000000000..ce18b68db4 --- /dev/null +++ b/src/test/conf_examples/example_2/error @@ -0,0 +1 @@ +Unknown option 'JumpingJellyjars' diff --git a/src/test/conf_examples/example_2/torrc b/src/test/conf_examples/example_2/torrc new file mode 100644 index 0000000000..8ec8133b24 --- /dev/null +++ b/src/test/conf_examples/example_2/torrc @@ -0,0 +1 @@ +JumpingJellyjars 1 diff --git a/src/test/conf_examples/example_3/cmdline b/src/test/conf_examples/example_3/cmdline new file mode 100644 index 0000000000..5b2fadcebb --- /dev/null +++ b/src/test/conf_examples/example_3/cmdline @@ -0,0 +1 @@ +--socksport 99 diff --git a/src/test/conf_examples/example_3/expected b/src/test/conf_examples/example_3/expected new file mode 100644 index 0000000000..867fb8bcc8 --- /dev/null +++ b/src/test/conf_examples/example_3/expected @@ -0,0 +1 @@ +SocksPort 99 diff --git a/src/test/conf_examples/example_3/torrc b/src/test/conf_examples/example_3/torrc new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/src/test/conf_examples/example_3/torrc diff --git a/src/test/conf_examples/include_1/expected b/src/test/conf_examples/include_1/expected new file mode 100644 index 0000000000..4bbf52ce9f --- /dev/null +++ b/src/test/conf_examples/include_1/expected @@ -0,0 +1,3 @@ +ContactInfo includefile@example.com +Nickname nested +ORPort 8008 diff --git a/src/test/conf_examples/include_1/included.inc b/src/test/conf_examples/include_1/included.inc new file mode 100644 index 0000000000..8d1834345d --- /dev/null +++ b/src/test/conf_examples/include_1/included.inc @@ -0,0 +1,4 @@ + +ContactInfo includefile@example.com + +%include "nested.inc"
\ No newline at end of file diff --git a/src/test/conf_examples/include_1/nested.inc b/src/test/conf_examples/include_1/nested.inc new file mode 100644 index 0000000000..789b044a2b --- /dev/null +++ b/src/test/conf_examples/include_1/nested.inc @@ -0,0 +1,2 @@ + +Nickname nested
\ No newline at end of file diff --git a/src/test/conf_examples/include_1/torrc b/src/test/conf_examples/include_1/torrc new file mode 100644 index 0000000000..2ed4074f6e --- /dev/null +++ b/src/test/conf_examples/include_1/torrc @@ -0,0 +1,4 @@ + +%include "included.inc" + +ORPort 8008 diff --git a/src/test/conf_examples/include_bug_31408/expected b/src/test/conf_examples/include_bug_31408/expected new file mode 100644 index 0000000000..2e822f1a78 --- /dev/null +++ b/src/test/conf_examples/include_bug_31408/expected @@ -0,0 +1,2 @@ +Nickname test31408 +ORPort 31408 diff --git a/src/test/conf_examples/include_bug_31408/included/01_nickname.inc b/src/test/conf_examples/include_bug_31408/included/01_nickname.inc new file mode 100644 index 0000000000..508dd89a35 --- /dev/null +++ b/src/test/conf_examples/include_bug_31408/included/01_nickname.inc @@ -0,0 +1 @@ +Nickname test31408 diff --git a/src/test/conf_examples/include_bug_31408/included/02_no_configs.inc b/src/test/conf_examples/include_bug_31408/included/02_no_configs.inc new file mode 100644 index 0000000000..140e927f19 --- /dev/null +++ b/src/test/conf_examples/include_bug_31408/included/02_no_configs.inc @@ -0,0 +1,3 @@ +# Bug 31048 is triggered when the last file in a config directory: +# * contains no configuration options, +# * but is non-empty: that is, it contains comments or whitespace. diff --git a/src/test/conf_examples/include_bug_31408/torrc b/src/test/conf_examples/include_bug_31408/torrc new file mode 100644 index 0000000000..a42685e93c --- /dev/null +++ b/src/test/conf_examples/include_bug_31408/torrc @@ -0,0 +1,2 @@ +%include "included" +ORPort 31408 diff --git a/src/test/conf_examples/large_1/expected b/src/test/conf_examples/large_1/expected new file mode 100644 index 0000000000..5866f5823e --- /dev/null +++ b/src/test/conf_examples/large_1/expected @@ -0,0 +1,159 @@ +AccountingMax 10737418240 +AccountingRule sum +AccountingStart day 05:15 +Address 128.66.8.8 +AllowNonRFC953Hostnames 1 +AndroidIdentityTag droidy +AutomapHostsOnResolve 1 +AutomapHostsSuffixes .onions +AvoidDiskWrites 1 +BandwidthBurst 2147483647 +BandwidthRate 1610612736 +Bridge 128.66.1.10:80 +CacheDirectory /this-is-a-cache +CellStatistics 1 +CircuitBuildTimeout 200 +CircuitsAvailableTimeout 10 +CircuitStreamTimeout 20 +ClientAutoIPv6ORPort 1 +ClientOnly 1 +ClientPreferIPv6DirPort 1 +ClientPreferIPv6ORPort 1 +ClientRejectInternalAddresses 0 +ClientUseIPv4 0 +ClientUseIPv6 1 +ConnDirectionStatistics 1 +ConnectionPadding 1 +ConnLimit 64 +ConsensusParams wombat=7 +ConstrainedSockets 1 +ConstrainedSockSize 10240 +ContactInfo long_config@example.com +ControlPortFileGroupReadable 1 +ControlPort 9058 +CookieAuthentication 1 +CookieAuthFile /control/cookie +CookieAuthFileGroupReadable 1 +CountPrivateBandwidth 1 +DataDirectory /data/dir +DirAllowPrivateAddresses 1 +DirPolicy reject 128.66.1.1/32, accept *:* +DirPortFrontPage /dirport/frontpage +DirPort 99 +DirReqStatistics 0 +DisableDebuggerAttachment 0 +DisableNetwork 1 +DisableOOSCheck 0 +DNSPort 53535 +DormantCanceledByStartup 1 +DormantClientTimeout 1260 +DormantOnFirstStartup 1 +DormantTimeoutDisabledByIdleStreams 0 +DoSCircuitCreationBurst 1000 +DoSCircuitCreationDefenseTimePeriod 300 +DoSCircuitCreationDefenseType 2 +DoSCircuitCreationEnabled 1 +DoSCircuitCreationMinConnections 10 +DoSCircuitCreationRate 100 +DoSConnectionDefenseType 2 +DoSConnectionEnabled 1 +DoSConnectionMaxConcurrentCount 6 +DoSRefuseSingleHopClientRendezvous 0 +DownloadExtraInfo 1 +EnforceDistinctSubnets 0 +EntryNodes potrzebie,triffid,cromulent +EntryStatistics 1 +ExcludeExitNodes blaznort,kriffid,zeppelin +ExcludeNodes 128.66.7.6 +ExitNodes 128.66.7.7,128.66.128.0/17,exitexit +ExitPolicy accept *:80,reject *:* +ExitPolicyRejectLocalInterfaces 1 +ExitPolicyRejectPrivate 0 +ExitPortStatistics 1 +ExitRelay 1 +ExtendAllowPrivateAddresses 1 +ExtendByEd25519ID 1 +ExtORPortCookieAuthFile /foobar +ExtORPort 99 +FascistFirewall 1 +FetchDirInfoEarly 1 +FetchDirInfoExtraEarly 1 +FetchUselessDescriptors 1 +FirewallPorts 80,443,999 +GeoIPExcludeUnknown 1 +GeoIPFile /geoip +GuardfractionFile /gff +GuardLifetime 691200 +HeartbeatPeriod 2700 +IPv6Exit 1 +KeepalivePeriod 540 +KeyDirectory /keyz +KISTSchedRunInterval 1 +Log notice file /logfile +Log info file /logfile-verbose +LogTimeGranularity 60000 +LongLivedPorts 9090 +MainloopStats 1 +MapAddress www.example.com:10.0.0.6 +MaxAdvertisedBandwidth 100 +MaxCircuitDirtiness 3600 +MaxClientCircuitsPending 127 +MaxConsensusAgeForDiffs 2629728 +MaxMemInQueues 314572800 +MaxOnionQueueDelay 60000 +MaxUnparseableDescSizeToLog 1048576 +MiddleNodes grommit,truffle,parcheesi +MyFamily $ffffffffffffffffffffffffffffffffffffffff +NewCircuitPeriod 7200 +Nickname nickname +NodeFamily $ffffffffffffffffffffffffffffffffffffffff,$dddddddddddddddddddddddddddddddddddddddd +NumCPUs 3 +NumDirectoryGuards 4 +NumEntryGuards 5 +NumPrimaryGuards 8 +OfflineMasterKey 1 +OptimisticData 1 +ORPort 2222 +OutboundBindAddress 10.0.0.7 +OutboundBindAddressExit 10.0.0.8 +OutboundBindAddressOR 10.0.0.9 +PerConnBWBurst 10485760 +PerConnBWRate 102400 +PidFile /piddy +ProtocolWarnings 1 +PublishHidServDescriptors 0 +PublishServerDescriptor 0 +ReachableAddresses 0.0.0.0, *:* +ReachableDirAddresses 128.0.0.0/1 +ReachableORAddresses 128.0.0.0/8 +RejectPlaintextPorts 23 +RelayBandwidthBurst 10000 +RelayBandwidthRate 1000 +RendPostPeriod 600 +RephistTrackTime 600 +SafeLogging 0 +Schedulers Vanilla,KISTLite,Kist +ShutdownWaitLength 10 +SigningKeyLifetime 4838400 +Socks5Proxy 128.66.99.99:99 +Socks5ProxyPassword flynn +Socks5ProxyUsername spaceparanoids +SocksPolicy accept 127.0.0.0/24, reject *:* +SocksPort 9099 +SocksTimeout 600 +SSLKeyLifetime 86400 +StrictNodes 1 +SyslogIdentityTag tortor +TestSocks 1 +TokenBucketRefillInterval 1000 +TrackHostExits www.example.com +TrackHostExitsExpire 3600 +TruncateLogFile 1 +UnixSocksGroupWritable 1 +UpdateBridgesFromAuthority 1 +UseDefaultFallbackDirs 0 +UseGuardFraction 1 +UseMicrodescriptors 0 +VirtualAddrNetworkIPv4 18.66.0.0/16 +VirtualAddrNetworkIPv6 [ff00::]/16 +WarnPlaintextPorts 7,11,23,1001 diff --git a/src/test/conf_examples/large_1/torrc b/src/test/conf_examples/large_1/torrc new file mode 100644 index 0000000000..e99acd9fb7 --- /dev/null +++ b/src/test/conf_examples/large_1/torrc @@ -0,0 +1,167 @@ +AccountingMax 10 GB +AccountingRule sum +AccountingStart day 05:15 +Address 128.66.8.8 +AllowNonRFC953Hostnames 1 +AndroidIdentityTag droidy +AutomapHostsOnResolve 1 +AutomapHostsSuffixes .onions +AvoidDiskWrites 1 +BandwidthBurst 2 GB +BandwidthRate 1.5 GB +Bridge 128.66.1.10:80 +CacheDirectory /this-is-a-cache +CellStatistics 1 +CircuitBuildTimeout 200 +CircuitPadding 1 +CircuitsAvailableTimeout 10 +CircuitStreamTimeout 20 +ClientAutoIPv6ORPort 1 +ClientOnly 1 +ClientPreferIPv6DirPort 1 +ClientPreferIPv6ORPort 1 +ClientRejectInternalAddresses 0 +ClientUseIPv4 0 +ClientUseIPv6 1 +ConnDirectionStatistics 1 +ConnectionPadding 1 +ConnLimit 64 +ConsensusParams wombat=7 +ConstrainedSockets 1 +ConstrainedSockSize 10240 +ContactInfo long_config@example.com +ControlPortFileGroupReadable 1 +ControlPort 9058 +CookieAuthentication 1 +CookieAuthFile /control/cookie +CookieAuthFileGroupReadable 1 +CountPrivateBandwidth 1 +DataDirectory /data/dir +DirAllowPrivateAddresses 1 +DirPolicy reject 128.66.1.1/32, accept *:* +DirReqStatistics 0 +DirPort 99 +DirPortFrontPage /dirport/frontpage +DisableDebuggerAttachment 0 +DisableNetwork 1 +DisableOOSCheck 0 +DNSPort 53535 +DormantCanceledByStartup 1 +DormantClientTimeout 21 minutes +DormantOnFirstStartup 1 +DormantTimeoutDisabledByIdleStreams 0 +DoSCircuitCreationBurst 1000 +DoSCircuitCreationDefenseTimePeriod 5 minutes +DoSCircuitCreationDefenseType 2 +DoSCircuitCreationEnabled 1 +DoSCircuitCreationMinConnections 10 +DoSCircuitCreationRate 100 +DoSConnectionDefenseType 2 +DoSConnectionEnabled 1 +DoSConnectionMaxConcurrentCount 6 +DoSRefuseSingleHopClientRendezvous 0 +DownloadExtraInfo 1 +EnforceDistinctSubnets 0 +EntryNodes potrzebie,triffid,cromulent +EntryStatistics 1 +ExcludeExitNodes blaznort,kriffid,zeppelin +ExcludeNodes 128.66.7.6 +ExitNodes 128.66.7.7,128.66.128.0/17,exitexit +ExitPolicy accept *:80,reject *:* +ExitPolicyRejectLocalInterfaces 1 +ExitPolicyRejectPrivate 0 +ExitPortStatistics 1 +ExitRelay 1 +ExtendAllowPrivateAddresses 1 +ExtendByEd25519ID 1 +ExtORPort 99 +ExtORPortCookieAuthFile /foobar +ExtraInfoStatistics 1 +FascistFirewall 1 +FetchDirInfoEarly 1 +FetchDirInfoExtraEarly 1 +FetchHidServDescriptors 1 +FetchServerDescriptors 1 +FetchUselessDescriptors 1 +FirewallPorts 80,443,999 +GeoIPExcludeUnknown 1 +GeoIPFile /geoip +GuardfractionFile /gff +GuardLifetime 8 days +HeartbeatPeriod 45 minutes +IPv6Exit 1 +KeepalivePeriod 9 minutes +KeyDirectory /keyz +KISTSchedRunInterval 1 msec +LearnCircuitBuildTimeout 1 +Log notice file /logfile +Log info file /logfile-verbose +LogTimeGranularity 1 minute +LongLivedPorts 9090 +MainloopStats 1 +MapAddress www.example.com:10.0.0.6 +MaxAdvertisedBandwidth 100 +MaxCircuitDirtiness 1 hour +MaxClientCircuitsPending 127 +MaxConsensusAgeForDiffs 1 month +MaxMemInQueues 300 MB +MaxOnionQueueDelay 60 seconds +MaxUnparseableDescSizeToLog 1 MB +MiddleNodes grommit, truffle, parcheesi +MyFamily $ffffffffffffffffffffffffffffffffffffffff +NewCircuitPeriod 2 hours +Nickname nickname +NodeFamily $ffffffffffffffffffffffffffffffffffffffff,$dddddddddddddddddddddddddddddddddddddddd +NumCPUs 3 +NumDirectoryGuards 4 +NumEntryGuards 5 +NumPrimaryGuards 8 +OfflineMasterKey 1 +OptimisticData 1 +ORPort 2222 +OutboundBindAddress 10.0.0.7 +OutboundBindAddressExit 10.0.0.8 +OutboundBindAddressOR 10.0.0.9 +PaddingStatistics 1 +PerConnBWBurst 10 MB +PerConnBWRate 100 kb +PidFile /piddy +ProtocolWarnings 1 +PublishHidServDescriptors 0 +PublishServerDescriptor 0 +ReachableAddresses 0.0.0.0, *:* +ReachableDirAddresses 128.0.0.0/1 +ReachableORAddresses 128.0.0.0/8 +RejectPlaintextPorts 23 +RelayBandwidthBurst 10000 +RelayBandwidthRate 1000 +RendPostPeriod 10 minutes +RephistTrackTime 10 minutes +SafeLogging 0 +SafeSocks 0 +Schedulers Vanilla,KISTLite,Kist +ShutdownWaitLength 10 seconds +SigningKeyLifetime 8 weeks +Socks5Proxy 128.66.99.99:99 +Socks5ProxyPassword flynn +Socks5ProxyUsername spaceparanoids +SocksPolicy accept 127.0.0.0/24, reject *:* +SocksPort 9099 +SocksTimeout 10 minutes +SSLKeyLifetime 1 day +StrictNodes 1 +SyslogIdentityTag tortor +TestSocks 1 +TokenBucketRefillInterval 1 second +TrackHostExits www.example.com +TrackHostExitsExpire 1 hour +TruncateLogFile 1 +UnixSocksGroupWritable 1 +UpdateBridgesFromAuthority 1 +UseDefaultFallbackDirs 0 +UseEntryGuards 1 +UseGuardFraction 1 +UseMicrodescriptors 0 +VirtualAddrNetworkIPv4 18.66.0.0/16 +VirtualAddrNetworkIPv6 [ff00::]/16 +WarnPlaintextPorts 7,11,23,1001 diff --git a/src/test/conf_examples/obsolete_1/expected b/src/test/conf_examples/obsolete_1/expected new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/src/test/conf_examples/obsolete_1/expected diff --git a/src/test/conf_examples/obsolete_1/torrc b/src/test/conf_examples/obsolete_1/torrc new file mode 100644 index 0000000000..3cd9a6d777 --- /dev/null +++ b/src/test/conf_examples/obsolete_1/torrc @@ -0,0 +1,68 @@ +# These options are obsolete as of 0.4.2 +AllowDotExit +AllowInvalidNodes +AllowSingleHopCircuits +AllowSingleHopExits +AlternateHSAuthority +AuthDirBadDir +AuthDirBadDirCCs +AuthDirRejectUnlisted +AuthDirListBadDirs +AuthDirMaxServersPerAuthAddr +CircuitIdleTimeout +ControlListenAddress +DirListenAddress +DisableIOCP +DisableV2DirectoryInfo_ +DynamicDHGroups +DNSListenAddress +TestingEnableTbEmptyEvent +ExcludeSingleHopRelays +FallbackNetworkstatusFile +FastFirstHopPK +FetchV2Networkstatus +Group +HidServDirectoryV2 +CloseHSClientCircuitsImmediatelyOnTimeout +CloseHSServiceRendCircuitsImmediatelyOnTimeout +MaxOnionsPending +NamingAuthoritativeDirectory +NATDListenAddress +PredictedPortsRelevanceTime +WarnUnsafeSocks +ORListenAddress +PathBiasDisableRate +PathBiasScaleFactor +PathBiasMultFactor +PathBiasUseCloseCounts +PortForwarding +PortForwardingHelper +PreferTunneledDirConns +RecommendedPackages +RunTesting +SchedulerLowWaterMark__ +SchedulerHighWaterMark__ +SchedulerMaxFlushCells__ +SocksListenAddress +StrictEntryNodes +StrictExitNodes +Support022HiddenServices +Tor2webMode +Tor2webRendezvousPoints +TLSECGroup +TransListenAddress +TunnelDirConns +UseEntryGuardsAsDirGuards +UseNTorHandshake +UserspaceIOCPBuffers +V1AuthoritativeDirectory +V2AuthoritativeDirectory +VoteOnHidServDirectoriesV2 +UseFilteringSSLBufferevents +__UseFilteringSSLBufferevents +TestingConsensusMaxDownloadTries +ClientBootstrapConsensusMaxDownloadTries +ClientBootstrapConsensusAuthorityOnlyMaxDownloadTries +TestingDescriptorMaxDownloadTries +TestingMicrodescMaxDownloadTries +TestingCertMaxDownloadTries diff --git a/src/test/conf_examples/obsolete_2/expected b/src/test/conf_examples/obsolete_2/expected new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/src/test/conf_examples/obsolete_2/expected diff --git a/src/test/conf_examples/obsolete_2/torrc b/src/test/conf_examples/obsolete_2/torrc new file mode 100644 index 0000000000..4f78d47625 --- /dev/null +++ b/src/test/conf_examples/obsolete_2/torrc @@ -0,0 +1,2 @@ +# This option has been obsolete for some time +AllowDotExit diff --git a/src/test/conf_examples/ops_1/cmdline b/src/test/conf_examples/ops_1/cmdline new file mode 100644 index 0000000000..2bb9bfa132 --- /dev/null +++ b/src/test/conf_examples/ops_1/cmdline @@ -0,0 +1 @@ +ORPort 1000 diff --git a/src/test/conf_examples/ops_1/expected b/src/test/conf_examples/ops_1/expected new file mode 100644 index 0000000000..84be6a70e2 --- /dev/null +++ b/src/test/conf_examples/ops_1/expected @@ -0,0 +1,2 @@ +Nickname Unnamed +ORPort 1000 diff --git a/src/test/conf_examples/ops_1/torrc b/src/test/conf_examples/ops_1/torrc new file mode 100644 index 0000000000..daf8ae60fe --- /dev/null +++ b/src/test/conf_examples/ops_1/torrc @@ -0,0 +1,3 @@ +# We'll replace this option on the command line. + +ORPort 9999 diff --git a/src/test/conf_examples/ops_2/cmdline b/src/test/conf_examples/ops_2/cmdline new file mode 100644 index 0000000000..fdd48a045c --- /dev/null +++ b/src/test/conf_examples/ops_2/cmdline @@ -0,0 +1 @@ +/ORPort diff --git a/src/test/conf_examples/ops_2/expected b/src/test/conf_examples/ops_2/expected new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/src/test/conf_examples/ops_2/expected diff --git a/src/test/conf_examples/ops_2/torrc b/src/test/conf_examples/ops_2/torrc new file mode 100644 index 0000000000..21fcc93f9a --- /dev/null +++ b/src/test/conf_examples/ops_2/torrc @@ -0,0 +1,3 @@ +# We'll remove this option on the command line, and not replace it. + +ORPort 9999 diff --git a/src/test/conf_examples/ops_3/cmdline b/src/test/conf_examples/ops_3/cmdline new file mode 100644 index 0000000000..e4965d26f8 --- /dev/null +++ b/src/test/conf_examples/ops_3/cmdline @@ -0,0 +1 @@ ++ORPort 1000 diff --git a/src/test/conf_examples/ops_3/expected b/src/test/conf_examples/ops_3/expected new file mode 100644 index 0000000000..569d26b577 --- /dev/null +++ b/src/test/conf_examples/ops_3/expected @@ -0,0 +1,3 @@ +Nickname Unnamed +ORPort 9999 +ORPort 1000 diff --git a/src/test/conf_examples/ops_3/torrc b/src/test/conf_examples/ops_3/torrc new file mode 100644 index 0000000000..14adf87d7f --- /dev/null +++ b/src/test/conf_examples/ops_3/torrc @@ -0,0 +1,3 @@ +# We will extend this option on the command line + +ORPort 9999 diff --git a/src/test/conf_examples/ops_4/expected b/src/test/conf_examples/ops_4/expected new file mode 100644 index 0000000000..bf52f6a330 --- /dev/null +++ b/src/test/conf_examples/ops_4/expected @@ -0,0 +1,2 @@ +Nickname Unnamed +ORPort 9099 diff --git a/src/test/conf_examples/ops_4/torrc b/src/test/conf_examples/ops_4/torrc new file mode 100644 index 0000000000..dcec2aa95d --- /dev/null +++ b/src/test/conf_examples/ops_4/torrc @@ -0,0 +1,3 @@ +# This value is unadorned, so replaces the one from defaults.torrc. + +ORPort 9099 diff --git a/src/test/conf_examples/ops_4/torrc.defaults b/src/test/conf_examples/ops_4/torrc.defaults new file mode 100644 index 0000000000..04cd0393c6 --- /dev/null +++ b/src/test/conf_examples/ops_4/torrc.defaults @@ -0,0 +1 @@ +ORPort 9000 diff --git a/src/test/conf_examples/ops_5/expected b/src/test/conf_examples/ops_5/expected new file mode 100644 index 0000000000..288721da53 --- /dev/null +++ b/src/test/conf_examples/ops_5/expected @@ -0,0 +1,3 @@ +Nickname Unnamed +ORPort 9000 +ORPort 9099 diff --git a/src/test/conf_examples/ops_5/torrc b/src/test/conf_examples/ops_5/torrc new file mode 100644 index 0000000000..3284fc1c55 --- /dev/null +++ b/src/test/conf_examples/ops_5/torrc @@ -0,0 +1,3 @@ +# This value has a plus, and so extends the one from defaults.torrc. + ++ORPort 9099 diff --git a/src/test/conf_examples/ops_5/torrc.defaults b/src/test/conf_examples/ops_5/torrc.defaults new file mode 100644 index 0000000000..04cd0393c6 --- /dev/null +++ b/src/test/conf_examples/ops_5/torrc.defaults @@ -0,0 +1 @@ +ORPort 9000 diff --git a/src/test/conf_examples/ops_6/expected b/src/test/conf_examples/ops_6/expected new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/src/test/conf_examples/ops_6/expected diff --git a/src/test/conf_examples/ops_6/torrc b/src/test/conf_examples/ops_6/torrc new file mode 100644 index 0000000000..4d51caaff7 --- /dev/null +++ b/src/test/conf_examples/ops_6/torrc @@ -0,0 +1,3 @@ +# This value has a slash, and so clears the one from defaults.torrc. + +/ORPort diff --git a/src/test/conf_examples/ops_6/torrc.defaults b/src/test/conf_examples/ops_6/torrc.defaults new file mode 100644 index 0000000000..04cd0393c6 --- /dev/null +++ b/src/test/conf_examples/ops_6/torrc.defaults @@ -0,0 +1 @@ +ORPort 9000 diff --git a/src/test/conf_examples/relpath_rad/error b/src/test/conf_examples/relpath_rad/error new file mode 100644 index 0000000000..e131744475 --- /dev/null +++ b/src/test/conf_examples/relpath_rad/error @@ -0,0 +1 @@ +RunAsDaemon is not compatible with relative paths. diff --git a/src/test/conf_examples/relpath_rad/torrc b/src/test/conf_examples/relpath_rad/torrc new file mode 100644 index 0000000000..fe02441c3f --- /dev/null +++ b/src/test/conf_examples/relpath_rad/torrc @@ -0,0 +1,4 @@ + +# Relative-path data directories are incompatible with RunAsDaemon +DataDirectory ./datadir +RunAsDaemon 1 diff --git a/src/test/fuzz/fuzz_hsdescv3.c b/src/test/fuzz/fuzz_hsdescv3.c index 2cbd655898..9d4a6dbb55 100644 --- a/src/test/fuzz/fuzz_hsdescv3.c +++ b/src/test/fuzz/fuzz_hsdescv3.c @@ -35,16 +35,21 @@ mock_rsa_ed25519_crosscert_check(const uint8_t *crosscert, static size_t mock_decrypt_desc_layer(const hs_descriptor_t *desc, - const uint8_t *encrypted_blob, - size_t encrypted_blob_size, const uint8_t *descriptor_cookie, - int is_superencrypted_layer, + bool is_superencrypted_layer, char **decrypted_out) { (void)is_superencrypted_layer; (void)desc; (void)descriptor_cookie; const size_t overhead = HS_DESC_ENCRYPTED_SALT_LEN + DIGEST256_LEN; + const uint8_t *encrypted_blob = (is_superencrypted_layer) + ? desc->plaintext_data.superencrypted_blob + : desc->superencrypted_data.encrypted_blob; + size_t encrypted_blob_size = (is_superencrypted_layer) + ? desc->plaintext_data.superencrypted_blob_size + : desc->superencrypted_data.encrypted_blob_size; + if (encrypted_blob_size < overhead) return 0; *decrypted_out = tor_memdup_nulterm( diff --git a/src/test/fuzz/fuzzing_common.c b/src/test/fuzz/fuzzing_common.c index 6d0f9d7d60..604aba7a7f 100644 --- a/src/test/fuzz/fuzzing_common.c +++ b/src/test/fuzz/fuzzing_common.c @@ -1,6 +1,7 @@ /* Copyright (c) 2016-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define CRYPTO_ED25519_PRIVATE +#define CONFIG_PRIVATE #include "orconfig.h" #include "core/or/or.h" #include "app/main/subsysmgr.h" @@ -111,7 +112,7 @@ global_init(void) } /* set up the options. */ - mock_options = tor_malloc_zero(sizeof(or_options_t)); + mock_options = options_new(); MOCK(get_options, mock_get_options); /* Make BUG() and nonfatal asserts crash */ @@ -137,7 +138,7 @@ LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) return fuzz_main(Data, Size); } -#else /* !(defined(LLVM_FUZZ)) */ +#else /* !defined(LLVM_FUZZ) */ int main(int argc, char **argv) @@ -166,7 +167,7 @@ main(int argc, char **argv) memset(&s, 0, sizeof(s)); set_log_severity_config(loglevel, LOG_ERR, &s); /* ALWAYS log bug warnings. */ - s.masks[LOG_WARN-LOG_ERR] |= LD_BUG; + s.masks[SEVERITY_MASK_IDX(LOG_WARN)] |= LD_BUG; add_stream_log(&s, "", fileno(stdout)); } @@ -189,7 +190,7 @@ main(int argc, char **argv) if (fuzz_cleanup() < 0) abort(); - tor_free(mock_options); + or_options_free(mock_options); UNMOCK(get_options); return 0; } diff --git a/src/test/include.am b/src/test/include.am index 85f9c9f880..d8e25dea9f 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -23,7 +23,9 @@ TESTSCRIPTS = \ src/test/test_workqueue_pipe.sh \ src/test/test_workqueue_pipe2.sh \ src/test/test_workqueue_socketpair.sh \ - src/test/test_switch_id.sh + src/test/test_switch_id.sh \ + src/test/test_cmdline.sh \ + src/test/test_parseconf.sh if USE_RUST TESTSCRIPTS += \ @@ -31,7 +33,11 @@ TESTSCRIPTS += \ endif if USEPYTHON -TESTSCRIPTS += src/test/test_ntor.sh src/test/test_hs_ntor.sh src/test/test_bt.sh +TESTSCRIPTS += \ + src/test/test_ntor.sh \ + src/test/test_hs_ntor.sh \ + src/test/test_bt.sh \ + scripts/maint/practracker/test_practracker.sh if COVERAGE_ENABLED # ... @@ -96,6 +102,7 @@ src_test_test_SOURCES += \ src/test/log_test_helpers.c \ src/test/hs_test_helpers.c \ src/test/rend_test_helpers.c \ + src/test/resolve_test_helpers.c \ src/test/rng_test_helpers.c \ src/test/test.c \ src/test/test_accounting.c \ @@ -120,6 +127,8 @@ src_test_test_SOURCES += \ src/test/test_circuitstats.c \ src/test/test_compat_libevent.c \ src/test/test_config.c \ + src/test/test_confmgr.c \ + src/test/test_confparse.c \ src/test/test_connection.c \ src/test/test_conscache.c \ src/test/test_consdiff.c \ @@ -153,6 +162,7 @@ src_test_test_SOURCES += \ src/test/test_handles.c \ src/test/test_hs_cache.c \ src/test/test_hs_descriptor.c \ + src/test/test_hs_dos.c \ src/test/test_introduce.c \ src/test/test_keypin.c \ src/test/test_link_handshake.c \ @@ -169,6 +179,7 @@ src_test_test_SOURCES += \ src/test/test_periodic_event.c \ src/test/test_policy.c \ src/test/test_process.c \ + src/test/test_process_descs.c \ src/test/test_prob_distr.c \ src/test/test_procmon.c \ src/test/test_proto_http.c \ @@ -193,6 +204,7 @@ src_test_test_SOURCES += \ src/test/test_status.c \ src/test/test_storagedir.c \ src/test/test_threads.c \ + src/test/test_token_bucket.c \ src/test/test_tortls.c \ src/test/test_util.c \ src/test/test_util_format.c \ @@ -330,6 +342,7 @@ noinst_HEADERS+= \ src/test/hs_test_helpers.h \ src/test/log_test_helpers.h \ src/test/rend_test_helpers.h \ + src/test/resolve_test_helpers.h \ src/test/rng_test_helpers.h \ src/test/test.h \ src/test/ptr_helpers.h \ @@ -404,7 +417,9 @@ EXTRA_DIST += \ src/test/test_workqueue_efd2.sh \ src/test/test_workqueue_pipe.sh \ src/test/test_workqueue_pipe2.sh \ - src/test/test_workqueue_socketpair.sh + src/test/test_workqueue_socketpair.sh \ + src/test/test_cmdline.sh \ + src/test/test_parseconf.sh test-rust: $(TESTS_ENVIRONMENT) "$(abs_top_srcdir)/src/test/test_rust.sh" diff --git a/src/test/resolve_test_helpers.c b/src/test/resolve_test_helpers.c new file mode 100644 index 0000000000..73ea730149 --- /dev/null +++ b/src/test/resolve_test_helpers.c @@ -0,0 +1,85 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file resolve_test_helpers.c + * @brief Helper functions for mocking libc's blocking hostname lookup + * facilities. + **/ + +#define RESOLVE_PRIVATE +#include "orconfig.h" +#include "test/resolve_test_helpers.h" +#include "lib/net/address.h" +#include "lib/net/resolve.h" +#include "test/test.h" + +#include <stdio.h> +#include <string.h> + +/** + * Mock replacement for our getaddrinfo/gethostbyname wrapper. + **/ +static int +replacement_host_lookup(const char *name, uint16_t family, tor_addr_t *addr) +{ + static const struct lookup_table_ent { + const char *name; + const char *ipv4; + const char *ipv6; + } entries[] = { + { "localhost", "127.0.0.1", "::1" }, + { "torproject.org", "198.51.100.6", "2001:DB8::700" }, + { NULL, NULL, NULL }, + }; + + int r = -1; + + for (unsigned i = 0; entries[i].name != NULL; ++i) { + if (!strcasecmp(name, entries[i].name)) { + if (family == AF_INET6) { + int s = tor_addr_parse(addr, entries[i].ipv6); + tt_int_op(s, OP_EQ, AF_INET6); + } else { + int s = tor_addr_parse(addr, entries[i].ipv4); + tt_int_op(s, OP_EQ, AF_INET); + } + r = 0; + break; + } + } + + log_debug(LD_GENERAL, "resolve(%s,%d) => %s", + name, family, r == 0 ? fmt_addr(addr) : "-1"); + + return r; + done: + return -1; +} + +/** + * Set up a mock replacement for our wrapper on libc's resolver code. + * + * According to our replacement, only "localhost" and "torproject.org" + * are real addresses; everything else doesn't exist. + * + * Use this function to avoid using the DNS resolver during unit tests; + * call unmock_hostname_resolver() when you're done. + **/ +void +mock_hostname_resolver(void) +{ + MOCK(tor_addr_lookup_host_impl, replacement_host_lookup); +} + +/** + * Unmock our wrappers for libc's blocking hostname resolver code. + **/ +void +unmock_hostname_resolver(void) +{ + UNMOCK(tor_addr_lookup_host_impl); +} diff --git a/src/test/resolve_test_helpers.h b/src/test/resolve_test_helpers.h new file mode 100644 index 0000000000..e7d2e29373 --- /dev/null +++ b/src/test/resolve_test_helpers.h @@ -0,0 +1,18 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file resolve_test_helpers.h + * @brief Header for test/resolve_test_helpers.c + **/ + +#ifndef TOR_TEST_RESOLVE_TEST_HELPERS_H +#define TOR_TEST_RESOLVE_TEST_HELPERS_H + +void mock_hostname_resolver(void); +void unmock_hostname_resolver(void); + +#endif /* !defined(TOR_TEST_RESOLVE_TEST_HELPERS_H) */ diff --git a/src/test/test.c b/src/test/test.c index cac98dd839..6dbec26fa8 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -840,6 +840,8 @@ struct testgroup_t testgroups[] = { { "circuituse/", circuituse_tests }, { "compat/libevent/", compat_libevent_tests }, { "config/", config_tests }, + { "config/mgr/", confmgr_tests }, + { "config/parse/", confparse_tests }, { "connection/", connection_tests }, { "conscache/", conscache_tests }, { "consdiff/", consdiff_tests }, @@ -857,6 +859,7 @@ struct testgroup_t testgroups[] = { { "crypto/pem/", pem_tests }, { "crypto/rng/", crypto_rng_tests }, { "dir/", dir_tests }, + { "dir/auth/process_descs/", process_descs_tests }, { "dir/md/", microdesc_tests }, { "dir/voting/flags/", voting_flags_tests }, { "dir/voting/schedule/", voting_schedule_tests }, @@ -876,6 +879,7 @@ struct testgroup_t testgroups[] = { { "hs_config/", hs_config_tests }, { "hs_control/", hs_control_tests }, { "hs_descriptor/", hs_descriptor }, + { "hs_dos/", hs_dos_tests }, { "hs_intropoint/", hs_intropoint_tests }, { "hs_ntor/", hs_ntor_tests }, { "hs_service/", hs_service_tests }, @@ -916,6 +920,7 @@ struct testgroup_t testgroups[] = { { "socks/", socks_tests }, { "status/" , status_tests }, { "storagedir/", storagedir_tests }, + { "token_bucket/", token_bucket_tests }, { "tortls/", tortls_tests }, #ifndef ENABLE_NSS { "tortls/openssl/", tortls_openssl_tests }, diff --git a/src/test/test.h b/src/test/test.h index 167fd090ac..76c4c0ec75 100644 --- a/src/test/test.h +++ b/src/test/test.h @@ -197,6 +197,8 @@ extern struct testcase_t circuitstats_tests[]; extern struct testcase_t circuituse_tests[]; extern struct testcase_t compat_libevent_tests[]; extern struct testcase_t config_tests[]; +extern struct testcase_t confmgr_tests[]; +extern struct testcase_t confparse_tests[]; extern struct testcase_t connection_tests[]; extern struct testcase_t conscache_tests[]; extern struct testcase_t consdiff_tests[]; @@ -226,6 +228,7 @@ extern struct testcase_t hs_common_tests[]; extern struct testcase_t hs_config_tests[]; extern struct testcase_t hs_control_tests[]; extern struct testcase_t hs_descriptor[]; +extern struct testcase_t hs_dos_tests[]; extern struct testcase_t hs_intropoint_tests[]; extern struct testcase_t hs_ntor_tests[]; extern struct testcase_t hs_service_tests[]; @@ -250,6 +253,7 @@ extern struct testcase_t prob_distr_tests[]; extern struct testcase_t slow_stochastic_prob_distr_tests[]; extern struct testcase_t procmon_tests[]; extern struct testcase_t process_tests[]; +extern struct testcase_t process_descs_tests[]; extern struct testcase_t proto_http_tests[]; extern struct testcase_t proto_misc_tests[]; extern struct testcase_t protover_tests[]; @@ -272,6 +276,7 @@ extern struct testcase_t sr_tests[]; extern struct testcase_t status_tests[]; extern struct testcase_t storagedir_tests[]; extern struct testcase_t thread_tests[]; +extern struct testcase_t token_bucket_tests[]; extern struct testcase_t tortls_openssl_tests[]; extern struct testcase_t tortls_tests[]; extern struct testcase_t util_format_tests[]; diff --git a/src/test/test_addr.c b/src/test/test_addr.c index 05d8bf6c7b..c89c6e78d4 100644 --- a/src/test/test_addr.c +++ b/src/test/test_addr.c @@ -12,6 +12,7 @@ #include "test/log_test_helpers.h" #include "lib/net/resolve.h" #include "test/rng_test_helpers.h" +#include "test/resolve_test_helpers.h" #ifdef HAVE_SYS_UN_H #include <sys/un.h> @@ -724,158 +725,559 @@ test_addr_ip6_helpers(void *arg) ; } +/* Test that addr_str successfully parses, and: + * - the address has family expect_family, + * - the fmt_decorated result of tor_addr_to_str() is expect_str. + */ +#define TEST_ADDR_PARSE_FMT(addr_str, expect_family, fmt_decorated, \ + expect_str) \ + STMT_BEGIN \ + r = tor_addr_parse(&addr, addr_str); \ + tt_int_op(r, OP_EQ, expect_family); \ + sv = tor_addr_to_str(buf, &addr, sizeof(buf), fmt_decorated); \ + tt_str_op(sv, OP_EQ, buf); \ + tt_str_op(buf, OP_EQ, expect_str); \ + STMT_END + +/* Test that addr_str fails to parse, and: + * - the returned address is null. + */ +#define TEST_ADDR_PARSE_XFAIL(addr_str) \ + STMT_BEGIN \ + r = tor_addr_parse(&addr, addr_str); \ + tt_int_op(r, OP_EQ, -1); \ + tt_assert(tor_addr_is_null(&addr)); \ + STMT_END + +/* Test that addr_port_str and default_port successfully parse, and: + * - the address has family expect_family, + * - the fmt_decorated result of tor_addr_to_str() is expect_str, + * - the port is expect_port. + */ +#define TEST_ADDR_PORT_PARSE_FMT(addr_port_str, default_port, expect_family, \ + fmt_decorated, expect_str, expect_port) \ + STMT_BEGIN \ + r = tor_addr_port_parse(LOG_DEBUG, addr_port_str, &addr, &port, \ + default_port); \ + tt_int_op(r, OP_EQ, 0); \ + tt_int_op(tor_addr_family(&addr), OP_EQ, expect_family); \ + sv = tor_addr_to_str(buf, &addr, sizeof(buf), fmt_decorated); \ + tt_str_op(sv, OP_EQ, buf); \ + tt_str_op(buf, OP_EQ, expect_str); \ + tt_int_op(port, OP_EQ, expect_port); \ + STMT_END + +/* Test that addr_port_str and default_port fail to parse, and: + * - the returned address is null, + * - the returned port is 0. + */ +#define TEST_ADDR_PORT_PARSE_XFAIL(addr_port_str, default_port) \ + STMT_BEGIN \ + r = tor_addr_port_parse(LOG_DEBUG, addr_port_str, &addr, &port, \ + default_port); \ + tt_int_op(r, OP_EQ, -1); \ + tt_assert(tor_addr_is_null(&addr)); \ + tt_int_op(port, OP_EQ, 0); \ + STMT_END + +/* Test that addr_str successfully parses as an IPv4 address using + * tor_lookup_hostname(), and: + * - the fmt_addr32() of the result is expect_str. + */ +#define TEST_ADDR_V4_LOOKUP_HOSTNAME(addr_str, expect_str) \ + STMT_BEGIN \ + r = tor_lookup_hostname(addr_str, &addr32h); \ + tt_int_op(r, OP_EQ, 0); \ + tt_str_op(fmt_addr32(addr32h), OP_EQ, expect_str); \ + STMT_END + +/* Test that bad_str fails to parse using tor_lookup_hostname(), with a + * permanent failure, and: + * - the returned address is 0. + */ +#define TEST_ADDR_V4_LOOKUP_XFAIL(bad_str) \ + STMT_BEGIN \ + r = tor_lookup_hostname(bad_str, &addr32h); \ + tt_int_op(r, OP_EQ, -1); \ + tt_int_op(addr32h, OP_EQ, 0); \ + STMT_END + +/* Test that looking up host_str as an IPv4 address using tor_lookup_hostname() + * does something sensible: + * - the result is -1, 0, or 1. + * - if the result is a failure, the returned address is 0. + * We can't rely on the result of this function, because it depends on the + * network. + */ +#define TEST_HOST_V4_LOOKUP(host_str) \ + STMT_BEGIN \ + r = tor_lookup_hostname(host_str, &addr32h); \ + tt_int_op(r, OP_GE, -1); \ + tt_int_op(r, OP_LE, 1); \ + if (r != 0) \ + tt_int_op(addr32h, OP_EQ, 0); \ + STMT_END + +/* Test that addr_str successfully parses as a require_family IP address using + * tor_addr_lookup(), and: + * - the address has family expect_family, + * - the fmt_decorated result of tor_addr_to_str() is expect_str. + */ +#define TEST_ADDR_LOOKUP_FMT(addr_str, require_family, expect_family, \ + fmt_decorated, expect_str) \ + STMT_BEGIN \ + r = tor_addr_lookup(addr_str, require_family, &addr); \ + tt_int_op(r, OP_EQ, 0); \ + tt_int_op(tor_addr_family(&addr), OP_EQ, expect_family); \ + sv = tor_addr_to_str(buf, &addr, sizeof(buf), fmt_decorated); \ + tt_str_op(sv, OP_EQ, buf); \ + tt_str_op(buf, OP_EQ, expect_str); \ + STMT_END + +/* Test that bad_str fails to parse as a require_family IP address using + * tor_addr_lookup(), with a permanent failure, and: + * - the returned address is null. + */ +#define TEST_ADDR_LOOKUP_XFAIL(bad_str, require_family) \ + STMT_BEGIN \ + r = tor_addr_lookup(bad_str, require_family, &addr); \ + tt_int_op(r, OP_EQ, -1); \ + tt_assert(tor_addr_is_null(&addr)); \ + STMT_END + +/* Test that looking up host_string as a require_family IP address using + * tor_addr_lookup(), does something sensible: + * - the result is -1, 0, or 1. + * - if the result is a failure, the returned address is null. + * We can't rely on the result of this function, because it depends on the + * network. + */ +#define TEST_HOST_LOOKUP(host_str, require_family) \ + STMT_BEGIN \ + r = tor_addr_lookup(host_str, require_family, &addr); \ + tt_int_op(r, OP_GE, -1); \ + tt_int_op(r, OP_LE, 1); \ + if (r != 0) \ + tt_assert(tor_addr_is_null(&addr)); \ + STMT_END + +/* Test that addr_port_str successfully parses as an IP address and port + * using tor_addr_port_lookup(), and: + * - the address has family expect_family, + * - the fmt_decorated result of tor_addr_to_str() is expect_str, + * - the port is expect_port. + */ +#define TEST_ADDR_PORT_LOOKUP_FMT(addr_port_str, expect_family, \ + fmt_decorated, expect_str, expect_port) \ + STMT_BEGIN \ + r = tor_addr_port_lookup(addr_port_str, &addr, &port); \ + tt_int_op(r, OP_EQ, 0); \ + tt_int_op(tor_addr_family(&addr), OP_EQ, expect_family); \ + sv = tor_addr_to_str(buf, &addr, sizeof(buf), fmt_decorated); \ + tt_str_op(sv, OP_EQ, buf); \ + tt_str_op(buf, OP_EQ, expect_str); \ + tt_int_op(port, OP_EQ, expect_port); \ + STMT_END + +/* Test that bad_str fails to parse as an IP address and port + * using tor_addr_port_lookup(), and: + * - the returned address is null, + * - the returned port is 0. + */ +#define TEST_ADDR_PORT_LOOKUP_XFAIL(bad_str) \ + STMT_BEGIN \ + r = tor_addr_port_lookup(bad_str, &addr, &port); \ + tt_int_op(r, OP_EQ, -1); \ + tt_assert(tor_addr_is_null(&addr)); \ + tt_int_op(port, OP_EQ, 0); \ + STMT_END + +/* Test that looking up host_port_str as an IP address using + * tor_addr_port_lookup(), does something sensible: + * - the result is -1 or 0. + * - if the result is a failure, the returned address is null, and the + * returned port is zero, + * - if the result is a success, the returned port is expect_success_port, + * and the returned family is AF_INET or AF_INET6. + * We can't rely on the result of this function, because it depends on the + * network. + */ +#define TEST_HOST_PORT_LOOKUP(host_port_str, expect_success_port) \ + STMT_BEGIN \ + r = tor_addr_port_lookup(host_port_str, &addr, &port); \ + tt_int_op(r, OP_GE, -1); \ + tt_int_op(r, OP_LE, 0); \ + if (r == -1) { \ + tt_assert(tor_addr_is_null(&addr)); \ + tt_int_op(port, OP_EQ, 0); \ + } else { \ + tt_assert(tor_addr_family(&addr) == AF_INET || \ + tor_addr_family(&addr) == AF_INET6); \ + tt_int_op(port, OP_EQ, expect_success_port); \ + } \ + STMT_END + +/* Test that addr_str successfully parses as a canonical IPv4 address. + * Check for successful parsing using: + * - tor_addr_parse(), + * - tor_addr_port_parse() with a default port, + * - tor_lookup_hostname(), + * - tor_addr_lookup() with AF_INET, + * - tor_addr_lookup() with AF_UNSPEC, + * - tor_addr_port_lookup(), with a zero port. + * Check for failures using: + * - tor_addr_port_parse() without a default port, because there is no port, + * - tor_addr_lookup() with AF_INET6, + * - tor_addr_port_lookup(), because there is no port. + */ +#define TEST_ADDR_V4_PARSE_CANONICAL(addr_str) \ + STMT_BEGIN \ + TEST_ADDR_PARSE_FMT(addr_str, AF_INET, 0, addr_str); \ + TEST_ADDR_PORT_PARSE_FMT(addr_str, 111, AF_INET, 0, \ + addr_str, 111); \ + TEST_ADDR_V4_LOOKUP_HOSTNAME(addr_str, addr_str); \ + TEST_ADDR_PORT_LOOKUP_FMT(addr_str, AF_INET, 0, addr_str, 0); \ + TEST_ADDR_LOOKUP_FMT(addr_str, AF_INET, AF_INET, 0, addr_str); \ + TEST_ADDR_LOOKUP_FMT(addr_str, AF_UNSPEC, AF_INET, 0, addr_str); \ + TEST_ADDR_PORT_PARSE_XFAIL(addr_str, -1); \ + TEST_ADDR_LOOKUP_XFAIL(addr_str, AF_INET6); \ + STMT_END + +/* Test that addr_str successfully parses as a canonical fmt_decorated + * IPv6 address. + * Check for successful parsing using: + * - tor_addr_parse(), + * - tor_addr_port_parse() with a default port, + * - tor_addr_lookup() with AF_INET6, + * - tor_addr_lookup() with AF_UNSPEC, + * - tor_addr_port_lookup(), with a zero port. + * Check for failures using: + * - tor_addr_port_parse() without a default port, because there is no port, + * - tor_lookup_hostname(), because it only supports IPv4, + * - tor_addr_lookup() with AF_INET. + */ +#define TEST_ADDR_V6_PARSE_CANONICAL(addr_str, fmt_decorated) \ + STMT_BEGIN \ + TEST_ADDR_PARSE_FMT(addr_str, AF_INET6, fmt_decorated, addr_str); \ + TEST_ADDR_PORT_PARSE_FMT(addr_str, 222, AF_INET6, fmt_decorated, \ + addr_str, 222); \ + TEST_ADDR_LOOKUP_FMT(addr_str, AF_INET6, AF_INET6, fmt_decorated, \ + addr_str); \ + TEST_ADDR_LOOKUP_FMT(addr_str, AF_UNSPEC, AF_INET6, fmt_decorated, \ + addr_str); \ + TEST_ADDR_PORT_LOOKUP_FMT(addr_str, AF_INET6, fmt_decorated, addr_str, \ + 0); \ + TEST_ADDR_PORT_PARSE_XFAIL(addr_str, -1); \ + TEST_ADDR_V4_LOOKUP_XFAIL(addr_str); \ + TEST_ADDR_LOOKUP_XFAIL(addr_str, AF_INET); \ + STMT_END + +/* Test that addr_str successfully parses, and the fmt_decorated canonical + * IPv6 string is expect_str. + * Check for successful parsing using: + * - tor_addr_parse(), + * - tor_addr_port_parse() with a default port, + * - tor_addr_lookup() with AF_INET6, + * - tor_addr_lookup() with AF_UNSPEC, + * - tor_addr_port_lookup(), with a zero port. + * Check for failures using: + * - tor_addr_port_parse() without a default port, because there is no port. + * - tor_lookup_hostname(), because it only supports IPv4, + * - tor_addr_lookup() with AF_INET. + */ +#define TEST_ADDR_V6_PARSE(addr_str, fmt_decorated, expect_str) \ + STMT_BEGIN \ + TEST_ADDR_PARSE_FMT(addr_str, AF_INET6, fmt_decorated, expect_str); \ + TEST_ADDR_PORT_PARSE_FMT(addr_str, 333, AF_INET6, fmt_decorated, \ + expect_str, 333); \ + TEST_ADDR_LOOKUP_FMT(addr_str, AF_INET6, AF_INET6, fmt_decorated, \ + expect_str); \ + TEST_ADDR_LOOKUP_FMT(addr_str, AF_UNSPEC, AF_INET6, fmt_decorated, \ + expect_str); \ + TEST_ADDR_PORT_LOOKUP_FMT(addr_str, AF_INET6, fmt_decorated, expect_str, \ + 0); \ + TEST_ADDR_PORT_PARSE_XFAIL(addr_str, -1); \ + TEST_ADDR_V4_LOOKUP_XFAIL(addr_str); \ + TEST_ADDR_LOOKUP_XFAIL(addr_str, AF_INET); \ + STMT_END + +/* Test that addr_port_str successfully parses to the canonical IPv4 address + * string expect_str, and port expect_port. + * Check for successful parsing using: + * - tor_addr_port_parse() without a default port, + * - tor_addr_port_parse() with a default port, + * - tor_addr_port_lookup(). + * Check for failures using: + * - tor_addr_parse(), because there is a port, + * - tor_lookup_hostname(), because there is a port. + * - tor_addr_lookup(), regardless of the address family, because there is a + * port. + */ +#define TEST_ADDR_V4_PORT_PARSE(addr_port_str, expect_str, expect_port) \ + STMT_BEGIN \ + TEST_ADDR_PORT_PARSE_FMT(addr_port_str, -1, AF_INET, 0, expect_str, \ + expect_port); \ + TEST_ADDR_PORT_PARSE_FMT(addr_port_str, 444, AF_INET, 0, expect_str, \ + expect_port); \ + TEST_ADDR_PORT_LOOKUP_FMT(addr_port_str, AF_INET, 0, expect_str, \ + expect_port); \ + TEST_ADDR_PARSE_XFAIL(addr_port_str); \ + TEST_ADDR_V4_LOOKUP_XFAIL(addr_port_str); \ + TEST_ADDR_LOOKUP_XFAIL(addr_port_str, AF_INET); \ + TEST_ADDR_LOOKUP_XFAIL(addr_port_str, AF_UNSPEC); \ + TEST_ADDR_LOOKUP_XFAIL(addr_port_str, AF_INET6); \ + STMT_END + +/* Test that addr_port_str successfully parses to the canonical undecorated + * IPv6 address string expect_str, and port expect_port. + * Check for successful parsing using: + * - tor_addr_port_parse() without a default port, + * - tor_addr_port_parse() with a default port, + * - tor_addr_port_lookup(). + * Check for failures using: + * - tor_addr_parse(), because there is a port, + * - tor_lookup_hostname(), because there is a port, and because it only + * supports IPv4, + * - tor_addr_lookup(), regardless of the address family, because there is a + * port. + */ +#define TEST_ADDR_V6_PORT_PARSE(addr_port_str, expect_str, expect_port) \ + STMT_BEGIN \ + TEST_ADDR_PORT_PARSE_FMT(addr_port_str, -1, AF_INET6, 0, expect_str, \ + expect_port); \ + TEST_ADDR_PORT_PARSE_FMT(addr_port_str, 555, AF_INET6, 0, expect_str, \ + expect_port); \ + TEST_ADDR_PORT_LOOKUP_FMT(addr_port_str, AF_INET6, 0, expect_str, \ + expect_port); \ + TEST_ADDR_PARSE_XFAIL(addr_port_str); \ + TEST_ADDR_V4_LOOKUP_XFAIL(addr_port_str); \ + TEST_ADDR_LOOKUP_XFAIL(addr_port_str, AF_INET6); \ + TEST_ADDR_LOOKUP_XFAIL(addr_port_str, AF_UNSPEC); \ + TEST_ADDR_LOOKUP_XFAIL(addr_port_str, AF_INET); \ + STMT_END + +/* Test that bad_str fails to parse due to a bad address or port. + * Check for failures using: + * - tor_addr_parse(), + * - tor_addr_port_parse() without a default port, + * - tor_addr_port_parse() with a default port, + * - tor_lookup_hostname(), + * - tor_addr_lookup(), regardless of the address family, + * - tor_addr_port_lookup(). + */ +#define TEST_ADDR_PARSE_XFAIL_MALFORMED(bad_str) \ + STMT_BEGIN \ + TEST_ADDR_PARSE_XFAIL(bad_str); \ + TEST_ADDR_PORT_PARSE_XFAIL(bad_str, -1); \ + TEST_ADDR_PORT_PARSE_XFAIL(bad_str, 666); \ + TEST_ADDR_V4_LOOKUP_XFAIL(bad_str); \ + TEST_ADDR_LOOKUP_XFAIL(bad_str, AF_UNSPEC); \ + TEST_ADDR_LOOKUP_XFAIL(bad_str, AF_INET); \ + TEST_ADDR_LOOKUP_XFAIL(bad_str, AF_INET6); \ + TEST_ADDR_PORT_LOOKUP_XFAIL(bad_str); \ + STMT_END + +/* Test that host_str is treated as a hostname, and not an address. + * Check for success or failure using the network-dependent functions: + * - tor_lookup_hostname(), + * - tor_addr_lookup(), regardless of the address family, + * - tor_addr_port_lookup(), expecting a zero port. + * Check for failures using: + * - tor_addr_parse(), + * - tor_addr_port_parse() without a default port, + * - tor_addr_port_parse() with a default port. + */ +#define TEST_HOSTNAME(host_str) \ + STMT_BEGIN \ + TEST_HOST_V4_LOOKUP(host_str); \ + TEST_HOST_LOOKUP(host_str, AF_UNSPEC); \ + TEST_HOST_LOOKUP(host_str, AF_INET); \ + TEST_HOST_LOOKUP(host_str, AF_INET6); \ + TEST_HOST_PORT_LOOKUP(host_str, 0); \ + TEST_ADDR_PARSE_XFAIL(host_str); \ + TEST_ADDR_PORT_PARSE_XFAIL(host_str, -1); \ + TEST_ADDR_PORT_PARSE_XFAIL(host_str, 777); \ + STMT_END + +/* Test that host_port_str is treated as a hostname and port, and not a + * hostname or an address. + * Check for success or failure using the network-dependent function: + * - tor_addr_port_lookup(), expecting expect_success_port if the lookup is + * successful. + * Check for failures using: + * - tor_addr_parse(), + * - tor_addr_port_parse() without a default port, + * - tor_addr_port_parse() with a default port, + * - tor_lookup_hostname(), because it doesn't support ports, + * - tor_addr_lookup(), regardless of the address family, because it doesn't + * support ports. + */ +#define TEST_HOSTNAME_PORT(host_port_str, expect_success_port) \ + STMT_BEGIN \ + TEST_HOST_PORT_LOOKUP(host_port_str, expect_success_port); \ + TEST_ADDR_PARSE_XFAIL(host_port_str); \ + TEST_ADDR_PORT_PARSE_XFAIL(host_port_str, -1); \ + TEST_ADDR_PORT_PARSE_XFAIL(host_port_str, 888); \ + TEST_ADDR_V4_LOOKUP_XFAIL(host_port_str); \ + TEST_ADDR_LOOKUP_XFAIL(host_port_str, AF_UNSPEC); \ + TEST_ADDR_LOOKUP_XFAIL(host_port_str, AF_INET); \ + TEST_ADDR_LOOKUP_XFAIL(host_port_str, AF_INET6); \ + STMT_END + +static void +test_addr_parse_canonical(void *arg) +{ + int r; + tor_addr_t addr; + uint16_t port; + const char *sv; + uint32_t addr32h; + char buf[TOR_ADDR_BUF_LEN]; + + (void)arg; + + /* Correct calls. */ + TEST_ADDR_V4_PARSE_CANONICAL("192.0.2.1"); + TEST_ADDR_V4_PARSE_CANONICAL("192.0.2.2"); + + TEST_ADDR_V6_PARSE_CANONICAL("[11:22::33:44]", 1); + TEST_ADDR_V6_PARSE_CANONICAL("[::1]", 1); + TEST_ADDR_V6_PARSE_CANONICAL("[::]", 1); + TEST_ADDR_V6_PARSE_CANONICAL("[2::]", 1); + TEST_ADDR_V6_PARSE_CANONICAL("[11:22:33:44:55:66:77:88]", 1); + + /* Allow IPv6 without square brackets, when there is no port, but only if + * there is a default port */ + TEST_ADDR_V6_PARSE_CANONICAL("11:22::33:44", 0); + TEST_ADDR_V6_PARSE_CANONICAL("::1", 0); + TEST_ADDR_V6_PARSE_CANONICAL("::", 0); + TEST_ADDR_V6_PARSE_CANONICAL("2::", 0); + TEST_ADDR_V6_PARSE_CANONICAL("11:22:33:44:55:66:77:88", 0); + done: + ; +} + /** Test tor_addr_parse() and tor_addr_port_parse(). */ static void test_addr_parse(void *arg) { + int r; tor_addr_t addr; + uint16_t port; + const char *sv; + uint32_t addr32h; char buf[TOR_ADDR_BUF_LEN]; - uint16_t port = 0; - /* Correct call. */ (void)arg; - r= tor_addr_parse(&addr, "192.0.2.1"); - tt_int_op(r,OP_EQ, AF_INET); - tor_addr_to_str(buf, &addr, sizeof(buf), 0); - tt_str_op(buf,OP_EQ, "192.0.2.1"); - - r= tor_addr_parse(&addr, "11:22::33:44"); - tt_int_op(r,OP_EQ, AF_INET6); - tor_addr_to_str(buf, &addr, sizeof(buf), 0); - tt_str_op(buf,OP_EQ, "11:22::33:44"); - - r= tor_addr_parse(&addr, "[11:22::33:44]"); - tt_int_op(r,OP_EQ, AF_INET6); - tor_addr_to_str(buf, &addr, sizeof(buf), 0); - tt_str_op(buf,OP_EQ, "11:22::33:44"); - - r= tor_addr_parse(&addr, "11:22:33:44:55:66:1.2.3.4"); - tt_int_op(r,OP_EQ, AF_INET6); - tor_addr_to_str(buf, &addr, sizeof(buf), 0); - tt_str_op(buf,OP_EQ, "11:22:33:44:55:66:102:304"); - - r= tor_addr_parse(&addr, "11:22::33:44:1.2.3.4"); - tt_int_op(r,OP_EQ, AF_INET6); - tor_addr_to_str(buf, &addr, sizeof(buf), 0); - tt_str_op(buf,OP_EQ, "11:22::33:44:102:304"); - /* Empty string. */ - r= tor_addr_parse(&addr, ""); - tt_int_op(r,OP_EQ, -1); + mock_hostname_resolver(); - /* Square brackets around IPv4 address. */ - r= tor_addr_parse(&addr, "[192.0.2.1]"); - tt_int_op(r,OP_EQ, -1); + /* IPv6-mapped IPv4 addresses. Tor doesn't really use these. */ + TEST_ADDR_V6_PARSE("11:22:33:44:55:66:1.2.3.4", 0, + "11:22:33:44:55:66:102:304"); - /* Only left square bracket. */ - r= tor_addr_parse(&addr, "[11:22::33:44"); - tt_int_op(r,OP_EQ, -1); + TEST_ADDR_V6_PARSE("11:22::33:44:1.2.3.4", 0, + "11:22::33:44:102:304"); - /* Only right square bracket. */ - r= tor_addr_parse(&addr, "11:22::33:44]"); - tt_int_op(r,OP_EQ, -1); + /* Ports. */ + TEST_ADDR_V4_PORT_PARSE("192.0.2.1:1234", "192.0.2.1", 1234); + TEST_ADDR_V6_PORT_PARSE("[::1]:1234", "::1", 1234); - /* Leading colon. */ - r= tor_addr_parse(&addr, ":11:22::33:44"); - tt_int_op(r,OP_EQ, -1); + /* Host names. */ + TEST_HOSTNAME("localhost"); + TEST_HOSTNAME_PORT("localhost:1234", 1234); + TEST_HOSTNAME_PORT("localhost:0", 0); - /* Trailing colon. */ - r= tor_addr_parse(&addr, "11:22::33:44:"); - tt_int_op(r,OP_EQ, -1); - - /* Too many hex words in IPv4-mapped IPv6 address. */ - r= tor_addr_parse(&addr, "11:22:33:44:55:66:77:88:1.2.3.4"); - tt_int_op(r,OP_EQ, -1); - - /* Correct call. */ - r= tor_addr_port_parse(LOG_DEBUG, - "192.0.2.1:1234", - &addr, &port, -1); - tt_int_op(r, OP_EQ, 0); - tor_addr_to_str(buf, &addr, sizeof(buf), 0); - tt_str_op(buf,OP_EQ, "192.0.2.1"); - tt_int_op(port,OP_EQ, 1234); - - r= tor_addr_port_parse(LOG_DEBUG, - "[::1]:1234", - &addr, &port, -1); - tt_int_op(r, OP_EQ, 0); - tor_addr_to_str(buf, &addr, sizeof(buf), 0); - tt_str_op(buf,OP_EQ, "::1"); - tt_int_op(port,OP_EQ, 1234); - - /* Domain name. */ - r= tor_addr_port_parse(LOG_DEBUG, - "torproject.org:1234", - &addr, &port, -1); - tt_int_op(r, OP_EQ, -1); + TEST_HOSTNAME("torproject.org"); + TEST_HOSTNAME_PORT("torproject.org:56", 56); - /* Only IP. */ - r= tor_addr_port_parse(LOG_DEBUG, - "192.0.2.2", - &addr, &port, -1); - tt_int_op(r, OP_EQ, -1); + TEST_HOSTNAME("probably-not-a-valid-dns.name-tld"); + TEST_HOSTNAME_PORT("probably-not-a-valid-dns.name-tld:789", 789); - r= tor_addr_port_parse(LOG_DEBUG, - "192.0.2.2", - &addr, &port, 200); - tt_int_op(r, OP_EQ, 0); - tt_int_op(port,OP_EQ,200); - - r= tor_addr_port_parse(LOG_DEBUG, - "[::1]", - &addr, &port, -1); - tt_int_op(r, OP_EQ, -1); + /* Malformed addresses. */ + /* Empty string. */ + TEST_ADDR_PARSE_XFAIL_MALFORMED(""); - r= tor_addr_port_parse(LOG_DEBUG, - "[::1]", - &addr, &port, 400); - tt_int_op(r, OP_EQ, 0); - tt_int_op(port,OP_EQ,400); + /* Square brackets around IPv4 address. */ + TEST_ADDR_PARSE_XFAIL_MALFORMED("[192.0.2.1]"); + TEST_ADDR_PARSE_XFAIL_MALFORMED("[192.0.2.3]:12345"); - /* Bad port. */ - r= tor_addr_port_parse(LOG_DEBUG, - "192.0.2.2:66666", - &addr, &port, -1); - tt_int_op(r, OP_EQ, -1); - r= tor_addr_port_parse(LOG_DEBUG, - "192.0.2.2:66666", - &addr, &port, 200); - tt_int_op(r, OP_EQ, -1); + /* Only left square bracket. */ + TEST_ADDR_PARSE_XFAIL_MALFORMED("[11:22::33:44"); - /* Only domain name */ - r= tor_addr_port_parse(LOG_DEBUG, - "torproject.org", - &addr, &port, -1); - tt_int_op(r, OP_EQ, -1); - r= tor_addr_port_parse(LOG_DEBUG, - "torproject.org", - &addr, &port, 200); - tt_int_op(r, OP_EQ, -1); + /* Only right square bracket. */ + TEST_ADDR_PARSE_XFAIL_MALFORMED("11:22::33:44]"); - /* Bad IP address */ - r= tor_addr_port_parse(LOG_DEBUG, - "192.0.2:1234", - &addr, &port, -1); - tt_int_op(r, OP_EQ, -1); + /* Leading colon. */ + TEST_ADDR_PARSE_XFAIL_MALFORMED(":11:22::33:44"); - /* Make sure that the default port has lower priority than the real - one */ - r= tor_addr_port_parse(LOG_DEBUG, - "192.0.2.2:1337", - &addr, &port, 200); - tt_int_op(r, OP_EQ, 0); - tt_int_op(port,OP_EQ,1337); + /* Trailing colon. */ + TEST_ADDR_PARSE_XFAIL_MALFORMED("11:22::33:44:"); + TEST_ADDR_PARSE_XFAIL_MALFORMED("[::1]:"); + TEST_ADDR_PARSE_XFAIL_MALFORMED("localhost:"); - r= tor_addr_port_parse(LOG_DEBUG, - "[::1]:1369", - &addr, &port, 200); - tt_int_op(r, OP_EQ, 0); - tt_int_op(port,OP_EQ,1369); + /* Bad port. */ + TEST_ADDR_PARSE_XFAIL_MALFORMED("192.0.2.2:66666"); + TEST_ADDR_PARSE_XFAIL_MALFORMED("[::1]:77777"); + TEST_ADDR_PARSE_XFAIL_MALFORMED("::1:88888"); + TEST_ADDR_PARSE_XFAIL_MALFORMED("localhost:99999"); + + TEST_ADDR_PARSE_XFAIL_MALFORMED("192.0.2.2:-1"); + TEST_ADDR_PARSE_XFAIL_MALFORMED("[::1]:-2"); + TEST_ADDR_PARSE_XFAIL_MALFORMED("::1:-3"); + TEST_ADDR_PARSE_XFAIL_MALFORMED("localhost:-4"); + + TEST_ADDR_PARSE_XFAIL_MALFORMED("192.0.2.2:1 bad"); + TEST_ADDR_PARSE_XFAIL_MALFORMED("192.0.2.2:bad-port"); + TEST_ADDR_PARSE_XFAIL_MALFORMED("[::1]:bad-port-1"); + TEST_ADDR_PARSE_XFAIL_MALFORMED("::1:1-bad-port"); + TEST_ADDR_PARSE_XFAIL_MALFORMED("localhost:1-bad-port"); + TEST_ADDR_PARSE_XFAIL_MALFORMED("localhost:1-bad-port-1"); + + /* Bad hostname */ + TEST_ADDR_PARSE_XFAIL_MALFORMED("definitely invalid"); + TEST_ADDR_PARSE_XFAIL_MALFORMED("definitely invalid:22222"); + + /* Ambiguous cases */ + /* Too many hex words in IPv4-mapped IPv6 address. + * But some OS host lookup routines accept it as a hostname, or + * as an IP address?? (I assume they discard unused characters). */ + TEST_HOSTNAME("11:22:33:44:55:66:77:88:1.2.3.4"); + + /* IPv6 address with port and no brackets + * We reject it, but some OS host lookup routines accept it as an + * IPv6 address:port ? */ + TEST_HOSTNAME_PORT("11:22::33:44:12345", 12345); + /* Is it a port, or are there too many hex words? + * We reject it either way, but some OS host lookup routines accept it as an + * IPv6 address:port */ + TEST_HOSTNAME_PORT("11:22:33:44:55:66:77:88:99", 99); + /* But we accept it if it has square brackets. */ + TEST_ADDR_V6_PORT_PARSE("[11:22:33:44:55:66:77:88]:99", + "11:22:33:44:55:66:77:88",99); + + /* Bad IPv4 address + * We reject it, but some OS host lookup routines accept it as an + * IPv4 address[:port], with a zero last octet */ + TEST_HOSTNAME("192.0.1"); + TEST_HOSTNAME_PORT("192.0.2:1234", 1234); + + /* More bad IPv6 addresses and ports: no brackets + * We reject it, but some OS host lookup routines accept it as an + * IPv6 address[:port] */ + TEST_HOSTNAME_PORT("::1:12345", 12345); + TEST_HOSTNAME_PORT("11:22::33:44:12345", 12345); + + /* And this is an ambiguous case, which is interpreted as an IPv6 address. */ + TEST_ADDR_V6_PARSE_CANONICAL("11:22::88:99", 0); + /* Use square brackets to resolve the ambiguity */ + TEST_ADDR_V6_PARSE_CANONICAL("[11:22::88:99]", 1); + TEST_ADDR_V6_PORT_PARSE("[11:22::88]:99", + "11:22::88",99); done: - ; + unmock_hostname_resolver(); } static void @@ -1254,6 +1656,7 @@ struct testcase_t addr_tests[] = { ADDR_LEGACY(basic), ADDR_LEGACY(ip6_helpers), ADDR_LEGACY(parse), + ADDR_LEGACY(parse_canonical), { "virtaddr", test_virtaddrmap, 0, NULL, NULL }, { "virtaddr_persist", test_virtaddrmap_persist, TT_FORK, NULL, NULL }, { "localname", test_addr_localname, 0, NULL, NULL }, diff --git a/src/test/test_address.c b/src/test/test_address.c index bf9ca047dc..32fb4aa232 100644 --- a/src/test/test_address.c +++ b/src/test/test_address.c @@ -24,6 +24,8 @@ #endif /* defined(HAVE_IFCONF_TO_SMARTLIST) */ #include "core/or/or.h" +#include "app/config/config.h" +#include "feature/dirauth/process_descs.h" #include "feature/nodelist/routerinfo_st.h" #include "feature/nodelist/node_st.h" #include "feature/nodelist/nodelist.h" @@ -1244,8 +1246,95 @@ test_address_tor_node_in_same_network_family(void *ignored) helper_free_mock_node(node_b); } +static or_options_t mock_options; + +static const or_options_t * +mock_get_options(void) +{ + return &mock_options; +} + +/* Test dirserv_router_has_valid_address() on a stub routerinfo, with only its + * address fields set. Use IPv4 ipv4_addr_str and IPv6 ipv6_addr_str. + * Fail if it does not return rv. */ +#define TEST_ROUTER_VALID_ADDRESS_HELPER(ipv4_addr_str, ipv6_addr_str, rv) \ + STMT_BEGIN \ + ri = tor_malloc_zero(sizeof(routerinfo_t)); \ + tor_addr_t addr; \ + tor_addr_parse(&addr, (ipv4_addr_str)); \ + ri->addr = tor_addr_to_ipv4h(&addr); \ + tor_addr_parse(&ri->ipv6_addr, (ipv6_addr_str)); \ + tt_int_op(dirserv_router_has_valid_address(ri), OP_EQ, (rv)); \ + tor_free(ri); \ + STMT_END + +/* Like TEST_ROUTER_VALID_ADDRESS_HELPER(), but always passes a null + * IPv6 address. */ +#define CHECK_RI_ADDR(ipv4_addr_str, rv) \ + TEST_ROUTER_VALID_ADDRESS_HELPER(ipv4_addr_str, "::", rv) + +/* Like TEST_ROUTER_VALID_ADDRESS_HELPER(), but always passes a non-internal + * IPv4 address, so that the IPv6 check is reached. */ +#define CHECK_RI_ADDR6(ipv6_addr_str, rv) \ + TEST_ROUTER_VALID_ADDRESS_HELPER("1.0.0.1", ipv6_addr_str, rv) + +static void +test_address_dirserv_router_addr_private(void *opt_dir_allow_private) +{ + /* A stub routerinfo structure, with only its address fields set. */ + routerinfo_t *ri = NULL; + /* The expected return value for private addresses. + * Modified if DirAllowPrivateAddresses is 1. */ + int private_rv = -1; + + memset(&mock_options, 0, sizeof(or_options_t)); + MOCK(get_options, mock_get_options); + + if (opt_dir_allow_private) { + mock_options.DirAllowPrivateAddresses = 1; + private_rv = 0; + } + + CHECK_RI_ADDR("1.0.0.1", 0); + CHECK_RI_ADDR("10.0.0.1", private_rv); + + CHECK_RI_ADDR6("2600::1", 0); + CHECK_RI_ADDR6("fe80::1", private_rv); + + /* Null addresses */ + /* IPv4 null fails, regardless of IPv6 */ + CHECK_RI_ADDR("0.0.0.0", private_rv); + TEST_ROUTER_VALID_ADDRESS_HELPER("0.0.0.0", "::", private_rv); + + /* IPv6 null succeeds, because IPv4 is not null */ + CHECK_RI_ADDR6("::", 0); + + /* Byte-zeroed null addresses */ + /* IPv4 null fails, regardless of IPv6 */ + { + ri = tor_malloc_zero(sizeof(routerinfo_t)); + tt_int_op(dirserv_router_has_valid_address(ri), OP_EQ, private_rv); + tor_free(ri); + } + + /* IPv6 null succeeds, because IPv4 is not internal */ + { + ri = tor_malloc_zero(sizeof(routerinfo_t)); + ri->addr = 16777217; /* 1.0.0.1 */ + tt_int_op(dirserv_router_has_valid_address(ri), OP_EQ, 0); + tor_free(ri); + } + + done: + tor_free(ri); + UNMOCK(get_options); +} + #define ADDRESS_TEST(name, flags) \ { #name, test_address_ ## name, flags, NULL, NULL } +#define ADDRESS_TEST_STR_ARG(name, flags, str_arg) \ + { #name "/" str_arg, test_address_ ## name, flags, &passthrough_setup, \ + (void *)(str_arg) } struct testcase_t address_tests[] = { ADDRESS_TEST(udp_socket_trick_whitebox, TT_FORK), @@ -1277,5 +1366,7 @@ struct testcase_t address_tests[] = { ADDRESS_TEST(tor_addr_eq_ipv4h, 0), ADDRESS_TEST(tor_addr_in_same_network_family, 0), ADDRESS_TEST(tor_node_in_same_network_family, 0), + ADDRESS_TEST(dirserv_router_addr_private, 0), + ADDRESS_TEST_STR_ARG(dirserv_router_addr_private, 0, "allow_private"), END_OF_TESTCASES }; diff --git a/src/test/test_btrack.c b/src/test/test_btrack.c index 48486fb5a1..80da7829ae 100644 --- a/src/test/test_btrack.c +++ b/src/test/test_btrack.c @@ -4,6 +4,7 @@ #include "core/or/or.h" #include "test/test.h" +#include "test/test_helpers.h" #include "test/log_test_helpers.h" #define OCIRC_EVENT_PRIVATE @@ -12,48 +13,75 @@ #include "core/or/orconn_event.h" static void +send_state(const orconn_state_msg_t *msg_in) +{ + orconn_state_msg_t *msg = tor_malloc(sizeof(*msg)); + + *msg = *msg_in; + orconn_state_publish(msg); +} + +static void +send_status(const orconn_status_msg_t *msg_in) +{ + orconn_status_msg_t *msg = tor_malloc(sizeof(*msg)); + + *msg = *msg_in; + orconn_status_publish(msg); +} + +static void +send_chan(const ocirc_chan_msg_t *msg_in) +{ + ocirc_chan_msg_t *msg = tor_malloc(sizeof(*msg)); + + *msg = *msg_in; + ocirc_chan_publish(msg); +} + +static void test_btrack_launch(void *arg) { - orconn_event_msg_t conn; - ocirc_event_msg_t circ; + orconn_state_msg_t conn; + ocirc_chan_msg_t circ; + memset(&conn, 0, sizeof(conn)); + memset(&circ, 0, sizeof(circ)); (void)arg; - conn.type = ORCONN_MSGTYPE_STATE; - conn.u.state.gid = 1; - conn.u.state.chan = 1; - conn.u.state.proxy_type = PROXY_NONE; - conn.u.state.state = OR_CONN_STATE_CONNECTING; + conn.gid = 1; + conn.chan = 1; + conn.proxy_type = PROXY_NONE; + conn.state = OR_CONN_STATE_CONNECTING; setup_full_capture_of_logs(LOG_DEBUG); - orconn_event_publish(&conn); + send_state(&conn); expect_log_msg_containing("ORCONN gid=1 chan=1 proxy_type=0 state=1"); expect_no_log_msg_containing("ORCONN BEST_"); teardown_capture_of_logs(); - circ.type = OCIRC_MSGTYPE_CHAN; - circ.u.chan.chan = 1; - circ.u.chan.onehop = true; + circ.chan = 1; + circ.onehop = true; setup_full_capture_of_logs(LOG_DEBUG); - ocirc_event_publish(&circ); + send_chan(&circ); expect_log_msg_containing("ORCONN LAUNCH chan=1 onehop=1"); expect_log_msg_containing("ORCONN BEST_ANY state -1->1 gid=1"); teardown_capture_of_logs(); - conn.u.state.gid = 2; - conn.u.state.chan = 2; + conn.gid = 2; + conn.chan = 2; setup_full_capture_of_logs(LOG_DEBUG); - orconn_event_publish(&conn); + send_state(&conn); expect_log_msg_containing("ORCONN gid=2 chan=2 proxy_type=0 state=1"); expect_no_log_msg_containing("ORCONN BEST_"); teardown_capture_of_logs(); - circ.u.chan.chan = 2; - circ.u.chan.onehop = false; + circ.chan = 2; + circ.onehop = false; setup_full_capture_of_logs(LOG_DEBUG); - ocirc_event_publish(&circ); + send_chan(&circ); expect_log_msg_containing("ORCONN LAUNCH chan=2 onehop=0"); expect_log_msg_containing("ORCONN BEST_AP state -1->1 gid=2"); teardown_capture_of_logs(); @@ -65,27 +93,28 @@ test_btrack_launch(void *arg) static void test_btrack_delete(void *arg) { - orconn_event_msg_t conn; + orconn_state_msg_t state; + orconn_status_msg_t status; + memset(&state, 0, sizeof(state)); + memset(&status, 0, sizeof(status)); (void)arg; - conn.type = ORCONN_MSGTYPE_STATE; - conn.u.state.gid = 1; - conn.u.state.chan = 1; - conn.u.state.proxy_type = PROXY_NONE; - conn.u.state.state = OR_CONN_STATE_CONNECTING; + state.gid = 1; + state.chan = 1; + state.proxy_type = PROXY_NONE; + state.state = OR_CONN_STATE_CONNECTING; setup_full_capture_of_logs(LOG_DEBUG); - orconn_event_publish(&conn); + send_state(&state); expect_log_msg_containing("ORCONN gid=1 chan=1 proxy_type=0"); teardown_capture_of_logs(); - conn.type = ORCONN_MSGTYPE_STATUS; - conn.u.status.gid = 1; - conn.u.status.status = OR_CONN_EVENT_CLOSED; - conn.u.status.reason = 0; + status.gid = 1; + status.status = OR_CONN_EVENT_CLOSED; + status.reason = 0; setup_full_capture_of_logs(LOG_DEBUG); - orconn_event_publish(&conn); + send_status(&status); expect_log_msg_containing("ORCONN DELETE gid=1 status=3 reason=0"); teardown_capture_of_logs(); @@ -94,7 +123,7 @@ test_btrack_delete(void *arg) } struct testcase_t btrack_tests[] = { - { "launch", test_btrack_launch, TT_FORK, 0, NULL }, - { "delete", test_btrack_delete, TT_FORK, 0, NULL }, + { "launch", test_btrack_launch, TT_FORK, &helper_pubsub_setup, NULL }, + { "delete", test_btrack_delete, TT_FORK, &helper_pubsub_setup, NULL }, END_OF_TESTCASES }; diff --git a/src/test/test_channelpadding.c b/src/test/test_channelpadding.c index 5d012e462b..885246628e 100644 --- a/src/test/test_channelpadding.c +++ b/src/test/test_channelpadding.c @@ -289,8 +289,6 @@ test_channelpadding_timers(void *arg) channel_t *chans[CHANNELS_TO_TEST]; (void)arg; - tor_libevent_postfork(); - if (!connection_array) connection_array = smartlist_new(); @@ -393,7 +391,6 @@ test_channelpadding_killonehop(void *arg) channelpadding_decision_t decision; int64_t new_time; (void)arg; - tor_libevent_postfork(); routerstatus_t *relay = tor_malloc_zero(sizeof(routerstatus_t)); monotime_init(); @@ -502,8 +499,6 @@ test_channelpadding_consensus(void *arg) int64_t new_time; (void)arg; - tor_libevent_postfork(); - /* * Params tested: * nf_pad_before_usage @@ -898,8 +893,6 @@ test_channelpadding_decide_to_pad_channel(void *arg) connection_array = smartlist_new(); (void)arg; - tor_libevent_postfork(); - monotime_init(); monotime_enable_test_mocking(); monotime_set_mock_time_nsec(1); diff --git a/src/test/test_circuitbuild.c b/src/test/test_circuitbuild.c index 0c23091594..7291e04d6a 100644 --- a/src/test/test_circuitbuild.c +++ b/src/test/test_circuitbuild.c @@ -167,6 +167,7 @@ test_upgrade_from_guard_wait(void *arg) tt_assert(!list); done: + smartlist_free(list); circuit_free(circ); entry_guard_free_(guard); } @@ -177,6 +178,6 @@ struct testcase_t circuitbuild_tests[] = { { "unsafe_exit", test_new_route_len_unsafe_exit, 0, NULL, NULL }, { "unhandled_exit", test_new_route_len_unhandled_exit, 0, NULL, NULL }, { "upgrade_from_guard_wait", test_upgrade_from_guard_wait, TT_FORK, - NULL, NULL }, + &helper_pubsub_setup, NULL }, END_OF_TESTCASES }; diff --git a/src/test/test_circuitpadding.c b/src/test/test_circuitpadding.c index 25f8fd311b..934ddb0208 100644 --- a/src/test/test_circuitpadding.c +++ b/src/test/test_circuitpadding.c @@ -4,9 +4,11 @@ #define CIRCUITPADDING_MACHINES_PRIVATE #define NETWORKSTATUS_PRIVATE #define CRYPT_PATH_PRIVATE +#define RELAY_PRIVATE #include "core/or/or.h" #include "test/test.h" +#include "test/log_test_helpers.h" #include "lib/testsupport/testsupport.h" #include "core/or/connection_or.h" #include "core/or/channel.h" @@ -3152,6 +3154,29 @@ test_circuitpadding_hs_machines(void *arg) UNMOCK(circpad_machine_schedule_padding); } +/** Test that we effectively ignore non-padding cells in padding circuits. */ +static void +test_circuitpadding_ignore_non_padding_cells(void *arg) +{ + int retval; + relay_header_t rh; + + (void) arg; + + client_side = (circuit_t *)origin_circuit_new(); + client_side->purpose = CIRCUIT_PURPOSE_C_CIRCUIT_PADDING; + + rh.command = RELAY_COMMAND_BEGIN; + + setup_full_capture_of_logs(LOG_INFO); + retval = handle_relay_cell_command(NULL, client_side, NULL, NULL, &rh, 0); + tt_int_op(retval, OP_EQ, 0); + expect_log_msg_containing("Ignored cell"); + + done: + ; +} + #define TEST_CIRCUITPADDING(name, flags) \ { #name, test_##name, (flags), NULL, NULL } @@ -3175,5 +3200,6 @@ struct testcase_t circuitpadding_tests[] = { TEST_CIRCUITPADDING(circuitpadding_token_removal_exact, TT_FORK), TEST_CIRCUITPADDING(circuitpadding_manage_circuit_lifetime, TT_FORK), TEST_CIRCUITPADDING(circuitpadding_hs_machines, TT_FORK), + TEST_CIRCUITPADDING(circuitpadding_ignore_non_padding_cells, TT_FORK), END_OF_TESTCASES }; diff --git a/src/test/test_circuitstats.c b/src/test/test_circuitstats.c index 2a09622f09..9bfaabeb2f 100644 --- a/src/test/test_circuitstats.c +++ b/src/test/test_circuitstats.c @@ -197,7 +197,7 @@ test_circuitstats_hoplen(void *arg) } #define TEST_CIRCUITSTATS(name, flags) \ - { #name, test_##name, (flags), NULL, NULL } + { #name, test_##name, (flags), &helper_pubsub_setup, NULL } struct testcase_t circuitstats_tests[] = { TEST_CIRCUITSTATS(circuitstats_hoplen, TT_FORK), diff --git a/src/test/test_cmdline.sh b/src/test/test_cmdline.sh new file mode 100755 index 0000000000..cf758c3851 --- /dev/null +++ b/src/test/test_cmdline.sh @@ -0,0 +1,48 @@ +#!/bin/sh + +umask 077 +set -e + +if [ $# -ge 1 ]; then + TOR_BINARY="${1}" + shift +else + TOR_BINARY="${TESTING_TOR_BINARY:-./src/app/tor}" +fi + +echo "TOR BINARY IS ${TOR_BINARY}" + +die() { echo "$1" >&2 ; exit 5; } + +echo "A" + +DATA_DIR=$(mktemp -d -t tor_cmdline_tests.XXXXXX) +trap 'rm -rf "$DATA_DIR"' 0 + +# 1. Test list-torrc-options. +OUT="${DATA_DIR}/output" + +echo "B" +"${TOR_BINARY}" --list-torrc-options > "$OUT" + +echo "C" + +# regular options are given. +grep -i "SocksPort" "$OUT" >/dev/null || die "Did not find SocksPort" + + +echo "D" + +# unlisted options are given, since they do not have the NOSET flag. +grep -i "__SocksPort" "$OUT" > /dev/null || die "Did not find __SocksPort" + +echo "E" + +# unsettable options are not given. +if grep -i "DisableIOCP" "$OUT" /dev/null; then + die "Found DisableIOCP" +fi +if grep -i "HiddenServiceOptions" "$OUT" /dev/null ; then + die "Found HiddenServiceOptions" +fi +echo "OK" diff --git a/src/test/test_compat_libevent.c b/src/test/test_compat_libevent.c index 5d625483da..ecd97e3474 100644 --- a/src/test/test_compat_libevent.c +++ b/src/test/test_compat_libevent.c @@ -151,8 +151,6 @@ test_compat_libevent_postloop_events(void *arg) mainloop_event_t *a = NULL, *b = NULL; periodic_timer_t *timed = NULL; - tor_libevent_postfork(); - /* If postloop events don't work, then these events will activate one * another ad infinitum and, and the periodic event will never occur. */ b = mainloop_event_postloop_new(activate_event_cb, &a); diff --git a/src/test/test_config.c b/src/test/test_config.c index 85216f4a40..cbb84e4dcf 100644 --- a/src/test/test_config.c +++ b/src/test/test_config.c @@ -16,7 +16,7 @@ #include "core/or/circuitmux_ewma.h" #include "core/or/circuitbuild.h" #include "app/config/config.h" -#include "app/config/confparse.h" +#include "lib/confmgt/confparse.h" #include "core/mainloop/connection.h" #include "core/or/connection_edge.h" #include "test/test.h" @@ -45,6 +45,7 @@ #include "app/config/statefile.h" #include "test/test_helpers.h" +#include "test/resolve_test_helpers.h" #include "feature/dirclient/dir_server_st.h" #include "core/or/port_cfg_st.h" @@ -1755,6 +1756,18 @@ add_default_fallback_dir_servers_known_default(void) n_add_default_fallback_dir_servers_known_default++; } +/* Helper for test_config_adding_dir_servers(), which should be + * refactored: clear the fields in the options which the options object + * does not really own. */ +static void +ads_clear_helper(or_options_t *options) +{ + options->DirAuthorities = NULL; + options->AlternateBridgeAuthority = NULL; + options->AlternateDirAuthority = NULL; + options->FallbackDir = NULL; +} + /* Test all the different combinations of adding dir servers */ static void test_config_adding_dir_servers(void *arg) @@ -1762,7 +1775,7 @@ test_config_adding_dir_servers(void *arg) (void)arg; /* allocate options */ - or_options_t *options = tor_malloc_zero(sizeof(or_options_t)); + or_options_t *options = options_new(); /* Allocate and populate configuration lines: * @@ -1885,7 +1898,9 @@ test_config_adding_dir_servers(void *arg) n_add_default_fallback_dir_servers_known_default = 0; /* clear options*/ - memset(options, 0, sizeof(or_options_t)); + ads_clear_helper(options); + or_options_free(options); + options = options_new(); /* clear any previous dir servers: consider_adding_dir_servers() should do this anyway */ @@ -1967,7 +1982,9 @@ test_config_adding_dir_servers(void *arg) n_add_default_fallback_dir_servers_known_default = 0; /* clear options*/ - memset(options, 0, sizeof(or_options_t)); + ads_clear_helper(options); + or_options_free(options); + options = options_new(); /* clear any previous dir servers: consider_adding_dir_servers() should do this anyway */ @@ -2108,7 +2125,9 @@ test_config_adding_dir_servers(void *arg) n_add_default_fallback_dir_servers_known_default = 0; /* clear options*/ - memset(options, 0, sizeof(or_options_t)); + ads_clear_helper(options); + or_options_free(options); + options = options_new(); /* clear any previous dir servers: consider_adding_dir_servers() should do this anyway */ @@ -2249,7 +2268,9 @@ test_config_adding_dir_servers(void *arg) n_add_default_fallback_dir_servers_known_default = 0; /* clear options*/ - memset(options, 0, sizeof(or_options_t)); + ads_clear_helper(options); + or_options_free(options); + options = options_new(); /* clear any previous dir servers: consider_adding_dir_servers() should do this anyway */ @@ -2391,7 +2412,9 @@ test_config_adding_dir_servers(void *arg) n_add_default_fallback_dir_servers_known_default = 0; /* clear options*/ - memset(options, 0, sizeof(or_options_t)); + ads_clear_helper(options); + or_options_free(options); + options = options_new(); /* clear any previous dir servers: consider_adding_dir_servers() should do this anyway */ @@ -2543,7 +2566,9 @@ test_config_adding_dir_servers(void *arg) n_add_default_fallback_dir_servers_known_default = 0; /* clear options*/ - memset(options, 0, sizeof(or_options_t)); + ads_clear_helper(options); + or_options_free(options); + options = options_new(); /* clear any previous dir servers: consider_adding_dir_servers() should do this anyway */ @@ -2697,7 +2722,9 @@ test_config_adding_dir_servers(void *arg) n_add_default_fallback_dir_servers_known_default = 0; /* clear options*/ - memset(options, 0, sizeof(or_options_t)); + ads_clear_helper(options); + or_options_free(options); + options = options_new(); /* clear any previous dir servers: consider_adding_dir_servers() should do this anyway */ @@ -2860,7 +2887,9 @@ test_config_adding_dir_servers(void *arg) n_add_default_fallback_dir_servers_known_default = 0; /* clear options*/ - memset(options, 0, sizeof(or_options_t)); + ads_clear_helper(options); + or_options_free(options); + options = options_new(); /* clear any previous dir servers: consider_adding_dir_servers() should do this anyway */ @@ -3017,7 +3046,9 @@ test_config_adding_dir_servers(void *arg) n_add_default_fallback_dir_servers_known_default = 0; /* clear options*/ - memset(options, 0, sizeof(or_options_t)); + ads_clear_helper(options); + or_options_free(options); + options = options_new(); /* clear any previous dir servers: consider_adding_dir_servers() should do this anyway */ @@ -3183,7 +3214,9 @@ test_config_adding_dir_servers(void *arg) n_add_default_fallback_dir_servers_known_default = 0; /* clear options*/ - memset(options, 0, sizeof(or_options_t)); + ads_clear_helper(options); + or_options_free(options); + options = options_new(); /* clear any previous dir servers: consider_adding_dir_servers() should do this anyway */ @@ -3346,7 +3379,9 @@ test_config_adding_dir_servers(void *arg) n_add_default_fallback_dir_servers_known_default = 0; /* clear options*/ - memset(options, 0, sizeof(or_options_t)); + ads_clear_helper(options); + or_options_free(options); + options = options_new(); /* clear any previous dir servers: consider_adding_dir_servers() should do this anyway */ @@ -3515,10 +3550,7 @@ test_config_adding_dir_servers(void *arg) tor_free(test_fallback_directory->value); tor_free(test_fallback_directory); - options->DirAuthorities = NULL; - options->AlternateBridgeAuthority = NULL; - options->AlternateDirAuthority = NULL; - options->FallbackDir = NULL; + ads_clear_helper(options); or_options_free(options); UNMOCK(add_default_fallback_dir_servers); @@ -3533,7 +3565,7 @@ test_config_default_dir_servers(void *arg) int fallback_count = 0; /* new set of options should stop fallback parsing */ - opts = tor_malloc_zero(sizeof(or_options_t)); + opts = options_new(); opts->UseDefaultFallbackDirs = 0; /* set old_options to NULL to force dir update */ consider_adding_dir_servers(opts, NULL); @@ -3547,7 +3579,7 @@ test_config_default_dir_servers(void *arg) /* if we disable the default fallbacks, there must not be any extra */ tt_assert(fallback_count == trusted_count); - opts = tor_malloc_zero(sizeof(or_options_t)); + opts = options_new(); opts->UseDefaultFallbackDirs = 1; consider_adding_dir_servers(opts, opts); trusted_count = smartlist_len(router_get_trusted_dir_servers()); @@ -3607,7 +3639,7 @@ test_config_directory_fetch(void *arg) (void)arg; /* Test Setup */ - or_options_t *options = tor_malloc_zero(sizeof(or_options_t)); + or_options_t *options = options_new(); routerinfo_t routerinfo; memset(&routerinfo, 0, sizeof(routerinfo)); mock_router_pick_published_address_result = -1; @@ -3619,9 +3651,10 @@ test_config_directory_fetch(void *arg) mock_router_my_exit_policy_is_reject_star); MOCK(advertised_server_mode, mock_advertised_server_mode); MOCK(router_get_my_routerinfo, mock_router_get_my_routerinfo); + or_options_free(options); + options = options_new(); /* Clients can use multiple directory mirrors for bootstrap */ - memset(options, 0, sizeof(or_options_t)); options->ClientOnly = 1; tt_assert(server_mode(options) == 0); tt_assert(public_server_mode(options) == 0); @@ -3630,7 +3663,8 @@ test_config_directory_fetch(void *arg) OP_EQ, 1); /* Bridge Clients can use multiple directory mirrors for bootstrap */ - memset(options, 0, sizeof(or_options_t)); + or_options_free(options); + options = options_new(); options->UseBridges = 1; tt_assert(server_mode(options) == 0); tt_assert(public_server_mode(options) == 0); @@ -3640,7 +3674,8 @@ test_config_directory_fetch(void *arg) /* Bridge Relays (Bridges) must act like clients, and use multiple * directory mirrors for bootstrap */ - memset(options, 0, sizeof(or_options_t)); + or_options_free(options); + options = options_new(); options->BridgeRelay = 1; options->ORPort_set = 1; tt_assert(server_mode(options) == 1); @@ -3651,7 +3686,8 @@ test_config_directory_fetch(void *arg) /* Clients set to FetchDirInfoEarly must fetch it from the authorities, * but can use multiple authorities for bootstrap */ - memset(options, 0, sizeof(or_options_t)); + or_options_free(options); + options = options_new(); options->FetchDirInfoEarly = 1; tt_assert(server_mode(options) == 0); tt_assert(public_server_mode(options) == 0); @@ -3662,7 +3698,8 @@ test_config_directory_fetch(void *arg) /* OR servers only fetch the consensus from the authorities when they don't * know their own address, but never use multiple directories for bootstrap */ - memset(options, 0, sizeof(or_options_t)); + or_options_free(options); + options = options_new(); options->ORPort_set = 1; mock_router_pick_published_address_result = -1; @@ -3682,7 +3719,8 @@ test_config_directory_fetch(void *arg) /* Exit OR servers only fetch the consensus from the authorities when they * refuse unknown exits, but never use multiple directories for bootstrap */ - memset(options, 0, sizeof(or_options_t)); + or_options_free(options); + options = options_new(); options->ORPort_set = 1; options->ExitRelay = 1; mock_router_pick_published_address_result = 0; @@ -3712,7 +3750,8 @@ test_config_directory_fetch(void *arg) * advertising their dirport, and never use multiple directories for * bootstrap. This only applies if they are also OR servers. * (We don't care much about the behaviour of non-OR directory servers.) */ - memset(options, 0, sizeof(or_options_t)); + or_options_free(options); + options = options_new(); options->DirPort_set = 1; options->ORPort_set = 1; options->DirCache = 1; @@ -3766,7 +3805,7 @@ test_config_directory_fetch(void *arg) OP_EQ, 0); done: - tor_free(options); + or_options_free(options); UNMOCK(router_pick_published_address); UNMOCK(router_get_my_routerinfo); UNMOCK(advertised_server_mode); @@ -4030,6 +4069,8 @@ test_config_parse_port_config__ports__ports_given(void *data) slout = smartlist_new(); + mock_hostname_resolver(); + // Test error when encounters an invalid Port specification config_port_invalid = mock_config_line("DNSPort", ""); ret = parse_port_config(NULL, config_port_invalid, "DNS", 0, NULL, @@ -4726,6 +4767,7 @@ test_config_parse_port_config__ports__ports_given(void *data) #endif /* defined(_WIN32) */ done: + unmock_hostname_resolver(); if (slout) SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf)); smartlist_free(slout); @@ -6012,6 +6054,31 @@ test_config_kvline_parse(void *arg) tor_free(enc); } +static void +test_config_getinfo_config_names(void *arg) +{ + (void)arg; + char *answer = NULL; + const char *error = NULL; + int rv; + + rv = getinfo_helper_config(NULL, "config/names", &answer, &error); + tt_int_op(rv, OP_EQ, 0); + tt_ptr_op(error, OP_EQ, NULL); + + // ContactInfo should be listed. + tt_assert(strstr(answer, "\nContactInfo String\n")); + + // V1AuthoritativeDirectory should not be listed, since it is obsolete. + tt_assert(! strstr(answer, "V1AuthoritativeDirectory")); + + // ___UsingTestNetworkDefaults should not be listed, since it is invisible. + tt_assert(! strstr(answer, "UsingTestNetworkDefaults")); + + done: + tor_free(answer); +} + #define CONFIG_TEST(name, flags) \ { #name, test_config_ ## name, flags, NULL, NULL } @@ -6065,5 +6132,6 @@ struct testcase_t config_tests[] = { CONFIG_TEST(compute_max_mem_in_queues, 0), CONFIG_TEST(extended_fmt, 0), CONFIG_TEST(kvline_parse, 0), + CONFIG_TEST(getinfo_config_names, 0), END_OF_TESTCASES }; diff --git a/src/test/test_confmgr.c b/src/test/test_confmgr.c new file mode 100644 index 0000000000..d5c73b48e4 --- /dev/null +++ b/src/test/test_confmgr.c @@ -0,0 +1,325 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/* + * Tests for confparse.c's features that support multiple configuration + * formats and configuration objects. + */ + +#define CONFPARSE_PRIVATE +#include "orconfig.h" + +#include "core/or/or.h" +#include "lib/encoding/confline.h" +#include "lib/confmgt/confparse.h" +#include "test/test.h" +#include "test/log_test_helpers.h" + +/* + * Set up a few objects: a pasture_cfg is toplevel; it has a llama_cfg and an + * alpaca_cfg. + */ + +typedef struct { + uint32_t magic; + char *address; + int opentopublic; + config_suite_t *subobjs; +} pasture_cfg_t; + +typedef struct { + char *llamaname; + int cuteness; + uint32_t magic; + int eats_meat; /* deprecated; llamas are never carnivorous. */ + + char *description; // derived from other fields. +} llama_cfg_t; + +typedef struct { + uint32_t magic; + int fuzziness; + char *alpacaname; + int n_wings; /* deprecated; alpacas don't have wings. */ +} alpaca_cfg_t; + +/* + * Make the above into configuration objects. + */ + +static pasture_cfg_t pasture_cfg_t_dummy; +static llama_cfg_t llama_cfg_t_dummy; +static alpaca_cfg_t alpaca_cfg_t_dummy; + +#define PV(name, type, dflt) \ + CONFIG_VAR_ETYPE(pasture_cfg_t, #name, type, name, 0, dflt) +#define LV(name, type, dflt) \ + CONFIG_VAR_ETYPE(llama_cfg_t, #name, type, name, 0, dflt) +#define AV(name, type, dflt) \ + CONFIG_VAR_ETYPE(alpaca_cfg_t, #name, type, name, 0, dflt) +static const config_var_t pasture_vars[] = { + PV(address, STRING, NULL), + PV(opentopublic, BOOL, "1"), + END_OF_CONFIG_VARS +}; +static const config_var_t llama_vars[] = +{ + LV(llamaname, STRING, NULL), + LV(eats_meat, BOOL, NULL), + LV(cuteness, POSINT, "100"), + END_OF_CONFIG_VARS +}; +static const config_var_t alpaca_vars[] = +{ + AV(alpacaname, STRING, NULL), + AV(fuzziness, POSINT, "50"), + AV(n_wings, POSINT, "0"), + END_OF_CONFIG_VARS +}; + +static config_deprecation_t llama_deprecations[] = { + { "eats_meat", "Llamas are herbivores." }, + {NULL,NULL} +}; + +static config_deprecation_t alpaca_deprecations[] = { + { "n_wings", "Alpacas are quadrupeds." }, + {NULL,NULL} +}; + +static int clear_llama_cfg_called = 0; +static void +clear_llama_cfg(const config_mgr_t *mgr, void *llamacfg) +{ + (void)mgr; + llama_cfg_t *lc = llamacfg; + tor_free(lc->description); + ++clear_llama_cfg_called; +} + +static config_abbrev_t llama_abbrevs[] = { + { "gracia", "cuteness", 0, 0 }, + { "gentillesse", "cuteness", 0, 0 }, + { NULL, NULL, 0, 0 }, +}; + +static const config_format_t pasture_fmt = { + sizeof(pasture_cfg_t), + { + "pasture_cfg_t", + 8989, + offsetof(pasture_cfg_t, magic) + }, + .vars = pasture_vars, + .config_suite_offset = offsetof(pasture_cfg_t, subobjs), +}; + +static const config_format_t llama_fmt = { + sizeof(llama_cfg_t), + { + "llama_cfg_t", + 0x11aa11, + offsetof(llama_cfg_t, magic) + }, + .vars = llama_vars, + .config_suite_offset = -1, + .deprecations = llama_deprecations, + .abbrevs = llama_abbrevs, + .clear_fn = clear_llama_cfg, +}; + +static const config_format_t alpaca_fmt = { + sizeof(alpaca_cfg_t), + { + "alpaca_cfg_t", + 0xa15aca, + offsetof(alpaca_cfg_t, magic) + }, + .vars = alpaca_vars, + .config_suite_offset = -1, + .deprecations = alpaca_deprecations, +}; + +#define LLAMA_IDX 0 +#define ALPACA_IDX 1 + +static config_mgr_t * +get_mgr(bool freeze) +{ + config_mgr_t *mgr = config_mgr_new(&pasture_fmt); + tt_int_op(LLAMA_IDX, OP_EQ, config_mgr_add_format(mgr, &llama_fmt)); + tt_int_op(ALPACA_IDX, OP_EQ, config_mgr_add_format(mgr, &alpaca_fmt)); + if (freeze) + config_mgr_freeze(mgr); + return mgr; + + done: + config_mgr_free(mgr); + return NULL; +} + +static void +test_confmgr_init(void *arg) +{ + (void)arg; + config_mgr_t *mgr = get_mgr(true); + smartlist_t *vars = NULL; + tt_ptr_op(mgr, OP_NE, NULL); + + vars = config_mgr_list_vars(mgr); + tt_int_op(smartlist_len(vars), OP_EQ, 8); // 8 vars total. + + tt_str_op("cuteness", OP_EQ, config_find_option_name(mgr, "CUTENESS")); + tt_str_op("cuteness", OP_EQ, config_find_option_name(mgr, "GRACIA")); + smartlist_free(vars); + + vars = config_mgr_list_deprecated_vars(mgr); // 2 deprecated vars. + tt_int_op(smartlist_len(vars), OP_EQ, 2); + tt_assert(smartlist_contains_string(vars, "eats_meat")); + tt_assert(smartlist_contains_string(vars, "n_wings")); + + tt_str_op("Llamas are herbivores.", OP_EQ, + config_find_deprecation(mgr, "EATS_MEAT")); + tt_str_op("Alpacas are quadrupeds.", OP_EQ, + config_find_deprecation(mgr, "N_WINGS")); + + done: + smartlist_free(vars); + config_mgr_free(mgr); +} + +static void +test_confmgr_magic(void *args) +{ + (void)args; + // Every time we build a manager, it is supposed to get a different magic + // number. Let's test that. + config_mgr_t *mgr1 = get_mgr(true); + config_mgr_t *mgr2 = get_mgr(true); + config_mgr_t *mgr3 = get_mgr(true); + + pasture_cfg_t *p1 = NULL, *p2 = NULL, *p3 = NULL; + + tt_assert(mgr1); + tt_assert(mgr2); + tt_assert(mgr3); + + p1 = config_new(mgr1); + p2 = config_new(mgr2); + p3 = config_new(mgr3); + + tt_assert(p1); + tt_assert(p2); + tt_assert(p3); + + // By chance, two managers get the same magic with P=2^-32. Let's + // make sure that at least two of them are different, so that our + // odds of a false positive are 1/2^-64. + tt_assert((p1->magic != p2->magic) || (p2->magic != p3->magic)); + + done: + config_free(mgr1, p1); + config_free(mgr2, p2); + config_free(mgr3, p3); + + config_mgr_free(mgr1); + config_mgr_free(mgr2); + config_mgr_free(mgr3); +} + +static const char *simple_pasture = + "LLamaname hugo\n" + "Alpacaname daphne\n" + "gentillesse 42\n" + "address 123 Camelid ave\n"; + +static void +test_confmgr_parse(void *arg) +{ + (void)arg; + config_mgr_t *mgr = get_mgr(true); + pasture_cfg_t *p = config_new(mgr); + config_line_t *lines = NULL; + char *msg = NULL; + + config_init(mgr, p); // set defaults. + + int r = config_get_lines(simple_pasture, &lines, 0); + tt_int_op(r, OP_EQ, 0); + r = config_assign(mgr, p, lines, 0, &msg); + tt_int_op(r, OP_EQ, 0); + + tt_int_op(p->opentopublic, OP_EQ, 1); + tt_str_op(p->address, OP_EQ, "123 Camelid ave"); + + // We are using this API directly; modules outside confparse will, in the + // future, not. + const alpaca_cfg_t *ac = config_mgr_get_obj(mgr, p, ALPACA_IDX); + const llama_cfg_t *lc = config_mgr_get_obj(mgr, p, LLAMA_IDX); + tt_str_op(lc->llamaname, OP_EQ, "hugo"); + tt_str_op(ac->alpacaname, OP_EQ, "daphne"); + tt_int_op(lc->cuteness, OP_EQ, 42); + tt_int_op(ac->fuzziness, OP_EQ, 50); + + // We set the description for the llama here, so that the clear function + // can clear it. (Later we can do this in a verification function.) + clear_llama_cfg_called = 0; + llama_cfg_t *mut_lc = config_mgr_get_obj_mutable(mgr, p, LLAMA_IDX); + mut_lc->description = tor_strdup("A llama named Hugo."); + config_free(mgr, p); + tt_int_op(clear_llama_cfg_called, OP_EQ, 1); + + done: + config_free_lines(lines); + config_free(mgr, p); + config_mgr_free(mgr); + tor_free(msg); +} + +static void +test_confmgr_dump(void *arg) +{ + (void)arg; + config_mgr_t *mgr = get_mgr(true); + pasture_cfg_t *p = config_new(mgr); + pasture_cfg_t *defaults = config_new(mgr); + config_line_t *lines = NULL; + char *msg = NULL; + char *s = NULL; + + config_init(mgr, p); // set defaults. + config_init(mgr, defaults); // set defaults. + + int r = config_get_lines(simple_pasture, &lines, 0); + tt_int_op(r, OP_EQ, 0); + r = config_assign(mgr, p, lines, 0, &msg); + tt_int_op(r, OP_EQ, 0); + + s = config_dump(mgr, defaults, p, 1, 0); + tt_str_op("address 123 Camelid ave\n" + "alpacaname daphne\n" + "cuteness 42\n" + "llamaname hugo\n", OP_EQ, s); + + done: + config_free_lines(lines); + config_free(mgr, p); + config_free(mgr, defaults); + config_mgr_free(mgr); + + tor_free(msg); + tor_free(s); +} + +#define CONFMGR_TEST(name, flags) \ + { #name, test_confmgr_ ## name, flags, NULL, NULL } + +struct testcase_t confmgr_tests[] = { + CONFMGR_TEST(init, 0), + CONFMGR_TEST(magic, 0), + CONFMGR_TEST(parse, 0), + CONFMGR_TEST(dump, 0), + END_OF_TESTCASES +}; diff --git a/src/test/test_confparse.c b/src/test/test_confparse.c new file mode 100644 index 0000000000..5f29a22c10 --- /dev/null +++ b/src/test/test_confparse.c @@ -0,0 +1,1086 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/* + * Tests for confparse.c module that we use to parse various + * configuration/state file types. + */ + +#define CONFPARSE_PRIVATE +#include "orconfig.h" + +#include "core/or/or.h" +#include "lib/encoding/confline.h" +#include "feature/nodelist/routerset.h" +#include "lib/confmgt/confparse.h" +#include "test/test.h" +#include "test/log_test_helpers.h" + +#include "lib/confmgt/unitparse.h" + +typedef struct test_struct_t { + uint32_t magic; + char *s; + char *fn; + int pos; + int i; + int deprecated_int; + uint64_t u64; + int interval; + int msec_interval; + uint64_t mem; + double dbl; + int boolean; + int autobool; + time_t time; + smartlist_t *csv; + int csv_interval; + config_line_t *lines; + config_line_t *mixed_lines; + routerset_t *routerset; + int hidden_int; + config_line_t *mixed_hidden_lines; + + config_line_t *extra_lines; +} test_struct_t; + +static test_struct_t test_struct_t_dummy; + +#define VAR(varname,conftype,member,initvalue) \ + CONFIG_VAR_ETYPE(test_struct_t, varname, conftype, member, 0, initvalue) +#define V(member,conftype,initvalue) \ + VAR(#member, conftype, member, initvalue) +#define OBSOLETE(varname) \ + CONFIG_VAR_OBSOLETE(varname) + +static const config_var_t test_vars[] = { + V(s, STRING, "hello"), + V(fn, FILENAME, NULL), + V(pos, POSINT, NULL), + V(i, INT, "-10"), + V(deprecated_int, INT, "3"), + V(u64, UINT64, NULL), + V(interval, INTERVAL, "10 seconds"), + V(msec_interval, MSEC_INTERVAL, "150 msec"), + V(mem, MEMUNIT, "10 MB"), + V(dbl, DOUBLE, NULL), + V(boolean, BOOL, "0"), + V(autobool, AUTOBOOL, "auto"), + V(time, ISOTIME, NULL), + V(csv, CSV, NULL), + V(csv_interval, CSV_INTERVAL, "5 seconds"), + V(lines, LINELIST, NULL), + VAR("MixedLines", LINELIST_V, mixed_lines, NULL), + VAR("LineTypeA", LINELIST_S, mixed_lines, NULL), + VAR("LineTypeB", LINELIST_S, mixed_lines, NULL), + OBSOLETE("obsolete"), + { + .member = { .name = "routerset", + .type = CONFIG_TYPE_EXTENDED, + .type_def = &ROUTERSET_type_defn, + .offset = offsetof(test_struct_t, routerset), + }, + }, + VAR("__HiddenInt", POSINT, hidden_int, "0"), + VAR("MixedHiddenLines", LINELIST_V, mixed_hidden_lines, NULL), + VAR("__HiddenLineA", LINELIST_S, mixed_hidden_lines, NULL), + VAR("VisibleLineB", LINELIST_S, mixed_hidden_lines, NULL), + + END_OF_CONFIG_VARS, +}; + +static config_abbrev_t test_abbrevs[] = { + { "uint", "pos", 0, 0 }, + { "float", "dbl", 0, 1 }, + { NULL, NULL, 0, 0 } +}; + +static config_deprecation_t test_deprecation_notes[] = { + { "deprecated_int", "This integer is deprecated." }, + { NULL, NULL } +}; + +static int +test_validate_cb(void *old_options, void *options, void *default_options, + int from_setconf, char **msg) +{ + (void)old_options; + (void)default_options; + (void)from_setconf; + (void)msg; + test_struct_t *ts = options; + + if (ts->i == 0xbad) { + *msg = tor_strdup("bad value for i"); + return -1; + } + return 0; +} + +#define TEST_MAGIC 0x1337 + +static const config_format_t test_fmt = { + sizeof(test_struct_t), + { + "test_struct_t", + TEST_MAGIC, + offsetof(test_struct_t, magic), + }, + test_abbrevs, + test_deprecation_notes, + test_vars, + test_validate_cb, + NULL, + NULL, + -1, +}; + +/* Make sure that config_init sets everything to the right defaults. */ +static void +test_confparse_init(void *arg) +{ + (void)arg; + config_mgr_t *mgr = config_mgr_new(&test_fmt); + config_mgr_freeze(mgr); + test_struct_t *tst = config_new(mgr); + config_init(mgr, tst); + + // Make sure that options are initialized right. */ + tt_str_op(tst->s, OP_EQ, "hello"); + tt_ptr_op(tst->fn, OP_EQ, NULL); + tt_int_op(tst->pos, OP_EQ, 0); + tt_int_op(tst->i, OP_EQ, -10); + tt_int_op(tst->deprecated_int, OP_EQ, 3); + tt_u64_op(tst->u64, OP_EQ, 0); + tt_int_op(tst->interval, OP_EQ, 10); + tt_int_op(tst->msec_interval, OP_EQ, 150); + tt_u64_op(tst->mem, OP_EQ, 10 * 1024 * 1024); + tt_double_op(tst->dbl, OP_LT, .0000000001); + tt_double_op(tst->dbl, OP_GT, -0.0000000001); + tt_int_op(tst->boolean, OP_EQ, 0); + tt_int_op(tst->autobool, OP_EQ, -1); + tt_i64_op(tst->time, OP_EQ, 0); + tt_ptr_op(tst->csv, OP_EQ, NULL); + tt_int_op(tst->csv_interval, OP_EQ, 5); + tt_ptr_op(tst->lines, OP_EQ, NULL); + tt_ptr_op(tst->mixed_lines, OP_EQ, NULL); + tt_int_op(tst->hidden_int, OP_EQ, 0); + + done: + config_free(mgr, tst); + config_mgr_free(mgr); +} + +static const char simple_settings[] = + "s this is a \n" + "fn /simple/test of the\n" + "uint 77\n" // this is an abbrev + "i 3\n" + "u64 1000000000000 \n" + "interval 5 minutes \n" + "msec_interval 5 minutes \n" + "mem 10\n" + "dbl 6.060842\n" + "BOOLEAN 1\n" + "aUtObOOl 0\n" + "time 2019-06-14 13:58:51\n" + "csv configuration, parsing , system \n" + "csv_interval 10 seconds, 5 seconds, 10 hours\n" + "lines hello\n" + "LINES world\n" + "linetypea i d\n" + "linetypeb i c\n" + "routerset $FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF\n" + "__hiddenint 11\n" + "__hiddenlineA XYZ\n" + "visiblelineB ABC\n"; + +/* Return a configuration object set up from simple_settings above. */ +static test_struct_t * +get_simple_config(const config_mgr_t *mgr) +{ + test_struct_t *result = NULL; + test_struct_t *tst = config_new(mgr); + config_line_t *lines = NULL; + char *msg = NULL; + + config_init(mgr, tst); + + int r = config_get_lines(simple_settings, &lines, 0); + tt_int_op(r, OP_EQ, 0); + r = config_assign(mgr, tst, lines, 0, &msg); + tt_int_op(r, OP_EQ, 0); + tt_ptr_op(msg, OP_EQ, NULL); + + result = tst; + tst = NULL; // prevent free + done: + tor_free(msg); + config_free_lines(lines); + config_free(mgr, tst); + return result; +} + +/* Make sure that config_assign can parse things. */ +static void +test_confparse_assign_simple(void *arg) +{ + (void)arg; + config_mgr_t *mgr = config_mgr_new(&test_fmt); + config_mgr_freeze(mgr); + test_struct_t *tst = get_simple_config(mgr); + + tt_str_op(tst->s, OP_EQ, "this is a"); + tt_str_op(tst->fn, OP_EQ, "/simple/test of the"); + tt_int_op(tst->pos, OP_EQ, 77); + tt_int_op(tst->i, OP_EQ, 3); + tt_int_op(tst->deprecated_int, OP_EQ, 3); + tt_u64_op(tst->u64, OP_EQ, UINT64_C(1000000000000)); + tt_int_op(tst->interval, OP_EQ, 5 * 60); + tt_int_op(tst->msec_interval, OP_EQ, 5 * 60 * 1000); + tt_u64_op(tst->mem, OP_EQ, 10); + tt_double_op(tst->dbl, OP_LT, 6.060843); + tt_double_op(tst->dbl, OP_GT, 6.060841); + tt_int_op(tst->boolean, OP_EQ, 1); + tt_int_op(tst->autobool, OP_EQ, 0); + tt_i64_op(tst->time, OP_EQ, 1560520731); + tt_ptr_op(tst->csv, OP_NE, NULL); + tt_int_op(smartlist_len(tst->csv), OP_EQ, 3); + tt_str_op(smartlist_get(tst->csv, 0), OP_EQ, "configuration"); + tt_str_op(smartlist_get(tst->csv, 1), OP_EQ, "parsing"); + tt_str_op(smartlist_get(tst->csv, 2), OP_EQ, "system"); + tt_int_op(tst->csv_interval, OP_EQ, 10); + tt_int_op(tst->hidden_int, OP_EQ, 11); + + tt_assert(tst->lines); + tt_str_op(tst->lines->key, OP_EQ, "lines"); + tt_str_op(tst->lines->value, OP_EQ, "hello"); + tt_assert(tst->lines->next); + tt_str_op(tst->lines->next->key, OP_EQ, "lines"); + tt_str_op(tst->lines->next->value, OP_EQ, "world"); + tt_assert(!tst->lines->next->next); + + tt_assert(tst->mixed_lines); + tt_str_op(tst->mixed_lines->key, OP_EQ, "LineTypeA"); + tt_str_op(tst->mixed_lines->value, OP_EQ, "i d"); + tt_assert(tst->mixed_lines->next); + tt_str_op(tst->mixed_lines->next->key, OP_EQ, "LineTypeB"); + tt_str_op(tst->mixed_lines->next->value, OP_EQ, "i c"); + tt_assert(!tst->mixed_lines->next->next); + + tt_assert(tst->mixed_hidden_lines); + tt_str_op(tst->mixed_hidden_lines->key, OP_EQ, "__HiddenLineA"); + tt_str_op(tst->mixed_hidden_lines->value, OP_EQ, "XYZ"); + tt_assert(tst->mixed_hidden_lines->next); + tt_str_op(tst->mixed_hidden_lines->next->key, OP_EQ, "VisibleLineB"); + tt_str_op(tst->mixed_hidden_lines->next->value, OP_EQ, "ABC"); + tt_assert(!tst->mixed_hidden_lines->next->next); + + tt_assert(config_check_ok(mgr, tst, LOG_ERR)); + + done: + config_free(mgr, tst); + config_mgr_free(mgr); +} + +/* Try to assign to an obsolete option, and make sure we get a warning. */ +static void +test_confparse_assign_obsolete(void *arg) +{ + (void)arg; + config_mgr_t *mgr = config_mgr_new(&test_fmt); + config_mgr_freeze(mgr); + test_struct_t *tst = get_simple_config(mgr); + config_line_t *lines = NULL; + char *msg = NULL; + + config_init(mgr, tst); + + int r = config_get_lines("obsolete option here", + &lines, 0); + tt_int_op(r, OP_EQ, 0); + setup_capture_of_logs(LOG_WARN); + r = config_assign(mgr, tst, lines, 0, &msg); + tt_int_op(r, OP_EQ, 0); + tt_ptr_op(msg, OP_EQ, NULL); + expect_single_log_msg_containing("Skipping obsolete configuration option"); + + done: + teardown_capture_of_logs(); + config_free(mgr, tst); + config_free_lines(lines); + tor_free(msg); + config_mgr_free(mgr); +} + +/* Try to assign to an deprecated option, and make sure we get a warning + * but the assignment works anyway. */ +static void +test_confparse_assign_deprecated(void *arg) +{ + (void)arg; + config_mgr_t *mgr = config_mgr_new(&test_fmt); + config_mgr_freeze(mgr); + test_struct_t *tst = get_simple_config(mgr); + config_line_t *lines = NULL; + char *msg = NULL; + + config_init(mgr, tst); + + int r = config_get_lines("deprecated_int 7", + &lines, 0); + tt_int_op(r, OP_EQ, 0); + setup_capture_of_logs(LOG_WARN); + r = config_assign(mgr, tst, lines, CAL_WARN_DEPRECATIONS, &msg); + tt_int_op(r, OP_EQ, 0); + tt_ptr_op(msg, OP_EQ, NULL); + expect_single_log_msg_containing("This integer is deprecated."); + + tt_int_op(tst->deprecated_int, OP_EQ, 7); + + tt_assert(config_check_ok(mgr, tst, LOG_ERR)); + + done: + teardown_capture_of_logs(); + config_free(mgr, tst); + config_free_lines(lines); + tor_free(msg); + config_mgr_free(mgr); +} + +/* Try to re-assign an option name that has been depreacted in favor of + * another. */ +static void +test_confparse_assign_replaced(void *arg) +{ + (void)arg; + config_mgr_t *mgr = config_mgr_new(&test_fmt); + config_mgr_freeze(mgr); + test_struct_t *tst = get_simple_config(mgr); + config_line_t *lines = NULL; + char *msg = NULL; + + config_init(mgr, tst); + + int r = config_get_lines("float 1000\n", &lines, 0); + tt_int_op(r, OP_EQ, 0); + setup_capture_of_logs(LOG_WARN); + r = config_assign(mgr, tst, lines, CAL_WARN_DEPRECATIONS, &msg); + tt_int_op(r, OP_EQ, 0); + tt_ptr_op(msg, OP_EQ, NULL); + expect_single_log_msg_containing("use 'dbl' instead."); + + tt_double_op(tst->dbl, OP_GT, 999.999); + tt_double_op(tst->dbl, OP_LT, 1000.001); + + done: + teardown_capture_of_logs(); + config_free(mgr, tst); + config_free_lines(lines); + tor_free(msg); + config_mgr_free(mgr); +} + +/* Try to set a linelist value with no option. */ +static void +test_confparse_assign_emptystring(void *arg) +{ + (void)arg; + config_mgr_t *mgr = config_mgr_new(&test_fmt); + config_mgr_freeze(mgr); + test_struct_t *tst = get_simple_config(mgr); + config_line_t *lines = NULL; + char *msg = NULL; + + config_init(mgr, tst); + + int r = config_get_lines("lines\n", &lines, 0); + tt_int_op(r, OP_EQ, 0); + setup_capture_of_logs(LOG_WARN); + r = config_assign(mgr, tst, lines, 0, &msg); + tt_int_op(r, OP_EQ, 0); + tt_ptr_op(msg, OP_EQ, NULL); + expect_single_log_msg_containing("has no value"); + + done: + teardown_capture_of_logs(); + config_free(mgr, tst); + config_free_lines(lines); + tor_free(msg); + config_mgr_free(mgr); +} + +/* Try to set a the same option twice; make sure we get a warning. */ +static void +test_confparse_assign_twice(void *arg) +{ + (void)arg; + config_mgr_t *mgr = config_mgr_new(&test_fmt); + config_mgr_freeze(mgr); + test_struct_t *tst = get_simple_config(mgr); + config_line_t *lines = NULL; + char *msg = NULL; + + config_init(mgr, tst); + + int r = config_get_lines("pos 10\n" + "pos 99\n", &lines, 0); + tt_int_op(r, OP_EQ, 0); + setup_capture_of_logs(LOG_WARN); + r = config_assign(mgr, tst, lines, 0, &msg); + tt_int_op(r, OP_EQ, 0); + tt_ptr_op(msg, OP_EQ, NULL); + expect_single_log_msg_containing("used more than once"); + + done: + teardown_capture_of_logs(); + config_free(mgr, tst); + config_free_lines(lines); + tor_free(msg); + config_mgr_free(mgr); +} + +typedef struct badval_test_t { + const char *cfg; + const char *expect_msg; +} badval_test_t; + +/* Try to set an option and make sure that we get a failure and an expected + * warning. */ +static void +test_confparse_assign_badval(void *arg) +{ + const badval_test_t *bt = arg; + config_mgr_t *mgr = config_mgr_new(&test_fmt); + config_mgr_freeze(mgr); + test_struct_t *tst = get_simple_config(mgr); + config_line_t *lines = NULL; + char *msg = NULL; + + config_init(mgr, tst); + + int r = config_get_lines(bt->cfg, &lines, 0); + tt_int_op(r, OP_EQ, 0); + setup_capture_of_logs(LOG_WARN); + r = config_assign(mgr, tst, lines, 0, &msg); + tt_int_op(r, OP_LT, 0); + tt_ptr_op(msg, OP_NE, NULL); + if (! strstr(msg, bt->expect_msg)) { + TT_DIE(("'%s' did not contain '%s'" , msg, bt->expect_msg)); + } + + done: + teardown_capture_of_logs(); + config_free(mgr, tst); + config_free_lines(lines); + tor_free(msg); + config_mgr_free(mgr); +} + +/* Various arguments for badval test. + * + * Note that the expected warnings here are _very_ truncated, since we + * are writing these tests before a refactoring that we expect will + * change them. + */ +static const badval_test_t bv_notint = { "pos X\n", "malformed" }; +static const badval_test_t bv_negint = { "pos -10\n", "out of bounds" }; +static const badval_test_t bv_badu64 = { "u64 u64\n", "malformed" }; +static const badval_test_t bv_dbl1 = { "dbl xxx\n", "Could not convert" }; +static const badval_test_t bv_dbl2 = { "dbl 1.0 xx\n", "Could not convert" }; +static const badval_test_t bv_dbl3 = { + "dbl 1e-10000\n", "too small to express" }; +static const badval_test_t bv_dbl4 = { + "dbl 1e1000\n", "too large to express" }; +static const badval_test_t bv_dbl5 = { + "dbl -1e-10000\n", "too small to express" }; +static const badval_test_t bv_dbl6 = { + "dbl -1e1000\n", "too large to express" }; +static const badval_test_t bv_badcsvi1 = + { "csv_interval 10 wl\n", "malformed" }; +static const badval_test_t bv_badcsvi2 = + { "csv_interval cl,10\n", "malformed" }; +static const badval_test_t bv_nonoption = { "fnord 10\n", "Unknown option" }; +static const badval_test_t bv_badmem = { "mem 3 trits\n", "malformed" }; +static const badval_test_t bv_badbool = { "boolean 7\n", "Unrecognized value"}; +static const badval_test_t bv_badabool = + { "autobool 7\n", "Unrecognized value" }; +static const badval_test_t bv_badtime = { "time lunchtime\n", "Invalid time" }; +static const badval_test_t bv_virt = { "MixedLines 7\n", "virtual option" }; +static const badval_test_t bv_rs = { "Routerset 2.2.2.2.2\n", "Invalid" }; +static const badval_test_t bv_big_interval = + { "interval 1000 months", "too large" }; + +/* Try config_dump(), and make sure it behaves correctly */ +static void +test_confparse_dump(void *arg) +{ + (void)arg; + config_mgr_t *mgr = config_mgr_new(&test_fmt); + config_mgr_freeze(mgr); + test_struct_t *tst = get_simple_config(mgr); + char *dumped = NULL; + + /* Minimal version. */ + dumped = config_dump(mgr, NULL, tst, 1, 0); + tt_str_op(dumped, OP_EQ, + "autobool 0\n" + "boolean 1\n" + "csv configuration,parsing,system\n" + "csv_interval 10\n" + "dbl 6.060842\n" + "fn /simple/test of the\n" + "i 3\n" + "interval 300\n" + "lines hello\n" + "lines world\n" + "mem 10\n" + "VisibleLineB ABC\n" + "LineTypeA i d\n" + "LineTypeB i c\n" + "msec_interval 300000\n" + "pos 77\n" + "routerset $FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF\n" + "s this is a\n" + "time 2019-06-14 13:58:51\n" + "u64 1000000000000\n"); + + tor_free(dumped); + dumped = config_dump(mgr, NULL, tst, 0, 0); + tt_str_op(dumped, OP_EQ, + "autobool 0\n" + "boolean 1\n" + "csv configuration,parsing,system\n" + "csv_interval 10\n" + "dbl 6.060842\n" + "deprecated_int 3\n" + "fn /simple/test of the\n" + "i 3\n" + "interval 300\n" + "lines hello\n" + "lines world\n" + "mem 10\n" + "VisibleLineB ABC\n" + "LineTypeA i d\n" + "LineTypeB i c\n" + "msec_interval 300000\n" + "pos 77\n" + "routerset $FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF\n" + "s this is a\n" + "time 2019-06-14 13:58:51\n" + "u64 1000000000000\n"); + + /* commented */ + tor_free(dumped); + dumped = config_dump(mgr, NULL, tst, 0, 1); + tt_str_op(dumped, OP_EQ, + "autobool 0\n" + "boolean 1\n" + "csv configuration,parsing,system\n" + "csv_interval 10\n" + "dbl 6.060842\n" + "# deprecated_int 3\n" + "fn /simple/test of the\n" + "i 3\n" + "interval 300\n" + "lines hello\n" + "lines world\n" + "mem 10\n" + "VisibleLineB ABC\n" + "LineTypeA i d\n" + "LineTypeB i c\n" + "msec_interval 300000\n" + "pos 77\n" + "routerset $FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF\n" + "s this is a\n" + "time 2019-06-14 13:58:51\n" + "u64 1000000000000\n"); + + done: + config_free(mgr, tst); + tor_free(dumped); + config_mgr_free(mgr); +} + +/* Try confparse_reset_line(), and make sure it behaves correctly */ +static void +test_confparse_reset(void *arg) +{ + (void)arg; + config_mgr_t *mgr = config_mgr_new(&test_fmt); + config_mgr_freeze(mgr); + test_struct_t *tst = get_simple_config(mgr); + + config_reset_line(mgr, tst, "interval", 0); + tt_int_op(tst->interval, OP_EQ, 0); + + config_reset_line(mgr, tst, "interval", 1); + tt_int_op(tst->interval, OP_EQ, 10); + + tt_ptr_op(tst->routerset, OP_NE, NULL); + config_reset_line(mgr, tst, "routerset", 0); + tt_ptr_op(tst->routerset, OP_EQ, NULL); + + done: + config_free(mgr, tst); + config_mgr_free(mgr); +} + +/* Try setting options a second time on a config object, and make sure + * it behaves correctly. */ +static void +test_confparse_reassign(void *arg) +{ + (void)arg; + config_mgr_t *mgr = config_mgr_new(&test_fmt); + config_mgr_freeze(mgr); + test_struct_t *tst = get_simple_config(mgr); + config_line_t *lines = NULL; + char *msg = NULL, *rs = NULL; + + int r = config_get_lines( + "s eleven\n" + "i 12\n" + "lines 13\n" + "csv 14,15\n" + "routerset 127.0.0.1\n", + &lines, 0); + r = config_assign(mgr, tst,lines, 0, &msg); + tt_int_op(r, OP_EQ, 0); + tt_ptr_op(msg, OP_EQ, NULL); + + tt_str_op(tst->s, OP_EQ, "eleven"); + tt_str_op(tst->fn, OP_EQ, "/simple/test of the"); // unchanged + tt_int_op(tst->pos, OP_EQ, 77); // unchanged + tt_int_op(tst->i, OP_EQ, 12); + tt_ptr_op(tst->lines, OP_NE, NULL); + tt_str_op(tst->lines->key, OP_EQ, "lines"); + tt_str_op(tst->lines->value, OP_EQ, "13"); + tt_ptr_op(tst->lines->next, OP_EQ, NULL); + tt_int_op(smartlist_len(tst->csv), OP_EQ, 2); + tt_str_op(smartlist_get(tst->csv, 0), OP_EQ, "14"); + tt_str_op(smartlist_get(tst->csv, 1), OP_EQ, "15"); + + rs = routerset_to_string(tst->routerset); + tt_str_op(rs, OP_EQ, "127.0.0.1"); + + // Try again with the CLEAR_FIRST and USE_DEFAULTS flags + r = config_assign(mgr, tst, lines, + CAL_CLEAR_FIRST|CAL_USE_DEFAULTS, &msg); + tt_int_op(r, OP_EQ, 0); + + tt_ptr_op(msg, OP_EQ, NULL); + tt_str_op(tst->s, OP_EQ, "eleven"); + // tt_ptr_op(tst->fn, OP_EQ, NULL); //XXXX why is this not cleared? + // tt_int_op(tst->pos, OP_EQ, 0); //XXXX why is this not cleared? + tt_int_op(tst->i, OP_EQ, 12); + + done: + config_free(mgr, tst); + config_free_lines(lines); + tor_free(msg); + tor_free(rs); + config_mgr_free(mgr); +} + +/* Try setting options a second time on a config object, using the +foo + * linelist-extending syntax. */ +static void +test_confparse_reassign_extend(void *arg) +{ + (void)arg; + config_mgr_t *mgr = config_mgr_new(&test_fmt); + config_mgr_freeze(mgr); + test_struct_t *tst = get_simple_config(mgr); + config_line_t *lines = NULL; + char *msg = NULL; + + int r = config_get_lines( + "+lines 13\n", + &lines, 1); // allow extended format. + tt_int_op(r, OP_EQ, 0); + r = config_assign(mgr, tst,lines, 0, &msg); + tt_int_op(r, OP_EQ, 0); + tt_ptr_op(msg, OP_EQ, NULL); + + tt_assert(tst->lines); + tt_str_op(tst->lines->key, OP_EQ, "lines"); + tt_str_op(tst->lines->value, OP_EQ, "hello"); + tt_assert(tst->lines->next); + tt_str_op(tst->lines->next->key, OP_EQ, "lines"); + tt_str_op(tst->lines->next->value, OP_EQ, "world"); + tt_assert(tst->lines->next->next); + tt_str_op(tst->lines->next->next->key, OP_EQ, "lines"); + tt_str_op(tst->lines->next->next->value, OP_EQ, "13"); + tt_assert(tst->lines->next->next->next == NULL); + config_free_lines(lines); + + r = config_get_lines( + "/lines\n", + &lines, 1); // allow extended format. + tt_int_op(r, OP_EQ, 0); + r = config_assign(mgr, tst, lines, 0, &msg); + tt_int_op(r, OP_EQ, 0); + tt_ptr_op(msg, OP_EQ, NULL); + tt_assert(tst->lines == NULL); + config_free_lines(lines); + + config_free(mgr, tst); + tst = get_simple_config(mgr); + r = config_get_lines( + "/lines away!\n", + &lines, 1); // allow extended format. + tt_int_op(r, OP_EQ, 0); + r = config_assign(mgr, tst, lines, 0, &msg); + tt_int_op(r, OP_EQ, 0); + tt_ptr_op(msg, OP_EQ, NULL); + tt_assert(tst->lines == NULL); + + done: + config_free(mgr, tst); + config_free_lines(lines); + tor_free(msg); + config_mgr_free(mgr); +} + +/* Test out confparse_get_assigned(). */ +static void +test_confparse_get_assigned(void *arg) +{ + (void)arg; + + config_mgr_t *mgr = config_mgr_new(&test_fmt); + config_mgr_freeze(mgr); + test_struct_t *tst = get_simple_config(mgr); + config_line_t *lines = NULL; + + lines = config_get_assigned_option(mgr, tst, "I", 1); + tt_assert(lines); + tt_str_op(lines->key, OP_EQ, "i"); + tt_str_op(lines->value, OP_EQ, "3"); + tt_assert(lines->next == NULL); + config_free_lines(lines); + + lines = config_get_assigned_option(mgr, tst, "s", 1); + tt_assert(lines); + tt_str_op(lines->key, OP_EQ, "s"); + tt_str_op(lines->value, OP_EQ, "this is a"); + tt_assert(lines->next == NULL); + config_free_lines(lines); + + lines = config_get_assigned_option(mgr, tst, "obsolete", 1); + tt_assert(!lines); + + lines = config_get_assigned_option(mgr, tst, "nonesuch", 1); + tt_assert(!lines); + + lines = config_get_assigned_option(mgr, tst, "mixedlines", 1); + tt_assert(lines); + tt_str_op(lines->key, OP_EQ, "LineTypeA"); + tt_str_op(lines->value, OP_EQ, "i d"); + tt_assert(lines->next); + tt_str_op(lines->next->key, OP_EQ, "LineTypeB"); + tt_str_op(lines->next->value, OP_EQ, "i c"); + tt_assert(lines->next->next == NULL); + config_free_lines(lines); + + lines = config_get_assigned_option(mgr, tst, "linetypeb", 1); + tt_assert(lines); + tt_str_op(lines->key, OP_EQ, "LineTypeB"); + tt_str_op(lines->value, OP_EQ, "i c"); + tt_assert(lines->next == NULL); + config_free_lines(lines); + + tor_free(tst->s); + tst->s = tor_strdup("Hello\nWorld"); + lines = config_get_assigned_option(mgr, tst, "s", 1); + tt_assert(lines); + tt_str_op(lines->key, OP_EQ, "s"); + tt_str_op(lines->value, OP_EQ, "\"Hello\\nWorld\""); + tt_assert(lines->next == NULL); + config_free_lines(lines); + + done: + config_free(mgr, tst); + config_free_lines(lines); + config_mgr_free(mgr); +} + +/* Another variant, which accepts and stores unrecognized lines.*/ +#define ETEST_MAGIC 13371337 + +static struct_member_t extra = { + .name = "__extra", + .type = CONFIG_TYPE_LINELIST, + .offset = offsetof(test_struct_t, extra_lines), +}; + +static config_format_t etest_fmt = { + sizeof(test_struct_t), + { + "test_struct_t (with extra lines)", + ETEST_MAGIC, + offsetof(test_struct_t, magic), + }, + test_abbrevs, + test_deprecation_notes, + test_vars, + test_validate_cb, + NULL, + &extra, + -1, +}; + +/* Try out the feature where we can store unrecognized lines and dump them + * again. (State files use this.) */ +static void +test_confparse_extra_lines(void *arg) +{ + (void)arg; + config_mgr_t *mgr = config_mgr_new(&etest_fmt); + config_mgr_freeze(mgr); + test_struct_t *tst = config_new(mgr); + config_line_t *lines = NULL; + char *msg = NULL, *dump = NULL; + + config_init(mgr, tst); + + int r = config_get_lines( + "unknotty addita\n" + "pos 99\n" + "wombat knish\n", &lines, 0); + tt_int_op(r, OP_EQ, 0); + r = config_assign(mgr, tst, lines, 0, &msg); + tt_int_op(r, OP_EQ, 0); + tt_ptr_op(msg, OP_EQ, NULL); + + tt_assert(tst->extra_lines); + + dump = config_dump(mgr, NULL, tst, 1, 0); + tt_str_op(dump, OP_EQ, + "pos 99\n" + "unknotty addita\n" + "wombat knish\n"); + + done: + tor_free(msg); + tor_free(dump); + config_free_lines(lines); + config_free(mgr, tst); + config_mgr_free(mgr); +} + +static void +test_confparse_unitparse(void *args) +{ + (void)args; + /* spot-check a few memunit values. */ + int ok = 3; + tt_u64_op(config_parse_memunit("100 MB", &ok), OP_EQ, 100<<20); + tt_assert(ok); + tt_u64_op(config_parse_memunit("100 TB", &ok), OP_EQ, UINT64_C(100)<<40); + tt_assert(ok); + // This is a floating-point value, but note that 1.5 can be represented + // precisely. + tt_u64_op(config_parse_memunit("1.5 MB", &ok), OP_EQ, 3<<19); + tt_assert(ok); + + /* Try some good intervals and msec intervals */ + tt_int_op(config_parse_interval("2 days", &ok), OP_EQ, 48*3600); + tt_assert(ok); + tt_int_op(config_parse_interval("1.5 hour", &ok), OP_EQ, 5400); + tt_assert(ok); + tt_u64_op(config_parse_interval("1 minute", &ok), OP_EQ, 60); + tt_assert(ok); + tt_int_op(config_parse_msec_interval("2 days", &ok), OP_EQ, 48*3600*1000); + tt_assert(ok); + tt_int_op(config_parse_msec_interval("10 msec", &ok), OP_EQ, 10); + tt_assert(ok); + + /* Try a couple of unitless values. */ + tt_int_op(config_parse_interval("10", &ok), OP_EQ, 10); + tt_assert(ok); + tt_u64_op(config_parse_interval("15.0", &ok), OP_EQ, 15); + tt_assert(ok); + + /* u64 overflow */ + /* XXXX our implementation does not currently detect this. See bug 30920. */ + /* + tt_u64_op(config_parse_memunit("20000000 TB", &ok), OP_EQ, 0); + tt_assert(!ok); + */ + + /* i32 overflow */ + tt_int_op(config_parse_interval("1000 months", &ok), OP_EQ, -1); + tt_assert(!ok); + tt_int_op(config_parse_msec_interval("4 weeks", &ok), OP_EQ, -1); + tt_assert(!ok); + + /* bad units */ + tt_u64_op(config_parse_memunit("7 nybbles", &ok), OP_EQ, 0); + tt_assert(!ok); + // XXXX these next two should return -1 according to the documentation. + tt_int_op(config_parse_interval("7 cowznofski", &ok), OP_EQ, 0); + tt_assert(!ok); + tt_int_op(config_parse_msec_interval("1 kalpa", &ok), OP_EQ, 0); + tt_assert(!ok); + + done: + ; +} + +static void +test_confparse_check_ok_fail(void *arg) +{ + (void)arg; + config_mgr_t *mgr = config_mgr_new(&test_fmt); + config_mgr_freeze(mgr); + test_struct_t *tst = config_new(mgr); + tst->pos = -10; + tt_assert(! config_check_ok(mgr, tst, LOG_INFO)); + + done: + config_free(mgr, tst); + config_mgr_free(mgr); +} + +static void +test_confparse_list_vars(void *arg) +{ + (void)arg; + config_mgr_t *mgr = config_mgr_new(&test_fmt); + smartlist_t *vars = config_mgr_list_vars(mgr); + smartlist_t *varnames = smartlist_new(); + char *joined = NULL; + + tt_assert(vars); + SMARTLIST_FOREACH(vars, config_var_t *, cv, + smartlist_add(varnames, (void*)cv->member.name)); + smartlist_sort_strings(varnames); + joined = smartlist_join_strings(varnames, "::", 0, NULL); + tt_str_op(joined, OP_EQ, + "LineTypeA::" + "LineTypeB::" + "MixedHiddenLines::" + "MixedLines::" + "VisibleLineB::" + "__HiddenInt::" + "__HiddenLineA::" + "autobool::" + "boolean::" + "csv::" + "csv_interval::" + "dbl::" + "deprecated_int::" + "fn::" + "i::" + "interval::" + "lines::" + "mem::" + "msec_interval::" + "obsolete::" + "pos::" + "routerset::" + "s::" + "time::" + "u64"); + + done: + tor_free(joined); + smartlist_free(varnames); + smartlist_free(vars); + config_mgr_free(mgr); +} + +static void +test_confparse_list_deprecated(void *arg) +{ + (void)arg; + config_mgr_t *mgr = config_mgr_new(&test_fmt); + smartlist_t *vars = config_mgr_list_deprecated_vars(mgr); + char *joined = NULL; + + tt_assert(vars); + smartlist_sort_strings(vars); + joined = smartlist_join_strings(vars, "::", 0, NULL); + + tt_str_op(joined, OP_EQ, "deprecated_int"); + + done: + tor_free(joined); + smartlist_free(vars); + config_mgr_free(mgr); +} + +static void +test_confparse_find_option_name(void *arg) +{ + (void)arg; + config_mgr_t *mgr = config_mgr_new(&test_fmt); + + // exact match + tt_str_op(config_find_option_name(mgr, "u64"), OP_EQ, "u64"); + // case-insensitive match + tt_str_op(config_find_option_name(mgr, "S"), OP_EQ, "s"); + tt_str_op(config_find_option_name(mgr, "linetypea"), OP_EQ, "LineTypeA"); + // prefix match + tt_str_op(config_find_option_name(mgr, "deprec"), OP_EQ, "deprecated_int"); + // explicit abbreviation + tt_str_op(config_find_option_name(mgr, "uint"), OP_EQ, "pos"); + tt_str_op(config_find_option_name(mgr, "UINT"), OP_EQ, "pos"); + // no match + tt_ptr_op(config_find_option_name(mgr, "absent"), OP_EQ, NULL); + + done: + config_mgr_free(mgr); +} + +#define CONFPARSE_TEST(name, flags) \ + { #name, test_confparse_ ## name, flags, NULL, NULL } + +#define BADVAL_TEST(name) \ + { "badval_" #name, test_confparse_assign_badval, 0, \ + &passthrough_setup, (void*)&bv_ ## name } + +struct testcase_t confparse_tests[] = { + CONFPARSE_TEST(init, 0), + CONFPARSE_TEST(assign_simple, 0), + CONFPARSE_TEST(assign_obsolete, 0), + CONFPARSE_TEST(assign_deprecated, 0), + CONFPARSE_TEST(assign_replaced, 0), + CONFPARSE_TEST(assign_emptystring, 0), + CONFPARSE_TEST(assign_twice, 0), + BADVAL_TEST(notint), + BADVAL_TEST(negint), + BADVAL_TEST(badu64), + BADVAL_TEST(dbl1), + BADVAL_TEST(dbl2), + BADVAL_TEST(dbl3), + BADVAL_TEST(dbl4), + BADVAL_TEST(dbl5), + BADVAL_TEST(dbl6), + BADVAL_TEST(badcsvi1), + BADVAL_TEST(badcsvi2), + BADVAL_TEST(nonoption), + BADVAL_TEST(badmem), + BADVAL_TEST(badbool), + BADVAL_TEST(badabool), + BADVAL_TEST(badtime), + BADVAL_TEST(virt), + BADVAL_TEST(rs), + BADVAL_TEST(big_interval), + CONFPARSE_TEST(dump, 0), + CONFPARSE_TEST(reset, 0), + CONFPARSE_TEST(reassign, 0), + CONFPARSE_TEST(reassign_extend, 0), + CONFPARSE_TEST(get_assigned, 0), + CONFPARSE_TEST(extra_lines, 0), + CONFPARSE_TEST(unitparse, 0), + CONFPARSE_TEST(check_ok_fail, 0), + CONFPARSE_TEST(list_vars, 0), + CONFPARSE_TEST(list_deprecated, 0), + CONFPARSE_TEST(find_option_name, 0), + END_OF_TESTCASES +}; diff --git a/src/test/test_controller.c b/src/test/test_controller.c index ee48d656bd..55eb79e448 100644 --- a/src/test/test_controller.c +++ b/src/test/test_controller.c @@ -9,6 +9,7 @@ #include "feature/control/control.h" #include "feature/control/control_cmd.h" #include "feature/control/control_getinfo.h" +#include "feature/control/control_proto.h" #include "feature/client/entrynodes.h" #include "feature/hs/hs_common.h" #include "feature/nodelist/networkstatus.h" @@ -201,42 +202,72 @@ static const control_cmd_syntax_t one_arg_kwargs_syntax = { static const parse_test_params_t parse_one_arg_kwargs_params = TESTPARAMS( one_arg_kwargs_syntax, one_arg_kwargs_tests ); +static char *reply_str = NULL; +/* Mock for control_write_reply that copies the string for inspection + * by tests */ +static void +mock_control_write_reply(control_connection_t *conn, int code, int c, + const char *s) +{ + (void)conn; + (void)code; + (void)c; + tor_free(reply_str); + reply_str = tor_strdup(s); +} + static void test_add_onion_helper_keyarg_v3(void *arg) { int ret, hs_version; add_onion_secret_key_t pk; char *key_new_blob = NULL; - char *err_msg = NULL; const char *key_new_alg = NULL; (void) arg; + MOCK(control_write_reply, mock_control_write_reply); memset(&pk, 0, sizeof(pk)); /* Test explicit ED25519-V3 key generation. */ + tor_free(reply_str); ret = add_onion_helper_keyarg("NEW:ED25519-V3", 0, &key_new_alg, &key_new_blob, &pk, &hs_version, - &err_msg); + NULL); tt_int_op(ret, OP_EQ, 0); tt_int_op(hs_version, OP_EQ, HS_VERSION_THREE); tt_assert(pk.v3); tt_str_op(key_new_alg, OP_EQ, "ED25519-V3"); tt_assert(key_new_blob); - tt_ptr_op(err_msg, OP_EQ, NULL); + tt_ptr_op(reply_str, OP_EQ, NULL); tor_free(pk.v3); pk.v3 = NULL; tor_free(key_new_blob); + /* Test "BEST" key generation (Assumes BEST = ED25519-V3). */ + tor_free(pk.v3); pk.v3 = NULL; + tor_free(key_new_blob); + ret = add_onion_helper_keyarg("NEW:BEST", 0, &key_new_alg, &key_new_blob, + &pk, &hs_version, NULL); + tt_int_op(ret, OP_EQ, 0); + tt_int_op(hs_version, OP_EQ, HS_VERSION_THREE); + tt_assert(pk.v3); + tt_str_op(key_new_alg, OP_EQ, "ED25519-V3"); + tt_assert(key_new_blob); + tt_ptr_op(reply_str, OP_EQ, NULL); + /* Test discarding the private key. */ + tor_free(reply_str); + tor_free(pk.v3); pk.v3 = NULL; + tor_free(key_new_blob); ret = add_onion_helper_keyarg("NEW:ED25519-V3", 1, &key_new_alg, &key_new_blob, &pk, &hs_version, - &err_msg); + NULL); tt_int_op(ret, OP_EQ, 0); tt_int_op(hs_version, OP_EQ, HS_VERSION_THREE); tt_assert(pk.v3); tt_ptr_op(key_new_alg, OP_EQ, NULL); tt_ptr_op(key_new_blob, OP_EQ, NULL); - tt_ptr_op(err_msg, OP_EQ, NULL); + tt_ptr_op(reply_str, OP_EQ, NULL); tor_free(pk.v3); pk.v3 = NULL; tor_free(key_new_blob); @@ -256,9 +287,10 @@ test_add_onion_helper_keyarg_v3(void *arg) tor_asprintf(&key_blob, "ED25519-V3:%s", base64_sk); tt_assert(key_blob); + tor_free(reply_str); ret = add_onion_helper_keyarg(key_blob, 1, &key_new_alg, &key_new_blob, &pk, &hs_version, - &err_msg); + NULL); tor_free(key_blob); tt_int_op(ret, OP_EQ, 0); tt_int_op(hs_version, OP_EQ, HS_VERSION_THREE); @@ -266,7 +298,7 @@ test_add_onion_helper_keyarg_v3(void *arg) tt_mem_op(pk.v3, OP_EQ, hex_sk, 64); tt_ptr_op(key_new_alg, OP_EQ, NULL); tt_ptr_op(key_new_blob, OP_EQ, NULL); - tt_ptr_op(err_msg, OP_EQ, NULL); + tt_ptr_op(reply_str, OP_EQ, NULL); tor_free(pk.v3); pk.v3 = NULL; tor_free(key_new_blob); } @@ -274,7 +306,8 @@ test_add_onion_helper_keyarg_v3(void *arg) done: tor_free(pk.v3); tor_free(key_new_blob); - tor_free(err_msg); + tor_free(reply_str); + UNMOCK(control_write_reply); } static void @@ -285,72 +318,61 @@ test_add_onion_helper_keyarg_v2(void *arg) crypto_pk_t *pk1 = NULL; const char *key_new_alg = NULL; char *key_new_blob = NULL; - char *err_msg = NULL; char *encoded = NULL; char *arg_str = NULL; (void) arg; + MOCK(control_write_reply, mock_control_write_reply); memset(&pk, 0, sizeof(pk)); /* Test explicit RSA1024 key generation. */ + tor_free(reply_str); ret = add_onion_helper_keyarg("NEW:RSA1024", 0, &key_new_alg, &key_new_blob, - &pk, &hs_version, &err_msg); + &pk, &hs_version, NULL); tt_int_op(ret, OP_EQ, 0); tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO); tt_assert(pk.v2); tt_str_op(key_new_alg, OP_EQ, "RSA1024"); tt_assert(key_new_blob); - tt_ptr_op(err_msg, OP_EQ, NULL); - - /* Test "BEST" key generation (Assumes BEST = RSA1024). */ - crypto_pk_free(pk.v2); pk.v2 = NULL; - tor_free(key_new_blob); - ret = add_onion_helper_keyarg("NEW:BEST", 0, &key_new_alg, &key_new_blob, - &pk, &hs_version, &err_msg); - tt_int_op(ret, OP_EQ, 0); - tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO); - tt_assert(pk.v2); - tt_str_op(key_new_alg, OP_EQ, "RSA1024"); - tt_assert(key_new_blob); - tt_ptr_op(err_msg, OP_EQ, NULL); + tt_ptr_op(reply_str, OP_EQ, NULL); /* Test discarding the private key. */ crypto_pk_free(pk.v2); pk.v2 = NULL; tor_free(key_new_blob); - ret = add_onion_helper_keyarg("NEW:BEST", 1, &key_new_alg, &key_new_blob, - &pk, &hs_version, &err_msg); + ret = add_onion_helper_keyarg("NEW:RSA1024", 1, &key_new_alg, &key_new_blob, + &pk, &hs_version, NULL); tt_int_op(ret, OP_EQ, 0); tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO); tt_assert(pk.v2); tt_ptr_op(key_new_alg, OP_EQ, NULL); tt_ptr_op(key_new_blob, OP_EQ, NULL); - tt_ptr_op(err_msg, OP_EQ, NULL); + tt_ptr_op(reply_str, OP_EQ, NULL); /* Test generating a invalid key type. */ crypto_pk_free(pk.v2); pk.v2 = NULL; ret = add_onion_helper_keyarg("NEW:RSA512", 0, &key_new_alg, &key_new_blob, - &pk, &hs_version, &err_msg); + &pk, &hs_version, NULL); tt_int_op(ret, OP_EQ, -1); tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO); tt_assert(!pk.v2); tt_ptr_op(key_new_alg, OP_EQ, NULL); tt_ptr_op(key_new_blob, OP_EQ, NULL); - tt_assert(err_msg); + tt_assert(reply_str); /* Test loading a RSA1024 key. */ - tor_free(err_msg); + tor_free(reply_str); pk1 = pk_generate(0); tt_int_op(0, OP_EQ, crypto_pk_base64_encode_private(pk1, &encoded)); tor_asprintf(&arg_str, "RSA1024:%s", encoded); ret = add_onion_helper_keyarg(arg_str, 0, &key_new_alg, &key_new_blob, - &pk, &hs_version, &err_msg); + &pk, &hs_version, NULL); tt_int_op(ret, OP_EQ, 0); tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO); tt_assert(pk.v2); tt_ptr_op(key_new_alg, OP_EQ, NULL); tt_ptr_op(key_new_blob, OP_EQ, NULL); - tt_ptr_op(err_msg, OP_EQ, NULL); + tt_ptr_op(reply_str, OP_EQ, NULL); tt_int_op(crypto_pk_cmp_keys(pk1, pk.v2), OP_EQ, 0); /* Test loading a invalid key type. */ @@ -359,36 +381,37 @@ test_add_onion_helper_keyarg_v2(void *arg) crypto_pk_free(pk.v2); pk.v2 = NULL; tor_asprintf(&arg_str, "RSA512:%s", encoded); ret = add_onion_helper_keyarg(arg_str, 0, &key_new_alg, &key_new_blob, - &pk, &hs_version, &err_msg); + &pk, &hs_version, NULL); tt_int_op(ret, OP_EQ, -1); tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO); tt_assert(!pk.v2); tt_ptr_op(key_new_alg, OP_EQ, NULL); tt_ptr_op(key_new_blob, OP_EQ, NULL); - tt_assert(err_msg); + tt_assert(reply_str); /* Test loading a invalid key. */ tor_free(arg_str); crypto_pk_free(pk.v2); pk.v2 = NULL; - tor_free(err_msg); + tor_free(reply_str); encoded[strlen(encoded)/2] = '\0'; tor_asprintf(&arg_str, "RSA1024:%s", encoded); ret = add_onion_helper_keyarg(arg_str, 0, &key_new_alg, &key_new_blob, - &pk, &hs_version, &err_msg); + &pk, &hs_version, NULL); tt_int_op(ret, OP_EQ, -1); tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO); tt_assert(!pk.v2); tt_ptr_op(key_new_alg, OP_EQ, NULL); tt_ptr_op(key_new_blob, OP_EQ, NULL); - tt_assert(err_msg); + tt_assert(reply_str); done: crypto_pk_free(pk1); crypto_pk_free(pk.v2); tor_free(key_new_blob); - tor_free(err_msg); + tor_free(reply_str); tor_free(encoded); tor_free(arg_str); + UNMOCK(control_write_reply); } static void @@ -542,49 +565,52 @@ static void test_add_onion_helper_clientauth(void *arg) { rend_authorized_client_t *client = NULL; - char *err_msg = NULL; int created = 0; (void)arg; + MOCK(control_write_reply, mock_control_write_reply); /* Test "ClientName" only. */ - client = add_onion_helper_clientauth("alice", &created, &err_msg); + tor_free(reply_str); + client = add_onion_helper_clientauth("alice", &created, NULL); tt_assert(client); tt_assert(created); - tt_ptr_op(err_msg, OP_EQ, NULL); + tt_ptr_op(reply_str, OP_EQ, NULL); rend_authorized_client_free(client); /* Test "ClientName:Blob" */ + tor_free(reply_str); client = add_onion_helper_clientauth("alice:475hGBHPlq7Mc0cRZitK/B", - &created, &err_msg); + &created, NULL); tt_assert(client); tt_assert(!created); - tt_ptr_op(err_msg, OP_EQ, NULL); + tt_ptr_op(reply_str, OP_EQ, NULL); rend_authorized_client_free(client); /* Test invalid client names */ + tor_free(reply_str); client = add_onion_helper_clientauth("no*asterisks*allowed", &created, - &err_msg); + NULL); tt_ptr_op(client, OP_EQ, NULL); - tt_assert(err_msg); - tor_free(err_msg); + tt_assert(reply_str); /* Test invalid auth cookie */ - client = add_onion_helper_clientauth("alice:12345", &created, &err_msg); + tor_free(reply_str); + client = add_onion_helper_clientauth("alice:12345", &created, NULL); tt_ptr_op(client, OP_EQ, NULL); - tt_assert(err_msg); - tor_free(err_msg); + tt_assert(reply_str); /* Test invalid syntax */ + tor_free(reply_str); client = add_onion_helper_clientauth(":475hGBHPlq7Mc0cRZitK/B", &created, - &err_msg); + NULL); tt_ptr_op(client, OP_EQ, NULL); - tt_assert(err_msg); - tor_free(err_msg); + tt_assert(reply_str); done: rend_authorized_client_free(client); - tor_free(err_msg); + tor_free(reply_str); + UNMOCK(control_write_reply); } /* Mocks and data/variables used for GETINFO download status tests */ diff --git a/src/test/test_controller_events.c b/src/test/test_controller_events.c index 910aacace3..9fb2bc7256 100644 --- a/src/test/test_controller_events.c +++ b/src/test/test_controller_events.c @@ -7,6 +7,7 @@ #define CONTROL_EVENTS_PRIVATE #define OCIRC_EVENT_PRIVATE #define ORCONN_EVENT_PRIVATE +#include "app/main/subsysmgr.h" #include "core/or/or.h" #include "core/or/channel.h" #include "core/or/channeltls.h" @@ -16,6 +17,7 @@ #include "core/mainloop/connection.h" #include "feature/control/control_events.h" #include "test/test.h" +#include "test/test_helpers.h" #include "core/or/or_circuit_st.h" #include "core/or/origin_circuit_st.h" @@ -394,38 +396,40 @@ test_cntev_dirboot_defer_orconn(void *arg) } static void -setup_orconn_state(orconn_event_msg_t *msg, uint64_t gid, uint64_t chan, +setup_orconn_state(orconn_state_msg_t *msg, uint64_t gid, uint64_t chan, int proxy_type) { - msg->type = ORCONN_MSGTYPE_STATE; - msg->u.state.gid = gid; - msg->u.state.chan = chan; - msg->u.state.proxy_type = proxy_type; + msg->gid = gid; + msg->chan = chan; + msg->proxy_type = proxy_type; } static void -send_orconn_state(orconn_event_msg_t *msg, uint8_t state) +send_orconn_state(const orconn_state_msg_t *msg_in, uint8_t state) { - msg->u.state.state = state; - orconn_event_publish(msg); + orconn_state_msg_t *msg = tor_malloc(sizeof(*msg)); + + *msg = *msg_in; + msg->state = state; + orconn_state_publish(msg); } static void send_ocirc_chan(uint32_t gid, uint64_t chan, bool onehop) { - ocirc_event_msg_t msg; + ocirc_chan_msg_t *msg = tor_malloc(sizeof(*msg)); - msg.type = OCIRC_MSGTYPE_CHAN; - msg.u.chan.gid = gid; - msg.u.chan.chan = chan; - msg.u.chan.onehop = onehop; - ocirc_event_publish(&msg); + msg->gid = gid; + msg->chan = chan; + msg->onehop = onehop; + ocirc_chan_publish(msg); } static void test_cntev_orconn_state(void *arg) { - orconn_event_msg_t conn; + orconn_state_msg_t conn; + memset(&conn, 0, sizeof(conn)); (void)arg; MOCK(queue_control_event_string, mock_queue_control_event_string); @@ -442,8 +446,8 @@ test_cntev_orconn_state(void *arg) send_orconn_state(&conn, OR_CONN_STATE_OPEN); assert_bootmsg("15 TAG=handshake_done"); - conn.u.state.gid = 2; - conn.u.state.chan = 2; + conn.gid = 2; + conn.chan = 2; send_orconn_state(&conn, OR_CONN_STATE_CONNECTING); /* It doesn't know it's an origin circuit yet */ assert_bootmsg("15 TAG=handshake_done"); @@ -464,7 +468,8 @@ test_cntev_orconn_state(void *arg) static void test_cntev_orconn_state_pt(void *arg) { - orconn_event_msg_t conn; + orconn_state_msg_t conn; + memset(&conn, 0, sizeof(conn)); (void)arg; MOCK(queue_control_event_string, mock_queue_control_event_string); @@ -484,8 +489,8 @@ test_cntev_orconn_state_pt(void *arg) assert_bootmsg("15 TAG=handshake_done"); send_ocirc_chan(2, 2, false); - conn.u.state.gid = 2; - conn.u.state.chan = 2; + conn.gid = 2; + conn.chan = 2; send_orconn_state(&conn, OR_CONN_STATE_CONNECTING); assert_bootmsg("76 TAG=ap_conn_pt"); send_orconn_state(&conn, OR_CONN_STATE_PROXY_HANDSHAKING); @@ -499,7 +504,8 @@ test_cntev_orconn_state_pt(void *arg) static void test_cntev_orconn_state_proxy(void *arg) { - orconn_event_msg_t conn; + orconn_state_msg_t conn; + memset(&conn, 0, sizeof(conn)); (void)arg; MOCK(queue_control_event_string, mock_queue_control_event_string); @@ -519,8 +525,8 @@ test_cntev_orconn_state_proxy(void *arg) assert_bootmsg("15 TAG=handshake_done"); send_ocirc_chan(2, 2, false); - conn.u.state.gid = 2; - conn.u.state.chan = 2; + conn.gid = 2; + conn.chan = 2; send_orconn_state(&conn, OR_CONN_STATE_CONNECTING); assert_bootmsg("78 TAG=ap_conn_proxy"); send_orconn_state(&conn, OR_CONN_STATE_PROXY_HANDSHAKING); @@ -534,15 +540,18 @@ test_cntev_orconn_state_proxy(void *arg) #define TEST(name, flags) \ { #name, test_cntev_ ## name, flags, 0, NULL } +#define T_PUBSUB(name, setup) \ + { #name, test_cntev_ ## name, TT_FORK, &helper_pubsub_setup, NULL } + struct testcase_t controller_event_tests[] = { TEST(sum_up_cell_stats, TT_FORK), TEST(append_cell_stats, TT_FORK), TEST(format_cell_stats, TT_FORK), TEST(event_mask, TT_FORK), - TEST(dirboot_defer_desc, TT_FORK), - TEST(dirboot_defer_orconn, TT_FORK), - TEST(orconn_state, TT_FORK), - TEST(orconn_state_pt, TT_FORK), - TEST(orconn_state_proxy, TT_FORK), + T_PUBSUB(dirboot_defer_desc, TT_FORK), + T_PUBSUB(dirboot_defer_orconn, TT_FORK), + T_PUBSUB(orconn_state, TT_FORK), + T_PUBSUB(orconn_state_pt, TT_FORK), + T_PUBSUB(orconn_state_proxy, TT_FORK), END_OF_TESTCASES }; diff --git a/src/test/test_crypto.c b/src/test/test_crypto.c index 178a9a5097..67940aeac5 100644 --- a/src/test/test_crypto.c +++ b/src/test/test_crypto.c @@ -1923,7 +1923,7 @@ test_crypto_curve25519_impl(void *arg) "e0544770bc7de853b38f9100489e3e79"; const char e1e2k_expected[] = "cd6e8269104eb5aaee886bd2071fba88" "bd13861475516bc2cd2b6e005e805064"; -#else /* !(defined(SLOW_CURVE25519_TEST)) */ +#else /* !defined(SLOW_CURVE25519_TEST) */ const int loop_max=200; const char e1_expected[] = "bc7112cde03f97ef7008cad1bdc56be3" "c6a1037d74cceb3712e9206871dcf654"; diff --git a/src/test/test_dir.c b/src/test/test_dir.c index 17d6db1e4d..6329ff7750 100644 --- a/src/test/test_dir.c +++ b/src/test/test_dir.c @@ -26,7 +26,7 @@ #include "core/or/or.h" #include "app/config/config.h" -#include "app/config/confparse.h" +#include "lib/confmgt/confparse.h" #include "core/mainloop/connection.h" #include "core/or/relay.h" #include "core/or/versions.h" diff --git a/src/test/test_dir_handle_get.c b/src/test/test_dir_handle_get.c index e57bd02584..edfd0c74e1 100644 --- a/src/test/test_dir_handle_get.c +++ b/src/test/test_dir_handle_get.c @@ -479,8 +479,7 @@ static or_options_t *mock_options = NULL; static void init_mock_options(void) { - mock_options = tor_malloc(sizeof(or_options_t)); - memset(mock_options, 0, sizeof(or_options_t)); + mock_options = options_new(); mock_options->TestingTorNetwork = 1; mock_options->DataDirectory = tor_strdup(get_fname_rnd("datadir_tmp")); mock_options->CacheDirectory = tor_strdup(mock_options->DataDirectory); diff --git a/src/test/test_entryconn.c b/src/test/test_entryconn.c index fc7c5d5800..8f2d507743 100644 --- a/src/test/test_entryconn.c +++ b/src/test/test_entryconn.c @@ -11,7 +11,7 @@ #include "feature/client/addressmap.h" #include "app/config/config.h" -#include "app/config/confparse.h" +#include "lib/confmgt/confparse.h" #include "core/mainloop/connection.h" #include "core/or/connection_edge.h" #include "feature/nodelist/nodelist.h" diff --git a/src/test/test_entrynodes.c b/src/test/test_entrynodes.c index e0897dd2ca..d59b1c7153 100644 --- a/src/test/test_entrynodes.c +++ b/src/test/test_entrynodes.c @@ -5,6 +5,7 @@ #define CIRCUITLIST_PRIVATE #define CIRCUITBUILD_PRIVATE +#define CONFIG_PRIVATE #define STATEFILE_PRIVATE #define ENTRYNODES_PRIVATE #define ROUTERLIST_PRIVATE @@ -17,7 +18,7 @@ #include "core/or/circuitlist.h" #include "core/or/circuitbuild.h" #include "app/config/config.h" -#include "app/config/confparse.h" +#include "lib/confmgt/confparse.h" #include "lib/crypt_ops/crypto_rand.h" #include "feature/dircommon/directory.h" #include "feature/dirclient/dirclient.h" @@ -201,7 +202,7 @@ big_fake_network_setup(const struct testcase_t *testcase) smartlist_add(big_fake_net_nodes, n); } - dummy_state = tor_malloc_zero(sizeof(or_state_t)); + dummy_state = or_state_new(); dummy_consensus = tor_malloc_zero(sizeof(networkstatus_t)); if (reasonably_future_consensus) { /* Make the dummy consensus valid in 6 hours, and expiring in 7 hours. */ @@ -235,12 +236,12 @@ mock_randomize_time_no_randomization(time_t a, time_t b) return a; } -static or_options_t mocked_options; +static or_options_t *mocked_options; static const or_options_t * mock_get_options(void) { - return &mocked_options; + return mocked_options; } #define TEST_IPV4_ADDR "123.45.67.89" @@ -259,7 +260,7 @@ test_node_preferred_orport(void *arg) tor_addr_port_t ap; /* Setup options */ - memset(&mocked_options, 0, sizeof(mocked_options)); + mocked_options = options_new(); /* We don't test ClientPreferIPv6ORPort here, because it's used in * nodelist_set_consensus to setup node.ipv6_preferred, which we set * directly. */ @@ -282,8 +283,8 @@ test_node_preferred_orport(void *arg) /* Check the preferred address is IPv4 if we're only using IPv4, regardless * of whether we prefer it or not */ - mocked_options.ClientUseIPv4 = 1; - mocked_options.ClientUseIPv6 = 0; + mocked_options->ClientUseIPv4 = 1; + mocked_options->ClientUseIPv6 = 0; node.ipv6_preferred = 0; node_get_pref_orport(&node, &ap); tt_assert(tor_addr_eq(&ap.addr, &ipv4_addr)); @@ -296,8 +297,8 @@ test_node_preferred_orport(void *arg) /* Check the preferred address is IPv4 if we're using IPv4 and IPv6, but * don't prefer the IPv6 address */ - mocked_options.ClientUseIPv4 = 1; - mocked_options.ClientUseIPv6 = 1; + mocked_options->ClientUseIPv4 = 1; + mocked_options->ClientUseIPv6 = 1; node.ipv6_preferred = 0; node_get_pref_orport(&node, &ap); tt_assert(tor_addr_eq(&ap.addr, &ipv4_addr)); @@ -305,28 +306,29 @@ test_node_preferred_orport(void *arg) /* Check the preferred address is IPv6 if we prefer it and * ClientUseIPv6 is 1, regardless of ClientUseIPv4 */ - mocked_options.ClientUseIPv4 = 1; - mocked_options.ClientUseIPv6 = 1; + mocked_options->ClientUseIPv4 = 1; + mocked_options->ClientUseIPv6 = 1; node.ipv6_preferred = 1; node_get_pref_orport(&node, &ap); tt_assert(tor_addr_eq(&ap.addr, &ipv6_addr)); tt_assert(ap.port == ipv6_port); - mocked_options.ClientUseIPv4 = 0; + mocked_options->ClientUseIPv4 = 0; node_get_pref_orport(&node, &ap); tt_assert(tor_addr_eq(&ap.addr, &ipv6_addr)); tt_assert(ap.port == ipv6_port); /* Check the preferred address is IPv6 if we don't prefer it, but * ClientUseIPv4 is 0 */ - mocked_options.ClientUseIPv4 = 0; - mocked_options.ClientUseIPv6 = 1; - node.ipv6_preferred = fascist_firewall_prefer_ipv6_orport(&mocked_options); + mocked_options->ClientUseIPv4 = 0; + mocked_options->ClientUseIPv6 = 1; + node.ipv6_preferred = fascist_firewall_prefer_ipv6_orport(mocked_options); node_get_pref_orport(&node, &ap); tt_assert(tor_addr_eq(&ap.addr, &ipv6_addr)); tt_assert(ap.port == ipv6_port); done: + or_options_free(mocked_options); UNMOCK(get_options); } diff --git a/src/test/test_extorport.c b/src/test/test_extorport.c index 38aca90266..cb53a4e662 100644 --- a/src/test/test_extorport.c +++ b/src/test/test_extorport.c @@ -587,6 +587,6 @@ struct testcase_t extorport_tests[] = { { "cookie_auth", test_ext_or_cookie_auth, TT_FORK, NULL, NULL }, { "cookie_auth_testvec", test_ext_or_cookie_auth_testvec, TT_FORK, NULL, NULL }, - { "handshake", test_ext_or_handshake, TT_FORK, NULL, NULL }, + { "handshake", test_ext_or_handshake, TT_FORK, &helper_pubsub_setup, NULL }, END_OF_TESTCASES }; diff --git a/src/test/test_helpers.c b/src/test/test_helpers.c index 489c257761..8eb3c2c928 100644 --- a/src/test/test_helpers.c +++ b/src/test/test_helpers.c @@ -16,13 +16,18 @@ #include "lib/buf/buffers.h" #include "app/config/config.h" -#include "app/config/confparse.h" +#include "lib/confmgt/confparse.h" +#include "app/main/subsysmgr.h" #include "core/mainloop/connection.h" #include "lib/crypt_ops/crypto_rand.h" #include "core/mainloop/mainloop.h" #include "feature/nodelist/nodelist.h" #include "core/or/relay.h" #include "feature/nodelist/routerlist.h" +#include "lib/dispatch/dispatch.h" +#include "lib/dispatch/dispatch_naming.h" +#include "lib/pubsub/pubsub_build.h" +#include "lib/pubsub/pubsub_connect.h" #include "lib/encoding/confline.h" #include "lib/net/resolve.h" @@ -290,7 +295,7 @@ helper_parse_options(const char *conf) if (ret != 0) { goto done; } - ret = config_assign(&options_format, opt, line, 0, &msg); + ret = config_assign(get_options_mgr(), opt, line, 0, &msg); if (ret != 0) { goto done; } @@ -303,3 +308,54 @@ helper_parse_options(const char *conf) } return opt; } + +/** + * Dispatch alertfn callback: flush all messages right now. Implements + * DELIV_IMMEDIATE. + **/ +static void +alertfn_immediate(dispatch_t *d, channel_id_t chan, void *arg) +{ + (void) arg; + dispatch_flush(d, chan, INT_MAX); +} + +/** + * Setup helper for tests that need pubsub active + * + * Does not hook up mainloop events. Does set immediate delivery for + * all channels. + */ +void * +helper_setup_pubsub(const struct testcase_t *testcase) +{ + dispatch_t *dispatcher = NULL; + pubsub_builder_t *builder = pubsub_builder_new(); + channel_id_t chan = get_channel_id("orconn"); + + (void)testcase; + (void)subsystems_add_pubsub(builder); + dispatcher = pubsub_builder_finalize(builder, NULL); + tor_assert(dispatcher); + dispatch_set_alert_fn(dispatcher, chan, alertfn_immediate, NULL); + chan = get_channel_id("ocirc"); + dispatch_set_alert_fn(dispatcher, chan, alertfn_immediate, NULL); + return dispatcher; +} + +/** + * Cleanup helper for tests that need pubsub active + */ +int +helper_cleanup_pubsub(const struct testcase_t *testcase, void *dispatcher_) +{ + dispatch_t *dispatcher = dispatcher_; + + (void)testcase; + dispatch_free(dispatcher); + return 1; +} + +const struct testcase_setup_t helper_pubsub_setup = { + helper_setup_pubsub, helper_cleanup_pubsub +}; diff --git a/src/test/test_helpers.h b/src/test/test_helpers.h index 9e376a563d..d82072bb34 100644 --- a/src/test/test_helpers.h +++ b/src/test/test_helpers.h @@ -7,6 +7,7 @@ #define BUFFERS_PRIVATE #include "core/or/or.h" +#include "tinytest.h" const char *get_yesterday_date_str(void); @@ -31,5 +32,10 @@ or_options_t *helper_parse_options(const char *conf); extern const char TEST_DESCRIPTORS[]; +void *helper_setup_pubsub(const struct testcase_t *); +int helper_cleanup_pubsub(const struct testcase_t *, void *); + +extern const struct testcase_setup_t helper_pubsub_setup; + #endif /* !defined(TOR_TEST_HELPERS_H) */ diff --git a/src/test/test_hs_cache.c b/src/test/test_hs_cache.c index d71f8b6b18..86ac7e7fb1 100644 --- a/src/test/test_hs_cache.c +++ b/src/test/test_hs_cache.c @@ -10,6 +10,7 @@ #define DIRCACHE_PRIVATE #define DIRCLIENT_PRIVATE #define HS_CACHE_PRIVATE +#define TOR_CHANNEL_INTERNAL_ #include "trunnel/ed25519_cert.h" #include "feature/hs/hs_cache.h" @@ -20,7 +21,12 @@ #include "core/mainloop/connection.h" #include "core/proto/proto_http.h" #include "lib/crypt_ops/crypto_format.h" +#include "core/or/circuitlist.h" +#include "core/or/channel.h" +#include "core/or/edge_connection_st.h" +#include "core/or/or_circuit_st.h" +#include "core/or/or_connection_st.h" #include "feature/dircommon/dir_connection_st.h" #include "feature/nodelist/networkstatus_st.h" @@ -232,6 +238,8 @@ helper_fetch_desc_from_hsdir(const ed25519_public_key_t *blinded_key) /* The dir conn we are going to simulate */ dir_connection_t *conn = NULL; + edge_connection_t *edge_conn = NULL; + or_circuit_t *or_circ = NULL; /* First extract the blinded public key that we are going to use in our query, and then build the actual query string. */ @@ -245,8 +253,16 @@ helper_fetch_desc_from_hsdir(const ed25519_public_key_t *blinded_key) /* Simulate an HTTP GET request to the HSDir */ conn = dir_connection_new(AF_INET); tt_assert(conn); + TO_CONN(conn)->linked = 1; /* Signal that it is encrypted. */ tor_addr_from_ipv4h(&conn->base_.addr, 0x7f000001); - TO_CONN(conn)->linked = 1;/* Pretend the conn is encrypted :) */ + + /* Pretend this conn is anonymous. */ + edge_conn = edge_connection_new(CONN_TYPE_EXIT, AF_INET); + TO_CONN(conn)->linked_conn = TO_CONN(edge_conn); + or_circ = or_circuit_new(0, NULL); + or_circ->p_chan = tor_malloc_zero(sizeof(channel_t)); + edge_conn->on_circuit = TO_CIRCUIT(or_circ); + retval = directory_handle_command_get(conn, hsdir_query_str, NULL, 0); tt_int_op(retval, OP_EQ, 0); @@ -263,8 +279,11 @@ helper_fetch_desc_from_hsdir(const ed25519_public_key_t *blinded_key) done: tor_free(hsdir_query_str); - if (conn) + if (conn) { + tor_free(or_circ->p_chan); + connection_free_minimal(TO_CONN(conn)->linked_conn); connection_free_minimal(TO_CONN(conn)); + } return received_desc; } diff --git a/src/test/test_hs_cell.c b/src/test/test_hs_cell.c index cdcbe23e69..403509fbc8 100644 --- a/src/test/test_hs_cell.c +++ b/src/test/test_hs_cell.c @@ -20,6 +20,7 @@ #include "feature/hs/hs_service.h" /* Trunnel. */ +#include "trunnel/hs/cell_common.h" #include "trunnel/hs/cell_establish_intro.h" /** We simulate the creation of an outgoing ESTABLISH_INTRO cell, and then we @@ -38,11 +39,13 @@ test_gen_establish_intro_cell(void *arg) /* Create outgoing ESTABLISH_INTRO cell and extract its payload so that we attempt to parse it. */ { + hs_service_config_t config; + memset(&config, 0, sizeof(config)); /* We only need the auth key pair here. */ hs_service_intro_point_t *ip = service_intro_point_new(NULL); /* Auth key pair is generated in the constructor so we are all set for * using this IP object. */ - ret = hs_cell_build_establish_intro(circ_nonce, ip, buf); + ret = hs_cell_build_establish_intro(circ_nonce, &config, ip, buf); service_intro_point_free(ip); tt_u64_op(ret, OP_GT, 0); } @@ -97,6 +100,9 @@ test_gen_establish_intro_cell_bad(void *arg) trn_cell_establish_intro_t *cell = NULL; char circ_nonce[DIGEST_LEN] = {0}; hs_service_intro_point_t *ip = NULL; + hs_service_config_t config; + + memset(&config, 0, sizeof(config)); MOCK(ed25519_sign_prefixed, mock_ed25519_sign_prefixed); @@ -108,7 +114,7 @@ test_gen_establish_intro_cell_bad(void *arg) cell = trn_cell_establish_intro_new(); tt_assert(cell); ip = service_intro_point_new(NULL); - cell_len = hs_cell_build_establish_intro(circ_nonce, ip, NULL); + cell_len = hs_cell_build_establish_intro(circ_nonce, &config, ip, NULL); service_intro_point_free(ip); expect_log_msg_containing("Unable to make signature for " "ESTABLISH_INTRO cell."); @@ -120,11 +126,97 @@ test_gen_establish_intro_cell_bad(void *arg) UNMOCK(ed25519_sign_prefixed); } +static void +test_gen_establish_intro_dos_ext(void *arg) +{ + ssize_t ret; + hs_service_config_t config; + hs_service_intro_point_t *ip = NULL; + trn_cell_extension_t *extensions = NULL; + trn_cell_extension_dos_t *dos = NULL; + + (void) arg; + + memset(&config, 0, sizeof(config)); + ip = service_intro_point_new(NULL); + tt_assert(ip); + ip->support_intro2_dos_defense = 1; + + /* Case 1: No DoS parameters so no extension to be built. */ + extensions = build_establish_intro_extensions(&config, ip); + tt_int_op(trn_cell_extension_get_num(extensions), OP_EQ, 0); + trn_cell_extension_free(extensions); + extensions = NULL; + + /* Case 2: Enable the DoS extension. Parameter set to 0 should indicate to + * disable the defense on the intro point but there should be an extension + * nonetheless in the cell. */ + config.has_dos_defense_enabled = 1; + extensions = build_establish_intro_extensions(&config, ip); + tt_int_op(trn_cell_extension_get_num(extensions), OP_EQ, 1); + /* Validate the extension. */ + const trn_cell_extension_field_t *field = + trn_cell_extension_getconst_fields(extensions, 0); + tt_int_op(trn_cell_extension_field_get_field_type(field), OP_EQ, + TRUNNEL_CELL_EXTENSION_TYPE_DOS); + ret = trn_cell_extension_dos_parse(&dos, + trn_cell_extension_field_getconstarray_field(field), + trn_cell_extension_field_getlen_field(field)); + tt_int_op(ret, OP_EQ, 19); + /* Rate per sec param. */ + const trn_cell_extension_dos_param_t *param = + trn_cell_extension_dos_getconst_params(dos, 0); + tt_int_op(trn_cell_extension_dos_param_get_type(param), OP_EQ, + TRUNNEL_DOS_PARAM_TYPE_INTRO2_RATE_PER_SEC); + tt_u64_op(trn_cell_extension_dos_param_get_value(param), OP_EQ, 0); + /* Burst per sec param. */ + param = trn_cell_extension_dos_getconst_params(dos, 1); + tt_int_op(trn_cell_extension_dos_param_get_type(param), OP_EQ, + TRUNNEL_DOS_PARAM_TYPE_INTRO2_BURST_PER_SEC); + tt_u64_op(trn_cell_extension_dos_param_get_value(param), OP_EQ, 0); + trn_cell_extension_dos_free(dos); dos = NULL; + trn_cell_extension_free(extensions); extensions = NULL; + + /* Case 3: Enable the DoS extension. Parameter set to some normal values. */ + config.has_dos_defense_enabled = 1; + config.intro_dos_rate_per_sec = 42; + config.intro_dos_burst_per_sec = 250; + extensions = build_establish_intro_extensions(&config, ip); + tt_int_op(trn_cell_extension_get_num(extensions), OP_EQ, 1); + /* Validate the extension. */ + field = trn_cell_extension_getconst_fields(extensions, 0); + tt_int_op(trn_cell_extension_field_get_field_type(field), OP_EQ, + TRUNNEL_CELL_EXTENSION_TYPE_DOS); + ret = trn_cell_extension_dos_parse(&dos, + trn_cell_extension_field_getconstarray_field(field), + trn_cell_extension_field_getlen_field(field)); + tt_int_op(ret, OP_EQ, 19); + /* Rate per sec param. */ + param = trn_cell_extension_dos_getconst_params(dos, 0); + tt_int_op(trn_cell_extension_dos_param_get_type(param), OP_EQ, + TRUNNEL_DOS_PARAM_TYPE_INTRO2_RATE_PER_SEC); + tt_u64_op(trn_cell_extension_dos_param_get_value(param), OP_EQ, 42); + /* Burst per sec param. */ + param = trn_cell_extension_dos_getconst_params(dos, 1); + tt_int_op(trn_cell_extension_dos_param_get_type(param), OP_EQ, + TRUNNEL_DOS_PARAM_TYPE_INTRO2_BURST_PER_SEC); + tt_u64_op(trn_cell_extension_dos_param_get_value(param), OP_EQ, 250); + trn_cell_extension_dos_free(dos); dos = NULL; + trn_cell_extension_free(extensions); extensions = NULL; + + done: + service_intro_point_free(ip); + trn_cell_extension_dos_free(dos); + trn_cell_extension_free(extensions); +} + struct testcase_t hs_cell_tests[] = { { "gen_establish_intro_cell", test_gen_establish_intro_cell, TT_FORK, NULL, NULL }, { "gen_establish_intro_cell_bad", test_gen_establish_intro_cell_bad, TT_FORK, NULL, NULL }, + { "gen_establish_intro_dos_ext", test_gen_establish_intro_dos_ext, TT_FORK, + NULL, NULL }, END_OF_TESTCASES }; diff --git a/src/test/test_hs_client.c b/src/test/test_hs_client.c index 0d25a98bb3..b777dafdfb 100644 --- a/src/test/test_hs_client.c +++ b/src/test/test_hs_client.c @@ -37,6 +37,7 @@ #include "feature/hs/hs_config.h" #include "feature/hs/hs_ident.h" #include "feature/hs/hs_cache.h" +#include "feature/rend/rendcache.h" #include "core/or/circuitlist.h" #include "core/or/circuitbuild.h" #include "core/mainloop/connection.h" @@ -159,8 +160,7 @@ helper_get_circ_and_stream_for_test(origin_circuit_t **circ_out, or_circ->rend_data = rend_data_dup(conn_rend_data); } else { /* prop224: Setup hs ident on the circuit */ - or_circ->hs_ident = hs_ident_circuit_new(&service_pk, - HS_IDENT_CIRCUIT_RENDEZVOUS); + or_circ->hs_ident = hs_ident_circuit_new(&service_pk); } TO_CIRCUIT(or_circ)->state = CIRCUIT_STATE_OPEN; @@ -963,8 +963,7 @@ test_close_intro_circuits_new_desc(void *arg) const hs_desc_intro_point_t *ip = smartlist_get(desc1->encrypted_data.intro_points, 0); tt_assert(ip); - ocirc->hs_ident = hs_ident_circuit_new(&service_kp.pubkey, - HS_IDENT_CIRCUIT_INTRO); + ocirc->hs_ident = hs_ident_circuit_new(&service_kp.pubkey); ed25519_pubkey_copy(ô->hs_ident->intro_auth_pk, &ip->auth_key_cert->signed_key); } @@ -1007,6 +1006,91 @@ test_close_intro_circuits_new_desc(void *arg) UNMOCK(networkstatus_get_live_consensus); } +static void +test_close_intro_circuits_cache_clean(void *arg) +{ + int ret; + ed25519_keypair_t service_kp; + circuit_t *circ = NULL; + origin_circuit_t *ocirc = NULL; + hs_descriptor_t *desc1 = NULL; + + (void) arg; + + hs_init(); + rend_cache_init(); + + /* This is needed because of the client cache expiration timestamp is based + * on having a consensus. See cached_client_descriptor_has_expired(). */ + MOCK(networkstatus_get_live_consensus, + mock_networkstatus_get_live_consensus); + + /* Set consensus time */ + parse_rfc1123_time("Sat, 26 Oct 1985 13:00:00 UTC", + &mock_ns.valid_after); + parse_rfc1123_time("Sat, 26 Oct 1985 14:00:00 UTC", + &mock_ns.fresh_until); + parse_rfc1123_time("Sat, 26 Oct 1985 16:00:00 UTC", + &mock_ns.valid_until); + + /* Generate service keypair */ + tt_int_op(0, OP_EQ, ed25519_keypair_generate(&service_kp, 0)); + + /* Create and add to the global list a dummy client introduction circuits. + * We'll then make sure the hs_ident is attached to a dummy descriptor. */ + circ = dummy_origin_circuit_new(0); + tt_assert(circ); + circ->purpose = CIRCUIT_PURPOSE_C_INTRODUCING; + ocirc = TO_ORIGIN_CIRCUIT(circ); + + /* Build the first descriptor and cache it. */ + { + char *encoded; + desc1 = hs_helper_build_hs_desc_with_ip(&service_kp); + tt_assert(desc1); + ret = hs_desc_encode_descriptor(desc1, &service_kp, NULL, &encoded); + tt_int_op(ret, OP_EQ, 0); + tt_assert(encoded); + + /* Store it */ + ret = hs_cache_store_as_client(encoded, &service_kp.pubkey); + tt_int_op(ret, OP_EQ, 0); + tor_free(encoded); + tt_assert(hs_cache_lookup_as_client(&service_kp.pubkey)); + } + + /* We'll pick one introduction point and associate it with the circuit. */ + { + const hs_desc_intro_point_t *ip = + smartlist_get(desc1->encrypted_data.intro_points, 0); + tt_assert(ip); + ocirc->hs_ident = hs_ident_circuit_new(&service_kp.pubkey); + ed25519_pubkey_copy(ô->hs_ident->intro_auth_pk, + &ip->auth_key_cert->signed_key); + } + + /* Before we are about to clean up the intro circuits, make sure it is + * actually there. */ + tt_assert(circuit_get_next_intro_circ(NULL, true)); + + /* Cleanup the client cache. The ns valid after time is what decides if the + * descriptor has expired so put it in the future enough (72h) so we are + * sure to always expire. */ + mock_ns.valid_after = approx_time() + (72 * 24 * 60 * 60); + hs_cache_clean_as_client(0); + + /* Once stored, our intro circuit should be closed because it is related to + * an old introduction point that doesn't exists anymore. */ + tt_assert(!circuit_get_next_intro_circ(NULL, true)); + + done: + circuit_free(circ); + hs_descriptor_free(desc1); + hs_free_all(); + rend_cache_free_all(); + UNMOCK(networkstatus_get_live_consensus); +} + struct testcase_t hs_client_tests[] = { { "e2e_rend_circuit_setup_legacy", test_e2e_rend_circuit_setup_legacy, TT_FORK, NULL, NULL }, @@ -1026,6 +1110,8 @@ struct testcase_t hs_client_tests[] = { TT_FORK, NULL, NULL }, { "close_intro_circuits_new_desc", test_close_intro_circuits_new_desc, TT_FORK, NULL, NULL }, + { "close_intro_circuits_cache_clean", test_close_intro_circuits_cache_clean, + TT_FORK, NULL, NULL }, END_OF_TESTCASES }; diff --git a/src/test/test_hs_common.c b/src/test/test_hs_common.c index abded6021e..de3f7e04f7 100644 --- a/src/test/test_hs_common.c +++ b/src/test/test_hs_common.c @@ -502,6 +502,7 @@ test_desc_reupload_logic(void *arg) pubkey_hex, strlen(pubkey_hex)); hs_build_address(&pubkey, HS_VERSION_THREE, onion_addr); service = tor_malloc_zero(sizeof(hs_service_t)); + tt_assert(service); memcpy(service->onion_address, onion_addr, sizeof(service->onion_address)); ed25519_secret_key_generate(&service->keys.identity_sk, 0); ed25519_public_key_generate(&service->keys.identity_pk, diff --git a/src/test/test_hs_config.c b/src/test/test_hs_config.c index c2c556307d..71e1529216 100644 --- a/src/test/test_hs_config.c +++ b/src/test/test_hs_config.c @@ -12,6 +12,7 @@ #include "test/test.h" #include "test/test_helpers.h" #include "test/log_test_helpers.h" +#include "test/resolve_test_helpers.h" #include "app/config/config.h" #include "feature/hs/hs_common.h" @@ -272,6 +273,7 @@ test_valid_service_v2(void *arg) int ret; (void) arg; + mock_hostname_resolver(); /* Valid complex configuration. Basic client authorization. */ { @@ -314,7 +316,7 @@ test_valid_service_v2(void *arg) } done: - ; + unmock_hostname_resolver(); } static void @@ -392,6 +394,7 @@ test_valid_service_v3(void *arg) int ret; (void) arg; + mock_hostname_resolver(); /* Valid complex configuration. */ { @@ -448,7 +451,7 @@ test_valid_service_v3(void *arg) } done: - ; + unmock_hostname_resolver(); } static void @@ -489,6 +492,111 @@ test_staging_service_v3(void *arg) hs_free_all(); } +static void +test_dos_parameters(void *arg) +{ + int ret; + + (void) arg; + + hs_init(); + + /* Valid configuration. */ + { + const char *conf = + "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs3\n" + "HiddenServiceVersion 3\n" + "HiddenServicePort 22 1.1.1.1:22\n" + "HiddenServiceEnableIntroDoSDefense 1\n" + "HiddenServiceEnableIntroDoSRatePerSec 42\n" + "HiddenServiceEnableIntroDoSBurstPerSec 87\n"; + + setup_full_capture_of_logs(LOG_INFO); + ret = helper_config_service(conf, 0); + tt_int_op(ret, OP_EQ, 0); + expect_log_msg_containing("Service INTRO2 DoS defenses rate set to: 42"); + expect_log_msg_containing("Service INTRO2 DoS defenses burst set to: 87"); + teardown_capture_of_logs(); + } + + /* Invalid rate. Value of 2^37. Max allowed is 2^31. */ + { + const char *conf = + "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs3\n" + "HiddenServiceVersion 3\n" + "HiddenServicePort 22 1.1.1.1:22\n" + "HiddenServiceEnableIntroDoSDefense 1\n" + "HiddenServiceEnableIntroDoSRatePerSec 137438953472\n" + "HiddenServiceEnableIntroDoSBurstPerSec 87\n"; + + setup_full_capture_of_logs(LOG_WARN); + ret = helper_config_service(conf, 0); + tt_int_op(ret, OP_EQ, -1); + expect_log_msg_containing("HiddenServiceEnableIntroDoSRatePerSec must " + "be between 0 and 2147483647, " + "not 137438953472"); + teardown_capture_of_logs(); + } + + /* Invalid burst. Value of 2^38. Max allowed is 2^31. */ + { + const char *conf = + "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs3\n" + "HiddenServiceVersion 3\n" + "HiddenServicePort 22 1.1.1.1:22\n" + "HiddenServiceEnableIntroDoSDefense 1\n" + "HiddenServiceEnableIntroDoSRatePerSec 42\n" + "HiddenServiceEnableIntroDoSBurstPerSec 274877906944\n"; + + setup_full_capture_of_logs(LOG_WARN); + ret = helper_config_service(conf, 0); + tt_int_op(ret, OP_EQ, -1); + expect_log_msg_containing("HiddenServiceEnableIntroDoSBurstPerSec must " + "be between 0 and 2147483647, " + "not 274877906944"); + teardown_capture_of_logs(); + } + + /* Burst is smaller than rate. */ + { + const char *conf = + "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs3\n" + "HiddenServiceVersion 3\n" + "HiddenServicePort 22 1.1.1.1:22\n" + "HiddenServiceEnableIntroDoSDefense 1\n" + "HiddenServiceEnableIntroDoSRatePerSec 42\n" + "HiddenServiceEnableIntroDoSBurstPerSec 27\n"; + + setup_full_capture_of_logs(LOG_WARN); + ret = helper_config_service(conf, 0); + tt_int_op(ret, OP_EQ, -1); + expect_log_msg_containing("Hidden service DoS defenses burst (27) can " + "not be smaller than the rate value (42)."); + teardown_capture_of_logs(); + } + + /* Negative value. */ + { + const char *conf = + "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs3\n" + "HiddenServiceVersion 3\n" + "HiddenServicePort 22 1.1.1.1:22\n" + "HiddenServiceEnableIntroDoSDefense 1\n" + "HiddenServiceEnableIntroDoSRatePerSec -1\n" + "HiddenServiceEnableIntroDoSBurstPerSec 42\n"; + + setup_full_capture_of_logs(LOG_WARN); + ret = helper_config_service(conf, 0); + tt_int_op(ret, OP_EQ, -1); + expect_log_msg_containing("HiddenServiceEnableIntroDoSRatePerSec must be " + "between 0 and 2147483647, not -1"); + teardown_capture_of_logs(); + } + + done: + hs_free_all(); +} + struct testcase_t hs_config_tests[] = { /* Invalid service not specific to any version. */ { "invalid_service", test_invalid_service, TT_FORK, @@ -512,6 +620,9 @@ struct testcase_t hs_config_tests[] = { { "staging_service_v3", test_staging_service_v3, TT_FORK, NULL, NULL }, + /* Test HS DoS parameters. */ + { "dos_parameters", test_dos_parameters, TT_FORK, + NULL, NULL }, + END_OF_TESTCASES }; - diff --git a/src/test/test_hs_dos.c b/src/test/test_hs_dos.c new file mode 100644 index 0000000000..f68639e24a --- /dev/null +++ b/src/test/test_hs_dos.c @@ -0,0 +1,176 @@ +/* Copyright (c) 2017-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file test_hs_cell.c + * \brief Test hidden service cell functionality. + */ + +#define CIRCUITLIST_PRIVATE +#define NETWORKSTATUS_PRIVATE +#define HS_DOS_PRIVATE +#define HS_INTROPOINT_PRIVATE + +#include "test/test.h" +#include "test/test_helpers.h" +#include "test/log_test_helpers.h" + +#include "app/config/config.h" + +#include "core/or/circuitlist.h" +#include "core/or/circuituse.h" +#include "core/or/or_circuit_st.h" + +#include "feature/hs/hs_dos.h" +#include "feature/hs/hs_intropoint.h" +#include "feature/nodelist/networkstatus.h" + +static void +setup_mock_consensus(void) +{ + current_ns_consensus = tor_malloc_zero(sizeof(networkstatus_t)); + current_ns_consensus->net_params = smartlist_new(); + smartlist_add(current_ns_consensus->net_params, + (void *) "HiddenServiceEnableIntroDoSDefense=1"); + hs_dos_consensus_has_changed(current_ns_consensus); +} + +static void +free_mock_consensus(void) +{ + smartlist_free(current_ns_consensus->net_params); + tor_free(current_ns_consensus); +} + +static void +test_can_send_intro2(void *arg) +{ + uint32_t now = (uint32_t) approx_time(); + or_circuit_t *or_circ = NULL; + + (void) arg; + + hs_init(); + hs_dos_init(); + + get_options_mutable()->ORPort_set = 1; + setup_mock_consensus(); + + or_circ = or_circuit_new(1, NULL); + + /* Make that circuit a service intro point. */ + circuit_change_purpose(TO_CIRCUIT(or_circ), CIRCUIT_PURPOSE_INTRO_POINT); + hs_dos_setup_default_intro2_defenses(or_circ); + or_circ->introduce2_dos_defense_enabled = 1; + + /* Brand new circuit, we should be able to send INTRODUCE2 cells. */ + tt_int_op(true, OP_EQ, hs_dos_can_send_intro2(or_circ)); + + /* Simulate that 10 cells have arrived in 1 second. There should be no + * refill since the bucket is already at maximum on the first cell. */ + update_approx_time(++now); + for (int i = 0; i < 10; i++) { + tt_int_op(true, OP_EQ, hs_dos_can_send_intro2(or_circ)); + } + tt_uint_op(token_bucket_ctr_get(&or_circ->introduce2_bucket), OP_EQ, + get_intro2_burst_consensus_param(NULL) - 10); + + /* Fully refill the bucket minus 1 cell. */ + update_approx_time(++now); + tt_int_op(true, OP_EQ, hs_dos_can_send_intro2(or_circ)); + tt_uint_op(token_bucket_ctr_get(&or_circ->introduce2_bucket), OP_EQ, + get_intro2_burst_consensus_param(NULL) - 1); + + /* Receive an INTRODUCE2 at each second. We should have the bucket full + * since at every second it gets refilled. */ + for (int i = 0; i < 10; i++) { + update_approx_time(++now); + tt_int_op(true, OP_EQ, hs_dos_can_send_intro2(or_circ)); + } + /* Last check if we can send the cell decrements the bucket so minus 1. */ + tt_uint_op(token_bucket_ctr_get(&or_circ->introduce2_bucket), OP_EQ, + get_intro2_burst_consensus_param(NULL) - 1); + + /* Manually reset bucket for next test. */ + token_bucket_ctr_reset(&or_circ->introduce2_bucket, now); + tt_uint_op(token_bucket_ctr_get(&or_circ->introduce2_bucket), OP_EQ, + get_intro2_burst_consensus_param(NULL)); + + /* Do a full burst in the current second which should empty the bucket and + * we shouldn't be allowed to send one more cell after that. We go minus 1 + * cell else the very last check if we can send the INTRO2 cell returns + * false because the bucket goes down to 0. */ + for (uint32_t i = 0; i < get_intro2_burst_consensus_param(NULL) - 1; i++) { + tt_int_op(true, OP_EQ, hs_dos_can_send_intro2(or_circ)); + } + tt_uint_op(token_bucket_ctr_get(&or_circ->introduce2_bucket), OP_EQ, 1); + /* Get the last remaining cell, we shouldn't be allowed to send it. */ + tt_int_op(false, OP_EQ, hs_dos_can_send_intro2(or_circ)); + tt_uint_op(token_bucket_ctr_get(&or_circ->introduce2_bucket), OP_EQ, 0); + + /* Make sure the next 100 cells aren't allowed and bucket stays at 0. */ + for (int i = 0; i < 100; i++) { + tt_int_op(false, OP_EQ, hs_dos_can_send_intro2(or_circ)); + tt_uint_op(token_bucket_ctr_get(&or_circ->introduce2_bucket), OP_EQ, 0); + } + + /* One second has passed, we should have the rate minus 1 cell added. */ + update_approx_time(++now); + tt_int_op(true, OP_EQ, hs_dos_can_send_intro2(or_circ)); + tt_uint_op(token_bucket_ctr_get(&or_circ->introduce2_bucket), OP_EQ, + get_intro2_rate_consensus_param(NULL) - 1); + + done: + circuit_free_(TO_CIRCUIT(or_circ)); + + hs_free_all(); + free_mock_consensus(); +} + +static void +test_validate_dos_extension_params(void *arg) +{ + bool ret; + + (void) arg; + + /* Validate the default values. */ + ret = cell_dos_extension_parameters_are_valid( + get_intro2_rate_consensus_param(NULL), + get_intro2_burst_consensus_param(NULL)); + tt_assert(ret); + + /* Valid custom rate/burst. */ + ret = cell_dos_extension_parameters_are_valid(17, 42); + tt_assert(ret); + ret = cell_dos_extension_parameters_are_valid(INT32_MAX, INT32_MAX); + tt_assert(ret); + + /* Invalid rate. */ + ret = cell_dos_extension_parameters_are_valid(UINT64_MAX, 42); + tt_assert(!ret); + + /* Invalid burst. */ + ret = cell_dos_extension_parameters_are_valid(42, UINT64_MAX); + tt_assert(!ret); + + /* Value of 0 is valid (but should disable defenses) */ + ret = cell_dos_extension_parameters_are_valid(0, 0); + tt_assert(ret); + + /* Can't have burst smaller than rate. */ + ret = cell_dos_extension_parameters_are_valid(42, 40); + tt_assert(!ret); + + done: + return; +} + +struct testcase_t hs_dos_tests[] = { + { "can_send_intro2", test_can_send_intro2, TT_FORK, + NULL, NULL }, + { "validate_dos_extension_params", test_validate_dos_extension_params, + TT_FORK, NULL, NULL }, + + END_OF_TESTCASES +}; diff --git a/src/test/test_hs_intropoint.c b/src/test/test_hs_intropoint.c index 732836fb5b..feb934d93c 100644 --- a/src/test/test_hs_intropoint.c +++ b/src/test/test_hs_intropoint.c @@ -16,6 +16,7 @@ #include "lib/crypt_ops/crypto_rand.h" #include "core/or/or.h" +#include "core/or/channel.h" #include "core/or/circuitlist.h" #include "core/or/circuituse.h" #include "ht.h" @@ -25,6 +26,8 @@ #include "feature/hs/hs_cell.h" #include "feature/hs/hs_circuitmap.h" #include "feature/hs/hs_common.h" +#include "feature/hs/hs_config.h" +#include "feature/hs/hs_dos.h" #include "feature/hs/hs_intropoint.h" #include "feature/hs/hs_service.h" @@ -43,6 +46,9 @@ new_establish_intro_cell(const char *circ_nonce, uint8_t buf[RELAY_PAYLOAD_SIZE] = {0}; trn_cell_establish_intro_t *cell = NULL; hs_service_intro_point_t *ip = NULL; + hs_service_config_t config; + + memset(&config, 0, sizeof(config)); /* Ensure that *cell_out is NULL such that we can use to check if we need to * free `cell` in case of an error. */ @@ -52,7 +58,7 @@ new_establish_intro_cell(const char *circ_nonce, * using this IP object. */ ip = service_intro_point_new(NULL); tt_assert(ip); - cell_len = hs_cell_build_establish_intro(circ_nonce, ip, buf); + cell_len = hs_cell_build_establish_intro(circ_nonce, &config, ip, buf); tt_i64_op(cell_len, OP_GT, 0); cell_len = trn_cell_establish_intro_parse(&cell, buf, sizeof(buf)); @@ -73,12 +79,15 @@ new_establish_intro_encoded_cell(const char *circ_nonce, uint8_t *cell_out) { ssize_t cell_len = 0; hs_service_intro_point_t *ip = NULL; + hs_service_config_t config; + + memset(&config, 0, sizeof(config)); /* Auth key pair is generated in the constructor so we are all set for * using this IP object. */ ip = service_intro_point_new(NULL); tt_assert(ip); - cell_len = hs_cell_build_establish_intro(circ_nonce, ip, cell_out); + cell_len = hs_cell_build_establish_intro(circ_nonce, &config, ip, cell_out); tt_i64_op(cell_len, OP_GT, 0); done: @@ -118,6 +127,8 @@ helper_create_intro_circuit(void) or_circuit_t *circ = or_circuit_new(0, NULL); tt_assert(circ); circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_OR); + token_bucket_ctr_init(&circ->introduce2_bucket, 100, 100, + (uint32_t) approx_time()); done: return circ; } @@ -693,6 +704,17 @@ test_introduce1_suitable_circuit(void *arg) tt_int_op(ret, OP_EQ, 0); } + /* Single hop circuit should not be allowed. */ + { + circ = or_circuit_new(0, NULL); + circ->p_chan = tor_malloc_zero(sizeof(channel_t)); + circ->p_chan->is_client = 1; + ret = circuit_is_suitable_for_introduce1(circ); + tor_free(circ->p_chan); + circuit_free_(TO_CIRCUIT(circ)); + tt_int_op(ret, OP_EQ, 0); + } + done: ; } @@ -888,43 +910,213 @@ test_received_introduce1_handling(void *arg) UNMOCK(relay_send_command_from_edge_); } +static void +test_received_establish_intro_dos_ext(void *arg) +{ + int ret; + ssize_t cell_len = 0; + uint8_t cell[RELAY_PAYLOAD_SIZE] = {0}; + char circ_nonce[DIGEST_LEN] = {0}; + hs_service_intro_point_t *ip = NULL; + hs_service_config_t config; + or_circuit_t *intro_circ = or_circuit_new(0,NULL); + + (void) arg; + + MOCK(relay_send_command_from_edge_, mock_relay_send_command_from_edge); + + hs_circuitmap_init(); + + /* Setup. */ + crypto_rand(circ_nonce, sizeof(circ_nonce)); + ip = service_intro_point_new(NULL); + tt_assert(ip); + ip->support_intro2_dos_defense = 1; + memset(&config, 0, sizeof(config)); + config.has_dos_defense_enabled = 1; + config.intro_dos_rate_per_sec = 13; + config.intro_dos_burst_per_sec = 42; + helper_prepare_circ_for_intro(intro_circ, circ_nonce); + /* The INTRO2 bucket should be 0 at this point. */ + tt_u64_op(token_bucket_ctr_get(&intro_circ->introduce2_bucket), OP_EQ, 0); + tt_u64_op(intro_circ->introduce2_bucket.cfg.rate, OP_EQ, 0); + tt_int_op(intro_circ->introduce2_bucket.cfg.burst, OP_EQ, 0); + tt_int_op(intro_circ->introduce2_dos_defense_enabled, OP_EQ, 0); + + /* Case 1: Build encoded cell. Usable DoS parameters. */ + cell_len = hs_cell_build_establish_intro(circ_nonce, &config, ip, cell); + tt_size_op(cell_len, OP_GT, 0); + /* Pass it to the intro point. */ + ret = hs_intro_received_establish_intro(intro_circ, cell, cell_len); + tt_int_op(ret, OP_EQ, 0); + /* Should be set to the burst value. */ + tt_u64_op(token_bucket_ctr_get(&intro_circ->introduce2_bucket), OP_EQ, 42); + /* Validate the config of the intro2 bucket. */ + tt_u64_op(intro_circ->introduce2_bucket.cfg.rate, OP_EQ, 13); + tt_int_op(intro_circ->introduce2_bucket.cfg.burst, OP_EQ, 42); + tt_int_op(intro_circ->introduce2_dos_defense_enabled, OP_EQ, 1); + + /* Need to reset the circuit in between test cases. */ + circuit_free_(TO_CIRCUIT(intro_circ)); + intro_circ = or_circuit_new(0,NULL); + helper_prepare_circ_for_intro(intro_circ, circ_nonce); + + /* Case 2: Build encoded cell. Bad DoS parameters. */ + config.has_dos_defense_enabled = 1; + config.intro_dos_rate_per_sec = UINT_MAX; + config.intro_dos_burst_per_sec = 13; + cell_len = hs_cell_build_establish_intro(circ_nonce, &config, ip, cell); + tt_size_op(cell_len, OP_GT, 0); + /* Pass it to the intro point. */ + ret = hs_intro_received_establish_intro(intro_circ, cell, cell_len); + tt_int_op(ret, OP_EQ, 0); + tt_u64_op(token_bucket_ctr_get(&intro_circ->introduce2_bucket), OP_EQ, + HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_DEFAULT); + tt_u64_op(intro_circ->introduce2_bucket.cfg.rate, OP_EQ, + HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_DEFAULT); + tt_int_op(intro_circ->introduce2_bucket.cfg.burst, OP_EQ, + HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_DEFAULT); + tt_int_op(intro_circ->introduce2_dos_defense_enabled, OP_EQ, + HS_CONFIG_V3_DOS_DEFENSE_DEFAULT); + + /* Need to reset the circuit in between test cases. */ + circuit_free_(TO_CIRCUIT(intro_circ)); + intro_circ = or_circuit_new(0,NULL); + helper_prepare_circ_for_intro(intro_circ, circ_nonce); + + /* Case 3: Build encoded cell. Burst is smaller than rate. Not allowed. */ + config.has_dos_defense_enabled = 1; + config.intro_dos_rate_per_sec = 87; + config.intro_dos_burst_per_sec = 45; + cell_len = hs_cell_build_establish_intro(circ_nonce, &config, ip, cell); + tt_size_op(cell_len, OP_GT, 0); + /* Pass it to the intro point. */ + ret = hs_intro_received_establish_intro(intro_circ, cell, cell_len); + tt_int_op(ret, OP_EQ, 0); + tt_u64_op(token_bucket_ctr_get(&intro_circ->introduce2_bucket), OP_EQ, + HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_DEFAULT); + tt_u64_op(intro_circ->introduce2_bucket.cfg.rate, OP_EQ, + HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_DEFAULT); + tt_int_op(intro_circ->introduce2_bucket.cfg.burst, OP_EQ, + HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_DEFAULT); + tt_int_op(intro_circ->introduce2_dos_defense_enabled, OP_EQ, + HS_CONFIG_V3_DOS_DEFENSE_DEFAULT); + + /* Need to reset the circuit in between test cases. */ + circuit_free_(TO_CIRCUIT(intro_circ)); + intro_circ = or_circuit_new(0,NULL); + helper_prepare_circ_for_intro(intro_circ, circ_nonce); + + /* Case 4: Build encoded cell. Rate is 0 but burst is not 0. Disables the + * defense. */ + config.has_dos_defense_enabled = 1; + config.intro_dos_rate_per_sec = 0; + config.intro_dos_burst_per_sec = 45; + cell_len = hs_cell_build_establish_intro(circ_nonce, &config, ip, cell); + tt_size_op(cell_len, OP_GT, 0); + /* Pass it to the intro point. */ + ret = hs_intro_received_establish_intro(intro_circ, cell, cell_len); + tt_int_op(ret, OP_EQ, 0); + tt_u64_op(token_bucket_ctr_get(&intro_circ->introduce2_bucket), OP_EQ, + HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_DEFAULT); + tt_u64_op(intro_circ->introduce2_bucket.cfg.rate, OP_EQ, + HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_DEFAULT); + tt_int_op(intro_circ->introduce2_bucket.cfg.burst, OP_EQ, + HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_DEFAULT); + tt_int_op(intro_circ->introduce2_dos_defense_enabled, OP_EQ, + HS_CONFIG_V3_DOS_DEFENSE_DEFAULT); + + /* Need to reset the circuit in between test cases. */ + circuit_free_(TO_CIRCUIT(intro_circ)); + intro_circ = or_circuit_new(0,NULL); + helper_prepare_circ_for_intro(intro_circ, circ_nonce); + + /* Case 5: Build encoded cell. Burst is 0 but rate is not 0. Disables the + * defense. */ + config.has_dos_defense_enabled = 1; + config.intro_dos_rate_per_sec = 45; + config.intro_dos_burst_per_sec = 0; + cell_len = hs_cell_build_establish_intro(circ_nonce, &config, ip, cell); + tt_size_op(cell_len, OP_GT, 0); + /* Pass it to the intro point. */ + ret = hs_intro_received_establish_intro(intro_circ, cell, cell_len); + tt_int_op(ret, OP_EQ, 0); + tt_u64_op(token_bucket_ctr_get(&intro_circ->introduce2_bucket), OP_EQ, + HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_DEFAULT); + tt_u64_op(intro_circ->introduce2_bucket.cfg.rate, OP_EQ, + HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_DEFAULT); + tt_int_op(intro_circ->introduce2_bucket.cfg.burst, OP_EQ, + HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_DEFAULT); + tt_int_op(intro_circ->introduce2_dos_defense_enabled, OP_EQ, + HS_CONFIG_V3_DOS_DEFENSE_DEFAULT); + + done: + circuit_free_(TO_CIRCUIT(intro_circ)); + service_intro_point_free(ip); + hs_circuitmap_free_all(); + UNMOCK(relay_send_command_from_edge_); +} + +static void * +hs_subsystem_setup_fn(const struct testcase_t *tc) +{ + (void) tc; + + return NULL; +} + +static int +hs_subsystem_cleanup_fn(const struct testcase_t *tc, void *arg) +{ + (void) tc; + (void) arg; + + return 1; +} + +static struct testcase_setup_t test_setup = { + hs_subsystem_setup_fn, hs_subsystem_cleanup_fn +}; + struct testcase_t hs_intropoint_tests[] = { { "intro_point_registration", - test_intro_point_registration, TT_FORK, NULL, NULL }, + test_intro_point_registration, TT_FORK, NULL, &test_setup}, { "receive_establish_intro_wrong_keytype", - test_establish_intro_wrong_keytype, TT_FORK, NULL, NULL }, + test_establish_intro_wrong_keytype, TT_FORK, NULL, &test_setup}, { "receive_establish_intro_wrong_keytype2", - test_establish_intro_wrong_keytype2, TT_FORK, NULL, NULL }, + test_establish_intro_wrong_keytype2, TT_FORK, NULL, &test_setup}, { "receive_establish_intro_wrong_purpose", - test_establish_intro_wrong_purpose, TT_FORK, NULL, NULL }, + test_establish_intro_wrong_purpose, TT_FORK, NULL, &test_setup}, { "receive_establish_intro_wrong_sig", - test_establish_intro_wrong_sig, TT_FORK, NULL, NULL }, + test_establish_intro_wrong_sig, TT_FORK, NULL, &test_setup}, { "receive_establish_intro_wrong_sig_len", - test_establish_intro_wrong_sig_len, TT_FORK, NULL, NULL }, + test_establish_intro_wrong_sig_len, TT_FORK, NULL, &test_setup}, { "receive_establish_intro_wrong_auth_key_len", - test_establish_intro_wrong_auth_key_len, TT_FORK, NULL, NULL }, + test_establish_intro_wrong_auth_key_len, TT_FORK, NULL, &test_setup}, { "receive_establish_intro_wrong_mac", - test_establish_intro_wrong_mac, TT_FORK, NULL, NULL }, + test_establish_intro_wrong_mac, TT_FORK, NULL, &test_setup}, { "introduce1_suitable_circuit", - test_introduce1_suitable_circuit, TT_FORK, NULL, NULL }, + test_introduce1_suitable_circuit, TT_FORK, NULL, &test_setup}, { "introduce1_is_legacy", - test_introduce1_is_legacy, TT_FORK, NULL, NULL }, + test_introduce1_is_legacy, TT_FORK, NULL, &test_setup}, { "introduce1_validation", - test_introduce1_validation, TT_FORK, NULL, NULL }, + test_introduce1_validation, TT_FORK, NULL, &test_setup}, { "received_introduce1_handling", - test_received_introduce1_handling, TT_FORK, NULL, NULL }, + test_received_introduce1_handling, TT_FORK, NULL, &test_setup}, + + { "received_establish_intro_dos_ext", + test_received_establish_intro_dos_ext, TT_FORK, NULL, &test_setup}, END_OF_TESTCASES }; - diff --git a/src/test/test_hs_service.c b/src/test/test_hs_service.c index a303f10411..66194cee3d 100644 --- a/src/test/test_hs_service.c +++ b/src/test/test_hs_service.c @@ -171,8 +171,7 @@ test_e2e_rend_circuit_setup(void *arg) tt_int_op(0, OP_EQ, ed25519_secret_key_generate(&sk, 0)); tt_int_op(0, OP_EQ, ed25519_public_key_generate(&service_pk, &sk)); - or_circ->hs_ident = hs_ident_circuit_new(&service_pk, - HS_IDENT_CIRCUIT_RENDEZVOUS); + or_circ->hs_ident = hs_ident_circuit_new(&service_pk); TO_CIRCUIT(or_circ)->state = CIRCUIT_STATE_OPEN; } @@ -675,9 +674,11 @@ test_service_intro_point(void *arg) (void) arg; + update_approx_time(1481621834); + /* Test simple creation of an object. */ { - time_t now = time(NULL); + time_t now = approx_time(); ip = helper_create_service_ip(); tt_assert(ip); /* Make sure the authentication keypair is not zeroes. */ @@ -1105,8 +1106,7 @@ test_closing_intro_circs(void *arg) /* Initialize intro circuit */ intro_circ = origin_circuit_init(CIRCUIT_PURPOSE_S_ESTABLISH_INTRO, flags); - intro_circ->hs_ident = hs_ident_circuit_new(&service->keys.identity_pk, - HS_IDENT_CIRCUIT_INTRO); + intro_circ->hs_ident = hs_ident_circuit_new(&service->keys.identity_pk); /* Register circuit in the circuitmap . */ hs_circuitmap_register_intro_circ_v3_service_side(intro_circ, &ip->auth_key_kp.pubkey); @@ -1132,8 +1132,7 @@ test_closing_intro_circs(void *arg) /* Now pretend that a new intro point circ was launched and opened. Check * that the intro point will be established correctly. */ intro_circ = origin_circuit_init(CIRCUIT_PURPOSE_S_ESTABLISH_INTRO, flags); - intro_circ->hs_ident = hs_ident_circuit_new(&service->keys.identity_pk, - HS_IDENT_CIRCUIT_INTRO); + intro_circ->hs_ident = hs_ident_circuit_new(&service->keys.identity_pk); ed25519_pubkey_copy(&intro_circ->hs_ident->intro_auth_pk, &ip->auth_key_kp.pubkey); /* Register circuit in the circuitmap . */ @@ -1181,7 +1180,7 @@ test_introduce2(void *arg) MOCK(get_or_state, get_or_state_replacement); - dummy_state = tor_malloc_zero(sizeof(or_state_t)); + dummy_state = or_state_new(); circ = helper_create_origin_circuit(CIRCUIT_PURPOSE_S_INTRO, flags); tt_assert(circ); @@ -1265,6 +1264,7 @@ test_service_event(void *arg) /* Set a service for this circuit. */ service = helper_create_service(); + tt_assert(service); ed25519_pubkey_copy(&circ->hs_ident->identity_pk, &service->keys.identity_pk); @@ -1300,8 +1300,14 @@ test_service_event(void *arg) run_housekeeping_event(now); tt_int_op(digest256map_size(service->desc_current->intro_points.map), OP_EQ, 1); + /* No removal if we have an established circuit after retries. */ + ip->circuit_retries = MAX_INTRO_POINT_CIRCUIT_RETRIES + 1; + run_housekeeping_event(now); + tt_int_op(digest256map_size(service->desc_current->intro_points.map), + OP_EQ, 1); /* Remove the IP object at once for the next test. */ ip->circuit_retries = MAX_INTRO_POINT_CIRCUIT_RETRIES + 1; + ip->circuit_established = 0; run_housekeeping_event(now); tt_int_op(digest256map_size(service->desc_current->intro_points.map), OP_EQ, 0); @@ -1345,7 +1351,7 @@ test_rotate_descriptors(void *arg) (void) arg; - dummy_state = tor_malloc_zero(sizeof(or_state_t)); + dummy_state = or_state_new(); hs_init(); MOCK(get_or_state, get_or_state_replacement); @@ -1462,7 +1468,7 @@ test_build_update_descriptors(void *arg) MOCK(networkstatus_get_live_consensus, mock_networkstatus_get_live_consensus); - dummy_state = tor_malloc_zero(sizeof(or_state_t)); + dummy_state = or_state_new(); ret = parse_rfc1123_time("Sat, 26 Oct 1985 03:00:00 UTC", &mock_ns.valid_after); @@ -1693,7 +1699,7 @@ test_build_descriptors(void *arg) MOCK(networkstatus_get_live_consensus, mock_networkstatus_get_live_consensus); - dummy_state = tor_malloc_zero(sizeof(or_state_t)); + dummy_state = or_state_new(); ret = parse_rfc1123_time("Sat, 26 Oct 1985 03:00:00 UTC", &mock_ns.valid_after); @@ -1794,7 +1800,7 @@ test_upload_descriptors(void *arg) MOCK(networkstatus_get_live_consensus, mock_networkstatus_get_live_consensus); - dummy_state = tor_malloc_zero(sizeof(or_state_t)); + dummy_state = or_state_new(); ret = parse_rfc1123_time("Sat, 26 Oct 1985 13:00:00 UTC", &mock_ns.valid_after); diff --git a/src/test/test_introduce.c b/src/test/test_introduce.c index 4a6d90d97e..104e973b1f 100644 --- a/src/test/test_introduce.c +++ b/src/test/test_introduce.c @@ -383,8 +383,10 @@ make_intro_from_plaintext( /* Output the cell */ *cell_out = cell; + cell = NULL; done: + tor_free(cell); return cell_len; } @@ -535,4 +537,3 @@ struct testcase_t introduce_tests[] = { INTRODUCE_LEGACY(late_parse_v3), END_OF_TESTCASES }; - diff --git a/src/test/test_logging.c b/src/test/test_logging.c index bb7018fe1c..203ce64e32 100644 --- a/src/test/test_logging.c +++ b/src/test/test_logging.c @@ -35,7 +35,7 @@ test_get_sigsafe_err_fds(void *arg) set_log_severity_config(LOG_WARN, LOG_ERR, &include_bug); set_log_severity_config(LOG_WARN, LOG_ERR, &no_bug); - no_bug.masks[0] &= ~(LD_BUG|LD_GENERAL); + no_bug.masks[SEVERITY_MASK_IDX(LOG_ERR)] &= ~(LD_BUG|LD_GENERAL); set_log_severity_config(LOG_INFO, LOG_NOTICE, &no_bug2); /* Add some logs; make sure the output is as expected. */ diff --git a/src/test/test_microdesc.c b/src/test/test_microdesc.c index 53ee799185..804e6c546a 100644 --- a/src/test/test_microdesc.c +++ b/src/test/test_microdesc.c @@ -21,6 +21,7 @@ #include "feature/nodelist/routerstatus_st.h" #include "test/test.h" +#include "test/log_test_helpers.h" #ifdef HAVE_SYS_STAT_H #include <sys/stat.h> @@ -648,6 +649,41 @@ static const char MD_PARSE_TEST_DATA[] = "ntor-onion-key k2yFqTU2vzMCQDEiE/j9UcEHxKrXMLpB3IL0or09sik=\n" "id rsa1024 2A8wYpHxnkKJ92orocvIQBzeHlE\n" "p6 allow 80\n" + /* Good 11: Normal, non-exit relay with ipv6 address */ + "onion-key\n" + "-----BEGIN RSA PUBLIC KEY-----\n" + "MIGJAoGBAM7uUtq5F6h63QNYIvC+4NcWaD0DjtnrOORZMkdpJhinXUOwce3cD5Dj\n" + "sgdN1wJpWpTQMXJ2DssfSgmOVXETP7qJuZyRprxalQhaEATMDNJA/66Ml1jSO9mZ\n" + "+8Xb7m/4q778lNtkSbsvMaYD2Dq6k2QQ3kMhr9z8oUtX0XA23+pfAgMBAAE=\n" + "-----END RSA PUBLIC KEY-----\n" + "a [::1:2:3:4]:9090\n" + "a 18.0.0.1:9999\n" + "ntor-onion-key k2yFqTU2vzMCQDEiE/j9UcEHxKrXMLpB3IL0or09sik=\n" + "id rsa1024 2A8wYpHxnkKJ92orocvIQBzeHlE\n" + /* Good 12: Normal, exit relay with ipv6 address */ + "onion-key\n" + "-----BEGIN RSA PUBLIC KEY-----\n" + "MIGJAoGBAM7uUtq5F6h63QNYIvC+4NcWaD0DjtnrOORZMkdpJhinXUOwce3cD5Dj\n" + "sgdN1wJpWpTQMXJ2DssfSgmOVXETP7qJuZyRprxalQhaEATMDNJA/66Ml1jSO9mZ\n" + "+8Xb7m/4q778lNtkSbsvMaYD2Dq6k2QQ3kMhr9z8oUtX0XA23+pfAgMBAAE=\n" + "-----END RSA PUBLIC KEY-----\n" + "a [::1:2:3:4]:9090\n" + "a 18.0.0.1:9999\n" + "ntor-onion-key k2yFqTU2vzMCQDEiE/j9UcEHxKrXMLpB3IL0or09sik=\n" + "p accept 20-23,43,53,79-81,88,110,143,194,220,389,443,464,531,543-544\n" + "id rsa1024 2A8wYpHxnkKJ92orocvIQBzeHlE\n" + /* Good 13: Normal, exit relay with only ipv6 exit policy */ + "onion-key\n" + "-----BEGIN RSA PUBLIC KEY-----\n" + "MIGJAoGBAM7uUtq5F6h63QNYIvC+4NcWaD0DjtnrOORZMkdpJhinXUOwce3cD5Dj\n" + "sgdN1wJpWpTQMXJ2DssfSgmOVXETP7qJuZyRprxalQhaEATMDNJA/66Ml1jSO9mZ\n" + "+8Xb7m/4q778lNtkSbsvMaYD2Dq6k2QQ3kMhr9z8oUtX0XA23+pfAgMBAAE=\n" + "-----END RSA PUBLIC KEY-----\n" + "a [::1:2:3:4]:9090\n" + "a 18.0.0.1:9999\n" + "ntor-onion-key k2yFqTU2vzMCQDEiE/j9UcEHxKrXMLpB3IL0or09sik=\n" + "p6 accept 20-23,43,53,79-81,88,110,143,194,220,389,443,464,531,543-544\n" + "id rsa1024 2A8wYpHxnkKJ92orocvIQBzeHlE\n" ; #ifdef HAVE_CFLAG_WOVERLENGTH_STRINGS ENABLE_GCC_WARNING(overlength-strings) @@ -665,7 +701,7 @@ test_md_parse(void *arg) smartlist_t *mds = microdescs_parse_from_string(MD_PARSE_TEST_DATA, NULL, 1, SAVED_NOWHERE, invalid); - tt_int_op(smartlist_len(mds), OP_EQ, 11); + tt_int_op(smartlist_len(mds), OP_EQ, 14); tt_int_op(smartlist_len(invalid), OP_EQ, 4); test_memeq_hex(smartlist_get(invalid,0), @@ -712,6 +748,21 @@ test_md_parse(void *arg) tt_assert(tor_addr_family(&md->ipv6_addr) == AF_INET6); tt_int_op(md->ipv6_orport, OP_EQ, 9090); + md = smartlist_get(mds, 11); + tt_assert(tor_addr_family(&md->ipv6_addr) == AF_INET6); + tt_int_op(md->ipv6_orport, OP_EQ, 9090); + tt_int_op(md->policy_is_reject_star, OP_EQ, 1); + + md = smartlist_get(mds, 12); + tt_assert(tor_addr_family(&md->ipv6_addr) == AF_INET6); + tt_int_op(md->ipv6_orport, OP_EQ, 9090); + tt_int_op(md->policy_is_reject_star, OP_EQ, 0); + + md = smartlist_get(mds, 13); + tt_assert(tor_addr_family(&md->ipv6_addr) == AF_INET6); + tt_int_op(md->ipv6_orport, OP_EQ, 9090); + tt_int_op(md->policy_is_reject_star, OP_EQ, 0); + done: SMARTLIST_FOREACH(mds, microdesc_t *, mdsc, microdesc_free(mdsc)); smartlist_free(mds); @@ -720,6 +771,80 @@ test_md_parse(void *arg) tor_free(mem_op_hex_tmp); } +static void +test_md_parse_id_ed25519(void *arg) +{ + (void)arg; + + /* A correct MD with an ed25519 ID ... and an unspecified ID type, + * which is permitted. */ + const char GOOD_MD[] = + "onion-key\n" + "-----BEGIN RSA PUBLIC KEY-----\n" + "MIGJAoGBAM7uUtq5F6h63QNYIvC+4NcWaD0DjtnrOORZMkdpJhinXUOwce3cD5Dj\n" + "sgdN1wJpWpTQMXJ2DssfSgmOVXETP7qJuZyRprxalQhaEATMDNJA/66Ml1jSO9mZ\n" + "+8Xb7m/4q778lNtkSbsvMaYD2Dq6k2QQ3kMhr9z8oUtX0XA23+pfAgMBAAE=\n" + "-----END RSA PUBLIC KEY-----\n" + "id ed25519 VGhpcyBpc24ndCBhY3R1YWxseSBhIHB1YmxpYyBrZXk\n" + "id wumpus dodecahedron\n"; + + smartlist_t *mds = NULL; + const microdesc_t *md; + + mds = microdescs_parse_from_string(GOOD_MD, + NULL, 1, SAVED_NOWHERE, NULL); + tt_assert(mds); + tt_int_op(smartlist_len(mds), OP_EQ, 1); + md = smartlist_get(mds, 0); + tt_mem_op(md->ed25519_identity_pkey, OP_EQ, + "This isn't actually a public key", ED25519_PUBKEY_LEN); + SMARTLIST_FOREACH(mds, microdesc_t *, m, microdesc_free(m)); + smartlist_free(mds); + + /* As above, but ed25519 ID key appears twice. */ + const char DUPLICATE_KEY[] = + "onion-key\n" + "-----BEGIN RSA PUBLIC KEY-----\n" + "MIGJAoGBAM7uUtq5F6h63QNYIvC+4NcWaD0DjtnrOORZMkdpJhinXUOwce3cD5Dj\n" + "sgdN1wJpWpTQMXJ2DssfSgmOVXETP7qJuZyRprxalQhaEATMDNJA/66Ml1jSO9mZ\n" + "+8Xb7m/4q778lNtkSbsvMaYD2Dq6k2QQ3kMhr9z8oUtX0XA23+pfAgMBAAE=\n" + "-----END RSA PUBLIC KEY-----\n" + "id ed25519 VGhpcyBpc24ndCBhY3R1YWxseSBhIHB1YmxpYyBrZXk\n" + "id ed25519 VGhpcyBpc24ndCBhY3R1YWxseSBhIHB1YmxpYyBrZXk\n"; + + setup_capture_of_logs(LOG_WARN); + mds = microdescs_parse_from_string(DUPLICATE_KEY, + NULL, 1, SAVED_NOWHERE, NULL); + tt_assert(mds); + tt_int_op(smartlist_len(mds), OP_EQ, 0); // no entries. + expect_single_log_msg_containing("Extra ed25519 key"); + mock_clean_saved_logs(); + smartlist_free(mds); + + /* As above, but ed25519 ID key is invalid. */ + const char BOGUS_KEY[] = + "onion-key\n" + "-----BEGIN RSA PUBLIC KEY-----\n" + "MIGJAoGBAM7uUtq5F6h63QNYIvC+4NcWaD0DjtnrOORZMkdpJhinXUOwce3cD5Dj\n" + "sgdN1wJpWpTQMXJ2DssfSgmOVXETP7qJuZyRprxalQhaEATMDNJA/66Ml1jSO9mZ\n" + "+8Xb7m/4q778lNtkSbsvMaYD2Dq6k2QQ3kMhr9z8oUtX0XA23+pfAgMBAAE=\n" + "-----END RSA PUBLIC KEY-----\n" + "id ed25519 VGhpcyBpc24ndCBhY3R1YWxseSBhIHB1YmxpYyZZZZZZZZZZZ\n"; + + mds = microdescs_parse_from_string(BOGUS_KEY, + NULL, 1, SAVED_NOWHERE, NULL); + tt_assert(mds); + tt_int_op(smartlist_len(mds), OP_EQ, 0); // no entries. + expect_single_log_msg_containing("Bogus ed25519 key"); + + done: + if (mds) { + SMARTLIST_FOREACH(mds, microdesc_t *, m, microdesc_free(m)); + smartlist_free(mds); + } + teardown_capture_of_logs(); +} + static int mock_rgsbd_called = 0; static routerstatus_t *mock_rgsbd_val_a = NULL; static routerstatus_t *mock_rgsbd_val_b = NULL; @@ -853,6 +978,7 @@ struct testcase_t microdesc_tests[] = { { "broken_cache", test_md_cache_broken, TT_FORK, NULL, NULL }, { "generate", test_md_generate, 0, NULL, NULL }, { "parse", test_md_parse, 0, NULL, NULL }, + { "parse_id_ed25519", test_md_parse_id_ed25519, 0, NULL, NULL }, { "reject_cache", test_md_reject_cache, TT_FORK, NULL, NULL }, { "corrupt_desc", test_md_corrupt_desc, TT_FORK, NULL, NULL }, END_OF_TESTCASES diff --git a/src/test/test_nodelist.c b/src/test/test_nodelist.c index 8d6d3cb974..b8ca56bb81 100644 --- a/src/test/test_nodelist.c +++ b/src/test/test_nodelist.c @@ -10,11 +10,13 @@ #include "core/or/or.h" #include "lib/crypt_ops/crypto_rand.h" +#include "feature/nodelist/describe.h" #include "feature/nodelist/networkstatus.h" #include "feature/nodelist/nodefamily.h" #include "feature/nodelist/nodelist.h" #include "feature/nodelist/torcert.h" +#include "core/or/extend_info_st.h" #include "feature/nodelist/microdesc_st.h" #include "feature/nodelist/networkstatus_st.h" #include "feature/nodelist/node_st.h" @@ -76,7 +78,7 @@ test_nodelist_node_get_verbose_nickname_not_named(void *arg) } /** A node should be considered a directory server if it has an open dirport - * of it accepts tunnelled directory requests. + * or it accepts tunnelled directory requests. */ static void test_nodelist_node_is_dir(void *arg) @@ -640,6 +642,610 @@ test_nodelist_nodefamily_canonicalize(void *arg) tor_free(c); } +/** format_node_description() should return + * "Fingerprint~Nickname at IPv4 and [IPv6]". + * The nickname and addresses are optional. + */ +static void +test_nodelist_format_node_description(void *arg) +{ + char mock_digest[DIGEST_LEN]; + char mock_nickname[MAX_NICKNAME_LEN+1]; + tor_addr_t mock_null_ip; + tor_addr_t mock_ipv4; + tor_addr_t mock_ipv6; + + char ndesc[NODE_DESC_BUF_LEN]; + const char *rv = NULL; + + (void) arg; + + /* Clear variables */ + memset(ndesc, 0, sizeof(ndesc)); + memset(mock_digest, 0, sizeof(mock_digest)); + memset(mock_nickname, 0, sizeof(mock_nickname)); + memset(&mock_null_ip, 0, sizeof(mock_null_ip)); + memset(&mock_ipv4, 0, sizeof(mock_ipv4)); + memset(&mock_ipv6, 0, sizeof(mock_ipv6)); + + /* Set variables */ + memcpy(mock_digest, + "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA" + "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA", + sizeof(mock_digest)); + strlcpy(mock_nickname, "TestOR7890123456789", sizeof(mock_nickname)); + tor_addr_parse(&mock_ipv4, "111.222.233.244"); + tor_addr_parse(&mock_ipv6, "[1111:2222:3333:4444:5555:6666:7777:8888]"); + + /* Test function with variables */ + rv = format_node_description(ndesc, + mock_digest, + NULL, + NULL, + 0); + tt_ptr_op(rv, OP_EQ, ndesc); + tt_str_op(ndesc, OP_EQ, "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"); + + /* format node description should use ~ because named is deprecated */ + rv = format_node_description(ndesc, + mock_digest, + mock_nickname, + NULL, + 0); + tt_ptr_op(rv, OP_EQ, ndesc); + tt_str_op(ndesc, OP_EQ, + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~""TestOR7890123456789"); + + /* Try a null IP address, rather than NULL */ + rv = format_node_description(ndesc, + mock_digest, + mock_nickname, + &mock_null_ip, + 0); + tt_ptr_op(rv, OP_EQ, ndesc); + tt_str_op(ndesc, OP_EQ, + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789"); + + /* Try some real IP addresses */ + rv = format_node_description(ndesc, + mock_digest, + NULL, + &mock_ipv4, + 0); + tt_ptr_op(rv, OP_EQ, ndesc); + tt_str_op(ndesc, OP_EQ, + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA at 111.222.233.244"); + + rv = format_node_description(ndesc, + mock_digest, + mock_nickname, + &mock_ipv6, + 0); + tt_ptr_op(rv, OP_EQ, ndesc); + tt_str_op(ndesc, OP_EQ, + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at " + "[1111:2222:3333:4444:5555:6666:7777:8888]"); + + rv = format_node_description(ndesc, + mock_digest, + mock_nickname, + &mock_ipv6, + tor_addr_to_ipv4h(&mock_ipv4)); + tt_ptr_op(rv, OP_EQ, ndesc); + tt_str_op(ndesc, OP_EQ, + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at " + "111.222.233.244 and [1111:2222:3333:4444:5555:6666:7777:8888]"); + + /* test NULL handling */ + rv = format_node_description(NULL, NULL, NULL, NULL, 0); + tt_str_op(rv, OP_EQ, "<NULL BUFFER>"); + + rv = format_node_description(ndesc, NULL, NULL, NULL, 0); + tt_ptr_op(rv, OP_EQ, ndesc); + tt_str_op(rv, OP_EQ, "<NULL ID DIGEST>"); + + done: + return; +} + +/** router_describe() is a wrapper for format_node_description(), see that + * test for details. + * + * The routerinfo-only node_describe() tests are in this function, + * so we can re-use the same mocked variables. + */ +static void +test_nodelist_router_describe(void *arg) +{ + char mock_nickname[MAX_NICKNAME_LEN+1]; + tor_addr_t mock_ipv4; + routerinfo_t mock_ri_ipv4; + routerinfo_t mock_ri_ipv6; + routerinfo_t mock_ri_dual; + + const char *rv = NULL; + + (void) arg; + + /* Clear variables */ + memset(mock_nickname, 0, sizeof(mock_nickname)); + memset(&mock_ipv4, 0, sizeof(mock_ipv4)); + memset(&mock_ri_ipv4, 0, sizeof(mock_ri_ipv4)); + memset(&mock_ri_ipv6, 0, sizeof(mock_ri_ipv6)); + memset(&mock_ri_dual, 0, sizeof(mock_ri_dual)); + + /* Set up the dual-stack routerinfo */ + memcpy(mock_ri_dual.cache_info.identity_digest, + "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA" + "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA", + sizeof(mock_ri_dual.cache_info.identity_digest)); + strlcpy(mock_nickname, "TestOR7890123456789", sizeof(mock_nickname)); + mock_ri_dual.nickname = mock_nickname; + tor_addr_parse(&mock_ipv4, "111.222.233.244"); + mock_ri_dual.addr = tor_addr_to_ipv4h(&mock_ipv4); + tor_addr_parse(&mock_ri_dual.ipv6_addr, + "[1111:2222:3333:4444:5555:6666:7777:8888]"); + + /* Create and modify the other routerinfos. + * mock_nickname is referenced from all 3 routerinfos. + * That's ok, all their memory is static. */ + memcpy(&mock_ri_ipv4, &mock_ri_dual, sizeof(mock_ri_ipv4)); + memcpy(&mock_ri_ipv6, &mock_ri_dual, sizeof(mock_ri_ipv6)); + /* Clear the unnecessary addresses */ + memset(&mock_ri_ipv4.ipv6_addr, 0, sizeof(mock_ri_ipv4.ipv6_addr)); + mock_ri_ipv6.addr = 0; + + /* We don't test the no-nickname and no-IP cases, because they're covered by + * format_node_description(), and we don't expect to see them in Tor code. */ + + /* Try some real IP addresses */ + rv = router_describe(&mock_ri_ipv4); + tt_str_op(rv, OP_EQ, + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at " + "111.222.233.244"); + + rv = router_describe(&mock_ri_ipv6); + tt_str_op(rv, OP_EQ, + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at " + "[1111:2222:3333:4444:5555:6666:7777:8888]"); + + rv = router_describe(&mock_ri_dual); + tt_str_op(rv, OP_EQ, + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at " + "111.222.233.244 and [1111:2222:3333:4444:5555:6666:7777:8888]"); + + /* test NULL handling */ + rv = router_describe(NULL); + tt_str_op(rv, OP_EQ, "<null>"); + + /* Now test a node with only these routerinfos */ + node_t mock_node; + memset(&mock_node, 0, sizeof(mock_node)); + memcpy(mock_node.identity, + "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA" + "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA", + sizeof(mock_node.identity)); + + /* Try some real IP addresses */ + mock_node.ri = &mock_ri_ipv4; + rv = node_describe(&mock_node); + tt_str_op(rv, OP_EQ, + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at " + "111.222.233.244"); + + mock_node.ri = &mock_ri_ipv6; + rv = node_describe(&mock_node); + tt_str_op(rv, OP_EQ, + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at " + "[1111:2222:3333:4444:5555:6666:7777:8888]"); + + mock_node.ri = &mock_ri_dual; + rv = node_describe(&mock_node); + tt_str_op(rv, OP_EQ, + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at " + "111.222.233.244 and [1111:2222:3333:4444:5555:6666:7777:8888]"); + + done: + return; +} + +/** node_describe() is a wrapper for format_node_description(), see that + * test for details. + * + * The routerinfo-only and routerstatus-only node_describe() tests are in + * test_nodelist_router_describe() and test_nodelist_routerstatus_describe(), + * so we can re-use their mocked variables. + */ +static void +test_nodelist_node_describe(void *arg) +{ + char mock_nickname[MAX_NICKNAME_LEN+1]; + tor_addr_t mock_ipv4; + + const char *rv = NULL; + + (void) arg; + + /* Routerinfos */ + routerinfo_t mock_ri_dual; + + /* Clear variables */ + memset(mock_nickname, 0, sizeof(mock_nickname)); + memset(&mock_ipv4, 0, sizeof(mock_ipv4)); + memset(&mock_ri_dual, 0, sizeof(mock_ri_dual)); + + /* Set up the dual-stack routerinfo */ + memcpy(mock_ri_dual.cache_info.identity_digest, + "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA" + "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA", + sizeof(mock_ri_dual.cache_info.identity_digest)); + strlcpy(mock_nickname, "TestOR7890123456789", sizeof(mock_nickname)); + mock_ri_dual.nickname = mock_nickname; + tor_addr_parse(&mock_ipv4, "111.222.233.244"); + mock_ri_dual.addr = tor_addr_to_ipv4h(&mock_ipv4); + tor_addr_parse(&mock_ri_dual.ipv6_addr, + "[1111:2222:3333:4444:5555:6666:7777:8888]"); + + /* Routerstatuses */ + routerstatus_t mock_rs_ipv4; + routerstatus_t mock_rs_dual; + + /* Clear variables */ + memset(&mock_ipv4, 0, sizeof(mock_ipv4)); + memset(&mock_rs_ipv4, 0, sizeof(mock_rs_ipv4)); + memset(&mock_rs_dual, 0, sizeof(mock_rs_dual)); + + /* Set up the dual-stack routerstatus */ + memcpy(mock_rs_dual.identity_digest, + "\xBB\xBB\xBB\xBB\xBB\xBB\xBB\xBB\xBB\xBB" + "\xBB\xBB\xBB\xBB\xBB\xBB\xBB\xBB\xBB\xBB", + sizeof(mock_rs_dual.identity_digest)); + strlcpy(mock_rs_dual.nickname, "Bbb", + sizeof(mock_rs_dual.nickname)); + tor_addr_parse(&mock_ipv4, "2.2.2.2"); + mock_rs_dual.addr = tor_addr_to_ipv4h(&mock_ipv4); + tor_addr_parse(&mock_rs_dual.ipv6_addr, + "[bbbb::bbbb]"); + + /* Create and modify the other routerstatus. */ + memcpy(&mock_rs_ipv4, &mock_rs_dual, sizeof(mock_rs_ipv4)); + /* Clear the unnecessary IPv6 address */ + memset(&mock_rs_ipv4.ipv6_addr, 0, sizeof(mock_rs_ipv4.ipv6_addr)); + + /* Microdescs */ + microdesc_t mock_md_null; + microdesc_t mock_md_ipv6; + + /* Clear variables */ + memset(&mock_md_null, 0, sizeof(mock_md_null)); + memset(&mock_md_ipv6, 0, sizeof(mock_md_ipv6)); + + /* Set up the microdesc */ + tor_addr_parse(&mock_md_ipv6.ipv6_addr, + "[eeee::6000:6000]"); + + /* Set up the node */ + node_t mock_node; + memset(&mock_node, 0, sizeof(mock_node)); + memcpy(mock_node.identity, + "\xCC\xCC\xCC\xCC\xCC\xCC\xCC\xCC\xCC\xCC" + "\xCC\xCC\xCC\xCC\xCC\xCC\xCC\xCC\xCC\xCC", + sizeof(mock_node.identity)); + + /* Test that the routerinfo and routerstatus work separately, but the + * identity comes from the node */ + mock_node.ri = &mock_ri_dual; + mock_node.rs = NULL; + mock_node.md = NULL; + rv = node_describe(&mock_node); + tt_str_op(rv, OP_EQ, + "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~TestOR7890123456789 at " + "111.222.233.244 and [1111:2222:3333:4444:5555:6666:7777:8888]"); + + mock_node.ri = NULL; + mock_node.rs = &mock_rs_ipv4; + mock_node.md = NULL; + rv = node_describe(&mock_node); + tt_str_op(rv, OP_EQ, + "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at " + "2.2.2.2"); + + mock_node.ri = NULL; + mock_node.rs = &mock_rs_dual; + mock_node.md = NULL; + rv = node_describe(&mock_node); + tt_str_op(rv, OP_EQ, + "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at " + "2.2.2.2 and [bbbb::bbbb]"); + + /* Test that the routerstatus overrides the routerinfo */ + mock_node.ri = &mock_ri_dual; + mock_node.rs = &mock_rs_ipv4; + mock_node.md = NULL; + rv = node_describe(&mock_node); + tt_str_op(rv, OP_EQ, + "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at " + "2.2.2.2"); + + mock_node.ri = &mock_ri_dual; + mock_node.rs = &mock_rs_dual; + mock_node.md = NULL; + rv = node_describe(&mock_node); + tt_str_op(rv, OP_EQ, + "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at " + "2.2.2.2 and [bbbb::bbbb]"); + + /* Test that the microdesc IPv6 is used if the routerinfo doesn't have IPv6 + */ + mock_node.ri = NULL; + mock_node.rs = &mock_rs_ipv4; + mock_node.md = &mock_md_ipv6; + rv = node_describe(&mock_node); + tt_str_op(rv, OP_EQ, + "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at " + "2.2.2.2 and [eeee::6000:6000]"); + + mock_node.ri = NULL; + mock_node.rs = &mock_rs_ipv4; + mock_node.md = &mock_md_null; + rv = node_describe(&mock_node); + tt_str_op(rv, OP_EQ, + "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at " + "2.2.2.2"); + + mock_node.ri = NULL; + mock_node.rs = &mock_rs_dual; + mock_node.md = &mock_md_ipv6; + rv = node_describe(&mock_node); + tt_str_op(rv, OP_EQ, + "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at " + "2.2.2.2 and [bbbb::bbbb]"); + + mock_node.ri = NULL; + mock_node.rs = &mock_rs_dual; + mock_node.md = &mock_md_null; + rv = node_describe(&mock_node); + tt_str_op(rv, OP_EQ, + "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at " + "2.2.2.2 and [bbbb::bbbb]"); + + /* Test that the routerinfo doesn't change the results above + */ + mock_node.ri = &mock_ri_dual; + mock_node.rs = &mock_rs_ipv4; + mock_node.md = &mock_md_ipv6; + rv = node_describe(&mock_node); + tt_str_op(rv, OP_EQ, + "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at " + "2.2.2.2 and [eeee::6000:6000]"); + + mock_node.ri = &mock_ri_dual; + mock_node.rs = &mock_rs_ipv4; + mock_node.md = &mock_md_null; + rv = node_describe(&mock_node); + tt_str_op(rv, OP_EQ, + "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at " + "2.2.2.2"); + + mock_node.ri = &mock_ri_dual; + mock_node.rs = &mock_rs_dual; + mock_node.md = &mock_md_ipv6; + rv = node_describe(&mock_node); + tt_str_op(rv, OP_EQ, + "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at " + "2.2.2.2 and [bbbb::bbbb]"); + + mock_node.ri = &mock_ri_dual; + mock_node.rs = &mock_rs_dual; + mock_node.md = &mock_md_null; + rv = node_describe(&mock_node); + tt_str_op(rv, OP_EQ, + "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at " + "2.2.2.2 and [bbbb::bbbb]"); + + /* test NULL handling */ + rv = node_describe(NULL); + tt_str_op(rv, OP_EQ, "<null>"); + + mock_node.ri = NULL; + mock_node.rs = NULL; + mock_node.md = NULL; + rv = node_describe(&mock_node); + tt_str_op(rv, OP_EQ, + "<null rs and ri>"); + + done: + return; +} + +/** routerstatus_describe() is a wrapper for format_node_description(), see + * that test for details. + * + * The routerstatus-only node_describe() tests are in this function, + * so we can re-use the same mocked variables. + */ +static void +test_nodelist_routerstatus_describe(void *arg) +{ + tor_addr_t mock_ipv4; + routerstatus_t mock_rs_ipv4; + routerstatus_t mock_rs_ipv6; + routerstatus_t mock_rs_dual; + + const char *rv = NULL; + + (void) arg; + + /* Clear variables */ + memset(&mock_ipv4, 0, sizeof(mock_ipv4)); + memset(&mock_rs_ipv4, 0, sizeof(mock_rs_ipv4)); + memset(&mock_rs_ipv6, 0, sizeof(mock_rs_ipv6)); + memset(&mock_rs_dual, 0, sizeof(mock_rs_dual)); + + /* Set up the dual-stack routerstatus */ + memcpy(mock_rs_dual.identity_digest, + "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA" + "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA", + sizeof(mock_rs_dual.identity_digest)); + strlcpy(mock_rs_dual.nickname, "TestOR7890123456789", + sizeof(mock_rs_dual.nickname)); + tor_addr_parse(&mock_ipv4, "111.222.233.244"); + mock_rs_dual.addr = tor_addr_to_ipv4h(&mock_ipv4); + tor_addr_parse(&mock_rs_dual.ipv6_addr, + "[1111:2222:3333:4444:5555:6666:7777:8888]"); + + /* Create and modify the other routerstatuses. */ + memcpy(&mock_rs_ipv4, &mock_rs_dual, sizeof(mock_rs_ipv4)); + memcpy(&mock_rs_ipv6, &mock_rs_dual, sizeof(mock_rs_ipv6)); + /* Clear the unnecessary addresses */ + memset(&mock_rs_ipv4.ipv6_addr, 0, sizeof(mock_rs_ipv4.ipv6_addr)); + mock_rs_ipv6.addr = 0; + + /* We don't test the no-nickname and no-IP cases, because they're covered by + * format_node_description(), and we don't expect to see them in Tor code. */ + + /* Try some real IP addresses */ + rv = routerstatus_describe(&mock_rs_ipv4); + tt_str_op(rv, OP_EQ, + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at " + "111.222.233.244"); + + rv = routerstatus_describe(&mock_rs_ipv6); + tt_str_op(rv, OP_EQ, + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at " + "[1111:2222:3333:4444:5555:6666:7777:8888]"); + + rv = routerstatus_describe(&mock_rs_dual); + tt_str_op(rv, OP_EQ, + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at " + "111.222.233.244 and [1111:2222:3333:4444:5555:6666:7777:8888]"); + + /* test NULL handling */ + rv = routerstatus_describe(NULL); + tt_str_op(rv, OP_EQ, "<null>"); + + /* Now test a node with only these routerstatuses */ + node_t mock_node; + memset(&mock_node, 0, sizeof(mock_node)); + memcpy(mock_node.identity, + "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA" + "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA", + sizeof(mock_node.identity)); + + /* Try some real IP addresses */ + mock_node.rs = &mock_rs_ipv4; + rv = node_describe(&mock_node); + tt_str_op(rv, OP_EQ, + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at " + "111.222.233.244"); + + mock_node.rs = &mock_rs_ipv6; + rv = node_describe(&mock_node); + tt_str_op(rv, OP_EQ, + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at " + "[1111:2222:3333:4444:5555:6666:7777:8888]"); + + mock_node.rs = &mock_rs_dual; + rv = node_describe(&mock_node); + tt_str_op(rv, OP_EQ, + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at " + "111.222.233.244 and [1111:2222:3333:4444:5555:6666:7777:8888]"); + + done: + return; +} + +/** extend_info_describe() is a wrapper for format_node_description(), see + * that test for details. + */ +static void +test_nodelist_extend_info_describe(void *arg) +{ + extend_info_t mock_ei_ipv4; + extend_info_t mock_ei_ipv6; + + const char *rv = NULL; + + (void) arg; + + /* Clear variables */ + memset(&mock_ei_ipv4, 0, sizeof(mock_ei_ipv4)); + memset(&mock_ei_ipv6, 0, sizeof(mock_ei_ipv6)); + + /* Set up the IPv4 extend info */ + memcpy(mock_ei_ipv4.identity_digest, + "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA" + "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA", + sizeof(mock_ei_ipv4.identity_digest)); + strlcpy(mock_ei_ipv4.nickname, "TestOR7890123456789", + sizeof(mock_ei_ipv4.nickname)); + tor_addr_parse(&mock_ei_ipv4.addr, "111.222.233.244"); + + /* Create and modify the other extend info. */ + memcpy(&mock_ei_ipv6, &mock_ei_ipv4, sizeof(mock_ei_ipv6)); + tor_addr_parse(&mock_ei_ipv6.addr, + "[1111:2222:3333:4444:5555:6666:7777:8888]"); + + /* We don't test the no-nickname and no-IP cases, because they're covered by + * format_node_description(), and we don't expect to see them in Tor code. */ + + /* Try some real IP addresses */ + rv = extend_info_describe(&mock_ei_ipv4); + tt_str_op(rv, OP_EQ, + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at " + "111.222.233.244"); + + rv = extend_info_describe(&mock_ei_ipv6); + tt_str_op(rv, OP_EQ, + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at " + "[1111:2222:3333:4444:5555:6666:7777:8888]"); + + /* Extend infos only have one IP address, so there is no dual case */ + + /* test NULL handling */ + rv = extend_info_describe(NULL); + tt_str_op(rv, OP_EQ, "<null>"); + + done: + return; +} + +/** router_get_verbose_nickname() should return "Fingerprint~Nickname" + */ +static void +test_nodelist_router_get_verbose_nickname(void *arg) +{ + routerinfo_t mock_ri; + char mock_nickname[MAX_NICKNAME_LEN+1]; + + char vname[MAX_VERBOSE_NICKNAME_LEN+1]; + + (void) arg; + + memset(&mock_ri, 0, sizeof(routerinfo_t)); + memset(mock_nickname, 0, sizeof(mock_nickname)); + mock_ri.nickname = mock_nickname; + + /* verbose nickname should use ~ because named is deprecated */ + strlcpy(mock_nickname, "TestOR", sizeof(mock_nickname)); + memcpy(mock_ri.cache_info.identity_digest, + "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA" + "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA", + DIGEST_LEN); + router_get_verbose_nickname(vname, &mock_ri); + tt_str_op(vname, OP_EQ, "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR"); + + /* test NULL router handling */ + router_get_verbose_nickname(vname, NULL); + tt_str_op(vname, OP_EQ, "<null>"); + + router_get_verbose_nickname(NULL, &mock_ri); + router_get_verbose_nickname(NULL, NULL); + + done: + return; +} + #define NODE(name, flags) \ { #name, test_nodelist_##name, (flags), NULL, NULL } @@ -654,5 +1260,11 @@ struct testcase_t nodelist_tests[] = { NODE(nickname_matches, 0), NODE(node_nodefamily, TT_FORK), NODE(nodefamily_canonicalize, 0), + NODE(format_node_description, 0), + NODE(router_describe, 0), + NODE(node_describe, 0), + NODE(routerstatus_describe, 0), + NODE(extend_info_describe, 0), + NODE(router_get_verbose_nickname, 0), END_OF_TESTCASES }; diff --git a/src/test/test_options.c b/src/test/test_options.c index 7009910b0f..69407a999b 100644 --- a/src/test/test_options.c +++ b/src/test/test_options.c @@ -5,7 +5,7 @@ #define CONFIG_PRIVATE #include "core/or/or.h" -#include "app/config/confparse.h" +#include "lib/confmgt/confparse.h" #include "app/config/config.h" #include "test/test.h" #include "lib/geoip/geoip.h" @@ -14,6 +14,7 @@ #include "feature/nodelist/routerset.h" #include "core/mainloop/mainloop.h" #include "test/log_test_helpers.h" +#include "test/resolve_test_helpers.h" #include "lib/sandbox/sandbox.h" #include "lib/memarea/memarea.h" @@ -54,9 +55,9 @@ setup_log_callback(void) { log_severity_list_t lst; memset(&lst, 0, sizeof(lst)); - lst.masks[LOG_ERR - LOG_ERR] = ~0; - lst.masks[LOG_WARN - LOG_ERR] = ~0; - lst.masks[LOG_NOTICE - LOG_ERR] = ~0; + lst.masks[SEVERITY_MASK_IDX(LOG_ERR)] = LD_ALL_DOMAINS; + lst.masks[SEVERITY_MASK_IDX(LOG_WARN)] = LD_ALL_DOMAINS; + lst.masks[SEVERITY_MASK_IDX(LOG_NOTICE)] = LD_ALL_DOMAINS; add_callback_log(&lst, log_cback); mark_logs_temp(); } @@ -96,7 +97,7 @@ clear_log_messages(void) opt->command = CMD_RUN_TOR; \ options_init(opt); \ \ - dflt = config_dup(&options_format, opt); \ + dflt = config_dup(get_options_mgr(), opt); \ clear_log_messages(); \ } while (0) @@ -196,7 +197,7 @@ test_options_validate_impl(const char *configuration, if (r) goto done; - r = config_assign(&options_format, opt, cl, 0, &msg); + r = config_assign(get_options_mgr(), opt, cl, 0, &msg); if (phase == PH_ASSIGN) { if (test_options_checkmsgs(configuration, expect_errmsg, expect_log_severity, @@ -241,6 +242,7 @@ test_options_validate(void *arg) (void)arg; setup_log_callback(); sandbox_disable_getaddrinfo_cache(); + mock_hostname_resolver(); WANT_ERR("ExtORPort 500000", "Invalid ExtORPort", PH_VALIDATE); @@ -258,13 +260,17 @@ test_options_validate(void *arg) WANT_ERR("BridgeRelay 1\nDirCache 0", "We're a bridge but DirCache is disabled.", PH_VALIDATE); + // XXXX We should replace this with a more full error message once #29211 + // XXXX is done. It is truncated for now because at the current stage + // XXXX of refactoring, we can't give a full error message like before. WANT_ERR_LOG("HeartbeatPeriod 21 snarks", - "Interval 'HeartbeatPeriod 21 snarks' is malformed or" - " out of bounds.", LOG_WARN, "Unknown unit 'snarks'.", + "malformed or out of bounds", LOG_WARN, + "Unknown unit 'snarks'.", PH_ASSIGN); + // XXXX As above. WANT_ERR_LOG("LogTimeGranularity 21 snarks", - "Msec interval 'LogTimeGranularity 21 snarks' is malformed or" - " out of bounds.", LOG_WARN, "Unknown unit 'snarks'.", + "malformed or out of bounds", LOG_WARN, + "Unknown unit 'snarks'.", PH_ASSIGN); OK("HeartbeatPeriod 1 hour", PH_VALIDATE); OK("LogTimeGranularity 100 milliseconds", PH_VALIDATE); @@ -278,6 +284,7 @@ test_options_validate(void *arg) close_temp_logs(); clear_log_messages(); + unmock_hostname_resolver(); return; } @@ -300,7 +307,7 @@ test_have_enough_mem_for_dircache(void *arg) r = config_get_lines(configuration, &cl, 1); tt_int_op(r, OP_EQ, 0); - r = config_assign(&options_format, opt, cl, 0, &msg); + r = config_assign(get_options_mgr(), opt, cl, 0, &msg); tt_int_op(r, OP_EQ, 0); /* 300 MB RAM available, DirCache enabled */ @@ -323,7 +330,7 @@ test_have_enough_mem_for_dircache(void *arg) r = config_get_lines(configuration, &cl, 1); tt_int_op(r, OP_EQ, 0); - r = config_assign(&options_format, opt, cl, 0, &msg); + r = config_assign(get_options_mgr(), opt, cl, 0, &msg); tt_int_op(r, OP_EQ, 0); /* 300 MB RAM available, DirCache enabled, Bridge */ @@ -346,7 +353,7 @@ test_have_enough_mem_for_dircache(void *arg) r = config_get_lines(configuration, &cl, 1); tt_int_op(r, OP_EQ, 0); - r = config_assign(&options_format, opt, cl, 0, &msg); + r = config_assign(get_options_mgr(), opt, cl, 0, &msg); tt_int_op(r, OP_EQ, 0); /* 200 MB RAM available, DirCache disabled */ @@ -434,7 +441,7 @@ get_options_test_data(const char *conf) rv = config_get_lines(conf, &cl, 1); tt_int_op(rv, OP_EQ, 0); - rv = config_assign(&options_format, result->opt, cl, 0, &msg); + rv = config_assign(get_options_mgr(), result->opt, cl, 0, &msg); if (msg) { /* Display the parse error message by comparing it with an empty string */ tt_str_op(msg, OP_EQ, ""); @@ -445,7 +452,7 @@ get_options_test_data(const char *conf) result->opt->TokenBucketRefillInterval = 1; rv = config_get_lines(TEST_OPTIONS_OLD_VALUES, &cl, 1); tt_int_op(rv, OP_EQ, 0); - rv = config_assign(&options_format, result->def_opt, cl, 0, &msg); + rv = config_assign(get_options_mgr(), result->def_opt, cl, 0, &msg); if (msg) { /* Display the parse error message by comparing it with an empty string */ tt_str_op(msg, OP_EQ, ""); @@ -1168,7 +1175,7 @@ test_options_validate__transproxy(void *ignored) // Assert that a test has run for some TransProxyType tt_assert(tdata); -#else /* !(defined(USE_TRANSPARENT)) */ +#else /* !defined(USE_TRANSPARENT) */ tdata = get_options_test_data("TransPort 127.0.0.1:555\n"); ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg); @@ -1343,29 +1350,6 @@ test_options_validate__token_bucket(void *ignored) } static void -test_options_validate__recommended_packages(void *ignored) -{ - (void)ignored; - int ret; - char *msg; - setup_capture_of_logs(LOG_WARN); - options_test_data_t *tdata = get_options_test_data( - "RecommendedPackages foo 1.2 http://foo.com sha1=123123123123\n" - "RecommendedPackages invalid-package-line\n"); - - ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg); - tt_int_op(ret, OP_EQ, -1); - expect_no_log_msg("Invalid RecommendedPackage line " - "invalid-package-line will be ignored\n"); - - done: - escaped(NULL); // This will free the leaking memory from the previous escaped - teardown_capture_of_logs(); - free_options_test_data(tdata); - tor_free(msg); -} - -static void test_options_validate__fetch_dir(void *ignored) { (void)ignored; @@ -4200,7 +4184,6 @@ struct testcase_t options_tests[] = { LOCAL_VALIDATE_TEST(exclude_nodes), LOCAL_VALIDATE_TEST(node_families), LOCAL_VALIDATE_TEST(token_bucket), - LOCAL_VALIDATE_TEST(recommended_packages), LOCAL_VALIDATE_TEST(fetch_dir), LOCAL_VALIDATE_TEST(conn_limit), LOCAL_VALIDATE_TEST(paths_needed), diff --git a/src/test/test_parseconf.sh b/src/test/test_parseconf.sh new file mode 100755 index 0000000000..eeb80cdfa7 --- /dev/null +++ b/src/test/test_parseconf.sh @@ -0,0 +1,204 @@ +#!/bin/sh +# Copyright 2019, The Tor Project, Inc. +# See LICENSE for licensing information + +# Integration test script for verifying that Tor configurations are parsed as +# we expect. +# +# Valid configurations are tested with --dump-config, which parses and +# validates the configuration before writing it out. We then make sure that +# the result is what we expect, before parsing and dumping it again to make +# sure that there is no change. +# +# Invalid configurations are tested with --verify-config, which parses +# and validates the configuration. We capture its output and make sure that +# it contains the error message we expect. + +# This script looks for its test cases as individual directories in +# src/test/conf_examples/. Each test may have these files: +# +# torrc -- Usually needed. This file is passed to Tor on the command line +# with the "-f" flag. (If you omit it, you'll test Tor's behavior when +# it receives a nonexistent configuration file.) +# +# torrc.defaults -- Optional. If present, it is passed to Tor on the command +# line with the --defaults-torrc option. If this file is absent, an empty +# file is passed instead to prevent Tor from reading the system defaults. +# +# cmdline -- Optional. If present, it contains command-line arguments that +# will be passed to Tor. +# +# expected -- If this file is present, then it should be the expected result +# of "--dump-config short" for this test case. Exactly one of +# "expected" or "error" must be present, or the test will fail. +# +# error -- If this file is present, then it contains a regex that must be +# matched by some line in the output of "--verify-config", which must +# fail. Exactly one of "expected" or "error" must be present, or the +# test will fail. + +umask 077 +set -e + +# emulate realpath(), in case coreutils or equivalent is not installed. +abspath() { + f="$*" + if [ -d "$f" ]; then + dir="$f" + base="" + else + dir="$(dirname "$f")" + base="/$(basename "$f")" + fi + dir="$(cd "$dir" && pwd)" + echo "$dir$base" +} + +# find the tor binary +if [ $# -ge 1 ]; then + TOR_BINARY="${1}" + shift +else + TOR_BINARY="${TESTING_TOR_BINARY:-./src/app/tor}" +fi + +TOR_BINARY="$(abspath "$TOR_BINARY")" + +# make a safe space for temporary files +DATA_DIR=$(mktemp -d -t tor_parseconf_tests.XXXXXX) +trap 'rm -rf "$DATA_DIR"' 0 + +# This is where we look for examples +EXAMPLEDIR="$(dirname "$0")"/conf_examples + +case "$(uname -s)" in + CYGWIN*) WINDOWS=1;; + MINGW*) WINDOWS=1;; + MSYS*) WINDOWS=1;; + *) WINDOWS=0;; +esac + +#### +# BUG WORKAROUND FOR 31757: +# On Appveyor, it seems that Tor sometimes randomly fails to produce +# output with --dump-config. Whil we are figuring this out, do not treat +# windows errors as hard failures. +#### +if test "$WINDOWS" = 1; then + EXITCODE=0 +else + EXITCODE=1 +fi + +die() { echo "$1" >&2 ; exit "$EXITCODE"; } + +if test "$WINDOWS" = 1; then + FILTER="dos2unix" +else + FILTER="cat" +fi + +touch "${DATA_DIR}/EMPTY" || die "Couldn't create empty file." + +for dir in "${EXAMPLEDIR}"/*; do + if ! test -d "${dir}"; then + # Only count directories. + continue + fi + + testname="$(basename "${dir}")" + # We use printf since "echo -n" is not standard + printf "%s: " "$testname" + + PREV_DIR="$(pwd)" + cd "${dir}" + + if test -f "./torrc.defaults"; then + DEFAULTS="./torrc.defaults" + else + DEFAULTS="${DATA_DIR}/EMPTY" + fi + + if test -f "./cmdline"; then + CMDLINE="$(cat ./cmdline)" + else + CMDLINE="" + fi + + if test -f "./expected"; then + if test -f "./error"; then + echo "FAIL: Found both ${dir}/expected and ${dir}/error." + echo "(Only one of these files should exist.)" + exit $EXITCODE + fi + + # This case should succeed: run dump-config and see if it does. + + "${TOR_BINARY}" -f "./torrc" \ + --defaults-torrc "${DEFAULTS}" \ + --dump-config short \ + ${CMDLINE} \ + | "${FILTER}" > "${DATA_DIR}/output.${testname}" \ + || die "Failure: Tor exited." + + if cmp "./expected" "${DATA_DIR}/output.${testname}">/dev/null ; then + # Check round-trip. + "${TOR_BINARY}" -f "${DATA_DIR}/output.${testname}" \ + --defaults-torrc "${DATA_DIR}/empty" \ + --dump-config short \ + | "${FILTER}" \ + > "${DATA_DIR}/output_2.${testname}" \ + || die "Failure: Tor exited on round-trip." + + if ! cmp "${DATA_DIR}/output.${testname}" \ + "${DATA_DIR}/output_2.${testname}"; then + echo "Failure: did not match on round-trip." + exit $EXITCODE + fi + + echo "OK" + else + echo "FAIL" + if test "$(wc -c < "${DATA_DIR}/output.${testname}")" = 0; then + # There was no output -- probably we failed. + "${TOR_BINARY}" -f "./torrc" \ + --defaults-torrc "${DEFAULTS}" \ + --verify-config \ + ${CMDLINE} || true + fi + diff -u "./expected" "${DATA_DIR}/output.${testname}" || /bin/true + exit $EXITCODE + fi + + elif test -f "./error"; then + # This case should fail: run verify-config and see if it does. + + "${TOR_BINARY}" --verify-config \ + -f ./torrc \ + --defaults-torrc "${DEFAULTS}" \ + ${CMDLINE} \ + > "${DATA_DIR}/output.${testname}" \ + && die "Failure: Tor did not report an error." + + expect_err="$(cat ./error)" + if grep "${expect_err}" "${DATA_DIR}/output.${testname}" >/dev/null; then + echo "OK" + else + echo "FAIL" + echo "Expected error: ${expect_err}" + echo "Tor said:" + cat "${DATA_DIR}/output.${testname}" + exit $EXITCODE + fi + + else + # This case is not actually configured with a success or a failure. + # call that an error. + + echo "FAIL: Did not find ${dir}/expected or ${dir}/error." + exit $EXITCODE + fi + + cd "${PREV_DIR}" + +done diff --git a/src/test/test_process_descs.c b/src/test/test_process_descs.c new file mode 100644 index 0000000000..7dc9abde31 --- /dev/null +++ b/src/test/test_process_descs.c @@ -0,0 +1,67 @@ +/* Copyright (c) 2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" + +#include "core/or/or.h" +#include "feature/dirauth/process_descs.h" + +#include "test/test.h" + +static void +test_process_descs_versions(void *arg) +{ + (void)arg; + struct { + const char *version; + bool should_reject; + } cases[] = { + // a very old version: reject. + { "Tor 0.1.2.3-alpha", true }, + // a non-tor program: don't reject. + { "Wombat 0.1.2.3-alpha", false }, + // a slightly old version: reject + { "Tor 0.2.9.4-alpha", true }, + // a slightly old version: just new enough to support. + { "Tor 0.2.9.5-alpha", false }, + // a newer 0.2.9 version: supported. + { "Tor 0.2.9.100", false }, + // some unsupported versions: reject. + { "Tor 0.3.0.0-alpha-dev", true }, + { "Tor 0.3.0.2-alpha", true }, + { "Tor 0.3.0.5", true }, + { "Tor 0.3.1.4", true }, + { "Tor 0.3.2.4", true }, + { "Tor 0.3.3.4", true }, + { "Tor 0.3.4.1-alpha", true }, + { "Tor 0.3.4.100", true }, + { "Tor 0.3.5.1-alpha", true }, + { "Tor 0.3.5.6-rc", true}, + // new enough to be supported + { "Tor 0.3.5.7", false }, + { "Tor 0.3.5.8", false }, + { "Tor 0.4.0.1-alpha", false }, + { "Tor 0.4.1.5", false }, + // Very far in the future + { "Tor 100.100.1.5", false }, + }; + size_t n_cases = ARRAY_LENGTH(cases); + + for (unsigned i = 0; i < n_cases; ++i) { + const char *msg = NULL; + bool rejected = dirserv_rejects_tor_version(cases[i].version, &msg); + tt_int_op(rejected, OP_EQ, cases[i].should_reject); + tt_int_op(msg == NULL, OP_EQ, rejected == false); + } + + done: + ; +} + +#define T(name,flags) \ + { #name, test_process_descs_##name, (flags), NULL, NULL } + +struct testcase_t process_descs_tests[] = { + T(versions,0), + END_OF_TESTCASES +}; diff --git a/src/test/test_protover.c b/src/test/test_protover.c index 1759aef97d..0254501165 100644 --- a/src/test/test_protover.c +++ b/src/test/test_protover.c @@ -22,7 +22,7 @@ test_protover_parse(void *arg) tt_skip(); done: ; -#else /* !(defined(HAVE_RUST)) */ +#else /* !defined(HAVE_RUST) */ char *re_encoded = NULL; const char *orig = "Foo=1,3 Bar=3 Baz= Quux=9-12,14,15-16,900"; diff --git a/src/test/test_pt.c b/src/test/test_pt.c index 87e3ba356c..8f3ce03c42 100644 --- a/src/test/test_pt.c +++ b/src/test/test_pt.c @@ -11,7 +11,7 @@ #define PROCESS_PRIVATE #include "core/or/or.h" #include "app/config/config.h" -#include "app/config/confparse.h" +#include "lib/confmgt/confparse.h" #include "feature/control/control.h" #include "feature/control/control_events.h" #include "feature/client/transports.h" @@ -352,7 +352,7 @@ test_pt_configure_proxy(void *arg) managed_proxy_t *mp = NULL; (void) arg; - dummy_state = tor_malloc_zero(sizeof(or_state_t)); + dummy_state = or_state_new(); MOCK(process_read_stdout, process_read_stdout_replacement); MOCK(get_or_state, diff --git a/src/test/test_pubsub_build.c b/src/test/test_pubsub_build.c index ce5bf60080..021323fbf1 100644 --- a/src/test/test_pubsub_build.c +++ b/src/test/test_pubsub_build.c @@ -493,48 +493,6 @@ test_pubsub_build_sub_many(void *arg) tor_free(sysname); } -/* The same subsystem can only declare one publish or subscribe. */ -static void -test_pubsub_build_pubsub_redundant(void *arg) -{ - (void)arg; - pubsub_builder_t *b = NULL; - dispatch_t *dispatcher = NULL; - pubsub_connector_t *c = NULL; - - b = pubsub_builder_new(); - seed_pubsub_builder_basic(b); - pub_binding_t btmp; - - { - c = pubsub_connector_for_subsystem(b, get_subsys_id("sys2")); - DISPATCH_ADD_SUB(c, main, bunch_of_coconuts); - pubsub_add_pub_(c, &btmp, get_channel_id("main"), - get_message_id("yes_we_have_no"), - get_msg_type_id("string"), - 0 /* flags */, - "somewhere.c", 22); - pubsub_connector_free(c); - }; - - setup_full_capture_of_logs(LOG_WARN); - dispatcher = pubsub_builder_finalize(b, NULL); - b = NULL; - tt_assert(dispatcher == NULL); - - expect_log_msg_containing( - "Message \"yes_we_have_no\" is configured to be published by " - "subsystem \"sys2\" more than once."); - expect_log_msg_containing( - "Message \"bunch_of_coconuts\" is configured to be subscribed by " - "subsystem \"sys2\" more than once."); - - done: - pubsub_builder_free(b); - dispatch_free(dispatcher); - teardown_capture_of_logs(); -} - /* It's fine to declare the excl flag. */ static void test_pubsub_build_excl_ok(void *arg) @@ -614,7 +572,6 @@ struct testcase_t pubsub_build_tests[] = { T(pubsub_same, TT_FORK), T(pubsub_multi, TT_FORK), T(sub_many, TT_FORK), - T(pubsub_redundant, TT_FORK), T(excl_ok, TT_FORK), T(excl_bad, TT_FORK), END_OF_TESTCASES diff --git a/src/test/test_rebind.sh b/src/test/test_rebind.sh index e0d8394d38..d6d9d86668 100755 --- a/src/test/test_rebind.sh +++ b/src/test/test_rebind.sh @@ -12,8 +12,6 @@ if test "$UNAME_OS" = 'CYGWIN' || \ fi fi -exitcode=0 - tmpdir= clean () { if [ -n "$tmpdir" ] && [ -d "$tmpdir" ]; then diff --git a/src/test/test_switch_id.c b/src/test/test_switch_id.c index baddf8d66e..19483713f7 100644 --- a/src/test/test_switch_id.c +++ b/src/test/test_switch_id.c @@ -87,7 +87,7 @@ main(int argc, char **argv) fprintf(stderr, "This test is not supported on your OS.\n"); return 77; -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ const char *username; const char *testname; if (argc != 3) { diff --git a/src/test/test_token_bucket.c b/src/test/test_token_bucket.c new file mode 100644 index 0000000000..31670718d9 --- /dev/null +++ b/src/test/test_token_bucket.c @@ -0,0 +1,152 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file test_bwmgt.c + * \brief tests for bandwidth management / token bucket functions + */ + +#define TOKEN_BUCKET_PRIVATE + +#include "core/or/or.h" +#include "test/test.h" + +#include "lib/evloop/token_bucket.h" + +// an imaginary time, in timestamp units. Chosen so it will roll over. +static const uint32_t START_TS = UINT32_MAX - 1000; +static const uint32_t RATE = 10; +static const uint32_t BURST = 50; + +static void +test_token_bucket_ctr_init(void *arg) +{ + (void) arg; + token_bucket_ctr_t tb; + + token_bucket_ctr_init(&tb, RATE, BURST, START_TS); + tt_uint_op(tb.cfg.rate, OP_EQ, RATE); + tt_uint_op(tb.cfg.burst, OP_EQ, BURST); + tt_uint_op(tb.last_refilled_at_timestamp, OP_EQ, START_TS); + tt_int_op(tb.counter.bucket, OP_EQ, BURST); + + done: + ; +} + +static void +test_token_bucket_ctr_adjust(void *arg) +{ + (void) arg; + token_bucket_ctr_t tb; + + token_bucket_ctr_init(&tb, RATE, BURST, START_TS); + + /* Increase burst. */ + token_bucket_ctr_adjust(&tb, RATE, BURST * 2); + tt_uint_op(tb.cfg.rate, OP_EQ, RATE); + tt_uint_op(tb.counter.bucket, OP_EQ, BURST); + tt_uint_op(tb.cfg.burst, OP_EQ, BURST * 2); + + /* Decrease burst but still above bucket value. */ + token_bucket_ctr_adjust(&tb, RATE, BURST + 10); + tt_uint_op(tb.cfg.rate, OP_EQ, RATE); + tt_uint_op(tb.counter.bucket, OP_EQ, BURST); + tt_uint_op(tb.cfg.burst, OP_EQ, BURST + 10); + + /* Decrease burst below bucket value. */ + token_bucket_ctr_adjust(&tb, RATE, BURST - 1); + tt_uint_op(tb.cfg.rate, OP_EQ, RATE); + tt_uint_op(tb.counter.bucket, OP_EQ, BURST - 1); + tt_uint_op(tb.cfg.burst, OP_EQ, BURST - 1); + + /* Change rate. */ + token_bucket_ctr_adjust(&tb, RATE * 2, BURST); + tt_uint_op(tb.cfg.rate, OP_EQ, RATE * 2); + tt_uint_op(tb.counter.bucket, OP_EQ, BURST - 1); + tt_uint_op(tb.cfg.burst, OP_EQ, BURST); + + done: + ; +} + +static void +test_token_bucket_ctr_dec(void *arg) +{ + (void) arg; + token_bucket_ctr_t tb; + + token_bucket_ctr_init(&tb, RATE, BURST, START_TS); + + /* Simple decrement by one. */ + tt_uint_op(0, OP_EQ, token_bucket_ctr_dec(&tb, 1)); + tt_uint_op(tb.counter.bucket, OP_EQ, BURST - 1); + + /* Down to 0. Becomes empty. */ + tt_uint_op(true, OP_EQ, token_bucket_ctr_dec(&tb, BURST - 1)); + tt_uint_op(tb.counter.bucket, OP_EQ, 0); + + /* Reset and try to underflow. */ + token_bucket_ctr_init(&tb, RATE, BURST, START_TS); + tt_uint_op(true, OP_EQ, token_bucket_ctr_dec(&tb, BURST + 1)); + tt_int_op(tb.counter.bucket, OP_EQ, -1); + + /* Keep underflowing shouldn't flag the bucket as empty. */ + tt_uint_op(false, OP_EQ, token_bucket_ctr_dec(&tb, BURST)); + tt_int_op(tb.counter.bucket, OP_EQ, - (int32_t) (BURST + 1)); + + done: + ; +} + +static void +test_token_bucket_ctr_refill(void *arg) +{ + (void) arg; + token_bucket_ctr_t tb; + + token_bucket_ctr_init(&tb, RATE, BURST, START_TS); + + /* Reduce of half the bucket and let a single second go before refill. */ + token_bucket_ctr_dec(&tb, BURST / 2); + tt_int_op(tb.counter.bucket, OP_EQ, BURST / 2); + token_bucket_ctr_refill(&tb, START_TS + 1); + tt_int_op(tb.counter.bucket, OP_EQ, (BURST / 2) + RATE); + tt_int_op(tb.last_refilled_at_timestamp, OP_EQ, START_TS + 1); + + /* No time change, nothing should move. */ + token_bucket_ctr_refill(&tb, START_TS + 1); + tt_int_op(tb.counter.bucket, OP_EQ, (BURST / 2) + RATE); + tt_int_op(tb.last_refilled_at_timestamp, OP_EQ, START_TS + 1); + + /* Add 99 seconds, bucket should be back to a full BURST. */ + token_bucket_ctr_refill(&tb, START_TS + 99); + tt_int_op(tb.counter.bucket, OP_EQ, BURST); + tt_int_op(tb.last_refilled_at_timestamp, OP_EQ, START_TS + 99); + + /* Empty bucket at once. */ + token_bucket_ctr_dec(&tb, BURST); + tt_int_op(tb.counter.bucket, OP_EQ, 0); + /* On second passes. */ + token_bucket_ctr_refill(&tb, START_TS + 100); + tt_int_op(tb.last_refilled_at_timestamp, OP_EQ, START_TS + 100); + tt_int_op(tb.counter.bucket, OP_EQ, RATE); + /* A second second passes. */ + token_bucket_ctr_refill(&tb, START_TS + 101); + tt_int_op(tb.last_refilled_at_timestamp, OP_EQ, START_TS + 101); + tt_int_op(tb.counter.bucket, OP_EQ, RATE * 2); + + done: + ; +} + +#define TOKEN_BUCKET(name) \ + { #name, test_token_bucket_ ## name , 0, NULL, NULL } + +struct testcase_t token_bucket_tests[] = { + TOKEN_BUCKET(ctr_init), + TOKEN_BUCKET(ctr_adjust), + TOKEN_BUCKET(ctr_dec), + TOKEN_BUCKET(ctr_refill), + END_OF_TESTCASES +}; diff --git a/src/test/test_util.c b/src/test/test_util.c index b4d8a4d767..aebefe64c5 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -5399,6 +5399,13 @@ test_util_socketpair(void *arg) tt_skip(); } #endif /* defined(__FreeBSD__) */ +#ifdef ENETUNREACH + if (ersatz && socketpair_result == -ENETUNREACH) { + /* We can also fail with -ENETUNREACH if we have no network stack at + * all. */ + tt_skip(); + } +#endif /* defined(ENETUNREACH) */ tt_int_op(0, OP_EQ, socketpair_result); tt_assert(SOCKET_OK(fds[0])); @@ -6116,7 +6123,7 @@ test_util_log_mallinfo(void *arg) } else { tt_u64_op(mem1, OP_LT, mem2); } -#else /* !(defined(HAVE_MALLINFO)) */ +#else /* !defined(HAVE_MALLINFO) */ tt_skip(); #endif /* defined(HAVE_MALLINFO) */ done: @@ -6175,7 +6182,7 @@ test_util_map_anon_nofork(void *arg) tt_skip(); done: ; -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ /* We have the right OS support. We're going to try marking the buffer as * either zero-on-fork or as drop-on-fork, whichever is supported. Then we * will fork and send a byte back to the parent process. This will either diff --git a/src/test/test_workqueue.c b/src/test/test_workqueue.c index c58634da5c..ba478a45a4 100644 --- a/src/test/test_workqueue.c +++ b/src/test/test_workqueue.c @@ -63,7 +63,7 @@ mark_handled(int serial) tor_assert(! bitarray_is_set(handled, serial)); bitarray_set(handled, serial); tor_mutex_release(&bitmap_mutex); -#else /* !(defined(TRACK_RESPONSES)) */ +#else /* !defined(TRACK_RESPONSES) */ (void)serial; #endif /* defined(TRACK_RESPONSES) */ } diff --git a/src/test/testing_common.c b/src/test/testing_common.c index ad22898ce5..ff6028ddb4 100644 --- a/src/test/testing_common.c +++ b/src/test/testing_common.c @@ -89,7 +89,7 @@ setup_directory(void) (int)getpid(), rnd32); r = mkdir(temp_dir); } -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ tor_snprintf(temp_dir, sizeof(temp_dir), "/tmp/tor_test_%d_%s", (int) getpid(), rnd32); r = mkdir(temp_dir, 0700); @@ -295,7 +295,7 @@ main(int c, const char **v) memset(&s, 0, sizeof(s)); set_log_severity_config(loglevel, LOG_ERR, &s); /* ALWAYS log bug warnings. */ - s.masks[LOG_WARN-LOG_ERR] |= LD_BUG; + s.masks[SEVERITY_MASK_IDX(LOG_WARN)] |= LD_BUG; add_stream_log(&s, "", fileno(stdout)); } { @@ -303,7 +303,7 @@ main(int c, const char **v) log_severity_list_t s; memset(&s, 0, sizeof(s)); set_log_severity_config(LOG_ERR, LOG_ERR, &s); - s.masks[LOG_WARN-LOG_ERR] |= LD_BUG; + s.masks[SEVERITY_MASK_IDX(LOG_WARN)] |= LD_BUG; add_callback_log(&s, log_callback_failure); } flush_log_messages_from_startup(); diff --git a/src/test/testing_rsakeys.c b/src/test/testing_rsakeys.c index 8ba6bf9fe4..727449e6eb 100644 --- a/src/test/testing_rsakeys.c +++ b/src/test/testing_rsakeys.c @@ -468,7 +468,7 @@ pk_generate_internal(int bits) *idxp += crypto_rand_int_range(1,3); *idxp %= n_pregen; return crypto_pk_dup_key(pregen_array[*idxp]); -#else /* !(defined(USE_PREGENERATED_RSA_KEYS)) */ +#else /* !defined(USE_PREGENERATED_RSA_KEYS) */ crypto_pk_t *result; int res; result = crypto_pk_new(); diff --git a/src/tools/tor-print-ed-signing-cert.c b/src/tools/tor-print-ed-signing-cert.c index 1f1a01ab5c..43a1d7bcbd 100644 --- a/src/tools/tor-print-ed-signing-cert.c +++ b/src/tools/tor-print-ed-signing-cert.c @@ -10,11 +10,13 @@ #include "lib/cc/torint.h" /* TOR_PRIdSZ */ #include "lib/crypt_ops/crypto_format.h" #include "lib/malloc/malloc.h" +#include "lib/encoding/time_fmt.h" int main(int argc, char **argv) { ed25519_cert_t *cert = NULL; + char rfc1123_buf[RFC1123_TIME_LEN+1] = ""; if (argc != 2) { fprintf(stderr, "Usage:\n"); @@ -59,6 +61,11 @@ main(int argc, char **argv) printf("Expires at: %s", ctime(&expires_at)); + format_rfc1123_time(rfc1123_buf, expires_at); + printf("RFC 1123 timestamp: %s\n", rfc1123_buf); + + printf("UNIX timestamp: %ld\n", (long int)expires_at); + ed25519_cert_free(cert); return 0; diff --git a/src/trunnel/channelpadding_negotiation.c b/src/trunnel/channelpadding_negotiation.c index 59e6b38384..d96496e90c 100644 --- a/src/trunnel/channelpadding_negotiation.c +++ b/src/trunnel/channelpadding_negotiation.c @@ -1,4 +1,4 @@ -/* channelpadding_negotiation.c -- generated by Trunnel v1.5.2. +/* channelpadding_negotiation.c -- generated by Trunnel v1.5.3. * https://gitweb.torproject.org/trunnel.git * You probably shouldn't edit this file. */ diff --git a/src/trunnel/channelpadding_negotiation.h b/src/trunnel/channelpadding_negotiation.h index fcfc232fea..3f96174f68 100644 --- a/src/trunnel/channelpadding_negotiation.h +++ b/src/trunnel/channelpadding_negotiation.h @@ -1,4 +1,4 @@ -/* channelpadding_negotiation.h -- generated by Trunnel v1.5.2. +/* channelpadding_negotiation.h -- generated by Trunnel v1.5.3. * https://gitweb.torproject.org/trunnel.git * You probably shouldn't edit this file. */ diff --git a/src/trunnel/circpad_negotiation.c b/src/trunnel/circpad_negotiation.c index 236be06ada..547818f2ec 100644 --- a/src/trunnel/circpad_negotiation.c +++ b/src/trunnel/circpad_negotiation.c @@ -1,4 +1,4 @@ -/* circpad_negotiation.c -- generated by Trunnel v1.5.2. +/* circpad_negotiation.c -- generated by Trunnel v1.5.3. * https://gitweb.torproject.org/trunnel.git * You probably shouldn't edit this file. */ diff --git a/src/trunnel/circpad_negotiation.h b/src/trunnel/circpad_negotiation.h index d09080dc16..ba9155019e 100644 --- a/src/trunnel/circpad_negotiation.h +++ b/src/trunnel/circpad_negotiation.h @@ -1,4 +1,4 @@ -/* circpad_negotiation.h -- generated by Trunnel v1.5.2. +/* circpad_negotiation.h -- generated by Trunnel v1.5.3. * https://gitweb.torproject.org/trunnel.git * You probably shouldn't edit this file. */ diff --git a/src/trunnel/ed25519_cert.c b/src/trunnel/ed25519_cert.c index 1276c7a505..86b79ef9b6 100644 --- a/src/trunnel/ed25519_cert.c +++ b/src/trunnel/ed25519_cert.c @@ -1,4 +1,4 @@ -/* ed25519_cert.c -- generated by Trunnel v1.5.2. +/* ed25519_cert.c -- generated by Trunnel v1.5.3. * https://gitweb.torproject.org/trunnel.git * You probably shouldn't edit this file. */ diff --git a/src/trunnel/ed25519_cert.h b/src/trunnel/ed25519_cert.h index e086c6fced..bd91ce1055 100644 --- a/src/trunnel/ed25519_cert.h +++ b/src/trunnel/ed25519_cert.h @@ -1,4 +1,4 @@ -/* ed25519_cert.h -- generated by Trunnel v1.5.2. +/* ed25519_cert.h -- generated by Trunnel v1.5.3. * https://gitweb.torproject.org/trunnel.git * You probably shouldn't edit this file. */ diff --git a/src/trunnel/hs/cell_common.c b/src/trunnel/hs/cell_common.c index af223560c1..1f50961d69 100644 --- a/src/trunnel/hs/cell_common.c +++ b/src/trunnel/hs/cell_common.c @@ -1,4 +1,4 @@ -/* cell_common.c -- generated by Trunnel v1.5.2. +/* cell_common.c -- generated by Trunnel v1.5.3. * https://gitweb.torproject.org/trunnel.git * You probably shouldn't edit this file. */ @@ -28,10 +28,10 @@ int cellcommon_deadcode_dummy__ = 0; } \ } while (0) -trn_cell_extension_fields_t * -trn_cell_extension_fields_new(void) +trn_cell_extension_field_t * +trn_cell_extension_field_new(void) { - trn_cell_extension_fields_t *val = trunnel_calloc(1, sizeof(trn_cell_extension_fields_t)); + trn_cell_extension_field_t *val = trunnel_calloc(1, sizeof(trn_cell_extension_field_t)); if (NULL == val) return NULL; return val; @@ -40,7 +40,7 @@ trn_cell_extension_fields_new(void) /** Release all storage held inside 'obj', but do not free 'obj'. */ static void -trn_cell_extension_fields_clear(trn_cell_extension_fields_t *obj) +trn_cell_extension_field_clear(trn_cell_extension_field_t *obj) { (void) obj; TRUNNEL_DYNARRAY_WIPE(&obj->field); @@ -48,62 +48,62 @@ trn_cell_extension_fields_clear(trn_cell_extension_fields_t *obj) } void -trn_cell_extension_fields_free(trn_cell_extension_fields_t *obj) +trn_cell_extension_field_free(trn_cell_extension_field_t *obj) { if (obj == NULL) return; - trn_cell_extension_fields_clear(obj); - trunnel_memwipe(obj, sizeof(trn_cell_extension_fields_t)); + trn_cell_extension_field_clear(obj); + trunnel_memwipe(obj, sizeof(trn_cell_extension_field_t)); trunnel_free_(obj); } uint8_t -trn_cell_extension_fields_get_field_type(const trn_cell_extension_fields_t *inp) +trn_cell_extension_field_get_field_type(const trn_cell_extension_field_t *inp) { return inp->field_type; } int -trn_cell_extension_fields_set_field_type(trn_cell_extension_fields_t *inp, uint8_t val) +trn_cell_extension_field_set_field_type(trn_cell_extension_field_t *inp, uint8_t val) { inp->field_type = val; return 0; } uint8_t -trn_cell_extension_fields_get_field_len(const trn_cell_extension_fields_t *inp) +trn_cell_extension_field_get_field_len(const trn_cell_extension_field_t *inp) { return inp->field_len; } int -trn_cell_extension_fields_set_field_len(trn_cell_extension_fields_t *inp, uint8_t val) +trn_cell_extension_field_set_field_len(trn_cell_extension_field_t *inp, uint8_t val) { inp->field_len = val; return 0; } size_t -trn_cell_extension_fields_getlen_field(const trn_cell_extension_fields_t *inp) +trn_cell_extension_field_getlen_field(const trn_cell_extension_field_t *inp) { return TRUNNEL_DYNARRAY_LEN(&inp->field); } uint8_t -trn_cell_extension_fields_get_field(trn_cell_extension_fields_t *inp, size_t idx) +trn_cell_extension_field_get_field(trn_cell_extension_field_t *inp, size_t idx) { return TRUNNEL_DYNARRAY_GET(&inp->field, idx); } uint8_t -trn_cell_extension_fields_getconst_field(const trn_cell_extension_fields_t *inp, size_t idx) +trn_cell_extension_field_getconst_field(const trn_cell_extension_field_t *inp, size_t idx) { - return trn_cell_extension_fields_get_field((trn_cell_extension_fields_t*)inp, idx); + return trn_cell_extension_field_get_field((trn_cell_extension_field_t*)inp, idx); } int -trn_cell_extension_fields_set_field(trn_cell_extension_fields_t *inp, size_t idx, uint8_t elt) +trn_cell_extension_field_set_field(trn_cell_extension_field_t *inp, size_t idx, uint8_t elt) { TRUNNEL_DYNARRAY_SET(&inp->field, idx, elt); return 0; } int -trn_cell_extension_fields_add_field(trn_cell_extension_fields_t *inp, uint8_t elt) +trn_cell_extension_field_add_field(trn_cell_extension_field_t *inp, uint8_t elt) { #if SIZE_MAX >= UINT8_MAX if (inp->field.n_ == UINT8_MAX) @@ -117,17 +117,17 @@ trn_cell_extension_fields_add_field(trn_cell_extension_fields_t *inp, uint8_t el } uint8_t * -trn_cell_extension_fields_getarray_field(trn_cell_extension_fields_t *inp) +trn_cell_extension_field_getarray_field(trn_cell_extension_field_t *inp) { return inp->field.elts_; } const uint8_t * -trn_cell_extension_fields_getconstarray_field(const trn_cell_extension_fields_t *inp) +trn_cell_extension_field_getconstarray_field(const trn_cell_extension_field_t *inp) { - return (const uint8_t *)trn_cell_extension_fields_getarray_field((trn_cell_extension_fields_t*)inp); + return (const uint8_t *)trn_cell_extension_field_getarray_field((trn_cell_extension_field_t*)inp); } int -trn_cell_extension_fields_setlen_field(trn_cell_extension_fields_t *inp, size_t newlen) +trn_cell_extension_field_setlen_field(trn_cell_extension_field_t *inp, size_t newlen) { uint8_t *newptr; #if UINT8_MAX < SIZE_MAX @@ -147,7 +147,7 @@ trn_cell_extension_fields_setlen_field(trn_cell_extension_fields_t *inp, size_t return -1; } const char * -trn_cell_extension_fields_check(const trn_cell_extension_fields_t *obj) +trn_cell_extension_field_check(const trn_cell_extension_field_t *obj) { if (obj == NULL) return "Object was NULL"; @@ -159,11 +159,11 @@ trn_cell_extension_fields_check(const trn_cell_extension_fields_t *obj) } ssize_t -trn_cell_extension_fields_encoded_len(const trn_cell_extension_fields_t *obj) +trn_cell_extension_field_encoded_len(const trn_cell_extension_field_t *obj) { ssize_t result = 0; - if (NULL != trn_cell_extension_fields_check(obj)) + if (NULL != trn_cell_extension_field_check(obj)) return -1; @@ -178,24 +178,24 @@ trn_cell_extension_fields_encoded_len(const trn_cell_extension_fields_t *obj) return result; } int -trn_cell_extension_fields_clear_errors(trn_cell_extension_fields_t *obj) +trn_cell_extension_field_clear_errors(trn_cell_extension_field_t *obj) { int r = obj->trunnel_error_code_; obj->trunnel_error_code_ = 0; return r; } ssize_t -trn_cell_extension_fields_encode(uint8_t *output, const size_t avail, const trn_cell_extension_fields_t *obj) +trn_cell_extension_field_encode(uint8_t *output, const size_t avail, const trn_cell_extension_field_t *obj) { ssize_t result = 0; size_t written = 0; uint8_t *ptr = output; const char *msg; #ifdef TRUNNEL_CHECK_ENCODED_LEN - const ssize_t encoded_len = trn_cell_extension_fields_encoded_len(obj); + const ssize_t encoded_len = trn_cell_extension_field_encoded_len(obj); #endif - if (NULL != (msg = trn_cell_extension_fields_check(obj))) + if (NULL != (msg = trn_cell_extension_field_check(obj))) goto check_failed; #ifdef TRUNNEL_CHECK_ENCODED_LEN @@ -252,11 +252,11 @@ trn_cell_extension_fields_encode(uint8_t *output, const size_t avail, const trn_ return result; } -/** As trn_cell_extension_fields_parse(), but do not allocate the +/** As trn_cell_extension_field_parse(), but do not allocate the * output object. */ static ssize_t -trn_cell_extension_fields_parse_into(trn_cell_extension_fields_t *obj, const uint8_t *input, const size_t len_in) +trn_cell_extension_field_parse_into(trn_cell_extension_field_t *obj, const uint8_t *input, const size_t len_in) { const uint8_t *ptr = input; size_t remaining = len_in; @@ -290,15 +290,15 @@ trn_cell_extension_fields_parse_into(trn_cell_extension_fields_t *obj, const uin } ssize_t -trn_cell_extension_fields_parse(trn_cell_extension_fields_t **output, const uint8_t *input, const size_t len_in) +trn_cell_extension_field_parse(trn_cell_extension_field_t **output, const uint8_t *input, const size_t len_in) { ssize_t result; - *output = trn_cell_extension_fields_new(); + *output = trn_cell_extension_field_new(); if (NULL == *output) return -1; - result = trn_cell_extension_fields_parse_into(*output, input, len_in); + result = trn_cell_extension_field_parse_into(*output, input, len_in); if (result < 0) { - trn_cell_extension_fields_free(*output); + trn_cell_extension_field_free(*output); *output = NULL; } return result; @@ -322,7 +322,7 @@ trn_cell_extension_clear(trn_cell_extension_t *obj) unsigned idx; for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->fields); ++idx) { - trn_cell_extension_fields_free(TRUNNEL_DYNARRAY_GET(&obj->fields, idx)); + trn_cell_extension_field_free(TRUNNEL_DYNARRAY_GET(&obj->fields, idx)); } } TRUNNEL_DYNARRAY_WIPE(&obj->fields); @@ -356,66 +356,66 @@ trn_cell_extension_getlen_fields(const trn_cell_extension_t *inp) return TRUNNEL_DYNARRAY_LEN(&inp->fields); } -struct trn_cell_extension_fields_st * +struct trn_cell_extension_field_st * trn_cell_extension_get_fields(trn_cell_extension_t *inp, size_t idx) { return TRUNNEL_DYNARRAY_GET(&inp->fields, idx); } - const struct trn_cell_extension_fields_st * + const struct trn_cell_extension_field_st * trn_cell_extension_getconst_fields(const trn_cell_extension_t *inp, size_t idx) { return trn_cell_extension_get_fields((trn_cell_extension_t*)inp, idx); } int -trn_cell_extension_set_fields(trn_cell_extension_t *inp, size_t idx, struct trn_cell_extension_fields_st * elt) +trn_cell_extension_set_fields(trn_cell_extension_t *inp, size_t idx, struct trn_cell_extension_field_st * elt) { - trn_cell_extension_fields_t *oldval = TRUNNEL_DYNARRAY_GET(&inp->fields, idx); + trn_cell_extension_field_t *oldval = TRUNNEL_DYNARRAY_GET(&inp->fields, idx); if (oldval && oldval != elt) - trn_cell_extension_fields_free(oldval); + trn_cell_extension_field_free(oldval); return trn_cell_extension_set0_fields(inp, idx, elt); } int -trn_cell_extension_set0_fields(trn_cell_extension_t *inp, size_t idx, struct trn_cell_extension_fields_st * elt) +trn_cell_extension_set0_fields(trn_cell_extension_t *inp, size_t idx, struct trn_cell_extension_field_st * elt) { TRUNNEL_DYNARRAY_SET(&inp->fields, idx, elt); return 0; } int -trn_cell_extension_add_fields(trn_cell_extension_t *inp, struct trn_cell_extension_fields_st * elt) +trn_cell_extension_add_fields(trn_cell_extension_t *inp, struct trn_cell_extension_field_st * elt) { #if SIZE_MAX >= UINT8_MAX if (inp->fields.n_ == UINT8_MAX) goto trunnel_alloc_failed; #endif - TRUNNEL_DYNARRAY_ADD(struct trn_cell_extension_fields_st *, &inp->fields, elt, {}); + TRUNNEL_DYNARRAY_ADD(struct trn_cell_extension_field_st *, &inp->fields, elt, {}); return 0; trunnel_alloc_failed: TRUNNEL_SET_ERROR_CODE(inp); return -1; } -struct trn_cell_extension_fields_st * * +struct trn_cell_extension_field_st * * trn_cell_extension_getarray_fields(trn_cell_extension_t *inp) { return inp->fields.elts_; } -const struct trn_cell_extension_fields_st * const * +const struct trn_cell_extension_field_st * const * trn_cell_extension_getconstarray_fields(const trn_cell_extension_t *inp) { - return (const struct trn_cell_extension_fields_st * const *)trn_cell_extension_getarray_fields((trn_cell_extension_t*)inp); + return (const struct trn_cell_extension_field_st * const *)trn_cell_extension_getarray_fields((trn_cell_extension_t*)inp); } int trn_cell_extension_setlen_fields(trn_cell_extension_t *inp, size_t newlen) { - struct trn_cell_extension_fields_st * *newptr; + struct trn_cell_extension_field_st * *newptr; #if UINT8_MAX < SIZE_MAX if (newlen > UINT8_MAX) goto trunnel_alloc_failed; #endif newptr = trunnel_dynarray_setlen(&inp->fields.allocated_, &inp->fields.n_, inp->fields.elts_, newlen, - sizeof(inp->fields.elts_[0]), (trunnel_free_fn_t) trn_cell_extension_fields_free, + sizeof(inp->fields.elts_[0]), (trunnel_free_fn_t) trn_cell_extension_field_free, &inp->trunnel_error_code_); if (newlen != 0 && newptr == NULL) goto trunnel_alloc_failed; @@ -437,7 +437,7 @@ trn_cell_extension_check(const trn_cell_extension_t *obj) unsigned idx; for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->fields); ++idx) { - if (NULL != (msg = trn_cell_extension_fields_check(TRUNNEL_DYNARRAY_GET(&obj->fields, idx)))) + if (NULL != (msg = trn_cell_extension_field_check(TRUNNEL_DYNARRAY_GET(&obj->fields, idx)))) return msg; } } @@ -458,12 +458,12 @@ trn_cell_extension_encoded_len(const trn_cell_extension_t *obj) /* Length of u8 num */ result += 1; - /* Length of struct trn_cell_extension_fields fields[num] */ + /* Length of struct trn_cell_extension_field fields[num] */ { unsigned idx; for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->fields); ++idx) { - result += trn_cell_extension_fields_encoded_len(TRUNNEL_DYNARRAY_GET(&obj->fields, idx)); + result += trn_cell_extension_field_encoded_len(TRUNNEL_DYNARRAY_GET(&obj->fields, idx)); } } return result; @@ -500,13 +500,13 @@ trn_cell_extension_encode(uint8_t *output, const size_t avail, const trn_cell_ex trunnel_set_uint8(ptr, (obj->num)); written += 1; ptr += 1; - /* Encode struct trn_cell_extension_fields fields[num] */ + /* Encode struct trn_cell_extension_field fields[num] */ { unsigned idx; for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->fields); ++idx) { trunnel_assert(written <= avail); - result = trn_cell_extension_fields_encode(ptr, avail - written, TRUNNEL_DYNARRAY_GET(&obj->fields, idx)); + result = trn_cell_extension_field_encode(ptr, avail - written, TRUNNEL_DYNARRAY_GET(&obj->fields, idx)); if (result < 0) goto fail; /* XXXXXXX !*/ written += result; ptr += result; @@ -553,18 +553,18 @@ trn_cell_extension_parse_into(trn_cell_extension_t *obj, const uint8_t *input, c obj->num = (trunnel_get_uint8(ptr)); remaining -= 1; ptr += 1; - /* Parse struct trn_cell_extension_fields fields[num] */ - TRUNNEL_DYNARRAY_EXPAND(trn_cell_extension_fields_t *, &obj->fields, obj->num, {}); + /* Parse struct trn_cell_extension_field fields[num] */ + TRUNNEL_DYNARRAY_EXPAND(trn_cell_extension_field_t *, &obj->fields, obj->num, {}); { - trn_cell_extension_fields_t * elt; + trn_cell_extension_field_t * elt; unsigned idx; for (idx = 0; idx < obj->num; ++idx) { - result = trn_cell_extension_fields_parse(&elt, ptr, remaining); + result = trn_cell_extension_field_parse(&elt, ptr, remaining); if (result < 0) goto relay_fail; trunnel_assert((size_t)result <= remaining); remaining -= result; ptr += result; - TRUNNEL_DYNARRAY_ADD(trn_cell_extension_fields_t *, &obj->fields, elt, {trn_cell_extension_fields_free(elt);}); + TRUNNEL_DYNARRAY_ADD(trn_cell_extension_field_t *, &obj->fields, elt, {trn_cell_extension_field_free(elt);}); } } trunnel_assert(ptr + remaining == input + len_in); diff --git a/src/trunnel/hs/cell_common.h b/src/trunnel/hs/cell_common.h index e08eedfdb3..beb65e015f 100644 --- a/src/trunnel/hs/cell_common.h +++ b/src/trunnel/hs/cell_common.h @@ -1,4 +1,4 @@ -/* cell_common.h -- generated by Trunnel v1.5.2. +/* cell_common.h -- generated by Trunnel v1.5.3. * https://gitweb.torproject.org/trunnel.git * You probably shouldn't edit this file. */ @@ -8,112 +8,112 @@ #include <stdint.h> #include "trunnel.h" -#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_EXTENSION_FIELDS) -struct trn_cell_extension_fields_st { +#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_EXTENSION_FIELD) +struct trn_cell_extension_field_st { uint8_t field_type; uint8_t field_len; TRUNNEL_DYNARRAY_HEAD(, uint8_t) field; uint8_t trunnel_error_code_; }; #endif -typedef struct trn_cell_extension_fields_st trn_cell_extension_fields_t; +typedef struct trn_cell_extension_field_st trn_cell_extension_field_t; #if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_EXTENSION) struct trn_cell_extension_st { uint8_t num; - TRUNNEL_DYNARRAY_HEAD(, struct trn_cell_extension_fields_st *) fields; + TRUNNEL_DYNARRAY_HEAD(, struct trn_cell_extension_field_st *) fields; uint8_t trunnel_error_code_; }; #endif typedef struct trn_cell_extension_st trn_cell_extension_t; -/** Return a newly allocated trn_cell_extension_fields with all +/** Return a newly allocated trn_cell_extension_field with all * elements set to zero. */ -trn_cell_extension_fields_t *trn_cell_extension_fields_new(void); -/** Release all storage held by the trn_cell_extension_fields in +trn_cell_extension_field_t *trn_cell_extension_field_new(void); +/** Release all storage held by the trn_cell_extension_field in * 'victim'. (Do nothing if 'victim' is NULL.) */ -void trn_cell_extension_fields_free(trn_cell_extension_fields_t *victim); -/** Try to parse a trn_cell_extension_fields from the buffer in +void trn_cell_extension_field_free(trn_cell_extension_field_t *victim); +/** Try to parse a trn_cell_extension_field from the buffer in * 'input', using up to 'len_in' bytes from the input buffer. On * success, return the number of bytes consumed and set *output to the - * newly allocated trn_cell_extension_fields_t. On failure, return -2 + * newly allocated trn_cell_extension_field_t. On failure, return -2 * if the input appears truncated, and -1 if the input is otherwise * invalid. */ -ssize_t trn_cell_extension_fields_parse(trn_cell_extension_fields_t **output, const uint8_t *input, const size_t len_in); +ssize_t trn_cell_extension_field_parse(trn_cell_extension_field_t **output, const uint8_t *input, const size_t len_in); /** Return the number of bytes we expect to need to encode the - * trn_cell_extension_fields in 'obj'. On failure, return a negative + * trn_cell_extension_field in 'obj'. On failure, return a negative * value. Note that this value may be an overestimate, and can even be * an underestimate for certain unencodeable objects. */ -ssize_t trn_cell_extension_fields_encoded_len(const trn_cell_extension_fields_t *obj); -/** Try to encode the trn_cell_extension_fields from 'input' into the +ssize_t trn_cell_extension_field_encoded_len(const trn_cell_extension_field_t *obj); +/** Try to encode the trn_cell_extension_field from 'input' into the * buffer at 'output', using up to 'avail' bytes of the output buffer. * On success, return the number of bytes used. On failure, return -2 * if the buffer was not long enough, and -1 if the input was invalid. */ -ssize_t trn_cell_extension_fields_encode(uint8_t *output, size_t avail, const trn_cell_extension_fields_t *input); -/** Check whether the internal state of the trn_cell_extension_fields +ssize_t trn_cell_extension_field_encode(uint8_t *output, size_t avail, const trn_cell_extension_field_t *input); +/** Check whether the internal state of the trn_cell_extension_field * in 'obj' is consistent. Return NULL if it is, and a short message * if it is not. */ -const char *trn_cell_extension_fields_check(const trn_cell_extension_fields_t *obj); +const char *trn_cell_extension_field_check(const trn_cell_extension_field_t *obj); /** Clear any errors that were set on the object 'obj' by its setter * functions. Return true iff errors were cleared. */ -int trn_cell_extension_fields_clear_errors(trn_cell_extension_fields_t *obj); +int trn_cell_extension_field_clear_errors(trn_cell_extension_field_t *obj); /** Return the value of the field_type field of the - * trn_cell_extension_fields_t in 'inp' + * trn_cell_extension_field_t in 'inp' */ -uint8_t trn_cell_extension_fields_get_field_type(const trn_cell_extension_fields_t *inp); +uint8_t trn_cell_extension_field_get_field_type(const trn_cell_extension_field_t *inp); /** Set the value of the field_type field of the - * trn_cell_extension_fields_t in 'inp' to 'val'. Return 0 on success; + * trn_cell_extension_field_t in 'inp' to 'val'. Return 0 on success; * return -1 and set the error code on 'inp' on failure. */ -int trn_cell_extension_fields_set_field_type(trn_cell_extension_fields_t *inp, uint8_t val); +int trn_cell_extension_field_set_field_type(trn_cell_extension_field_t *inp, uint8_t val); /** Return the value of the field_len field of the - * trn_cell_extension_fields_t in 'inp' + * trn_cell_extension_field_t in 'inp' */ -uint8_t trn_cell_extension_fields_get_field_len(const trn_cell_extension_fields_t *inp); +uint8_t trn_cell_extension_field_get_field_len(const trn_cell_extension_field_t *inp); /** Set the value of the field_len field of the - * trn_cell_extension_fields_t in 'inp' to 'val'. Return 0 on success; + * trn_cell_extension_field_t in 'inp' to 'val'. Return 0 on success; * return -1 and set the error code on 'inp' on failure. */ -int trn_cell_extension_fields_set_field_len(trn_cell_extension_fields_t *inp, uint8_t val); +int trn_cell_extension_field_set_field_len(trn_cell_extension_field_t *inp, uint8_t val); /** Return the length of the dynamic array holding the field field of - * the trn_cell_extension_fields_t in 'inp'. + * the trn_cell_extension_field_t in 'inp'. */ -size_t trn_cell_extension_fields_getlen_field(const trn_cell_extension_fields_t *inp); +size_t trn_cell_extension_field_getlen_field(const trn_cell_extension_field_t *inp); /** Return the element at position 'idx' of the dynamic array field - * field of the trn_cell_extension_fields_t in 'inp'. + * field of the trn_cell_extension_field_t in 'inp'. */ -uint8_t trn_cell_extension_fields_get_field(trn_cell_extension_fields_t *inp, size_t idx); -/** As trn_cell_extension_fields_get_field, but take and return a - * const pointer +uint8_t trn_cell_extension_field_get_field(trn_cell_extension_field_t *inp, size_t idx); +/** As trn_cell_extension_field_get_field, but take and return a const + * pointer */ -uint8_t trn_cell_extension_fields_getconst_field(const trn_cell_extension_fields_t *inp, size_t idx); +uint8_t trn_cell_extension_field_getconst_field(const trn_cell_extension_field_t *inp, size_t idx); /** Change the element at position 'idx' of the dynamic array field - * field of the trn_cell_extension_fields_t in 'inp', so that it will + * field of the trn_cell_extension_field_t in 'inp', so that it will * hold the value 'elt'. */ -int trn_cell_extension_fields_set_field(trn_cell_extension_fields_t *inp, size_t idx, uint8_t elt); +int trn_cell_extension_field_set_field(trn_cell_extension_field_t *inp, size_t idx, uint8_t elt); /** Append a new element 'elt' to the dynamic array field field of the - * trn_cell_extension_fields_t in 'inp'. + * trn_cell_extension_field_t in 'inp'. */ -int trn_cell_extension_fields_add_field(trn_cell_extension_fields_t *inp, uint8_t elt); +int trn_cell_extension_field_add_field(trn_cell_extension_field_t *inp, uint8_t elt); /** Return a pointer to the variable-length array field field of * 'inp'. */ -uint8_t * trn_cell_extension_fields_getarray_field(trn_cell_extension_fields_t *inp); -/** As trn_cell_extension_fields_get_field, but take and return a - * const pointer +uint8_t * trn_cell_extension_field_getarray_field(trn_cell_extension_field_t *inp); +/** As trn_cell_extension_field_get_field, but take and return a const + * pointer */ -const uint8_t * trn_cell_extension_fields_getconstarray_field(const trn_cell_extension_fields_t *inp); +const uint8_t * trn_cell_extension_field_getconstarray_field(const trn_cell_extension_field_t *inp); /** Change the length of the variable-length array field field of * 'inp' to 'newlen'.Fill extra elements with 0. Return 0 on success; * return -1 and set the error code on 'inp' on failure. */ -int trn_cell_extension_fields_setlen_field(trn_cell_extension_fields_t *inp, size_t newlen); +int trn_cell_extension_field_setlen_field(trn_cell_extension_field_t *inp, size_t newlen); /** Return a newly allocated trn_cell_extension with all elements set * to zero. */ @@ -166,32 +166,32 @@ size_t trn_cell_extension_getlen_fields(const trn_cell_extension_t *inp); /** Return the element at position 'idx' of the dynamic array field * fields of the trn_cell_extension_t in 'inp'. */ -struct trn_cell_extension_fields_st * trn_cell_extension_get_fields(trn_cell_extension_t *inp, size_t idx); +struct trn_cell_extension_field_st * trn_cell_extension_get_fields(trn_cell_extension_t *inp, size_t idx); /** As trn_cell_extension_get_fields, but take and return a const * pointer */ - const struct trn_cell_extension_fields_st * trn_cell_extension_getconst_fields(const trn_cell_extension_t *inp, size_t idx); + const struct trn_cell_extension_field_st * trn_cell_extension_getconst_fields(const trn_cell_extension_t *inp, size_t idx); /** Change the element at position 'idx' of the dynamic array field * fields of the trn_cell_extension_t in 'inp', so that it will hold * the value 'elt'. Free the previous value, if any. */ -int trn_cell_extension_set_fields(trn_cell_extension_t *inp, size_t idx, struct trn_cell_extension_fields_st * elt); +int trn_cell_extension_set_fields(trn_cell_extension_t *inp, size_t idx, struct trn_cell_extension_field_st * elt); /** As trn_cell_extension_set_fields, but does not free the previous * value. */ -int trn_cell_extension_set0_fields(trn_cell_extension_t *inp, size_t idx, struct trn_cell_extension_fields_st * elt); +int trn_cell_extension_set0_fields(trn_cell_extension_t *inp, size_t idx, struct trn_cell_extension_field_st * elt); /** Append a new element 'elt' to the dynamic array field fields of * the trn_cell_extension_t in 'inp'. */ -int trn_cell_extension_add_fields(trn_cell_extension_t *inp, struct trn_cell_extension_fields_st * elt); +int trn_cell_extension_add_fields(trn_cell_extension_t *inp, struct trn_cell_extension_field_st * elt); /** Return a pointer to the variable-length array field fields of * 'inp'. */ -struct trn_cell_extension_fields_st * * trn_cell_extension_getarray_fields(trn_cell_extension_t *inp); +struct trn_cell_extension_field_st * * trn_cell_extension_getarray_fields(trn_cell_extension_t *inp); /** As trn_cell_extension_get_fields, but take and return a const * pointer */ -const struct trn_cell_extension_fields_st * const * trn_cell_extension_getconstarray_fields(const trn_cell_extension_t *inp); +const struct trn_cell_extension_field_st * const * trn_cell_extension_getconstarray_fields(const trn_cell_extension_t *inp); /** Change the length of the variable-length array field fields of * 'inp' to 'newlen'.Fill extra elements with NULL; free removed * elements. Return 0 on success; return -1 and set the error code on diff --git a/src/trunnel/hs/cell_common.trunnel b/src/trunnel/hs/cell_common.trunnel index 1aa6999de7..7e99cbfa66 100644 --- a/src/trunnel/hs/cell_common.trunnel +++ b/src/trunnel/hs/cell_common.trunnel @@ -1,6 +1,6 @@ /* This file contains common data structure that cells use. */ -struct trn_cell_extension_fields { +struct trn_cell_extension_field { u8 field_type; u8 field_len; u8 field[field_len]; @@ -8,5 +8,5 @@ struct trn_cell_extension_fields { struct trn_cell_extension { u8 num; - struct trn_cell_extension_fields fields[num]; + struct trn_cell_extension_field fields[num]; }; diff --git a/src/trunnel/hs/cell_establish_intro.c b/src/trunnel/hs/cell_establish_intro.c index ae3b7b1bc8..f31404c55f 100644 --- a/src/trunnel/hs/cell_establish_intro.c +++ b/src/trunnel/hs/cell_establish_intro.c @@ -1,4 +1,4 @@ -/* cell_establish_intro.c -- generated by Trunnel v1.5.2. +/* cell_establish_intro.c -- generated by Trunnel v1.5.3. * https://gitweb.torproject.org/trunnel.git * You probably shouldn't edit this file. */ @@ -36,6 +36,185 @@ ssize_t trn_cell_extension_encoded_len(const trn_cell_extension_t *obj); ssize_t trn_cell_extension_encode(uint8_t *output, size_t avail, const trn_cell_extension_t *input); const char *trn_cell_extension_check(const trn_cell_extension_t *obj); int trn_cell_extension_clear_errors(trn_cell_extension_t *obj); +trn_cell_extension_dos_param_t * +trn_cell_extension_dos_param_new(void) +{ + trn_cell_extension_dos_param_t *val = trunnel_calloc(1, sizeof(trn_cell_extension_dos_param_t)); + if (NULL == val) + return NULL; + return val; +} + +/** Release all storage held inside 'obj', but do not free 'obj'. + */ +static void +trn_cell_extension_dos_param_clear(trn_cell_extension_dos_param_t *obj) +{ + (void) obj; +} + +void +trn_cell_extension_dos_param_free(trn_cell_extension_dos_param_t *obj) +{ + if (obj == NULL) + return; + trn_cell_extension_dos_param_clear(obj); + trunnel_memwipe(obj, sizeof(trn_cell_extension_dos_param_t)); + trunnel_free_(obj); +} + +uint8_t +trn_cell_extension_dos_param_get_type(const trn_cell_extension_dos_param_t *inp) +{ + return inp->type; +} +int +trn_cell_extension_dos_param_set_type(trn_cell_extension_dos_param_t *inp, uint8_t val) +{ + inp->type = val; + return 0; +} +uint64_t +trn_cell_extension_dos_param_get_value(const trn_cell_extension_dos_param_t *inp) +{ + return inp->value; +} +int +trn_cell_extension_dos_param_set_value(trn_cell_extension_dos_param_t *inp, uint64_t val) +{ + inp->value = val; + return 0; +} +const char * +trn_cell_extension_dos_param_check(const trn_cell_extension_dos_param_t *obj) +{ + if (obj == NULL) + return "Object was NULL"; + if (obj->trunnel_error_code_) + return "A set function failed on this object"; + return NULL; +} + +ssize_t +trn_cell_extension_dos_param_encoded_len(const trn_cell_extension_dos_param_t *obj) +{ + ssize_t result = 0; + + if (NULL != trn_cell_extension_dos_param_check(obj)) + return -1; + + + /* Length of u8 type */ + result += 1; + + /* Length of u64 value */ + result += 8; + return result; +} +int +trn_cell_extension_dos_param_clear_errors(trn_cell_extension_dos_param_t *obj) +{ + int r = obj->trunnel_error_code_; + obj->trunnel_error_code_ = 0; + return r; +} +ssize_t +trn_cell_extension_dos_param_encode(uint8_t *output, const size_t avail, const trn_cell_extension_dos_param_t *obj) +{ + ssize_t result = 0; + size_t written = 0; + uint8_t *ptr = output; + const char *msg; +#ifdef TRUNNEL_CHECK_ENCODED_LEN + const ssize_t encoded_len = trn_cell_extension_dos_param_encoded_len(obj); +#endif + + if (NULL != (msg = trn_cell_extension_dos_param_check(obj))) + goto check_failed; + +#ifdef TRUNNEL_CHECK_ENCODED_LEN + trunnel_assert(encoded_len >= 0); +#endif + + /* Encode u8 type */ + trunnel_assert(written <= avail); + if (avail - written < 1) + goto truncated; + trunnel_set_uint8(ptr, (obj->type)); + written += 1; ptr += 1; + + /* Encode u64 value */ + trunnel_assert(written <= avail); + if (avail - written < 8) + goto truncated; + trunnel_set_uint64(ptr, trunnel_htonll(obj->value)); + written += 8; ptr += 8; + + + trunnel_assert(ptr == output + written); +#ifdef TRUNNEL_CHECK_ENCODED_LEN + { + trunnel_assert(encoded_len >= 0); + trunnel_assert((size_t)encoded_len == written); + } + +#endif + + return written; + + truncated: + result = -2; + goto fail; + check_failed: + (void)msg; + result = -1; + goto fail; + fail: + trunnel_assert(result < 0); + return result; +} + +/** As trn_cell_extension_dos_param_parse(), but do not allocate the + * output object. + */ +static ssize_t +trn_cell_extension_dos_param_parse_into(trn_cell_extension_dos_param_t *obj, const uint8_t *input, const size_t len_in) +{ + const uint8_t *ptr = input; + size_t remaining = len_in; + ssize_t result = 0; + (void)result; + + /* Parse u8 type */ + CHECK_REMAINING(1, truncated); + obj->type = (trunnel_get_uint8(ptr)); + remaining -= 1; ptr += 1; + + /* Parse u64 value */ + CHECK_REMAINING(8, truncated); + obj->value = trunnel_ntohll(trunnel_get_uint64(ptr)); + remaining -= 8; ptr += 8; + trunnel_assert(ptr + remaining == input + len_in); + return len_in - remaining; + + truncated: + return -2; +} + +ssize_t +trn_cell_extension_dos_param_parse(trn_cell_extension_dos_param_t **output, const uint8_t *input, const size_t len_in) +{ + ssize_t result; + *output = trn_cell_extension_dos_param_new(); + if (NULL == *output) + return -1; + result = trn_cell_extension_dos_param_parse_into(*output, input, len_in); + if (result < 0) { + trn_cell_extension_dos_param_free(*output); + *output = NULL; + } + return result; +} trn_cell_establish_intro_t * trn_cell_establish_intro_new(void) { @@ -561,6 +740,296 @@ trn_cell_establish_intro_parse(trn_cell_establish_intro_t **output, const uint8_ } return result; } +trn_cell_extension_dos_t * +trn_cell_extension_dos_new(void) +{ + trn_cell_extension_dos_t *val = trunnel_calloc(1, sizeof(trn_cell_extension_dos_t)); + if (NULL == val) + return NULL; + return val; +} + +/** Release all storage held inside 'obj', but do not free 'obj'. + */ +static void +trn_cell_extension_dos_clear(trn_cell_extension_dos_t *obj) +{ + (void) obj; + { + + unsigned idx; + for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->params); ++idx) { + trn_cell_extension_dos_param_free(TRUNNEL_DYNARRAY_GET(&obj->params, idx)); + } + } + TRUNNEL_DYNARRAY_WIPE(&obj->params); + TRUNNEL_DYNARRAY_CLEAR(&obj->params); +} + +void +trn_cell_extension_dos_free(trn_cell_extension_dos_t *obj) +{ + if (obj == NULL) + return; + trn_cell_extension_dos_clear(obj); + trunnel_memwipe(obj, sizeof(trn_cell_extension_dos_t)); + trunnel_free_(obj); +} + +uint8_t +trn_cell_extension_dos_get_n_params(const trn_cell_extension_dos_t *inp) +{ + return inp->n_params; +} +int +trn_cell_extension_dos_set_n_params(trn_cell_extension_dos_t *inp, uint8_t val) +{ + inp->n_params = val; + return 0; +} +size_t +trn_cell_extension_dos_getlen_params(const trn_cell_extension_dos_t *inp) +{ + return TRUNNEL_DYNARRAY_LEN(&inp->params); +} + +struct trn_cell_extension_dos_param_st * +trn_cell_extension_dos_get_params(trn_cell_extension_dos_t *inp, size_t idx) +{ + return TRUNNEL_DYNARRAY_GET(&inp->params, idx); +} + + const struct trn_cell_extension_dos_param_st * +trn_cell_extension_dos_getconst_params(const trn_cell_extension_dos_t *inp, size_t idx) +{ + return trn_cell_extension_dos_get_params((trn_cell_extension_dos_t*)inp, idx); +} +int +trn_cell_extension_dos_set_params(trn_cell_extension_dos_t *inp, size_t idx, struct trn_cell_extension_dos_param_st * elt) +{ + trn_cell_extension_dos_param_t *oldval = TRUNNEL_DYNARRAY_GET(&inp->params, idx); + if (oldval && oldval != elt) + trn_cell_extension_dos_param_free(oldval); + return trn_cell_extension_dos_set0_params(inp, idx, elt); +} +int +trn_cell_extension_dos_set0_params(trn_cell_extension_dos_t *inp, size_t idx, struct trn_cell_extension_dos_param_st * elt) +{ + TRUNNEL_DYNARRAY_SET(&inp->params, idx, elt); + return 0; +} +int +trn_cell_extension_dos_add_params(trn_cell_extension_dos_t *inp, struct trn_cell_extension_dos_param_st * elt) +{ +#if SIZE_MAX >= UINT8_MAX + if (inp->params.n_ == UINT8_MAX) + goto trunnel_alloc_failed; +#endif + TRUNNEL_DYNARRAY_ADD(struct trn_cell_extension_dos_param_st *, &inp->params, elt, {}); + return 0; + trunnel_alloc_failed: + TRUNNEL_SET_ERROR_CODE(inp); + return -1; +} + +struct trn_cell_extension_dos_param_st * * +trn_cell_extension_dos_getarray_params(trn_cell_extension_dos_t *inp) +{ + return inp->params.elts_; +} +const struct trn_cell_extension_dos_param_st * const * +trn_cell_extension_dos_getconstarray_params(const trn_cell_extension_dos_t *inp) +{ + return (const struct trn_cell_extension_dos_param_st * const *)trn_cell_extension_dos_getarray_params((trn_cell_extension_dos_t*)inp); +} +int +trn_cell_extension_dos_setlen_params(trn_cell_extension_dos_t *inp, size_t newlen) +{ + struct trn_cell_extension_dos_param_st * *newptr; +#if UINT8_MAX < SIZE_MAX + if (newlen > UINT8_MAX) + goto trunnel_alloc_failed; +#endif + newptr = trunnel_dynarray_setlen(&inp->params.allocated_, + &inp->params.n_, inp->params.elts_, newlen, + sizeof(inp->params.elts_[0]), (trunnel_free_fn_t) trn_cell_extension_dos_param_free, + &inp->trunnel_error_code_); + if (newlen != 0 && newptr == NULL) + goto trunnel_alloc_failed; + inp->params.elts_ = newptr; + return 0; + trunnel_alloc_failed: + TRUNNEL_SET_ERROR_CODE(inp); + return -1; +} +const char * +trn_cell_extension_dos_check(const trn_cell_extension_dos_t *obj) +{ + if (obj == NULL) + return "Object was NULL"; + if (obj->trunnel_error_code_) + return "A set function failed on this object"; + { + const char *msg; + + unsigned idx; + for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->params); ++idx) { + if (NULL != (msg = trn_cell_extension_dos_param_check(TRUNNEL_DYNARRAY_GET(&obj->params, idx)))) + return msg; + } + } + if (TRUNNEL_DYNARRAY_LEN(&obj->params) != obj->n_params) + return "Length mismatch for params"; + return NULL; +} + +ssize_t +trn_cell_extension_dos_encoded_len(const trn_cell_extension_dos_t *obj) +{ + ssize_t result = 0; + + if (NULL != trn_cell_extension_dos_check(obj)) + return -1; + + + /* Length of u8 n_params */ + result += 1; + + /* Length of struct trn_cell_extension_dos_param params[n_params] */ + { + + unsigned idx; + for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->params); ++idx) { + result += trn_cell_extension_dos_param_encoded_len(TRUNNEL_DYNARRAY_GET(&obj->params, idx)); + } + } + return result; +} +int +trn_cell_extension_dos_clear_errors(trn_cell_extension_dos_t *obj) +{ + int r = obj->trunnel_error_code_; + obj->trunnel_error_code_ = 0; + return r; +} +ssize_t +trn_cell_extension_dos_encode(uint8_t *output, const size_t avail, const trn_cell_extension_dos_t *obj) +{ + ssize_t result = 0; + size_t written = 0; + uint8_t *ptr = output; + const char *msg; +#ifdef TRUNNEL_CHECK_ENCODED_LEN + const ssize_t encoded_len = trn_cell_extension_dos_encoded_len(obj); +#endif + + if (NULL != (msg = trn_cell_extension_dos_check(obj))) + goto check_failed; + +#ifdef TRUNNEL_CHECK_ENCODED_LEN + trunnel_assert(encoded_len >= 0); +#endif + + /* Encode u8 n_params */ + trunnel_assert(written <= avail); + if (avail - written < 1) + goto truncated; + trunnel_set_uint8(ptr, (obj->n_params)); + written += 1; ptr += 1; + + /* Encode struct trn_cell_extension_dos_param params[n_params] */ + { + + unsigned idx; + for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->params); ++idx) { + trunnel_assert(written <= avail); + result = trn_cell_extension_dos_param_encode(ptr, avail - written, TRUNNEL_DYNARRAY_GET(&obj->params, idx)); + if (result < 0) + goto fail; /* XXXXXXX !*/ + written += result; ptr += result; + } + } + + + trunnel_assert(ptr == output + written); +#ifdef TRUNNEL_CHECK_ENCODED_LEN + { + trunnel_assert(encoded_len >= 0); + trunnel_assert((size_t)encoded_len == written); + } + +#endif + + return written; + + truncated: + result = -2; + goto fail; + check_failed: + (void)msg; + result = -1; + goto fail; + fail: + trunnel_assert(result < 0); + return result; +} + +/** As trn_cell_extension_dos_parse(), but do not allocate the output + * object. + */ +static ssize_t +trn_cell_extension_dos_parse_into(trn_cell_extension_dos_t *obj, const uint8_t *input, const size_t len_in) +{ + const uint8_t *ptr = input; + size_t remaining = len_in; + ssize_t result = 0; + (void)result; + + /* Parse u8 n_params */ + CHECK_REMAINING(1, truncated); + obj->n_params = (trunnel_get_uint8(ptr)); + remaining -= 1; ptr += 1; + + /* Parse struct trn_cell_extension_dos_param params[n_params] */ + TRUNNEL_DYNARRAY_EXPAND(trn_cell_extension_dos_param_t *, &obj->params, obj->n_params, {}); + { + trn_cell_extension_dos_param_t * elt; + unsigned idx; + for (idx = 0; idx < obj->n_params; ++idx) { + result = trn_cell_extension_dos_param_parse(&elt, ptr, remaining); + if (result < 0) + goto relay_fail; + trunnel_assert((size_t)result <= remaining); + remaining -= result; ptr += result; + TRUNNEL_DYNARRAY_ADD(trn_cell_extension_dos_param_t *, &obj->params, elt, {trn_cell_extension_dos_param_free(elt);}); + } + } + trunnel_assert(ptr + remaining == input + len_in); + return len_in - remaining; + + truncated: + return -2; + relay_fail: + trunnel_assert(result < 0); + return result; + trunnel_alloc_failed: + return -1; +} + +ssize_t +trn_cell_extension_dos_parse(trn_cell_extension_dos_t **output, const uint8_t *input, const size_t len_in) +{ + ssize_t result; + *output = trn_cell_extension_dos_new(); + if (NULL == *output) + return -1; + result = trn_cell_extension_dos_parse_into(*output, input, len_in); + if (result < 0) { + trn_cell_extension_dos_free(*output); + *output = NULL; + } + return result; +} trn_cell_intro_established_t * trn_cell_intro_established_new(void) { diff --git a/src/trunnel/hs/cell_establish_intro.h b/src/trunnel/hs/cell_establish_intro.h index ccaef5488c..1924d9cab6 100644 --- a/src/trunnel/hs/cell_establish_intro.h +++ b/src/trunnel/hs/cell_establish_intro.h @@ -1,4 +1,4 @@ -/* cell_establish_intro.h -- generated by Trunnel v1.5.2. +/* cell_establish_intro.h -- generated by Trunnel v1.5.3. * https://gitweb.torproject.org/trunnel.git * You probably shouldn't edit this file. */ @@ -10,6 +10,17 @@ struct trn_cell_extension_st; #define TRUNNEL_SHA3_256_LEN 32 +#define TRUNNEL_CELL_EXTENSION_TYPE_DOS 1 +#define TRUNNEL_DOS_PARAM_TYPE_INTRO2_RATE_PER_SEC 1 +#define TRUNNEL_DOS_PARAM_TYPE_INTRO2_BURST_PER_SEC 2 +#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_EXTENSION_DOS_PARAM) +struct trn_cell_extension_dos_param_st { + uint8_t type; + uint64_t value; + uint8_t trunnel_error_code_; +}; +#endif +typedef struct trn_cell_extension_dos_param_st trn_cell_extension_dos_param_t; #if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_ESTABLISH_INTRO) struct trn_cell_establish_intro_st { const uint8_t *start_cell; @@ -26,6 +37,14 @@ struct trn_cell_establish_intro_st { }; #endif typedef struct trn_cell_establish_intro_st trn_cell_establish_intro_t; +#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_EXTENSION_DOS) +struct trn_cell_extension_dos_st { + uint8_t n_params; + TRUNNEL_DYNARRAY_HEAD(, struct trn_cell_extension_dos_param_st *) params; + uint8_t trunnel_error_code_; +}; +#endif +typedef struct trn_cell_extension_dos_st trn_cell_extension_dos_t; #if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_INTRO_ESTABLISHED) struct trn_cell_intro_established_st { struct trn_cell_extension_st *extensions; @@ -33,6 +52,62 @@ struct trn_cell_intro_established_st { }; #endif typedef struct trn_cell_intro_established_st trn_cell_intro_established_t; +/** Return a newly allocated trn_cell_extension_dos_param with all + * elements set to zero. + */ +trn_cell_extension_dos_param_t *trn_cell_extension_dos_param_new(void); +/** Release all storage held by the trn_cell_extension_dos_param in + * 'victim'. (Do nothing if 'victim' is NULL.) + */ +void trn_cell_extension_dos_param_free(trn_cell_extension_dos_param_t *victim); +/** Try to parse a trn_cell_extension_dos_param from the buffer in + * 'input', using up to 'len_in' bytes from the input buffer. On + * success, return the number of bytes consumed and set *output to the + * newly allocated trn_cell_extension_dos_param_t. On failure, return + * -2 if the input appears truncated, and -1 if the input is otherwise + * invalid. + */ +ssize_t trn_cell_extension_dos_param_parse(trn_cell_extension_dos_param_t **output, const uint8_t *input, const size_t len_in); +/** Return the number of bytes we expect to need to encode the + * trn_cell_extension_dos_param in 'obj'. On failure, return a + * negative value. Note that this value may be an overestimate, and + * can even be an underestimate for certain unencodeable objects. + */ +ssize_t trn_cell_extension_dos_param_encoded_len(const trn_cell_extension_dos_param_t *obj); +/** Try to encode the trn_cell_extension_dos_param from 'input' into + * the buffer at 'output', using up to 'avail' bytes of the output + * buffer. On success, return the number of bytes used. On failure, + * return -2 if the buffer was not long enough, and -1 if the input + * was invalid. + */ +ssize_t trn_cell_extension_dos_param_encode(uint8_t *output, size_t avail, const trn_cell_extension_dos_param_t *input); +/** Check whether the internal state of the + * trn_cell_extension_dos_param in 'obj' is consistent. Return NULL if + * it is, and a short message if it is not. + */ +const char *trn_cell_extension_dos_param_check(const trn_cell_extension_dos_param_t *obj); +/** Clear any errors that were set on the object 'obj' by its setter + * functions. Return true iff errors were cleared. + */ +int trn_cell_extension_dos_param_clear_errors(trn_cell_extension_dos_param_t *obj); +/** Return the value of the type field of the + * trn_cell_extension_dos_param_t in 'inp' + */ +uint8_t trn_cell_extension_dos_param_get_type(const trn_cell_extension_dos_param_t *inp); +/** Set the value of the type field of the + * trn_cell_extension_dos_param_t in 'inp' to 'val'. Return 0 on + * success; return -1 and set the error code on 'inp' on failure. + */ +int trn_cell_extension_dos_param_set_type(trn_cell_extension_dos_param_t *inp, uint8_t val); +/** Return the value of the value field of the + * trn_cell_extension_dos_param_t in 'inp' + */ +uint64_t trn_cell_extension_dos_param_get_value(const trn_cell_extension_dos_param_t *inp); +/** Set the value of the value field of the + * trn_cell_extension_dos_param_t in 'inp' to 'val'. Return 0 on + * success; return -1 and set the error code on 'inp' on failure. + */ +int trn_cell_extension_dos_param_set_value(trn_cell_extension_dos_param_t *inp, uint64_t val); /** Return a newly allocated trn_cell_establish_intro with all * elements set to zero. */ @@ -216,6 +291,90 @@ const uint8_t * trn_cell_establish_intro_getconstarray_sig(const trn_cell_estab * -1 and set the error code on 'inp' on failure. */ int trn_cell_establish_intro_setlen_sig(trn_cell_establish_intro_t *inp, size_t newlen); +/** Return a newly allocated trn_cell_extension_dos with all elements + * set to zero. + */ +trn_cell_extension_dos_t *trn_cell_extension_dos_new(void); +/** Release all storage held by the trn_cell_extension_dos in + * 'victim'. (Do nothing if 'victim' is NULL.) + */ +void trn_cell_extension_dos_free(trn_cell_extension_dos_t *victim); +/** Try to parse a trn_cell_extension_dos from the buffer in 'input', + * using up to 'len_in' bytes from the input buffer. On success, + * return the number of bytes consumed and set *output to the newly + * allocated trn_cell_extension_dos_t. On failure, return -2 if the + * input appears truncated, and -1 if the input is otherwise invalid. + */ +ssize_t trn_cell_extension_dos_parse(trn_cell_extension_dos_t **output, const uint8_t *input, const size_t len_in); +/** Return the number of bytes we expect to need to encode the + * trn_cell_extension_dos in 'obj'. On failure, return a negative + * value. Note that this value may be an overestimate, and can even be + * an underestimate for certain unencodeable objects. + */ +ssize_t trn_cell_extension_dos_encoded_len(const trn_cell_extension_dos_t *obj); +/** Try to encode the trn_cell_extension_dos from 'input' into the + * buffer at 'output', using up to 'avail' bytes of the output buffer. + * On success, return the number of bytes used. On failure, return -2 + * if the buffer was not long enough, and -1 if the input was invalid. + */ +ssize_t trn_cell_extension_dos_encode(uint8_t *output, size_t avail, const trn_cell_extension_dos_t *input); +/** Check whether the internal state of the trn_cell_extension_dos in + * 'obj' is consistent. Return NULL if it is, and a short message if + * it is not. + */ +const char *trn_cell_extension_dos_check(const trn_cell_extension_dos_t *obj); +/** Clear any errors that were set on the object 'obj' by its setter + * functions. Return true iff errors were cleared. + */ +int trn_cell_extension_dos_clear_errors(trn_cell_extension_dos_t *obj); +/** Return the value of the n_params field of the + * trn_cell_extension_dos_t in 'inp' + */ +uint8_t trn_cell_extension_dos_get_n_params(const trn_cell_extension_dos_t *inp); +/** Set the value of the n_params field of the + * trn_cell_extension_dos_t in 'inp' to 'val'. Return 0 on success; + * return -1 and set the error code on 'inp' on failure. + */ +int trn_cell_extension_dos_set_n_params(trn_cell_extension_dos_t *inp, uint8_t val); +/** Return the length of the dynamic array holding the params field of + * the trn_cell_extension_dos_t in 'inp'. + */ +size_t trn_cell_extension_dos_getlen_params(const trn_cell_extension_dos_t *inp); +/** Return the element at position 'idx' of the dynamic array field + * params of the trn_cell_extension_dos_t in 'inp'. + */ +struct trn_cell_extension_dos_param_st * trn_cell_extension_dos_get_params(trn_cell_extension_dos_t *inp, size_t idx); +/** As trn_cell_extension_dos_get_params, but take and return a const + * pointer + */ + const struct trn_cell_extension_dos_param_st * trn_cell_extension_dos_getconst_params(const trn_cell_extension_dos_t *inp, size_t idx); +/** Change the element at position 'idx' of the dynamic array field + * params of the trn_cell_extension_dos_t in 'inp', so that it will + * hold the value 'elt'. Free the previous value, if any. + */ +int trn_cell_extension_dos_set_params(trn_cell_extension_dos_t *inp, size_t idx, struct trn_cell_extension_dos_param_st * elt); +/** As trn_cell_extension_dos_set_params, but does not free the + * previous value. + */ +int trn_cell_extension_dos_set0_params(trn_cell_extension_dos_t *inp, size_t idx, struct trn_cell_extension_dos_param_st * elt); +/** Append a new element 'elt' to the dynamic array field params of + * the trn_cell_extension_dos_t in 'inp'. + */ +int trn_cell_extension_dos_add_params(trn_cell_extension_dos_t *inp, struct trn_cell_extension_dos_param_st * elt); +/** Return a pointer to the variable-length array field params of + * 'inp'. + */ +struct trn_cell_extension_dos_param_st * * trn_cell_extension_dos_getarray_params(trn_cell_extension_dos_t *inp); +/** As trn_cell_extension_dos_get_params, but take and return a const + * pointer + */ +const struct trn_cell_extension_dos_param_st * const * trn_cell_extension_dos_getconstarray_params(const trn_cell_extension_dos_t *inp); +/** Change the length of the variable-length array field params of + * 'inp' to 'newlen'.Fill extra elements with NULL; free removed + * elements. Return 0 on success; return -1 and set the error code on + * 'inp' on failure. + */ +int trn_cell_extension_dos_setlen_params(trn_cell_extension_dos_t *inp, size_t newlen); /** Return a newly allocated trn_cell_intro_established with all * elements set to zero. */ diff --git a/src/trunnel/hs/cell_establish_intro.trunnel b/src/trunnel/hs/cell_establish_intro.trunnel index 011ee62a15..e30938f6c2 100644 --- a/src/trunnel/hs/cell_establish_intro.trunnel +++ b/src/trunnel/hs/cell_establish_intro.trunnel @@ -39,3 +39,26 @@ struct trn_cell_intro_established { /* Extension(s). Reserved fields. */ struct trn_cell_extension extensions; }; + +/* + * ESTABLISH_INTRO cell extensions. + */ + +const TRUNNEL_CELL_EXTENSION_TYPE_DOS = 0x01; + +/* DoS Parameter types. */ +const TRUNNEL_DOS_PARAM_TYPE_INTRO2_RATE_PER_SEC = 0x01; +const TRUNNEL_DOS_PARAM_TYPE_INTRO2_BURST_PER_SEC = 0x02; + +/* + * DoS Parameters Extension. See proposal 305 for more details. + */ +struct trn_cell_extension_dos_param { + u8 type; + u64 value; +}; + +struct trn_cell_extension_dos { + u8 n_params; + struct trn_cell_extension_dos_param params[n_params]; +}; diff --git a/src/trunnel/hs/cell_introduce1.c b/src/trunnel/hs/cell_introduce1.c index 53b3d299f2..016c9fa8d6 100644 --- a/src/trunnel/hs/cell_introduce1.c +++ b/src/trunnel/hs/cell_introduce1.c @@ -1,4 +1,4 @@ -/* cell_introduce1.c -- generated by Trunnel v1.5.2. +/* cell_introduce1.c -- generated by Trunnel v1.5.3. * https://gitweb.torproject.org/trunnel.git * You probably shouldn't edit this file. */ diff --git a/src/trunnel/hs/cell_introduce1.h b/src/trunnel/hs/cell_introduce1.h index 986a531ca7..8dabff3cb5 100644 --- a/src/trunnel/hs/cell_introduce1.h +++ b/src/trunnel/hs/cell_introduce1.h @@ -1,4 +1,4 @@ -/* cell_introduce1.h -- generated by Trunnel v1.5.2. +/* cell_introduce1.h -- generated by Trunnel v1.5.3. * https://gitweb.torproject.org/trunnel.git * You probably shouldn't edit this file. */ diff --git a/src/trunnel/hs/cell_rendezvous.c b/src/trunnel/hs/cell_rendezvous.c index 53cb609138..1204e93cfc 100644 --- a/src/trunnel/hs/cell_rendezvous.c +++ b/src/trunnel/hs/cell_rendezvous.c @@ -1,4 +1,4 @@ -/* cell_rendezvous.c -- generated by Trunnel v1.5.2. +/* cell_rendezvous.c -- generated by Trunnel v1.5.3. * https://gitweb.torproject.org/trunnel.git * You probably shouldn't edit this file. */ diff --git a/src/trunnel/hs/cell_rendezvous.h b/src/trunnel/hs/cell_rendezvous.h index 39e14da25b..5a8c2ff52a 100644 --- a/src/trunnel/hs/cell_rendezvous.h +++ b/src/trunnel/hs/cell_rendezvous.h @@ -1,4 +1,4 @@ -/* cell_rendezvous.h -- generated by Trunnel v1.5.2. +/* cell_rendezvous.h -- generated by Trunnel v1.5.3. * https://gitweb.torproject.org/trunnel.git * You probably shouldn't edit this file. */ diff --git a/src/trunnel/include.am b/src/trunnel/include.am index ce15570b15..6c3a5ff06b 100644 --- a/src/trunnel/include.am +++ b/src/trunnel/include.am @@ -11,7 +11,7 @@ TRUNNELINPUTS = \ src/trunnel/link_handshake.trunnel \ src/trunnel/pwbox.trunnel \ src/trunnel/channelpadding_negotiation.trunnel \ - src/trunnel/sendme.trunnel \ + src/trunnel/sendme_cell.trunnel \ src/trunnel/socks5.trunnel \ src/trunnel/circpad_negotiation.trunnel @@ -25,7 +25,7 @@ TRUNNELSOURCES = \ src/trunnel/hs/cell_introduce1.c \ src/trunnel/hs/cell_rendezvous.c \ src/trunnel/channelpadding_negotiation.c \ - src/trunnel/sendme.c \ + src/trunnel/sendme_cell.c \ src/trunnel/socks5.c \ src/trunnel/netinfo.c \ src/trunnel/circpad_negotiation.c @@ -42,7 +42,7 @@ TRUNNELHEADERS = \ src/trunnel/hs/cell_introduce1.h \ src/trunnel/hs/cell_rendezvous.h \ src/trunnel/channelpadding_negotiation.h \ - src/trunnel/sendme.h \ + src/trunnel/sendme_cell.h \ src/trunnel/socks5.h \ src/trunnel/netinfo.h \ src/trunnel/circpad_negotiation.h diff --git a/src/trunnel/link_handshake.c b/src/trunnel/link_handshake.c index 03ead31c62..76db4b0e29 100644 --- a/src/trunnel/link_handshake.c +++ b/src/trunnel/link_handshake.c @@ -1,4 +1,4 @@ -/* link_handshake.c -- generated by Trunnel v1.5.2. +/* link_handshake.c -- generated by Trunnel v1.5.3. * https://gitweb.torproject.org/trunnel.git * You probably shouldn't edit this file. */ diff --git a/src/trunnel/link_handshake.h b/src/trunnel/link_handshake.h index 6a23483adc..0c7ac36b1b 100644 --- a/src/trunnel/link_handshake.h +++ b/src/trunnel/link_handshake.h @@ -1,4 +1,4 @@ -/* link_handshake.h -- generated by Trunnel v1.5.2. +/* link_handshake.h -- generated by Trunnel v1.5.3. * https://gitweb.torproject.org/trunnel.git * You probably shouldn't edit this file. */ diff --git a/src/trunnel/netinfo.c b/src/trunnel/netinfo.c index 5d815b9b12..d7d0cddc89 100644 --- a/src/trunnel/netinfo.c +++ b/src/trunnel/netinfo.c @@ -1,4 +1,4 @@ -/* netinfo.c -- generated by Trunnel v1.5.2. +/* netinfo.c -- generated by Trunnel v1.5.3. * https://gitweb.torproject.org/trunnel.git * You probably shouldn't edit this file. */ diff --git a/src/trunnel/netinfo.h b/src/trunnel/netinfo.h index ac46e603ba..37c2ae3c2d 100644 --- a/src/trunnel/netinfo.h +++ b/src/trunnel/netinfo.h @@ -1,4 +1,4 @@ -/* netinfo.h -- generated by Trunnel v1.5.2. +/* netinfo.h -- generated by Trunnel v1.5.3. * https://gitweb.torproject.org/trunnel.git * You probably shouldn't edit this file. */ diff --git a/src/trunnel/pwbox.c b/src/trunnel/pwbox.c index c356515d36..c159a5e687 100644 --- a/src/trunnel/pwbox.c +++ b/src/trunnel/pwbox.c @@ -1,4 +1,4 @@ -/* pwbox.c -- generated by Trunnel v1.5.2. +/* pwbox.c -- generated by Trunnel v1.5.3. * https://gitweb.torproject.org/trunnel.git * You probably shouldn't edit this file. */ diff --git a/src/trunnel/pwbox.h b/src/trunnel/pwbox.h index a9a421408a..36d595f4ef 100644 --- a/src/trunnel/pwbox.h +++ b/src/trunnel/pwbox.h @@ -1,4 +1,4 @@ -/* pwbox.h -- generated by Trunnel v1.5.2. +/* pwbox.h -- generated by Trunnel v1.5.3. * https://gitweb.torproject.org/trunnel.git * You probably shouldn't edit this file. */ diff --git a/src/trunnel/sendme.c b/src/trunnel/sendme_cell.c index 262b915234..b9f8fe967f 100644 --- a/src/trunnel/sendme.c +++ b/src/trunnel/sendme_cell.c @@ -1,11 +1,11 @@ -/* sendme.c -- generated by Trunnel v1.5.2. +/* sendme_cell.c -- generated by Trunnel v1.5.3. * https://gitweb.torproject.org/trunnel.git * You probably shouldn't edit this file. */ #include <stdlib.h> #include "trunnel-impl.h" -#include "sendme.h" +#include "sendme_cell.h" #define TRUNNEL_SET_ERROR_CODE(obj) \ do { \ @@ -15,8 +15,8 @@ #if defined(__COVERITY__) || defined(__clang_analyzer__) /* If we're running a static analysis tool, we don't want it to complain * that some of our remaining-bytes checks are dead-code. */ -int sendme_deadcode_dummy__ = 0; -#define OR_DEADCODE_DUMMY || sendme_deadcode_dummy__ +int sendmecell_deadcode_dummy__ = 0; +#define OR_DEADCODE_DUMMY || sendmecell_deadcode_dummy__ #else #define OR_DEADCODE_DUMMY #endif diff --git a/src/trunnel/sendme.h b/src/trunnel/sendme_cell.h index f3c3dd78c4..45efb9f10d 100644 --- a/src/trunnel/sendme.h +++ b/src/trunnel/sendme_cell.h @@ -1,9 +1,9 @@ -/* sendme.h -- generated by Trunnel v1.5.2. +/* sendme_cell.h -- generated by Trunnel v1.5.3. * https://gitweb.torproject.org/trunnel.git * You probably shouldn't edit this file. */ -#ifndef TRUNNEL_SENDME_H -#define TRUNNEL_SENDME_H +#ifndef TRUNNEL_SENDME_CELL_H +#define TRUNNEL_SENDME_CELL_H #include <stdint.h> #include "trunnel.h" diff --git a/src/trunnel/sendme.trunnel b/src/trunnel/sendme_cell.trunnel index 300963e679..300963e679 100644 --- a/src/trunnel/sendme.trunnel +++ b/src/trunnel/sendme_cell.trunnel diff --git a/src/trunnel/socks5.c b/src/trunnel/socks5.c index 057a52b042..f32862e353 100644 --- a/src/trunnel/socks5.c +++ b/src/trunnel/socks5.c @@ -1,4 +1,4 @@ -/* socks5.c -- generated by Trunnel v1.5.2. +/* socks5.c -- generated by Trunnel v1.5.3. * https://gitweb.torproject.org/trunnel.git * You probably shouldn't edit this file. */ diff --git a/src/trunnel/socks5.h b/src/trunnel/socks5.h index d3bea152e7..23ac64faba 100644 --- a/src/trunnel/socks5.h +++ b/src/trunnel/socks5.h @@ -1,4 +1,4 @@ -/* socks5.h -- generated by Trunnel v1.5.2. +/* socks5.h -- generated by Trunnel v1.5.3. * https://gitweb.torproject.org/trunnel.git * You probably shouldn't edit this file. */ diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h index c23dfe7bb7..556c8b8056 100644 --- a/src/win32/orconfig.h +++ b/src/win32/orconfig.h @@ -218,7 +218,7 @@ #define USING_TWOS_COMPLEMENT /* Version number of package */ -#define VERSION "0.4.1.8-dev" +#define VERSION "0.4.2.6-dev" |