diff options
489 files changed, 29562 insertions, 8343 deletions
diff --git a/.appveyor.yml b/.appveyor.yml index 818e074a4e..7e08602fe3 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -95,6 +95,8 @@ test_script: $buildpath = @("C:\msys64\${env:compiler_path}\bin") + $oldpath $env:Path = $buildpath -join ';' Set-Location "${env:build}" + Copy-Item "C:/msys64/${env:compiler_path}/bin/libssp-0.dll" -Destination "${env:build}/src/test" + Copy-Item "C:/msys64/${env:compiler_path}/bin/zlib1.dll" -Destination "${env:build}/src/test" Execute-Bash "VERBOSE=1 make -k -j2 check" } diff --git a/.gitignore b/.gitignore index f1ce903a11..a40cda02dd 100644 --- a/.gitignore +++ b/.gitignore @@ -158,6 +158,8 @@ uptime-*.json # /src/lib /src/lib/libcurve25519_donna.a +/src/lib/libtor-buf.a +/src/lib/libtor-buf-testing.a /src/lib/libtor-compress.a /src/lib/libtor-compress-testing.a /src/lib/libtor-container.a @@ -213,6 +215,8 @@ uptime-*.json /src/lib/libtor-tls.a /src/lib/libtor-tls-testing.a /src/lib/libtor-trace.a +/src/lib/libtor-version.a +/src/lib/libtor-version-testing.a /src/lib/libtor-wallclock.a /src/lib/libtor-wallclock-testing.a @@ -240,20 +244,22 @@ uptime-*.json /src/test/test /src/test/test-slow /src/test/test-bt-cl -/src/test/test-child +/src/test/test-process /src/test/test-memwipe /src/test/test-ntor-cl /src/test/test-hs-ntor-cl +/src/test/test-rng /src/test/test-switch-id /src/test/test-timers /src/test/test_workqueue /src/test/test.exe /src/test/test-slow.exe /src/test/test-bt-cl.exe -/src/test/test-child.exe +/src/test/test-process.exe /src/test/test-ntor-cl.exe /src/test/test-hs-ntor-cl.exe /src/test/test-memwipe.exe +/src/test/test-rng.exe /src/test/test-switch-id.exe /src/test/test-timers.exe /src/test/test_workqueue.exe diff --git a/.travis.yml b/.travis.yml index 7f7e7a5723..c491bab88f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -116,6 +116,7 @@ addons: - libscrypt-dev - libseccomp-dev - libzstd-dev + - shellcheck ## Conditional build dependencies ## Always installed, so we don't need sudo - asciidoc @@ -144,6 +145,7 @@ addons: - pkg-config ## Optional build dependencies - ccache + - shellcheck ## Conditional build dependencies ## Always installed, because manual brew installs are hard to get right - asciidoc @@ -1,3 +1,853 @@ +Changes in version 0.4.0.1-alpha - 2019-01-18 + Tor 0.4.0.1-alpha is the first release in the new 0.4.0.x series. It + introduces improved features for power and bandwidth conservation, + more accurate reporting of bootstrap progress for user interfaces, and + an experimental backend for an exciting new adaptive padding feature. + There is also the usual assortment of bugfixes and minor features, all + described below. + + o Major features (battery management, client, dormant mode): + - When Tor is running as a client, and it is unused for a long time, + it can now enter a "dormant" state. When Tor is dormant, it avoids + network and CPU activity until it is reawoken either by a user + request or by a controller command. For more information, see the + configuration options starting with "Dormant". Implements tickets + 2149 and 28335. + - The client's memory of whether it is "dormant", and how long it + has spent idle, persists across invocations. Implements + ticket 28624. + - There is a DormantOnFirstStartup option that integrators can use + if they expect that in many cases, Tor will be installed but + not used. + + o Major features (bootstrap reporting): + - When reporting bootstrap progress, report the first connection + uniformly, regardless of whether it's a connection for building + application circuits. This allows finer-grained reporting of early + progress than previously possible, with the improvements of ticket + 27169. Closes tickets 27167 and 27103. Addresses ticket 27308. + - When reporting bootstrap progress, treat connecting to a proxy or + pluggable transport as separate from having successfully used that + proxy or pluggable transport to connect to a relay. Closes tickets + 27100 and 28884. + + o Major features (circuit padding): + - Implement preliminary support for the circuit padding portion of + Proposal 254. The implementation supports Adaptive Padding (aka + WTF-PAD) state machines for use between experimental clients and + relays. Support is also provided for APE-style state machines that + use probability distributions instead of histograms to specify + inter-packet delay. At the moment, Tor does not provide any + padding state machines that are used in normal operation: for now, + this feature exists solely for experimentation. Closes + ticket 28142. + + o Major features (refactoring): + - Tor now uses an explicit list of its own subsystems when + initializing and shutting down. Previously, these systems were + managed implicitly in various places throughout the codebase. + (There may still be some subsystems using the old system.) Closes + ticket 28330. + + o Minor features (bootstrap reporting): + - When reporting bootstrap progress, stop distinguishing between + situations where only internal paths are available and situations + where external paths are available. Previously, Tor would often + erroneously report that it had only internal paths. Closes + ticket 27402. + + o Minor features (continuous integration): + - Log Python version during each Travis CI job. Resolves + issue 28551. + + o Minor features (controller): + - Add a DROPOWNERSHIP command to undo the effects of TAKEOWNERSHIP. + Implements ticket 28843. + + o Minor features (developer tooling): + - Provide a git hook script to prevent "fixup!" and "squash!" + commits from ending up in the master branch, as scripts/main/pre- + push.git-hook. Closes ticket 27993. + + o Minor features (directory authority): + - Directory authorities support a new consensus algorithm, under + which the family lines in microdescriptors are encoded in a + canonical form. This change makes family lines more compressible + in transit, and on the client. Closes ticket 28266; implements + proposal 298. + + o Minor features (directory authority, relay): + - Authorities now vote on a "StaleDesc" flag to indicate that a + relay's descriptor is so old that the relay should upload again + soon. Relays treat this flag as a signal to upload a new + descriptor. This flag will eventually let us remove the + 'published' date from routerstatus entries, and make our consensus + diffs much smaller. Closes ticket 26770; implements proposal 293. + + o Minor features (fallback directory mirrors): + - Update the fallback whitelist based on operator opt-ins and opt- + outs. Closes ticket 24805, patch by Phoul. + + o Minor features (FreeBSD): + - On FreeBSD-based systems, warn relay operators if the + "net.inet.ip.random_id" sysctl (IP ID randomization) is disabled. + Closes ticket 28518. + + o Minor features (HTTP standards compliance): + - Stop sending the header "Content-type: application/octet-stream" + along with transparently compressed documents: this confused + browsers. Closes ticket 28100. + + o Minor features (IPv6): + - We add an option ClientAutoIPv6ORPort, to make clients randomly + prefer a node's IPv4 or IPv6 ORPort. The random preference is set + every time a node is loaded from a new consensus or bridge config. + We expect that this option will enable clients to bootstrap more + quickly without having to determine whether they support IPv4, + IPv6, or both. Closes ticket 27490. Patch by Neel Chauhan. + - When using addrs_in_same_network_family(), avoid choosing circuit + paths that pass through the same IPv6 subnet more than once. + Previously, we only checked IPv4 subnets. Closes ticket 24393. + Patch by Neel Chauhan. + + o Minor features (log messages): + - Improve log message in v3 onion services that could print out + negative revision counters. Closes ticket 27707. Patch + by "ffmancera". + + o Minor features (memory usage): + - Save memory by storing microdescriptor family lists with a more + compact representation. Closes ticket 27359. + - Tor clients now use mmap() to read consensus files from disk, so + that they no longer need keep the full text of a consensus in + memory when parsing it or applying a diff. Closes ticket 27244. + + o Minor features (parsing): + - Directory authorities now validate that router descriptors and + ExtraInfo documents are in a valid subset of UTF-8, and reject + them if they are not. Closes ticket 27367. + + o Minor features (performance): + - Cache the results of summarize_protocol_flags(), so that we don't + have to parse the same protocol-versions string over and over. + This should save us a huge number of malloc calls on startup, and + may reduce memory fragmentation with some allocators. Closes + ticket 27225. + - Remove a needless memset() call from get_token_arguments, thereby + speeding up the tokenization of directory objects by about 20%. + Closes ticket 28852. + - Replace parse_short_policy() with a faster implementation, to + improve microdescriptor parsing time. Closes ticket 28853. + - Speed up directory parsing a little by avoiding use of the non- + inlined strcmp_len() function. Closes ticket 28856. + - Speed up microdescriptor parsing by about 30%, to help improve + startup time. Closes ticket 28839. + + o Minor features (pluggable transports): + - Add support for emitting STATUS updates to Tor's control port from + a pluggable transport process. Closes ticket 28846. + - Add support for logging to Tor's logging subsystem from a + pluggable transport process. Closes ticket 28180. + + o Minor features (process management): + - Add a new process API for handling child processes. This new API + allows Tor to have bi-directional communication with child + processes on both Unix and Windows. Closes ticket 28179. + - Use the subsystem manager to initialize and shut down the process + module. Closes ticket 28847. + + o Minor features (relay): + - When listing relay families, list them in canonical form including + the relay's own identity, and try to give a more useful set of + warnings. Part of ticket 28266 and proposal 298. + + o Minor features (required protocols): + - Before exiting because of a missing required protocol, Tor will + now check the publication time of the consensus, and not exit + unless the consensus is newer than the Tor program's own release + date. Previously, Tor would not check the consensus publication + time, and so might exit because of a missing protocol that might + no longer be required in a current consensus. Implements proposal + 297; closes ticket 27735. + + o Minor features (testing): + - Allow a HeartbeatPeriod of less than 30 minutes in testing Tor + networks. Closes ticket 28840. Patch by Rob Jansen. + + o Minor bugfixes (client, clock skew): + - Bootstrap successfully even when Tor's clock is behind the clocks + on the authorities. Fixes bug 28591; bugfix on 0.2.0.9-alpha. + - Select guards even if the consensus has expired, as long as the + consensus is still reasonably live. Fixes bug 24661; bugfix + on 0.3.0.1-alpha. + + o Minor bugfixes (compilation): + - Compile correctly on OpenBSD; previously, we were missing some + headers required in order to detect it properly. Fixes bug 28938; + bugfix on 0.3.5.1-alpha. Patch from Kris Katterjohn. + + o Minor bugfixes (directory clients): + - Mark outdated dirservers when Tor only has a reasonably live + consensus. Fixes bug 28569; bugfix on 0.3.2.5-alpha. + + o Minor bugfixes (directory mirrors): + - Even when a directory mirror's clock is behind the clocks on the + authorities, we now allow the mirror to serve "future" + consensuses. Fixes bug 28654; bugfix on 0.3.0.1-alpha. + + o Minor bugfixes (DNS): + - Gracefully handle an empty or absent resolve.conf file by falling + back to using "localhost" as a DNS server (and hoping it works). + Previously, we would just stop running as an exit. Fixes bug + 21900; bugfix on 0.2.1.10-alpha. + + o Minor bugfixes (guards): + - In count_acceptable_nodes(), the minimum number is now one bridge + or guard node, and two non-guard nodes for a circuit. Previously, + we had added up the sum of all nodes with a descriptor, but that + could cause us to build failing circuits when we had either too + many bridges or not enough guard nodes. Fixes bug 25885; bugfix on + 0.3.6.1-alpha. Patch by Neel Chauhan. + + o Minor bugfixes (IPv6): + - Fix tor_ersatz_socketpair on IPv6-only systems. Previously, the + IPv6 socket was bound using an address family of AF_INET instead + of AF_INET6. Fixes bug 28995; bugfix on 0.3.5.1-alpha. Patch from + Kris Katterjohn. + + o Minor bugfixes (logging): + - Rework rep_hist_log_link_protocol_counts() to iterate through all + link protocol versions when logging incoming/outgoing connection + counts. Tor no longer skips version 5, and we won't have to + remember to update this function when new link protocol version is + developed. Fixes bug 28920; bugfix on 0.2.6.10. + + o Minor bugfixes (networking): + - Introduce additional checks into tor_addr_parse() to reject + certain incorrect inputs that previously were not detected. Fixes + bug 23082; bugfix on 0.2.0.10-alpha. + + o Minor bugfixes (onion service v3, client): + - Stop logging a "BUG()" warning and stacktrace when we find a SOCKS + connection waiting for a descriptor that we actually have in the + cache. It turns out that this can actually happen, though it is + rare. Now, tor will recover and retry the descriptor. Fixes bug + 28669; bugfix on 0.3.2.4-alpha. + + o Minor bugfixes (periodic events): + - Refrain from calling routerlist_remove_old_routers() from + check_descriptor_callback(). Instead, create a new hourly periodic + event. Fixes bug 27929; bugfix on 0.2.8.1-alpha. + + o Minor bugfixes (pluggable transports): + - Make sure that data is continously read from standard output and + standard error pipes of a pluggable transport child-process, to + avoid deadlocking when a pipe's buffer is full. Fixes bug 26360; + bugfix on 0.2.3.6-alpha. + + o Minor bugfixes (unit tests): + - Instead of relying on hs_free_all() to clean up all onion service + objects in test_build_descriptors(), we now deallocate them one by + one. This lets Coverity know that we are not leaking memory there + and fixes CID 1442277. Fixes bug 28989; bugfix on 0.3.5.1-alpha. + + o Minor bugfixes (usability): + - Stop saying "Your Guard ..." in pathbias_measure_{use,close}_rate(). + Some users took this phrasing to mean that the mentioned guard was + under their control or responsibility, which it is not. Fixes bug + 28895; bugfix on Tor 0.3.0.1-alpha. + + o Code simplification and refactoring: + - Reimplement NETINFO cell parsing and generation to rely on + trunnel-generated wire format handling code. Closes ticket 27325. + - Remove unnecessary unsafe code from the Rust macro "cstr!". Closes + ticket 28077. + - Rework SOCKS wire format handling to rely on trunnel-generated + parsing/generation code. Resolves ticket 27620. + - Split out bootstrap progress reporting from control.c into a + separate file. Part of ticket 27402. + - The .may_include files that we use to describe our directory-by- + directory dependency structure now describe a noncircular + dependency graph over the directories that they cover. Our + checkIncludes.py tool now enforces this noncircularity. Closes + ticket 28362. + + o Documentation: + - Mention that you cannot add a new onion service if Tor is already + running with Sandbox enabled. Closes ticket 28560. + - Improve ControlPort documentation. Mention that it accepts + address:port pairs, and can be used multiple times. Closes + ticket 28805. + - Document the exact output of "tor --version". Closes ticket 28889. + + o Removed features: + - Stop responding to the 'GETINFO status/version/num-concurring' and + 'GETINFO status/version/num-versioning' control port commands, as + those were deprecated back in 0.2.0.30. Also stop listing them in + output of 'GETINFO info/names'. Resolves ticket 28757. + - The scripts used to generate and maintain the list of fallback + directories have been extracted into a new "fallback-scripts" + repository. Closes ticket 27914. + + o Testing: + - Run shellcheck for scripts in the in scripts/ directory. Closes + ticket 28058. + - Add unit tests for tokenize_string() and get_next_token() + functions. Resolves ticket 27625. + + o Code simplification and refactoring (onion service v3): + - Consolidate the authorized client descriptor cookie computation + code from client and service into one function. Closes + ticket 27549. + + o Code simplification and refactoring (shell scripts): + - Cleanup scan-build.sh to silence shellcheck warnings. Closes + ticket 28007. + - Fix issues that shellcheck found in chutney-git-bisect.sh. + Resolves ticket 28006. + - Fix issues that shellcheck found in updateRustDependencies.sh. + Resolves ticket 28012. + - Fix shellcheck warnings in cov-diff script. Resolves issue 28009. + - Fix shellcheck warnings in run_calltool.sh. Resolves ticket 28011. + - Fix shellcheck warnings in run_trunnel.sh. Resolves issue 28010. + - Fix shellcheck warnings in scripts/test/coverage. Resolves + issue 28008. + + +Changes in version 0.3.3.11 - 2019-01-07 + Tor 0.3.3.11 backports numerous fixes from later versions of Tor. + numerous fixes, including an important fix for anyone using OpenSSL + 1.1.1. Anyone running an earlier version of Tor 0.3.3 should upgrade + to this version, or to a later series. + + As a reminder, support the Tor 0.3.3 series will end on 22 Feb 2019. + We anticipate that this will be the last release of Tor 0.3.3, unless + some major bug is before then. Some time between now and then, users + should switch to either the Tor 0.3.4 series (supported until at least + 10 June 2019), or the Tor 0.3.5 series, which will receive long-term + support until at least 1 Feb 2022. + + o Major bugfixes (OpenSSL, portability, backport from 0.3.5.5-alpha): + - Fix our usage of named groups when running as a TLS 1.3 client in + OpenSSL 1.1.1. Previously, we only initialized EC groups when + running as a relay, which caused clients to fail to negotiate TLS + 1.3 with relays. Fixes bug 28245; bugfix on 0.2.9.15 (when TLS 1.3 + support was added). + + o Major bugfixes (restart-in-process, backport from 0.3.5.1-alpha): + - Fix a use-after-free error that could be caused by passing Tor an + impossible set of options that would fail during options_act(). + Fixes bug 27708; bugfix on 0.3.3.1-alpha. + + o Minor features (continuous integration, backport from 0.3.5.1-alpha): + - Only run one online rust build in Travis, to reduce network + errors. Skip offline rust builds on Travis for Linux gcc, because + they're redundant. Implements ticket 27252. + - Skip gcc on OSX in Travis CI, because it's rarely used. Skip a + duplicate hardening-off build in Travis on Tor 0.2.9. Skip gcc on + Linux with default settings, because all the non-default builds + use gcc on Linux. Implements ticket 27252. + + o Minor features (continuous integration, backport from 0.3.5.3-alpha): + - Use the Travis Homebrew addon to install packages on macOS during + Travis CI. The package list is the same, but the Homebrew addon + does not do a `brew update` by default. Implements ticket 27738. + + o Minor features (fallback directory list, backport from 0.3.5.6-rc): + - Replace the 150 fallbacks originally introduced in Tor + 0.3.3.1-alpha in January 2018 (of which ~115 were still + functional), with a list of 157 fallbacks (92 new, 65 existing, 85 + removed) generated in December 2018. Closes ticket 24803. + + o Minor features (geoip): + - Update geoip and geoip6 to the January 3 2019 Maxmind GeoLite2 + Country database. Closes ticket 29012. + + o Minor features (OpenSSL bug workaround, backport from 0.3.5.7): + - Work around a bug in OpenSSL 1.1.1a, which prevented the TLS 1.3 + key export function from handling long labels. When this bug is + detected, Tor will disable TLS 1.3. We recommend upgrading to a + version of OpenSSL without this bug when it becomes available. + Closes ticket 28973. + + o Minor bugfixes (relay statistics, backport from 0.3.5.7): + - Update relay descriptor on bandwidth changes only when the uptime + is smaller than 24h, in order to reduce the efficiency of guard + discovery attacks. Fixes bug 24104; bugfix on 0.1.1.6-alpha. + + o Minor bugfixes (C correctness, backport from 0.3.5.4-alpha): + - Avoid undefined behavior in an end-of-string check when parsing + the BEGIN line in a directory object. Fixes bug 28202; bugfix + on 0.2.0.3-alpha. + + o Minor bugfixes (code safety, backport from 0.3.5.3-alpha): + - Rewrite our assertion macros so that they no longer suppress the + compiler's -Wparentheses warnings. Fixes bug 27709; bugfix + + o Minor bugfixes (compilation, backport from 0.3.5.5-alpha): + - Initialize a variable unconditionally in aes_new_cipher(), since + some compilers cannot tell that we always initialize it before + use. Fixes bug 28413; bugfix on 0.2.9.3-alpha. + + o Minor bugfixes (directory authority, backport from 0.3.5.4-alpha): + - Log additional info when we get a relay that shares an ed25519 ID + with a different relay, instead making a BUG() warning. Fixes bug + 27800; bugfix on 0.3.2.1-alpha. + + o Minor bugfixes (directory permissions, backport form 0.3.5.3-alpha): + - When a user requests a group-readable DataDirectory, give it to + them. Previously, when the DataDirectory and the CacheDirectory + were the same, the default setting (0) for + CacheDirectoryGroupReadable would override the setting for + DataDirectoryGroupReadable. Fixes bug 26913; bugfix + on 0.3.3.1-alpha. + + o Minor bugfixes (onion service v3, backport from 0.3.5.1-alpha): + - When the onion service directory can't be created or has the wrong + permissions, do not log a stack trace. Fixes bug 27335; bugfix + on 0.3.2.1-alpha. + + o Minor bugfixes (onion service v3, backport from 0.3.5.2-alpha): + - Close all SOCKS request (for the same .onion) if the newly fetched + descriptor is unusable. Before that, we would close only the first + one leaving the other hanging and let to time out by themselves. + Fixes bug 27410; bugfix on 0.3.2.1-alpha. + + o Minor bugfixes (onion service v3, backport from 0.3.5.3-alpha): + - Don't warn so loudly when Tor is unable to decode an onion + descriptor. This can now happen as a normal use case if a client + gets a descriptor with client authorization but the client is not + authorized. Fixes bug 27550; bugfix on 0.3.5.1-alpha. + + o Minor bugfixes (onion service v3, backport from 0.3.5.6-rc): + - When deleting an ephemeral onion service (DEL_ONION), do not close + any rendezvous circuits in order to let the existing client + connections finish by themselves or closed by the application. The + HS v2 is doing that already so now we have the same behavior for + all versions. Fixes bug 28619; bugfix on 0.3.3.1-alpha. + + o Minor bugfixes (HTTP tunnel): + - Fix a bug warning when closing an HTTP tunnel connection due to + an HTTP request we couldn't handle. Fixes bug 26470; bugfix on + 0.3.2.1-alpha. + + o Minor bugfixes (memory leaks, backport from 0.3.5.5-alpha): + - Fix a harmless memory leak in libtorrunner.a. Fixes bug 28419; + bugfix on 0.3.3.1-alpha. Patch from Martin Kepplinger. + + o Minor bugfixes (netflow padding, backport from 0.3.5.1-alpha): + - Ensure circuitmux queues are empty before scheduling or sending + padding. Fixes bug 25505; bugfix on 0.3.1.1-alpha. + + o Minor bugfixes (protover, backport from 0.3.5.3-alpha): + - Reject protocol names containing bytes other than alphanumeric + characters and hyphens ([A-Za-z0-9-]). Fixes bug 27316; bugfix + on 0.2.9.4-alpha. + + o Minor bugfixes (rust, backport from 0.3.5.1-alpha): + - Compute protover votes correctly in the rust version of the + protover code. Previously, the protover rewrite in 24031 allowed + repeated votes from the same voter for the same protocol version + to be counted multiple times in protover_compute_vote(). Fixes bug + 27649; bugfix on 0.3.3.5-rc. + - Reject protover names that contain invalid characters. Fixes bug + 27687; bugfix on 0.3.3.1-alpha. + + o Minor bugfixes (rust, backport from 0.3.5.2-alpha): + - protover_all_supported() would attempt to allocate up to 16GB on + some inputs, leading to a potential memory DoS. Fixes bug 27206; + bugfix on 0.3.3.5-rc. + + o Minor bugfixes (rust, backport from 0.3.5.4-alpha): + - Fix a potential null dereference in protover_all_supported(). Add + a test for it. Fixes bug 27804; bugfix on 0.3.3.1-alpha. + - Return a string that can be safely freed by C code, not one + created by the rust allocator, in protover_all_supported(). Fixes + bug 27740; bugfix on 0.3.3.1-alpha. + - Fix an API mismatch in the rust implementation of + protover_compute_vote(). This bug could have caused crashes on any + directory authorities running Tor with Rust (which we do not yet + recommend). Fixes bug 27741; bugfix on 0.3.3.6. + + o Minor bugfixes (testing, backport from 0.3.5.1-alpha): + - If a unit test running in a subprocess exits abnormally or with a + nonzero status code, treat the test as having failed, even if the + test reported success. Without this fix, memory leaks don't cause + the tests to fail, even with LeakSanitizer. Fixes bug 27658; + bugfix on 0.2.2.4-alpha. + + o Minor bugfixes (testing, backport from 0.3.5.4-alpha): + - Treat backtrace test failures as expected on BSD-derived systems + (NetBSD, OpenBSD, and macOS/Darwin) until we solve bug 17808. + (FreeBSD failures have been treated as expected since 18204 in + 0.2.8.) Fixes bug 27948; bugfix on 0.2.5.2-alpha. + + o Minor bugfixes (unit tests, guard selection, backport from 0.3.5.6-rc): + - Stop leaking memory in an entry guard unit test. Fixes bug 28554; + bugfix on 0.3.0.1-alpha. + + +Changes in version 0.3.4.10 - 2019-01-07 + Tor 0.3.4.9 is the second stable release in its series; it backports + numerous fixes, including an important fix for relays, and for anyone + using OpenSSL 1.1.1. Anyone running an earlier version of Tor 0.3.4 + should upgrade. + + As a reminder, the Tor 0.3.4 series will be supported until 10 June + 2019. Some time between now and then, users should switch to the Tor + 0.3.5 series, which will receive long-term support until at least 1 + Feb 2022. + + o Major bugfixes (OpenSSL, portability, backport from 0.3.5.5-alpha): + - Fix our usage of named groups when running as a TLS 1.3 client in + OpenSSL 1.1.1. Previously, we only initialized EC groups when + running as a relay, which caused clients to fail to negotiate TLS + 1.3 with relays. Fixes bug 28245; bugfix on 0.2.9.15 (when TLS 1.3 + support was added). + + o Major bugfixes (relay, directory, backport from 0.3.5.7): + - Always reactivate linked connections in the main loop so long as + any linked connection has been active. Previously, connections + serving directory information wouldn't get reactivated after the + first chunk of data was sent (usually 32KB), which would prevent + clients from bootstrapping. Fixes bug 28912; bugfix on + 0.3.4.1-alpha. Patch by "cypherpunks3". + + o Minor features (continuous integration, Windows, backport from 0.3.5.6-rc): + - Always show the configure and test logs, and upload them as build + artifacts, when building for Windows using Appveyor CI. + Implements 28459. + + o Minor features (controller, backport from 0.3.5.1-alpha): + - For purposes of CIRC_BW-based dropped cell detection, track half- + closed stream ids, and allow their ENDs, SENDMEs, DATA and path + bias check cells to arrive without counting it as dropped until + either the END arrives, or the windows are empty. Closes + ticket 25573. + + o Minor features (fallback directory list, backport from 0.3.5.6-rc): + - Replace the 150 fallbacks originally introduced in Tor + 0.3.3.1-alpha in January 2018 (of which ~115 were still + functional), with a list of 157 fallbacks (92 new, 65 existing, 85 + removed) generated in December 2018. Closes ticket 24803. + + o Minor features (geoip): + - Update geoip and geoip6 to the November 6 2018 Maxmind GeoLite2 + Country database. Closes ticket 28395. + + o Minor features (OpenSSL bug workaround, backport from 0.3.5.7): + - Work around a bug in OpenSSL 1.1.1a, which prevented the TLS 1.3 + key export function from handling long labels. When this bug is + detected, Tor will disable TLS 1.3. We recommend upgrading to a + version of OpenSSL without this bug when it becomes available. + Closes ticket 28973. + + o Minor bugfixes (compilation, backport from 0.3.5.5-alpha): + - Initialize a variable unconditionally in aes_new_cipher(), since + some compilers cannot tell that we always initialize it before + use. Fixes bug 28413; bugfix on 0.2.9.3-alpha. + + o Minor bugfixes (connection, relay, backport from 0.3.5.5-alpha): + - Avoid a logging a BUG() stacktrace when closing connection held + open because the write side is rate limited but not the read side. + Now, the connection read side is simply shut down until Tor is + able to flush the connection and close it. Fixes bug 27750; bugfix + on 0.3.4.1-alpha. + + o Minor bugfixes (continuous integration, Windows, backport from 0.3.5.5-alpha): + - Manually configure the zstd compiler options, when building using + mingw on Appveyor Windows CI. The MSYS2 mingw zstd package does + not come with a pkg-config file. Fixes bug 28454; bugfix + on 0.3.4.1-alpha. + - Stop using an external OpenSSL install, and stop installing MSYS2 + packages, when building using mingw on Appveyor Windows CI. Fixes + bug 28399; bugfix on 0.3.4.1-alpha. + + o Minor bugfixes (continuous integration, Windows, backport from 0.3.5.6-rc): + - Explicitly specify the path to the OpenSSL library and do not + download OpenSSL from Pacman, but instead use the library that is + already provided by AppVeyor. Fixes bug 28574; bugfix on master. + + o Minor bugfixes (directory permissions, backport form 0.3.5.3-alpha): + - When a user requests a group-readable DataDirectory, give it to + them. Previously, when the DataDirectory and the CacheDirectory + were the same, the default setting (0) for + CacheDirectoryGroupReadable would override the setting for + DataDirectoryGroupReadable. Fixes bug 26913; bugfix + on 0.3.3.1-alpha. + + o Minor bugfixes (memory leaks, backport from 0.3.5.5-alpha): + - Fix a harmless memory leak in libtorrunner.a. Fixes bug 28419; + bugfix on 0.3.3.1-alpha. Patch from Martin Kepplinger. + + o Minor bugfixes (onion service v3, backport from 0.3.5.3-alpha): + - Don't warn so loudly when Tor is unable to decode an onion + descriptor. This can now happen as a normal use case if a client + gets a descriptor with client authorization but the client is not + authorized. Fixes bug 27550; bugfix on 0.3.5.1-alpha. + + o Minor bugfixes (onion service v3, backport from 0.3.5.6-rc): + - When deleting an ephemeral onion service (DEL_ONION), do not close + any rendezvous circuits in order to let the existing client + connections finish by themselves or closed by the application. The + HS v2 is doing that already so now we have the same behavior for + all versions. Fixes bug 28619; bugfix on 0.3.3.1-alpha. + + o Minor bugfixes (relay statistics, backport from 0.3.5.7): + - Update relay descriptor on bandwidth changes only when the uptime + is smaller than 24h, in order to reduce the efficiency of guard + discovery attacks. Fixes bug 24104; bugfix on 0.1.1.6-alpha. + + o Minor bugfixes (unit tests, guard selection, backport from 0.3.5.6-rc): + - Stop leaking memory in an entry guard unit test. Fixes bug 28554; + bugfix on 0.3.0.1-alpha. + + +Changes in version 0.3.5.7 - 2019-01-07 + Tor 0.3.5.7 is the first stable release in its series; it includes + compilation and portability fixes, and a fix for a severe problem + affecting directory caches. + + The Tor 0.3.5 series includes several new features and performance + improvements, including client authorization for v3 onion services, + cleanups to bootstrap reporting, support for improved bandwidth- + measurement tools, experimental support for NSS in place of OpenSSL, + and much more. It also begins a full reorganization of Tor's code + layout, for improved modularity and maintainability in the future. + Finally, there is the usual set of performance improvements and + bugfixes that we try to do in every release series. + + There are a couple of changes in the 0.3.5 that may affect + compatibility. First, the default version for newly created onion + services is now v3. Use the HiddenServiceVersion option if you want to + override this. Second, some log messages related to bootstrapping have + changed; if you use stem, you may need to update to the latest version + so it will recognize them. + + We have designated 0.3.5 as a "long-term support" (LTS) series: we + will continue to patch major bugs in typical configurations of 0.3.5 + until at least 1 Feb 2022. (We do not plan to provide long-term + support for embedding, Rust support, NSS support, running a directory + authority, or unsupported platforms. For these, you will need to stick + with the latest stable release.) + + Below are the changes since 0.3.5.6-rc. For a complete list of changes + since 0.3.4.9, see the ReleaseNotes file. + + o Major bugfixes (relay, directory): + - Always reactivate linked connections in the main loop so long as + any linked connection has been active. Previously, connections + serving directory information wouldn't get reactivated after the + first chunk of data was sent (usually 32KB), which would prevent + clients from bootstrapping. Fixes bug 28912; bugfix on + 0.3.4.1-alpha. Patch by "cypherpunks3". + + o Minor features (compilation): + - When possible, place our warning flags in a separate file, to + avoid flooding verbose build logs. Closes ticket 28924. + + o Minor features (geoip): + - Update geoip and geoip6 to the January 3 2019 Maxmind GeoLite2 + Country database. Closes ticket 29012. + + o Minor features (OpenSSL bug workaround): + - Work around a bug in OpenSSL 1.1.1a, which prevented the TLS 1.3 + key export function from handling long labels. When this bug is + detected, Tor will disable TLS 1.3. We recommend upgrading to a + version of OpenSSL without this bug when it becomes available. + Closes ticket 28973. + + o Minor features (performance): + - Remove about 96% of the work from the function that we run at + startup to test our curve25519_basepoint implementation. Since + this function has yet to find an actual failure, we now only run + it for 8 iterations instead of 200. Based on our profile + information, this change should save around 8% of our startup time + on typical desktops, and may have a similar effect on other + platforms. Closes ticket 28838. + - Stop re-validating our hardcoded Diffie-Hellman parameters on + every startup. Doing this wasted time and cycles, especially on + low-powered devices. Closes ticket 28851. + + o Minor bugfixes (compilation): + - Fix compilation for Android by adding a missing header to + freespace.c. Fixes bug 28974; bugfix on 0.3.5.1-alpha. + + o Minor bugfixes (correctness): + - Fix an unreached code path where we checked the value of + "hostname" inside send_resolved_hostname_cell(). Previously, we + used it before checking it; now we check it first. Fixes bug + 28879; bugfix on 0.1.2.7-alpha. + + o Minor bugfixes (testing): + - Make sure that test_rebind.py actually obeys its timeout, even + when it receives a large number of log messages. Fixes bug 28883; + bugfix on 0.3.5.4-alpha. + - Stop running stem's unit tests as part of "make test-stem", but + continue to run stem's unit and online tests during "make test- + stem-full". Fixes bug 28568; bugfix on 0.2.6.3-alpha. + + o Minor bugfixes (windows services): + - Make Tor start correctly as an NT service again: previously it was + broken by refactoring. Fixes bug 28612; bugfix on 0.3.5.3-alpha. + + o Code simplification and refactoring: + - When parsing a port configuration, make it more obvious to static + analyzer tools that we always initialize the address. Closes + ticket 28881. + + +Changes in version 0.3.5.6-rc - 2018-12-18 + Tor 0.3.5.6-rc fixes numerous small bugs in earlier versions of Tor. + It is the first release candidate in the 0.3.5.x series; if no further + huge bugs are found, our next release may be the stable 0.3.5.x. + + o Minor features (continuous integration, Windows): + - Always show the configure and test logs, and upload them as build + artifacts, when building for Windows using Appveyor CI. + Implements 28459. + + o Minor features (fallback directory list): + - Replace the 150 fallbacks originally introduced in Tor + 0.3.3.1-alpha in January 2018 (of which ~115 were still + functional), with a list of 157 fallbacks (92 new, 65 existing, 85 + removed) generated in December 2018. Closes ticket 24803. + + o Minor features (geoip): + - Update geoip and geoip6 to the December 5 2018 Maxmind GeoLite2 + Country database. Closes ticket 28744. + + o Minor bugfixes (compilation): + - Add missing dependency on libgdi32.dll for tor-print-ed-signing- + cert.exe on Windows. Fixes bug 28485; bugfix on 0.3.5.1-alpha. + + o Minor bugfixes (continuous integration, Windows): + - Explicitly specify the path to the OpenSSL library and do not + download OpenSSL from Pacman, but instead use the library that is + already provided by AppVeyor. Fixes bug 28574; bugfix on master. + + o Minor bugfixes (onion service v3): + - When deleting an ephemeral onion service (DEL_ONION), do not close + any rendezvous circuits in order to let the existing client + connections finish by themselves or closed by the application. The + HS v2 is doing that already so now we have the same behavior for + all versions. Fixes bug 28619; bugfix on 0.3.3.1-alpha. + + o Minor bugfixes (restart-in-process, boostrap): + - Add missing resets of bootstrap tracking state when shutting down + (regression caused by ticket 27169). Fixes bug 28524; bugfix + on 0.3.5.1-alpha. + + o Minor bugfixes (testing): + - Use a separate DataDirectory for the test_rebind script. + Previously, this script would run using the default DataDirectory, + and sometimes fail. Fixes bug 28562; bugfix on 0.3.5.1-alpha. + Patch from Taylor R Campbell. + - Stop leaking memory in an entry guard unit test. Fixes bug 28554; + bugfix on 0.3.0.1-alpha. + + o Minor bugfixes (Windows): + - Correctly identify Windows 8.1, Windows 10, and Windows Server + 2008 and later from their NT versions. Fixes bug 28096; bugfix on + 0.2.2.34; reported by Keifer Bly. + - On recent Windows versions, the GetVersionEx() function may report + an earlier Windows version than the running OS. To avoid user + confusion, add "[or later]" to Tor's version string on affected + versions of Windows. Fixes bug 28096; bugfix on 0.2.2.34; reported + by Keifer Bly. + - Remove Windows versions that were never supported by the + GetVersionEx() function. Stop duplicating the latest Windows + version in get_uname(). Fixes bug 28096; bugfix on 0.2.2.34; + reported by Keifer Bly. + + o Testing: + - Increase logging and tag all log entries with timestamps in + test_rebind.py. Provides diagnostics for issue 28229. + + o Code simplification and refactoring (shared random, dirauth): + - Change many tor_assert() to use BUG() instead. The idea is to not + crash a dirauth but rather scream loudly with a stacktrace and let + it continue run. The shared random subsystem is very resilient and + if anything wrong happens with it, at worst a non coherent value + will be put in the vote and discarded by the other authorities. + Closes ticket 19566. + + o Documentation (onion services): + - Document in the man page that changing ClientOnionAuthDir value or + adding a new file in the directory will not work at runtime upon + sending a HUP if Sandbox 1. Closes ticket 28128. + - Note in the man page that the only real way to fully revoke an + onion service v3 client authorization is by restarting the tor + process. Closes ticket 28275. + + +Changes in version 0.3.5.5-alpha - 2018-11-16 + Tor 0.3.5.5-alpha includes numerous bugfixes on earlier releases, + including several that we hope to backport to older release series in + the future. + + o Major bugfixes (OpenSSL, portability): + - Fix our usage of named groups when running as a TLS 1.3 client in + OpenSSL 1.1.1. Previously, we only initialized EC groups when + running as a relay, which caused clients to fail to negotiate TLS + 1.3 with relays. Fixes bug 28245; bugfix on 0.2.9.15 (when TLS 1.3 + support was added). + + o Minor features (geoip): + - Update geoip and geoip6 to the November 6 2018 Maxmind GeoLite2 + Country database. Closes ticket 28395. + + o Minor bugfixes (compilation): + - Initialize a variable unconditionally in aes_new_cipher(), since + some compilers cannot tell that we always initialize it before + use. Fixes bug 28413; bugfix on 0.2.9.3-alpha. + + o Minor bugfixes (connection, relay): + - Avoid a logging a BUG() stacktrace when closing connection held + open because the write side is rate limited but not the read side. + Now, the connection read side is simply shut down until Tor is + able to flush the connection and close it. Fixes bug 27750; bugfix + on 0.3.4.1-alpha. + + o Minor bugfixes (continuous integration, Windows): + - Manually configure the zstd compiler options, when building using + mingw on Appveyor Windows CI. The MSYS2 mingw zstd package does + not come with a pkg-config file. Fixes bug 28454; bugfix + on 0.3.4.1-alpha. + - Stop using an external OpenSSL install, and stop installing MSYS2 + packages, when building using mingw on Appveyor Windows CI. Fixes + bug 28399; bugfix on 0.3.4.1-alpha. + + o Minor bugfixes (documentation): + - Make Doxygen work again after the code movement in the 0.3.5 + source tree. Fixes bug 28435; bugfix on 0.3.5.1-alpha. + + o Minor bugfixes (Linux seccomp2 sandbox): + - Permit the "shutdown()" system call, which is apparently used by + OpenSSL under some circumstances. Fixes bug 28183; bugfix + on 0.2.5.1-alpha. + + o Minor bugfixes (logging): + - Stop talking about the Named flag in log messages. Clients have + ignored the Named flag since 0.3.2. Fixes bug 28441; bugfix + on 0.3.2.1-alpha. + + o Minor bugfixes (memory leaks): + - Fix a harmless memory leak in libtorrunner.a. Fixes bug 28419; + bugfix on 0.3.3.1-alpha. Patch from Martin Kepplinger. + + o Minor bugfixes (onion services): + - On an intro point for a version 3 onion service, stop closing + introduction circuits on an NACK. This lets the client decide + whether to reuse the circuit or discard it. Previously, we closed + intro circuits when sending NACKs. Fixes bug 27841; bugfix on + 0.3.2.1-alpha. Patch by Neel Chaunan. + - When replacing a descriptor in the client cache, make sure to + close all client introduction circuits for the old descriptor, so + we don't end up with unusable leftover circuits. Fixes bug 27471; + bugfix on 0.3.2.1-alpha. + + Changes in version 0.3.5.4-alpha - 2018-11-08 Tor 0.3.5.4-alpha includes numerous bugfixes on earlier versions and improves our continuous integration support. It continues our attempts diff --git a/Makefile.am b/Makefile.am index 5d18666edc..4e5681fb41 100644 --- a/Makefile.am +++ b/Makefile.am @@ -31,9 +31,7 @@ TESTING_TOR_BINARY=$(top_builddir)/src/app/tor$(EXEEXT) endif if USE_RUST -## this MUST be $(), otherwise am__DEPENDENCIES will not track it -rust_ldadd=$(top_builddir)/$(TOR_RUST_LIB_PATH) \ - $(TOR_RUST_EXTRA_LIBS) +rust_ldadd=$(top_builddir)/$(TOR_RUST_LIB_PATH) else rust_ldadd= endif @@ -42,6 +40,7 @@ endif TOR_UTIL_LIBS = \ src/lib/libtor-geoip.a \ src/lib/libtor-process.a \ + src/lib/libtor-buf.a \ src/lib/libtor-time.a \ src/lib/libtor-fs.a \ src/lib/libtor-encoding.a \ @@ -62,6 +61,7 @@ TOR_UTIL_LIBS = \ src/lib/libtor-malloc.a \ src/lib/libtor-wallclock.a \ src/lib/libtor-err.a \ + src/lib/libtor-version.a \ src/lib/libtor-intmath.a \ src/lib/libtor-ctime.a @@ -71,6 +71,7 @@ if UNITTESTS_ENABLED TOR_UTIL_TESTING_LIBS = \ src/lib/libtor-geoip-testing.a \ src/lib/libtor-process-testing.a \ + src/lib/libtor-buf-testing.a \ src/lib/libtor-time-testing.a \ src/lib/libtor-fs-testing.a \ src/lib/libtor-encoding-testing.a \ @@ -91,6 +92,7 @@ TOR_UTIL_TESTING_LIBS = \ src/lib/libtor-malloc-testing.a \ src/lib/libtor-wallclock-testing.a \ src/lib/libtor-err-testing.a \ + src/lib/libtor-version-testing.a \ src/lib/libtor-intmath.a \ src/lib/libtor-ctime-testing.a endif @@ -213,7 +215,16 @@ doxygen: test: all $(top_builddir)/src/test/test -check-local: check-spaces check-changes check-includes +shellcheck: + # Only use shellcheck if it is present + if command -v shellcheck; then \ + find $(top_srcdir)/scripts/ -name "*.sh" -exec shellcheck {} +; \ + if [ -d "$(top_srcdir)/scripts/test" ]; then \ + shellcheck $(top_srcdir)/scripts/test/cov-diff $(top_srcdir)/scripts/test/coverage; \ + fi; \ + fi + +check-local: check-spaces check-changes check-includes shellcheck need-chutney-path: @if test ! -d "$$CHUTNEY_PATH"; then \ @@ -412,13 +423,13 @@ endif check-changes: if USEPYTHON @if test -d "$(top_srcdir)/changes"; then \ - $(PYTHON) $(top_srcdir)/scripts/maint/lintChanges.py $(top_srcdir)/changes; \ + PACKAGE_VERSION=$(PACKAGE_VERSION) $(PYTHON) $(top_srcdir)/scripts/maint/lintChanges.py $(top_srcdir)/changes; \ fi endif .PHONY: update-versions update-versions: - $(PERL) $(top_builddir)/scripts/maint/updateVersions.pl + abs_top_srcdir="$(abs_top_srcdir)" $(PYTHON) $(top_srcdir)/scripts/maint/update_versions.py .PHONY: callgraph callgraph: diff --git a/ReleaseNotes b/ReleaseNotes index e0a25a74b7..6c9aa3c294 100644 --- a/ReleaseNotes +++ b/ReleaseNotes @@ -2,6 +2,1023 @@ This document summarizes new features and bugfixes in each stable release of Tor. If you want to see more detailed descriptions of the changes in each development snapshot, see the ChangeLog file. +Changes in version 0.3.3.11 - 2019-01-07 + Tor 0.3.3.11 backports numerous fixes from later versions of Tor. + numerous fixes, including an important fix for anyone using OpenSSL + 1.1.1. Anyone running an earlier version of Tor 0.3.3 should upgrade + to this version, or to a later series. + + As a reminder, support the Tor 0.3.3 series will end on 22 Feb 2019. + We anticipate that this will be the last release of Tor 0.3.3, unless + some major bug is before then. Some time between now and then, users + should switch to either the Tor 0.3.4 series (supported until at least + 10 June 2019), or the Tor 0.3.5 series, which will receive long-term + support until at least 1 Feb 2022. + + o Major bugfixes (OpenSSL, portability, backport from 0.3.5.5-alpha): + - Fix our usage of named groups when running as a TLS 1.3 client in + OpenSSL 1.1.1. Previously, we only initialized EC groups when + running as a relay, which caused clients to fail to negotiate TLS + 1.3 with relays. Fixes bug 28245; bugfix on 0.2.9.15 (when TLS 1.3 + support was added). + + o Major bugfixes (restart-in-process, backport from 0.3.5.1-alpha): + - Fix a use-after-free error that could be caused by passing Tor an + impossible set of options that would fail during options_act(). + Fixes bug 27708; bugfix on 0.3.3.1-alpha. + + o Minor features (continuous integration, backport from 0.3.5.1-alpha): + - Only run one online rust build in Travis, to reduce network + errors. Skip offline rust builds on Travis for Linux gcc, because + they're redundant. Implements ticket 27252. + - Skip gcc on OSX in Travis CI, because it's rarely used. Skip a + duplicate hardening-off build in Travis on Tor 0.2.9. Skip gcc on + Linux with default settings, because all the non-default builds + use gcc on Linux. Implements ticket 27252. + + o Minor features (continuous integration, backport from 0.3.5.3-alpha): + - Use the Travis Homebrew addon to install packages on macOS during + Travis CI. The package list is the same, but the Homebrew addon + does not do a `brew update` by default. Implements ticket 27738. + + o Minor features (fallback directory list, backport from 0.3.5.6-rc): + - Replace the 150 fallbacks originally introduced in Tor + 0.3.3.1-alpha in January 2018 (of which ~115 were still + functional), with a list of 157 fallbacks (92 new, 65 existing, 85 + removed) generated in December 2018. Closes ticket 24803. + + o Minor features (geoip): + - Update geoip and geoip6 to the January 3 2019 Maxmind GeoLite2 + Country database. Closes ticket 29012. + + o Minor features (OpenSSL bug workaround, backport from 0.3.5.7): + - Work around a bug in OpenSSL 1.1.1a, which prevented the TLS 1.3 + key export function from handling long labels. When this bug is + detected, Tor will disable TLS 1.3. We recommend upgrading to a + version of OpenSSL without this bug when it becomes available. + Closes ticket 28973. + + o Minor bugfixes (relay statistics, backport from 0.3.5.7): + - Update relay descriptor on bandwidth changes only when the uptime + is smaller than 24h, in order to reduce the efficiency of guard + discovery attacks. Fixes bug 24104; bugfix on 0.1.1.6-alpha. + + o Minor bugfixes (C correctness, backport from 0.3.5.4-alpha): + - Avoid undefined behavior in an end-of-string check when parsing + the BEGIN line in a directory object. Fixes bug 28202; bugfix + on 0.2.0.3-alpha. + + o Minor bugfixes (code safety, backport from 0.3.5.3-alpha): + - Rewrite our assertion macros so that they no longer suppress the + compiler's -Wparentheses warnings. Fixes bug 27709; bugfix + + o Minor bugfixes (compilation, backport from 0.3.5.5-alpha): + - Initialize a variable unconditionally in aes_new_cipher(), since + some compilers cannot tell that we always initialize it before + use. Fixes bug 28413; bugfix on 0.2.9.3-alpha. + + o Minor bugfixes (directory authority, backport from 0.3.5.4-alpha): + - Log additional info when we get a relay that shares an ed25519 ID + with a different relay, instead making a BUG() warning. Fixes bug + 27800; bugfix on 0.3.2.1-alpha. + + o Minor bugfixes (directory permissions, backport form 0.3.5.3-alpha): + - When a user requests a group-readable DataDirectory, give it to + them. Previously, when the DataDirectory and the CacheDirectory + were the same, the default setting (0) for + CacheDirectoryGroupReadable would override the setting for + DataDirectoryGroupReadable. Fixes bug 26913; bugfix + on 0.3.3.1-alpha. + + o Minor bugfixes (onion service v3, backport from 0.3.5.1-alpha): + - When the onion service directory can't be created or has the wrong + permissions, do not log a stack trace. Fixes bug 27335; bugfix + on 0.3.2.1-alpha. + + o Minor bugfixes (onion service v3, backport from 0.3.5.2-alpha): + - Close all SOCKS request (for the same .onion) if the newly fetched + descriptor is unusable. Before that, we would close only the first + one leaving the other hanging and let to time out by themselves. + Fixes bug 27410; bugfix on 0.3.2.1-alpha. + + o Minor bugfixes (onion service v3, backport from 0.3.5.3-alpha): + - Don't warn so loudly when Tor is unable to decode an onion + descriptor. This can now happen as a normal use case if a client + gets a descriptor with client authorization but the client is not + authorized. Fixes bug 27550; bugfix on 0.3.5.1-alpha. + + o Minor bugfixes (onion service v3, backport from 0.3.5.6-rc): + - When deleting an ephemeral onion service (DEL_ONION), do not close + any rendezvous circuits in order to let the existing client + connections finish by themselves or closed by the application. The + HS v2 is doing that already so now we have the same behavior for + all versions. Fixes bug 28619; bugfix on 0.3.3.1-alpha. + + o Minor bugfixes (HTTP tunnel): + - Fix a bug warning when closing an HTTP tunnel connection due to + an HTTP request we couldn't handle. Fixes bug 26470; bugfix on + 0.3.2.1-alpha. + + o Minor bugfixes (memory leaks, backport from 0.3.5.5-alpha): + - Fix a harmless memory leak in libtorrunner.a. Fixes bug 28419; + bugfix on 0.3.3.1-alpha. Patch from Martin Kepplinger. + + o Minor bugfixes (netflow padding, backport from 0.3.5.1-alpha): + - Ensure circuitmux queues are empty before scheduling or sending + padding. Fixes bug 25505; bugfix on 0.3.1.1-alpha. + + o Minor bugfixes (protover, backport from 0.3.5.3-alpha): + - Reject protocol names containing bytes other than alphanumeric + characters and hyphens ([A-Za-z0-9-]). Fixes bug 27316; bugfix + on 0.2.9.4-alpha. + + o Minor bugfixes (rust, backport from 0.3.5.1-alpha): + - Compute protover votes correctly in the rust version of the + protover code. Previously, the protover rewrite in 24031 allowed + repeated votes from the same voter for the same protocol version + to be counted multiple times in protover_compute_vote(). Fixes bug + 27649; bugfix on 0.3.3.5-rc. + - Reject protover names that contain invalid characters. Fixes bug + 27687; bugfix on 0.3.3.1-alpha. + + o Minor bugfixes (rust, backport from 0.3.5.2-alpha): + - protover_all_supported() would attempt to allocate up to 16GB on + some inputs, leading to a potential memory DoS. Fixes bug 27206; + bugfix on 0.3.3.5-rc. + + o Minor bugfixes (rust, backport from 0.3.5.4-alpha): + - Fix a potential null dereference in protover_all_supported(). Add + a test for it. Fixes bug 27804; bugfix on 0.3.3.1-alpha. + - Return a string that can be safely freed by C code, not one + created by the rust allocator, in protover_all_supported(). Fixes + bug 27740; bugfix on 0.3.3.1-alpha. + - Fix an API mismatch in the rust implementation of + protover_compute_vote(). This bug could have caused crashes on any + directory authorities running Tor with Rust (which we do not yet + recommend). Fixes bug 27741; bugfix on 0.3.3.6. + + o Minor bugfixes (testing, backport from 0.3.5.1-alpha): + - If a unit test running in a subprocess exits abnormally or with a + nonzero status code, treat the test as having failed, even if the + test reported success. Without this fix, memory leaks don't cause + the tests to fail, even with LeakSanitizer. Fixes bug 27658; + bugfix on 0.2.2.4-alpha. + + o Minor bugfixes (testing, backport from 0.3.5.4-alpha): + - Treat backtrace test failures as expected on BSD-derived systems + (NetBSD, OpenBSD, and macOS/Darwin) until we solve bug 17808. + (FreeBSD failures have been treated as expected since 18204 in + 0.2.8.) Fixes bug 27948; bugfix on 0.2.5.2-alpha. + + o Minor bugfixes (unit tests, guard selection, backport from 0.3.5.6-rc): + - Stop leaking memory in an entry guard unit test. Fixes bug 28554; + bugfix on 0.3.0.1-alpha. + + +Changes in version 0.3.4.10 - 2019-01-07 + Tor 0.3.4.9 is the second stable release in its series; it backports + numerous fixes, including an important fix for relays, and for anyone + using OpenSSL 1.1.1. Anyone running an earlier version of Tor 0.3.4 + should upgrade. + + As a reminder, the Tor 0.3.4 series will be supported until 10 June + 2019. Some time between now and then, users should switch to the Tor + 0.3.5 series, which will receive long-term support until at least 1 + Feb 2022. + + o Major bugfixes (OpenSSL, portability, backport from 0.3.5.5-alpha): + - Fix our usage of named groups when running as a TLS 1.3 client in + OpenSSL 1.1.1. Previously, we only initialized EC groups when + running as a relay, which caused clients to fail to negotiate TLS + 1.3 with relays. Fixes bug 28245; bugfix on 0.2.9.15 (when TLS 1.3 + support was added). + + o Major bugfixes (relay, directory, backport from 0.3.5.7): + - Always reactivate linked connections in the main loop so long as + any linked connection has been active. Previously, connections + serving directory information wouldn't get reactivated after the + first chunk of data was sent (usually 32KB), which would prevent + clients from bootstrapping. Fixes bug 28912; bugfix on + 0.3.4.1-alpha. Patch by "cypherpunks3". + + o Minor features (continuous integration, Windows, backport from 0.3.5.6-rc): + - Always show the configure and test logs, and upload them as build + artifacts, when building for Windows using Appveyor CI. + Implements 28459. + + o Minor features (controller, backport from 0.3.5.1-alpha): + - For purposes of CIRC_BW-based dropped cell detection, track half- + closed stream ids, and allow their ENDs, SENDMEs, DATA and path + bias check cells to arrive without counting it as dropped until + either the END arrives, or the windows are empty. Closes + ticket 25573. + + o Minor features (fallback directory list, backport from 0.3.5.6-rc): + - Replace the 150 fallbacks originally introduced in Tor + 0.3.3.1-alpha in January 2018 (of which ~115 were still + functional), with a list of 157 fallbacks (92 new, 65 existing, 85 + removed) generated in December 2018. Closes ticket 24803. + + o Minor features (geoip): + - Update geoip and geoip6 to the November 6 2018 Maxmind GeoLite2 + Country database. Closes ticket 28395. + + o Minor features (OpenSSL bug workaround, backport from 0.3.5.7): + - Work around a bug in OpenSSL 1.1.1a, which prevented the TLS 1.3 + key export function from handling long labels. When this bug is + detected, Tor will disable TLS 1.3. We recommend upgrading to a + version of OpenSSL without this bug when it becomes available. + Closes ticket 28973. + + o Minor bugfixes (compilation, backport from 0.3.5.5-alpha): + - Initialize a variable unconditionally in aes_new_cipher(), since + some compilers cannot tell that we always initialize it before + use. Fixes bug 28413; bugfix on 0.2.9.3-alpha. + + o Minor bugfixes (connection, relay, backport from 0.3.5.5-alpha): + - Avoid a logging a BUG() stacktrace when closing connection held + open because the write side is rate limited but not the read side. + Now, the connection read side is simply shut down until Tor is + able to flush the connection and close it. Fixes bug 27750; bugfix + on 0.3.4.1-alpha. + + o Minor bugfixes (continuous integration, Windows, backport from 0.3.5.5-alpha): + - Manually configure the zstd compiler options, when building using + mingw on Appveyor Windows CI. The MSYS2 mingw zstd package does + not come with a pkg-config file. Fixes bug 28454; bugfix + on 0.3.4.1-alpha. + - Stop using an external OpenSSL install, and stop installing MSYS2 + packages, when building using mingw on Appveyor Windows CI. Fixes + bug 28399; bugfix on 0.3.4.1-alpha. + + o Minor bugfixes (continuous integration, Windows, backport from 0.3.5.6-rc): + - Explicitly specify the path to the OpenSSL library and do not + download OpenSSL from Pacman, but instead use the library that is + already provided by AppVeyor. Fixes bug 28574; bugfix on master. + + o Minor bugfixes (directory permissions, backport form 0.3.5.3-alpha): + - When a user requests a group-readable DataDirectory, give it to + them. Previously, when the DataDirectory and the CacheDirectory + were the same, the default setting (0) for + CacheDirectoryGroupReadable would override the setting for + DataDirectoryGroupReadable. Fixes bug 26913; bugfix + on 0.3.3.1-alpha. + + o Minor bugfixes (memory leaks, backport from 0.3.5.5-alpha): + - Fix a harmless memory leak in libtorrunner.a. Fixes bug 28419; + bugfix on 0.3.3.1-alpha. Patch from Martin Kepplinger. + + o Minor bugfixes (onion service v3, backport from 0.3.5.3-alpha): + - Don't warn so loudly when Tor is unable to decode an onion + descriptor. This can now happen as a normal use case if a client + gets a descriptor with client authorization but the client is not + authorized. Fixes bug 27550; bugfix on 0.3.5.1-alpha. + + o Minor bugfixes (onion service v3, backport from 0.3.5.6-rc): + - When deleting an ephemeral onion service (DEL_ONION), do not close + any rendezvous circuits in order to let the existing client + connections finish by themselves or closed by the application. The + HS v2 is doing that already so now we have the same behavior for + all versions. Fixes bug 28619; bugfix on 0.3.3.1-alpha. + + o Minor bugfixes (relay statistics, backport from 0.3.5.7): + - Update relay descriptor on bandwidth changes only when the uptime + is smaller than 24h, in order to reduce the efficiency of guard + discovery attacks. Fixes bug 24104; bugfix on 0.1.1.6-alpha. + + o Minor bugfixes (unit tests, guard selection, backport from 0.3.5.6-rc): + - Stop leaking memory in an entry guard unit test. Fixes bug 28554; + bugfix on 0.3.0.1-alpha. + + +Changes in version 0.3.5.7 - 2019-01-07 + Tor 0.3.5.7 is the first stable release in its series; it includes + compilation and portability fixes, and a fix for a severe problem + affecting directory caches. + + The Tor 0.3.5 series includes several new features and performance + improvements, including client authorization for v3 onion services, + cleanups to bootstrap reporting, support for improved bandwidth- + measurement tools, experimental support for NSS in place of OpenSSL, + and much more. It also begins a full reorganization of Tor's code + layout, for improved modularity and maintainability in the future. + Finally, there is the usual set of performance improvements and + bugfixes that we try to do in every release series. + + There are a couple of changes in the 0.3.5 that may affect + compatibility. First, the default version for newly created onion + services is now v3. Use the HiddenServiceVersion option if you want to + override this. Second, some log messages related to bootstrapping have + changed; if you use stem, you may need to update to the latest version + so it will recognize them. + + We have designated 0.3.5 as a "long-term support" (LTS) series: we + will continue to patch major bugs in typical configurations of 0.3.5 + until at least 1 Feb 2022. (We do not plan to provide long-term + support for embedding, Rust support, NSS support, running a directory + authority, or unsupported platforms. For these, you will need to stick + with the latest stable release.) + + Below are the changes since 0.3.4.9. For a complete list of changes + since 0.3.5.6-rc, see the ChangeLog file. + + o Major features (bootstrap): + - Don't report directory progress until after a connection to a + relay or bridge has succeeded. Previously, we'd report 80% + progress based on cached directory information when we couldn't + even connect to the network. Closes ticket 27169. + + o Major features (new code layout): + - Nearly all of Tor's source code has been moved around into more + logical places. The "common" directory is now divided into a set + of libraries in "lib", and files in the "or" directory have been + split into "core" (logic absolutely needed for onion routing), + "feature" (independent modules in Tor), and "app" (to configure + and invoke the rest of Tor). See doc/HACKING/CodeStructure.md for + more information. Closes ticket 26481. + + This refactoring is not complete: although the libraries have been + refactored to be acyclic, the main body of Tor is still too + interconnected. We will attempt to improve this in the future. + + o Major features (onion services v3): + - Implement onion service client authorization at the descriptor + level: only authorized clients can decrypt a service's descriptor + to find out how to contact it. A new torrc option was added to + control this client side: ClientOnionAuthDir <path>. On the + service side, if the "authorized_clients/" directory exists in the + onion service directory path, client configurations are read from + the files within. See the manpage for more details. Closes ticket + 27547. Patch done by Suphanat Chunhapanya (haxxpop). + - Improve revision counter generation in next-gen onion services. + Onion services can now scale by hosting multiple instances on + different hosts without synchronization between them, which was + previously impossible because descriptors would get rejected by + HSDirs. Addresses ticket 25552. + - Version 3 onion services can now use the per-service + HiddenServiceExportCircuitID option to differentiate client + circuits. It communicates with the service by using the HAProxy + protocol to assign virtual IP addresses to inbound client + circuits. Closes ticket 4700. Patch by Mahrud Sayrafi. + + o Major features (onion services, UI change): + - For a newly created onion service, the default version is now 3. + Tor still supports existing version 2 services, but the operator + now needs to set "HiddenServiceVersion 2" in order to create a new + version 2 service. For existing services, Tor now learns the + version by reading the key file. Closes ticket 27215. + + o Major features (portability, cryptography, experimental, TLS): + - Tor now has the option to compile with the NSS library instead of + OpenSSL. This feature is experimental, and we expect that bugs may + remain. It is mainly intended for environments where Tor's + performance is not CPU-bound, and where NSS is already known to be + installed. To try it out, configure Tor with the --enable-nss + flag. Closes tickets 26631, 26815, and 26816. + + If you are experimenting with this option and using an old cached + consensus, Tor may fail to start. To solve this, delete your + "cached-consensus" and "cached-microdesc-consensus" files, + (if present), and restart Tor. + + o Major features (relay, UI change): + - Relays no longer run as exits by default. If the "ExitRelay" + option is auto (or unset), and no exit policy is specified with + ExitPolicy or ReducedExitPolicy, we now treat ExitRelay as 0. + Previously in this case, we allowed exit traffic and logged a + warning message. Closes ticket 21530. Patch by Neel Chauhan. + - Tor now validates that the ContactInfo config option is valid UTF- + 8 when parsing torrc. Closes ticket 27428. + + o Major bugfixes (compilation): + - Fix compilation on ARM (and other less-used CPUs) when compiling + with OpenSSL before 1.1. Fixes bug 27781; bugfix on 0.3.4.1-alpha. + + o Major bugfixes (compilation, rust): + - Rust tests can now build and run successfully with the + --enable-fragile-hardening option enabled. Doing this currently + requires the rust beta channel; it will be possible with stable + rust once Rust version 1.31 is released. Patch from Alex Crichton. + Fixes bugs 27272, 27273, and 27274. Bugfix on 0.3.1.1-alpha. + + o Major bugfixes (directory authority): + - Actually check that the address we get from DirAuthority + configuration line is valid IPv4. Explicitly disallow DirAuthority + address to be a DNS hostname. Fixes bug 26488; bugfix + on 0.1.2.10-rc. + + o Major bugfixes (embedding, main loop): + - When DisableNetwork becomes set, actually disable periodic events + that are already enabled. (Previously, we would refrain from + enabling new ones, but we would leave the old ones turned on.) + Fixes bug 28348; bugfix on 0.3.4.1-alpha. + + o Major bugfixes (main loop, bootstrap): + - Make sure Tor bootstraps and works properly if only the + ControlPort is set. Prior to this fix, Tor would only bootstrap + when a client port was set (Socks, Trans, NATD, DNS or HTTPTunnel + port). Fixes bug 27849; bugfix on 0.3.4.1-alpha. + + o Major bugfixes (onion service v3): + - On an intro point for a version 3 onion service, stop closing + introduction circuits on a NACK. This lets the client decide + whether to reuse the circuit or discard it. Previously, we closed + intro circuits when sending NACKs. Fixes bug 27841; bugfix on + 0.3.2.1-alpha. Patch by Neel Chaunan. + + o Major bugfixes (OpenSSL, portability): + - Fix our usage of named groups when running as a TLS 1.3 client in + OpenSSL 1.1.1. Previously, we only initialized EC groups when + running as a relay, which caused clients to fail to negotiate TLS + 1.3 with relays. Fixes bug 28245; bugfix on 0.2.9.15 (when TLS 1.3 + support was added). + + o Major bugfixes (relay bandwidth statistics): + - When we close relayed circuits, report the data in the circuit + queues as being written in our relay bandwidth stats. This + mitigates guard discovery and other attacks that close circuits + for the explicit purpose of noticing this discrepancy in + statistics. Fixes bug 23512; bugfix on 0.0.8pre3. + + o Major bugfixes (relay): + - When our write bandwidth limit is exhausted, stop writing on the + connection. Previously, we had a typo in the code that would make + us stop reading instead, leading to relay connections being stuck + indefinitely and consuming kernel RAM. Fixes bug 28089; bugfix + on 0.3.4.1-alpha. + - Always reactivate linked connections in the main loop so long as + any linked connection has been active. Previously, connections + serving directory information wouldn't get reactivated after the + first chunk of data was sent (usually 32KB), which would prevent + clients from bootstrapping. Fixes bug 28912; bugfix on + 0.3.4.1-alpha. Patch by "cypherpunks3". + + o Major bugfixes (restart-in-process): + - Fix a use-after-free error that could be caused by passing Tor an + impossible set of options that would fail during options_act(). + Fixes bug 27708; bugfix on 0.3.3.1-alpha. + + o Minor features (admin tools): + - Add a new --key-expiration option to print the expiration date of + the signing cert in an ed25519_signing_cert file. Resolves + issue 19506. + + o Minor features (build): + - If you pass the "--enable-pic" option to configure, Tor will try + to tell the compiler to build position-independent code suitable + to link into a dynamic library. (The default remains -fPIE, for + code suitable for a relocatable executable.) Closes ticket 23846. + + o Minor features (code correctness, testing): + - Tor's build process now includes a "check-includes" make target to + verify that no module of Tor relies on any headers from a higher- + level module. We hope to use this feature over time to help + refactor our codebase. Closes ticket 26447. + + o Minor features (code layout): + - We have a new "lowest-level" error-handling API for use by code + invoked from within the logging module. With this interface, the + logging code is no longer at risk of calling into itself if a + failure occurs while it is trying to log something. Closes + ticket 26427. + + o Minor features (compilation): + - When possible, place our warning flags in a separate file, to + avoid flooding verbose build logs. Closes ticket 28924. + - Tor's configure script now supports a --with-malloc= option to + select your malloc implementation. Supported options are + "tcmalloc", "jemalloc", "openbsd" (deprecated), and "system" (the + default). Addresses part of ticket 20424. Based on a patch from + Alex Xu. + + o Minor features (config): + - The "auto" keyword in torrc is now case-insensitive. Closes + ticket 26663. + + o Minor features (continuous integration): + - Add a Travis CI build for --enable-nss on Linux gcc. Closes + ticket 27751. + - Add new CI job to Travis configuration to run stem-based + integration tests. Closes ticket 27913. + - Use the Travis Homebrew addon to install packages on macOS during + Travis CI. The package list is the same, but the Homebrew addon + does not do a `brew update` by default. Implements ticket 27738. + - Report what program produced the mysterious core file that we + occasionally see on Travis CI during make distcheck. Closes + ticket 28024. + - Don't do a distcheck with --disable-module-dirauth in Travis. + Implements ticket 27252. + - Install libcap-dev and libseccomp2-dev so these optional + dependencies get tested on Travis CI. Closes ticket 26560. + - Only run one online rust build in Travis, to reduce network + errors. Skip offline rust builds on Travis for Linux gcc, because + they're redundant. Implements ticket 27252. + - Skip gcc on OSX in Travis CI, because it's rarely used. Skip a + duplicate hardening-off build in Travis on Tor 0.2.9. Skip gcc on + Linux with default settings, because all the non-default builds + use gcc on Linux. Implements ticket 27252. + + o Minor features (continuous integration, Windows): + - Always show the configure and test logs, and upload them as build + artifacts, when building for Windows using Appveyor CI. + Implements 28459. + - Build tor on Windows Server 2012 R2 and Windows Server 2016 using + Appveyor's CI. Closes ticket 28318. + + o Minor features (controller): + - Emit CIRC_BW events as soon as we detect that we processed an + invalid or otherwise dropped cell on a circuit. This allows + vanguards and other controllers to react more quickly to dropped + cells. Closes ticket 27678. + - For purposes of CIRC_BW-based dropped cell detection, track half- + closed stream ids, and allow their ENDs, SENDMEs, DATA and path + bias check cells to arrive without counting it as dropped until + either the END arrives, or the windows are empty. Closes + ticket 25573. + - Implement a 'GETINFO md/all' controller command to enable getting + all known microdescriptors. Closes ticket 8323. + - The GETINFO command now support an "uptime" argument, to return + Tor's uptime in seconds. Closes ticket 25132. + + o Minor features (denial-of-service avoidance): + - Make our OOM handler aware of the DNS cache so that it doesn't + fill up the memory. This check is important for our DoS mitigation + subsystem. Closes ticket 18642. Patch by Neel Chauhan. + + o Minor features (development): + - Tor's makefile now supports running the "clippy" Rust style tool + on our Rust code. Closes ticket 22156. + + o Minor features (directory authority): + - There is no longer an artificial upper limit on the length of + bandwidth lines. Closes ticket 26223. + - When a bandwidth file is used to obtain the bandwidth measurements, + include this bandwidth file headers in the votes. Closes + ticket 3723. + - Improved support for networks with only a single authority or a + single fallback directory. Patch from Gabriel Somlo. Closes + ticket 25928. + + o Minor features (embedding API): + - The Tor controller API now supports a function to launch Tor with + a preconstructed owning controller FD, so that embedding + applications don't need to manage controller ports and + authentication. Closes ticket 24204. + - The Tor controller API now has a function that returns the name + and version of the backend implementing the API. Closes + ticket 26947. + + o Minor features (fallback directory list): + - Replace the 150 fallbacks originally introduced in Tor + 0.3.3.1-alpha in January 2018 (of which ~115 were still + functional), with a list of 157 fallbacks (92 new, 65 existing, 85 + removed) generated in December 2018. Closes ticket 24803. + + o Minor features (geoip): + - Update geoip and geoip6 to the January 3 2019 Maxmind GeoLite2 + Country database. Closes ticket 29012. + + o Minor features (memory management): + - Get Libevent to use the same memory allocator as Tor, by calling + event_set_mem_functions() during initialization. Resolves + ticket 8415. + + o Minor features (memory usage): + - When not using them, store legacy TAP public onion keys in DER- + encoded format, rather than as expanded public keys. This should + save several megabytes on typical clients. Closes ticket 27246. + + o Minor features (OpenSSL bug workaround): + - Work around a bug in OpenSSL 1.1.1a, which prevented the TLS 1.3 + key export function from handling long labels. When this bug is + detected, Tor will disable TLS 1.3. We recommend upgrading to a + version of OpenSSL without this bug when it becomes available. + Closes ticket 28973. + + o Minor features (OpenSSL): + - When possible, use RFC5869 HKDF implementation from OpenSSL rather + than our own. Resolves ticket 19979. + + o Minor features (performance): + - Remove about 96% of the work from the function that we run at + startup to test our curve25519_basepoint implementation. Since + this function has yet to find an actual failure, we now only run + it for 8 iterations instead of 200. Based on our profile + information, this change should save around 8% of our startup time + on typical desktops, and may have a similar effect on other + platforms. Closes ticket 28838. + - Stop re-validating our hardcoded Diffie-Hellman parameters on + every startup. Doing this wasted time and cycles, especially on + low-powered devices. Closes ticket 28851. + + o Minor features (Rust, code quality): + - Improve rust code quality in the rust protover implementation by + making it more idiomatic. Includes changing an internal API to + take &str instead of &String. Closes ticket 26492. + + o Minor features (testing): + - Add scripts/test/chutney-git-bisect.sh, for bisecting using + chutney. Implements ticket 27211. + + o Minor features (tor-resolve): + - The tor-resolve utility can now be used with IPv6 SOCKS proxies. + Side-effect of the refactoring for ticket 26526. + + o Minor features (UI): + - Log each included configuration file or directory as we read it, + to provide more visibility about where Tor is reading from. Patch + from Unto Sten; closes ticket 27186. + - Lower log level of "Scheduler type KIST has been enabled" to INFO. + Closes ticket 26703. + + o Minor bugfixes (32-bit OSX and iOS, timing): + - Fix an integer overflow bug in our optimized 32-bit millisecond- + difference algorithm for 32-bit Apple platforms. Previously, it + would overflow when calculating the difference between two times + more than 47 days apart. Fixes part of bug 27139; bugfix + on 0.3.4.1-alpha. + - Improve the precision of our 32-bit millisecond difference + algorithm for 32-bit Apple platforms. Fixes part of bug 27139; + bugfix on 0.3.4.1-alpha. + - Relax the tolerance on the mainloop/update_time_jumps test when + running on 32-bit Apple platforms. Fixes part of bug 27139; bugfix + on 0.3.4.1-alpha. + + o Minor bugfixes (bootstrap): + - Try harder to get descriptors in non-exit test networks, by using + the mid weight for the third hop when there are no exits. Fixes + bug 27237; bugfix on 0.2.6.2-alpha. + + o Minor bugfixes (C correctness): + - Avoid casting smartlist index to int implicitly, as it may trigger + a warning (-Wshorten-64-to-32). Fixes bug 26282; bugfix on + 0.2.3.13-alpha, 0.2.7.1-alpha and 0.2.1.1-alpha. + - Use time_t for all values in + predicted_ports_prediction_time_remaining(). Rework the code that + computes difference between durations/timestamps. Fixes bug 27165; + bugfix on 0.3.1.1-alpha. + + o Minor bugfixes (client, memory usage): + - When not running as a directory cache, there is no need to store + the text of the current consensus networkstatus in RAM. + Previously, however, clients would store it anyway, at a cost of + over 5 MB. Now, they do not. Fixes bug 27247; bugfix + on 0.3.0.1-alpha. + + o Minor bugfixes (client, ReachableAddresses): + - Instead of adding a "reject *:*" line to ReachableAddresses when + loading the configuration, add one to the policy after parsing it + in parse_reachable_addresses(). This prevents extra "reject *.*" + lines from accumulating on reloads. Fixes bug 20874; bugfix on + 0.1.1.5-alpha. Patch by Neel Chauhan. + + o Minor bugfixes (code quality): + - Rename sandbox_getaddrinfo() and other functions to no longer + misleadingly suggest that they are sandbox-only. Fixes bug 26525; + bugfix on 0.2.7.1-alpha. + + o Minor bugfixes (code safety): + - Rewrite our assertion macros so that they no longer suppress the + compiler's -Wparentheses warnings. Fixes bug 27709; bugfix + on 0.0.6. + + o Minor bugfixes (compilation): + - Initialize a variable unconditionally in aes_new_cipher(), since + some compilers cannot tell that we always initialize it before + use. Fixes bug 28413; bugfix on 0.2.9.3-alpha. + + o Minor bugfixes (configuration): + - Refuse to start with relative file paths and RunAsDaemon set + (regression from the fix for bug 22731). Fixes bug 28298; bugfix + on 0.3.3.1-alpha. + + o Minor bugfixes (configuration, Onion Services): + - In rend_service_parse_port_config(), disallow any input to remain + after address-port pair was parsed. This will catch address and + port being whitespace-separated by mistake of the user. Fixes bug + 27044; bugfix on 0.2.9.10. + + o Minor bugfixes (connection, relay): + - Avoid a logging a BUG() stacktrace when closing connection held + open because the write side is rate limited but not the read side. + Now, the connection read side is simply shut down until Tor is + able to flush the connection and close it. Fixes bug 27750; bugfix + on 0.3.4.1-alpha. + + o Minor bugfixes (continuous integration, Windows): + - Stop reinstalling identical packages in our Windows CI. Fixes bug + 27464; bugfix on 0.3.4.1-alpha. + - Install only the necessary mingw packages during our appveyor + builds. This change makes the build a little faster, and prevents + a conflict with a preinstalled mingw openssl that appveyor now + ships. Fixes bugs 27765 and 27943; bugfix on 0.3.4.2-alpha. + - Explicitly specify the path to the OpenSSL library and do not + download OpenSSL from Pacman, but instead use the library that is + already provided by AppVeyor. Fixes bug 28574; bugfix on master. + - Manually configure the zstd compiler options, when building using + mingw on Appveyor Windows CI. The MSYS2 mingw zstd package does + not come with a pkg-config file. Fixes bug 28454; bugfix + on 0.3.4.1-alpha. + - Stop using an external OpenSSL install, and stop installing MSYS2 + packages, when building using mingw on Appveyor Windows CI. Fixes + bug 28399; bugfix on 0.3.4.1-alpha. + + o Minor bugfixes (controller): + - Consider all routerinfo errors other than "not a server" to be + transient for the purpose of "GETINFO exit-policy/*" controller + request. Print stacktrace in the unlikely case of failing to + recompute routerinfo digest. Fixes bug 27034; bugfix + on 0.3.4.1-alpha. + + o Minor bugfixes (correctness): + - Fix an unreached code path where we checked the value of + "hostname" inside send_resolved_hostname_cell(). Previously, we + used it before checking it; now we check it first. Fixes bug + 28879; bugfix on 0.1.2.7-alpha. + + o Minor bugfixes (directory connection shutdown): + - Avoid a double-close when shutting down a stalled directory + connection. Fixes bug 26896; bugfix on 0.3.4.1-alpha. + + o Minor bugfixes (directory permissions): + - When a user requests a group-readable DataDirectory, give it to + them. Previously, when the DataDirectory and the CacheDirectory + were the same, the default setting (0) for + CacheDirectoryGroupReadable would override the setting for + DataDirectoryGroupReadable. Fixes bug 26913; bugfix + on 0.3.3.1-alpha. + + o Minor bugfixes (HTTP tunnel): + - Fix a bug warning when closing an HTTP tunnel connection due to an + HTTP request we couldn't handle. Fixes bug 26470; bugfix + on 0.3.2.1-alpha. + + o Minor bugfixes (ipv6): + - In addrs_in_same_network_family(), we choose the subnet size based + on the IP version (IPv4 or IPv6). Previously, we chose a fixed + subnet size of /16 for both IPv4 and IPv6 addresses. Fixes bug + 15518; bugfix on 0.2.3.1-alpha. Patch by Neel Chauhan. + + o Minor bugfixes (Linux seccomp2 sandbox): + - Permit the "shutdown()" system call, which is apparently used by + OpenSSL under some circumstances. Fixes bug 28183; bugfix + on 0.2.5.1-alpha. + + o Minor bugfixes (logging): + - Stop talking about the Named flag in log messages. Clients have + ignored the Named flag since 0.3.2. Fixes bug 28441; bugfix + on 0.3.2.1-alpha. + - As a precaution, do an early return from log_addr_has_changed() if + Tor is running as client. Also, log a stack trace for debugging as + this function should only be called when Tor runs as server. Fixes + bug 26892; bugfix on 0.1.1.9-alpha. + - Refrain from mentioning bug 21018 in the logs, as it is already + fixed. Fixes bug 25477; bugfix on 0.2.9.8. + + o Minor bugfixes (logging, documentation): + - When SafeLogging is enabled, scrub IP address in + channel_tls_process_netinfo_cell(). Also, add a note to manpage + that scrubbing is not guaranteed on loglevels below Notice. Fixes + bug 26882; bugfix on 0.2.4.10-alpha. + + o Minor bugfixes (memory leaks): + - Fix a harmless memory leak in libtorrunner.a. Fixes bug 28419; + bugfix on 0.3.3.1-alpha. Patch from Martin Kepplinger. + - Fix a small memory leak when calling Tor with --dump-config. Fixes + bug 27893; bugfix on 0.3.2.1-alpha. + + o Minor bugfixes (netflow padding): + - Ensure circuitmux queues are empty before scheduling or sending + padding. Fixes bug 25505; bugfix on 0.3.1.1-alpha. + + o Minor bugfixes (onion service v2): + - Log at level "info", not "warning", in the case that we do not + have a consensus when a .onion request comes in. This can happen + normally while bootstrapping. Fixes bug 27040; bugfix + on 0.2.8.2-alpha. + + o Minor bugfixes (onion service v3): + - When deleting an ephemeral onion service (DEL_ONION), do not close + any rendezvous circuits in order to let the existing client + connections finish by themselves or closed by the application. The + HS v2 is doing that already so now we have the same behavior for + all versions. Fixes bug 28619; bugfix on 0.3.3.1-alpha. + - Build the service descriptor's signing key certificate before + uploading, so we always have a fresh one: leaving no chances for + it to expire service side. Fixes bug 27838; bugfix + on 0.3.2.1-alpha. + - Stop dumping a stack trace when trying to connect to an intro + point without having a descriptor for it. Fixes bug 27774; bugfix + on 0.3.2.1-alpha. + - When selecting a v3 rendezvous point, don't only look at the + protover, but also check whether the curve25519 onion key is + present. This way we avoid picking a relay that supports the v3 + rendezvous but for which we don't have the microdescriptor. Fixes + bug 27797; bugfix on 0.3.2.1-alpha. + - Close all SOCKS request (for the same .onion) if the newly fetched + descriptor is unusable. Before that, we would close only the first + one leaving the other hanging and let to time out by themselves. + Fixes bug 27410; bugfix on 0.3.2.1-alpha. + - When the onion service directory can't be created or has the wrong + permissions, do not log a stack trace. Fixes bug 27335; bugfix + on 0.3.2.1-alpha. + - When replacing a descriptor in the client cache, make sure to + close all client introduction circuits for the old descriptor, so + we don't end up with unusable leftover circuits. Fixes bug 27471; + bugfix on 0.3.2.1-alpha. + + o Minor bugfixes (OS compatibility): + - Properly handle configuration changes that move a listener to/from + wildcard IP address. If the first attempt to bind a socket fails, + close the old listener and try binding the socket again. Fixes bug + 17873; bugfix on 0.0.8pre-1. + + o Minor bugfixes (performance):: + - Rework node_is_a_configured_bridge() to no longer call + node_get_all_orports(), which was performing too many memory + allocations. Fixes bug 27224; bugfix on 0.2.3.9. + + o Minor bugfixes (protover): + - Reject protocol names containing bytes other than alphanumeric + characters and hyphens ([A-Za-z0-9-]). Fixes bug 27316; bugfix + on 0.2.9.4-alpha. + + o Minor bugfixes (protover, rust): + - Reject extra commas in version strings. Fixes bug 27197; bugfix + on 0.3.3.3-alpha. + - protover_all_supported() would attempt to allocate up to 16GB on + some inputs, leading to a potential memory DoS. Fixes bug 27206; + bugfix on 0.3.3.5-rc. + - Compute protover votes correctly in the rust version of the + protover code. Previously, the protover rewrite in 24031 allowed + repeated votes from the same voter for the same protocol version + to be counted multiple times in protover_compute_vote(). Fixes bug + 27649; bugfix on 0.3.3.5-rc. + - Reject protover names that contain invalid characters. Fixes bug + 27687; bugfix on 0.3.3.1-alpha. + + o Minor bugfixes (relay shutdown, systemd): + - Notify systemd of ShutdownWaitLength so it can be set to longer + than systemd's TimeoutStopSec. In Tor's systemd service file, set + TimeoutSec to 60 seconds to allow Tor some time to shut down. + Fixes bug 28113; bugfix on 0.2.6.2-alpha. + + o Minor bugfixes (relay statistics): + - Update relay descriptor on bandwidth changes only when the uptime + is smaller than 24h, in order to reduce the efficiency of guard + discovery attacks. Fixes bug 24104; bugfix on 0.1.1.6-alpha. + + o Minor bugfixes (relay): + - Consider the fact that we'll be making direct connections to our + entry and guard nodes when computing the fraction of nodes that + have their descriptors. Also, if we are using bridges and there is + at least one bridge with a full descriptor, treat the fraction of + guards available as 100%. Fixes bug 25886; bugfix on 0.2.4.10-alpha. + Patch by Neel Chauhan. + - Update the message logged on relays when DirCache is disabled. + Since 0.3.3.5-rc, authorities require DirCache (V2Dir) for the + Guard flag. Fixes bug 24312; bugfix on 0.3.3.5-rc. + + o Minor bugfixes (testing): + - Stop running stem's unit tests as part of "make test-stem", but + continue to run stem's unit and online tests during "make test- + stem-full". Fixes bug 28568; bugfix on 0.2.6.3-alpha. + - Stop leaking memory in an entry guard unit test. Fixes bug 28554; + bugfix on 0.3.0.1-alpha. + - Make the hs_service tests use the same time source when creating + the introduction point and when testing it. Now tests work better + on very slow systems like ARM or Travis. Fixes bug 27810; bugfix + on 0.3.2.1-alpha. + - Revise the "conditionvar_timeout" test so that it succeeds even on + heavily loaded systems where the test threads are not scheduled + within 200 msec. Fixes bug 27073; bugfix on 0.2.6.3-alpha. + - Fix two unit tests to work when HOME environment variable is not + set. Fixes bug 27096; bugfix on 0.2.8.1-alpha. + - If a unit test running in a subprocess exits abnormally or with a + nonzero status code, treat the test as having failed, even if the + test reported success. Without this fix, memory leaks don't cause + the tests to fail, even with LeakSanitizer. Fixes bug 27658; + bugfix on 0.2.2.4-alpha. + - When logging a version mismatch in our openssl_version tests, + report the actual offending version strings. Fixes bug 26152; + bugfix on 0.2.9.1-alpha. + - Fix forking tests on Windows when there is a space somewhere in + the path. Fixes bug 26437; bugfix on 0.2.2.4-alpha. + + o Minor bugfixes (Windows): + - Correctly identify Windows 8.1, Windows 10, and Windows Server + 2008 and later from their NT versions. Fixes bug 28096; bugfix on + 0.2.2.34; reported by Keifer Bly. + - On recent Windows versions, the GetVersionEx() function may report + an earlier Windows version than the running OS. To avoid user + confusion, add "[or later]" to Tor's version string on affected + versions of Windows. Fixes bug 28096; bugfix on 0.2.2.34; reported + by Keifer Bly. + - Remove Windows versions that were never supported by the + GetVersionEx() function. Stop duplicating the latest Windows + version in get_uname(). Fixes bug 28096; bugfix on 0.2.2.34; + reported by Keifer Bly. + + o Code simplification and refactoring: + - When parsing a port configuration, make it more obvious to static + analyzer tools that we always initialize the address. Closes + ticket 28881. + - Divide more large Tor source files -- especially ones that span + multiple areas of functionality -- into smaller parts, including + onion.c and main.c. Closes ticket 26747. + - Divide the "routerparse.c" module into separate modules for each + group of parsed objects. Closes ticket 27924. + - Move protover_rust.c to the same place protover.c was moved to. + Closes ticket 27814. + - Split directory.c into separate pieces for client, server, and + common functionality. Closes ticket 26744. + - Split the non-statistics-related parts from the rephist.c and + geoip.c modules. Closes ticket 27892. + - Split the router.c file into relay-only and shared components, to + help with future modularization. Closes ticket 27864. + - Divide the routerlist.c and dirserv.c modules into smaller parts. + Closes ticket 27799. + - 'updateFallbackDirs.py' now ignores the blacklist file, as it's not + longer needed. Closes ticket 26502. + - Include paths to header files within Tor are now qualified by + directory within the top-level src directory. + - Many structures have been removed from the centralized "or.h" + header, and moved into their own headers. This will allow us to + reduce the number of places in the code that rely on each + structure's contents and layout. Closes ticket 26383. + - Remove ATTR_NONNULL macro from codebase. Resolves ticket 26527. + - Remove GetAdaptersAddresses_fn_t. The code that used it was + removed as part of the 26481 refactor. Closes ticket 27467. + - Rework Tor SOCKS server code to use Trunnel and benefit from + autogenerated functions for parsing and generating SOCKS wire + format. New implementation is cleaner, more maintainable and + should be less prone to heartbleed-style vulnerabilities. + Implements a significant fraction of ticket 3569. + - Split sampled_guards_update_from_consensus() and + select_entry_guard_for_circuit() into subfunctions. In + entry_guards_update_primary() unite three smartlist enumerations + into one and move smartlist comparison code out of the function. + Closes ticket 21349. + - Tor now assumes that you have standards-conformant stdint.h and + inttypes.h headers when compiling. Closes ticket 26626. + - Unify our bloom filter logic. Previously we had two copies of this + code: one for routerlist filtering, and one for address set + calculations. Closes ticket 26510. + - Use the simpler strcmpstart() helper in + rend_parse_v2_service_descriptor instead of strncmp(). Closes + ticket 27630. + - Utility functions that can perform a DNS lookup are now wholly + separated from those that can't, in separate headers and C + modules. Closes ticket 26526. + + o Documentation: + - In the tor-resolve(1) manpage, fix the reference to socks- + extensions.txt by adding a web URL. Resolves ticket 27853. + - Mention that we require Python to be 2.7 or newer for some + integration tests that we ship with Tor. Resolves ticket 27677. + - Copy paragraph and URL to Tor's code of conduct document from + CONTRIBUTING to new CODE_OF_CONDUCT file. Resolves ticket 26638. + - Remove old instructions from INSTALL document. Closes ticket 26588. + - Warn users that they should not include MyFamily line(s) in their + torrc when running Tor bridge. Closes ticket 26908. + + o Removed features: + - Tor no longer supports building with the dmalloc library. For + debugging memory issues, we suggest using gperftools or msan + instead. Closes ticket 26426. + - Tor no longer attempts to run on Windows environments without the + GetAdaptersAddresses() function. This function has existed since + Windows XP, which is itself already older than we support. + - Remove Tor2web functionality for version 2 onion services. The + Tor2webMode and Tor2webRendezvousPoints options are now obsolete. + (This feature was never shipped in vanilla Tor and it was only + possible to use this feature by building the support at compile + time. Tor2webMode is not implemented for version 3 onion services.) + Closes ticket 26367. + + o Testing: + - Increase logging and tag all log entries with timestamps in + test_rebind.py. Provides diagnostics for issue 28229. + + o Code simplification and refactoring (shared random, dirauth): + - Change many tor_assert() to use BUG() instead. The idea is to not + crash a dirauth but rather scream loudly with a stacktrace and let + it continue run. The shared random subsystem is very resilient and + if anything wrong happens with it, at worst a non coherent value + will be put in the vote and discarded by the other authorities. + Closes ticket 19566. + + o Documentation (onion services): + - Improve HSv3 client authorization by making some options more + explicit and detailed. Closes ticket 28026. Patch by Mike Tigas. + - Document in the man page that changing ClientOnionAuthDir value or + adding a new file in the directory will not work at runtime upon + sending a HUP if Sandbox 1. Closes ticket 28128. + - Note in the man page that the only real way to fully revoke an + onion service v3 client authorization is by restarting the tor + process. Closes ticket 28275. + + Changes in version 0.3.4.9 - 2018-11-02 Tor 0.3.4.9 is the second stable release in its series; it backports numerous fixes, including a fix for a bandwidth management bug that diff --git a/changes/bug24661 b/changes/bug24661 deleted file mode 100644 index a915a93e0e..0000000000 --- a/changes/bug24661 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (client, guard selection): - - When Tor's consensus has expired, but is still reasonably live, use it - to select guards. Fixes bug 24661; bugfix on 0.3.0.1-alpha. diff --git a/changes/bug27197 b/changes/bug27197 deleted file mode 100644 index e389f85065..0000000000 --- a/changes/bug27197 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (protover, rust): - - Reject extra commas in version string. Fixes bug 27197; bugfix on - 0.3.3.3-alpha. diff --git a/changes/bug27740 b/changes/bug27740 deleted file mode 100644 index 76a17b7dda..0000000000 --- a/changes/bug27740 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (rust): - - Return a string that can be safely freed by C code, not one created by - the rust allocator, in protover_all_supported(). Fixes bug 27740; bugfix - on 0.3.3.1-alpha. diff --git a/changes/bug27741 b/changes/bug27741 deleted file mode 100644 index 531e264b63..0000000000 --- a/changes/bug27741 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (rust, directory authority): - - Fix an API mismatch in the rust implementation of - protover_compute_vote(). This bug could have caused crashes on any - directory authorities running Tor with Rust (which we do not yet - recommend). Fixes bug 27741; bugfix on 0.3.3.6. diff --git a/changes/bug27750 b/changes/bug27750 deleted file mode 100644 index c234788b1c..0000000000 --- a/changes/bug27750 +++ /dev/null @@ -1,6 +0,0 @@ - o Minor bugfixes (connection, relay): - - Avoid a wrong BUG() stacktrace in case a closing connection is being held - open because the write side is rate limited but not the read side. Now, - the connection read side is simply shutdown instead of kept open until tor - is able to flush the connection and then fully close it. Fixes bug 27750; - bugfix on 0.3.4.1-alpha. diff --git a/changes/bug27800 b/changes/bug27800 deleted file mode 100644 index 63d5dbc681..0000000000 --- a/changes/bug27800 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (directory authority): - - Log additional info when we get a relay that shares an ed25519 - ID with a different relay, instead making a BUG() warning. - Fixes bug 27800; bugfix on 0.3.2.1-alpha. diff --git a/changes/bug27804 b/changes/bug27804 deleted file mode 100644 index fa7fec0bc5..0000000000 --- a/changes/bug27804 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (rust): - - Fix a potential null dereference in protover_all_supported(). - Add a test for it. Fixes bug 27804; bugfix on 0.3.3.1-alpha. diff --git a/changes/bug27841 b/changes/bug27841 deleted file mode 100644 index 9cd1da7275..0000000000 --- a/changes/bug27841 +++ /dev/null @@ -1,7 +0,0 @@ - o Minor bugfixes (onion services): - - On an intro point for a version 3 onion service, we do not close - an introduction circuit on an NACK. This lets the client decide - whether to reuse the circuit or discard it. Previously, we closed - intro circuits on NACKs. Fixes bug 27841; bugfix on 0.3.2.1-alpha. - Patch by Neel Chaunan - diff --git a/changes/bug27948 b/changes/bug27948 deleted file mode 100644 index fea16f3d0f..0000000000 --- a/changes/bug27948 +++ /dev/null @@ -1,6 +0,0 @@ - o Minor bugfixes (tests): - - Treat backtrace test failures as expected on BSD-derived systems - (NetBSD, OpenBSD, and macOS/Darwin) until we solve bug 17808. - (FreeBSD failures have been treated as expected since 18204 in 0.2.8.) - Fixes bug 27948; bugfix on 0.2.5.2-alpha. - diff --git a/changes/bug27963_timeradd b/changes/bug27963_timeradd deleted file mode 100644 index 34b361cf8d..0000000000 --- a/changes/bug27963_timeradd +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (compilation, opensolaris): - - Add a missing include to compat_pthreads.c, to fix compilation - on OpenSolaris and its descendants. Fixes bug 27963; bugfix - on 0.3.5.1-alpha. diff --git a/changes/bug27968 b/changes/bug27968 deleted file mode 100644 index 78c8eee33a..0000000000 --- a/changes/bug27968 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (testing): - - Avoid hangs and race conditions in test_rebind.py. - Fixes bug 27968; bugfix on 0.3.5.1-alpha. diff --git a/changes/bug28096 b/changes/bug28096 deleted file mode 100644 index 6847df9798..0000000000 --- a/changes/bug28096 +++ /dev/null @@ -1,13 +0,0 @@ - o Minor bugfixes (Windows): - - Correctly identify Windows 8.1, Windows 10, and Windows Server 2008 - and later from their NT versions. - Fixes bug 28096; bugfix on 0.2.2.34; reported by Keifer Bly. - - On recent Windows versions, the GetVersionEx() function may report - an earlier Windows version than the running OS. To avoid user - confusion, add "[or later]" to Tor's version string on affected - versions of Windows. - Fixes bug 28096; bugfix on 0.2.2.34; reported by Keifer Bly. - - Remove Windows versions that were never supported by the - GetVersionEx() function. Stop duplicating the latest Windows - version in get_uname(). - Fixes bug 28096; bugfix on 0.2.2.34; reported by Keifer Bly. diff --git a/changes/bug28115 b/changes/bug28115 deleted file mode 100644 index e3e29968eb..0000000000 --- a/changes/bug28115 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (portability): - - Make the OPE code (which is used for v3 onion services) run correctly - on big-endian platforms. Fixes bug 28115; bugfix on 0.3.5.1-alpha. diff --git a/changes/bug28127 b/changes/bug28127 deleted file mode 100644 index 541128c88e..0000000000 --- a/changes/bug28127 +++ /dev/null @@ -1,7 +0,0 @@ - o Minor bugfixes (onion services): - - Unless we have explicitly set HiddenServiceVersion, detect the onion - service version and then look for invalid options. Previously, we - did the reverse, but that broke existing configs which were pointed - to a v2 hidden service and had options like HiddenServiceAuthorizeClient - set Fixes bug 28127; bugfix on 0.3.5.1-alpha. Patch by Neel Chauhan. - diff --git a/changes/bug28183 b/changes/bug28183 deleted file mode 100644 index 8d35dcdc01..0000000000 --- a/changes/bug28183 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (Linux seccomp2 sandbox): - - Permit the "shutdown()" system call, which is apparently - used by OpenSSL under some circumstances. Fixes bug 28183; - bugfix on 0.2.5.1-alpha. diff --git a/changes/bug28202 b/changes/bug28202 deleted file mode 100644 index 182daac4f1..0000000000 --- a/changes/bug28202 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (C correctness): - - Avoid undefined behavior in an end-of-string check when parsing the - BEGIN line in a directory object. Fixes bug 28202; bugfix on - 0.2.0.3-alpha. diff --git a/changes/bug28245 b/changes/bug28245 deleted file mode 100644 index d7e6deb810..0000000000 --- a/changes/bug28245 +++ /dev/null @@ -1,6 +0,0 @@ - o Major bugfixes (OpenSSL, portability): - - Fix our usage of named groups when running as a TLS 1.3 client in - OpenSSL 1.1.1. Previously, we only initialized EC groups when running - as a server, which caused clients to fail to negotiate TLS 1.3 with - relays. Fixes bug 28245; bugfix on 0.2.9.15 when TLS 1.3 support was - added. diff --git a/changes/bug28298 b/changes/bug28298 deleted file mode 100644 index 8db340f3df..0000000000 --- a/changes/bug28298 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (configuration): - - Resume refusing to start with relative file paths and RunAsDaemon - set (regression from the fix for bug 22731). Fixes bug 28298; - bugfix on 0.3.3.1-alpha. diff --git a/changes/bug28303 b/changes/bug28303 deleted file mode 100644 index 80f1302e5e..0000000000 --- a/changes/bug28303 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (compilation): - - Fix a pair of missing headers on OpenBSD. Fixes bug 28303; - bugfix on 0.3.5.1-alpha. Patch from Kris Katterjohn. diff --git a/changes/bug28348_034 b/changes/bug28348_034 deleted file mode 100644 index 3913c03a4c..0000000000 --- a/changes/bug28348_034 +++ /dev/null @@ -1,5 +0,0 @@ - o Major bugfixes (embedding, main loop): - - When DisableNetwork becomes set, actually disable periodic events that - are already enabled. (Previously, we would refrain from enabling new - ones, but we would leave the old ones turned on.) - Fixes bug 28348; bugfix on 0.3.4.1-alpha. diff --git a/changes/bug28399 b/changes/bug28399 deleted file mode 100644 index 9096db70b0..0000000000 --- a/changes/bug28399 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (continuous integration, Windows): - - Stop using an external OpenSSL install, and stop installing MSYS2 - packages, when building using mingw on Appveyor Windows CI. - Fixes bug 28399; bugfix on 0.3.4.1-alpha. diff --git a/changes/bug28413 b/changes/bug28413 deleted file mode 100644 index 4c88bea7e7..0000000000 --- a/changes/bug28413 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (compilation): - - Initialize a variable in aes_new_cipher(), since some compilers - cannot tell that we always initialize it before use. Fixes bug 28413; - bugfix on 0.2.9.3-alpha. diff --git a/changes/bug28419 b/changes/bug28419 deleted file mode 100644 index 52ceb0a2a7..0000000000 --- a/changes/bug28419 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (memory leaks): - - Fix a harmless memory leak in libtorrunner.a. Fixes bug 28419; - bugfix on 0.3.3.1-alpha. Patch from Martin Kepplinger.
\ No newline at end of file diff --git a/changes/bug28435 b/changes/bug28435 deleted file mode 100644 index 2a886cb8b7..0000000000 --- a/changes/bug28435 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (documentation): - - Make Doxygen work again after the 0.3.5 source tree moves. - Fixes bug 28435; bugfix on 0.3.5.1-alpha. diff --git a/changes/bug28441 b/changes/bug28441 deleted file mode 100644 index d259b9f742..0000000000 --- a/changes/bug28441 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (logging): - - Stop talking about the Named flag in log messages. Clients have - ignored the Named flag since 0.3.2. Fixes bug 28441; - bugfix on 0.3.2.1-alpha. diff --git a/changes/bug28454 b/changes/bug28454 deleted file mode 100644 index ca46ae2777..0000000000 --- a/changes/bug28454 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (continuous integration, Windows): - - Manually configure the zstd compiler options, when building using - mingw on Appveyor Windows CI. The MSYS2 mingw zstd package does not - come with a pkg-config file. Fixes bug 28454; bugfix on 0.3.4.1-alpha. diff --git a/changes/bug28485 b/changes/bug28485 deleted file mode 100644 index a8309ae21f..0000000000 --- a/changes/bug28485 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (compilation): - - Add missing dependency on libgdi32.dll for tor-print-ed-signing-cert.exe - on Windows. Fixes bug 28485; bugfix on 0.3.5.1-alpha. diff --git a/changes/bug28524 b/changes/bug28524 deleted file mode 100644 index 1cad700422..0000000000 --- a/changes/bug28524 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (restart-in-process, boostrap): - - Add missing resets of bootstrap tracking state when shutting - down (regression caused by ticket 27169). Fixes bug 28524; - bugfix on 0.3.5.1-alpha. diff --git a/changes/bug28554 b/changes/bug28554 deleted file mode 100644 index 9a0b281406..0000000000 --- a/changes/bug28554 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (unit tests, guard selection): - - Stop leaking memory in an entry guard unit test. Fixes bug 28554; - bugfix on 0.3.0.1-alpha. diff --git a/changes/bug28562 b/changes/bug28562 deleted file mode 100644 index e14362164d..0000000000 --- a/changes/bug28562 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (testing): - - Use a separate DataDirectory for the test_rebind script. - Previously, this script would run using the default DataDirectory, - and sometimes fail. Fixes bug 28562; bugfix on 0.3.5.1-alpha. - Patch from Taylor R Campbell. diff --git a/changes/bug28568 b/changes/bug28568 deleted file mode 100644 index 919ec08903..0000000000 --- a/changes/bug28568 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (testing): - - Stop running stem's unit tests as part of "make test-stem". But continue - to run stem's unit and online tests during "make test-stem-full". - Fixes bug 28568; bugfix on 0.2.6.3-alpha. diff --git a/changes/bug28569 b/changes/bug28569 deleted file mode 100644 index 45a57a80ae..0000000000 --- a/changes/bug28569 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (unit tests, directory clients): - - Mark outdated dirservers when Tor only has a reasonably live consensus. - Fixes bug 28569; bugfix on 0.3.2.5-alpha. diff --git a/changes/bug28612 b/changes/bug28612 deleted file mode 100644 index 559f254234..0000000000 --- a/changes/bug28612 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (windows services): - - Make Tor start correctly as an NT service again: previously it - was broken by refactoring. Fixes bug 28612; bugfix on 0.3.5.3-alpha. - diff --git a/changes/bug28614_better_logging b/changes/bug28614_better_logging new file mode 100644 index 0000000000..26d19c3c11 --- /dev/null +++ b/changes/bug28614_better_logging @@ -0,0 +1,6 @@ + o Minor bugfixes (logging): + - On Windows, when errors cause us to reload a consensus from disk, tell + the user that we are retrying at log level "notice". Previously we only + logged this information at "info", which was confusing because the + errors themselves were logged at "warning". Improves previous fix for + 28614. Fixes bug 30004; bugfix on 0.4.0.2-alpha. diff --git a/changes/bug28619 b/changes/bug28619 deleted file mode 100644 index 86be8cb2fb..0000000000 --- a/changes/bug28619 +++ /dev/null @@ -1,6 +0,0 @@ - o Minor bugfixes (hidden service v3): - - When deleting an ephemeral onion service (DEL_ONION), do not close any - rendezvous circuits in order to let the existing client connections - finish by themselves or closed by the application. The HS v2 is doing - that already so now we have the same behavior for all versions. Fixes - bug 28619; bugfix on 0.3.3.1-alpha. diff --git a/changes/bug28895 b/changes/bug28895 deleted file mode 100644 index 25fb167b2e..0000000000 --- a/changes/bug28895 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (usability): - - Stop saying "Your Guard ..." in pathbias_measure_{use,close}_rate() - as that confusingly suggests that mentioned guard node is under control - and responsibility of end user, which it is not. Fixes bug 28895; - bugfix on Tor 0.3.0.1-alpha. diff --git a/changes/bug28920 b/changes/bug28920 deleted file mode 100644 index e698686a6d..0000000000 --- a/changes/bug28920 +++ /dev/null @@ -1,6 +0,0 @@ - o Minor bugfixes (logging): - - Rework rep_hist_log_link_protocol_counts() to iterate through all link - protocol versions when logging incoming/outgoing connection counts. Tor - no longer skips version 5 and we don't have to remember to update this - function when new link protocol version is developed. Fixes bug 28920; - bugfix on 0.2.6.10. diff --git a/changes/bug28925 b/changes/bug28925 new file mode 100644 index 0000000000..a867443885 --- /dev/null +++ b/changes/bug28925 @@ -0,0 +1,4 @@ + o Minor bugfixes (bootstrap reporting): + - During bootstrap reporting, correctly distinguish pluggable + transports from plain proxies. Fixes bug 28925; bugfix on + 0.4.0.1-alpha. diff --git a/changes/bug28938 b/changes/bug28938 deleted file mode 100644 index de6c5f7b79..0000000000 --- a/changes/bug28938 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (compilation): - - Fix missing headers required for proper detection of - OpenBSD. Fixes bug 28938; bugfix on 0.3.5.1-alpha. - Patch from Kris Katterjohn. diff --git a/changes/bug28974 b/changes/bug28974 deleted file mode 100644 index 2d74f5674f..0000000000 --- a/changes/bug28974 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (compilation): - - Fix compilation for Android by adding a missing header to - freespace.c. Fixes bug 28974; bugfix on 0.3.5.1-alpha. diff --git a/changes/bug28995 b/changes/bug28995 deleted file mode 100644 index f76b6a085a..0000000000 --- a/changes/bug28995 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfix (IPv6): - Fix tor_ersatz_socketpair on IPv6-only systems. Previously, - the IPv6 socket was bound using an address family of AF_INET - instead of AF_INET6. Fixes bug 28995; bugfix on 0.3.5.1-alpha. - Patch from Kris Katterjohn. diff --git a/changes/bug29122 b/changes/bug29122 new file mode 100644 index 0000000000..020052ff8f --- /dev/null +++ b/changes/bug29122 @@ -0,0 +1,3 @@ + o Minor bugfixes (unit tests): + - Fix intermittent failures on an adaptive padding unittest. Fixes bug + 29122; bugfix on 0.4.0.1-alpha diff --git a/changes/bug29150 b/changes/bug29150 new file mode 100644 index 0000000000..7696b90378 --- /dev/null +++ b/changes/bug29150 @@ -0,0 +1,3 @@ + o Minor bugfixes (linux seccomp sandbox): + - Fix startup crash when experimental sandbox support is enabled. + Fixes bug 29150; bugfix on 0.4.0.1-alpha. Patch by Peter Gerber. diff --git a/changes/bug29169 b/changes/bug29169 new file mode 100644 index 0000000000..41d4b76ef5 --- /dev/null +++ b/changes/bug29169 @@ -0,0 +1,3 @@ + o Minor bugfixes (compilation): + - Fix compilation warnings in test_circuitpadding.c. Fixes bug 29169; + bugfix on 0.4.0.1-alpha. diff --git a/changes/bug29204 b/changes/bug29204 new file mode 100644 index 0000000000..ec2cf67b2f --- /dev/null +++ b/changes/bug29204 @@ -0,0 +1,4 @@ + o Minor bugfixes (circuitpadding): + - Inspect circuit-level cell queue before sending padding, to avoid + sending padding while too much data is queued. Fixes bug 29204; + bugfix on 0.4.0.1-alpha. diff --git a/changes/bug29298 b/changes/bug29298 new file mode 100644 index 0000000000..df12db77d7 --- /dev/null +++ b/changes/bug29298 @@ -0,0 +1,5 @@ + o Minor bugfixes (testing, circuit padding): + - Disabled unstable circuit padding unittest that was causing intermittent + test failures because of ill-defined small histogram. Such histograms + will be allowed again after 29298 is implemented. Fixes second case of + bug 29122; bugfix on 0.4.0.1-alpha.
\ No newline at end of file diff --git a/changes/bug29500 b/changes/bug29500 new file mode 100644 index 0000000000..16550935b2 --- /dev/null +++ b/changes/bug29500 @@ -0,0 +1,3 @@ + o Minor bugfixes (circuitpadding testing): + - Minor tweaks to avoid very rare test failures related to timers and + monotime. Fixes bug 29500; bugfix on 0.4.0.1-alpha diff --git a/changes/bug29508 b/changes/bug29508 new file mode 100644 index 0000000000..ee728bbbc9 --- /dev/null +++ b/changes/bug29508 @@ -0,0 +1,3 @@ + o Minor bugfixes (scheduler): + - When readding channels to the pending list, check the correct channel's + sched_heap_idx. Fixes bug 29508; bugfix on 0.3.2.10 diff --git a/changes/bug29527 b/changes/bug29527 new file mode 100644 index 0000000000..6f36a9e1a0 --- /dev/null +++ b/changes/bug29527 @@ -0,0 +1,5 @@ + o Minor features (circuit padding): + - Stop warning about undefined behavior in the probability distribution + tests. Float division by zero may technically be undefined behaviour in + C, but it's well-defined in IEEE 754. Partial backport of 29298. + Closes ticket 29527; bugfix on 0.4.0.1-alpha. diff --git a/changes/bug29562 b/changes/bug29562 new file mode 100644 index 0000000000..0621cd09a0 --- /dev/null +++ b/changes/bug29562 @@ -0,0 +1,4 @@ + o Minor bugfixes (pluggable transports): + - Fix an assertion failure crash bug when a pluggable transport process is + terminated during the bootstrap phase. Fixes bug 29562; bugfix on + 0.4.0.1-alpha. diff --git a/changes/bug29693 b/changes/bug29693 new file mode 100644 index 0000000000..33ce051c40 --- /dev/null +++ b/changes/bug29693 @@ -0,0 +1,3 @@ + o Minor bugfixes (unit tests): + - Decrease the false positive rate of stochastic probability distribution + tests. Fixes bug 29693; bugfix on 0.4.0.1-alpha.
\ No newline at end of file diff --git a/changes/bug29706_refactor b/changes/bug29706_refactor new file mode 100644 index 0000000000..ba1d0c7edd --- /dev/null +++ b/changes/bug29706_refactor @@ -0,0 +1,4 @@ + o Minor bugfixes (memory management): + - Refactor the shared random state's memory management so that it actually + takes ownership of the shared random value pointers. + Fixes bug 29706; bugfix on 0.2.9.1-alpha. diff --git a/changes/bug29874 b/changes/bug29874 new file mode 100644 index 0000000000..8534753b51 --- /dev/null +++ b/changes/bug29874 @@ -0,0 +1,4 @@ + o Minor bugfixes (pluggable transports): + - Restore old behaviour when it comes to discovering the path of a given + Pluggable Transport exe-file. Fixes bug 29874; bugfix on 0.4.0.1-alpha. + diff --git a/changes/bug29930 b/changes/bug29930 new file mode 100644 index 0000000000..a99b11430b --- /dev/null +++ b/changes/bug29930 @@ -0,0 +1,4 @@ + o Minor bugfixes (UI): + - Lower log level of unlink() errors during bootstrap. Fixes bug 29930; + bugfix on 0.4.0.1-alpha. + diff --git a/changes/bug29959-040 b/changes/bug29959-040 new file mode 100644 index 0000000000..3740e0169a --- /dev/null +++ b/changes/bug29959-040 @@ -0,0 +1,3 @@ + o Minor bugfixes (directory authorities): + - Actually include the bandwidth-file-digest line in directory authority + votes. Fixes bug 29959; bugfix on 0.4.0.2-alpha. diff --git a/changes/bug30001 b/changes/bug30001 new file mode 100644 index 0000000000..52e58872ef --- /dev/null +++ b/changes/bug30001 @@ -0,0 +1,7 @@ + o Minor features (testing): + - Use the approx_time() function when setting the "Expires" header + in directory replies, to make them more testable. Needed for + ticket 30001. + o Minor bug fixes (testing): + - Check the time in the "Expires" header with approx_time(). + Fixes bug 30001; bugfix on 0.4.0.4-rc. diff --git a/changes/bug30263 b/changes/bug30263 new file mode 100644 index 0000000000..ba81c1b8a1 --- /dev/null +++ b/changes/bug30263 @@ -0,0 +1,3 @@ + o Minor bugfixes (shellcheck): + - Stop looking for scripts in the build directory during + "make shellcheck". Fixes bug 30263; bugfix on 0.4.0.1-alpha. diff --git a/changes/bug30614 b/changes/bug30614 new file mode 100644 index 0000000000..9f904bd115 --- /dev/null +++ b/changes/bug30614 @@ -0,0 +1,4 @@ + o Minor bugfixes (NetBSD): + - Fix usage of minherit() on NetBSD and other platforms that define + MAP_INHERIT_{ZERO,NONE} instead of INHERIT_{ZERO,NONE}. Fixes bug + 30614; bugfix on 0.4.0.2-alpha. Patch from Taylor Campbell. diff --git a/changes/bug30649 b/changes/bug30649 new file mode 100644 index 0000000000..4b2c603171 --- /dev/null +++ b/changes/bug30649 @@ -0,0 +1,4 @@ + o Minor bugfixes (circuit padding): + - On relays, properly check that a padding machine is absent before + logging a warn about it being absent. Fixes bug 30649; + bugfix on 0.4.0.1-alpha. diff --git a/changes/bug31810 b/changes/bug31810 new file mode 100644 index 0000000000..628d12f09b --- /dev/null +++ b/changes/bug31810 @@ -0,0 +1,4 @@ + o Minor bugfixes (process management): + - Remove assertion in the Unix process backend. This assertion would trigger + when a new process is spawned where the executable is not found leading to + a stack trace from the child process. Fixes bug 31810; bugfix on 0.4.0.1-alpha. diff --git a/changes/bug32108 b/changes/bug32108 new file mode 100644 index 0000000000..2806fa3e5d --- /dev/null +++ b/changes/bug32108 @@ -0,0 +1,8 @@ + o Major bugfixes (relay): + - Relays now respect their AccountingMax bandwidth again. When relays + entered "soft" hibernation (which typically starts when we've hit + 90% of our AccountingMax), we had stopped checking whether we should + enter hard hibernation. Soft hibernation refuses new connections and + new circuits, but the existing circuits can continue, meaning that + relays could have exceeded their configured AccountingMax. Fixes + bug 32108; bugfix on 0.4.0.1-alpha. diff --git a/changes/diagnostic_28223_redux b/changes/diagnostic_28223_redux new file mode 100644 index 0000000000..0d7499832e --- /dev/null +++ b/changes/diagnostic_28223_redux @@ -0,0 +1,4 @@ + o Minor features (diagnostic): + - Add more diagnostic log messages in an attempt to solve + the issue of NUL bytes appearing in a microdescriptor cache. + Related to ticket 28223. diff --git a/changes/doc28623 b/changes/doc28623 new file mode 100644 index 0000000000..3c3313abdd --- /dev/null +++ b/changes/doc28623 @@ -0,0 +1,3 @@ + o Documentation: + - In manpage entry describing MapAddress torrc setting, use example + IP addresses from ranges specified by RFC 5737. Resolves issue 28623. diff --git a/changes/doc29121 b/changes/doc29121 new file mode 100644 index 0000000000..dd31cc9c70 --- /dev/null +++ b/changes/doc29121 @@ -0,0 +1,3 @@ + o Documentation: + - Clarify that Tor performs stream isolation between *Port listeners by + default. Resolves issue 29121. diff --git a/changes/feature28976 b/changes/feature28976 new file mode 100644 index 0000000000..c7ebc207f7 --- /dev/null +++ b/changes/feature28976 @@ -0,0 +1,4 @@ + o Minor features (developer tooling): + - Provide a git pre-commit hook that disallows commiting if we have any + failures in our code and changelog formatting checks. It is now available + in scripts/maint/pre-commit.git-hook. Implements feature 28976. diff --git a/changes/geoip-2018-11-06 b/changes/geoip-2018-11-06 deleted file mode 100644 index 5c18ea4244..0000000000 --- a/changes/geoip-2018-11-06 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (geoip): - - Update geoip and geoip6 to the November 6 2018 Maxmind GeoLite2 - Country database. Closes ticket 28395. - diff --git a/changes/geoip-2018-12-05 b/changes/geoip-2018-12-05 deleted file mode 100644 index 20ccf2d8a5..0000000000 --- a/changes/geoip-2018-12-05 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (geoip): - - Update geoip and geoip6 to the December 5 2018 Maxmind GeoLite2 - Country database. Closes ticket 28744. - diff --git a/changes/geoip-2019-01-03 b/changes/geoip-2019-01-03 deleted file mode 100644 index 27ffb7f460..0000000000 --- a/changes/geoip-2019-01-03 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (geoip): - - Update geoip and geoip6 to the January 3 2019 Maxmind GeoLite2 - Country database. Closes ticket 29012. - diff --git a/changes/rust_asan b/changes/rust_asan deleted file mode 100644 index 1ca7ae6888..0000000000 --- a/changes/rust_asan +++ /dev/null @@ -1,8 +0,0 @@ - o Major bugfixes (compilation, rust): - - Rust tests can now build and run successfully with the - --enable-fragile-hardening option enabled. - Doing this currently requires the rust beta channel; it will - be possible with stable rust as of rust version 1.31 is out. - Patch from Alex Crichton. - Fixes bugs 27272, 27273, and 27274. - Bugfix on 0.3.1.1-alpha. diff --git a/changes/ticket19566 b/changes/ticket19566 deleted file mode 100644 index bf7071e660..0000000000 --- a/changes/ticket19566 +++ /dev/null @@ -1,6 +0,0 @@ - o Code simplification and refactoring (shared random, dirauth): - - Change many tor_assert() to use BUG() instead. The idea is to not crash - a dirauth but rather scream loudly with a stacktrace and let it continue - run. The shared random subsystem is very resilient and if anything wrong - happens with it, at worst a non coherent value will be put in the vote - and discarded by the other authorities. Closes ticket 19566. diff --git a/changes/ticket21377 b/changes/ticket21377 new file mode 100644 index 0000000000..2bf5149a0a --- /dev/null +++ b/changes/ticket21377 @@ -0,0 +1,4 @@ + o Minor features (dircache): + - When a directory authority is using a bandwidth file to obtain the + bandwidth values that will be included in the next vote, serve this + bandwidth file at /tor/status-vote/next/bandwidth. Closes ticket 21377.
\ No newline at end of file diff --git a/changes/ticket24803 b/changes/ticket24803 deleted file mode 100644 index e76a9eeab9..0000000000 --- a/changes/ticket24803 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor features (fallback directory list): - - Replace the 150 fallbacks originally introduced in Tor 0.3.3.1-alpha in - January 2018 (of which ~115 were still functional), with a list of - 157 fallbacks (92 new, 65 existing, 85 removed) generated in - December 2018. Closes ticket 24803. diff --git a/changes/ticket26698 b/changes/ticket26698 new file mode 100644 index 0000000000..6b029a1b73 --- /dev/null +++ b/changes/ticket26698 @@ -0,0 +1,4 @@ + o Minor features (directory authority): + - When a directory authority is using a bandwidth file to obtain the + bandwidth values, include the digest of the file in the vote. + Closes ticket 26698. diff --git a/changes/ticket27471 b/changes/ticket27471 deleted file mode 100644 index ffe77d268e..0000000000 --- a/changes/ticket27471 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (hidden service v3, client): - - When replacing a descriptor in the client cache with a newer descriptor, - make sure to close all client introduction circuits of the old - descriptor so we don't end up with unusable leftover circuits. Fixes bug - 27471; bugfix on 0.3.2.1-alpha. diff --git a/changes/ticket27751 b/changes/ticket27751 deleted file mode 100644 index 593c473b61..0000000000 --- a/changes/ticket27751 +++ /dev/null @@ -1,2 +0,0 @@ - o Minor features (continuous integration): - - Add a Travis CI build for --enable-nss on Linux gcc. Closes ticket 27751. diff --git a/changes/ticket27761 b/changes/ticket27761 new file mode 100644 index 0000000000..35106ee9c6 --- /dev/null +++ b/changes/ticket27761 @@ -0,0 +1,4 @@ + o Minor features (changelogs): + - Check that bugfix versions in changes files look like Tor versions + from the versions spec. Warn when bugfixes claim to be on a future + release. Closes ticket 27761. diff --git a/changes/ticket27838 b/changes/ticket27838 deleted file mode 100644 index 1699730d7a..0000000000 --- a/changes/ticket27838 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (hidden service v3): - - Build the service descriptor signing key certificate before uploading so - we always have a fresh one leaving no chances for it to expire service - side. Fixes bug 27838; bugfix on 0.3.2.1-alpha. diff --git a/changes/ticket27913 b/changes/ticket27913 deleted file mode 100644 index 81ce725932..0000000000 --- a/changes/ticket27913 +++ /dev/null @@ -1,3 +0,0 @@ - o Testing: - - Add new CI job to Travis configuration that runs stem-based - integration tests. Closes ticket 27913. diff --git a/changes/ticket27995 b/changes/ticket27995 deleted file mode 100644 index 8c75425749..0000000000 --- a/changes/ticket27995 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (hidden service v3, client authorization): - - Fix an assert() when adding a client authorization for the first time - and then sending a HUP signal to the service. Before that, tor would - stop abruptly. Fixes bug 27995; bugfix on 0.3.5.1-alpha. diff --git a/changes/ticket28026 b/changes/ticket28026 deleted file mode 100644 index a6911c2cab..0000000000 --- a/changes/ticket28026 +++ /dev/null @@ -1,3 +0,0 @@ - o Documentation (hidden service manpage): - - Improve HSv3 client authorization by making some options more explicit - and detailed. Closes ticket 28026. Patch by "mtigas". diff --git a/changes/ticket28113 b/changes/ticket28113 deleted file mode 100644 index 30dd825a9b..0000000000 --- a/changes/ticket28113 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (relay shutdown, systemd): - - Notify systemd of ShutdownWaitLength so it can be set to longer than - systemd's TimeoutStopSec. In tor's systemd service file, set - TimeoutSec to 60 seconds, to allow tor some time to shut down. - Fixes bug 28113; bugfix on 0.2.6.2-alpha. diff --git a/changes/ticket28128 b/changes/ticket28128 deleted file mode 100644 index 6d08c74242..0000000000 --- a/changes/ticket28128 +++ /dev/null @@ -1,4 +0,0 @@ - o Documentation (hidden service manpage, sandbox): - - Document in the man page that changing ClientOnionAuthDir value or - adding a new file in the directory will not work at runtime upon sending - a HUP if Sandbox 1. Closes ticket 28128. diff --git a/changes/ticket28229_diag b/changes/ticket28229_diag deleted file mode 100644 index cd02b81faa..0000000000 --- a/changes/ticket28229_diag +++ /dev/null @@ -1,3 +0,0 @@ - o Testing: - - Increase logging and tag all log entries with timestamps - in test_rebind.py. Provides diagnostics for issue 28229. diff --git a/changes/ticket28275 b/changes/ticket28275 deleted file mode 100644 index eadca86b7b..0000000000 --- a/changes/ticket28275 +++ /dev/null @@ -1,4 +0,0 @@ - o Documentation (hidden service v3, man page): - - Note in the man page that the only real way to fully revoke an onion - service v3 client authorization is by restarting the tor process. Closes - ticket 28275. diff --git a/changes/ticket28318 b/changes/ticket28318 deleted file mode 100644 index 24dc1e9580..0000000000 --- a/changes/ticket28318 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor features (Windows, continuous integration): - - Build tor on Windows Server 2012 R2 and Windows Server 2016 using - Appveyor's CI. Closes ticket 28318. diff --git a/changes/ticket28459 b/changes/ticket28459 deleted file mode 100644 index 6b5839b52b..0000000000 --- a/changes/ticket28459 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (continuous integration, Windows): - - Always show the configure and test logs, and upload them as build - artifacts, when building for Windows using Appveyor CI. - Implements 28459. diff --git a/changes/ticket28574 b/changes/ticket28574 deleted file mode 100644 index 562810f511..0000000000 --- a/changes/ticket28574 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (continuous integration, Windows): - - Explicitly specify the path to the OpenSSL library and do not download - OpenSSL from Pacman, but instead use the library that is already provided - by AppVeyor. Fixes bug 28574; bugfix on master. diff --git a/changes/ticket28614 b/changes/ticket28614 new file mode 100644 index 0000000000..3c93313726 --- /dev/null +++ b/changes/ticket28614 @@ -0,0 +1,8 @@ + o Major bugfixes (windows, startup): + - When writing a consensus file to disk, always write in + "binary" mode so that we can safely map it into memory later. + Fixes part of bug 28614; bugfix on 0.4.0.1-alpha. + - When reading a consensus file from disk, detect whether it + was written in text mode, and re-read it in text mode if so. + Fixes part of bug 28614; bugfix on 0.4.0.1-alpha. + diff --git a/changes/ticket28669 b/changes/ticket28669 deleted file mode 100644 index 32c6114ffc..0000000000 --- a/changes/ticket28669 +++ /dev/null @@ -1,6 +0,0 @@ - o Minor bugfix (hidden service v3, client): - - Avoid a BUG() stacktrace in case a SOCKS connection is found waiting for - the descriptor while we do have it in the cache. There is a rare case - when this can happen. Now, tor will recover and retry the descriptor. - Fixes bug 28669; bugfix on 0.3.2.4-alpha. - diff --git a/changes/ticket28816 b/changes/ticket28816 new file mode 100644 index 0000000000..02878ccfdc --- /dev/null +++ b/changes/ticket28816 @@ -0,0 +1,4 @@ + o Code simplification and refactoring: + - Introduce a connection_dir_buf_add() helper function that checks for + compress_state of dir_connection_t and automatically writes a string to + directory connection with or without compression. Resolves issue 28816. diff --git a/changes/ticket28838 b/changes/ticket28838 deleted file mode 100644 index 6c290bf82b..0000000000 --- a/changes/ticket28838 +++ /dev/null @@ -1,8 +0,0 @@ - o Minor features (performance): - - Remove about 96% of the work from the function that we run at - startup to test our curve25519_basepoint implementation. Since - this function has yet to find an actual failure, we'll only - run it for 8 iterations instead of 200. Based on our profile - information, this change should save around 8% of our startup - time on typical desktops, and may have a similar effect on - other platforms. Closes ticket 28838. diff --git a/changes/ticket28851 b/changes/ticket28851 deleted file mode 100644 index bab0318662..0000000000 --- a/changes/ticket28851 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (performance): - - Stop re-validating our hardcoded Diffie-Hellman parameters on every - startup. Doing this wasted time and cycles, especially on low-powered - devices. Closes ticket 28851. diff --git a/changes/ticket28879 b/changes/ticket28879 deleted file mode 100644 index 126420f6ca..0000000000 --- a/changes/ticket28879 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (correctness): - - Fix an unreached code-path where we checked the value of "hostname" - inside send_resolved_hostnam_cell(). Previously, we used it before - checking it; now we check it first. Fixes bug 28879; bugfix on - 0.1.2.7-alpha. diff --git a/changes/ticket28881 b/changes/ticket28881 deleted file mode 100644 index 1b015a6c37..0000000000 --- a/changes/ticket28881 +++ /dev/null @@ -1,4 +0,0 @@ - o Code simplification and refactoring: - - When parsing a port configuration, make it more - obvious to static analyzer tools that we will always initialize the - address. Closes ticket 28881. diff --git a/changes/ticket28883 b/changes/ticket28883 deleted file mode 100644 index 1d8b6cb416..0000000000 --- a/changes/ticket28883 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (testing): - - Make sure that test_rebind.py actually obeys its timeout, even - when it receives a large number of log messages. Fixes bug 28883; - bugfix on 0.3.5.4-alpha. diff --git a/changes/ticket28912 b/changes/ticket28912 deleted file mode 100644 index 4119b778bc..0000000000 --- a/changes/ticket28912 +++ /dev/null @@ -1,6 +0,0 @@ - o Major bugfixes (relay, directory): - - A connection serving directory information wouldn't get reactivated after - the first chunk of data was sent (usually 32KB). Tor now always activate - the main loop event that goes through these connections as long as at - least one connection is still active. Fixes bug 28912; bugfix on - 0.3.4.1-alpha. Patch by "cypherpunks3". diff --git a/changes/ticket28924 b/changes/ticket28924 deleted file mode 100644 index 055a6cf285..0000000000 --- a/changes/ticket28924 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (compilation): - - When possible, place our warning flags in a separate file, to avoid - flooding verbose build logs to an unacceptable amount. Closes ticket - 28924. diff --git a/changes/ticket28973 b/changes/ticket28973 deleted file mode 100644 index b1d208ee51..0000000000 --- a/changes/ticket28973 +++ /dev/null @@ -1,6 +0,0 @@ - o Minor features (OpenSSL bug workaround): - - Work around a bug in OpenSSL 1.1.1a, which prevented the TLS 1.3 - key export function from handling long labels. When this bug - is detected, Tor will disable TLS 1.3. We recommend upgrading to - a version of OpenSSL without this bug when it becomes available. - Closes ticket 28973. diff --git a/changes/ticket29072 b/changes/ticket29072 new file mode 100644 index 0000000000..3526330f30 --- /dev/null +++ b/changes/ticket29072 @@ -0,0 +1,2 @@ + o Removed features: + - Remove check-tor script from repository. Resolves issue 29072. diff --git a/changes/ticket29357 b/changes/ticket29357 new file mode 100644 index 0000000000..3aab930cd4 --- /dev/null +++ b/changes/ticket29357 @@ -0,0 +1,7 @@ + o Minor features (dormant mode): + - Add a DormantCanceledByStartup option to tell Tor that it should + treat a startup event as cancelling any previous dormant state. + Integrators should use this option with caution: it should + only be used if Tor is being started because of something that the + user did, and not if Tor is being automatically started in the + background. Closes ticket 29357. diff --git a/changes/ticket29631 b/changes/ticket29631 new file mode 100644 index 0000000000..9fc194ba96 --- /dev/null +++ b/changes/ticket29631 @@ -0,0 +1,4 @@ + o Minor bugfixes (Rust, protover): + - The Rust implementation of protover was missing the "Padding" value in + the translate function from C to Rust. Fixes bug 29631; bugfix on + 0.4.0.1-alpha. diff --git a/changes/ticket29897 b/changes/ticket29897 new file mode 100644 index 0000000000..232a79fbce --- /dev/null +++ b/changes/ticket29897 @@ -0,0 +1,3 @@ + o Code simplification and refactoring: + - Refactor handle_get_next_bandwidth() to use connection_dir_buf_add(). + Implements ticket 29897. diff --git a/changes/ticket31091 b/changes/ticket31091 new file mode 100644 index 0000000000..3cb9a2c37b --- /dev/null +++ b/changes/ticket31091 @@ -0,0 +1,3 @@ + o Minor bugfixes (pluggable transports): + - Remove overly strict assertions that triggers when a pluggable transport + is spawned in an unsuccessful manner. Fixes bug 31091; bugfix on 0.4.0.1-alpha. diff --git a/configure.ac b/configure.ac index 834fea073a..8990cd9fc7 100644 --- a/configure.ac +++ b/configure.ac @@ -4,10 +4,19 @@ dnl Copyright (c) 2007-2019, The Tor Project, Inc. dnl See LICENSE for licensing information AC_PREREQ([2.63]) -AC_INIT([tor],[0.3.5.9-dev]) +AC_INIT([tor],[0.4.0.6-dev]) AC_CONFIG_SRCDIR([src/app/main/tor_main.c]) AC_CONFIG_MACRO_DIR([m4]) +# DO NOT EDIT THIS DEFINITION BY HAND UNLESS YOU KNOW WHAT YOU'RE DOING. +# +# The update_versions.py script updates this definition when the +# version number changes. Tor uses it to make sure that it +# only shuts down for missing "required protocols" when those protocols +# are listed as required by a consensus after this date. +AC_DEFINE(APPROX_RELEASE_DATE, ["2019-12-09"], # for 0.4.0.6-dev + [Approximate date when this software was released. (Updated when the version changes.)]) + # "foreign" means we don't follow GNU package layout standards # "1.11" means we require automake version 1.11 or newer # "subdir-objects" means put .o files in the same directory as the .c files @@ -503,15 +512,6 @@ if test "x$enable_rust" = "xyes"; then fi fi - dnl This is a workaround for #46797 - dnl (a.k.a https://github.com/rust-lang/rust/issues/46797 ). Once the - dnl upstream bug is fixed, we can remove this workaround. - case "$host_os" in - darwin*) - TOR_RUST_EXTRA_LIBS="-lresolv" - ;; - esac - dnl For now both MSVC and MinGW rust libraries will output static libs with dnl the MSVC naming convention. if test "$bwin32" = "true"; then @@ -556,8 +556,6 @@ if test "x$enable_rust" = "xyes"; then AC_MSG_RESULT([$RUSTC_VERSION]) fi -AC_SUBST(TOR_RUST_EXTRA_LIBS) - AC_SEARCH_LIBS(socket, [socket network]) AC_SEARCH_LIBS(gethostbyname, [nsl]) AC_SEARCH_LIBS(dlopen, [dl]) @@ -607,8 +605,10 @@ AC_CHECK_FUNCS( llround \ localtime_r \ lround \ + madvise \ memmem \ memset_s \ + minherit \ mmap \ pipe \ pipe2 \ @@ -812,6 +812,8 @@ fi dnl Now check for particular libevent functions. AC_CHECK_FUNCS([evutil_secure_rng_set_urandom_device_file \ evutil_secure_rng_add_bytes \ + evdns_base_get_nameserver_addr \ + ]) LIBS="$save_LIBS" @@ -1196,6 +1198,11 @@ if test "$fragile_hardening" = "yes"; then AC_MSG_ERROR([The compiler supports -fsanitize=undefined, but for some reason I was not able to link when using it. Are you missing run-time support? With GCC you need libasan.*, and with Clang you need libclang_rt.ubsan*]) fi + TOR_TRY_COMPILE_WITH_CFLAGS([-fno-sanitize=float-divide-by-zero], also_link, CFLAGS_UBSAN="-fno-sanitize=float-divide-by-zero", true) + if test "$tor_cv_cflags__fno_sanitize_float_divide_by_zero" = "yes" && test "$tor_can_link__fno_sanitize_float_divide_by_zero" != "yes"; then + AC_MSG_ERROR([The compiler supports -fno-sanitize=float-divide-by-zero, but for some reason I was not able to link when using it. Are you missing run-time support? With GCC you need libasan.*, and with Clang you need libclang_rt.ubsan*]) + fi + TOR_CHECK_CFLAGS([-fno-omit-frame-pointer]) fi @@ -1450,6 +1457,7 @@ AC_CHECK_HEADERS([errno.h \ inttypes.h \ limits.h \ linux/types.h \ + mach/vm_inherit.h \ machine/limits.h \ malloc.h \ malloc/malloc.h \ @@ -2445,7 +2453,6 @@ AC_CONFIG_FILES([ src/config/torrc.minimal src/rust/.cargo/config scripts/maint/checkOptionDocs.pl - scripts/maint/updateVersions.pl warning_flags ]) diff --git a/contrib/or-tools/check-tor b/contrib/or-tools/check-tor deleted file mode 100755 index e981a35fcc..0000000000 --- a/contrib/or-tools/check-tor +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/sh - -## Originally written by Peter Palfrader. - -## This script lets you quickly check if a given router (by nickname) -## will let you do a TLS handshake, or will let you download a directory. - -## Usage: check-tor nickname - -#set -x - -router="$1" -dirserver="http://belegost.seul.org:80/tor/" - -lines=$( wget -q $dirserver --proxy=off -O - | grep -A5 '^router '"$router"' ' ) -line=$( echo "$lines" | head -n1 ) - -if [ -z "$line" ]; then - echo "Not found" >&2 - exit 1 -fi - -echo "$lines" -echo - -ipor=$( echo "$line" | awk '{printf "%s:%s", $3, $4}' ) - -op=$( echo "$line" | awk '{printf $6}' ) -ipop=$( echo "$line" | awk '{printf "%s:%s", $3, $6}' ) - -echo -echo ">>" openssl s_client -connect "$ipor" -timeout 5 openssl s_client -connect "$ipor" < /dev/null -if [ "$op" != "0" ]; then - echo - echo ">>" wget --proxy=off -O - http://$ipop/tor/ - timeout 5 wget --proxy=off -O - http://$ipop/tor/ | head -n3 -fi - -echo -echo -n "$router "; echo "$lines" | grep 'fingerprint' | sed -e 's/^opt //' -e 's/^fingerprint //'; diff --git a/contrib/win32build/tor-mingw.nsi.in b/contrib/win32build/tor-mingw.nsi.in index 9eeb328223..93f3f72ec7 100644 --- a/contrib/win32build/tor-mingw.nsi.in +++ b/contrib/win32build/tor-mingw.nsi.in @@ -8,7 +8,7 @@ !include "LogicLib.nsh" !include "FileFunc.nsh" !insertmacro GetParameters -!define VERSION "0.3.5.9-dev" +!define VERSION "0.4.0.6-dev" !define INSTALLER "tor-${VERSION}-win32.exe" !define WEBSITE "https://www.torproject.org/" !define LICENSE "LICENSE" diff --git a/doc/HACKING/Maintaining.md b/doc/HACKING/Maintaining.md new file mode 100644 index 0000000000..4d5a7f6b76 --- /dev/null +++ b/doc/HACKING/Maintaining.md @@ -0,0 +1,113 @@ +# Maintaining Tor + +This document details the duties and processes on maintaining the Tor code +base. + +The first section describes who is the current Tor maintainer and what are the +responsibilities. Tor has one main single maintainer but does have many +committers and subsystem maintainers. + +The second third section describes how the **alpha and master** branches are +maintained and by whom. + +Finally, the last section describes how the **stable** branches are maintained +and by whom. + +This document does not cover how Tor is released, please see +[ReleasingTor.md](ReleasingTor.md) for that information. + +## Tor Maintainer + +The current maintainer is Nick Mathewson <nickm@torproject.org>. + +The maintainer takes final decisions in terms of engineering, architecture and +protocol design. Releasing Tor falls under their responsibility. + +## Alpha and Master Branches + +The Tor repository always has at all times a **master** branch which contains +the upstream ongoing development. + +It may also contain a branch for a released feature freezed version which is +called the **alpha** branch. The git tag and version number is always +postfixed with `-alpha[-dev]`. For example: `tor-0.3.5.0-alpha-dev` or +`tor-0.3.5.3-alpha`. + +Tor is separated into subsystems and some of those are maintained by other +developers than the main maintainer. Those people have commit access to the +code base but only commit (in most cases) into the subsystem they maintain. + +Upstream merges are restricted to the alpha and master branches. Subsystem +maintainers should never push a patch into a stable branch which is the +responsibility of the [stable branch maintainer](#stable-branches). + +### Who + +In alphabetical order, the following people have upstream commit access and +maintain the following subsystems: + +- David Goulet <dgoulet@torproject.org> + * Onion Service (including Shared Random). + ***keywords:*** *[tor-hs]* + * Channels, Circuitmux, Connection, Scheduler. + ***keywords:*** *[tor-chan, tor-cmux, tor-sched, tor-conn]* + * Cell Logic (Handling/Parsing). + ***keywords:*** *[tor-cell]* + * Threading backend. + ***keywords:*** *[tor-thread]* + +- George Kadianakis <asn@torproject.org> + * Onion Service (including Shared Random). + ***keywords:*** *[tor-hs]* + * Guard. + ***keywords:*** *[tor-guard]* + * Pluggable Transport (excluding Bridge networking). + ***keywords:*** *[tor-pt]* + +### Tasks + +These are the tasks of a subsystem maintainer: + +1. Regularly go over `merge_ready` tickets relevant to the related subsystem + and for the current alpha or development (master branch) Milestone. + +2. A subsystem maintainer is expected to contribute to any design changes + (including proposals) or large patch set about the subsystem. + +3. Leave their ego at the door. Mistakes will be made but they have to be + taking care of seriously. Learn and move on quickly. + +### Merging Policy + +These are few important items to follow when merging code upstream: + +1. To merge code upstream, the patch must have passed our CI (currently + github.com/torproject), have a corresponding ticket and reviewed by + **at least** one person that is not the original coder. + + Example A: If Alice writes a patch then Bob, a Tor network team member, + reviews it and flags it `merge_ready`. Then, the maintainer is required + to look at the patch and makes a decision. + + Example B: If the maintainer writes a patch then Bob, a Tor network + team member, reviews it and flags it `merge_ready`, then the maintainer + can merge the code upstream. + +2. Maintainer makes sure the commit message should describe what was fixed + and, if it applies, how was it fixed. It should also always refer to + the ticket number. + +3. Trivial patches such as comment change, documentation, syntax issues or + typos can be merged without a ticket or reviewers. + +4. Tor uses the "merge forward" method, that is, if a patch applies to the + alpha branch, it has to be merged there first and then merged forward + into master. + +5. Maintainer should always consult with the network team about any doubts, + mis-understandings or unknowns of a patch. Final word will always go to the + main Tor maintainer. + +## Stable Branches + +(Currently being drafted and reviewed by the network team.) diff --git a/doc/HACKING/ReleasingTor.md b/doc/HACKING/ReleasingTor.md index 55a40fc89b..7334b1b34a 100644 --- a/doc/HACKING/ReleasingTor.md +++ b/doc/HACKING/ReleasingTor.md @@ -20,30 +20,29 @@ new Tor release: === I. Make sure it works -1. Use it for a while, as a client, as a relay, as a hidden service, - and as a directory authority. See if it has any obvious bugs, and - resolve those. +1. Make sure that CI passes: have a look at Travis, Appveyor, and + Jenkins. Make sure you're looking at the right branches. - As applicable, merge the `maint-X` branch into the `release-X` branch. - But you've been doing that all along, right? + If there are any unexplained failures, try to fix them or figure them + out. -2. Are all of the jenkins builders happy? See jenkins.torproject.org. +2. Verify that there are no big outstanding issues. You might find such + issues -- - What about the bsd buildbots? - See http://buildbot.pixelminers.net/builders/ + * On Trac - What about Coverity Scan? + * On coverity scan - What about clang scan-build? + * On OSS-Fuzz - Does 'make distcheck' complain? +3. Run checks that aren't covered above, including: - How about 'make test-stem' and 'make test-network' and - `make test-network-full`? + * clang scan-build. (See the script in ./scripts/test/scan_build.sh) - - Are all those tests still happy with --enable-expensive-hardening ? + * make test-network and make test-network-all (with + --enable-expensive-hardening) - Any memory leaks? + * Running Tor yourself and making sure that it actually works for you. === II. Write a changelog @@ -55,11 +54,14 @@ new Tor release: of them and reordering to focus on what users and funders would find interesting and understandable. - To do this, first run `./scripts/maint/lintChanges.py changes/*` and - fix as many warnings as you can. Then run `./scripts/maint/sortChanges.py - changes/* > changelog.in` to combine headings and sort the entries. - After that, it's time to hand-edit and fix the issues that lintChanges - can't find: + To do this, run + `./scripts/maint/sortChanges.py changes/* > changelog.in` + to combine headings and sort the entries. Copy the changelog.in file + into the ChangeLog. Run 'format_changelog.py' (see below) to clean + up the line breaks. + + After that, it's time to hand-edit and fix the issues that + lintChanges can't find: 1. Within each section, sort by "version it's a bugfix on", else by numerical ticket order. @@ -68,8 +70,6 @@ new Tor release: Make stuff very terse - Make sure each section name ends with a colon - Describe the user-visible problem right away Mention relevant config options by name. If they're rare or unusual, @@ -79,7 +79,9 @@ new Tor release: Present and imperative tense: not past. - 'Relays', not 'servers' or 'nodes' or 'Tor relays'. + "Relays", not "servers" or "nodes" or "Tor relays". + + "Onion services", not "hidden services". "Stop FOOing", not "Fix a bug where we would FOO". @@ -100,12 +102,14 @@ new Tor release: For stable releases that backport things from later, we try to compose their releases, we try to make sure that we keep the changelog entries - identical to their original versions, with a 'backport from 0.x.y.z' + identical to their original versions, with a "backport from 0.x.y.z" note added to each section. So in this case, once you have the items from the changes files copied together, don't use them to build a new changelog: instead, look up the corrected versions that were merged into ChangeLog in the master branch, and use those. + Add "backport from X.Y.Z" in the section header for these entries. + 2. Compose a short release blurb to highlight the user-facing changes. Insert said release blurb into the ChangeLog stanza. If it's a stable release, add it to the ReleaseNotes file too. If we're adding @@ -131,44 +135,57 @@ new Tor release: === III. Making the source release. 1. In `maint-0.?.x`, bump the version number in `configure.ac` and run - `perl scripts/maint/updateVersions.pl` to update version numbers in other + `make update-versions` to update version numbers in other places, and commit. Then merge `maint-0.?.x` into `release-0.?.x`. - (NOTE: To bump the version number, edit `configure.ac`, and then run - either `make`, or `perl scripts/maint/updateVersions.pl`, depending on - your version.) - When you merge the maint branch forward to the next maint branch, or into master, merge it with "-s ours" to avoid a needless version bump. 2. Make distcheck, put the tarball up in somewhere (how about your - homedir on your homedir on people.torproject.org?) , and tell `#tor` - about it. Wait a while to see if anybody has problems building it. - (Though jenkins is usually pretty good about catching these things.) + homedir on your homedir on people.torproject.org?) , and tell `#tor-dev` + about it. + + If you want, wait until at least one person has built it + successfully. (We used to say "wait for others to test it", but our + CI has successfully caught these kinds of errors for the last several + years.) + + +3. Make sure that the new version is recommended in the latest consensus. + (Otherwise, users will get confused when it complains to them + about its status.) + + If it is not, you'll need to poke Roger, Weasel, and Sebastian again: see + item 0.1 at the start of this document. === IV. Commit, upload, announce 1. Sign the tarball, then sign and push the git tag: gpg -ba <the_tarball> - git tag -u <keyid> tor-0.3.x.y-status - git push origin tag tor-0.3.x.y-status + git tag -s tor-0.4.x.y-<status> + git push origin tag tor-0.4.x.y-<status> + + (You must do this before you update the website: the website scripts + rely on finding the version by tag.) - (You must do this before you update the website: it relies on finding - the version by tag.) + (If your default PGP key is not the one you want to sign with, then say + "-u <keyid>" instead of "-s".) 2. scp the tarball and its sig to the dist website, i.e. - `/srv/dist-master.torproject.org/htdocs/` on dist-master. When you want - it to go live, you run "static-update-component dist.torproject.org" - on dist-master. + `/srv/dist-master.torproject.org/htdocs/` on dist-master. Run + "static-update-component dist.torproject.org" on dist-master. In the webwml.git repository, `include/versions.wmi` and `Makefile` - to note the new version. + to note the new version. Push these changes to master. (NOTE: Due to #17805, there can only be one stable version listed at once. Nonetheless, do not call your version "alpha" if it is stable, or people will get confused.) + (NOTE: It will take a while for the website update scripts to update + the website.) + 3. Email the packagers (cc'ing tor-team) that a new tarball is up. The current list of packagers is: @@ -186,29 +203,35 @@ new Tor release: Also, email tor-packagers@lists.torproject.org. + Mention where to download the tarball (https://dist.torproject.org). + + Include a link to the changelog. + + 4. Add the version number to Trac. To do this, go to Trac, log in, select "Admin" near the top of the screen, then select "Versions" from the menu on the left. At the right, there will be an "Add version" box. By convention, we enter the version in the form "Tor: - 0.2.2.23-alpha" (or whatever the version is), and we select the date as + 0.4.0.1-alpha" (or whatever the version is), and we select the date as the date in the ChangeLog. -5. Double-check: did the version get recommended in the consensus yet? Is - the website updated? If not, don't announce until they have the - up-to-date versions, or people will get confused. +5. Wait for the download page to be updated. (If you don't do this before you + announce, people will be confused.) 6. Mail the release blurb and ChangeLog to tor-talk (development release) or tor-announce (stable). Post the changelog on the blog as well. You can generate a - blog-formatted version of the changelog with the -B option to - format-changelog. + blog-formatted version of the changelog with + `./scripts/maint/format_changelog.py --B` When you post, include an estimate of when the next TorBrowser releases will come out that include this Tor release. This will usually track https://wiki.mozilla.org/RapidRelease/Calendar , but it can vary. + For templates to use when announcing, see: + https://trac.torproject.org/projects/tor/wiki/org/teams/NetworkTeam/AnnouncementTemplates === V. Aftermath and cleanup @@ -216,7 +239,7 @@ new Tor release: `maint-x.y.z` branch to "newversion-dev", and do a `merge -s ours` merge to avoid taking that change into master. -2. Forward-port the ChangeLog (and ReleaseNotes if appropriate). +2. Forward-port the ChangeLog (and ReleaseNotes if appropriate) to the + master branch. 3. Keep an eye on the blog post, to moderate comments and answer questions. - diff --git a/doc/tor.1.txt b/doc/tor.1.txt index 21b482802e..a02b1372c8 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -93,7 +93,9 @@ COMMAND-LINE OPTIONS into Tor. (Any module not listed is not optional in this version of Tor.) [[opt-version]] **--version**:: - Display Tor version and exit. + Display Tor version and exit. The output is a single line of the format + "Tor version [version number]." (The version number format + is as specified in version-spec.txt.) [[opt-quiet]] **--quiet**|**--hush**:: Override the default console log. By default, Tor starts out logging @@ -352,7 +354,7 @@ GENERAL OPTIONS all sockets will be set to this limit. Must be a value between 2048 and 262144, in 1024 byte increments. Default of 8192 is recommended. -[[ControlPort]] **ControlPort** __PORT__|**unix:**__path__|**auto** [__flags__]:: +[[ControlPort]] **ControlPort** \['address':]__port__|**unix:**__path__|**auto** [__flags__]:: If set, Tor will accept connections on this port and allow those connections to control the Tor process using the Tor Control Protocol (described in control-spec.txt in @@ -363,7 +365,8 @@ GENERAL OPTIONS methods means either method is sufficient to authenticate to Tor.) This option is required for many Tor controllers; most use the value of 9051. If a unix domain socket is used, you may quote the path using standard - C escape sequences. + C escape sequences. You can specify this directive multiple times, to + bind to multiple address/port pairs. Set it to "auto" to have Tor pick a port for you. (Default: 0) + + Recognized flags are... @@ -602,20 +605,26 @@ GENERAL OPTIONS Otherwise the sandbox will be disabled. The option is currently an experimental feature. It only works on Linux-based operating systems, and only when Tor has been built with the libseccomp library. This option - can not be changed while tor is running. + can not be changed while tor is running. + + - When the Sandbox is 1, the following options can not be changed when tor + When the **Sandbox** is 1, the following options can not be changed when tor is running: - Address - ConnLimit - CookieAuthFile - DirPortFrontPage - ExtORPortCookieAuthFile - Logs - ServerDNSResolvConfFile - Tor must remain in client or server mode (some changes to ClientOnly and - ORPort are not allowed). - ClientOnionAuthDir and any files in it won't reload on HUP signal. + **Address**, + **ConnLimit**, + **CookieAuthFile**, + **DirPortFrontPage**, + **ExtORPortCookieAuthFile**, + **Logs**, + **ServerDNSResolvConfFile**, + **ClientOnionAuthDir** (and any files in it won't reload on HUP signal). + + + Launching new Onion Services through the control port is not supported + with current syscall sandboxing implementation. + + + Tor must remain in client or server mode (some changes to **ClientOnly** + and **ORPort** are not allowed). Currently, if **Sandbox** is 1, + **ControlPort** command "GETINFO address" will not work. + + (Default: 0) [[Socks4Proxy]] **Socks4Proxy** __host__[:__port__]:: @@ -673,7 +682,8 @@ GENERAL OPTIONS + The currently recognized domains are: general, crypto, net, config, fs, protocol, mm, http, app, control, circ, rend, bug, dir, dirserv, or, edge, - acct, hist, handshake, heartbeat, channel, sched, guard, consdiff, and dos. + acct, hist, handshake, heartbeat, channel, sched, guard, consdiff, dos, + process, pt, and btrack. Domain names are case-insensitive. + + For example, "`Log [handshake]debug [~net,~mm]info notice stdout`" sends @@ -1015,6 +1025,26 @@ The following options are useful only for clients (that is, if The .exit address notation, if enabled via MapAddress, overrides this option. +[[MiddleNodes]] **MiddleNodes** __node__,__node__,__...__:: + A list of identity fingerprints and country codes of nodes + to use for "middle" hops in your normal circuits. + Normal circuits include all circuits except for direct connections + to directory servers. Middle hops are all hops other than exit and entry. + ++ + This is an **experimental** feature that is meant to be used by researchers + and developers to test new features in the Tor network safely. Using it + without care will strongly influence your anonymity. This feature might get + removed in the future. ++ + The HSLayer2Node and HSLayer3Node options override this option for onion + service circuits, if they are set. The vanguards addon will read this + option, and if set, it will set HSLayer2Nodes and HSLayer3Nodes to nodes + from this set. ++ + The ExcludeNodes option overrides this option: any node listed in both + MiddleNodes and ExcludeNodes is treated as excluded. See + the **ExcludeNodes** option for more information on how to specify nodes. + [[EntryNodes]] **EntryNodes** __node__,__node__,__...__:: A list of identity fingerprints and country codes of nodes to use for the first hop in your normal circuits. @@ -1031,13 +1061,14 @@ The following options are useful only for clients (that is, if If StrictNodes is set to 1, Tor will treat solely the ExcludeNodes option as a requirement to follow for all the circuits you generate, even if doing so will break functionality for you (StrictNodes applies to neither - ExcludeExitNodes nor to ExitNodes). If StrictNodes is set to 0, Tor will - still try to avoid nodes in the ExcludeNodes list, but it will err on the - side of avoiding unexpected errors. Specifically, StrictNodes 0 tells Tor - that it is okay to use an excluded node when it is *necessary* to perform - relay reachability self-tests, connect to a hidden service, provide a - hidden service to a client, fulfill a .exit request, upload directory - information, or download directory information. (Default: 0) + ExcludeExitNodes nor to ExitNodes, nor to MiddleNodes). If StrictNodes + is set to 0, Tor will still try to avoid nodes in the ExcludeNodes list, + but it will err on the side of avoiding unexpected errors. + Specifically, StrictNodes 0 tells Tor that it is okay to use an excluded + node when it is *necessary* to perform relay reachability self-tests, + connect to a hidden service, provide a hidden service to a client, + fulfill a .exit request, upload directory information, or download + directory information. (Default: 0) [[FascistFirewall]] **FascistFirewall** **0**|**1**:: If 1, Tor will only create outgoing connections to ORs running on ports @@ -1132,17 +1163,18 @@ The following options are useful only for clients (that is, if 1. When evaluating MapAddress expressions Tor stops when it hits the most recently added expression that matches the requested address. So if you - have the following in your torrc, www.torproject.org will map to 1.1.1.1: + have the following in your torrc, www.torproject.org will map to + 198.51.100.1: - MapAddress www.torproject.org 2.2.2.2 - MapAddress www.torproject.org 1.1.1.1 + MapAddress www.torproject.org 192.0.2.1 + MapAddress www.torproject.org 198.51.100.1 2. Tor evaluates the MapAddress configuration until it finds no matches. So if you have the following in your torrc, www.torproject.org will map to - 2.2.2.2: + 203.0.113.1: - MapAddress 1.1.1.1 2.2.2.2 - MapAddress www.torproject.org 1.1.1.1 + MapAddress 198.51.100.1 203.0.113.1 + MapAddress www.torproject.org 198.51.100.1 3. The following MapAddress expression is invalid (and will be ignored) because you cannot map from a specific address to a wildcard @@ -1202,6 +1234,8 @@ The following options are useful only for clients (that is, if information to anybody watching your network, and allow anybody to use your computer as an open proxy. + + + If multiple entries of this option are present in your configuration + file, Tor will perform stream isolation between listeners by default. The _isolation flags_ arguments give Tor rules for which streams received on this SocksPort are allowed to share circuits with one another. Recognized isolation flags are: @@ -1444,14 +1478,18 @@ The following options are useful only for clients (that is, if protocol instead of SOCKS. Set this to 0 if you don't want to allow "HTTP CONNECT" connections. Set the port to "auto" to have Tor pick a port for you. This directive can be - specified multiple times to bind to multiple addresses/ports. See + specified multiple times to bind to multiple addresses/ports. If multiple + entries of this option are present in your configuration file, Tor will + perform stream isolation between listeners by default. See SOCKSPort for an explanation of isolation flags. (Default: 0) [[TransPort]] **TransPort** \['address':]__port__|**auto** [_isolation flags_]:: Open this port to listen for transparent proxy connections. Set this to 0 if you don't want to allow transparent proxy connections. Set the port to "auto" to have Tor pick a port for you. This directive can be - specified multiple times to bind to multiple addresses/ports. See + specified multiple times to bind to multiple addresses/ports. If multiple + entries of this option are present in your configuration file, Tor will + perform stream isolation between listeners by default. See SOCKSPort for an explanation of isolation flags. + + TransPort requires OS support for transparent proxies, such as BSDs' pf or @@ -1488,7 +1526,9 @@ The following options are useful only for clients (that is, if included in old versions of FreeBSD, etc) using the NATD protocol. Use 0 if you don't want to allow NATD connections. Set the port to "auto" to have Tor pick a port for you. This directive can be - specified multiple times to bind to multiple addresses/ports. See + specified multiple times to bind to multiple addresses/ports. If multiple + entries of this option are present in your configuration file, Tor will + perform stream isolation between listeners by default. See SocksPort for an explanation of isolation flags. + + This option is only for people who cannot use TransPort. (Default: 0) @@ -1753,6 +1793,12 @@ The following options are useful only for clients (that is, if other clients prefer IPv4. Other things may influence the choice. This option breaks a tie to the favor of IPv6. (Default: auto) +[[ClientAutoIPv6ORPort]] **ClientAutoIPv6ORPort** **0**|**1**:: + If this option is set to 1, Tor clients randomly prefer a node's IPv4 or + IPv6 ORPort. The random preference is set every time a node is loaded + from a new consensus or bridge config. When this option is set to 1, + **ClientPreferIPv6ORPort** is ignored. (Default: 0) + [[PathsNeededToBuildCircuits]] **PathsNeededToBuildCircuits** __NUM__:: Tor clients don't build circuits for user traffic until they know about enough of the network so that they could potentially construct @@ -1794,6 +1840,43 @@ The following options are useful only for clients (that is, if Try this many simultaneous connections to download a consensus before waiting for one to complete, timeout, or error out. (Default: 3) +[[DormantClientTimeout]] **DormantClientTimeout** __N__ **minutes**|**hours**|**days**|**weeks**:: + If Tor spends this much time without any client activity, + enter a dormant state where automatic circuits are not built, and + directory information is not fetched. + Does not affect servers or onion services. Must be at least 10 minutes. + (Default: 24 hours) + +[[DormantTimeoutDisabledByIdleStreams]] **DormantTimeoutDisabledByIdleStreams** **0**|**1**:: + If true, then any open client stream (even one not reading or writing) + counts as client activity for the purpose of DormantClientTimeout. + If false, then only network activity counts. (Default: 1) + +[[DormantOnFirstStartup]] **DormantOnFirstStartup** **0**|**1**:: + If true, then the first time Tor starts up with a fresh DataDirectory, + it starts in dormant mode, and takes no actions until the user has made + a request. (This mode is recommended if installing a Tor client for a + user who might not actually use it.) If false, Tor bootstraps the first + time it is started, whether it sees a user request or not. + + + After the first time Tor starts, it begins in dormant mode if it was + dormant before, and not otherwise. (Default: 0) + +[[DormantCanceledByStartup]] **DormantCanceledByStartup** **0**|**1**:: + By default, Tor starts in active mode if it was active the last time + it was shut down, and in dormant mode if it was dormant. But if + this option is true, Tor treats every startup event as user + activity, and Tor will never start in Dormant mode, even if it has + been unused for a long time on previous runs. (Default: 0) + + + Note: Packagers and application developers should change the value of + this option only with great caution: it has the potential to + create spurious traffic on the network. This option should only + be used if Tor is started by an affirmative user activity (like + clicking on an applcation or running a command), and not if Tor + is launched for some other reason (for example, by a startup + process, or by an application that launches itself on every login.) + SERVER OPTIONS -------------- @@ -2217,7 +2300,8 @@ is non-zero): __filename__. The file format is the same as the standard Unix "**resolv.conf**" file (7). This option, like all other ServerDNS options, only affects name lookups that your server does on behalf of clients. - (Defaults to use the system DNS configuration.) + (Defaults to use the system DNS configuration or a localhost DNS service + in case no nameservers are found in a given configuration.) [[ServerDNSAllowBrokenConfig]] **ServerDNSAllowBrokenConfig** **0**|**1**:: If this option is false, Tor exits immediately if there are problems @@ -2806,6 +2890,8 @@ The following options are used to configure a hidden service. Store data files for a hidden service in DIRECTORY. Every hidden service must have a separate directory. You may use this option multiple times to specify multiple services. If DIRECTORY does not exist, Tor will create it. + Please note that you cannot add new Onion Service to already running Tor + instance if **Sandbox** is enabled. (Note: in current versions of Tor, if DIRECTORY is a relative path, it will be relative to the current working directory of Tor instance, not to its DataDirectory. Do not @@ -3260,10 +3346,6 @@ __CacheDirectory__**/cached-microdescs** and **cached-microdescs.new**:: router. The ".new" file is an append-only journal; when it gets too large, all entries are merged into a new cached-microdescs file. -__CacheDirectory__**/cached-routers** and **cached-routers.new**:: - Obsolete versions of cached-descriptors and cached-descriptors.new. When - Tor can't find the newer files, it looks here instead. - __DataDirectory__**/state**:: A set of persistent key-value mappings. These are documented in the file. These include: diff --git a/scripts/codegen/fuzzing_include_am.py b/scripts/codegen/fuzzing_include_am.py index 3c948d87cf..a944584453 100755 --- a/scripts/codegen/fuzzing_include_am.py +++ b/scripts/codegen/fuzzing_include_am.py @@ -13,6 +13,7 @@ FUZZERS = """ iptsv2 microdesc socks + strops vrs """ @@ -23,12 +24,12 @@ FUZZING_CPPFLAGS = \ FUZZING_CFLAGS = \ $(AM_CFLAGS) $(TEST_CFLAGS) FUZZING_LDFLAG = \ - @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ @TOR_LDFLAGS_libevent@ + @TOR_LDFLAGS_zlib@ $(TOR_LDFLAGS_CRYPTLIB) @TOR_LDFLAGS_libevent@ FUZZING_LIBS = \ $(TOR_INTERNAL_TESTING_LIBS) \ $(rust_ldadd) \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \ - @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ \ + @TOR_LIBEVENT_LIBS@ $(TOR_LIBS_CRYPTLIB) \ @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ @CURVE25519_LIBS@ \ @TOR_SYSTEMD_LIBS@ \ @TOR_LZMA_LIBS@ \ diff --git a/scripts/codegen/run_trunnel.sh b/scripts/codegen/run_trunnel.sh index 428804342a..645b3c2158 100755 --- a/scripts/codegen/run_trunnel.sh +++ b/scripts/codegen/run_trunnel.sh @@ -9,9 +9,7 @@ OPTIONS="--require-version=1.5.1" # Get all .trunnel files recursively from that directory so we can support # multiple sub-directories. -for file in `find ./src/trunnel/ -name '*.trunnel'`; do - python -m trunnel ${OPTIONS} $file -done +find ./src/trunnel/ -name '*.trunnel' -exec python -m trunnel ${OPTIONS} {} \; python -m trunnel ${OPTIONS} --write-c-files --target-dir=./src/ext/trunnel/ diff --git a/scripts/maint/checkIncludes.py b/scripts/maint/checkIncludes.py index 46a3f39638..3afd9bbebe 100755 --- a/scripts/maint/checkIncludes.py +++ b/scripts/maint/checkIncludes.py @@ -33,6 +33,9 @@ else: def open_file(fname): return open(fname, 'r', encoding='utf-8') +def warn(msg): + print(msg, file=sys.stderr) + def err(msg): """ Declare that an error has happened, and remember that there has been an error. """ @@ -48,14 +51,34 @@ def fname_is_c(fname): INCLUDE_PATTERN = re.compile(r'\s*#\s*include\s+"([^"]*)"') RULES_FNAME = ".may_include" +ALLOWED_PATTERNS = [ + re.compile(r'^.*\*\.(h|inc)$'), + re.compile(r'^.*/.*\.h$'), + re.compile(r'^ext/.*\.c$'), + re.compile(r'^orconfig.h$'), + re.compile(r'^micro-revision.i$'), +] + +def pattern_is_normal(s): + for p in ALLOWED_PATTERNS: + if p.match(s): + return True + return False + class Rules(object): """ A 'Rules' object is the parsed version of a .may_include file. """ def __init__(self, dirpath): self.dirpath = dirpath + if dirpath.startswith("src/"): + self.incpath = dirpath[4:] + else: + self.incpath = dirpath self.patterns = [] self.usedPatterns = set() def addPattern(self, pattern): + if not pattern_is_normal(pattern): + warn("Unusual pattern {} in {}".format(pattern, self.dirpath)) self.patterns.append(pattern) def includeOk(self, path): @@ -86,6 +109,20 @@ class Rules(object): if p not in self.usedPatterns: print("Pattern {} in {} was never used.".format(p, self.dirpath)) + def getAllowedDirectories(self): + allowed = [] + for p in self.patterns: + m = re.match(r'^(.*)/\*\.(h|inc)$', p) + if m: + allowed.append(m.group(1)) + continue + m = re.match(r'^(.*)/[^/]*$', p) + if m: + allowed.append(m.group(1)) + continue + + return allowed + def load_include_rules(fname): """ Read a rules file from 'fname', and return it as a Rules object. """ result = Rules(os.path.split(fname)[0]) @@ -98,6 +135,9 @@ def load_include_rules(fname): return result list_unused = False +log_sorted_levels = False + +uses_dirs = { } for dirpath, dirnames, fnames in os.walk("src"): if ".may_include" in fnames: @@ -108,8 +148,34 @@ for dirpath, dirnames, fnames in os.walk("src"): if list_unused: rules.noteUnusedRules() + uses_dirs[rules.incpath] = rules.getAllowedDirectories() + if trouble: err( """To change which includes are allowed in a C file, edit the {} files in its enclosing directory.""".format(RULES_FNAME)) sys.exit(1) + +all_levels = [] + +n = 0 +while uses_dirs: + n += 0 + cur_level = [] + for k in list(uses_dirs): + uses_dirs[k] = [ d for d in uses_dirs[k] + if (d in uses_dirs and d != k)] + if uses_dirs[k] == []: + cur_level.append(k) + for k in cur_level: + del uses_dirs[k] + n += 1 + if cur_level and log_sorted_levels: + print(n, cur_level) + if n > 100: + break + +if uses_dirs: + print("There are circular .may_include dependencies in here somewhere:", + uses_dirs) + sys.exit(1) diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist deleted file mode 100644 index 79551948c6..0000000000 --- a/scripts/maint/fallback.whitelist +++ /dev/null @@ -1,997 +0,0 @@ -# updateFallbackDirs.py directory mirror whitelist -# -# Format: -# IPv4:DirPort orport=<ORPort> id=<ID> [ ipv6=<IPv6>:<IPv6 ORPort> ] -# or use: -# scripts/maint/generateFallbackDirLine.py fingerprint ... -# -# All attributes must match for the directory mirror to be included. -# If the fallback has an ipv6 key, the whitelist line must also have -# it, and vice versa, otherwise they don't match. -# (The blacklist overrides the whitelist.) - -# To replace this list with the hard-coded fallback list (for testing), use -# a command similar to: -# cat src/app/config/fallback_dirs.inc | grep \" | grep -v weight | \ -# tr -d '\n' | \ -# sed 's/"" / /g' | sed 's/""/"/g' | tr \" '\n' | grep -v '^$' \ -# > scripts/maint/fallback.whitelist -# -# When testing before a release, exclusions due to changed details will result -# in a warning, unless the IPv4 address or port change happened recently. -# Then it is only logged at info level, as part of the eligibility check. -# Exclusions due to stability also are only shown at info level. -# -# Add the number of selected, slow, and excluded relays, and compare that to -# the number of hard-coded relays. If it's less, use info-level logs to find -# out why each of the missing relays was excluded. - -# If a relay operator wants their relay to be a FallbackDir, -# enter the following information here: -# <IPv4>:<DirPort> orport=<ORPort> id=<ID> [ ipv6=<IPv6>:<IPv6 ORPort> ] - -# https://lists.torproject.org/pipermail/tor-relays/2015-December/008362.html -# https://trac.torproject.org/projects/tor/ticket/22321#comment:22 -78.47.18.110:443 orport=80 id=F8D27B163B9247B232A2EEE68DD8B698695C28DE ipv6=[2a01:4f8:120:4023::110]:80 # fluxe3 -131.188.40.188:1443 orport=80 id=EBE718E1A49EE229071702964F8DB1F318075FF8 ipv6=[2001:638:a000:4140::ffff:188]:80 # fluxe4 - -# https://lists.torproject.org/pipermail/tor-relays/2015-December/008366.html -5.39.88.19:9030 orport=9001 id=7CB8C31432A796731EA7B6BF4025548DFEB25E0C ipv6=[2001:41d0:8:9a13::1]:9050 - -# https://lists.torproject.org/pipermail/tor-relays/2015-December/008370.html -# https://lists.torproject.org/pipermail/tor-relays/2016-January/008517.html -# https://lists.torproject.org/pipermail/tor-relays/2016-January/008555.html -212.47.237.95:9030 orport=9001 id=3F5D8A879C58961BB45A3D26AC41B543B40236D6 -212.47.237.95:9130 orport=9101 id=6FB38EB22E57EF7ED5EF00238F6A48E553735D88 - -# https://lists.torproject.org/pipermail/tor-relays/2015-December/008372.html -# IPv6 tunnel available on request (is this a good idea?) -108.53.208.157:80 orport=443 id=4F0DB7E687FC7C0AE55C8F243DA8B0EB27FBF1F2 - -# https://lists.torproject.org/pipermail/tor-relays/2015-December/008373.html -167.114.35.28:9030 orport=9001 id=E65D300F11E1DB12C534B0146BDAB6972F1A8A48 - -# https://lists.torproject.org/pipermail/tor-relays/2015-December/008378.html -144.76.14.145:110 orport=143 id=14419131033443AE6E21DA82B0D307F7CAE42BDB ipv6=[2a01:4f8:190:9490::dead]:443 - -# https://lists.torproject.org/pipermail/tor-relays/2015-December/008379.html -# Email sent directly to teor, verified using relay contact info -91.121.84.137:4951 orport=4051 id=6DE61A6F72C1E5418A66BFED80DFB63E4C77668F -91.121.84.137:4952 orport=4052 id=9FBEB75E8BC142565F12CBBE078D63310236A334 - -# https://lists.torproject.org/pipermail/tor-relays/2015-December/008381.html -# Sent additional emails to teor with updated relays -81.7.11.96:9030 orport=9001 id=8FA37B93397015B2BC5A525C908485260BE9F422 # Doedel22 -# 9F5068310818ED7C70B0BC4087AB55CB12CB4377 not found in current consensus -178.254.19.101:80 orport=443 id=F9246DEF2B653807236DA134F2AEAB103D58ABFE # Freebird31 -178.254.19.101:9030 orport=9001 id=0C475BA4D3AA3C289B716F95954CAD616E50C4E5 # Freebird32 -81.7.14.253:9001 orport=443 id=1AE039EE0B11DB79E4B4B29CBA9F752864A0259E # Ichotolot60 -81.7.11.186:1080 orport=443 id=B86137AE9681701901C6720E55C16805B46BD8E3 # BeastieJoy60 -85.25.213.211:465 orport=80 id=CE47F0356D86CF0A1A2008D97623216D560FB0A8 # BeastieJoy61 -85.25.159.65:995 orport=80 id=52BFADA8BEAA01BA46C8F767F83C18E2FE50C1B9 # BeastieJoy63 -81.7.3.67:993 orport=443 id=A2E6BB5C391CD46B38C55B4329C35304540771F1 # BeastieJoy62 -81.7.14.31:9001 orport=443 id=7600680249A22080ECC6173FBBF64D6FCF330A61 # Ichotolot62 - -# https://lists.torproject.org/pipermail/tor-relays/2015-December/008382.html -51.255.33.237:9091 orport=9001 id=A360C21FA87FFA2046D92C17086A6B47E5C68109 - -# https://lists.torproject.org/pipermail/tor-relays/2015-December/008383.html -81.7.14.246:80 orport=443 id=CE75BF0972ADD52AF8807602374E495C815DB304 ipv6=[2a02:180:a:51::dead]:443 - -# https://lists.torproject.org/pipermail/tor-relays/2015-December/008384.html -# Sent additional email to teor with fingerprint change -149.202.98.161:80 orport=443 id=FC64CD763F8C1A319BFBBF62551684F4E1E42332 ipv6=[2001:41d0:8:4528::161]:443 -193.111.136.162:80 orport=443 id=C79552275DFCD486B942510EF663ED36ACA1A84B ipv6=[2001:4ba0:cafe:10d0::1]:443 - -# https://lists.torproject.org/pipermail/tor-relays/2015-December/008416.html -185.100.84.212:80 orport=443 id=330CD3DB6AD266DC70CDB512B036957D03D9BC59 ipv6=[2a06:1700:0:7::1]:443 - -# https://lists.torproject.org/pipermail/tor-relays/2015-December/008417.html -178.16.208.56:80 orport=443 id=2CDCFED0142B28B002E89D305CBA2E26063FADE2 ipv6=[2a00:1c20:4089:1234:cd49:b58a:9ebe:67ec]:443 -178.16.208.57:80 orport=443 id=92CFD9565B24646CAC2D172D3DB503D69E777B8A ipv6=[2a00:1c20:4089:1234:7825:2c5d:1ecd:c66f]:443 - -# https://lists.torproject.org/pipermail/tor-relays/2016-January/008513.html -178.62.173.203:9030 orport=9001 id=DD85503F2D1F52EF9EAD621E942298F46CD2FC10 ipv6=[2a03:b0c0:0:1010::a4:b001]:9001 - -# https://lists.torproject.org/pipermail/tor-relays/2016-January/008534.html -5.9.110.236:9030 orport=9001 id=0756B7CD4DFC8182BE23143FAC0642F515182CEB ipv6=[2a01:4f8:162:51e2::2]:9001 - -# https://lists.torproject.org/pipermail/tor-relays/2016-January/008542.html -178.62.199.226:80 orport=443 id=CBEFF7BA4A4062045133C053F2D70524D8BBE5BE ipv6=[2a03:b0c0:2:d0::b7:5001]:443 - -# Email sent directly to teor, verified using relay contact info -94.23.204.175:9030 orport=9001 id=5665A3904C89E22E971305EE8C1997BCA4123C69 - -# Email sent directly to teor, verified using relay contact info -171.25.193.77:80 orport=443 id=A10C4F666D27364036B562823E5830BC448E046A ipv6=[2001:67c:289c:3::77]:443 -171.25.193.78:80 orport=443 id=A478E421F83194C114F41E94F95999672AED51FE ipv6=[2001:67c:289c:3::78]:443 -171.25.193.20:80 orport=443 id=DD8BD7307017407FCC36F8D04A688F74A0774C02 ipv6=[2001:67c:289c::20]:443 -# same machine as DD8BD7307017407FCC36F8D04A688F74A0774C02 -171.25.193.25:80 orport=443 id=185663B7C12777F052B2C2D23D7A239D8DA88A0F ipv6=[2001:67c:289c::25]:443 - -# Email sent directly to teor, verified using relay contact info -212.47.229.2:9030 orport=9001 id=20462CBA5DA4C2D963567D17D0B7249718114A68 ipv6=[2001:bc8:4400:2100::f03]:9001 -93.115.97.242:9030 orport=9001 id=B5212DB685A2A0FCFBAE425738E478D12361710D -46.28.109.231:9030 orport=9001 id=F70B7C5CD72D74C7F9F2DC84FA9D20D51BA13610 ipv6=[2a02:2b88:2:1::4205:1]:9001 - -# Email sent directly to teor, verified using relay contact info -85.235.250.88:80 orport=443 id=72B2B12A3F60408BDBC98C6DF53988D3A0B3F0EE # TykRelay01 -185.96.88.29:80 orport=443 id=86C281AD135058238D7A337D546C902BE8505DDE # TykRelay051 -# This fallback opted-in in previous releases, then changed its details, -# and so we blacklisted it. Now we want to whitelist changes. -# Assume details update is permanent -185.96.180.29:80 orport=443 id=F93D8F37E35C390BCAD9F9069E13085B745EC216 # TykRelay06 - -# Email sent directly to teor, verified using relay contact info -185.11.180.67:80 orport=9001 id=794D8EA8343A4E820320265D05D4FA83AB6D1778 - -# Email sent directly to teor, verified using relay contact info -178.16.208.62:80 orport=443 id=5CF8AFA5E4B0BB88942A44A3F3AAE08C3BDFD60B ipv6=[2a00:1c20:4089:1234:a6a4:2926:d0af:dfee]:443 -46.165.221.166:80 orport=443 id=EE5F897C752D46BCFF531641B853FC6BC78DD4A7 -178.16.208.60:80 orport=443 id=B44FBE5366AD98B46D829754FA4AC599BAE41A6A ipv6=[2a00:1c20:4089:1234:67bc:79f3:61c0:6e49]:443 -178.16.208.55:80 orport=443 id=C4AEA05CF380BAD2230F193E083B8869B4A29937 ipv6=[2a00:1c20:4089:1234:7b2c:11c5:5221:903e]:443 -178.16.208.61:80 orport=443 id=3B52392E2256C35CDCF7801FF898FC88CE6D431A ipv6=[2a00:1c20:4089:1234:2712:a3d0:666b:88a6]:443 -81.89.96.88:80 orport=443 id=55ED4BB49F6D3F36D8D9499BE43500E017A5EF82 ipv6=[2a02:180:1:1:14c5:b0b7:2d7d:5f3a]:443 -209.222.8.196:80 orport=443 id=C86D2F3DEFE287A0EEB28D4887AF14E35C172733 ipv6=[2001:19f0:1620:41c1:426c:5adf:2ed5:4e88]:443 -81.89.96.89:80 orport=443 id=28651F419F5A1CF74511BB500C58112192DD4943 ipv6=[2a02:180:1:1:2ced:24e:32ea:a03b]:443 -46.165.221.166:9030 orport=9001 id=8C7106C880FE8AA1319DD71B59623FCB8914C9F1 -178.16.208.62:80 orport=443 id=5CF8AFA5E4B0BB88942A44A3F3AAE08C3BDFD60B ipv6=[2a00:1c20:4089:1234:a6a4:2926:d0af:dfee]:443" -46.165.221.166:80 orport=443 id=EE5F897C752D46BCFF531641B853FC6BC78DD4A7 -178.16.208.60:80 orport=443 id=B44FBE5366AD98B46D829754FA4AC599BAE41A6A ipv6=[2a00:1c20:4089:1234:67bc:79f3:61c0:6e49]:443 -178.16.208.55:80 orport=443 id=C4AEA05CF380BAD2230F193E083B8869B4A29937 ipv6=[2a00:1c20:4089:1234:7b2c:11c5:5221:903e]:443 -178.16.208.61:80 orport=443 id=3B52392E2256C35CDCF7801FF898FC88CE6D431A ipv6=[2a00:1c20:4089:1234:2712:a3d0:666b:88a6]:443 -81.89.96.88:80 orport=443 id=55ED4BB49F6D3F36D8D9499BE43500E017A5EF82 ipv6=[2a02:180:1:1:14c5:b0b7:2d7d:5f3a]:443 -209.222.8.196:80 orport=443 id=C86D2F3DEFE287A0EEB28D4887AF14E35C172733 ipv6=[2001:19f0:1620:41c1:426c:5adf:2ed5:4e88]:443 -81.89.96.89:80 orport=443 id=28651F419F5A1CF74511BB500C58112192DD4943 ipv6=[2a02:180:1:1:2ced:24e:32ea:a03b]:443 -46.165.221.166:9030 orport=9001 id=8C7106C880FE8AA1319DD71B59623FCB8914C9F1 -178.16.208.56:80 orport=443 id=2CDCFED0142B28B002E89D305CBA2E26063FADE2 ipv6=[2a00:1c20:4089:1234:cd49:b58a:9ebe:67ec]:443 -178.16.208.58:80 orport=443 id=A4C98CEA3F34E05299417E9F885A642C88EF6029 ipv6=[2a00:1c20:4089:1234:cdae:1b3e:cc38:3d45]:443 -178.16.208.57:80 orport=443 id=92CFD9565B24646CAC2D172D3DB503D69E777B8A ipv6=[2a00:1c20:4089:1234:7825:2c5d:1ecd:c66f]:443 -178.16.208.59:80 orport=443 id=136F9299A5009A4E0E96494E723BDB556FB0A26B ipv6=[2a00:1c20:4089:1234:bff6:e1bb:1ce3:8dc6]:443 - -# Email sent directly to teor, verified using relay contact info -5.39.76.158:80 orport=443 id=C41F60F8B00E7FEF5CCC5BC6BB514CA1B8AAB651 - -# Email sent directly to teor, verified using relay contact info -109.163.234.2:80 orport=443 id=14F92FF956105932E9DEC5B82A7778A0B1BD9A52 -109.163.234.4:80 orport=443 id=4888770464F0E900EFEF1BA181EA873D13F7713C -109.163.234.5:80 orport=443 id=5EB8D862E70981B8690DEDEF546789E26AB2BD24 -109.163.234.7:80 orport=443 id=23038A7F2845EBA2234ECD6651BD4A7762F51B18 -109.163.234.8:80 orport=443 id=0818DAE0E2DDF795AEDEAC60B15E71901084F281 -109.163.234.9:80 orport=443 id=ABF7FBF389C9A747938B639B20E80620B460B2A9 -62.102.148.67:80 orport=443 id=4A0C3E177AF684581EF780981AEAF51A98A6B5CF -# Assume details update is permanent -77.247.181.166:80 orport=443 id=77131D7E2EC1CA9B8D737502256DA9103599CE51 # CriticalMass -77.247.181.164:80 orport=443 id=204DFD2A2C6A0DC1FA0EACB495218E0B661704FD # HaveHeart -77.247.181.162:80 orport=443 id=7BFB908A3AA5B491DA4CA72CCBEE0E1F2A939B55 # sofia - -# https://twitter.com/biotimylated/status/718994247500718080 -212.47.252.149:9030 orport=9001 id=2CAC39BAA996791CEFAADC9D4754D65AF5EB77C0 - -# Email sent directly to teor, verified using relay contact info -46.165.230.5:80 orport=443 id=A0F06C2FADF88D3A39AA3072B406F09D7095AC9E - -# Email sent directly to teor, verified using relay contact info -94.242.246.24:23 orport=8080 id=EC116BCB80565A408CE67F8EC3FE3B0B02C3A065 ipv6=[2a01:608:ffff:ff07::1:24]:9004 -176.126.252.11:443 orport=9001 id=B0279A521375F3CB2AE210BDBFC645FDD2E1973A ipv6=[2a02:59e0:0:7::11]:9003 -176.126.252.12:21 orport=8080 id=379FB450010D17078B3766C2273303C358C3A442 ipv6=[2a02:59e0:0:7::12]:81 -94.242.246.23:443 orport=9001 id=F65E0196C94DFFF48AFBF2F5F9E3E19AAE583FD0 ipv6=[2a01:608:ffff:ff07::1:23]:9003 -85.248.227.164:444 orport=9002 id=B84F248233FEA90CAD439F292556A3139F6E1B82 ipv6=[2a00:1298:8011:212::164]:9004 -85.248.227.163:443 orport=9001 id=C793AB88565DDD3C9E4C6F15CCB9D8C7EF964CE9 ipv6=[2a00:1298:8011:212::163]:9003 - -# Email sent directly to teor, verified using relay contact info -148.251.190.229:9030 orport=9010 id=BF0FB582E37F738CD33C3651125F2772705BB8E8 ipv6=[2a01:4f8:211:c68::2]:9010 - -# Email sent directly to teor, verified using relay contact info -5.79.68.161:81 orport=443 id=9030DCF419F6E2FBF84F63CBACBA0097B06F557E ipv6=[2001:1af8:4700:a012:1::1]:443 -5.79.68.161:9030 orport=9001 id=B7EC0C02D7D9F1E31B0C251A6B058880778A0CD1 ipv6=[2001:1af8:4700:a012:1::1]:9001 - -# Email sent directly to teor, verified using relay contact info -62.210.92.11:9030 orport=9001 id=0266B0660F3F20A7D1F3D8335931C95EF50F6C6B ipv6=[2001:bc8:338c::1]:9001 -62.210.92.11:9130 orport=9101 id=387B065A38E4DAA16D9D41C2964ECBC4B31D30FF ipv6=[2001:bc8:338c::1]:9101 - -# Email sent directly to teor, verified using relay contact info -188.165.194.195:9030 orport=9001 id=49E7AD01BB96F6FE3AB8C3B15BD2470B150354DF - -# Message sent directly to teor, verified using relay contact info -95.215.44.110:80 orport=443 id=D56AA4A1AA71961F5279FB70A6DCF7AD7B993EB5 -95.215.44.122:80 orport=443 id=998D8FE06B867AA3F8D257A7D28FFF16964D53E2 -95.215.44.111:80 orport=443 id=A7C7FD510B20BC8BE8F2A1D911364E1A23FBD09F - -# Email sent directly to teor, verified using relay contact info -86.59.119.88:80 orport=443 id=ACD889D86E02EDDAB1AFD81F598C0936238DC6D0 -86.59.119.83:80 orport=443 id=FC9AC8EA0160D88BCCFDE066940D7DD9FA45495B - -# Email sent directly to teor, verified using relay contact info -193.11.164.243:9030 orport=9001 id=FFA72BD683BC2FCF988356E6BEC1E490F313FB07 ipv6=[2001:6b0:7:125::243]:9001 -109.105.109.162:52860 orport=60784 id=32EE911D968BE3E016ECA572BB1ED0A9EE43FC2F ipv6=[2001:948:7:2::163]:5001 - -# Email sent directly to teor, verified using relay contact info -146.0.32.144:9030 orport=9001 id=35E8B344F661F4F2E68B17648F35798B44672D7E - -# Email sent directly to teor, verified using relay contact info -46.252.26.2:45212 orport=49991 id=E589316576A399C511A9781A73DA4545640B479D - -# Email sent directly to teor, verified using relay contact info -89.187.142.208:80 orport=443 id=64186650FFE4469EBBE52B644AE543864D32F43C - -# Email sent directly to teor -# Assume details update is permanent -212.51.134.123:9030 orport=9001 id=50586E25BE067FD1F739998550EDDCB1A14CA5B2 # Jans - -# Email sent directly to teor, verified using relay contact info -46.101.143.173:80 orport=443 id=F960DF50F0FD4075AC9B505C1D4FFC8384C490FB - -# Email sent directly to teor, verified using relay contact info -193.171.202.146:9030 orport=9001 id=01A9258A46E97FF8B2CAC7910577862C14F2C524 - -# Email sent directly to teor, verified using relay contact info -# Assume details update is permanent -197.231.221.211:9030 orport=443 id=BC630CBBB518BE7E9F4E09712AB0269E9DC7D626 # IPredator - -# Email sent directly to teor, verified using relay contact info -185.61.138.18:8080 orport=4443 id=2541759BEC04D37811C2209A88E863320271EC9C - -# Email sent directly to teor, verified using relay contact info -193.11.114.45:9031 orport=9002 id=80AAF8D5956A43C197104CEF2550CD42D165C6FB -193.11.114.43:9030 orport=9001 id=12AD30E5D25AA67F519780E2111E611A455FDC89 ipv6=[2001:6b0:30:1000::99]:9050 -193.11.114.46:9032 orport=9003 id=B83DC1558F0D34353BB992EF93AFEAFDB226A73E - -# Email sent directly to teor, verified using relay contact info -138.201.250.33:9012 orport=9011 id=2BA2C8E96B2590E1072AECE2BDB5C48921BF8510 - -# Email sent directly to teor, verified using relay contact info -37.221.162.226:9030 orport=9001 id=D64366987CB39F61AD21DBCF8142FA0577B92811 - -# Email sent directly to teor, verified using relay contact info -91.219.237.244:80 orport=443 id=92ECC9E0E2AF81BB954719B189AC362E254AD4A5 - -# Email sent directly to teor, verified using relay contact info -185.21.100.50:9030 orport=9001 id=58ED9C9C35E433EE58764D62892B4FFD518A3CD0 ipv6=[2a00:1158:2:cd00:0:74:6f:72]:443 - -# Email sent directly to teor, verified using relay contact info -193.35.52.53:9030 orport=9001 id=DAA39FC00B196B353C2A271459C305C429AF09E4 - -# Email sent directly to teor, verified using relay contact info -134.119.3.164:9030 orport=9001 id=D1B8AAA98C65F3DF7D8BB3AF881CAEB84A33D8EE - -# Email sent directly to teor, verified using relay contact info -173.212.254.192:31336 orport=31337 id=99E246DB480B313A3012BC3363093CC26CD209C7 - -# Email sent directly to teor, verified using relay contact info -178.62.22.36:80 orport=443 id=A0766C0D3A667A3232C7D569DE94A28F9922FCB1 ipv6=[2a03:b0c0:1:d0::174:1]:9050 -188.166.23.127:80 orport=443 id=8672E8A01B4D3FA4C0BBE21C740D4506302EA487 ipv6=[2a03:b0c0:2:d0::27b:7001]:9050 -198.199.64.217:80 orport=443 id=B1D81825CFD7209BD1B4520B040EF5653C204A23 ipv6=[2604:a880:400:d0::1a9:b001]:9050 -159.203.32.149:80 orport=443 id=55C7554AFCEC1062DCBAC93E67B2E03C6F330EFC ipv6=[2604:a880:cad:d0::105:f001]:9050 - -# Email sent directly to teor, verified using relay contact info -5.196.31.80:9030 orport=9900 id=DFB2EB472643FAFCD5E73D2E37D51DB67203A695 ipv6=[2001:41d0:52:400::a65]:9900 - -# Email sent directly to teor, verified using relay contact info -188.138.112.60:1433 orport=1521 id=C414F28FD2BEC1553024299B31D4E726BEB8E788 - -# Email sent directly to teor, verified using relay contact info -213.61.66.118:9031 orport=9001 id=30648BC64CEDB3020F4A405E4AB2A6347FB8FA22 -213.61.66.117:9032 orport=9002 id=6E44A52E3D1FF7683FE5C399C3FB5E912DE1C6B4 -213.61.66.115:9034 orport=9004 id=480CCC94CEA04D2DEABC0D7373868E245D4C2AE2 -213.61.66.116:9033 orport=9003 id=A9DEB920B42B4EC1DE6249034039B06D61F38690 - -# Email sent directly to teor, verified using relay contact info -136.243.187.165:9030 orport=443 id=1AC65257D7BFDE7341046625470809693A8ED83E - -# Email sent directly to teor, verified using relay contact info -212.47.230.49:9030 orport=9001 id=3D6D0771E54056AEFC28BB1DE816951F11826E97 - -# Email sent directly to teor, verified using relay contact info -192.99.55.69:80 orport=443 id=0682DE15222A4A4A0D67DBA72A8132161992C023 -192.99.59.140:80 orport=443 id=3C9148DA49F20654730FAC83FFF693A4D49D0244 -51.254.215.13:80 orport=443 id=73C30C8ABDD6D9346C822966DE73B9F82CB6178A -51.254.215.129:80 orport=443 id=7B4491D05144B20AE8519AE784B94F0525A8BB79 -192.99.59.139:80 orport=443 id=82EC878ADA7C205146B9F5193A7310867FAA0D7B -51.254.215.124:80 orport=443 id=98999EBE89B5FA9AA0C58421F0B46C3D0AF51CBA -51.254.214.208:80 orport=443 id=C3F0D1417848EAFC41277A73DEB4A9F2AEC23DDF -192.99.59.141:80 orport=443 id=F45426551795B9DA78BEDB05CD5F2EACED8132E4 -192.99.59.14:80 orport=443 id=161A1B29A37EBF096D2F8A9B1E176D6487FE42AE - -# Email sent directly to teor, verified using relay contact info -151.80.42.103:9030 orport=9001 id=9007C1D8E4F03D506A4A011B907A9E8D04E3C605 ipv6=[2001:41d0:e:f67::114]:9001 - -# Email sent directly to teor, verified using relay contact info -5.39.92.199:80 orport=443 id=0BEA4A88D069753218EAAAD6D22EA87B9A1319D6 ipv6=[2001:41d0:8:b1c7::1]:443 - -# Email sent directly to teor, verified using relay contact info -176.31.159.231:80 orport=443 id=D5DBCC0B4F029F80C7B8D33F20CF7D97F0423BB1 -176.31.159.230:80 orport=443 id=631748AFB41104D77ADBB7E5CD4F8E8AE876E683 -195.154.79.128:80 orport=443 id=C697612CA5AED06B8D829FCC6065B9287212CB2F -195.154.9.161:80 orport=443 id=B6295A9960F89BD0C743EEBC5670450EA6A34685 -46.148.18.74:8080 orport=443 id=6CACF0B5F03C779672F3C5C295F37C8D234CA3F7 - -# Email sent directly to teor, verified using relay contact info -37.187.102.108:80 orport=443 id=F4263275CF54A6836EE7BD527B1328836A6F06E1 ipv6=[2001:41d0:a:266c::1]:443 # EvilMoe -212.47.241.21:80 orport=443 id=892F941915F6A0C6E0958E52E0A9685C190CF45C # EvilMoe - -# Email sent directly to teor, verified using relay contact info -212.129.38.254:9030 orport=9001 id=FDF845FC159C0020E2BDDA120C30C5C5038F74B4 - -# Email sent directly to teor -37.157.195.87:8030 orport=443 id=12FD624EE73CEF37137C90D38B2406A66F68FAA2 # thanatosCZ -5.189.169.190:8030 orport=8080 id=8D79F73DCD91FC4F5017422FAC70074D6DB8DD81 # thanatosDE - -# Email sent directly to teor, verified using relay contact info -37.187.7.74:80 orport=443 id=AEA43CB1E47BE5F8051711B2BF01683DB1568E05 ipv6=[2001:41d0:a:74a::1]:443 - -# Email sent directly to teor, verified using relay contact info -185.66.250.141:9030 orport=9001 id=B1726B94885CE3AC3910CA8B60622B97B98E2529 - -# Email sent directly to teor, verified using relay contact info -185.104.120.7:9030 orport=443 id=445F1C853966624FB3CF1E12442570DC553CC2EC ipv6=[2a06:3000::120:7]:443 -185.104.120.2:9030 orport=21 id=518FF8708698E1DA09C823C36D35DF89A2CAD956 -185.104.120.4:9030 orport=9001 id=F92B3CB9BBE0CB22409843FB1AE4DBCD5EFAC835 -185.104.120.3:9030 orport=21 id=707C1B61AC72227B34487B56D04BAA3BA1179CE8 ipv6=[2a06:3000::120:3]:21 - -# Email sent directly to teor, verified using relay contact info -37.187.102.186:9030 orport=9001 id=489D94333DF66D57FFE34D9D59CC2D97E2CB0053 ipv6=[2001:41d0:a:26ba::1]:9001 - -# Email sent directly to teor, verified using relay contact info -198.96.155.3:8080 orport=5001 id=BCEDF6C193AA687AE471B8A22EBF6BC57C2D285E - -# Email sent directly to teor, verified using relay contact info -212.83.154.33:8888 orport=443 id=3C79699D4FBC37DE1A212D5033B56DAE079AC0EF -212.83.154.33:8080 orport=8443 id=322C6E3A973BC10FC36DE3037AD27BC89F14723B - -# Email sent directly to teor, verified using relay contact info -51.255.41.65:9030 orport=9001 id=9231DF741915AA1630031A93026D88726877E93A - -# Email sent directly to teor, verified using relay contact info -78.142.142.246:80 orport=443 id=5A5E03355C1908EBF424CAF1F3ED70782C0D2F74 - -# Email sent directly to teor, verified using relay contact info -195.154.97.91:80 orport=443 id=BD33C50D50DCA2A46AAED54CA319A1EFEBF5D714 - -# Email sent directly to teor, verified using relay contact info -62.210.129.246:80 orport=443 id=79E169B25E4C7CE99584F6ED06F379478F23E2B8 - -# Email sent directly to teor, verified using relay contact info -5.196.74.215:9030 orport=9001 id=5818055DFBAF0FA7F67E8125FD63E3E7F88E28F6 - -# Email sent directly to teor, verified using relay contact info -212.47.233.86:9030 orport=9001 id=B4CAFD9CBFB34EC5DAAC146920DC7DFAFE91EA20 - -# Email sent directly to teor, verified using relay contact info -85.214.206.219:9030 orport=9001 id=98F8D5F359949E41DE8DF3DBB1975A86E96A84A0 - -# Email sent directly to teor, verified using relay contact info -46.166.170.4:80 orport=443 id=19F42DB047B72C7507F939F5AEA5CD1FA4656205 -46.166.170.5:80 orport=443 id=DA705AD4591E7B4708FA2CAC3D53E81962F3E6F6 - -# Email sent directly to teor, verified using relay contact info -5.189.157.56:80 orport=443 id=77F6D6A6B6EAFB8F5DADDC07A918BBF378ED6725 - -# Email sent directly to teor, verified using relay contact info -46.28.110.244:80 orport=443 id=9F7D6E6420183C2B76D3CE99624EBC98A21A967E -185.13.39.197:80 orport=443 id=001524DD403D729F08F7E5D77813EF12756CFA8D -95.130.12.119:80 orport=443 id=587E0A9552E4274B251F29B5B2673D38442EE4BF - -# Email sent directly to teor, verified using relay contact info -212.129.62.232:80 orport=443 id=B143D439B72D239A419F8DCE07B8A8EB1B486FA7 - -# Email sent directly to teor, verified using relay contact info -91.219.237.229:80 orport=443 id=1ECD73B936CB6E6B3CD647CC204F108D9DF2C9F7 - -# Email sent directly to teor, verified using relay contact info -46.101.151.222:80 orport=443 id=1DBAED235E3957DE1ABD25B4206BE71406FB61F8 -178.62.60.37:80 orport=443 id=175921396C7C426309AB03775A9930B6F611F794 - -# Email sent directly to teor, verified using relay contact info -178.62.197.82:80 orport=443 id=0D3EBA17E1C78F1E9900BABDB23861D46FCAF163 - -# Email sent directly to teor, verified using relay contact info -82.223.21.74:9030 orport=9001 id=7A32C9519D80CA458FC8B034A28F5F6815649A98 ipv6=[2001:470:53e0::cafe]:9050 - -# Email sent directly to teor, verified using relay contact info -146.185.177.103:80 orport=9030 id=9EC5E097663862DF861A18C32B37C5F82284B27D - -# Email sent directly to teor, verified using relay contact info -37.187.22.87:9030 orport=9001 id=36B9E7AC1E36B62A9D6F330ABEB6012BA7F0D400 ipv6=[2001:41d0:a:1657::1]:9001 - -# Email sent directly to teor, verified using relay contact info -37.59.46.159:9030 orport=9001 id=CBD0D1BD110EC52963082D839AC6A89D0AE243E7 - -# Email sent directly to teor, verified using relay contact info -212.47.250.243:9030 orport=9001 id=5B33EDBAEA92F446768B3753549F3B813836D477 -# Confirm with operator before adding these -#163.172.133.36:9030 orport=9001 id=D8C2BD36F01FA86F4401848A0928C4CB7E5FDFF9 -#158.69.216.70:9030 orport=9001 id=0ACE25A978D4422C742D6BC6345896719BF6A7EB - -# Email sent directly to teor, verified using relay contact info -5.199.142.236:9030 orport=9001 id=F4C0EDAA0BF0F7EC138746F8FEF1CE26C7860265 - -# Email sent directly to teor -188.166.133.133:9030 orport=9001 id=774555642FDC1E1D4FDF2E0C31B7CA9501C5C9C7 ipv6=[2a03:b0c0:2:d0::26c0:1]:9001 # dropsy - -# Email sent directly to teor, verified using relay contact info -46.8.249.10:80 orport=443 id=31670150090A7C3513CB7914B9610E786391A95D - -# Email sent directly to teor, verified using relay contact info -144.76.163.93:9030 orport=9001 id=22F08CF09764C4E8982640D77F71ED72FF26A9AC - -# Email sent directly to teor, verified using relay contact info -46.4.24.161:9030 orport=9001 id=DB4C76A3AD7E234DA0F00D6F1405D8AFDF4D8DED -46.4.24.161:9031 orport=9002 id=7460F3D12EBE861E4EE073F6233047AACFE46AB4 -46.38.51.132:9030 orport=9001 id=810DEFA7E90B6C6C383C063028EC397A71D7214A -163.172.194.53:9030 orport=9001 id=8C00FA7369A7A308F6A137600F0FA07990D9D451 ipv6=[2001:bc8:225f:142:6c69:7461:7669:73]:9001 - -# Email sent directly to teor, verified using relay contact info -176.10.107.180:9030 orport=9001 id=3D7E274A87D9A89AF064C13D1EE4CA1F184F2600 - -# Email sent directly to teor, verified using relay contact info -46.28.207.19:80 orport=443 id=5B92FA5C8A49D46D235735504C72DBB3472BA321 -46.28.207.141:80 orport=443 id=F69BED36177ED727706512BA6A97755025EEA0FB -46.28.205.170:80 orport=443 id=AF322D83A4D2048B22F7F1AF5F38AFF4D09D0B76 -95.183.48.12:80 orport=443 id=7187CED1A3871F837D0E60AC98F374AC541CB0DA - -# Email sent directly to teor, verified using relay contact info -93.180.156.84:9030 orport=9001 id=8844D87E9B038BE3270938F05AF797E1D3C74C0F - -# Email sent directly to teor, verified using relay contact info -37.187.115.157:9030 orport=9001 id=D5039E1EBFD96D9A3F9846BF99EC9F75EDDE902A - -# Email sent directly to teor, verified using relay contact info -5.34.183.205:80 orport=443 id=DDD7871C1B7FA32CB55061E08869A236E61BDDF8 - -# Email sent directly to teor, verified using relay contact info -51.254.246.203:9030 orport=9001 id=47B596B81C9E6277B98623A84B7629798A16E8D5 - -# Email sent directly to teor, verified using relay contact info -5.9.146.203:80 orport=443 id=1F45542A24A61BF9408F1C05E0DCE4E29F2CBA11 - -# Email sent directly to teor, verified using relay contact info -# Updated details from atlas based on ticket #20010 -163.172.176.167:80 orport=443 id=230A8B2A8BA861210D9B4BA97745AEC217A94207 -163.172.149.155:80 orport=443 id=0B85617241252517E8ECF2CFC7F4C1A32DCD153F -163.172.149.122:80 orport=443 id=A9406A006D6E7B5DA30F2C6D4E42A338B5E340B2 - -# Email sent directly to teor, verified using relay contact info -204.11.50.131:9030 orport=9001 id=185F2A57B0C4620582602761097D17DB81654F70 - -# Email sent directly to teor, verified using relay contact info -151.236.222.217:44607 orport=9001 id=94D58704C2589C130C9C39ED148BD8EA468DBA54 - -# Email sent directly to teor, verified using relay contact info -185.35.202.221:9030 orport=9001 id=C13B91384CDD52A871E3ECECE4EF74A7AC7DCB08 ipv6=[2a02:ed06::221]:9001 - -# Email sent directly to teor, verified using relay contact info -5.9.151.241:9030 orport=4223 id=9BF04559224F0F1C3C953D641F1744AF0192543A ipv6=[2a01:4f8:190:34f0::2]:4223 - -# Email sent directly to teor, verified using relay contact info -89.40.71.149:8081 orport=8080 id=EC639EDAA5121B47DBDF3D6B01A22E48A8CB6CC7 - -# Email sent directly to teor, verified using relay contact info -92.222.20.130:80 orport=443 id=0639612FF149AA19DF3BCEA147E5B8FED6F3C87C - -# Email sent directly to teor, verified using relay contact info -80.112.155.100:9030 orport=9001 id=53B000310984CD86AF47E5F3CD0BFF184E34B383 ipv6=[2001:470:7b02::38]:9001 - -# Email sent directly to teor, verified using relay contact info -83.212.99.68:80 orport=443 id=DDBB2A38252ADDA53E4492DDF982CA6CC6E10EC0 ipv6=[2001:648:2ffc:1225:a800:bff:fe3d:67b5]:443 - -# Email sent directly to teor, verified using relay contact info -95.130.11.147:9030 orport=443 id=6B697F3FF04C26123466A5C0E5D1F8D91925967A - -# Email sent directly to teor, verified using relay contact info -128.199.55.207:9030 orport=9001 id=BCEF908195805E03E92CCFE669C48738E556B9C5 ipv6=[2a03:b0c0:2:d0::158:3001]:9001 - -# Email sent directly to teor, verified using relay contact info -178.32.216.146:9030 orport=9001 id=17898F9A2EBC7D69DAF87C00A1BD2FABF3C9E1D2 - -# Email sent directly to teor, verified using relay contact info -212.83.40.238:9030 orport=9001 id=F409FA7902FD89270E8DE0D7977EA23BC38E5887 - -# Email sent directly to teor, verified using relay contact info -204.8.156.142:80 orport=443 id=94C4B7B8C50C86A92B6A20107539EE2678CF9A28 - -# Email sent directly to teor, verified using relay contact info -80.240.139.111:80 orport=443 id=DD3BE7382C221F31723C7B294310EF9282B9111B - -# Email sent directly to teor, verified using relay contact info -185.97.32.18:9030 orport=9001 id=04250C3835019B26AA6764E85D836088BE441088 - -# Email sent directly to teor -149.56.45.200:9030 orport=9001 id=FE296180018833AF03A8EACD5894A614623D3F76 ipv6=[2607:5300:201:3000::17d3]:9002 # PiotrTorpotkinOne - -# Email sent directly to teor, verified using relay contact info -81.2.209.10:443 orport=80 id=B6904ADD4C0D10CDA7179E051962350A69A63243 ipv6=[2001:15e8:201:1::d10a]:80 - -# Email sent directly to teor, verified using relay contact info -# IPv6 address unreliable -195.154.164.243:80 orport=443 id=AC66FFA4AB35A59EBBF5BF4C70008BF24D8A7A5C #ipv6=[2001:bc8:399f:f000::1]:993 -138.201.26.2:80 orport=443 id=6D3A3ED5671E4E3F58D4951438B10AE552A5FA0F -81.7.16.182:80 orport=443 id=51E1CF613FD6F9F11FE24743C91D6F9981807D82 ipv6=[2a02:180:1:1::517:10b6]:993 -134.119.36.135:80 orport=443 id=763C9556602BD6207771A7A3D958091D44C43228 ipv6=[2a00:1158:3::2a8]:993 -46.228.199.19:80 orport=443 id=E26AFC5F718E21AC502899B20C653AEFF688B0D2 ipv6=[2001:4ba0:cafe:4a::1]:993 -37.200.98.5:80 orport=443 id=231C2B9C8C31C295C472D031E06964834B745996 ipv6=[2a00:1158:3::11a]:993 -46.23.70.195:80 orport=443 id=C9933B3725239B6FAB5227BA33B30BE7B48BB485 -185.15.244.124:80 orport=443 id=935BABE2564F82016C19AEF63C0C40B5753BA3D2 ipv6=[2001:4ba0:cafe:e35::1]:993 -195.154.116.232:80 orport=443 id=B35C5739C8C5AB72094EB2B05738FD1F8EEF6EBD ipv6=[2001:bc8:399f:200::1]:993 -195.154.121.198:80 orport=443 id=0C77421C890D16B6D201283A2244F43DF5BC89DD ipv6=[2001:bc8:399f:100::1]:993 -37.187.20.59:80 orport=443 id=91D23D8A539B83D2FB56AA67ECD4D75CC093AC55 ipv6=[2001:41d0:a:143b::1]:993 -217.12.208.117:80 orport=443 id=E6E18151300F90C235D3809F90B31330737CEB43 ipv6=[2a00:1ca8:a7::1bb]:993 -81.7.10.251:80 orport=443 id=8073670F8F852971298F8AF2C5B23AE012645901 ipv6=[2a02:180:1:1::517:afb]:993 -46.36.39.50:80 orport=443 id=ED4B0DBA79AEF5521564FA0231455DCFDDE73BB6 ipv6=[2a02:25b0:aaaa:aaaa:8d49:b692:4852:0]:995 -91.194.90.103:80 orport=443 id=75C4495F4D80522CA6F6A3FB349F1B009563F4B7 ipv6=[2a02:c205:3000:5449::1]:993 -163.172.25.118:80 orport=22 id=0CF8F3E6590F45D50B70F2F7DA6605ECA6CD408F -188.138.88.42:80 orport=443 id=70C55A114C0EF3DC5784A4FAEE64388434A3398F -81.7.13.84:80 orport=443 id=0C1E7DD9ED0676C788933F68A9985ED853CA5812 ipv6=[2a02:180:1:1::5b8f:538c]:993 -213.246.56.95:80 orport=443 id=27E6E8E19C46751E7312420723C6162FF3356A4C ipv6=[2a00:c70:1:213:246:56:95:1]:993 -94.198.100.18:80 orport=443 id=BAACCB29197DB833F107E410E2BFAE5009EE7583 -217.12.203.46:80 orport=443 id=6A29FD8C00D573E6C1D47852345B0E5275BA3307 -212.117.180.107:80 orport=443 id=0B454C7EBA58657B91133A587C1BDAEDC6E23142 -217.12.199.190:80 orport=443 id=A37C47B03FF31CA6937D3D68366B157997FE7BCD ipv6=[2a02:27a8:0:2::486]:993 -216.230.230.247:80 orport=443 id=4C7BF55B1BFF47993DFF995A2926C89C81E4F04A -69.30.215.42:80 orport=443 id=510176C07005D47B23E6796F02C93241A29AA0E9 ipv6=[2604:4300:a:2e::2]:993 -89.46.100.162:80 orport=443 id=6B7191639E179965FD694612C9B2C8FB4267B27D -107.181.174.22:80 orport=443 id=5A551BF2E46BF26CC50A983F7435CB749C752553 ipv6=[2607:f7a0:3:4::4e]:993 - -# Email sent directly to teor, verified using relay contact info -212.238.208.48:9030 orport=9001 id=F406219CDD339026D160E53FCA0EF6857C70F109 ipv6=[2001:984:a8fb:1:ba27:ebff:feac:c109]:9001 - -# Email sent directly to teor -176.158.236.102:9030 orport=9001 id=DC163DDEF4B6F0C6BC226F9F6656A5A30C5C5686 # Underworld - -# Email sent directly to teor, verified using relay contact info -91.229.20.27:9030 orport=9001 id=9A0D54D3A6D2E0767596BF1515E6162A75B3293F - -# Email sent directly to teor, verified using relay contact info -80.127.137.19:80 orport=443 id=6EF897645B79B6CB35E853B32506375014DE3621 ipv6=[2001:981:47c1:1::6]:443 - -# Email sent directly to teor -163.172.138.22:80 orport=443 id=16102E458460349EE45C0901DAA6C30094A9BBEA ipv6=[2001:bc8:4400:2100::1:3]:443 # mkultra - -# Email sent directly to teor, verified using relay contact info -97.74.237.196:9030 orport=9001 id=2F0F32AB1E5B943CA7D062C03F18960C86E70D94 - -# Email sent directly to teor, verified using relay contact info -192.187.124.98:9030 orport=9001 id=FD1871854BFC06D7B02F10742073069F0528B5CC - -# Email sent directly to teor, verified using relay contact info -178.62.98.160:9030 orport=9001 id=8B92044763E880996A988831B15B2B0E5AD1544A - -# Email sent directly to teor, verified using relay contact info -163.172.217.50:9030 orport=9001 id=02ECD99ECD596013A8134D46531560816ECC4BE6 - -# Email sent directly to teor, verified using relay contact info -185.100.86.100:80 orport=443 id=0E8C0C8315B66DB5F703804B3889A1DD66C67CE0 -185.100.84.82:80 orport=443 id=7D05A38E39FC5D29AFE6BE487B9B4DC9E635D09E - -# Email sent directly to teor, verified using relay contact info -164.132.77.175:9030 orport=9001 id=3B33F6FCA645AD4E91428A3AF7DC736AD9FB727B -78.24.75.53:9030 orport=9001 id=DEB73705B2929AE9BE87091607388939332EF123 - -# Email sent directly to teor, verified using relay contact info -46.101.237.246:9030 orport=9001 id=75F1992FD3F403E9C082A5815EB5D12934CDF46C ipv6=[2a03:b0c0:3:d0::208:5001]:9050 -178.62.86.96:9030 orport=9001 id=439D0447772CB107B886F7782DBC201FA26B92D1 ipv6=[2a03:b0c0:1:d0::3cf:7001]:9050 - -# Email sent directly to teor, verified using relay contact info -# Very low bandwidth, stale consensues, excluded to cut down on warnings -#91.233.106.121:80 orport=443 id=896364B7996F5DFBA0E15D1A2E06D0B98B555DD6 - -# Email sent directly to teor, verified using relay contact info -167.114.113.48:9030 orport=403 id=2EC0C66EA700C44670444280AABAB1EC78B722A0 - -# Email sent directly to teor, verified using relay contact info -# Assume details update is permanent -213.141.138.174:9030 orport=9001 id=BD552C165E2ED2887D3F1CCE9CFF155DDA2D86E6 # Schakalium - -# Email sent directly to teor, verified using relay contact info -95.128.43.164:80 orport=443 id=616081EC829593AF4232550DE6FFAA1D75B37A90 ipv6=[2a02:ec0:209:10::4]:443 - -# Email sent directly to teor, verified using relay contact info -166.82.21.200:9030 orport=9029 id=D5C33F3E203728EDF8361EA868B2939CCC43FAFB - -# Email sent directly to teor, verified using relay contact info -91.121.54.8:9030 orport=9001 id=CBEE0F3303C8C50462A12107CA2AE061831931BC - -# Email sent directly to teor, verified using relay contact info -178.217.184.32:8080 orport=443 id=8B7F47AE1A5D954A3E58ACDE0865D09DBA5B738D - -# Email sent directly to teor, verified using relay contact info -85.10.201.47:9030 orport=9001 id=D8B7A3A6542AA54D0946B9DC0257C53B6C376679 ipv6=[2a01:4f8:a0:43eb::beef]:9001 - -# Email sent directly to teor, verified using relay contact info -120.29.217.46:80 orport=443 id=5E853C94AB1F655E9C908924370A0A6707508C62 - -# Email sent directly to teor, verified using relay contact info -37.153.1.10:9030 orport=9001 id=9772EFB535397C942C3AB8804FB35CFFAD012438 - -# Email sent directly to teor, verified using relay contact info -92.222.4.102:9030 orport=9001 id=1A6B8B8272632D8AD38442027F822A367128405C - -# Email sent directly to teor, verified using relay contact info -31.31.78.49:80 orport=443 id=46791D156C9B6C255C2665D4D8393EC7DBAA7798 - -# Email sent directly to teor -192.160.102.169:80 orport=9001 id=C0192FF43E777250084175F4E59AC1BA2290CE38 ipv6=[2620:132:300c:c01d::9]:9002 # manipogo -192.160.102.166:80 orport=9001 id=547DA56F6B88B6C596B3E3086803CDA4F0EF8F21 ipv6=[2620:132:300c:c01d::6]:9002 # chaucer -192.160.102.170:80 orport=9001 id=557ACEC850F54EEE65839F83CACE2B0825BE811E ipv6=[2620:132:300c:c01d::a]:9002 # ogopogo -192.160.102.164:80 orport=9001 id=823AA81E277F366505545522CEDC2F529CE4DC3F ipv6=[2620:132:300c:c01d::4]:9002 # snowfall -192.160.102.165:80 orport=9001 id=C90CA3B7FE01A146B8268D56977DC4A2C024B9EA ipv6=[2620:132:300c:c01d::5]:9002 # cowcat -192.160.102.168:80 orport=9001 id=F6A358DD367B3282D6EF5824C9D45E1A19C7E815 ipv6=[2620:132:300c:c01d::8]:9002 # prawksi - -# Email sent directly to teor, verified using relay contact info -136.243.214.137:80 orport=443 id=B291D30517D23299AD7CEE3E60DFE60D0E3A4664 - -# Email sent directly to teor, verified using relay contact info -192.87.28.28:9030 orport=9001 id=ED2338CAC2711B3E331392E1ED2831219B794024 -192.87.28.82:9030 orport=9001 id=844AE9CAD04325E955E2BE1521563B79FE7094B7 - -# Email sent directly to teor, verified using relay contact info -192.87.28.28:9030 orport=9001 id=ED2338CAC2711B3E331392E1ED2831219B794024 -# same machine as ED2338CAC2711B3E331392E1ED2831219B794024 -192.87.28.82:9030 orport=9001 id=844AE9CAD04325E955E2BE1521563B79FE7094B7 - -# https://twitter.com/kosjoli/status/719507270904758272 -85.10.202.87:9030 orport=9001 id=971AFB23C168DCD8EDA17473C1C452B359DE3A5A -176.9.5.116:9030 orport=9001 id=A1EB8D8F1EE28DB98BBB1EAA3B4BEDD303BAB911 -46.4.111.124:9030 orport=9001 id=D9065F9E57899B3D272AA212317AF61A9B14D204 - -# Email sent directly to teor, verified using relay contact info -185.100.85.61:80 orport=443 id=025B66CEBC070FCB0519D206CF0CF4965C20C96E - -# Email sent directly to teor, verified using relay contact info -108.166.168.158:80 orport=443 id=CDAB3AE06A8C9C6BF817B3B0F1877A4B91465699 - -# Email sent directly to teor, verified using relay contact info -91.219.236.222:80 orport=443 id=20704E7DD51501DC303FA51B738D7B7E61397CF6 - -# Email sent directly to teor, verified using relay contact info -185.14.185.240:9030 orport=443 id=D62FB817B0288085FAC38A6DC8B36DCD85B70260 -192.34.63.137:9030 orport=443 id=ABCB4965F1FEE193602B50A365425105C889D3F8 -128.199.197.16:9030 orport=443 id=DEE5298B3BA18CDE651421CD2DCB34A4A69F224D - -# Email sent directly to teor, verified using relay contact info -185.13.38.75:9030 orport=9001 id=D2A1703758A0FBBA026988B92C2F88BAB59F9361 - -# Email sent directly to teor, verified using relay contact info -128.204.39.106:9030 orport=9001 id=6F0F3C09AF9580F7606B34A7678238B3AF7A57B7 - -# Email sent directly to teor, verified using relay contact info -198.50.191.95:80 orport=443 id=39F096961ED2576975C866D450373A9913AFDC92 - -# Email sent directly to teor, verified using relay contact info -167.114.66.61:9696 orport=443 id=DE6CD5F09DF26076F26321B0BDFBE78ACD935C65 ipv6=[2607:5300:100::78d]:443 - -# Email sent directly to teor, verified using relay contact info -66.111.2.20:9030 orport=9001 id=9A68B85A02318F4E7E87F2828039FBD5D75B0142 -66.111.2.16:9030 orport=9001 id=3F092986E9B87D3FDA09B71FA3A602378285C77A - -# Email sent directly to teor, verified using relay contact info -92.222.38.67:80 orport=443 id=DED6892FF89DBD737BA689698A171B2392EB3E82 - -# Email sent directly to teor, verified using relay contact info -212.47.228.115:9030 orport=443 id=BCA017ACDA48330D02BB70716639ED565493E36E - -# Email sent directly to teor, verified using relay contact info -185.100.84.175:80 orport=443 id=39B59AF4FE54FAD8C5085FA9C15FDF23087250DB - -# Email sent directly to teor, verified using relay contact info -166.70.207.2:9030 orport=9001 id=E3DB2E354B883B59E8DC56B3E7A353DDFD457812 - -# Emails sent directly to teor, verified using relay contact info -69.162.139.9:9030 orport=9001 id=4791FC0692EAB60DF2BCCAFF940B95B74E7654F6 ipv6=[2607:f128:40:1212::45a2:8b09]:9001 - -# Email sent directly to teor, verified using relay contact info -213.239.217.18:1338 orport=1337 id=C37BC191AC389179674578C3E6944E925FE186C2 ipv6=[2a01:4f8:a0:746a:101:1:1:1]:1337 - -# Email sent directly to teor, verified using relay contact info -# Assume details update is permanent -188.40.128.246:9030 orport=9001 id=AD19490C7DBB26D3A68EFC824F67E69B0A96E601 ipv6=[2a01:4f8:221:1ac1:dead:beef:7005:9001]:9001 # sputnik - -# Email sent directly to teor, verified using relay contact info -88.198.253.13:9030 orport=9001 id=DF924196D69AAE3C00C115A9CCDF7BB62A175310 ipv6=[2a01:4f8:11a:b1f::2]:9001 - -# Email sent directly to teor, verified using relay contact info -185.100.86.128:9030 orport=9001 id=9B31F1F1C1554F9FFB3455911F82E818EF7C7883 -46.36.36.127:9030 orport=9001 id=C80DF89B21FF932DEC0D7821F679B6C79E1449C3 - -# Email sent directly to teor, verified using relay contact info -176.10.104.240:80 orport=443 id=0111BA9B604669E636FFD5B503F382A4B7AD6E80 -176.10.104.240:8080 orport=8443 id=AD86CD1A49573D52A7B6F4A35750F161AAD89C88 -176.10.104.243:80 orport=443 id=88487BDD980BF6E72092EE690E8C51C0AA4A538C -176.10.104.243:8080 orport=8443 id=95DA61AEF23A6C851028C1AA88AD8593F659E60F - -# Email sent directly to teor, verified using relay contact info -107.170.101.39:9030 orport=443 id=30973217E70AF00EBE51797FF6D9AA720A902EAA - -# Email sent directly to teor -193.70.112.165:80 orport=443 id=F10BDE279AE71515DDCCCC61DC19AC8765F8A3CC # ParkBenchInd001 - -# Email sent directly to teor -185.220.101.6:10006 orport=20006 id=C08DE49658E5B3CFC6F2A952B453C4B608C9A16A # niftyvolcanorabbit -185.220.101.13:10013 orport=20013 id=71AB4726D830FAE776D74AEF790CF04D8E0151B4 # niftycottontail -185.220.101.5:10005 orport=20005 id=1084200B44021D308EA4253F256794671B1D099A # niftyhedgehog -185.220.101.9:10009 orport=20009 id=14877C6384A9E793F422C8D1DDA447CACA4F7C4B # niftywoodmouse -185.220.101.8:10008 orport=20008 id=24E91955D969AEA1D80413C64FE106FAE7FD2EA9 # niftymouse -185.220.101.1:10001 orport=20001 id=28F4F392F8F19E3FBDE09616D9DB8143A1E2DDD3 # niftycottonmouse -185.220.101.21:10021 orport=20021 id=348B89013EDDD99E4755951D1EC284D9FED71226 # niftysquirrel -185.220.101.10:10010 orport=20010 id=4031460683AE9E0512D3620C2758D98758AC6C93 # niftyeuropeanrabbit -185.220.101.34:10034 orport=20034 id=47C42E2094EE482E7C9B586B10BABFB67557030B # niftyquokka -185.220.101.18:10018 orport=20018 id=5D5006E4992F2F97DF4F8B926C3688870EB52BD8 # niftyplagiodontia -185.220.101.28:10028 orport=20028 id=609E598FB6A00BCF7872906B602B705B64541C50 # niftychipmunk -185.220.101.20:10020 orport=20020 id=619349D82424C601CAEB94161A4CF778993DAEE7 # niftytucotuco -185.220.101.17:10017 orport=20017 id=644DECC5A1879C0FE23DE927DD7049F58BBDF349 # niftyhutia -185.220.101.0:10000 orport=20000 id=6E94866ED8CA098BACDFD36D4E8E2B459B8A734E # niftybeaver -185.220.101.30:10030 orport=20030 id=71CFDEB4D9E00CCC3E31EC4E8A29E109BBC1FB36 # niftypedetidae -185.220.101.29:10029 orport=20029 id=7DC52AE6667A30536BA2383CD102CFC24F20AD71 # niftyllipika -185.220.101.41:10041 orport=20041 id=7E281CD2C315C4F7A84BC7C8721C3BC974DDBFA3 # niftyporcupine -185.220.101.25:10025 orport=20025 id=8EE0534532EA31AA5172B1892F53B2F25C76EB02 # niftyjerboa -185.220.101.33:10033 orport=20033 id=906DCB390F2BA987AE258D745E60BAAABAD31DE8 # niftyquokka -185.220.101.26:10026 orport=20026 id=92A6085EABAADD928B6F8E871540A1A41CBC08BA # niftypedetes -185.220.101.40:10040 orport=20040 id=9A857254F379194D1CD76F4A79A20D2051BEDA3F # niftynutria -185.220.101.42:10042 orport=20042 id=9B816A5B3EB20B8E4E9B9D1FBA299BD3F40F0320 # niftypygmyjerboa -185.220.101.2:10002 orport=20002 id=B740BCECC4A9569232CDD45C0E1330BA0D030D33 # niftybunny -185.220.101.32:10032 orport=20032 id=B771AA877687F88E6F1CA5354756DF6C8A7B6B24 # niftypika -185.220.101.12:10012 orport=20012 id=BC82F2190DE2E97DE65F49B4A95572374BDC0789 # niftycapybara -185.220.101.22:10022 orport=20022 id=CA37CD46799449D83B6B98B8C22C649906307888 # niftyjackrabbit -185.220.101.4:10004 orport=20004 id=CDA2EA326E2272C57ACB26773D7252C211795B78 # niftygerbil -185.220.101.14:10014 orport=20014 id=E7EBA5D8A4E09684D11A1DF24F75362817333768 # niftyhare -185.220.101.16:10016 orport=20016 id=EC1997D51892E4607C68E800549A1E7E4694005A # niftyguineapig -185.220.101.24:10024 orport=20024 id=FDA70EC93DB01E3CB418CB6943B0C68464B18B4C # niftyrat - -# Email sent directly to teor, verified using relay contact info -64.113.32.29:9030 orport=9001 id=30C19B81981F450C402306E2E7CFB6C3F79CB6B2 - -# Emails sent directly to teor, verified using relay contact info -51.254.101.242:9002 orport=9001 id=4CC9CC9195EC38645B699A33307058624F660CCF - -# Emails sent directly to teor, verified using relay contact info -85.214.62.48:80 orport=443 id=6A7551EEE18F78A9813096E82BF84F740D32B911 - -# Email sent directly to teor, verified using relay contact info -173.255.245.116:9030 orport=9001 id=91E4015E1F82DAF0121D62267E54A1F661AB6DC7 - -# Email sent directly to teor, verified using relay contact info -62.216.5.120:9030 orport=9001 id=D032D4D617140D6B828FC7C4334860E45E414FBE - -# Email sent directly to teor, verified using relay contact info -51.254.136.195:80 orport=443 id=7BB70F8585DFC27E75D692970C0EEB0F22983A63 - -# Email sent directly to teor, verified using relay contact info -163.172.13.165:9030 orport=9001 id=33DA0CAB7C27812EFF2E22C9705630A54D101FEB ipv6=[2001:bc8:38cb:201::8]:9001 - -# Email sent directly to teor, verified using relay contact info -5.196.88.122:9030 orport=9001 id=0C2C599AFCB26F5CFC2C7592435924C1D63D9484 ipv6=[2001:41d0:a:fb7a::1]:9001 - -# Email sent directly to teor, verified using relay contact info -5.9.158.75:80 orport=443 id=1AF72E8906E6C49481A791A6F8F84F8DFEBBB2BA ipv6=[2a01:4f8:190:514a::2]:443 - -# Email sent directly to teor, verified using relay contact info -46.101.169.151:9030 orport=9001 id=D760C5B436E42F93D77EF2D969157EEA14F9B39C ipv6=[2a03:b0c0:3:d0::74f:a001]:9001 - -# Email sent directly to teor, verified using relay contact info -199.249.223.81:80 orport=443 id=F7447E99EB5CBD4D5EB913EE0E35AC642B5C1EF3 -199.249.223.79:80 orport=443 id=D33292FEDE24DD40F2385283E55C87F85C0943B6 -199.249.223.78:80 orport=443 id=EC15DB62D9101481F364DE52EB8313C838BDDC29 -199.249.223.77:80 orport=443 id=CC4A3AE960E3617F49BF9887B79186C14CBA6813 -199.249.223.76:80 orport=443 id=43209F6D50C657A56FE79AF01CA69F9EF19BD338 -199.249.223.75:80 orport=443 id=60D3667F56AEC5C69CF7E8F557DB21DDF6C36060 -199.249.223.74:80 orport=443 id=5F4CD12099AF20FAF9ADFDCEC65316A376D0201C -199.249.223.73:80 orport=443 id=5649CB2158DA94FB747415F26628BEC07FA57616 -199.249.223.72:80 orport=443 id=B028707969D8ED84E6DEA597A884F78AAD471971 -199.249.223.71:80 orport=443 id=B6320E44A230302C7BF9319E67597A9B87882241 -199.249.223.60:80 orport=443 id=B7047FBDE9C53C39011CA84E5CB2A8E3543066D0 -199.249.223.61:80 orport=443 id=40E7D6CE5085E4CDDA31D51A29D1457EB53F12AD -199.249.223.62:80 orport=443 id=0077BCBA7244DB3E6A5ED2746E86170066684887 -199.249.223.63:80 orport=443 id=1DB25DF59DAA01B5BE3D3CEB8AFED115940EBE8B -199.249.223.64:80 orport=443 id=9F2856F6D2B89AD4EF6D5723FAB167DB5A53519A -199.249.223.65:80 orport=443 id=9D21F034C3BFF4E7737D08CF775DC1745706801F -199.249.223.66:80 orport=443 id=C5A53BCC174EF8FD0DCB223E4AA929FA557DEDB2 -199.249.223.67:80 orport=443 id=155D6F57425F16C0624D77777641E4EB1B47C6F0 -199.249.223.68:80 orport=443 id=DF20497E487A979995D851A5BCEC313DF7E5BC51 -199.249.223.69:80 orport=443 id=7FA8E7E44F1392A4E40FFC3B69DB3B00091B7FD3 - -# https://lists.torproject.org/pipermail/tor-relays/2016-December/011114.html -86.105.212.130:9030 orport=443 id=9C900A7F6F5DD034CFFD192DAEC9CCAA813DB022 - -# Email sent directly to teor, verified using relay contact info -178.33.183.251:80 orport=443 id=DD823AFB415380A802DCAEB9461AE637604107FB ipv6=[2001:41d0:2:a683::251]:443 - -# Email sent directly to teor, verified using relay contact info -31.185.104.19:80 orport=443 id=9EAD5B2D3DBD96DBC80DCE423B0C345E920A758D -# same machine as 9EAD5B2D3DBD96DBC80DCE423B0C345E920A758D -31.185.104.20:80 orport=443 id=ADB2C26629643DBB9F8FE0096E7D16F9414B4F8D -31.185.104.21:80 orport=443 id=C2AAB088555850FC434E68943F551072042B85F1 -31.185.104.22:80 orport=443 id=5BA3A52760A0EABF7E7C3ED3048A77328FF0F148 - -# Email sent directly to teor, verified using relay contact info -185.34.60.114:80 orport=443 id=7F7A695DF6F2B8640A70B6ADD01105BC2EBC5135 - -# https://lists.torproject.org/pipermail/tor-relays/2017-December/013939.html -94.142.242.84:80 orport=443 id=AA0D167E03E298F9A8CD50F448B81FBD7FA80D56 ipv6=[2a02:898:24:84::1]:443 # rejozenger - -# Email sent directly to teor, verified using relay contact info -185.129.62.62:9030 orport=9001 id=ACDD9E85A05B127BA010466C13C8C47212E8A38F ipv6=[2a06:d380:0:3700::62]:9001 - -# Email sent directly to teor, verified using relay contact info -# The e84 part of the IPv6 address does not have a leading 0 in the consensus -81.30.158.213:9030 orport=9001 id=789EA6C9AE9ADDD8760903171CFA9AC5741B0C70 ipv6=[2001:4ba0:cafe:e84::1]:9001 - -# https://lists.torproject.org/pipermail/tor-relays/2016-December/011209.html -5.9.159.14:9030 orport=9001 id=0F100F60C7A63BED90216052324D29B08CFCF797 - -# Email sent directly to teor, verified using relay contact info -45.62.255.25:80 orport=443 id=3473ED788D9E63361D1572B7E82EC54338953D2A - -# Email sent directly to teor, verified using relay contact info -217.79.179.177:9030 orport=9001 id=3E53D3979DB07EFD736661C934A1DED14127B684 ipv6=[2001:4ba0:fff9:131:6c4f::90d3]:9001 - -# Email sent directly to teor, verified using relay contact info -212.47.244.38:8080 orport=443 id=E81EF60A73B3809F8964F73766B01BAA0A171E20 -163.172.157.213:8080 orport=443 id=4623A9EC53BFD83155929E56D6F7B55B5E718C24 -163.172.139.104:8080 orport=443 id=68F175CCABE727AA2D2309BCD8789499CEE36ED7 - -# Email sent directly to teor, verified using relay contact info -163.172.223.200:80 orport=443 id=998BF3ED7F70E33D1C307247B9626D9E7573C438 -195.154.122.54:80 orport=443 id=64E99CB34C595A02A3165484BD1215E7389322C6 - -# Email sent directly to teor, verified using relay contact info -185.100.86.128:9030 orport=9001 id=9B31F1F1C1554F9FFB3455911F82E818EF7C7883 -185.100.85.101:9030 orport=9001 id=4061C553CA88021B8302F0814365070AAE617270 -31.171.155.108:9030 orport=9001 id=D3E5EDDBE5159388704D6785BE51930AAFACEC6F - -# Email sent directly to teor, verified using relay contact info -89.163.247.43:9030 orport=9001 id=BC7ACFAC04854C77167C7D66B7E471314ED8C410 ipv6=[2001:4ba0:fff7:25::5]:9001 - -# Email sent directly to teor, verified using relay contact info -95.85.8.226:80 orport=443 id=1211AC1BBB8A1AF7CBA86BCE8689AA3146B86423 - -# Email sent directly to teor, verified using relay contact info -85.214.151.72:9030 orport=9001 id=722D365140C8C52DBB3C9FF6986E3CEFFE2BA812 - -# email sent directly to teor -72.52.75.27:9030 orport=9001 id=8567AD0A6369ED08527A8A8533A5162AC00F7678 # piecoopdotnet - -# Email sent directly to teor, verified using relay contact info -5.9.146.203:80 orport=443 id=1F45542A24A61BF9408F1C05E0DCE4E29F2CBA11 -5.9.159.14:9030 orport=9001 id=0F100F60C7A63BED90216052324D29B08CFCF797 - -# Email sent directly to teor, verified using relay contact info -# Assume details update is permanent -5.9.147.226:9030 orport=9001 id=B0553175AADB0501E5A61FC61CEA3970BE130FF2 ipv6=[2a01:4f8:190:30e1::2]:9001 # zwiubel - -# https://trac.torproject.org/projects/tor/ticket/22527#comment:1 -199.184.246.250:80 orport=443 id=1F6ABD086F40B890A33C93CC4606EE68B31C9556 ipv6=[2620:124:1009:1::171]:443 - -# https://trac.torproject.org/projects/tor/ticket/24695 -163.172.53.84:143 orport=21 id=1C90D3AEADFF3BCD079810632C8B85637924A58E ipv6=[2001:bc8:24f8::]:21 # Multivac - -# Email sent directly to teor -54.36.237.163:80 orport=443 id=DB2682153AC0CCAECD2BD1E9EBE99C6815807A1E # GermanCraft2 - -# Email sent directly to teor -62.138.7.171:9030 orport=9001 id=9844B981A80B3E4B50897098E2D65167E6AEF127 # 0x3d004 -62.138.7.171:8030 orport=8001 id=9285B22F7953D7874604EEE2B470609AD81C74E9 # 0x3d005 -91.121.23.100:9030 orport=9001 id=3711E80B5B04494C971FB0459D4209AB7F2EA799 # 0x3d002 -91.121.23.100:8030 orport=8001 id=CFBBA0D858F02E40B1432A65F6D13C9BDFE7A46B # 0x3d001 -51.15.13.245:9030 orport=9001 id=CED527EAC230E7B56E5B363F839671829C3BA01B # 0x3d006 -51.15.13.245:8030 orport=8001 id=8EBB8D1CF48FE2AB95C451DA8F10DB6235F40F8A # 0x3d007 - -# Email sent directly to teor -104.192.5.248:9030 orport=9001 id=BF735F669481EE1CCC348F0731551C933D1E2278 # Freeway11 - -# Email sent directly to teor -# https://lists.torproject.org/pipermail/tor-relays/2017-December/013961.html -178.17.174.14:9030 orport=9001 id=B06F093A3D4DFAD3E923F4F28A74901BD4F74EB1 # TorExitMoldova -178.17.170.156:9030 orport=9001 id=41C59606AFE1D1AA6EC6EF6719690B856F0B6587 # TorExitMoldova2 - -# Email sent directly to teor -163.172.221.44:59030 orport=59001 id=164604F5C86FC8CC9C0288BD9C02311958427597 # altego - -# Email sent directly to teor -46.38.237.221:9030 orport=9001 id=D30E9D4D639068611D6D96861C95C2099140B805 # mine - -# https://lists.torproject.org/pipermail/tor-relays/2017-December/013911.html -# https://lists.torproject.org/pipermail/tor-relays/2017-December/013912.html -199.249.223.62:80 orport=443 id=0077BCBA7244DB3E6A5ED2746E86170066684887 # Quintex13 -199.249.224.45:80 orport=443 id=041646640AB306EA74B001966E86169B04CC88D2 # QuintexAirVPN26 -199.249.223.67:80 orport=443 id=155D6F57425F16C0624D77777641E4EB1B47C6F0 # Quintex18 -199.249.223.45:80 orport=443 id=1AE949967F82BBE7534A3D6BA77A7EBE1CED4369 # Quintex36 -199.249.223.63:80 orport=443 id=1DB25DF59DAA01B5BE3D3CEB8AFED115940EBE8B # Quintex14 -199.249.224.63:80 orport=443 id=1E5136DDC52FAE1219208F0A6BADB0BA62587EE6 # Quintex43 -199.249.224.46:80 orport=443 id=2ED4D25766973713EB8C56A290BF07E06B85BF12 # QuintexAirVPN27 -199.249.223.42:80 orport=443 id=3687FEC7E73F61AC66F7AE251E7DEE6BBD8C0252 # Quintex33 -199.249.223.49:80 orport=443 id=36D68478366CB8627866757EBCE7FB3C17FC1CB8 # Quintex40 -199.249.224.49:80 orport=443 id=3CA0D15567024D2E0B557DC0CF3E962B37999A79 # QuintexAirVPN30 -199.249.223.61:80 orport=443 id=40E7D6CE5085E4CDDA31D51A29D1457EB53F12AD # Quintex12 -199.249.223.76:80 orport=443 id=43209F6D50C657A56FE79AF01CA69F9EF19BD338 # QuintexAirVPN5 -199.249.224.41:80 orport=443 id=54A4820B46E65509BF3E2B892E66930A41759DE9 # QuintexAirVPN22 -199.249.223.73:80 orport=443 id=5649CB2158DA94FB747415F26628BEC07FA57616 # QuintexAirVPN8 -199.249.223.74:80 orport=443 id=5F4CD12099AF20FAF9ADFDCEC65316A376D0201C # QuintexAirVPN7 -199.249.223.75:80 orport=443 id=60D3667F56AEC5C69CF7E8F557DB21DDF6C36060 # QuintexAirVPN6 -199.249.223.46:80 orport=443 id=66E19E8C4773086F669A1E06A3F8C23B6C079129 # Quintex37 -199.249.224.65:80 orport=443 id=764BF8A03868F84C8F323C1A676AA254B80DC3BF # Quintex45 -199.249.223.48:80 orport=443 id=7A3DD280EA4CD4DD16EF8C67B93D9BDE184D1A81 # Quintex39 -199.249.224.68:80 orport=443 id=7E6E9A6FDDB8DC7C92F0CFCC3CBE76C29F061799 # Quintex48 -199.249.223.69:80 orport=443 id=7FA8E7E44F1392A4E40FFC3B69DB3B00091B7FD3 # Quintex20 -199.249.223.44:80 orport=443 id=8B80169BEF71450FC4069A190853523B7AEA45E1 # Quintex35 -199.249.224.60:80 orport=443 id=9314BD9503B9014261A65C221D77E57389DBCCC1 # Quintex50 -199.249.224.40:80 orport=443 id=9C1E7D92115D431385B8CAEA6A7C15FB89CE236B # QuintexAirVPN21 -199.249.223.65:80 orport=443 id=9D21F034C3BFF4E7737D08CF775DC1745706801F # Quintex16 -199.249.224.67:80 orport=443 id=9E2D7C6981269404AA1970B53891701A20424EF8 # Quintex47 -199.249.223.64:80 orport=443 id=9F2856F6D2B89AD4EF6D5723FAB167DB5A53519A # Quintex15 -199.249.224.48:80 orport=443 id=A0DB820FEC87C0405F7BF05DEE5E4ADED2BB9904 # QuintexAirVPN29 -199.249.224.64:80 orport=443 id=A4A393FEF48640961AACE92D041934B55348CEF9 # Quintex44 -199.249.223.72:80 orport=443 id=B028707969D8ED84E6DEA597A884F78AAD471971 # QuintexAirVPN9 -199.249.223.40:80 orport=443 id=B0CD9F9B5B60651ADC5919C0F1EAA87DBA1D9249 # Quintex31 -199.249.224.61:80 orport=443 id=B2197C23A4FF5D1C49EE45BA7688BA8BCCD89A0B # Quintex41 -199.249.223.71:80 orport=443 id=B6320E44A230302C7BF9319E67597A9B87882241 # QuintexAirVPN10 -199.249.223.60:80 orport=443 id=B7047FBDE9C53C39011CA84E5CB2A8E3543066D0 # Quintex11 -199.249.224.66:80 orport=443 id=C78AFFEEE320EA0F860961763E613FD2FAC855F5 # Quintex46 -199.249.224.44:80 orport=443 id=CB7C0D841FE376EF43F7845FF201B0290C0A239E # QuintexAirVPN25 -199.249.223.47:80 orport=443 id=CC14C97F1D23EE97766828FC8ED8582E21E11665 # Quintex38 -199.249.223.77:80 orport=443 id=CC4A3AE960E3617F49BF9887B79186C14CBA6813 # QuintexAirVPN4 -199.249.223.41:80 orport=443 id=D25210CE07C49F2A4F2BC7A506EB0F5EA7F5E2C2 # Quintex32 -199.249.223.79:80 orport=443 id=D33292FEDE24DD40F2385283E55C87F85C0943B6 # QuintexAirVPN2 -199.249.224.47:80 orport=443 id=D6FF2697CEA5C0C7DA84797C2E71163814FC2466 # QuintexAirVPN28 -199.249.223.68:80 orport=443 id=DF20497E487A979995D851A5BCEC313DF7E5BC51 # Quintex19 -199.249.223.43:80 orport=443 id=E480D577F58E782A5BC4FA6F49A6650E9389302F # Quintex34 -199.249.224.69:80 orport=443 id=EABC2DD0D47B5DB11F2D37EB3C60C2A4D91C10F2 # Quintex49 -199.249.223.78:80 orport=443 id=EC15DB62D9101481F364DE52EB8313C838BDDC29 # QuintexAirVPN3 -199.249.224.42:80 orport=443 id=F21DE9C7DE31601D9716781E17E24380887883D1 # QuintexAirVPN23 -199.249.223.81:80 orport=443 id=F7447E99EB5CBD4D5EB913EE0E35AC642B5C1EF3 # QuintexAirVPN1 -199.249.224.43:80 orport=443 id=FDD700C791CC6BB0AC1C2099A82CBC367AD4B764 # QuintexAirVPN24 -199.249.224.62:80 orport=443 id=FE00A3A835680E67FBBC895A724E2657BB253E97 # Quintex42 -199.249.223.66:80 orport=443 id=C5A53BCC174EF8FD0DCB223E4AA929FA557DEDB2 # Quintex17 - -# https://lists.torproject.org/pipermail/tor-relays/2017-December/013914.html -5.196.23.64:9030 orport=9001 id=775B0FAFDE71AADC23FFC8782B7BEB1D5A92733E # Aerodynamik01 -217.182.75.181:9030 orport=9001 id=EFEACD781604EB80FBC025EDEDEA2D523AEAAA2F # Aerodynamik02 -193.70.43.76:9030 orport=9001 id=484A10BA2B8D48A5F0216674C8DD50EF27BC32F3 # Aerodynamik03 -149.56.141.138:9030 orport=9001 id=1938EBACBB1A7BFA888D9623C90061130E63BB3F # Aerodynamik04 - -# https://lists.torproject.org/pipermail/tor-relays/2017-December/013917.html -104.200.20.46:80 orport=9001 id=78E2BE744A53631B4AAB781468E94C52AB73968B # bynumlawtor - -# https://lists.torproject.org/pipermail/tor-relays/2017-December/013929.html -139.99.130.178:80 orport=443 id=867B95CACD64653FEEC4D2CEFC5C49B4620307A7 # coffswifi2 - -# https://lists.torproject.org/pipermail/tor-relays/2017-December/013946.html -172.98.193.43:80 orport=443 id=5E56738E7F97AA81DEEF59AF28494293DFBFCCDF # Backplane - -# Email sent directly to teor -62.210.254.132:80 orport=443 id=8456DFA94161CDD99E480C2A2992C366C6564410 # turingmachine - -# Email sent directly to teor -80.127.117.180:80 orport=443 id=328E54981C6DDD7D89B89E418724A4A7881E3192 ipv6=[2001:985:e77:10::4]:443 # sjc01 - -# https://lists.torproject.org/pipermail/tor-relays/2017-December/013960.html -51.15.205.214:9030 orport=9001 id=8B6556601612F1E2AFCE2A12FFFAF8482A76DD1F ipv6=[2001:bc8:4400:2500::5:b07]:9001 # titania1 -51.15.205.214:9031 orport=9002 id=5E363D72488276160D062DDD2DFA25CFEBAF5EA9 ipv6=[2001:bc8:4400:2500::5:b07]:9002 # titania2 - -# Email sent directly to teor -185.129.249.124:9030 orport=9001 id=1FA8F638298645BE58AC905276680889CB795A94 # treadstone - -# https://lists.torproject.org/pipermail/tor-relays/2017-December/014000.html -24.117.231.229:34175 orport=45117 id=CE24412AD69444954B4015E293AE53DDDAFEA3D6 # Anosognosia - -# https://lists.torproject.org/pipermail/tor-relays/2018-January/014012.html -128.31.0.13:80 orport=443 id=A53C46F5B157DD83366D45A8E99A244934A14C46 # csailmitexit - -# Email sent directly to teor -82.247.103.117:110 orport=995 id=C9B3C1661A9577BA24C1C2C6123918921A495509 # Casper01 -109.238.2.79:110 orport=995 id=7520892E3DD133D0B0464D01A158B54B8E2A8B75 # Casper02 -51.15.179.153:110 orport=995 id=BB60F5BA113A0B8B44B7B37DE3567FE561E92F78 # Casper04 - -# Email sent directly to teor -80.127.107.179:80 orport=443 id=BC6B2E2F62ACC5EDECBABE64DA1E48F84DD98B78 ipv6=[2001:981:4a22:c::6]:443 # TVISION02 - -# https://lists.torproject.org/pipermail/tor-relays/2018-January/014020.html -37.120.174.249:80 orport=443 id=11DF0017A43AF1F08825CD5D973297F81AB00FF3 ipv6=[2a03:4000:6:724c:df98:15f9:b34d:443]:443 # gGDHjdcC6zAlM8k08lX - -# These fallbacks opted-in in previous releases, then changed their details, -# and so we blacklisted them. Now we want to whitelist changes. -# Assume details update is permanent -85.230.184.93:9030 orport=443 id=855BC2DABE24C861CD887DB9B2E950424B49FC34 # Logforme -176.31.180.157:143 orport=22 id=E781F4EC69671B3F1864AE2753E0890351506329 ipv6=[2001:41d0:8:eb9d::1]:22 # armbrust - -# https://lists.torproject.org/pipermail/tor-relays/2018-January/014024.html -82.161.212.209:9030 orport=9001 id=4E8CE6F5651E7342C1E7E5ED031E82078134FB0D ipv6=[2001:980:d7ed:1:ff:b0ff:fe00:d0b]:9001 # ymkeo diff --git a/scripts/maint/format_changelog.py b/scripts/maint/format_changelog.py index 8dce4b6e51..08b2155fa3 100755 --- a/scripts/maint/format_changelog.py +++ b/scripts/maint/format_changelog.py @@ -401,7 +401,7 @@ class ChangeLog(object): self.dumpEndOfChangelog() # Let's turn bugs to html. -BUG_PAT = re.compile('(bug|ticket|feature)\s+(\d{4,5})', re.I) +BUG_PAT = re.compile('(bug|ticket|issue|feature)\s+(\d{4,5})', re.I) def bug_html(m): return "%s <a href='https://bugs.torproject.org/%s'>%s</a>" % (m.group(1), m.group(2), m.group(2)) diff --git a/scripts/maint/generateFallbackDirLine.py b/scripts/maint/generateFallbackDirLine.py deleted file mode 100755 index b856c938bf..0000000000 --- a/scripts/maint/generateFallbackDirLine.py +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env python - -# Generate a fallback directory whitelist/blacklist line for every fingerprint -# passed as an argument. -# -# Usage: -# generateFallbackDirLine.py fingerprint ... - -import sys -import urllib2 - -import stem.descriptor.remote -import stem.util.tor_tools - -if len(sys.argv) <= 1: - print('Usage: %s fingerprint ...' % sys.argv[0]) - sys.exit(1) - -for fingerprint in sys.argv[1:]: - if not stem.util.tor_tools.is_valid_fingerprint(fingerprint): - print("'%s' isn't a valid relay fingerprint" % fingerprint) - sys.exit(1) - - try: - desc = stem.descriptor.remote.get_server_descriptors(fingerprint).run()[0] - except urllib2.HTTPError as exc: - if exc.code == 404: - print('# %s not found in recent descriptors' % fingerprint) - continue - else: - raise - - if not desc.dir_port: - print("# %s needs a DirPort" % fingerprint) - else: - ipv6_addresses = [(address, port) for address, port, is_ipv6 in desc.or_addresses if is_ipv6] - ipv6_field = ' ipv6=[%s]:%s' % ipv6_addresses[0] if ipv6_addresses else '' - print('%s:%s orport=%s id=%s%s # %s' % (desc.address, desc.dir_port, desc.or_port, fingerprint, ipv6_field, desc.nickname)) diff --git a/scripts/maint/lintChanges.py b/scripts/maint/lintChanges.py index d5b8fcae5c..82c118f07e 100755 --- a/scripts/maint/lintChanges.py +++ b/scripts/maint/lintChanges.py @@ -35,6 +35,36 @@ NEEDS_SUBCATEGORIES = set([ "Major features", ]) +def split_tor_version(version): + ''' + Return the initial numeric components of the Tor version as a list of ints. + For versions earlier than 0.1.0, returns MAJOR, MINOR, and MICRO. + For versions 0.1.0 and later, returns MAJOR, MINOR, MICRO, and PATCHLEVEL if present. + + If the version is malformed, returns None. + ''' + version_match = re.match('([0-9]+)\.([0-9]+)\.([0-9]+)(\.([0-9]+))?', version) + if version_match is None: + return None + + version_groups = version_match.groups() + if version_groups is None: + return None + if len(version_groups) < 3: + return None + + if len(version_groups) != 5: + return None + version_components = version_groups[0:3] + version_components += version_groups[4:5] + + try: + version_list = [int(v) for v in version_components if v is not None] + except ValueError: + return None + + return version_list + def lintfile(fname): have_warned = [] @@ -87,6 +117,32 @@ def lintfile(fname): warn("Bugfix does not say 'Fixes bug X; bugfix on Y'") elif re.search('tor-([0-9]+)', contents): warn("Do not prefix versions with 'tor-'. ('0.1.2', not 'tor-0.1.2'.)") + else: + bugfix_match = re.search('bugfix on ([0-9]+\.[0-9]+\.[0-9]+)', contents) + if bugfix_match is None: + warn("Versions must have at least 3 digits. ('0.1.2', '0.3.4.8', or '0.3.5.1-alpha'.)") + elif bugfix_match.group(0) is None: + warn("Versions must have at least 3 digits. ('0.1.2', '0.3.4.8', or '0.3.5.1-alpha'.)") + else: + bugfix_match = re.search('bugfix on ([0-9a-z][-.0-9a-z]+[0-9a-z])', contents) + bugfix_group = bugfix_match.groups() if bugfix_match is not None else None + bugfix_version = bugfix_group[0] if bugfix_group is not None else None + package_version = os.environ.get('PACKAGE_VERSION', None) + if bugfix_version is None: + # This should be unreachable, unless the patterns are out of sync + warn("Malformed bugfix version.") + elif package_version is not None: + # If $PACKAGE_VERSION isn't set, skip this check + bugfix_split = split_tor_version(bugfix_version) + package_split = split_tor_version(package_version) + if bugfix_split is None: + # This should be unreachable, unless the patterns are out of sync + warn("Malformed bugfix version: '{}'.".format(bugfix_version)) + elif package_split is None: + # This should be unreachable, unless the patterns are out of sync, or the package versioning scheme has changed + warn("Malformed $PACKAGE_VERSION: '{}'.".format(package_version)) + elif bugfix_split > package_split: + warn("Bugfixes must be made on earlier versions (or this version). (Bugfix on version: '{}', current tor package version: '{}'.)".format(bugfix_version, package_version)) return have_warned != [] diff --git a/scripts/maint/lookupFallbackDirContact.py b/scripts/maint/lookupFallbackDirContact.py deleted file mode 100755 index 14c53d1282..0000000000 --- a/scripts/maint/lookupFallbackDirContact.py +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env python - -# Lookup fallback directory contact lines for every fingerprint passed as an -# argument. -# -# Usage: -# lookupFallbackDirContact.py fingerprint ... - -import sys - -import stem.descriptor.remote as remote - -if len(sys.argv) <= 1: - print "Usage: {} fingerprint ...".format(sys.argv[0]) - sys.exit(-1) - -# we need descriptors, because the consensus does not have contact infos -descriptor_list = remote.get_server_descriptors(fingerprints=sys.argv[1:]).run() - -descriptor_list_fingerprints = [] -for d in descriptor_list: - assert d.fingerprint in sys.argv[1:] - descriptor_list_fingerprints.append(d.fingerprint) - print "{} {}".format(d.fingerprint, d.contact) - -for fingerprint in sys.argv[1:]: - if fingerprint not in descriptor_list_fingerprints: - print "{} not found in current descriptors".format(fingerprint) diff --git a/scripts/maint/pre-commit.git-hook b/scripts/maint/pre-commit.git-hook new file mode 100755 index 0000000000..b4c4ce2061 --- /dev/null +++ b/scripts/maint/pre-commit.git-hook @@ -0,0 +1,26 @@ +#!/bin/bash +# +# To install this script, copy it to .git/hooks/pre-commit in local copy of +# tor git repo and make sure it has permission to execute. +# +# This is pre-commit git hook script that prevents commiting your changeset if +# it fails our code formatting or changelog entry formatting checkers. + +workdir=$(git rev-parse --show-toplevel) + +cd "$workdir" || exit 1 + +python scripts/maint/lintChanges.py ./changes/* + +perl scripts/maint/checkSpace.pl -C \ +src/lib/*/*.[ch] \ +src/core/*/*.[ch] \ +src/feature/*/*.[ch] \ +src/app/*/*.[ch] \ +src/test/*.[ch] \ +src/test/*/*.[ch] \ +src/tools/*.[ch] + +if test -e scripts/maint/checkIncludes.py; then + python scripts/maint/checkIncludes.py +fi diff --git a/scripts/maint/pre-push.git-hook b/scripts/maint/pre-push.git-hook new file mode 100755 index 0000000000..26296023fb --- /dev/null +++ b/scripts/maint/pre-push.git-hook @@ -0,0 +1,61 @@ +#!/bin/bash + +# To install this script, copy it into .git/hooks/pre-push path in your +# local copy of git repository. Make sure it has permission to execute. +# +# This is git pre-push hook script to prevent "fixup!" and "squash!" commits +# from ending up in upstream branches (master, release-* or maint-*). +# +# The following sample script was used as starting point: +# https://github.com/git/git/blob/master/templates/hooks--pre-push.sample + +z40=0000000000000000000000000000000000000000 + +CUR_BRANCH=$(git rev-parse --abbrev-ref HEAD) +if [ "$CUR_BRANCH" != "master" ] && [[ $CUR_BRANCH != release-* ]] && + [[ $CUR_BRANCH != maint-* ]] +then + exit 0 +fi + +echo "Running pre-push hook" + +# shellcheck disable=SC2034 +while read -r local_ref local_sha remote_ref remote_sha +do + if [ "$local_sha" = $z40 ] + then + # Handle delete + : + else + if [ "$remote_sha" = $z40 ] + then + # New branch, examine all commits + range="$local_sha" + else + # Update to existing branch, examine new commits + range="$remote_sha..$local_sha" + fi + + # Check for fixup! commit + commit=$(git rev-list -n 1 --grep '^fixup!' "$range") + if [ -n "$commit" ] + then + echo >&2 "Found fixup! commit in $local_ref, not pushing" + echo >&2 "If you really want to push this, use --no-verify." + exit 1 + fi + + # Check for squash! commit + commit=$(git rev-list -n 1 --grep '^squash!' "$range") + if [ -n "$commit" ] + then + echo >&2 "Found squash! commit in $local_ref, not pushing" + echo >&2 "If you really want to push this, use --no-verify." + exit 1 + fi + fi +done + +exit 0 + diff --git a/scripts/maint/run_calltool.sh b/scripts/maint/run_calltool.sh index efb8706fea..b0268322f4 100755 --- a/scripts/maint/run_calltool.sh +++ b/scripts/maint/run_calltool.sh @@ -15,10 +15,10 @@ SUBITEMS="fn_graph fn_invgraph fn_scc fn_scc_weaklinks module_graph module_invgr for calculation in $SUBITEMS; do echo "======== $calculation" - python -m calltool $calculation > callgraph/$calculation + python -m calltool "$calculation" > callgraph/"$calculation" done -echo <<EOF > callgraph/README +cat <<EOF > callgraph/README This directory holds output from calltool, as run on Tor. For more information about each of these files, see the NOTES and README files in the calltool distribution. diff --git a/scripts/maint/updateCopyright.pl b/scripts/maint/updateCopyright.pl index bd24377d38..36894b1baf 100755 --- a/scripts/maint/updateCopyright.pl +++ b/scripts/maint/updateCopyright.pl @@ -1,7 +1,7 @@ #!/usr/bin/perl -i -w -p -$NEWYEAR=2018; +$NEWYEAR=2019; -s/Copyright(.*) (201[^8]), The Tor Project/Copyright$1 $2-${NEWYEAR}, The Tor Project/; +s/Copyright(.*) (201[^9]), The Tor Project/Copyright$1 $2-${NEWYEAR}, The Tor Project/; s/Copyright(.*)-(20..), The Tor Project/Copyright$1-${NEWYEAR}, The Tor Project/; diff --git a/scripts/maint/updateFallbackDirs.py b/scripts/maint/updateFallbackDirs.py deleted file mode 100755 index 0ea3992d8f..0000000000 --- a/scripts/maint/updateFallbackDirs.py +++ /dev/null @@ -1,2216 +0,0 @@ -#!/usr/bin/env python - -# Usage: -# -# Regenerate the list: -# scripts/maint/updateFallbackDirs.py > src/app/config/fallback_dirs.inc 2> fallback_dirs.log -# -# Check the existing list: -# scripts/maint/updateFallbackDirs.py check_existing > fallback_dirs.inc.ok 2> fallback_dirs.log -# mv fallback_dirs.inc.ok src/app/config/fallback_dirs.inc -# -# This script should be run from a stable, reliable network connection, -# with no other network activity (and not over tor). -# If this is not possible, please disable: -# PERFORM_IPV4_DIRPORT_CHECKS and PERFORM_IPV6_DIRPORT_CHECKS -# -# Needs dateutil, stem, and potentially other python packages. -# Optionally uses ipaddress (python 3 builtin) or py2-ipaddress (package) -# for netblock analysis. -# -# Then read the logs to make sure the fallbacks aren't dominated by a single -# netblock or port. - -# Script by weasel, April 2015 -# Portions by gsathya & karsten, 2013 -# https://trac.torproject.org/projects/tor/attachment/ticket/8374/dir_list.2.py -# Modifications by teor, 2015 - -import StringIO -import string -import re -import datetime -import gzip -import os.path -import json -import math -import sys -import urllib -import urllib2 -import hashlib -import dateutil.parser -# bson_lazy provides bson -#from bson import json_util -import copy -import re - -from stem.descriptor import DocumentHandler -from stem.descriptor.remote import get_consensus, get_server_descriptors, MAX_FINGERPRINTS - -import logging -logging.root.name = '' - -HAVE_IPADDRESS = False -try: - # python 3 builtin, or install package py2-ipaddress - # there are several ipaddress implementations for python 2 - # with slightly different semantics with str typed text - # fortunately, all our IP addresses are in unicode - import ipaddress - HAVE_IPADDRESS = True -except ImportError: - # if this happens, we avoid doing netblock analysis - logging.warning('Unable to import ipaddress, please install py2-ipaddress.' + - ' A fallback list will be created, but optional netblock' + - ' analysis will not be performed.') - -## Top-Level Configuration - -# We use semantic versioning: https://semver.org -# In particular: -# * major changes include removing a mandatory field, or anything else that -# would break an appropriately tolerant parser, -# * minor changes include adding a field, -# * patch changes include changing header comments or other unstructured -# content -FALLBACK_FORMAT_VERSION = '2.0.0' -SECTION_SEPARATOR_BASE = '=====' -SECTION_SEPARATOR_COMMENT = '/* ' + SECTION_SEPARATOR_BASE + ' */' - -# Output all candidate fallbacks, or only output selected fallbacks? -OUTPUT_CANDIDATES = False - -# Perform DirPort checks over IPv4? -# Change this to False if IPv4 doesn't work for you, or if you don't want to -# download a consensus for each fallback -# Don't check ~1000 candidates when OUTPUT_CANDIDATES is True -PERFORM_IPV4_DIRPORT_CHECKS = False if OUTPUT_CANDIDATES else True - -# Perform DirPort checks over IPv6? -# If you know IPv6 works for you, set this to True -# This will exclude IPv6 relays without an IPv6 DirPort configured -# So it's best left at False until #18394 is implemented -# Don't check ~1000 candidates when OUTPUT_CANDIDATES is True -PERFORM_IPV6_DIRPORT_CHECKS = False if OUTPUT_CANDIDATES else False - -# Must relays be running now? -MUST_BE_RUNNING_NOW = (PERFORM_IPV4_DIRPORT_CHECKS - or PERFORM_IPV6_DIRPORT_CHECKS) - -# Clients have been using microdesc consensuses by default for a while now -DOWNLOAD_MICRODESC_CONSENSUS = True - -# If a relay delivers an expired consensus, if it expired less than this many -# seconds ago, we still allow the relay. This should never be less than -90, -# as all directory mirrors should have downloaded a consensus 90 minutes -# before it expires. It should never be more than 24 hours, because clients -# reject consensuses that are older than REASONABLY_LIVE_TIME. -# For the consensus expiry check to be accurate, the machine running this -# script needs an accurate clock. -# -# Relays on 0.3.0 and later return a 404 when they are about to serve an -# expired consensus. This makes them fail the download check. -# We use a tolerance of 0, so that 0.2.x series relays also fail the download -# check if they serve an expired consensus. -CONSENSUS_EXPIRY_TOLERANCE = 0 - -# Output fallback name, flags, bandwidth, and ContactInfo in a C comment? -OUTPUT_COMMENTS = True if OUTPUT_CANDIDATES else False - -# Output matching ContactInfo in fallbacks list? -# Useful if you're trying to contact operators -CONTACT_COUNT = True if OUTPUT_CANDIDATES else False - -# How the list should be sorted: -# fingerprint: is useful for stable diffs of fallback lists -# measured_bandwidth: is useful when pruning the list based on bandwidth -# contact: is useful for contacting operators once the list has been pruned -OUTPUT_SORT_FIELD = 'contact' if OUTPUT_CANDIDATES else 'fingerprint' - -## OnionOO Settings - -ONIONOO = 'https://onionoo.torproject.org/' -#ONIONOO = 'https://onionoo.thecthulhu.com/' - -# Don't bother going out to the Internet, just use the files available locally, -# even if they're very old -LOCAL_FILES_ONLY = False - -## Whitelist / Blacklist Filter Settings - -# The whitelist contains entries that are included if all attributes match -# (IPv4, dirport, orport, id, and optionally IPv6 and IPv6 orport) - -# What happens to entries not in whitelist? -# When True, they are included, when False, they are excluded -INCLUDE_UNLISTED_ENTRIES = True if OUTPUT_CANDIDATES else False - -WHITELIST_FILE_NAME = 'scripts/maint/fallback.whitelist' -FALLBACK_FILE_NAME = 'src/app/config/fallback_dirs.inc' - -# The number of bytes we'll read from a filter file before giving up -MAX_LIST_FILE_SIZE = 1024 * 1024 - -## Eligibility Settings - -# Require fallbacks to have the same address and port for a set amount of time -# We used to have this at 1 week, but that caused many fallback failures, which -# meant that we had to rebuild the list more often. We want fallbacks to be -# stable for 2 years, so we set it to a few months. -# -# If a relay changes address or port, that's it, it's not useful any more, -# because clients can't find it -ADDRESS_AND_PORT_STABLE_DAYS = 90 -# We ignore relays that have been down for more than this period -MAX_DOWNTIME_DAYS = 0 if MUST_BE_RUNNING_NOW else 7 -# FallbackDirs must have a time-weighted-fraction that is greater than or -# equal to: -# Mirrors that are down half the time are still useful half the time -CUTOFF_RUNNING = .50 -CUTOFF_V2DIR = .50 -# Guard flags are removed for some time after a relay restarts, so we ignore -# the guard flag. -CUTOFF_GUARD = .00 -# FallbackDirs must have a time-weighted-fraction that is less than or equal -# to: -# .00 means no bad exits -PERMITTED_BADEXIT = .00 - -# older entries' weights are adjusted with ALPHA^(age in days) -AGE_ALPHA = 0.99 - -# this factor is used to scale OnionOO entries to [0,1] -ONIONOO_SCALE_ONE = 999. - -## Fallback Count Limits - -# The target for these parameters is 20% of the guards in the network -# This is around 200 as of October 2015 -_FB_POG = 0.2 -FALLBACK_PROPORTION_OF_GUARDS = None if OUTPUT_CANDIDATES else _FB_POG - -# Limit the number of fallbacks (eliminating lowest by advertised bandwidth) -MAX_FALLBACK_COUNT = None if OUTPUT_CANDIDATES else 200 -# Emit a C #error if the number of fallbacks is less than expected -MIN_FALLBACK_COUNT = 0 if OUTPUT_CANDIDATES else MAX_FALLBACK_COUNT*0.5 - -# The maximum number of fallbacks on the same address, contact, or family -# -# With 150 fallbacks, this means each operator sees 5% of client bootstraps. -# For comparison: -# - We try to limit guard and exit operators to 5% of the network -# - The directory authorities used to see 11% of client bootstraps each -# -# We also don't want too much of the list to go down if a single operator -# has to move all their relays. -MAX_FALLBACKS_PER_IP = 1 -MAX_FALLBACKS_PER_IPV4 = MAX_FALLBACKS_PER_IP -MAX_FALLBACKS_PER_IPV6 = MAX_FALLBACKS_PER_IP -MAX_FALLBACKS_PER_CONTACT = 7 -MAX_FALLBACKS_PER_FAMILY = 7 - -## Fallback Bandwidth Requirements - -# Any fallback with the Exit flag has its bandwidth multiplied by this fraction -# to make sure we aren't further overloading exits -# (Set to 1.0, because we asked that only lightly loaded exits opt-in, -# and the extra load really isn't that much for large relays.) -EXIT_BANDWIDTH_FRACTION = 1.0 - -# If a single fallback's bandwidth is too low, it's pointless adding it -# We expect fallbacks to handle an extra 10 kilobytes per second of traffic -# Make sure they can support fifty times the expected extra load -# -# We convert this to a consensus weight before applying the filter, -# because all the bandwidth amounts are specified by the relay -MIN_BANDWIDTH = 50.0 * 10.0 * 1024.0 - -# Clients will time out after 30 seconds trying to download a consensus -# So allow fallback directories half that to deliver a consensus -# The exact download times might change based on the network connection -# running this script, but only by a few seconds -# There is also about a second of python overhead -CONSENSUS_DOWNLOAD_SPEED_MAX = 15.0 -# If the relay fails a consensus check, retry the download -# This avoids delisting a relay due to transient network conditions -CONSENSUS_DOWNLOAD_RETRY = True - -## Parsing Functions - -def parse_ts(t): - return datetime.datetime.strptime(t, "%Y-%m-%d %H:%M:%S") - -def remove_bad_chars(raw_string, bad_char_list): - # Remove each character in the bad_char_list - cleansed_string = raw_string - for c in bad_char_list: - cleansed_string = cleansed_string.replace(c, '') - return cleansed_string - -def cleanse_unprintable(raw_string): - # Remove all unprintable characters - cleansed_string = '' - for c in raw_string: - if c in string.printable: - cleansed_string += c - return cleansed_string - -def cleanse_whitespace(raw_string): - # Replace all whitespace characters with a space - cleansed_string = raw_string - for c in string.whitespace: - cleansed_string = cleansed_string.replace(c, ' ') - return cleansed_string - -def cleanse_c_multiline_comment(raw_string): - cleansed_string = raw_string - # Embedded newlines should be removed by tor/onionoo, but let's be paranoid - cleansed_string = cleanse_whitespace(cleansed_string) - # ContactInfo and Version can be arbitrary binary data - cleansed_string = cleanse_unprintable(cleansed_string) - # Prevent a malicious / unanticipated string from breaking out - # of a C-style multiline comment - # This removes '/*' and '*/' and '//' - bad_char_list = '*/' - # Prevent a malicious string from using C nulls - bad_char_list += '\0' - # Avoid confusing parsers by making sure there is only one comma per fallback - bad_char_list += ',' - # Avoid confusing parsers by making sure there is only one equals per field - bad_char_list += '=' - # Be safer by removing bad characters entirely - cleansed_string = remove_bad_chars(cleansed_string, bad_char_list) - # Some compilers may further process the content of comments - # There isn't much we can do to cover every possible case - # But comment-based directives are typically only advisory - return cleansed_string - -def cleanse_c_string(raw_string): - cleansed_string = raw_string - # Embedded newlines should be removed by tor/onionoo, but let's be paranoid - cleansed_string = cleanse_whitespace(cleansed_string) - # ContactInfo and Version can be arbitrary binary data - cleansed_string = cleanse_unprintable(cleansed_string) - # Prevent a malicious address/fingerprint string from breaking out - # of a C-style string - bad_char_list = '"' - # Prevent a malicious string from using escapes - bad_char_list += '\\' - # Prevent a malicious string from using C nulls - bad_char_list += '\0' - # Avoid confusing parsers by making sure there is only one comma per fallback - bad_char_list += ',' - # Avoid confusing parsers by making sure there is only one equals per field - bad_char_list += '=' - # Be safer by removing bad characters entirely - cleansed_string = remove_bad_chars(cleansed_string, bad_char_list) - # Some compilers may further process the content of strings - # There isn't much we can do to cover every possible case - # But this typically only results in changes to the string data - return cleansed_string - -## OnionOO Source Functions - -# a dictionary of source metadata for each onionoo query we've made -fetch_source = {} - -# register source metadata for 'what' -# assumes we only retrieve one document for each 'what' -def register_fetch_source(what, url, relays_published, version): - fetch_source[what] = {} - fetch_source[what]['url'] = url - fetch_source[what]['relays_published'] = relays_published - fetch_source[what]['version'] = version - -# list each registered source's 'what' -def fetch_source_list(): - return sorted(fetch_source.keys()) - -# given 'what', provide a multiline C comment describing the source -def describe_fetch_source(what): - desc = '/*' - desc += '\n' - desc += 'Onionoo Source: ' - desc += cleanse_c_multiline_comment(what) - desc += ' Date: ' - desc += cleanse_c_multiline_comment(fetch_source[what]['relays_published']) - desc += ' Version: ' - desc += cleanse_c_multiline_comment(fetch_source[what]['version']) - desc += '\n' - desc += 'URL: ' - desc += cleanse_c_multiline_comment(fetch_source[what]['url']) - desc += '\n' - desc += '*/' - return desc - -## File Processing Functions - -def write_to_file(str, file_name, max_len): - try: - with open(file_name, 'w') as f: - f.write(str[0:max_len]) - except EnvironmentError, error: - logging.error('Writing file %s failed: %d: %s'% - (file_name, - error.errno, - error.strerror) - ) - -def read_from_file(file_name, max_len): - try: - if os.path.isfile(file_name): - with open(file_name, 'r') as f: - return f.read(max_len) - except EnvironmentError, error: - logging.info('Loading file %s failed: %d: %s'% - (file_name, - error.errno, - error.strerror) - ) - return None - -def parse_fallback_file(file_name): - file_data = read_from_file(file_name, MAX_LIST_FILE_SIZE) - file_data = cleanse_unprintable(file_data) - file_data = remove_bad_chars(file_data, '\n"\0') - file_data = re.sub('/\*.*?\*/', '', file_data) - file_data = file_data.replace(',', '\n') - file_data = file_data.replace(' weight=10', '') - return file_data - -def load_possibly_compressed_response_json(response): - if response.info().get('Content-Encoding') == 'gzip': - buf = StringIO.StringIO( response.read() ) - f = gzip.GzipFile(fileobj=buf) - return json.load(f) - else: - return json.load(response) - -def load_json_from_file(json_file_name): - # An exception here may be resolved by deleting the .last_modified - # and .json files, and re-running the script - try: - with open(json_file_name, 'r') as f: - return json.load(f) - except EnvironmentError, error: - raise Exception('Reading not-modified json file %s failed: %d: %s'% - (json_file_name, - error.errno, - error.strerror) - ) - -## OnionOO Functions - -def datestr_to_datetime(datestr): - # Parse datetimes like: Fri, 02 Oct 2015 13:34:14 GMT - if datestr is not None: - dt = dateutil.parser.parse(datestr) - else: - # Never modified - use start of epoch - dt = datetime.datetime.utcfromtimestamp(0) - # strip any timezone out (in case they're supported in future) - dt = dt.replace(tzinfo=None) - return dt - -def onionoo_fetch(what, **kwargs): - params = kwargs - params['type'] = 'relay' - #params['limit'] = 10 - params['first_seen_days'] = '%d-'%(ADDRESS_AND_PORT_STABLE_DAYS) - params['last_seen_days'] = '-%d'%(MAX_DOWNTIME_DAYS) - params['flag'] = 'V2Dir' - url = ONIONOO + what + '?' + urllib.urlencode(params) - - # Unfortunately, the URL is too long for some OS filenames, - # but we still don't want to get files from different URLs mixed up - base_file_name = what + '-' + hashlib.sha1(url).hexdigest() - - full_url_file_name = base_file_name + '.full_url' - MAX_FULL_URL_LENGTH = 1024 - - last_modified_file_name = base_file_name + '.last_modified' - MAX_LAST_MODIFIED_LENGTH = 64 - - json_file_name = base_file_name + '.json' - - if LOCAL_FILES_ONLY: - # Read from the local file, don't write to anything - response_json = load_json_from_file(json_file_name) - else: - # store the full URL to a file for debugging - # no need to compare as long as you trust SHA-1 - write_to_file(url, full_url_file_name, MAX_FULL_URL_LENGTH) - - request = urllib2.Request(url) - request.add_header('Accept-encoding', 'gzip') - - # load the last modified date from the file, if it exists - last_mod_date = read_from_file(last_modified_file_name, - MAX_LAST_MODIFIED_LENGTH) - if last_mod_date is not None: - request.add_header('If-modified-since', last_mod_date) - - # Parse last modified date - last_mod = datestr_to_datetime(last_mod_date) - - # Not Modified and still recent enough to be useful - # Onionoo / Globe used to use 6 hours, but we can afford a day - required_freshness = datetime.datetime.utcnow() - # strip any timezone out (to match dateutil.parser) - required_freshness = required_freshness.replace(tzinfo=None) - required_freshness -= datetime.timedelta(hours=24) - - # Make the OnionOO request - response_code = 0 - try: - response = urllib2.urlopen(request) - response_code = response.getcode() - except urllib2.HTTPError, error: - response_code = error.code - if response_code == 304: # not modified - pass - else: - raise Exception("Could not get " + url + ": " - + str(error.code) + ": " + error.reason) - - if response_code == 200: # OK - last_mod = datestr_to_datetime(response.info().get('Last-Modified')) - - # Check for freshness - if last_mod < required_freshness: - if last_mod_date is not None: - # This check sometimes fails transiently, retry the script if it does - date_message = "Outdated data: last updated " + last_mod_date - else: - date_message = "No data: never downloaded " - raise Exception(date_message + " from " + url) - - # Process the data - if response_code == 200: # OK - - response_json = load_possibly_compressed_response_json(response) - - with open(json_file_name, 'w') as f: - # use the most compact json representation to save space - json.dump(response_json, f, separators=(',',':')) - - # store the last modified date in its own file - if response.info().get('Last-modified') is not None: - write_to_file(response.info().get('Last-Modified'), - last_modified_file_name, - MAX_LAST_MODIFIED_LENGTH) - - elif response_code == 304: # Not Modified - - response_json = load_json_from_file(json_file_name) - - else: # Unexpected HTTP response code not covered in the HTTPError above - raise Exception("Unexpected HTTP response code to " + url + ": " - + str(response_code)) - - register_fetch_source(what, - url, - response_json['relays_published'], - response_json['version']) - - return response_json - -def fetch(what, **kwargs): - #x = onionoo_fetch(what, **kwargs) - # don't use sort_keys, as the order of or_addresses is significant - #print json.dumps(x, indent=4, separators=(',', ': ')) - #sys.exit(0) - - return onionoo_fetch(what, **kwargs) - -## Fallback Candidate Class - -class Candidate(object): - CUTOFF_ADDRESS_AND_PORT_STABLE = (datetime.datetime.utcnow() - - datetime.timedelta(ADDRESS_AND_PORT_STABLE_DAYS)) - - def __init__(self, details): - for f in ['fingerprint', 'nickname', 'last_changed_address_or_port', - 'consensus_weight', 'or_addresses', 'dir_address']: - if not f in details: raise Exception("Document has no %s field."%(f,)) - - if not 'contact' in details: - details['contact'] = None - if not 'flags' in details or details['flags'] is None: - details['flags'] = [] - if (not 'advertised_bandwidth' in details - or details['advertised_bandwidth'] is None): - # relays without advertised bandwidth have it calculated from their - # consensus weight - details['advertised_bandwidth'] = 0 - if (not 'effective_family' in details - or details['effective_family'] is None): - details['effective_family'] = [] - if not 'platform' in details: - details['platform'] = None - details['last_changed_address_or_port'] = parse_ts( - details['last_changed_address_or_port']) - self._data = details - self._stable_sort_or_addresses() - - self._fpr = self._data['fingerprint'] - self._running = self._guard = self._v2dir = 0. - self._split_dirport() - self._compute_orport() - if self.orport is None: - raise Exception("Failed to get an orport for %s."%(self._fpr,)) - self._compute_ipv6addr() - if not self.has_ipv6(): - logging.debug("Failed to get an ipv6 address for %s."%(self._fpr,)) - self._compute_version() - self._extra_info_cache = None - - def _stable_sort_or_addresses(self): - # replace self._data['or_addresses'] with a stable ordering, - # sorting the secondary addresses in string order - # leave the received order in self._data['or_addresses_raw'] - self._data['or_addresses_raw'] = self._data['or_addresses'] - or_address_primary = self._data['or_addresses'][:1] - # subsequent entries in the or_addresses array are in an arbitrary order - # so we stabilise the addresses by sorting them in string order - or_addresses_secondaries_stable = sorted(self._data['or_addresses'][1:]) - or_addresses_stable = or_address_primary + or_addresses_secondaries_stable - self._data['or_addresses'] = or_addresses_stable - - def get_fingerprint(self): - return self._fpr - - # is_valid_ipv[46]_address by gsathya, karsten, 2013 - @staticmethod - def is_valid_ipv4_address(address): - if not isinstance(address, (str, unicode)): - return False - - # check if there are four period separated values - if address.count(".") != 3: - return False - - # checks that each value in the octet are decimal values between 0-255 - for entry in address.split("."): - if not entry.isdigit() or int(entry) < 0 or int(entry) > 255: - return False - elif entry[0] == "0" and len(entry) > 1: - return False # leading zeros, for instance in "1.2.3.001" - - return True - - @staticmethod - def is_valid_ipv6_address(address): - if not isinstance(address, (str, unicode)): - return False - - # remove brackets - address = address[1:-1] - - # addresses are made up of eight colon separated groups of four hex digits - # with leading zeros being optional - # https://en.wikipedia.org/wiki/IPv6#Address_format - - colon_count = address.count(":") - - if colon_count > 7: - return False # too many groups - elif colon_count != 7 and not "::" in address: - return False # not enough groups and none are collapsed - elif address.count("::") > 1 or ":::" in address: - return False # multiple groupings of zeros can't be collapsed - - found_ipv4_on_previous_entry = False - for entry in address.split(":"): - # If an IPv6 address has an embedded IPv4 address, - # it must be the last entry - if found_ipv4_on_previous_entry: - return False - if not re.match("^[0-9a-fA-f]{0,4}$", entry): - if not Candidate.is_valid_ipv4_address(entry): - return False - else: - found_ipv4_on_previous_entry = True - - return True - - def _split_dirport(self): - # Split the dir_address into dirip and dirport - (self.dirip, _dirport) = self._data['dir_address'].split(':', 2) - self.dirport = int(_dirport) - - def _compute_orport(self): - # Choose the first ORPort that's on the same IPv4 address as the DirPort. - # In rare circumstances, this might not be the primary ORPort address. - # However, _stable_sort_or_addresses() ensures we choose the same one - # every time, even if onionoo changes the order of the secondaries. - self._split_dirport() - self.orport = None - for i in self._data['or_addresses']: - if i != self._data['or_addresses'][0]: - logging.debug('Secondary IPv4 Address Used for %s: %s'%(self._fpr, i)) - (ipaddr, port) = i.rsplit(':', 1) - if (ipaddr == self.dirip) and Candidate.is_valid_ipv4_address(ipaddr): - self.orport = int(port) - return - - def _compute_ipv6addr(self): - # Choose the first IPv6 address that uses the same port as the ORPort - # Or, choose the first IPv6 address in the list - # _stable_sort_or_addresses() ensures we choose the same IPv6 address - # every time, even if onionoo changes the order of the secondaries. - self.ipv6addr = None - self.ipv6orport = None - # Choose the first IPv6 address that uses the same port as the ORPort - for i in self._data['or_addresses']: - (ipaddr, port) = i.rsplit(':', 1) - if (port == self.orport) and Candidate.is_valid_ipv6_address(ipaddr): - self.ipv6addr = ipaddr - self.ipv6orport = int(port) - return - # Choose the first IPv6 address in the list - for i in self._data['or_addresses']: - (ipaddr, port) = i.rsplit(':', 1) - if Candidate.is_valid_ipv6_address(ipaddr): - self.ipv6addr = ipaddr - self.ipv6orport = int(port) - return - - def _compute_version(self): - # parse the version out of the platform string - # The platform looks like: "Tor 0.2.7.6 on Linux" - self._data['version'] = None - if self._data['platform'] is None: - return - # be tolerant of weird whitespacing, use a whitespace split - tokens = self._data['platform'].split() - for token in tokens: - vnums = token.split('.') - # if it's at least a.b.c.d, with potentially an -alpha-dev, -alpha, -rc - if (len(vnums) >= 4 and vnums[0].isdigit() and vnums[1].isdigit() and - vnums[2].isdigit()): - self._data['version'] = token - return - - # From #20509 - # bug #20499 affects versions from 0.2.9.1-alpha-dev to 0.2.9.4-alpha-dev - # and version 0.3.0.0-alpha-dev - # Exhaustive lists are hard to get wrong - STALE_CONSENSUS_VERSIONS = ['0.2.9.1-alpha-dev', - '0.2.9.2-alpha', - '0.2.9.2-alpha-dev', - '0.2.9.3-alpha', - '0.2.9.3-alpha-dev', - '0.2.9.4-alpha', - '0.2.9.4-alpha-dev', - '0.3.0.0-alpha-dev' - ] - - def is_valid_version(self): - # call _compute_version before calling this - # is the version of the relay a version we want as a fallback? - # checks both recommended versions and bug #20499 / #20509 - # - # if the relay doesn't have a recommended version field, exclude the relay - if not self._data.has_key('recommended_version'): - log_excluded('%s not a candidate: no recommended_version field', - self._fpr) - return False - if not self._data['recommended_version']: - log_excluded('%s not a candidate: version not recommended', self._fpr) - return False - # if the relay doesn't have version field, exclude the relay - if not self._data.has_key('version'): - log_excluded('%s not a candidate: no version field', self._fpr) - return False - if self._data['version'] in Candidate.STALE_CONSENSUS_VERSIONS: - logging.warning('%s not a candidate: version delivers stale consensuses', - self._fpr) - return False - return True - - @staticmethod - def _extract_generic_history(history, which='unknown'): - # given a tree like this: - # { - # "1_month": { - # "count": 187, - # "factor": 0.001001001001001001, - # "first": "2015-02-27 06:00:00", - # "interval": 14400, - # "last": "2015-03-30 06:00:00", - # "values": [ - # 999, - # 999 - # ] - # }, - # "1_week": { - # "count": 169, - # "factor": 0.001001001001001001, - # "first": "2015-03-23 07:30:00", - # "interval": 3600, - # "last": "2015-03-30 07:30:00", - # "values": [ ...] - # }, - # "1_year": { - # "count": 177, - # "factor": 0.001001001001001001, - # "first": "2014-04-11 00:00:00", - # "interval": 172800, - # "last": "2015-03-29 00:00:00", - # "values": [ ...] - # }, - # "3_months": { - # "count": 185, - # "factor": 0.001001001001001001, - # "first": "2014-12-28 06:00:00", - # "interval": 43200, - # "last": "2015-03-30 06:00:00", - # "values": [ ...] - # } - # }, - # extract exactly one piece of data per time interval, - # using smaller intervals where available. - # - # returns list of (age, length, value) dictionaries. - - generic_history = [] - - periods = history.keys() - periods.sort(key = lambda x: history[x]['interval']) - now = datetime.datetime.utcnow() - newest = now - for p in periods: - h = history[p] - interval = datetime.timedelta(seconds = h['interval']) - this_ts = parse_ts(h['last']) - - if (len(h['values']) != h['count']): - logging.warning('Inconsistent value count in %s document for %s' - %(p, which)) - for v in reversed(h['values']): - if (this_ts <= newest): - agt1 = now - this_ts - agt2 = interval - agetmp1 = (agt1.microseconds + (agt1.seconds + agt1.days * 24 * 3600) - * 10**6) / 10**6 - agetmp2 = (agt2.microseconds + (agt2.seconds + agt2.days * 24 * 3600) - * 10**6) / 10**6 - generic_history.append( - { 'age': agetmp1, - 'length': agetmp2, - 'value': v - }) - newest = this_ts - this_ts -= interval - - if (this_ts + interval != parse_ts(h['first'])): - logging.warning('Inconsistent time information in %s document for %s' - %(p, which)) - - #print json.dumps(generic_history, sort_keys=True, - # indent=4, separators=(',', ': ')) - return generic_history - - @staticmethod - def _avg_generic_history(generic_history): - a = [] - for i in generic_history: - if i['age'] > (ADDRESS_AND_PORT_STABLE_DAYS * 24 * 3600): - continue - if (i['length'] is not None - and i['age'] is not None - and i['value'] is not None): - w = i['length'] * math.pow(AGE_ALPHA, i['age']/(3600*24)) - a.append( (i['value'] * w, w) ) - - sv = math.fsum(map(lambda x: x[0], a)) - sw = math.fsum(map(lambda x: x[1], a)) - - if sw == 0.0: - svw = 0.0 - else: - svw = sv/sw - return svw - - def _add_generic_history(self, history): - periods = r['read_history'].keys() - periods.sort(key = lambda x: r['read_history'][x]['interval'] ) - - print periods - - def add_running_history(self, history): - pass - - def add_uptime(self, uptime): - logging.debug('Adding uptime %s.'%(self._fpr,)) - - # flags we care about: Running, V2Dir, Guard - if not 'flags' in uptime: - logging.debug('No flags in document for %s.'%(self._fpr,)) - return - - for f in ['Running', 'Guard', 'V2Dir']: - if not f in uptime['flags']: - logging.debug('No %s in flags for %s.'%(f, self._fpr,)) - return - - running = self._extract_generic_history(uptime['flags']['Running'], - '%s-Running'%(self._fpr)) - guard = self._extract_generic_history(uptime['flags']['Guard'], - '%s-Guard'%(self._fpr)) - v2dir = self._extract_generic_history(uptime['flags']['V2Dir'], - '%s-V2Dir'%(self._fpr)) - if 'BadExit' in uptime['flags']: - badexit = self._extract_generic_history(uptime['flags']['BadExit'], - '%s-BadExit'%(self._fpr)) - - self._running = self._avg_generic_history(running) / ONIONOO_SCALE_ONE - self._guard = self._avg_generic_history(guard) / ONIONOO_SCALE_ONE - self._v2dir = self._avg_generic_history(v2dir) / ONIONOO_SCALE_ONE - self._badexit = None - if 'BadExit' in uptime['flags']: - self._badexit = self._avg_generic_history(badexit) / ONIONOO_SCALE_ONE - - def is_candidate(self): - try: - if (MUST_BE_RUNNING_NOW and not self.is_running()): - log_excluded('%s not a candidate: not running now, unable to check ' + - 'DirPort consensus download', self._fpr) - return False - if (self._data['last_changed_address_or_port'] > - self.CUTOFF_ADDRESS_AND_PORT_STABLE): - log_excluded('%s not a candidate: changed address/port recently (%s)', - self._fpr, self._data['last_changed_address_or_port']) - return False - if self._running < CUTOFF_RUNNING: - log_excluded('%s not a candidate: running avg too low (%lf)', - self._fpr, self._running) - return False - if self._v2dir < CUTOFF_V2DIR: - log_excluded('%s not a candidate: v2dir avg too low (%lf)', - self._fpr, self._v2dir) - return False - if self._badexit is not None and self._badexit > PERMITTED_BADEXIT: - log_excluded('%s not a candidate: badexit avg too high (%lf)', - self._fpr, self._badexit) - return False - # this function logs a message depending on which check fails - if not self.is_valid_version(): - return False - if self._guard < CUTOFF_GUARD: - log_excluded('%s not a candidate: guard avg too low (%lf)', - self._fpr, self._guard) - return False - if (not self._data.has_key('consensus_weight') - or self._data['consensus_weight'] < 1): - log_excluded('%s not a candidate: consensus weight invalid', self._fpr) - return False - except BaseException as e: - logging.warning("Exception %s when checking if fallback is a candidate", - str(e)) - return False - return True - - def is_in_whitelist(self, relaylist): - """ A fallback matches if each key in the whitelist line matches: - ipv4 - dirport - orport - id - ipv6 address and port (if present) - If the fallback has an ipv6 key, the whitelist line must also have - it, and vice versa, otherwise they don't match. """ - ipv6 = None - if self.has_ipv6(): - ipv6 = '%s:%d'%(self.ipv6addr, self.ipv6orport) - for entry in relaylist: - if entry['id'] != self._fpr: - # can't log here unless we match an IP and port, because every relay's - # fingerprint is compared to every entry's fingerprint - if entry['ipv4'] == self.dirip and int(entry['orport']) == self.orport: - logging.warning('%s excluded: has OR %s:%d changed fingerprint to ' + - '%s?', entry['id'], self.dirip, self.orport, - self._fpr) - if self.has_ipv6() and entry.has_key('ipv6') and entry['ipv6'] == ipv6: - logging.warning('%s excluded: has OR %s changed fingerprint to ' + - '%s?', entry['id'], ipv6, self._fpr) - continue - if entry['ipv4'] != self.dirip: - logging.warning('%s excluded: has it changed IPv4 from %s to %s?', - self._fpr, entry['ipv4'], self.dirip) - continue - if int(entry['dirport']) != self.dirport: - logging.warning('%s excluded: has it changed DirPort from %s:%d to ' + - '%s:%d?', self._fpr, self.dirip, int(entry['dirport']), - self.dirip, self.dirport) - continue - if int(entry['orport']) != self.orport: - logging.warning('%s excluded: has it changed ORPort from %s:%d to ' + - '%s:%d?', self._fpr, self.dirip, int(entry['orport']), - self.dirip, self.orport) - continue - if entry.has_key('ipv6') and self.has_ipv6(): - # if both entry and fallback have an ipv6 address, compare them - if entry['ipv6'] != ipv6: - logging.warning('%s excluded: has it changed IPv6 ORPort from %s ' + - 'to %s?', self._fpr, entry['ipv6'], ipv6) - continue - # if the fallback has an IPv6 address but the whitelist entry - # doesn't, or vice versa, the whitelist entry doesn't match - elif entry.has_key('ipv6') and not self.has_ipv6(): - logging.warning('%s excluded: has it lost its former IPv6 address %s?', - self._fpr, entry['ipv6']) - continue - elif not entry.has_key('ipv6') and self.has_ipv6(): - logging.warning('%s excluded: has it gained an IPv6 address %s?', - self._fpr, ipv6) - continue - return True - return False - - def cw_to_bw_factor(self): - # any relays with a missing or zero consensus weight are not candidates - # any relays with a missing advertised bandwidth have it set to zero - return self._data['advertised_bandwidth'] / self._data['consensus_weight'] - - # since advertised_bandwidth is reported by the relay, it can be gamed - # to avoid this, use the median consensus weight to bandwidth factor to - # estimate this relay's measured bandwidth, and make that the upper limit - def measured_bandwidth(self, median_cw_to_bw_factor): - cw_to_bw= median_cw_to_bw_factor - # Reduce exit bandwidth to make sure we're not overloading them - if self.is_exit(): - cw_to_bw *= EXIT_BANDWIDTH_FRACTION - measured_bandwidth = self._data['consensus_weight'] * cw_to_bw - if self._data['advertised_bandwidth'] != 0: - # limit advertised bandwidth (if available) to measured bandwidth - return min(measured_bandwidth, self._data['advertised_bandwidth']) - else: - return measured_bandwidth - - def set_measured_bandwidth(self, median_cw_to_bw_factor): - self._data['measured_bandwidth'] = self.measured_bandwidth( - median_cw_to_bw_factor) - - def is_exit(self): - return 'Exit' in self._data['flags'] - - def is_guard(self): - return 'Guard' in self._data['flags'] - - def is_running(self): - return 'Running' in self._data['flags'] - - # does this fallback have an IPv6 address and orport? - def has_ipv6(self): - return self.ipv6addr is not None and self.ipv6orport is not None - - # strip leading and trailing brackets from an IPv6 address - # safe to use on non-bracketed IPv6 and on IPv4 addresses - # also convert to unicode, and make None appear as '' - @staticmethod - def strip_ipv6_brackets(ip): - if ip is None: - return unicode('') - if len(ip) < 2: - return unicode(ip) - if ip[0] == '[' and ip[-1] == ']': - return unicode(ip[1:-1]) - return unicode(ip) - - # are ip_a and ip_b in the same netblock? - # mask_bits is the size of the netblock - # takes both IPv4 and IPv6 addresses - # the versions of ip_a and ip_b must be the same - # the mask must be valid for the IP version - @staticmethod - def netblocks_equal(ip_a, ip_b, mask_bits): - if ip_a is None or ip_b is None: - return False - ip_a = Candidate.strip_ipv6_brackets(ip_a) - ip_b = Candidate.strip_ipv6_brackets(ip_b) - a = ipaddress.ip_address(ip_a) - b = ipaddress.ip_address(ip_b) - if a.version != b.version: - raise Exception('Mismatching IP versions in %s and %s'%(ip_a, ip_b)) - if mask_bits > a.max_prefixlen: - logging.error('Bad IP mask %d for %s and %s'%(mask_bits, ip_a, ip_b)) - mask_bits = a.max_prefixlen - if mask_bits < 0: - logging.error('Bad IP mask %d for %s and %s'%(mask_bits, ip_a, ip_b)) - mask_bits = 0 - a_net = ipaddress.ip_network('%s/%d'%(ip_a, mask_bits), strict=False) - return b in a_net - - # is this fallback's IPv4 address (dirip) in the same netblock as other's - # IPv4 address? - # mask_bits is the size of the netblock - def ipv4_netblocks_equal(self, other, mask_bits): - return Candidate.netblocks_equal(self.dirip, other.dirip, mask_bits) - - # is this fallback's IPv6 address (ipv6addr) in the same netblock as - # other's IPv6 address? - # Returns False if either fallback has no IPv6 address - # mask_bits is the size of the netblock - def ipv6_netblocks_equal(self, other, mask_bits): - if not self.has_ipv6() or not other.has_ipv6(): - return False - return Candidate.netblocks_equal(self.ipv6addr, other.ipv6addr, mask_bits) - - # is this fallback's IPv4 DirPort the same as other's IPv4 DirPort? - def dirport_equal(self, other): - return self.dirport == other.dirport - - # is this fallback's IPv4 ORPort the same as other's IPv4 ORPort? - def ipv4_orport_equal(self, other): - return self.orport == other.orport - - # is this fallback's IPv6 ORPort the same as other's IPv6 ORPort? - # Returns False if either fallback has no IPv6 address - def ipv6_orport_equal(self, other): - if not self.has_ipv6() or not other.has_ipv6(): - return False - return self.ipv6orport == other.ipv6orport - - # does this fallback have the same DirPort, IPv4 ORPort, or - # IPv6 ORPort as other? - # Ignores IPv6 ORPort if either fallback has no IPv6 address - def port_equal(self, other): - return (self.dirport_equal(other) or self.ipv4_orport_equal(other) - or self.ipv6_orport_equal(other)) - - # return a list containing IPv4 ORPort, DirPort, and IPv6 ORPort (if present) - def port_list(self): - ports = [self.dirport, self.orport] - if self.has_ipv6() and not self.ipv6orport in ports: - ports.append(self.ipv6orport) - return ports - - # does this fallback share a port with other, regardless of whether the - # port types match? - # For example, if self's IPv4 ORPort is 80 and other's DirPort is 80, - # return True - def port_shared(self, other): - for p in self.port_list(): - if p in other.port_list(): - return True - return False - - # log how long it takes to download a consensus from dirip:dirport - # returns True if the download failed, False if it succeeded within max_time - @staticmethod - def fallback_consensus_download_speed(dirip, dirport, nickname, fingerprint, - max_time): - download_failed = False - # some directory mirrors respond to requests in ways that hang python - # sockets, which is why we log this line here - logging.info('Initiating %sconsensus download from %s (%s:%d) %s.', - 'microdesc ' if DOWNLOAD_MICRODESC_CONSENSUS else '', - nickname, dirip, dirport, fingerprint) - # there appears to be about 1 second of overhead when comparing stem's - # internal trace time and the elapsed time calculated here - TIMEOUT_SLOP = 1.0 - start = datetime.datetime.utcnow() - try: - consensus = get_consensus( - endpoints = [(dirip, dirport)], - timeout = (max_time + TIMEOUT_SLOP), - validate = True, - retries = 0, - fall_back_to_authority = False, - document_handler = DocumentHandler.BARE_DOCUMENT, - microdescriptor = DOWNLOAD_MICRODESC_CONSENSUS - ).run()[0] - end = datetime.datetime.utcnow() - time_since_expiry = (end - consensus.valid_until).total_seconds() - except Exception, stem_error: - end = datetime.datetime.utcnow() - log_excluded('Unable to retrieve a consensus from %s: %s', nickname, - stem_error) - status = 'error: "%s"' % (stem_error) - level = logging.WARNING - download_failed = True - elapsed = (end - start).total_seconds() - if download_failed: - # keep the error failure status, and avoid using the variables - pass - elif elapsed > max_time: - status = 'too slow' - level = logging.WARNING - download_failed = True - elif (time_since_expiry > 0): - status = 'outdated consensus, expired %ds ago'%(int(time_since_expiry)) - if time_since_expiry <= CONSENSUS_EXPIRY_TOLERANCE: - status += ', tolerating up to %ds'%(CONSENSUS_EXPIRY_TOLERANCE) - level = logging.INFO - else: - status += ', invalid' - level = logging.WARNING - download_failed = True - else: - status = 'ok' - level = logging.DEBUG - logging.log(level, 'Consensus download: %0.1fs %s from %s (%s:%d) %s, ' + - 'max download time %0.1fs.', elapsed, status, nickname, - dirip, dirport, fingerprint, max_time) - return download_failed - - # does this fallback download the consensus fast enough? - def check_fallback_download_consensus(self): - # include the relay if we're not doing a check, or we can't check (IPv6) - ipv4_failed = False - ipv6_failed = False - if PERFORM_IPV4_DIRPORT_CHECKS: - ipv4_failed = Candidate.fallback_consensus_download_speed(self.dirip, - self.dirport, - self._data['nickname'], - self._fpr, - CONSENSUS_DOWNLOAD_SPEED_MAX) - if self.has_ipv6() and PERFORM_IPV6_DIRPORT_CHECKS: - # Clients assume the IPv6 DirPort is the same as the IPv4 DirPort - ipv6_failed = Candidate.fallback_consensus_download_speed(self.ipv6addr, - self.dirport, - self._data['nickname'], - self._fpr, - CONSENSUS_DOWNLOAD_SPEED_MAX) - return ((not ipv4_failed) and (not ipv6_failed)) - - # if this fallback has not passed a download check, try it again, - # and record the result, available in get_fallback_download_consensus - def try_fallback_download_consensus(self): - if not self.get_fallback_download_consensus(): - self._data['download_check'] = self.check_fallback_download_consensus() - - # did this fallback pass the download check? - def get_fallback_download_consensus(self): - # if we're not performing checks, return True - if not PERFORM_IPV4_DIRPORT_CHECKS and not PERFORM_IPV6_DIRPORT_CHECKS: - return True - # if we are performing checks, but haven't done one, return False - if not self._data.has_key('download_check'): - return False - return self._data['download_check'] - - # output an optional header comment and info for this fallback - # try_fallback_download_consensus before calling this - def fallbackdir_line(self, fallbacks, prefilter_fallbacks): - s = '' - if OUTPUT_COMMENTS: - s += self.fallbackdir_comment(fallbacks, prefilter_fallbacks) - # if the download speed is ok, output a C string - # if it's not, but we OUTPUT_COMMENTS, output a commented-out C string - if self.get_fallback_download_consensus() or OUTPUT_COMMENTS: - s += self.fallbackdir_info(self.get_fallback_download_consensus()) - return s - - # output a header comment for this fallback - def fallbackdir_comment(self, fallbacks, prefilter_fallbacks): - # /* - # nickname - # flags - # adjusted bandwidth, consensus weight - # [contact] - # [identical contact counts] - # */ - # Multiline C comment - s = '/*' - s += '\n' - s += cleanse_c_multiline_comment(self._data['nickname']) - s += '\n' - s += 'Flags: ' - s += cleanse_c_multiline_comment(' '.join(sorted(self._data['flags']))) - s += '\n' - # this is an adjusted bandwidth, see calculate_measured_bandwidth() - bandwidth = self._data['measured_bandwidth'] - weight = self._data['consensus_weight'] - s += 'Bandwidth: %.1f MByte/s, Consensus Weight: %d'%( - bandwidth/(1024.0*1024.0), - weight) - s += '\n' - if self._data['contact'] is not None: - s += cleanse_c_multiline_comment(self._data['contact']) - if CONTACT_COUNT: - fallback_count = len([f for f in fallbacks - if f._data['contact'] == self._data['contact']]) - if fallback_count > 1: - s += '\n' - s += '%d identical contacts listed' % (fallback_count) - - # output the fallback info C string for this fallback - # this is the text that would go after FallbackDir in a torrc - # if this relay failed the download test and we OUTPUT_COMMENTS, - # comment-out the returned string - def fallbackdir_info(self, dl_speed_ok): - # "address:dirport orport=port id=fingerprint" - # (insert additional madatory fields here) - # "[ipv6=addr:orport]" - # (insert additional optional fields here) - # /* nickname=name */ - # /* extrainfo={0,1} */ - # (insert additional comment fields here) - # /* ===== */ - # , - # - # Do we want a C string, or a commented-out string? - c_string = dl_speed_ok - comment_string = not dl_speed_ok and OUTPUT_COMMENTS - # If we don't want either kind of string, bail - if not c_string and not comment_string: - return '' - s = '' - # Comment out the fallback directory entry if it's too slow - # See the debug output for which address and port is failing - if comment_string: - s += '/* Consensus download failed or was too slow:\n' - # Multi-Line C string with trailing comma (part of a string list) - # This makes it easier to diff the file, and remove IPv6 lines using grep - # Integers don't need escaping - s += '"%s orport=%d id=%s"'%( - cleanse_c_string(self._data['dir_address']), - self.orport, - cleanse_c_string(self._fpr)) - s += '\n' - # (insert additional madatory fields here) - if self.has_ipv6(): - s += '" ipv6=%s:%d"'%(cleanse_c_string(self.ipv6addr), self.ipv6orport) - s += '\n' - # (insert additional optional fields here) - if not comment_string: - s += '/* ' - s += 'nickname=%s'%(cleanse_c_string(self._data['nickname'])) - if not comment_string: - s += ' */' - s += '\n' - # if we know that the fallback is an extrainfo cache, flag it - # and if we don't know, assume it is not - if not comment_string: - s += '/* ' - s += 'extrainfo=%d'%(1 if self._extra_info_cache else 0) - if not comment_string: - s += ' */' - s += '\n' - # (insert additional comment fields here) - # The terminator and comma must be the last line in each fallback entry - if not comment_string: - s += '/* ' - s += SECTION_SEPARATOR_BASE - if not comment_string: - s += ' */' - s += '\n' - s += ',' - if comment_string: - s += '\n' - s += '*/' - return s - -## Fallback Candidate List Class - -class CandidateList(dict): - def __init__(self): - pass - - def _add_relay(self, details): - if not 'dir_address' in details: return - c = Candidate(details) - self[ c.get_fingerprint() ] = c - - def _add_uptime(self, uptime): - try: - fpr = uptime['fingerprint'] - except KeyError: - raise Exception("Document has no fingerprint field.") - - try: - c = self[fpr] - except KeyError: - logging.debug('Got unknown relay %s in uptime document.'%(fpr,)) - return - - c.add_uptime(uptime) - - def _add_details(self): - logging.debug('Loading details document.') - d = fetch('details', - fields=('fingerprint,nickname,contact,last_changed_address_or_port,' + - 'consensus_weight,advertised_bandwidth,or_addresses,' + - 'dir_address,recommended_version,flags,effective_family,' + - 'platform')) - logging.debug('Loading details document done.') - - if not 'relays' in d: raise Exception("No relays found in document.") - - for r in d['relays']: self._add_relay(r) - - def _add_uptimes(self): - logging.debug('Loading uptime document.') - d = fetch('uptime') - logging.debug('Loading uptime document done.') - - if not 'relays' in d: raise Exception("No relays found in document.") - for r in d['relays']: self._add_uptime(r) - - def add_relays(self): - self._add_details() - self._add_uptimes() - - def count_guards(self): - guard_count = 0 - for fpr in self.keys(): - if self[fpr].is_guard(): - guard_count += 1 - return guard_count - - # Find fallbacks that fit the uptime, stability, and flags criteria, - # and make an array of them in self.fallbacks - def compute_fallbacks(self): - self.fallbacks = map(lambda x: self[x], - filter(lambda x: self[x].is_candidate(), - self.keys())) - - # sort fallbacks by their consensus weight to advertised bandwidth factor, - # lowest to highest - # used to find the median cw_to_bw_factor() - def sort_fallbacks_by_cw_to_bw_factor(self): - self.fallbacks.sort(key=lambda f: f.cw_to_bw_factor()) - - # sort fallbacks by their measured bandwidth, highest to lowest - # calculate_measured_bandwidth before calling this - # this is useful for reviewing candidates in priority order - def sort_fallbacks_by_measured_bandwidth(self): - self.fallbacks.sort(key=lambda f: f._data['measured_bandwidth'], - reverse=True) - - # sort fallbacks by the data field data_field, lowest to highest - def sort_fallbacks_by(self, data_field): - self.fallbacks.sort(key=lambda f: f._data[data_field]) - - @staticmethod - def load_relaylist(file_obj): - """ Read each line in the file, and parse it like a FallbackDir line: - an IPv4 address and optional port: - <IPv4 address>:<port> - which are parsed into dictionary entries: - ipv4=<IPv4 address> - dirport=<port> - followed by a series of key=value entries: - orport=<port> - id=<fingerprint> - ipv6=<IPv6 address>:<IPv6 orport> - each line's key/value pairs are placed in a dictonary, - (of string -> string key/value pairs), - and these dictionaries are placed in an array. - comments start with # and are ignored """ - file_data = file_obj['data'] - file_name = file_obj['name'] - relaylist = [] - if file_data is None: - return relaylist - for line in file_data.split('\n'): - relay_entry = {} - # ignore comments - line_comment_split = line.split('#') - line = line_comment_split[0] - # cleanup whitespace - line = cleanse_whitespace(line) - line = line.strip() - if len(line) == 0: - continue - for item in line.split(' '): - item = item.strip() - if len(item) == 0: - continue - key_value_split = item.split('=') - kvl = len(key_value_split) - if kvl < 1 or kvl > 2: - print '#error Bad %s item: %s, format is key=value.'%( - file_name, item) - if kvl == 1: - # assume that entries without a key are the ipv4 address, - # perhaps with a dirport - ipv4_maybe_dirport = key_value_split[0] - ipv4_maybe_dirport_split = ipv4_maybe_dirport.split(':') - dirl = len(ipv4_maybe_dirport_split) - if dirl < 1 or dirl > 2: - print '#error Bad %s IPv4 item: %s, format is ipv4:port.'%( - file_name, item) - if dirl >= 1: - relay_entry['ipv4'] = ipv4_maybe_dirport_split[0] - if dirl == 2: - relay_entry['dirport'] = ipv4_maybe_dirport_split[1] - elif kvl == 2: - relay_entry[key_value_split[0]] = key_value_split[1] - relaylist.append(relay_entry) - return relaylist - - # apply the fallback whitelist - def apply_filter_lists(self, whitelist_obj): - excluded_count = 0 - logging.debug('Applying whitelist') - # parse the whitelist - whitelist = self.load_relaylist(whitelist_obj) - filtered_fallbacks = [] - for f in self.fallbacks: - in_whitelist = f.is_in_whitelist(whitelist) - if in_whitelist: - # include - filtered_fallbacks.append(f) - elif INCLUDE_UNLISTED_ENTRIES: - # include - filtered_fallbacks.append(f) - else: - # exclude - excluded_count += 1 - log_excluded('Excluding %s: not in whitelist.', - f._fpr) - self.fallbacks = filtered_fallbacks - return excluded_count - - @staticmethod - def summarise_filters(initial_count, excluded_count): - return '/* Whitelist excluded %d of %d candidates. */'%( - excluded_count, initial_count) - - # calculate each fallback's measured bandwidth based on the median - # consensus weight to advertised bandwidth ratio - def calculate_measured_bandwidth(self): - self.sort_fallbacks_by_cw_to_bw_factor() - median_fallback = self.fallback_median(True) - if median_fallback is not None: - median_cw_to_bw_factor = median_fallback.cw_to_bw_factor() - else: - # this will never be used, because there are no fallbacks - median_cw_to_bw_factor = None - for f in self.fallbacks: - f.set_measured_bandwidth(median_cw_to_bw_factor) - - # remove relays with low measured bandwidth from the fallback list - # calculate_measured_bandwidth for each relay before calling this - def remove_low_bandwidth_relays(self): - if MIN_BANDWIDTH is None: - return - above_min_bw_fallbacks = [] - for f in self.fallbacks: - if f._data['measured_bandwidth'] >= MIN_BANDWIDTH: - above_min_bw_fallbacks.append(f) - else: - # the bandwidth we log here is limited by the relay's consensus weight - # as well as its adverttised bandwidth. See set_measured_bandwidth - # for details - log_excluded('%s not a candidate: bandwidth %.1fMByte/s too low, ' + - 'must be at least %.1fMByte/s', f._fpr, - f._data['measured_bandwidth']/(1024.0*1024.0), - MIN_BANDWIDTH/(1024.0*1024.0)) - self.fallbacks = above_min_bw_fallbacks - - # the minimum fallback in the list - # call one of the sort_fallbacks_* functions before calling this - def fallback_min(self): - if len(self.fallbacks) > 0: - return self.fallbacks[-1] - else: - return None - - # the median fallback in the list - # call one of the sort_fallbacks_* functions before calling this - def fallback_median(self, require_advertised_bandwidth): - # use the low-median when there are an evan number of fallbacks, - # for consistency with the bandwidth authorities - if len(self.fallbacks) > 0: - median_position = (len(self.fallbacks) - 1) / 2 - if not require_advertised_bandwidth: - return self.fallbacks[median_position] - # if we need advertised_bandwidth but this relay doesn't have it, - # move to a fallback with greater consensus weight until we find one - while not self.fallbacks[median_position]._data['advertised_bandwidth']: - median_position += 1 - if median_position >= len(self.fallbacks): - return None - return self.fallbacks[median_position] - else: - return None - - # the maximum fallback in the list - # call one of the sort_fallbacks_* functions before calling this - def fallback_max(self): - if len(self.fallbacks) > 0: - return self.fallbacks[0] - else: - return None - - # return a new bag suitable for storing attributes - @staticmethod - def attribute_new(): - return dict() - - # get the count of attribute in attribute_bag - # if attribute is None or the empty string, return 0 - @staticmethod - def attribute_count(attribute, attribute_bag): - if attribute is None or attribute == '': - return 0 - if attribute not in attribute_bag: - return 0 - return attribute_bag[attribute] - - # does attribute_bag contain more than max_count instances of attribute? - # if so, return False - # if not, return True - # if attribute is None or the empty string, or max_count is invalid, - # always return True - @staticmethod - def attribute_allow(attribute, attribute_bag, max_count=1): - if attribute is None or attribute == '' or max_count <= 0: - return True - elif CandidateList.attribute_count(attribute, attribute_bag) >= max_count: - return False - else: - return True - - # add attribute to attribute_bag, incrementing the count if it is already - # present - # if attribute is None or the empty string, or count is invalid, - # do nothing - @staticmethod - def attribute_add(attribute, attribute_bag, count=1): - if attribute is None or attribute == '' or count <= 0: - pass - attribute_bag.setdefault(attribute, 0) - attribute_bag[attribute] += count - - # make sure there are only MAX_FALLBACKS_PER_IP fallbacks per IPv4 address, - # and per IPv6 address - # there is only one IPv4 address on each fallback: the IPv4 DirPort address - # (we choose the IPv4 ORPort which is on the same IPv4 as the DirPort) - # there is at most one IPv6 address on each fallback: the IPv6 ORPort address - # we try to match the IPv4 ORPort, but will use any IPv6 address if needed - # (clients only use the IPv6 ORPort) - # if there is no IPv6 address, only the IPv4 address is checked - # return the number of candidates we excluded - def limit_fallbacks_same_ip(self): - ip_limit_fallbacks = [] - ip_list = CandidateList.attribute_new() - for f in self.fallbacks: - if (CandidateList.attribute_allow(f.dirip, ip_list, - MAX_FALLBACKS_PER_IPV4) - and CandidateList.attribute_allow(f.ipv6addr, ip_list, - MAX_FALLBACKS_PER_IPV6)): - ip_limit_fallbacks.append(f) - CandidateList.attribute_add(f.dirip, ip_list) - if f.has_ipv6(): - CandidateList.attribute_add(f.ipv6addr, ip_list) - elif not CandidateList.attribute_allow(f.dirip, ip_list, - MAX_FALLBACKS_PER_IPV4): - log_excluded('Eliminated %s: already have %d fallback(s) on IPv4 %s' - %(f._fpr, CandidateList.attribute_count(f.dirip, ip_list), - f.dirip)) - elif (f.has_ipv6() and - not CandidateList.attribute_allow(f.ipv6addr, ip_list, - MAX_FALLBACKS_PER_IPV6)): - log_excluded('Eliminated %s: already have %d fallback(s) on IPv6 %s' - %(f._fpr, CandidateList.attribute_count(f.ipv6addr, - ip_list), - f.ipv6addr)) - original_count = len(self.fallbacks) - self.fallbacks = ip_limit_fallbacks - return original_count - len(self.fallbacks) - - # make sure there are only MAX_FALLBACKS_PER_CONTACT fallbacks for each - # ContactInfo - # if there is no ContactInfo, allow the fallback - # this check can be gamed by providing no ContactInfo, or by setting the - # ContactInfo to match another fallback - # However, given the likelihood that relays with the same ContactInfo will - # go down at similar times, its usefulness outweighs the risk - def limit_fallbacks_same_contact(self): - contact_limit_fallbacks = [] - contact_list = CandidateList.attribute_new() - for f in self.fallbacks: - if CandidateList.attribute_allow(f._data['contact'], contact_list, - MAX_FALLBACKS_PER_CONTACT): - contact_limit_fallbacks.append(f) - CandidateList.attribute_add(f._data['contact'], contact_list) - else: - log_excluded( - 'Eliminated %s: already have %d fallback(s) on ContactInfo %s' - %(f._fpr, CandidateList.attribute_count(f._data['contact'], - contact_list), - f._data['contact'])) - original_count = len(self.fallbacks) - self.fallbacks = contact_limit_fallbacks - return original_count - len(self.fallbacks) - - # make sure there are only MAX_FALLBACKS_PER_FAMILY fallbacks per effective - # family - # if there is no family, allow the fallback - # we use effective family, which ensures mutual family declarations - # but the check can be gamed by not declaring a family at all - # if any indirect families exist, the result depends on the order in which - # fallbacks are sorted in the list - def limit_fallbacks_same_family(self): - family_limit_fallbacks = [] - fingerprint_list = CandidateList.attribute_new() - for f in self.fallbacks: - if CandidateList.attribute_allow(f._fpr, fingerprint_list, - MAX_FALLBACKS_PER_FAMILY): - family_limit_fallbacks.append(f) - CandidateList.attribute_add(f._fpr, fingerprint_list) - for family_fingerprint in f._data['effective_family']: - CandidateList.attribute_add(family_fingerprint, fingerprint_list) - else: - # we already have a fallback with this fallback in its effective - # family - log_excluded( - 'Eliminated %s: already have %d fallback(s) in effective family' - %(f._fpr, CandidateList.attribute_count(f._fpr, fingerprint_list))) - original_count = len(self.fallbacks) - self.fallbacks = family_limit_fallbacks - return original_count - len(self.fallbacks) - - # try once to get the descriptors for fingerprint_list using stem - # returns an empty list on exception - @staticmethod - def get_fallback_descriptors_once(fingerprint_list): - desc_list = get_server_descriptors(fingerprints=fingerprint_list).run(suppress=True) - return desc_list - - # try up to max_retries times to get the descriptors for fingerprint_list - # using stem. Stops retrying when all descriptors have been retrieved. - # returns a list containing the descriptors that were retrieved - @staticmethod - def get_fallback_descriptors(fingerprint_list, max_retries=5): - # we can't use stem's retries=, because we want to support more than 96 - # descriptors - # - # add an attempt for every MAX_FINGERPRINTS (or part thereof) in the list - max_retries += (len(fingerprint_list) + MAX_FINGERPRINTS - 1) / MAX_FINGERPRINTS - remaining_list = fingerprint_list - desc_list = [] - for _ in xrange(max_retries): - if len(remaining_list) == 0: - break - new_desc_list = CandidateList.get_fallback_descriptors_once(remaining_list[0:MAX_FINGERPRINTS]) - for d in new_desc_list: - try: - remaining_list.remove(d.fingerprint) - except ValueError: - # warn and ignore if a directory mirror returned a bad descriptor - logging.warning("Directory mirror returned unwanted descriptor %s, ignoring", - d.fingerprint) - continue - desc_list.append(d) - return desc_list - - # find the fallbacks that cache extra-info documents - # Onionoo doesn't know this, so we have to use stem - def mark_extra_info_caches(self): - fingerprint_list = [ f._fpr for f in self.fallbacks ] - logging.info("Downloading fallback descriptors to find extra-info caches") - desc_list = CandidateList.get_fallback_descriptors(fingerprint_list) - for d in desc_list: - self[d.fingerprint]._extra_info_cache = d.extra_info_cache - missing_descriptor_list = [ f._fpr for f in self.fallbacks - if f._extra_info_cache is None ] - for f in missing_descriptor_list: - logging.warning("No descriptor for {}. Assuming extrainfo=0.".format(f)) - - # try a download check on each fallback candidate in order - # stop after max_count successful downloads - # but don't remove any candidates from the array - def try_download_consensus_checks(self, max_count): - dl_ok_count = 0 - for f in self.fallbacks: - f.try_fallback_download_consensus() - if f.get_fallback_download_consensus(): - # this fallback downloaded a consensus ok - dl_ok_count += 1 - if dl_ok_count >= max_count: - # we have enough fallbacks - return - - # put max_count successful candidates in the fallbacks array: - # - perform download checks on each fallback candidate - # - retry failed candidates if CONSENSUS_DOWNLOAD_RETRY is set - # - eliminate failed candidates - # - if there are more than max_count candidates, eliminate lowest bandwidth - # - if there are fewer than max_count candidates, leave only successful - # Return the number of fallbacks that failed the consensus check - def perform_download_consensus_checks(self, max_count): - self.sort_fallbacks_by_measured_bandwidth() - self.try_download_consensus_checks(max_count) - if CONSENSUS_DOWNLOAD_RETRY: - # try unsuccessful candidates again - # we could end up with more than max_count successful candidates here - self.try_download_consensus_checks(max_count) - # now we have at least max_count successful candidates, - # or we've tried them all - original_count = len(self.fallbacks) - self.fallbacks = filter(lambda x: x.get_fallback_download_consensus(), - self.fallbacks) - # some of these failed the check, others skipped the check, - # if we already had enough successful downloads - failed_count = original_count - len(self.fallbacks) - self.fallbacks = self.fallbacks[:max_count] - return failed_count - - # return a string that describes a/b as a percentage - @staticmethod - def describe_percentage(a, b): - if b != 0: - return '%d/%d = %.0f%%'%(a, b, (a*100.0)/b) - else: - # technically, 0/0 is undefined, but 0.0% is a sensible result - return '%d/%d = %.0f%%'%(a, b, 0.0) - - # return a dictionary of lists of fallbacks by IPv4 netblock - # the dictionary is keyed by the fingerprint of an arbitrary fallback - # in each netblock - # mask_bits is the size of the netblock - def fallbacks_by_ipv4_netblock(self, mask_bits): - netblocks = {} - for f in self.fallbacks: - found_netblock = False - for b in netblocks.keys(): - # we found an existing netblock containing this fallback - if f.ipv4_netblocks_equal(self[b], mask_bits): - # add it to the list - netblocks[b].append(f) - found_netblock = True - break - # make a new netblock based on this fallback's fingerprint - if not found_netblock: - netblocks[f._fpr] = [f] - return netblocks - - # return a dictionary of lists of fallbacks by IPv6 netblock - # where mask_bits is the size of the netblock - def fallbacks_by_ipv6_netblock(self, mask_bits): - netblocks = {} - for f in self.fallbacks: - # skip fallbacks without IPv6 addresses - if not f.has_ipv6(): - continue - found_netblock = False - for b in netblocks.keys(): - # we found an existing netblock containing this fallback - if f.ipv6_netblocks_equal(self[b], mask_bits): - # add it to the list - netblocks[b].append(f) - found_netblock = True - break - # make a new netblock based on this fallback's fingerprint - if not found_netblock: - netblocks[f._fpr] = [f] - return netblocks - - # log a message about the proportion of fallbacks in each IPv4 netblock, - # where mask_bits is the size of the netblock - def describe_fallback_ipv4_netblock_mask(self, mask_bits): - fallback_count = len(self.fallbacks) - shared_netblock_fallback_count = 0 - most_frequent_netblock = None - netblocks = self.fallbacks_by_ipv4_netblock(mask_bits) - for b in netblocks.keys(): - if len(netblocks[b]) > 1: - # how many fallbacks are in a netblock with other fallbacks? - shared_netblock_fallback_count += len(netblocks[b]) - # what's the netblock with the most fallbacks? - if (most_frequent_netblock is None - or len(netblocks[b]) > len(netblocks[most_frequent_netblock])): - most_frequent_netblock = b - logging.debug('Fallback IPv4 addresses in the same /%d:'%(mask_bits)) - for f in netblocks[b]: - logging.debug('%s - %s', f.dirip, f._fpr) - if most_frequent_netblock is not None: - logging.warning('There are %s fallbacks in the IPv4 /%d containing %s'%( - CandidateList.describe_percentage( - len(netblocks[most_frequent_netblock]), - fallback_count), - mask_bits, - self[most_frequent_netblock].dirip)) - if shared_netblock_fallback_count > 0: - logging.warning(('%s of fallbacks are in an IPv4 /%d with other ' + - 'fallbacks')%(CandidateList.describe_percentage( - shared_netblock_fallback_count, - fallback_count), - mask_bits)) - - # log a message about the proportion of fallbacks in each IPv6 netblock, - # where mask_bits is the size of the netblock - def describe_fallback_ipv6_netblock_mask(self, mask_bits): - fallback_count = len(self.fallbacks_with_ipv6()) - shared_netblock_fallback_count = 0 - most_frequent_netblock = None - netblocks = self.fallbacks_by_ipv6_netblock(mask_bits) - for b in netblocks.keys(): - if len(netblocks[b]) > 1: - # how many fallbacks are in a netblock with other fallbacks? - shared_netblock_fallback_count += len(netblocks[b]) - # what's the netblock with the most fallbacks? - if (most_frequent_netblock is None - or len(netblocks[b]) > len(netblocks[most_frequent_netblock])): - most_frequent_netblock = b - logging.debug('Fallback IPv6 addresses in the same /%d:'%(mask_bits)) - for f in netblocks[b]: - logging.debug('%s - %s', f.ipv6addr, f._fpr) - if most_frequent_netblock is not None: - logging.warning('There are %s fallbacks in the IPv6 /%d containing %s'%( - CandidateList.describe_percentage( - len(netblocks[most_frequent_netblock]), - fallback_count), - mask_bits, - self[most_frequent_netblock].ipv6addr)) - if shared_netblock_fallback_count > 0: - logging.warning(('%s of fallbacks are in an IPv6 /%d with other ' + - 'fallbacks')%(CandidateList.describe_percentage( - shared_netblock_fallback_count, - fallback_count), - mask_bits)) - - # log a message about the proportion of fallbacks in each IPv4 /8, /16, - # and /24 - def describe_fallback_ipv4_netblocks(self): - # this doesn't actually tell us anything useful - #self.describe_fallback_ipv4_netblock_mask(8) - self.describe_fallback_ipv4_netblock_mask(16) - #self.describe_fallback_ipv4_netblock_mask(24) - - # log a message about the proportion of fallbacks in each IPv6 /12 (RIR), - # /23 (smaller RIR blocks), /32 (LIR), /48 (Customer), and /64 (Host) - # https://www.iana.org/assignments/ipv6-unicast-address-assignments/ - def describe_fallback_ipv6_netblocks(self): - # these don't actually tell us anything useful - #self.describe_fallback_ipv6_netblock_mask(12) - #self.describe_fallback_ipv6_netblock_mask(23) - self.describe_fallback_ipv6_netblock_mask(32) - #self.describe_fallback_ipv6_netblock_mask(48) - self.describe_fallback_ipv6_netblock_mask(64) - - # log a message about the proportion of fallbacks in each IPv4 and IPv6 - # netblock - def describe_fallback_netblocks(self): - self.describe_fallback_ipv4_netblocks() - self.describe_fallback_ipv6_netblocks() - - # return a list of fallbacks which are on the IPv4 ORPort port - def fallbacks_on_ipv4_orport(self, port): - return filter(lambda x: x.orport == port, self.fallbacks) - - # return a list of fallbacks which are on the IPv6 ORPort port - def fallbacks_on_ipv6_orport(self, port): - return filter(lambda x: x.ipv6orport == port, self.fallbacks_with_ipv6()) - - # return a list of fallbacks which are on the DirPort port - def fallbacks_on_dirport(self, port): - return filter(lambda x: x.dirport == port, self.fallbacks) - - # log a message about the proportion of fallbacks on IPv4 ORPort port - # and return that count - def describe_fallback_ipv4_orport(self, port): - port_count = len(self.fallbacks_on_ipv4_orport(port)) - fallback_count = len(self.fallbacks) - logging.warning('%s of fallbacks are on IPv4 ORPort %d'%( - CandidateList.describe_percentage(port_count, - fallback_count), - port)) - return port_count - - # log a message about the proportion of IPv6 fallbacks on IPv6 ORPort port - # and return that count - def describe_fallback_ipv6_orport(self, port): - port_count = len(self.fallbacks_on_ipv6_orport(port)) - fallback_count = len(self.fallbacks_with_ipv6()) - logging.warning('%s of IPv6 fallbacks are on IPv6 ORPort %d'%( - CandidateList.describe_percentage(port_count, - fallback_count), - port)) - return port_count - - # log a message about the proportion of fallbacks on DirPort port - # and return that count - def describe_fallback_dirport(self, port): - port_count = len(self.fallbacks_on_dirport(port)) - fallback_count = len(self.fallbacks) - logging.warning('%s of fallbacks are on DirPort %d'%( - CandidateList.describe_percentage(port_count, - fallback_count), - port)) - return port_count - - # log a message about the proportion of fallbacks on each dirport, - # each IPv4 orport, and each IPv6 orport - def describe_fallback_ports(self): - fallback_count = len(self.fallbacks) - ipv4_or_count = fallback_count - ipv4_or_count -= self.describe_fallback_ipv4_orport(443) - ipv4_or_count -= self.describe_fallback_ipv4_orport(9001) - logging.warning('%s of fallbacks are on other IPv4 ORPorts'%( - CandidateList.describe_percentage(ipv4_or_count, - fallback_count))) - ipv6_fallback_count = len(self.fallbacks_with_ipv6()) - ipv6_or_count = ipv6_fallback_count - ipv6_or_count -= self.describe_fallback_ipv6_orport(443) - ipv6_or_count -= self.describe_fallback_ipv6_orport(9001) - logging.warning('%s of IPv6 fallbacks are on other IPv6 ORPorts'%( - CandidateList.describe_percentage(ipv6_or_count, - ipv6_fallback_count))) - dir_count = fallback_count - dir_count -= self.describe_fallback_dirport(80) - dir_count -= self.describe_fallback_dirport(9030) - logging.warning('%s of fallbacks are on other DirPorts'%( - CandidateList.describe_percentage(dir_count, - fallback_count))) - - # return a list of fallbacks which cache extra-info documents - def fallbacks_with_extra_info_cache(self): - return filter(lambda x: x._extra_info_cache, self.fallbacks) - - # log a message about the proportion of fallbacks that cache extra-info docs - def describe_fallback_extra_info_caches(self): - extra_info_falback_count = len(self.fallbacks_with_extra_info_cache()) - fallback_count = len(self.fallbacks) - logging.warning('%s of fallbacks cache extra-info documents'%( - CandidateList.describe_percentage(extra_info_falback_count, - fallback_count))) - - # return a list of fallbacks which have the Exit flag - def fallbacks_with_exit(self): - return filter(lambda x: x.is_exit(), self.fallbacks) - - # log a message about the proportion of fallbacks with an Exit flag - def describe_fallback_exit_flag(self): - exit_falback_count = len(self.fallbacks_with_exit()) - fallback_count = len(self.fallbacks) - logging.warning('%s of fallbacks have the Exit flag'%( - CandidateList.describe_percentage(exit_falback_count, - fallback_count))) - - # return a list of fallbacks which have an IPv6 address - def fallbacks_with_ipv6(self): - return filter(lambda x: x.has_ipv6(), self.fallbacks) - - # log a message about the proportion of fallbacks on IPv6 - def describe_fallback_ip_family(self): - ipv6_falback_count = len(self.fallbacks_with_ipv6()) - fallback_count = len(self.fallbacks) - logging.warning('%s of fallbacks are on IPv6'%( - CandidateList.describe_percentage(ipv6_falback_count, - fallback_count))) - - def summarise_fallbacks(self, eligible_count, operator_count, failed_count, - guard_count, target_count): - s = '' - # Report: - # whether we checked consensus download times - # the number of fallback directories (and limits/exclusions, if relevant) - # min & max fallback bandwidths - # #error if below minimum count - if PERFORM_IPV4_DIRPORT_CHECKS or PERFORM_IPV6_DIRPORT_CHECKS: - s += '/* Checked %s%s%s DirPorts served a consensus within %.1fs. */'%( - 'IPv4' if PERFORM_IPV4_DIRPORT_CHECKS else '', - ' and ' if (PERFORM_IPV4_DIRPORT_CHECKS - and PERFORM_IPV6_DIRPORT_CHECKS) else '', - 'IPv6' if PERFORM_IPV6_DIRPORT_CHECKS else '', - CONSENSUS_DOWNLOAD_SPEED_MAX) - else: - s += '/* Did not check IPv4 or IPv6 DirPort consensus downloads. */' - s += '\n' - # Multiline C comment with #error if things go bad - s += '/*' - s += '\n' - # Integers don't need escaping in C comments - fallback_count = len(self.fallbacks) - if FALLBACK_PROPORTION_OF_GUARDS is None: - fallback_proportion = '' - else: - fallback_proportion = ', Target %d (%d * %.2f)'%(target_count, - guard_count, - FALLBACK_PROPORTION_OF_GUARDS) - s += 'Final Count: %d (Eligible %d%s'%(fallback_count, eligible_count, - fallback_proportion) - if MAX_FALLBACK_COUNT is not None: - s += ', Max %d'%(MAX_FALLBACK_COUNT) - s += ')\n' - if eligible_count != fallback_count: - removed_count = eligible_count - fallback_count - excess_to_target_or_max = (eligible_count - operator_count - failed_count - - fallback_count) - # some 'Failed' failed the check, others 'Skipped' the check, - # if we already had enough successful downloads - s += ('Excluded: %d (Same Operator %d, Failed/Skipped Download %d, ' + - 'Excess %d)')%(removed_count, operator_count, failed_count, - excess_to_target_or_max) - s += '\n' - min_fb = self.fallback_min() - min_bw = min_fb._data['measured_bandwidth'] - max_fb = self.fallback_max() - max_bw = max_fb._data['measured_bandwidth'] - s += 'Bandwidth Range: %.1f - %.1f MByte/s'%(min_bw/(1024.0*1024.0), - max_bw/(1024.0*1024.0)) - s += '\n' - s += '*/' - if fallback_count < MIN_FALLBACK_COUNT: - # We must have a minimum number of fallbacks so they are always - # reachable, and are in diverse locations - s += '\n' - s += '#error Fallback Count %d is too low. '%(fallback_count) - s += 'Must be at least %d for diversity. '%(MIN_FALLBACK_COUNT) - s += 'Try adding entries to the whitelist, ' - s += 'or setting INCLUDE_UNLISTED_ENTRIES = True.' - return s - -def process_existing(): - logging.basicConfig(level=logging.INFO) - logging.getLogger('stem').setLevel(logging.INFO) - whitelist = {'data': parse_fallback_file(FALLBACK_FILE_NAME), - 'name': FALLBACK_FILE_NAME} - list_fallbacks(whitelist) - -def process_default(): - logging.basicConfig(level=logging.WARNING) - logging.getLogger('stem').setLevel(logging.WARNING) - whitelist = {'data': read_from_file(WHITELIST_FILE_NAME, MAX_LIST_FILE_SIZE), - 'name': WHITELIST_FILE_NAME} - list_fallbacks(whitelist) - -## Main Function -def main(): - if get_command() == 'check_existing': - process_existing() - else: - process_default() - -def get_command(): - if len(sys.argv) == 2: - return sys.argv[1] - else: - return None - -def log_excluded(msg, *args): - if get_command() == 'check_existing': - logging.warning(msg, *args) - else: - logging.info(msg, *args) - -def list_fallbacks(whitelist): - """ Fetches required onionoo documents and evaluates the - fallback directory criteria for each of the relays """ - - print "/* type=fallback */" - print ("/* version={} */" - .format(cleanse_c_multiline_comment(FALLBACK_FORMAT_VERSION))) - now = datetime.datetime.utcnow() - timestamp = now.strftime('%Y%m%d%H%M%S') - print ("/* timestamp={} */" - .format(cleanse_c_multiline_comment(timestamp))) - # end the header with a separator, to make it easier for parsers - print SECTION_SEPARATOR_COMMENT - - logging.warning('Downloading and parsing Onionoo data. ' + - 'This may take some time.') - # find relays that could be fallbacks - candidates = CandidateList() - candidates.add_relays() - - # work out how many fallbacks we want - guard_count = candidates.count_guards() - if FALLBACK_PROPORTION_OF_GUARDS is None: - target_count = guard_count - else: - target_count = int(guard_count * FALLBACK_PROPORTION_OF_GUARDS) - # the maximum number of fallbacks is the least of: - # - the target fallback count (FALLBACK_PROPORTION_OF_GUARDS * guard count) - # - the maximum fallback count (MAX_FALLBACK_COUNT) - if MAX_FALLBACK_COUNT is None: - max_count = target_count - else: - max_count = min(target_count, MAX_FALLBACK_COUNT) - - candidates.compute_fallbacks() - prefilter_fallbacks = copy.copy(candidates.fallbacks) - - # filter with the whitelist - # if a relay has changed IPv4 address or ports recently, it will be excluded - # as ineligible before we call apply_filter_lists, and so there will be no - # warning that the details have changed from those in the whitelist. - # instead, there will be an info-level log during the eligibility check. - initial_count = len(candidates.fallbacks) - excluded_count = candidates.apply_filter_lists(whitelist) - print candidates.summarise_filters(initial_count, excluded_count) - eligible_count = len(candidates.fallbacks) - - # calculate the measured bandwidth of each relay, - # then remove low-bandwidth relays - candidates.calculate_measured_bandwidth() - candidates.remove_low_bandwidth_relays() - - # print the raw fallback list - #for x in candidates.fallbacks: - # print x.fallbackdir_line(True) - # print json.dumps(candidates[x]._data, sort_keys=True, indent=4, - # separators=(',', ': '), default=json_util.default) - - # impose mandatory conditions here, like one per contact, family, IP - # in measured bandwidth order - candidates.sort_fallbacks_by_measured_bandwidth() - operator_count = 0 - # only impose these limits on the final list - operators can nominate - # multiple candidate fallbacks, and then we choose the best set - if not OUTPUT_CANDIDATES: - operator_count += candidates.limit_fallbacks_same_ip() - operator_count += candidates.limit_fallbacks_same_contact() - operator_count += candidates.limit_fallbacks_same_family() - - # check if each candidate can serve a consensus - # there's a small risk we've eliminated relays from the same operator that - # can serve a consensus, in favour of one that can't - # but given it takes up to 15 seconds to check each consensus download, - # the risk is worth it - if PERFORM_IPV4_DIRPORT_CHECKS or PERFORM_IPV6_DIRPORT_CHECKS: - logging.warning('Checking consensus download speeds. ' + - 'This may take some time.') - failed_count = candidates.perform_download_consensus_checks(max_count) - - # work out which fallbacks cache extra-infos - candidates.mark_extra_info_caches() - - # analyse and log interesting diversity metrics - # like netblock, ports, exit, IPv4-only - # (we can't easily analyse AS, and it's hard to accurately analyse country) - candidates.describe_fallback_ip_family() - # if we can't import the ipaddress module, we can't do netblock analysis - if HAVE_IPADDRESS: - candidates.describe_fallback_netblocks() - candidates.describe_fallback_ports() - candidates.describe_fallback_extra_info_caches() - candidates.describe_fallback_exit_flag() - - # output C comments summarising the fallback selection process - if len(candidates.fallbacks) > 0: - print candidates.summarise_fallbacks(eligible_count, operator_count, - failed_count, guard_count, - target_count) - else: - print '/* No Fallbacks met criteria */' - - # output C comments specifying the OnionOO data used to create the list - for s in fetch_source_list(): - print describe_fetch_source(s) - - # start the list with a separator, to make it easy for parsers - print SECTION_SEPARATOR_COMMENT - - # sort the list differently depending on why we've created it: - # if we're outputting the final fallback list, sort by fingerprint - # this makes diffs much more stable - # otherwise, if we're trying to find a bandwidth cutoff, or we want to - # contact operators in priority order, sort by bandwidth (not yet - # implemented) - # otherwise, if we're contacting operators, sort by contact - candidates.sort_fallbacks_by(OUTPUT_SORT_FIELD) - - for x in candidates.fallbacks: - print x.fallbackdir_line(candidates.fallbacks, prefilter_fallbacks) - -if __name__ == "__main__": - main() diff --git a/scripts/maint/updateRustDependencies.sh b/scripts/maint/updateRustDependencies.sh index a5a92579d3..6d0587351f 100755 --- a/scripts/maint/updateRustDependencies.sh +++ b/scripts/maint/updateRustDependencies.sh @@ -20,26 +20,26 @@ set -e -HERE=`dirname $(realpath $0)` -TOPLEVEL=`dirname $(dirname $HERE)` +HERE=$(dirname "$(realpath "$0")") +TOPLEVEL=$(dirname "$(dirname "$HERE")") TOML="$TOPLEVEL/src/rust/Cargo.toml" VENDORED="$TOPLEVEL/src/ext/rust/crates" -CARGO=`which cargo` +CARGO=$(command -v cargo) if ! test -f "$TOML" ; then - printf "Error: Couldn't find workspace Cargo.toml in expected location: %s\n" "$TOML" + printf "Error: Couldn't find workspace Cargo.toml in expected location: %s\\n" "$TOML" fi if ! test -d "$VENDORED" ; then - printf "Error: Couldn't find directory for Rust dependencies! Expected location: %s\n" "$VENDORED" + printf "Error: Couldn't find directory for Rust dependencies! Expected location: %s\\n" "$VENDORED" fi if test -z "$CARGO" ; then - printf "Error: cargo must be installed and in your \$PATH\n" + printf "Error: cargo must be installed and in your \$PATH\\n" fi -if test -z `cargo --list | grep vendor` ; then - printf "Error: cargo-vendor not installed\n" +if test -z "$(cargo --list | grep vendor)" ; then + printf "Error: cargo-vendor not installed\\n" fi -$CARGO vendor -v --locked --explicit-version --no-delete --sync $TOML $VENDORED +$CARGO vendor -v --locked --explicit-version --no-delete --sync "$TOML" "$VENDORED" diff --git a/scripts/maint/updateVersions.pl.in b/scripts/maint/updateVersions.pl.in deleted file mode 100755 index 65c51a1f2d..0000000000 --- a/scripts/maint/updateVersions.pl.in +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/perl -w - -$CONFIGURE_IN = '@abs_top_srcdir@/configure.ac'; -$ORCONFIG_H = '@abs_top_srcdir@/src/win32/orconfig.h'; -$TOR_NSI = '@abs_top_srcdir@/contrib/win32build/tor-mingw.nsi.in'; - -$quiet = 1; - -sub demand { - my $fn = shift; - die "Missing file $fn" unless (-f $fn); -} - -demand($CONFIGURE_IN); -demand($ORCONFIG_H); -demand($TOR_NSI); - -# extract version from configure.ac - -open(F, $CONFIGURE_IN) or die "$!"; -$version = undef; -while (<F>) { - if (/AC_INIT\(\[tor\],\s*\[([^\]]*)\]\)/) { - $version = $1; - last; - } -} -die "No version found" unless $version; -print "Tor version is $version\n" unless $quiet; -close F; - -sub correctversion { - my ($fn, $defchar) = @_; - undef $/; - open(F, $fn) or die "$!"; - my $s = <F>; - close F; - if ($s =~ /^$defchar(?:)define\s+VERSION\s+\"([^\"]+)\"/m) { - $oldver = $1; - if ($oldver ne $version) { - print "Version mismatch in $fn: It thinks that the version is $oldver. I think it's $version. Fixing.\n"; - $line = $defchar . "define VERSION \"$version\""; - open(F, ">$fn.bak"); - print F $s; - close F; - $s =~ s/^$defchar(?:)define\s+VERSION.*?$/$line/m; - open(F, ">$fn"); - print F $s; - close F; - } else { - print "$fn has the correct version. Good.\n" unless $quiet; - } - } else { - print "Didn't find a version line in $fn -- uh oh.\n"; - } -} - -correctversion($TOR_NSI, "!"); -correctversion($ORCONFIG_H, "#"); diff --git a/scripts/maint/update_versions.py b/scripts/maint/update_versions.py new file mode 100755 index 0000000000..8067f2c6c8 --- /dev/null +++ b/scripts/maint/update_versions.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python + +from __future__ import print_function + +import io +import os +import re +import sys +import time + +def P(path): + """ + Give 'path' as a path relative to the abs_top_srcdir environment + variable. + """ + return os.path.join( + os.environ.get('abs_top_srcdir', "."), + path) + +def warn(msg): + """ + Print an warning message. + """ + print("WARNING: {}".format(msg), file=sys.stderr) + +def find_version(infile): + """ + Given an open file (or some other iterator of lines) holding a + configure.ac file, find the current version line. + """ + for line in infile: + m = re.search(r'AC_INIT\(\[tor\],\s*\[([^\]]*)\]\)', line) + if m: + return m.group(1) + + return None + +def update_version_in(infile, outfile, regex, versionline): + """ + Copy every line from infile to outfile. If any line matches 'regex', + replace it with 'versionline'. Return True if any line was changed; + false otherwise. + + 'versionline' is either a string -- in which case it is used literally, + or a function that receives the output of 'regex.match'. + """ + found = False + have_changed = False + for line in infile: + m = regex.match(line) + if m: + found = True + oldline = line + if type(versionline) == type(u""): + line = versionline + else: + line = versionline(m) + if not line.endswith("\n"): + line += "\n" + if oldline != line: + have_changed = True + outfile.write(line) + + if not found: + warn("didn't find any version line to replace in {}".format(infile.name)) + + return have_changed + +def replace_on_change(fname, change): + """ + If "change" is true, replace fname with fname.tmp. Otherwise, + delete fname.tmp. Log what we're doing to stderr. + """ + if not change: + print("No change in {}".format(fname)) + os.unlink(fname+".tmp") + else: + print("Updating {}".format(fname)) + os.rename(fname+".tmp", fname) + + +def update_file(fname, + regex, + versionline, + encoding="utf-8"): + """ + Replace any line matching 'regex' in 'fname' with 'versionline'. + Do not modify 'fname' if there are no changes made. Use the + provided encoding to read and write. + """ + with io.open(fname, "r", encoding=encoding) as f, \ + io.open(fname+".tmp", "w", encoding=encoding) as outf: + have_changed = update_version_in(f, outf, regex, versionline) + + replace_on_change(fname, have_changed) + +# Find out our version +with open("configure.ac") as f: + version = find_version(f) + +# If we have no version, we can't proceed. +if version == None: + print("No version found in configure.ac", file=sys.stderr()) + sys.exit(1) + +print("The version is {}".format(version)) + +today = time.strftime("%Y-%m-%d", time.gmtime()) + +# In configure.ac, we replace the definition of APPROX_RELEASE_DATE +# with "{today} for {version}", but only if the version does not match +# what is already there. +def replace_fn(m): + if m.group(1) != version: + # The version changed -- we change the date. + return u'AC_DEFINE(APPROX_RELEASE_DATE, ["{}"], # for {}'.format(today, version) + else: + # No changes. + return m.group(0) +update_file(P("configure.ac"), + re.compile(r'AC_DEFINE\(APPROX_RELEASE_DATE.* for (.*)'), + replace_fn) + +# In tor-mingw.nsi.in, we replace the definition of VERSION. +update_file(P("contrib/win32build/tor-mingw.nsi.in"), + re.compile(r'!define VERSION .*'), + u'!define VERSION "{}"'.format(version), + encoding="iso-8859-1") + +# In src/win32/orconfig.h, we replace the definition of VERSION. +update_file(P("src/win32/orconfig.h"), + re.compile(r'#define VERSION .*'), + u'#define VERSION "{}"'.format(version)) diff --git a/scripts/test/chutney-git-bisect.sh b/scripts/test/chutney-git-bisect.sh index 8a3f2c70c8..dcf8ab1102 100755 --- a/scripts/test/chutney-git-bisect.sh +++ b/scripts/test/chutney-git-bisect.sh @@ -15,21 +15,21 @@ # Skips the test if <skip-flavour> fails (default no skip). CHUTNEY_TRIES=3 -if [ ! -z "$1" ]; then +if [ -n "$1" ]; then CHUTNEY_TRIES="$1" fi -if [ ! -z "$2" ]; then - cd "$2" +if [ -n "$2" ]; then + cd "$2" || exit fi CHUTNEY_TEST_CMD="make test-network-all" -if [ ! -z "$3" ]; then +if [ -n "$3" ]; then CHUTNEY_TEST_CMD="$CHUTNEY_PATH/tools/test-network.sh --flavour $3" fi CHUTNEY_SKIP_ON_FAIL_CMD="true" -if [ ! -z "$4" ]; then +if [ -n "$4" ]; then CHUTNEY_SKIP_ON_FAIL_CMD="$CHUTNEY_PATH/tools/test-network.sh --flavour $4" fi @@ -54,9 +54,9 @@ while [ "$i" -le "$CHUTNEY_TRIES" ]; do echo "test '$CHUTNEY_TEST_CMD' succeeded after $i/$CHUTNEY_TRIES attempts, good" exit 0 fi - i=$[$i+1] + i=$((i+1)) done -i=$[$i-1] +i=$((i-1)) echo "test '$CHUTNEY_TEST_CMD' failed $i/$CHUTNEY_TRIES attempts, bad" exit 1 diff --git a/scripts/test/cov-diff b/scripts/test/cov-diff index 6179dff63e..f3ca856888 100755 --- a/scripts/test/cov-diff +++ b/scripts/test/cov-diff @@ -7,9 +7,9 @@ DIRA="$1" DIRB="$2" -for B in $DIRB/*; do - A=$DIRA/`basename $B` - if [ -f $A ]; then +for B in "$DIRB"/*; do + A=$DIRA/$(basename "$B") + if [ -f "$A" ]; then perl -pe 's/^\s*\!*\d+(\*?):/ 1$1:/; s/^([^:]+:)[\d\s]+:/$1/; s/^ *-:(Runs|Programs):.*//;' "$A" > "$A.tmp" else cat /dev/null > "$A.tmp" diff --git a/scripts/test/coverage b/scripts/test/coverage index b6e17abe25..f61c83bc72 100755 --- a/scripts/test/coverage +++ b/scripts/test/coverage @@ -8,9 +8,9 @@ dst=$1 for fn in src/core/*/*.c src/feature/*/*.c src/app/*/*.c src/lib/*/*.c; do - BN=`basename $fn` - DN=`dirname $fn` - F=`echo $BN | sed -e 's/\.c$//;'` + BN=$(basename "$fn") + DN=$(dirname "$fn") + F=$(echo "$BN" | sed -e 's/\.c$//;') GC="${BN}.gcov" # Figure out the object file names ONS=$(echo "${DN}"/*testing_a-"${F}".o) @@ -20,18 +20,18 @@ for fn in src/core/*/*.c src/feature/*/*.c src/app/*/*.c src/lib/*/*.c; do then for on in $ONS; do # We should have a gcno file - GCNO=`echo $on | sed -e 's/\.o$/\.gcno/;'` - if [ -e $GCNO ] + GCNO=$(echo "$on" | sed -e 's/\.o$/\.gcno/;') + if [ -e "$GCNO" ] then # No need to test for gcda, since gcov assumes no execution # if it's absent - rm -f $GC - gcov -o $on $fn - if [ -e $GC ] + rm -f "$GC" + gcov -o "$on" "$fn" + if [ -e "$GC" ] then if [ -d "$dst" ] then - mv $GC $dst/$GC + mv "$GC" "$dst"/"$GC" fi else echo "gcov -o $on $fn didn't make a .gcov file" diff --git a/scripts/test/scan-build.sh b/scripts/test/scan-build.sh index 8d126cbcee..26e05ff101 100755 --- a/scripts/test/scan-build.sh +++ b/scripts/test/scan-build.sh @@ -33,6 +33,7 @@ CHECKERS="\ -enable-checker security.insecureAPI.strcpy \ " +# shellcheck disable=SC2034 # These have high false-positive rates. EXTRA_CHECKERS="\ -enable-checker alpha.security.ArrayBoundV2 \ @@ -40,6 +41,7 @@ EXTRA_CHECKERS="\ -enable-checker alpha.core.CastSize \ " +# shellcheck disable=SC2034 # These don't seem to generate anything useful NOISY_CHECKERS="\ -enable-checker alpha.clone.CloneChecker \ @@ -52,6 +54,7 @@ else OUTPUTARG="" fi +# shellcheck disable=SC2086 scan-build \ $CHECKERS \ ./configure @@ -61,11 +64,13 @@ scan-build \ # Make this not get scanned for dead assignments, since it has lots of # dead assignments we don't care about. +# shellcheck disable=SC2086 scan-build \ $CHECKERS \ -disable-checker deadcode.DeadStores \ make -j5 -k ./src/ext/ed25519/ref10/libed25519_ref10.a +# shellcheck disable=SC2086 scan-build \ $CHECKERS $OUTPUTARG \ make -j5 -k diff --git a/src/app/config/config.c b/src/app/config/config.c index 0b1b758d96..1676e9349a 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -64,6 +64,7 @@ #include "app/config/confparse.h" #include "app/config/statefile.h" #include "app/main/main.h" +#include "app/main/subsysmgr.h" #include "core/mainloop/connection.h" #include "core/mainloop/cpuworker.h" #include "core/mainloop/mainloop.h" @@ -112,9 +113,9 @@ #include "lib/crypt_ops/crypto_rand.h" #include "lib/crypt_ops/crypto_util.h" #include "lib/encoding/confline.h" -#include "lib/log/git_revision.h" #include "lib/net/resolve.h" #include "lib/sandbox/sandbox.h" +#include "lib/version/torversion.h" #ifdef ENABLE_NSS #include "lib/crypt_ops/crypto_nss_mgt.h" @@ -144,7 +145,7 @@ #include "lib/process/pidfile.h" #include "lib/process/restrict.h" #include "lib/process/setuid.h" -#include "lib/process/subprocess.h" +#include "lib/process/process.h" #include "lib/net/gethostname.h" #include "lib/thread/numcpus.h" @@ -342,6 +343,7 @@ static config_var_t option_vars_[] = { V(ClientOnly, BOOL, "0"), V(ClientPreferIPv6ORPort, AUTOBOOL, "auto"), V(ClientPreferIPv6DirPort, AUTOBOOL, "auto"), + V(ClientAutoIPv6ORPort, BOOL, "0"), V(ClientRejectInternalAddresses, BOOL, "1"), V(ClientTransportPlugin, LINELIST, NULL), V(ClientUseIPv6, BOOL, "0"), @@ -391,6 +393,10 @@ static config_var_t option_vars_[] = { OBSOLETE("DynamicDHGroups"), VPORT(DNSPort), OBSOLETE("DNSListenAddress"), + V(DormantClientTimeout, INTERVAL, "24 hours"), + V(DormantTimeoutDisabledByIdleStreams, BOOL, "1"), + V(DormantOnFirstStartup, BOOL, "0"), + V(DormantCanceledByStartup, BOOL, "0"), /* DoS circuit creation options. */ V(DoSCircuitCreationEnabled, AUTOBOOL, "auto"), V(DoSCircuitCreationMinConnections, UINT, "0"), @@ -416,6 +422,10 @@ static config_var_t option_vars_[] = { V(ExcludeExitNodes, ROUTERSET, NULL), OBSOLETE("ExcludeSingleHopRelays"), V(ExitNodes, ROUTERSET, NULL), + /* Researchers need a way to tell their clients to use specific + * middles that they also control, to allow safe live-network + * experimentation with new padding machines. */ + V(MiddleNodes, ROUTERSET, NULL), V(ExitPolicy, LINELIST, NULL), V(ExitPolicyRejectPrivate, BOOL, "1"), V(ExitPolicyRejectLocalInterfaces, BOOL, "0"), @@ -975,42 +985,6 @@ set_options(or_options_t *new_val, char **msg) return 0; } -/** The version of this Tor process, as parsed. */ -static char *the_tor_version = NULL; -/** A shorter version of this Tor process's version, for export in our router - * descriptor. (Does not include the git version, if any.) */ -static char *the_short_tor_version = NULL; - -/** Return the current Tor version. */ -const char * -get_version(void) -{ - if (the_tor_version == NULL) { - if (strlen(tor_git_revision)) { - tor_asprintf(&the_tor_version, "%s (git-%s)", get_short_version(), - tor_git_revision); - } else { - the_tor_version = tor_strdup(get_short_version()); - } - } - return the_tor_version; -} - -/** Return the current Tor version, without any git tag. */ -const char * -get_short_version(void) -{ - - if (the_short_tor_version == NULL) { -#ifdef TOR_BUILD_TAG - tor_asprintf(&the_short_tor_version, "%s (%s)", VERSION, TOR_BUILD_TAG); -#else - the_short_tor_version = tor_strdup(VERSION); -#endif - } - return the_short_tor_version; -} - /** Release additional memory allocated in options */ STATIC void @@ -1070,9 +1044,6 @@ config_free_all(void) tor_free(torrc_defaults_fname); tor_free(global_dirfrontpagecontents); - tor_free(the_short_tor_version); - tor_free(the_tor_version); - cleanup_protocol_warning_severity_level(); have_parsed_cmdline = 0; @@ -1443,10 +1414,10 @@ options_act_reversible(const or_options_t *old_options, char **msg) * processes. */ if (running_tor && options->RunAsDaemon) { if (! start_daemon_has_been_called()) - crypto_prefork(); + subsystems_prefork(); /* No need to roll back, since you can't change the value. */ if (start_daemon()) - crypto_postfork(); + subsystems_postfork(); } #ifdef HAVE_SYSTEMD @@ -1735,6 +1706,7 @@ options_need_geoip_info(const or_options_t *options, const char **reason_out) int routerset_usage = routerset_needs_geoip(options->EntryNodes) || routerset_needs_geoip(options->ExitNodes) || + routerset_needs_geoip(options->MiddleNodes) || routerset_needs_geoip(options->ExcludeExitNodes) || routerset_needs_geoip(options->ExcludeNodes) || routerset_needs_geoip(options->HSLayer2Nodes) || @@ -2042,9 +2014,6 @@ options_act(const or_options_t *old_options) finish_daemon(options->DataDirectory); } - /* See whether we need to enable/disable our once-a-second timer. */ - reschedule_per_second_timer(); - /* We want to reinit keys as needed before we do much of anything else: keys are important, and other things can depend on them. */ if (transition_affects_workers || @@ -2177,6 +2146,7 @@ options_act(const or_options_t *old_options) options->HSLayer2Nodes) || !routerset_equal(old_options->HSLayer3Nodes, options->HSLayer3Nodes) || + !routerset_equal(old_options->MiddleNodes, options->MiddleNodes) || options->StrictNodes != old_options->StrictNodes) { log_info(LD_CIRC, "Changed to using entry guards or bridges, or changed " @@ -3441,6 +3411,8 @@ options_validate(or_options_t *old_options, or_options_t *options, if (ContactInfo && !string_is_utf8(ContactInfo, strlen(ContactInfo))) REJECT("ContactInfo config option must be UTF-8."); + check_network_configuration(server_mode(options)); + /* Special case on first boot if no Log options are given. */ if (!options->Logs && !options->RunAsDaemon && !from_setconf) { if (quiet_level == 0) @@ -3592,7 +3564,8 @@ options_validate(or_options_t *old_options, or_options_t *options, "(Bridge/V3)AuthoritativeDir is set."); /* If we have a v3bandwidthsfile and it's broken, complain on startup */ if (options->V3BandwidthsFile && !old_options) { - dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL, NULL); + dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL, NULL, + NULL); } /* same for guardfraction file */ if (options->GuardfractionFile && !old_options) { @@ -3897,6 +3870,10 @@ options_validate(or_options_t *old_options, or_options_t *options, "default."); } + if (options->DormantClientTimeout < 10*60 && !options->TestingTorNetwork) { + REJECT("DormantClientTimeout is too low. It must be at least 10 minutes."); + } + if (options->PathBiasNoticeRate > 1.0) { tor_asprintf(msg, "PathBiasNoticeRate is too high. " @@ -3948,7 +3925,8 @@ options_validate(or_options_t *old_options, or_options_t *options, } if (options->HeartbeatPeriod && - options->HeartbeatPeriod < MIN_HEARTBEAT_PERIOD) { + options->HeartbeatPeriod < MIN_HEARTBEAT_PERIOD && + !options->TestingTorNetwork) { log_warn(LD_CONFIG, "HeartbeatPeriod option is too short; " "raising to %d seconds.", MIN_HEARTBEAT_PERIOD); options->HeartbeatPeriod = MIN_HEARTBEAT_PERIOD; diff --git a/src/app/config/config.h b/src/app/config/config.h index 301faf7067..46db02f944 100644 --- a/src/app/config/config.h +++ b/src/app/config/config.h @@ -41,8 +41,6 @@ const char *escaped_safe_str_client(const char *address); const char *escaped_safe_str(const char *address); void init_protocol_warning_severity_level(void); int get_protocol_warning_severity_level(void); -const char *get_version(void); -const char *get_short_version(void); /** An error from options_trial_assign() or options_init_from_string(). */ typedef enum setopt_err_t { diff --git a/src/app/config/or_options_st.h b/src/app/config/or_options_st.h index 74d2fefa16..bd707fd193 100644 --- a/src/app/config/or_options_st.h +++ b/src/app/config/or_options_st.h @@ -72,6 +72,9 @@ struct or_options_t { routerset_t *ExitNodes; /**< Structure containing nicknames, digests, * country codes and IP address patterns of ORs to * consider as exits. */ + routerset_t *MiddleNodes; /**< Structure containing nicknames, digests, + * country codes and IP address patterns of ORs to + * consider as middles. */ routerset_t *EntryNodes;/**< Structure containing nicknames, digests, * country codes and IP address patterns of ORs to * consider as entry points. */ @@ -666,6 +669,9 @@ struct or_options_t { * accessing this value directly. */ int ClientPreferIPv6DirPort; + /** If true, prefer an IPv4 or IPv6 OR port at random. */ + int ClientAutoIPv6ORPort; + /** The length of time that we think a consensus should be fresh. */ int V3AuthVotingInterval; /** The length of time we think it will take to distribute votes. */ @@ -1072,6 +1078,25 @@ struct or_options_t { /** Autobool: Do we refuse single hop client rendezvous? */ int DoSRefuseSingleHopClientRendezvous; + + /** Interval: how long without activity does it take for a client + * to become dormant? + **/ + int DormantClientTimeout; + + /** Boolean: true if having an idle stream is sufficient to prevent a client + * from becoming dormant. + **/ + int DormantTimeoutDisabledByIdleStreams; + + /** Boolean: true if Tor should be dormant the first time it starts with + * a datadirectory; false otherwise. */ + int DormantOnFirstStartup; + /** + * Boolean: true if Tor should treat every startup event as cancelling + * a possible previous dormant state. + **/ + int DormantCanceledByStartup; }; #endif diff --git a/src/app/config/or_state_st.h b/src/app/config/or_state_st.h index 5f8214d146..cdb9b38287 100644 --- a/src/app/config/or_state_st.h +++ b/src/app/config/or_state_st.h @@ -87,6 +87,13 @@ struct or_state_t { /** When did we last rotate our onion key? "0" for 'no idea'. */ time_t LastRotatedOnionKey; + + /** Number of minutes since the last user-initiated request (as defined by + * the dormant net-status system.) Set to zero if we are dormant. */ + int MinutesSinceUserActivity; + /** True if we were dormant when we last wrote the file; false if we + * weren't. "auto" on initial startup. */ + int Dormant; }; #endif diff --git a/src/app/config/statefile.c b/src/app/config/statefile.c index 89039a05b5..9681f6f8b3 100644 --- a/src/app/config/statefile.c +++ b/src/app/config/statefile.c @@ -34,6 +34,7 @@ #include "app/config/config.h" #include "app/config/confparse.h" #include "core/mainloop/mainloop.h" +#include "core/mainloop/netstatus.h" #include "core/mainloop/connection.h" #include "feature/control/control.h" #include "feature/client/entrynodes.h" @@ -45,6 +46,7 @@ #include "app/config/statefile.h" #include "lib/encoding/confline.h" #include "lib/net/resolve.h" +#include "lib/version/torversion.h" #include "app/config/or_state_st.h" @@ -131,6 +133,9 @@ static config_var_t state_vars_[] = { VAR("CircuitBuildTimeBin", LINELIST_S, BuildtimeHistogram, NULL), VAR("BuildtimeHistogram", LINELIST_V, BuildtimeHistogram, NULL), + V(MinutesSinceUserActivity, UINT, NULL), + V(Dormant, AUTOBOOL, "auto"), + END_OF_CONFIG_VARS }; @@ -308,6 +313,8 @@ or_state_set(or_state_t *new_state) get_circuit_build_times_mutable(),global_state) < 0) { ret = -1; } + netstatus_load_from_state(global_state, time(NULL)); + return ret; } @@ -499,6 +506,8 @@ or_state_save(time_t now) entry_guards_update_state(global_state); rep_hist_update_state(global_state); circuit_build_times_update_state(get_circuit_build_times(), global_state); + netstatus_flush_to_state(global_state, now); + if (accounting_is_enabled(get_options())) accounting_run_housekeeping(now); diff --git a/src/app/main/main.c b/src/app/main/main.c index 67f2181cd5..4b60763f75 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -15,12 +15,14 @@ #include "app/config/statefile.h" #include "app/main/main.h" #include "app/main/ntmain.h" +#include "app/main/subsysmgr.h" #include "core/mainloop/connection.h" #include "core/mainloop/cpuworker.h" #include "core/mainloop/mainloop.h" #include "core/mainloop/netstatus.h" #include "core/or/channel.h" #include "core/or/channelpadding.h" +#include "core/or/circuitpadding.h" #include "core/or/channeltls.h" #include "core/or/circuitlist.h" #include "core/or/circuitmux_ewma.h" @@ -33,6 +35,7 @@ #include "core/or/relay.h" #include "core/or/scheduler.h" #include "core/or/status.h" +#include "core/or/versions.h" #include "feature/api/tor_api.h" #include "feature/api/tor_api_internal.h" #include "feature/client/addressmap.h" @@ -65,11 +68,11 @@ #include "feature/stats/predict_ports.h" #include "feature/stats/rephist.h" #include "lib/compress/compress.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/crypt_ops/crypto_rand.h" #include "lib/crypt_ops/crypto_s2k.h" -#include "lib/err/backtrace.h" #include "lib/geoip/geoip.h" +#include "lib/net/resolve.h" #include "lib/process/waitpid.h" @@ -83,6 +86,7 @@ #include "lib/encoding/confline.h" #include "lib/evloop/timers.h" #include "lib/crypt_ops/crypto_init.h" +#include "lib/version/torversion.h" #include <event2/event.h> @@ -301,6 +305,19 @@ process_signal(int sig) log_heartbeat(time(NULL)); control_event_signal(sig); break; + case SIGACTIVE: + /* "SIGACTIVE" counts as ersatz user activity. */ + note_user_activity(approx_time()); + control_event_signal(sig); + break; + case SIGDORMANT: + /* "SIGDORMANT" means to ignore past user activity */ + log_notice(LD_GENERAL, "Going dormant because of controller request."); + reset_user_activity(0); + set_network_participation(false); + schedule_rescan_periodic_events(); + control_event_signal(sig); + break; } } @@ -426,18 +443,6 @@ dumpstats(int severity) rend_service_dump_stats(severity); } -/** Called by exit() as we shut down the process. - */ -static void -exit_function(void) -{ - /* NOTE: If we ever daemonize, this gets called immediately. That's - * okay for now, because we only use this on Windows. */ -#ifdef _WIN32 - WSACleanup(); -#endif -} - #ifdef _WIN32 #define UNIX_ONLY 0 #else @@ -482,6 +487,8 @@ static struct { { SIGNEWNYM, 0, NULL }, { SIGCLEARDNSCACHE, 0, NULL }, { SIGHEARTBEAT, 0, NULL }, + { SIGACTIVE, 0, NULL }, + { SIGDORMANT, 0, NULL }, { -1, -1, NULL } }; @@ -546,18 +553,13 @@ tor_init(int argc, char *argv[]) tor_snprintf(progname, sizeof(progname), "Tor %s", get_version()); log_set_application_name(progname); - /* Set up the crypto nice and early */ - if (crypto_early_init() < 0) { - log_err(LD_GENERAL, "Unable to initialize the crypto subsystem!"); - return -1; - } - /* Initialize the history structures. */ rep_hist_init(); /* Initialize the service cache. */ rend_cache_init(); addressmap_init(); /* Init the client dns cache. Do it always, since it's * cheap. */ + /* Initialize the HS subsystem. */ hs_init(); @@ -632,12 +634,6 @@ tor_init(int argc, char *argv[]) rust_log_welcome_string(); #endif /* defined(HAVE_RUST) */ - if (network_init()<0) { - log_err(LD_BUG,"Error initializing network; exiting."); - return -1; - } - atexit(exit_function); - int init_rv = options_init_from_torrc(argc,argv); if (init_rv < 0) { log_err(LD_CONFIG,"Reading config failed--see warnings above."); @@ -651,9 +647,13 @@ tor_init(int argc, char *argv[]) /* The options are now initialised */ const or_options_t *options = get_options(); - /* Initialize channelpadding parameters to defaults until we get - * a consensus */ + /* Initialize channelpadding and circpad parameters to defaults + * until we get a consensus */ channelpadding_new_consensus_params(NULL); + circpad_new_consensus_params(NULL); + + /* Initialize circuit padding to defaults+torrc until we get a consensus */ + circpad_machines_init(); /* Initialize predicted ports list after loading options */ predicted_ports_init(); @@ -772,6 +772,7 @@ tor_free_all(int postfork) dns_free_all(); clear_pending_onions(); circuit_free_all(); + circpad_machines_free(); entry_guards_free_all(); pt_free_all(); channel_tls_free_all(); @@ -784,7 +785,6 @@ tor_free_all(int postfork) routerparse_free_all(); ext_orport_free_all(); control_free_all(); - tor_free_getaddrinfo_cache(); protover_free_all(); bridges_free_all(); consdiffmgr_free_all(); @@ -792,6 +792,7 @@ tor_free_all(int postfork) dos_free_all(); circuitmux_ewma_free_all(); accounting_free_all(); + protover_summary_cache_free_all(); if (!postfork) { config_free_all(); @@ -801,7 +802,6 @@ tor_free_all(int postfork) policies_free_all(); } if (!postfork) { - tor_tls_free_all(); #ifndef _WIN32 tor_getpwnam(NULL); #endif @@ -814,12 +814,12 @@ tor_free_all(int postfork) release_lockfile(); } tor_libevent_free_all(); + + subsystems_shutdown(); + /* Stuff in util.c and address.c*/ if (!postfork) { - escaped(NULL); esc_router_info(NULL); - clean_up_backtrace_handler(); - logs_free_all(); /* free log strings. do this last so logs keep working. */ } } @@ -878,7 +878,6 @@ tor_cleanup(void) later, if it makes shutdown unacceptably slow. But for now, leave it here: it's helped us catch bugs in the past. */ - crypto_global_cleanup(); } /** Read/create keys as needed, and echo our fingerprint to stdout. */ @@ -1274,7 +1273,6 @@ int run_tor_main_loop(void) { handle_signals(); - monotime_init(); timers_initialize(); initialize_mainloop_events(); @@ -1386,54 +1384,13 @@ tor_run_main(const tor_main_configuration_t *tor_cfg) { int result = 0; -#ifdef _WIN32 -#ifndef HeapEnableTerminationOnCorruption -#define HeapEnableTerminationOnCorruption 1 -#endif - /* On heap corruption, just give up; don't try to play along. */ - HeapSetInformation(NULL, HeapEnableTerminationOnCorruption, NULL, 0); - - /* SetProcessDEPPolicy is only supported on 32-bit Windows. - * (On 64-bit Windows it always fails, and some compilers don't like the - * PSETDEP cast.) - * 32-bit Windows defines _WIN32. - * 64-bit Windows defines _WIN32 and _WIN64. */ -#ifndef _WIN64 - /* Call SetProcessDEPPolicy to permanently enable DEP. - The function will not resolve on earlier versions of Windows, - and failure is not dangerous. */ - HMODULE hMod = GetModuleHandleA("Kernel32.dll"); - if (hMod) { - typedef BOOL (WINAPI *PSETDEP)(DWORD); - PSETDEP setdeppolicy = (PSETDEP)GetProcAddress(hMod, - "SetProcessDEPPolicy"); - if (setdeppolicy) { - /* PROCESS_DEP_ENABLE | PROCESS_DEP_DISABLE_ATL_THUNK_EMULATION */ - setdeppolicy(3); - } - } -#endif /* !defined(_WIN64) */ -#endif /* defined(_WIN32) */ - - { - int bt_err = configure_backtrace_handler(get_version()); - if (bt_err < 0) { - log_warn(LD_BUG, "Unable to install backtrace handler: %s", - strerror(-bt_err)); - } - } - #ifdef EVENT_SET_MEM_FUNCTIONS_IMPLEMENTED event_set_mem_functions(tor_malloc_, tor_realloc_, tor_free_); #endif - init_protocol_warning_severity_level(); + subsystems_init(); - update_approx_time(time(NULL)); - tor_threads_init(); - tor_compress_init(); - init_logging(0); - monotime_init(); + init_protocol_warning_severity_level(); int argc = tor_cfg->argc + tor_cfg->argc_owned; char **argv = tor_calloc(argc, sizeof(char*)); @@ -1469,6 +1426,7 @@ tor_run_main(const tor_main_configuration_t *tor_cfg) tor_free_all(0); return -1; } + tor_make_getaddrinfo_cache_active(); // registering libevent rng #ifdef HAVE_EVUTIL_SECURE_RNG_SET_URANDOM_DEVICE_FILE diff --git a/src/app/main/subsysmgr.c b/src/app/main/subsysmgr.c new file mode 100644 index 0000000000..e0ca3ce4df --- /dev/null +++ b/src/app/main/subsysmgr.c @@ -0,0 +1,202 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#include "app/main/subsysmgr.h" +#include "lib/err/torerr.h" + +#include "lib/log/log.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +/** + * True iff we have checked tor_subsystems for consistency. + **/ +static bool subsystem_array_validated = false; + +/** + * True if a given subsystem is initialized. Expand this array if there + * are more than this number of subsystems. (We'd rather not + * dynamically allocate in this module.) + **/ +static bool sys_initialized[128]; + +/** + * Exit with a raw assertion if the subsystems list is inconsistent; + * initialize the subsystem_initialized array. + **/ +static void +check_and_setup(void) +{ + if (subsystem_array_validated) + return; + + raw_assert(ARRAY_LENGTH(sys_initialized) >= n_tor_subsystems); + memset(sys_initialized, 0, sizeof(sys_initialized)); + + int last_level = MIN_SUBSYS_LEVEL; + + for (unsigned i = 0; i < n_tor_subsystems; ++i) { + const subsys_fns_t *sys = tor_subsystems[i]; + if (sys->level < MIN_SUBSYS_LEVEL || sys->level > MAX_SUBSYS_LEVEL) { + fprintf(stderr, "BUG: Subsystem %s (at %u) has an invalid level %d. " + "It is supposed to be between %d and %d (inclusive).\n", + sys->name, i, sys->level, MIN_SUBSYS_LEVEL, MAX_SUBSYS_LEVEL); + raw_assert_unreached_msg("There is a bug in subsystem_list.c"); + } + if (sys->level < last_level) { + fprintf(stderr, "BUG: Subsystem %s (at #%u) is in the wrong position. " + "Its level is %d; but the previous subsystem's level was %d.\n", + sys->name, i, sys->level, last_level); + raw_assert_unreached_msg("There is a bug in subsystem_list.c"); + } + last_level = sys->level; + } + + subsystem_array_validated = true; +} + +/** + * Initialize all the subsystems; exit on failure. + **/ +int +subsystems_init(void) +{ + return subsystems_init_upto(MAX_SUBSYS_LEVEL); +} + +/** + * Initialize all the subsystems whose level is less than or equal to + * <b>target_level</b>; exit on failure. + **/ +int +subsystems_init_upto(int target_level) +{ + check_and_setup(); + + for (unsigned i = 0; i < n_tor_subsystems; ++i) { + const subsys_fns_t *sys = tor_subsystems[i]; + if (!sys->supported) + continue; + if (sys->level > target_level) + break; + if (sys_initialized[i]) + continue; + int r = 0; + if (sys->initialize) { + // Note that the logging subsystem is designed so that it does no harm + // to log a message in an uninitialized state. These messages will be + // discarded for now, however. + log_debug(LD_GENERAL, "Initializing %s", sys->name); + r = sys->initialize(); + } + if (r < 0) { + fprintf(stderr, "BUG: subsystem %s (at %u) initialization failed.\n", + sys->name, i); + raw_assert_unreached_msg("A subsystem couldn't be initialized."); + } + sys_initialized[i] = true; + } + + return 0; +} + +/** + * Shut down all the subsystems. + **/ +void +subsystems_shutdown(void) +{ + subsystems_shutdown_downto(MIN_SUBSYS_LEVEL - 1); +} + +/** + * Shut down all the subsystems whose level is above <b>target_level</b>. + **/ +void +subsystems_shutdown_downto(int target_level) +{ + check_and_setup(); + + for (int i = (int)n_tor_subsystems - 1; i >= 0; --i) { + const subsys_fns_t *sys = tor_subsystems[i]; + if (!sys->supported) + continue; + if (sys->level <= target_level) + break; + if (! sys_initialized[i]) + continue; + if (sys->shutdown) { + log_debug(LD_GENERAL, "Shutting down %s", sys->name); + sys->shutdown(); + } + sys_initialized[i] = false; + } +} + +/** + * Run pre-fork code on all subsystems that declare any + **/ +void +subsystems_prefork(void) +{ + check_and_setup(); + + for (int i = (int)n_tor_subsystems - 1; i >= 0; --i) { + const subsys_fns_t *sys = tor_subsystems[i]; + if (!sys->supported) + continue; + if (! sys_initialized[i]) + continue; + if (sys->prefork) { + log_debug(LD_GENERAL, "Pre-fork: %s", sys->name); + sys->prefork(); + } + } +} + +/** + * Run post-fork code on all subsystems that declare any + **/ +void +subsystems_postfork(void) +{ + check_and_setup(); + + for (unsigned i = 0; i < n_tor_subsystems; ++i) { + const subsys_fns_t *sys = tor_subsystems[i]; + if (!sys->supported) + continue; + if (! sys_initialized[i]) + continue; + if (sys->postfork) { + log_debug(LD_GENERAL, "Post-fork: %s", sys->name); + sys->postfork(); + } + } +} + +/** + * Run thread-cleanup code on all subsystems that declare any + **/ +void +subsystems_thread_cleanup(void) +{ + check_and_setup(); + + for (int i = (int)n_tor_subsystems - 1; i >= 0; --i) { + const subsys_fns_t *sys = tor_subsystems[i]; + if (!sys->supported) + continue; + if (! sys_initialized[i]) + continue; + if (sys->thread_cleanup) { + log_debug(LD_GENERAL, "Thread cleanup: %s", sys->name); + sys->thread_cleanup(); + } + } +} diff --git a/src/app/main/subsysmgr.h b/src/app/main/subsysmgr.h new file mode 100644 index 0000000000..a5e62f71d9 --- /dev/null +++ b/src/app/main/subsysmgr.h @@ -0,0 +1,24 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_SUBSYSMGR_T +#define TOR_SUBSYSMGR_T + +#include "lib/subsys/subsys.h" + +extern const struct subsys_fns_t *tor_subsystems[]; +extern const unsigned n_tor_subsystems; + +int subsystems_init(void); +int subsystems_init_upto(int level); + +void subsystems_shutdown(void); +void subsystems_shutdown_downto(int level); + +void subsystems_prefork(void); +void subsystems_postfork(void); +void subsystems_thread_cleanup(void); + +#endif diff --git a/src/app/main/subsystem_list.c b/src/app/main/subsystem_list.c new file mode 100644 index 0000000000..3834176182 --- /dev/null +++ b/src/app/main/subsystem_list.c @@ -0,0 +1,49 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#include "app/main/subsysmgr.h" +#include "lib/cc/compat_compiler.h" +#include "lib/cc/torint.h" + +#include "core/or/ocirc_event_sys.h" +#include "core/or/orconn_event_sys.h" +#include "feature/control/btrack_sys.h" +#include "lib/compress/compress_sys.h" +#include "lib/crypt_ops/crypto_sys.h" +#include "lib/err/torerr_sys.h" +#include "lib/log/log_sys.h" +#include "lib/net/network_sys.h" +#include "lib/process/winprocess_sys.h" +#include "lib/thread/thread_sys.h" +#include "lib/time/time_sys.h" +#include "lib/tls/tortls_sys.h" +#include "lib/wallclock/wallclock_sys.h" +#include "lib/process/process_sys.h" + +#include <stddef.h> + +/** + * Global list of the subsystems in Tor, in the order of their initialization. + **/ +const subsys_fns_t *tor_subsystems[] = { + &sys_winprocess, /* -100 */ + &sys_torerr, /* -100 */ + &sys_wallclock, /* -99 */ + &sys_threads, /* -95 */ + &sys_logging, /* -90 */ + &sys_time, /* -90 */ + &sys_network, /* -90 */ + &sys_compress, /* -70 */ + &sys_crypto, /* -60 */ + &sys_tortls, /* -50 */ + &sys_process, /* -35 */ + + &sys_orconn_event, /* -33 */ + &sys_ocirc_event, /* -32 */ + &sys_btrack, /* -30 */ +}; + +const unsigned n_tor_subsystems = ARRAY_LENGTH(tor_subsystems); diff --git a/src/config/mmdb-convert.py b/src/config/mmdb-convert.py index 3a454a3fc1..706a8b03cc 100644 --- a/src/config/mmdb-convert.py +++ b/src/config/mmdb-convert.py @@ -77,7 +77,7 @@ def to_int32(s): def to_int28(s): "Parse a pair of big-endian 28-bit integers from bytestring s." - a, b = unpack("!LL", s + b'\x00') + a, b = struct.unpack("!LL", s + b'\x00') return (((a & 0xf0) << 20) + (a >> 8)), ((a & 0x0f) << 24) + (b >> 8) class Tree(object): diff --git a/src/core/include.am b/src/core/include.am index 1b8ef2ac58..ae47c75e09 100644 --- a/src/core/include.am +++ b/src/core/include.am @@ -11,6 +11,8 @@ LIBTOR_APP_A_SOURCES = \ src/app/config/confparse.c \ src/app/config/statefile.c \ src/app/main/main.c \ + src/app/main/subsystem_list.c \ + src/app/main/subsysmgr.c \ src/core/crypto/hs_ntor.c \ src/core/crypto/onion_crypto.c \ src/core/crypto/onion_fast.c \ @@ -30,6 +32,7 @@ LIBTOR_APP_A_SOURCES = \ src/core/or/circuitlist.c \ src/core/or/circuitmux.c \ src/core/or/circuitmux_ewma.c \ + src/core/or/circuitpadding.c \ src/core/or/circuitstats.c \ src/core/or/circuituse.c \ src/core/or/command.c \ @@ -37,6 +40,8 @@ LIBTOR_APP_A_SOURCES = \ src/core/or/connection_or.c \ src/core/or/dos.c \ src/core/or/onion.c \ + src/core/or/ocirc_event.c \ + src/core/or/orconn_event.c \ src/core/or/policies.c \ src/core/or/protover.c \ src/core/or/protover_rust.c \ @@ -59,7 +64,13 @@ LIBTOR_APP_A_SOURCES = \ src/feature/client/dnsserv.c \ src/feature/client/entrynodes.c \ src/feature/client/transports.c \ + src/feature/control/btrack.c \ + src/feature/control/btrack_circuit.c \ + src/feature/control/btrack_orconn.c \ + src/feature/control/btrack_orconn_cevent.c \ + src/feature/control/btrack_orconn_maps.c \ src/feature/control/control.c \ + src/feature/control/control_bootstrap.c \ src/feature/control/fmt_serverstatus.c \ src/feature/control/getinfo_geoip.c \ src/feature/dirauth/keypin.c \ @@ -106,6 +117,7 @@ LIBTOR_APP_A_SOURCES = \ src/feature/nodelist/microdesc.c \ src/feature/nodelist/networkstatus.c \ src/feature/nodelist/nickname.c \ + src/feature/nodelist/nodefamily.c \ src/feature/nodelist/nodelist.c \ src/feature/nodelist/node_select.c \ src/feature/nodelist/routerinfo.c \ @@ -191,6 +203,7 @@ noinst_HEADERS += \ src/app/config/statefile.h \ src/app/main/main.h \ src/app/main/ntmain.h \ + src/app/main/subsysmgr.h \ src/core/crypto/hs_ntor.h \ src/core/crypto/onion_crypto.h \ src/core/crypto/onion_fast.h \ @@ -215,6 +228,7 @@ noinst_HEADERS += \ src/core/or/circuitmux.h \ src/core/or/circuitmux_ewma.h \ src/core/or/circuitstats.h \ + src/core/or/circuitpadding.h \ src/core/or/circuituse.h \ src/core/or/command.h \ src/core/or/connection_edge.h \ @@ -233,10 +247,14 @@ noinst_HEADERS += \ src/core/or/listener_connection_st.h \ src/core/or/onion.h \ src/core/or/or.h \ + src/core/or/orconn_event.h \ + src/core/or/orconn_event_sys.h \ src/core/or/or_circuit_st.h \ src/core/or/or_connection_st.h \ src/core/or/or_handshake_certs_st.h \ src/core/or/or_handshake_state_st.h \ + src/core/or/ocirc_event.h \ + src/core/or/ocirc_event_sys.h \ src/core/or/origin_circuit_st.h \ src/core/or/policies.h \ src/core/or/port_cfg_st.h \ @@ -263,6 +281,11 @@ noinst_HEADERS += \ src/feature/client/dnsserv.h \ src/feature/client/entrynodes.h \ src/feature/client/transports.h \ + src/feature/control/btrack_circuit.h \ + src/feature/control/btrack_orconn.h \ + src/feature/control/btrack_orconn_cevent.h \ + src/feature/control/btrack_orconn_maps.h \ + src/feature/control/btrack_sys.h \ src/feature/control/control.h \ src/feature/control/control_connection_st.h \ src/feature/control/fmt_serverstatus.h \ @@ -340,6 +363,8 @@ noinst_HEADERS += \ src/feature/nodelist/networkstatus_voter_info_st.h \ src/feature/nodelist/nickname.h \ src/feature/nodelist/node_st.h \ + src/feature/nodelist/nodefamily.h \ + src/feature/nodelist/nodefamily_st.h \ src/feature/nodelist/nodelist.h \ src/feature/nodelist/node_select.h \ src/feature/nodelist/routerinfo.h \ diff --git a/src/core/mainloop/connection.c b/src/core/mainloop/connection.c index 3595bba85c..21d7bd6d0d 100644 --- a/src/core/mainloop/connection.c +++ b/src/core/mainloop/connection.c @@ -57,7 +57,7 @@ #define CONNECTION_PRIVATE #include "core/or/or.h" #include "feature/client/bridges.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/tls/buffers_tls.h" #include "lib/err/backtrace.h" @@ -1466,6 +1466,20 @@ connection_listener_new(const struct sockaddr *listensockaddr, tor_socket_strerror(tor_socket_errno(s))); goto err; } + +#ifndef __APPLE__ + /* This code was introduced to help debug #28229. */ + int value; + socklen_t len = sizeof(value); + + if (!getsockopt(s, SOL_SOCKET, SO_ACCEPTCONN, &value, &len)) { + if (value == 0) { + log_err(LD_NET, "Could not listen on %s - " + "getsockopt(.,SO_ACCEPTCONN,.) yields 0.", address); + goto err; + } + } +#endif /* __APPLE__ */ #endif /* defined(HAVE_SYS_UN_H) */ } else { log_err(LD_BUG, "Got unexpected address family %d.", @@ -1867,7 +1881,7 @@ connection_init_accepted_conn(connection_t *conn, /* Initiate Extended ORPort authentication. */ return connection_ext_or_start_auth(TO_OR_CONN(conn)); case CONN_TYPE_OR: - control_event_or_conn_status(TO_OR_CONN(conn), OR_CONN_EVENT_NEW, 0); + connection_or_event_status(TO_OR_CONN(conn), OR_CONN_EVENT_NEW, 0); rv = connection_tls_start_handshake(TO_OR_CONN(conn), 1); if (rv < 0) { connection_or_close_for_error(TO_OR_CONN(conn), 0); @@ -1880,6 +1894,9 @@ connection_init_accepted_conn(connection_t *conn, TO_ENTRY_CONN(conn)->nym_epoch = get_signewnym_epoch(); TO_ENTRY_CONN(conn)->socks_request->listener_type = listener->base_.type; + /* Any incoming connection on an entry port counts as user activity. */ + note_user_activity(approx_time()); + switch (TO_CONN(listener)->type) { case CONN_TYPE_AP_LISTENER: conn->state = AP_CONN_STATE_SOCKS_WAIT; @@ -2079,6 +2096,11 @@ connection_connect_log_client_use_ip_version(const connection_t *conn) return; } + if (fascist_firewall_use_ipv6(options)) { + log_info(LD_NET, "Our outgoing connection is using IPv%d.", + tor_addr_family(&real_addr) == AF_INET6 ? 6 : 4); + } + /* Check if we couldn't satisfy an address family preference */ if ((!pref_ipv6 && tor_addr_family(&real_addr) == AF_INET6) || (pref_ipv6 && tor_addr_family(&real_addr) == AF_INET)) { @@ -2901,6 +2923,10 @@ retry_all_listeners(smartlist_t *new_conns, int close_all_noncontrol) retval = -1; #ifdef ENABLE_LISTENER_REBIND + if (smartlist_len(replacements)) + log_debug(LD_NET, "%d replacements - starting rebinding loop.", + smartlist_len(replacements)); + SMARTLIST_FOREACH_BEGIN(replacements, listener_replacement_t *, r) { int addr_in_use = 0; int skip = 0; @@ -2912,8 +2938,11 @@ retry_all_listeners(smartlist_t *new_conns, int close_all_noncontrol) connection_listener_new_for_port(r->new_port, &skip, &addr_in_use); connection_t *old_conn = r->old_conn; - if (skip) + if (skip) { + log_debug(LD_NET, "Skipping creating new listener for %s:%d", + old_conn->address, old_conn->port); continue; + } connection_close_immediate(old_conn); connection_mark_for_close(old_conn); @@ -4331,6 +4360,23 @@ connection_write_to_buf_impl_,(const char *string, size_t len, connection_write_to_buf_commit(conn, written); } +/** + * Write a <b>string</b> (of size <b>len</b> to directory connection + * <b>dir_conn</b>. Apply compression if connection is configured to use + * it and finalize it if <b>done</b> is true. + */ +void +connection_dir_buf_add(const char *string, size_t len, + dir_connection_t *dir_conn, int done) +{ + if (dir_conn->compress_state != NULL) { + connection_buf_add_compress(string, len, dir_conn, done); + return; + } + + connection_buf_add(string, len, TO_CONN(dir_conn)); +} + void connection_buf_add_compress(const char *string, size_t len, dir_connection_t *conn, int done) @@ -4445,6 +4491,16 @@ connection_get_by_type_state(int type, int state) CONN_GET_TEMPLATE(conn, conn->type == type && conn->state == state); } +/** + * Return a connection of type <b>type</b> that is not an internally linked + * connection, and is not marked for close. + **/ +MOCK_IMPL(connection_t *, +connection_get_by_type_nonlinked,(int type)) +{ + CONN_GET_TEMPLATE(conn, conn->type == type && !conn->linked); +} + /** Return a connection of type <b>type</b> that has rendquery equal * to <b>rendquery</b>, and that is not marked for close. If state * is non-zero, conn must be of that state too. @@ -5341,17 +5397,20 @@ assert_connection_ok(connection_t *conn, time_t now) } /** Fills <b>addr</b> and <b>port</b> with the details of the global - * proxy server we are using. - * <b>conn</b> contains the connection we are using the proxy for. + * proxy server we are using. Store a 1 to the int pointed to by + * <b>is_put_out</b> if the connection is using a pluggable + * transport; store 0 otherwise. <b>conn</b> contains the connection + * we are using the proxy for. * * Return 0 on success, -1 on failure. */ int get_proxy_addrport(tor_addr_t *addr, uint16_t *port, int *proxy_type, - const connection_t *conn) + int *is_pt_out, const connection_t *conn) { const or_options_t *options = get_options(); + *is_pt_out = 0; /* Client Transport Plugins can use another proxy, but that should be hidden * from the rest of tor (as the plugin is responsible for dealing with the * proxy), check it first, then check the rest of the proxy types to allow @@ -5367,6 +5426,7 @@ get_proxy_addrport(tor_addr_t *addr, uint16_t *port, int *proxy_type, tor_addr_copy(addr, &transport->addr); *port = transport->port; *proxy_type = transport->socks_version; + *is_pt_out = 1; return 0; } @@ -5403,11 +5463,13 @@ log_failed_proxy_connection(connection_t *conn) { tor_addr_t proxy_addr; uint16_t proxy_port; - int proxy_type; + int proxy_type, is_pt; - if (get_proxy_addrport(&proxy_addr, &proxy_port, &proxy_type, conn) != 0) + if (get_proxy_addrport(&proxy_addr, &proxy_port, &proxy_type, &is_pt, + conn) != 0) return; /* if we have no proxy set up, leave this function. */ + (void)is_pt; log_warn(LD_NET, "The connection to the %s proxy server at %s just failed. " "Make sure that the proxy server is up and running.", diff --git a/src/core/mainloop/connection.h b/src/core/mainloop/connection.h index 8ecdd6f06f..c93f1ef8e8 100644 --- a/src/core/mainloop/connection.h +++ b/src/core/mainloop/connection.h @@ -187,7 +187,7 @@ int connection_proxy_connect(connection_t *conn, int type); int connection_read_proxy_handshake(connection_t *conn); void log_failed_proxy_connection(connection_t *conn); int get_proxy_addrport(tor_addr_t *addr, uint16_t *port, int *proxy_type, - const connection_t *conn); + int *is_pt_out, const connection_t *conn); int retry_all_listeners(smartlist_t *new_conns, int close_all_noncontrol); @@ -226,6 +226,8 @@ MOCK_DECL(void, connection_write_to_buf_impl_, /* DOCDOC connection_write_to_buf */ static void connection_buf_add(const char *string, size_t len, connection_t *conn); +void connection_dir_buf_add(const char *string, size_t len, + dir_connection_t *dir_conn, int done); static inline void connection_buf_add(const char *string, size_t len, connection_t *conn) { @@ -240,6 +242,7 @@ size_t connection_get_outbuf_len(connection_t *conn); connection_t *connection_get_by_global_id(uint64_t id); connection_t *connection_get_by_type(int type); +MOCK_DECL(connection_t *,connection_get_by_type_nonlinked,(int type)); MOCK_DECL(connection_t *,connection_get_by_type_addr_port_purpose,(int type, const tor_addr_t *addr, uint16_t port, int purpose)); diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index f0aa37e8da..a42a8acc80 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -95,7 +95,7 @@ #include "feature/stats/geoip_stats.h" #include "feature/stats/predict_ports.h" #include "feature/stats/rephist.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/crypt_ops/crypto_rand.h" #include "lib/err/backtrace.h" #include "lib/tls/buffers_tls.h" @@ -200,12 +200,10 @@ static int can_complete_circuits = 0; #define LAZY_DESCRIPTOR_RETRY_INTERVAL (60) static int conn_close_if_marked(int i); -static int run_main_loop_until_done(void); static void connection_start_reading_from_linked_conn(connection_t *conn); static int connection_should_read_from_linked_conn(connection_t *conn); static void conn_read_callback(evutil_socket_t fd, short event, void *_conn); static void conn_write_callback(evutil_socket_t fd, short event, void *_conn); -static void second_elapsed_callback(periodic_timer_t *timer, void *args); static void shutdown_did_not_work_callback(evutil_socket_t fd, short event, void *arg) ATTR_NORETURN; @@ -1379,6 +1377,7 @@ CALLBACK(heartbeat); CALLBACK(hs_service); CALLBACK(launch_descriptor_fetches); CALLBACK(launch_reachability_tests); +CALLBACK(prune_old_routers); CALLBACK(reachability_warnings); CALLBACK(record_bridge_stats); CALLBACK(rend_cache_failure_clean); @@ -1391,78 +1390,93 @@ CALLBACK(save_stability); CALLBACK(save_state); CALLBACK(write_bridge_ns); CALLBACK(write_stats_file); +CALLBACK(control_per_second_events); +CALLBACK(second_elapsed); #undef CALLBACK /* Now we declare an array of periodic_event_item_t for each periodic event */ -#define CALLBACK(name, r, f) PERIODIC_EVENT(name, r, f) +#define CALLBACK(name, r, f) \ + PERIODIC_EVENT(name, PERIODIC_EVENT_ROLE_ ## r, f) +#define FL(name) (PERIODIC_EVENT_FLAG_ ## name) STATIC periodic_event_item_t periodic_events[] = { - /* Everyone needs to run those. */ - CALLBACK(add_entropy, PERIODIC_EVENT_ROLE_ALL, 0), - CALLBACK(check_expired_networkstatus, PERIODIC_EVENT_ROLE_ALL, 0), - CALLBACK(clean_caches, PERIODIC_EVENT_ROLE_ALL, 0), - CALLBACK(fetch_networkstatus, PERIODIC_EVENT_ROLE_ALL, - PERIODIC_EVENT_FLAG_NEED_NET), - CALLBACK(heartbeat, PERIODIC_EVENT_ROLE_ALL, 0), - CALLBACK(launch_descriptor_fetches, PERIODIC_EVENT_ROLE_ALL, - PERIODIC_EVENT_FLAG_NEED_NET), - CALLBACK(reset_padding_counts, PERIODIC_EVENT_ROLE_ALL, 0), - CALLBACK(retry_listeners, PERIODIC_EVENT_ROLE_ALL, - PERIODIC_EVENT_FLAG_NEED_NET), - CALLBACK(save_state, PERIODIC_EVENT_ROLE_ALL, 0), - CALLBACK(rotate_x509_certificate, PERIODIC_EVENT_ROLE_ALL, 0), - CALLBACK(write_stats_file, PERIODIC_EVENT_ROLE_ALL, 0), + + /* Everyone needs to run these. They need to have very long timeouts for + * that to be safe. */ + CALLBACK(add_entropy, ALL, 0), + CALLBACK(heartbeat, ALL, 0), + CALLBACK(reset_padding_counts, ALL, 0), + + /* This is a legacy catch-all callback that runs once per second if + * we are online and active. */ + CALLBACK(second_elapsed, NET_PARTICIPANT, + FL(RUN_ON_DISABLE)), + + /* XXXX Do we have a reason to do this on a callback? Does it do any good at + * all? For now, if we're dormant, we can let our listeners decay. */ + CALLBACK(retry_listeners, NET_PARTICIPANT, FL(NEED_NET)), + + /* We need to do these if we're participating in the Tor network. */ + CALLBACK(check_expired_networkstatus, NET_PARTICIPANT, 0), + CALLBACK(fetch_networkstatus, NET_PARTICIPANT, 0), + CALLBACK(launch_descriptor_fetches, NET_PARTICIPANT, FL(NEED_NET)), + CALLBACK(rotate_x509_certificate, NET_PARTICIPANT, 0), + CALLBACK(check_network_participation, NET_PARTICIPANT, 0), + + /* We need to do these if we're participating in the Tor network, and + * immediately before we stop. */ + CALLBACK(clean_caches, NET_PARTICIPANT, FL(RUN_ON_DISABLE)), + CALLBACK(save_state, NET_PARTICIPANT, FL(RUN_ON_DISABLE)), + CALLBACK(write_stats_file, NET_PARTICIPANT, FL(RUN_ON_DISABLE)), + CALLBACK(prune_old_routers, NET_PARTICIPANT, FL(RUN_ON_DISABLE)), /* Routers (bridge and relay) only. */ - CALLBACK(check_descriptor, PERIODIC_EVENT_ROLE_ROUTER, - PERIODIC_EVENT_FLAG_NEED_NET), - CALLBACK(check_ed_keys, PERIODIC_EVENT_ROLE_ROUTER, 0), - CALLBACK(check_for_reachability_bw, PERIODIC_EVENT_ROLE_ROUTER, - PERIODIC_EVENT_FLAG_NEED_NET), - CALLBACK(check_onion_keys_expiry_time, PERIODIC_EVENT_ROLE_ROUTER, 0), - CALLBACK(expire_old_ciruits_serverside, PERIODIC_EVENT_ROLE_ROUTER, - PERIODIC_EVENT_FLAG_NEED_NET), - CALLBACK(reachability_warnings, PERIODIC_EVENT_ROLE_ROUTER, - PERIODIC_EVENT_FLAG_NEED_NET), - CALLBACK(retry_dns, PERIODIC_EVENT_ROLE_ROUTER, 0), - CALLBACK(rotate_onion_key, PERIODIC_EVENT_ROLE_ROUTER, 0), + CALLBACK(check_descriptor, ROUTER, FL(NEED_NET)), + CALLBACK(check_ed_keys, ROUTER, 0), + CALLBACK(check_for_reachability_bw, ROUTER, FL(NEED_NET)), + CALLBACK(check_onion_keys_expiry_time, ROUTER, 0), + CALLBACK(expire_old_ciruits_serverside, ROUTER, FL(NEED_NET)), + CALLBACK(reachability_warnings, ROUTER, FL(NEED_NET)), + CALLBACK(retry_dns, ROUTER, 0), + CALLBACK(rotate_onion_key, ROUTER, 0), /* Authorities (bridge and directory) only. */ - CALLBACK(downrate_stability, PERIODIC_EVENT_ROLE_AUTHORITIES, 0), - CALLBACK(launch_reachability_tests, PERIODIC_EVENT_ROLE_AUTHORITIES, - PERIODIC_EVENT_FLAG_NEED_NET), - CALLBACK(save_stability, PERIODIC_EVENT_ROLE_AUTHORITIES, 0), + CALLBACK(downrate_stability, AUTHORITIES, 0), + CALLBACK(launch_reachability_tests, AUTHORITIES, FL(NEED_NET)), + CALLBACK(save_stability, AUTHORITIES, 0), /* Directory authority only. */ - CALLBACK(check_authority_cert, PERIODIC_EVENT_ROLE_DIRAUTH, 0), - CALLBACK(dirvote, PERIODIC_EVENT_ROLE_DIRAUTH, PERIODIC_EVENT_FLAG_NEED_NET), + CALLBACK(check_authority_cert, DIRAUTH, 0), + CALLBACK(dirvote, DIRAUTH, FL(NEED_NET)), /* Relay only. */ - CALLBACK(check_canonical_channels, PERIODIC_EVENT_ROLE_RELAY, - PERIODIC_EVENT_FLAG_NEED_NET), - CALLBACK(check_dns_honesty, PERIODIC_EVENT_ROLE_RELAY, - PERIODIC_EVENT_FLAG_NEED_NET), + CALLBACK(check_canonical_channels, RELAY, FL(NEED_NET)), + CALLBACK(check_dns_honesty, RELAY, FL(NEED_NET)), /* Hidden Service service only. */ - CALLBACK(hs_service, PERIODIC_EVENT_ROLE_HS_SERVICE, - PERIODIC_EVENT_FLAG_NEED_NET), + CALLBACK(hs_service, HS_SERVICE, FL(NEED_NET)), // XXXX break this down more /* Bridge only. */ - CALLBACK(record_bridge_stats, PERIODIC_EVENT_ROLE_BRIDGE, 0), + CALLBACK(record_bridge_stats, BRIDGE, 0), /* Client only. */ - CALLBACK(rend_cache_failure_clean, PERIODIC_EVENT_ROLE_CLIENT, 0), + /* XXXX this could be restricted to CLIENT+NET_PARTICIPANT */ + CALLBACK(rend_cache_failure_clean, NET_PARTICIPANT, FL(RUN_ON_DISABLE)), /* Bridge Authority only. */ - CALLBACK(write_bridge_ns, PERIODIC_EVENT_ROLE_BRIDGEAUTH, 0), + CALLBACK(write_bridge_ns, BRIDGEAUTH, 0), /* Directory server only. */ - CALLBACK(clean_consdiffmgr, PERIODIC_EVENT_ROLE_DIRSERVER, 0), + CALLBACK(clean_consdiffmgr, DIRSERVER, 0), + + /* Controller with per-second events only. */ + CALLBACK(control_per_second_events, CONTROLEV, 0), END_OF_PERIODIC_EVENTS }; #undef CALLBACK +#undef FL /* These are pointers to members of periodic_events[] that are used to * implement particular callbacks. We keep them separate here so that we @@ -1474,6 +1488,7 @@ static periodic_event_item_t *fetch_networkstatus_event=NULL; static periodic_event_item_t *launch_descriptor_fetches_event=NULL; static periodic_event_item_t *check_dns_honesty_event=NULL; static periodic_event_item_t *save_state_event=NULL; +static periodic_event_item_t *prune_old_routers_event=NULL; /** Reset all the periodic events so we'll do all our actions again as if we * just started up. @@ -1510,7 +1525,7 @@ get_my_roles(const or_options_t *options) { tor_assert(options); - int roles = 0; + int roles = PERIODIC_EVENT_ROLE_ALL; int is_bridge = options->BridgeRelay; int is_relay = server_mode(options); int is_dirauth = authdir_mode_v3(options); @@ -1518,6 +1533,8 @@ get_my_roles(const or_options_t *options) int is_hidden_service = !!hs_service_get_num_services() || !!rend_num_services(); int is_dirserver = dir_server_mode(options); + int sending_control_events = control_any_per_second_event_enabled(); + /* We also consider tor to have the role of a client if the ControlPort is * set because a lot of things can be done over the control port which * requires tor to have basic functionnalities. */ @@ -1525,6 +1542,9 @@ get_my_roles(const or_options_t *options) options->ControlPort_set || options->OwningControllerFD != UINT64_MAX; + int is_net_participant = is_participating_on_network() || + is_relay || is_hidden_service; + if (is_bridge) roles |= PERIODIC_EVENT_ROLE_BRIDGE; if (is_client) roles |= PERIODIC_EVENT_ROLE_CLIENT; if (is_relay) roles |= PERIODIC_EVENT_ROLE_RELAY; @@ -1532,6 +1552,8 @@ get_my_roles(const or_options_t *options) if (is_bridgeauth) roles |= PERIODIC_EVENT_ROLE_BRIDGEAUTH; if (is_hidden_service) roles |= PERIODIC_EVENT_ROLE_HS_SERVICE; if (is_dirserver) roles |= PERIODIC_EVENT_ROLE_DIRSERVER; + if (is_net_participant) roles |= PERIODIC_EVENT_ROLE_NET_PARTICIPANT; + if (sending_control_events) roles |= PERIODIC_EVENT_ROLE_CONTROLEV; return roles; } @@ -1576,6 +1598,7 @@ initialize_periodic_events(void) STMT_BEGIN name ## _event = find_periodic_event( #name ); STMT_END NAMED_CALLBACK(check_descriptor); + NAMED_CALLBACK(prune_old_routers); NAMED_CALLBACK(dirvote); NAMED_CALLBACK(fetch_networkstatus); NAMED_CALLBACK(launch_descriptor_fetches); @@ -1599,6 +1622,30 @@ teardown_periodic_events(void) periodic_events_initialized = 0; } +static mainloop_event_t *rescan_periodic_events_ev = NULL; + +/** Callback: rescan the periodic event list. */ +static void +rescan_periodic_events_cb(mainloop_event_t *event, void *arg) +{ + (void)event; + (void)arg; + rescan_periodic_events(get_options()); +} + +/** + * Schedule an event that will rescan which periodic events should run. + **/ +MOCK_IMPL(void, +schedule_rescan_periodic_events,(void)) +{ + if (!rescan_periodic_events_ev) { + rescan_periodic_events_ev = + mainloop_event_new(rescan_periodic_events_cb, NULL); + } + mainloop_event_activate(rescan_periodic_events_ev); +} + /** Do a pass at all our periodic events, disable those we don't need anymore * and enable those we need now using the given options. */ void @@ -1633,7 +1680,11 @@ rescan_periodic_events(const or_options_t *options) periodic_event_enable(item); } else { log_debug(LD_GENERAL, "Disabling periodic event %s", item->name); - periodic_event_disable(item); + if (item->flags & PERIODIC_EVENT_FLAG_RUN_ON_DISABLE) { + periodic_event_schedule_and_disable(item); + } else { + periodic_event_disable(item); + } } } } @@ -1708,6 +1759,30 @@ mainloop_schedule_postloop_cleanup(void) mainloop_event_activate(postloop_cleanup_ev); } +/** Event to run 'scheduled_shutdown_cb' */ +static mainloop_event_t *scheduled_shutdown_ev=NULL; + +/** Callback: run a scheduled shutdown */ +static void +scheduled_shutdown_cb(mainloop_event_t *ev, void *arg) +{ + (void)ev; + (void)arg; + log_notice(LD_GENERAL, "Clean shutdown finished. Exiting."); + tor_shutdown_event_loop_and_exit(0); +} + +/** Schedule the mainloop to exit after <b>delay_sec</b> seconds. */ +void +mainloop_schedule_shutdown(int delay_sec) +{ + const struct timeval delay_tv = { delay_sec, 0 }; + if (! scheduled_shutdown_ev) { + scheduled_shutdown_ev = mainloop_event_new(scheduled_shutdown_cb, NULL); + } + mainloop_event_schedule(scheduled_shutdown_ev, &delay_tv); +} + #define LONGEST_TIMER_PERIOD (30 * 86400) /** Helper: Return the number of seconds between <b>now</b> and <b>next</b>, * clipped to the range [1 second, LONGEST_TIMER_PERIOD]. */ @@ -1732,16 +1807,16 @@ safe_timer_diff(time_t now, time_t next) } /** Perform regular maintenance tasks. This function gets run once per - * second by second_elapsed_callback(). + * second. */ -static void -run_scheduled_events(time_t now) +static int +second_elapsed_callback(time_t now, const or_options_t *options) { - const or_options_t *options = get_options(); - - /* 0. See if we've been asked to shut down and our timeout has - * expired; or if our bandwidth limits are exhausted and we - * should hibernate; or if it's time to wake up from hibernation. + /* 0. See if our bandwidth limits are exhausted and we should hibernate + * + * Note: we have redundant mechanisms to handle the case where it's + * time to wake up from hibernation; or where we have a scheduled + * shutdown and it's time to run it, but this will also handle those. */ consider_hibernation(now); @@ -1751,10 +1826,13 @@ run_scheduled_events(time_t now) if (options->UseBridges && !net_is_disabled()) { /* Note: this check uses net_is_disabled(), not should_delay_dir_fetches() * -- the latter is only for fetching consensus-derived directory info. */ + // TODO: client + // Also, schedule this rather than probing 1x / sec fetch_bridge_descriptors(options, now); } if (accounting_is_enabled(options)) { + // TODO: refactor or rewrite? accounting_run_housekeeping(now); } @@ -1765,6 +1843,7 @@ run_scheduled_events(time_t now) */ /* (If our circuit build timeout can ever become lower than a second (which * it can't, currently), we should do this more often.) */ + // TODO: All expire stuff can become NET_PARTICIPANT, RUN_ON_DISABLE circuit_expire_building(); circuit_expire_waiting_for_better_guard(); @@ -1798,9 +1877,8 @@ run_scheduled_events(time_t now) run_connection_housekeeping(i, now); } - /* 11b. check pending unconfigured managed proxies */ - if (!net_is_disabled() && pt_proxies_configuration_pending()) - pt_configure_remaining_proxies(); + /* Run again in a second. */ + return 1; } /* Periodic callback: rotate the onion keys after the period defined by the @@ -1947,6 +2025,55 @@ add_entropy_callback(time_t now, const or_options_t *options) return ENTROPY_INTERVAL; } +/** Periodic callback: if there has been no network usage in a while, + * enter a dormant state. */ +STATIC int +check_network_participation_callback(time_t now, const or_options_t *options) +{ + /* If we're a server, we can't become dormant. */ + if (server_mode(options)) { + goto found_activity; + } + + /* If we're running an onion service, we can't become dormant. */ + /* XXXX this would be nice to change, so that we can be dormant with a + * service. */ + if (hs_service_get_num_services() || rend_num_services()) { + goto found_activity; + } + + /* If we have any currently open entry streams other than "linked" + * connections used for directory requests, those count as user activity. + */ + if (options->DormantTimeoutDisabledByIdleStreams) { + if (connection_get_by_type_nonlinked(CONN_TYPE_AP) != NULL) { + goto found_activity; + } + } + + /* XXXX Make this configurable? */ +/** How often do we check whether we have had network activity? */ +#define CHECK_PARTICIPATION_INTERVAL (5*60) + + /* Become dormant if there has been no user activity in a long time. + * (The funny checks below are in order to prevent overflow.) */ + time_t time_since_last_activity = 0; + if (get_last_user_activity_time() < now) + time_since_last_activity = now - get_last_user_activity_time(); + if (time_since_last_activity >= options->DormantClientTimeout) { + log_notice(LD_GENERAL, "No user activity in a long time: becoming" + " dormant."); + set_network_participation(false); + rescan_periodic_events(options); + } + + return CHECK_PARTICIPATION_INTERVAL; + + found_activity: + note_user_activity(now); + return CHECK_PARTICIPATION_INTERVAL; +} + /** * Periodic callback: if we're an authority, make sure we test * the routers on the network for reachability. @@ -2053,11 +2180,9 @@ check_expired_networkstatus_callback(time_t now, const or_options_t *options) (void)options; /* Check whether our networkstatus has expired. */ networkstatus_t *ns = networkstatus_get_latest_consensus(); - /*XXXX RD: This value needs to be the same as REASONABLY_LIVE_TIME in - * networkstatus_get_reasonably_live_consensus(), but that value is way - * way too high. Arma: is the bridge issue there resolved yet? -NM */ -#define NS_EXPIRY_SLOP (24*60*60) - if (ns && ns->valid_until < (now - NS_EXPIRY_SLOP) && + /* Use reasonably live consensuses until they are no longer reasonably live. + */ + if (ns && !networkstatus_consensus_reasonably_live(ns, now) && router_have_minimum_dir_info()) { router_dir_info_changed(); } @@ -2240,6 +2365,27 @@ retry_dns_callback(time_t now, const or_options_t *options) return RETRY_DNS_INTERVAL; } +/** + * Periodic callback: prune routerlist of old information about Tor network. + */ +static int +prune_old_routers_callback(time_t now, const or_options_t *options) +{ +#define ROUTERLIST_PRUNING_INTERVAL (60*60) // 1 hour. + (void)now; + (void)options; + + if (!net_is_disabled()) { + /* If any networkstatus documents are no longer recent, we need to + * update all the descriptors' running status. */ + /* Remove dead routers. */ + log_debug(LD_GENERAL, "Pruning routerlist..."); + routerlist_remove_old_routers(); + } + + return ROUTERLIST_PRUNING_INTERVAL; +} + /** Periodic callback: consider rebuilding or and re-uploading our descriptor * (if we've passed our internal checks). */ static int @@ -2259,12 +2405,6 @@ check_descriptor_callback(time_t now, const or_options_t *options) check_descriptor_ipaddress_changed(now); mark_my_descriptor_dirty_if_too_old(now); consider_publishable_server(0); - /* If any networkstatus documents are no longer recent, we need to - * update all the descriptors' running status. */ - /* Remove dead routers. */ - /* XXXX This doesn't belong here, but it was here in the pre- - * XXXX refactoring code. */ - routerlist_remove_old_routers(); } return CHECK_DESCRIPTOR_INTERVAL; @@ -2522,36 +2662,19 @@ hs_service_callback(time_t now, const or_options_t *options) return 1; } -/** Timer: used to invoke second_elapsed_callback() once per second. */ -static periodic_timer_t *second_timer = NULL; - -/** - * Enable or disable the per-second timer as appropriate, creating it if - * necessary. +/* + * Periodic callback: Send once-per-second events to the controller(s). + * This is called every second. */ -void -reschedule_per_second_timer(void) +static int +control_per_second_events_callback(time_t now, const or_options_t *options) { - struct timeval one_second; - one_second.tv_sec = 1; - one_second.tv_usec = 0; - - if (! second_timer) { - second_timer = periodic_timer_new(tor_libevent_get_base(), - &one_second, - second_elapsed_callback, - NULL); - tor_assert(second_timer); - } + (void) options; + (void) now; - const bool run_per_second_events = - control_any_per_second_event_enabled() || ! net_is_completely_disabled(); + control_per_second_events(); - if (run_per_second_events) { - periodic_timer_launch(second_timer, &one_second); - } else { - periodic_timer_disable(second_timer); - } + return 1; } /** Last time that update_current_time was called. */ @@ -2581,6 +2704,17 @@ update_current_time(time_t now) memcpy(&last_updated, ¤t_second_last_changed, sizeof(last_updated)); monotime_coarse_get(¤t_second_last_changed); + /** How much clock jumping means that we should adjust our idea of when + * to go dormant? */ +#define NUM_JUMPED_SECONDS_BEFORE_NETSTATUS_UPDATE 20 + + /* Don't go dormant early or late just because we jumped in time. */ + if (ABS(seconds_elapsed) >= NUM_JUMPED_SECONDS_BEFORE_NETSTATUS_UPDATE) { + if (is_participating_on_network()) { + netstatus_note_clock_jumped(seconds_elapsed); + } + } + /** How much clock jumping do we tolerate? */ #define NUM_JUMPED_SECONDS_BEFORE_WARN 100 @@ -2590,6 +2724,7 @@ update_current_time(time_t now) if (seconds_elapsed < -NUM_JUMPED_SECONDS_BEFORE_WARN) { // moving back in time is always a bad sign. circuit_note_clock_jumped(seconds_elapsed, false); + } else if (seconds_elapsed >= NUM_JUMPED_SECONDS_BEFORE_WARN) { /* Compare the monotonic clock to the result of time(). */ const int32_t monotime_msec_passed = @@ -2619,31 +2754,6 @@ update_current_time(time_t now) current_second = now; } -/** Libevent callback: invoked once every second. */ -static void -second_elapsed_callback(periodic_timer_t *timer, void *arg) -{ - /* XXXX This could be sensibly refactored into multiple callbacks, and we - * could use Libevent's timers for this rather than checking the current - * time against a bunch of timeouts every second. */ - time_t now; - (void)timer; - (void)arg; - - now = time(NULL); - - /* We don't need to do this once-per-second any more: time-updating is - * only in this callback _because it is a callback_. It should be fine - * to disable this callback, and the time will still get updated. - */ - update_current_time(now); - - /* Maybe some controller events are ready to fire */ - control_per_second_events(); - - run_scheduled_events(now); -} - #ifdef HAVE_SYSTEMD_209 static periodic_timer_t *systemd_watchdog_timer = NULL; @@ -2734,9 +2844,6 @@ do_main_loop(void) initialize_periodic_events(); initialize_mainloop_events(); - /* set up once-a-second callback. */ - reschedule_per_second_timer(); - #ifdef HAVE_SYSTEMD_209 uint64_t watchdog_delay; /* set up systemd watchdog notification. */ @@ -2758,10 +2865,6 @@ do_main_loop(void) } } #endif /* defined(HAVE_SYSTEMD_209) */ - - main_loop_should_exit = 0; - main_loop_exit_value = 0; - #ifdef ENABLE_RESTART_DEBUGGING { static int first_time = 1; @@ -2887,10 +2990,14 @@ run_main_loop_once(void) * * Shadow won't invoke this function, so don't fill it up with things. */ -static int +STATIC int run_main_loop_until_done(void) { int loop_result = 1; + + main_loop_should_exit = 0; + main_loop_exit_value = 0; + do { loop_result = run_main_loop_once(); } while (loop_result == 1); @@ -2921,7 +3028,6 @@ tor_mainloop_free_all(void) smartlist_free(connection_array); smartlist_free(closeable_connection_lst); smartlist_free(active_linked_connection_lst); - periodic_timer_free(second_timer); teardown_periodic_events(); tor_event_free(shutdown_did_not_work_event); tor_event_free(initialize_periodic_events_event); @@ -2929,6 +3035,8 @@ tor_mainloop_free_all(void) mainloop_event_free(schedule_active_linked_connections_event); mainloop_event_free(postloop_cleanup_ev); mainloop_event_free(handle_deferred_signewnym_ev); + mainloop_event_free(scheduled_shutdown_ev); + mainloop_event_free(rescan_periodic_events_ev); #ifdef HAVE_SYSTEMD_209 periodic_timer_free(systemd_watchdog_timer); diff --git a/src/core/mainloop/mainloop.h b/src/core/mainloop/mainloop.h index c5669fc4e0..6ed93fa900 100644 --- a/src/core/mainloop/mainloop.h +++ b/src/core/mainloop/mainloop.h @@ -65,6 +65,7 @@ void reschedule_or_state_save(void); void reschedule_dirvote(const or_options_t *options); void mainloop_schedule_postloop_cleanup(void); void rescan_periodic_events(const or_options_t *options); +MOCK_DECL(void, schedule_rescan_periodic_events,(void)); void update_current_time(time_t now); @@ -81,11 +82,12 @@ uint64_t get_main_loop_error_count(void); uint64_t get_main_loop_idle_count(void); void periodic_events_on_new_options(const or_options_t *options); -void reschedule_per_second_timer(void); void do_signewnym(time_t); time_t get_last_signewnym_time(void); +void mainloop_schedule_shutdown(int delay_sec); + void tor_init_connection_lists(void); void initialize_mainloop_events(void); void tor_mainloop_free_all(void); @@ -98,10 +100,14 @@ extern struct token_bucket_rw_t global_bucket; extern struct token_bucket_rw_t global_relayed_bucket; #ifdef MAINLOOP_PRIVATE +STATIC int run_main_loop_until_done(void); STATIC void close_closeable_connections(void); STATIC void initialize_periodic_events(void); STATIC void teardown_periodic_events(void); STATIC int get_my_roles(const or_options_t *); +STATIC int check_network_participation_callback(time_t now, + const or_options_t *options); + #ifdef TOR_UNIT_TESTS extern smartlist_t *connection_array; diff --git a/src/core/mainloop/netstatus.c b/src/core/mainloop/netstatus.c index 1444ca5db2..4924888598 100644 --- a/src/core/mainloop/netstatus.c +++ b/src/core/mainloop/netstatus.c @@ -6,9 +6,12 @@ #include "core/or/or.h" #include "core/mainloop/netstatus.h" +#include "core/mainloop/mainloop.h" #include "app/config/config.h" #include "feature/hibernate/hibernate.h" +#include "app/config/or_state_st.h" + /** Return true iff our network is in some sense disabled or shutting down: * either we're hibernating, entering hibernation, or the network is turned * off with DisableNetwork. */ @@ -26,3 +29,136 @@ net_is_completely_disabled(void) { return get_options()->DisableNetwork || we_are_fully_hibernating(); } + +/** + * The time at which we've last seen "user activity" -- that is, any activity + * that should keep us as a participant on the network. + * + * This is not actually the true time. We will adjust this forward if + * our clock jumps, or if Tor is shut down for a while, so that the time + * since our last activity remains as it was before the jump or shutdown. + */ +static time_t last_user_activity_seen = 0; + +/** + * True iff we are currently a "network participant" -- that is, we + * are building circuits, fetching directory information, and so on. + **/ +static bool participating_on_network = false; + +/** + * Record the fact that we have seen "user activity" at the time now. Move + * "last activity seen" time forwards, but never backwards. + * + * If we were previously not participating on the network, set our + * participation status to true, and launch periodic events as appropriate. + **/ +void +note_user_activity(time_t now) +{ + last_user_activity_seen = MAX(now, last_user_activity_seen); + + if (! participating_on_network) { + log_notice(LD_GENERAL, "Tor is no longer dormant."); + set_network_participation(true); + schedule_rescan_periodic_events(); + } +} + +/** + * Change the time at which "user activitiy" was last seen to <b>now</b>. + * + * Unlike note_user_actity, this function sets the time without checking + * whether it is in the past, and without causing any rescan of periodic events + * or change in participation status. + */ +void +reset_user_activity(time_t now) +{ + last_user_activity_seen = now; +} + +/** + * Return the most recent time at which we recorded "user activity". + **/ +time_t +get_last_user_activity_time(void) +{ + return last_user_activity_seen; +} + +/** + * Set the field that remembers whether we are currently participating on the + * network. Does not schedule or un-schedule periodic events. + **/ +void +set_network_participation(bool participation) +{ + participating_on_network = participation; +} + +/** + * Return true iff we are currently participating on the network. + **/ +bool +is_participating_on_network(void) +{ + return participating_on_network; +} + +/** + * Update 'state' with the last time at which we were active on the network. + **/ +void +netstatus_flush_to_state(or_state_t *state, time_t now) +{ + state->Dormant = ! participating_on_network; + if (participating_on_network) { + time_t sec_since_activity = MAX(0, now - last_user_activity_seen); + state->MinutesSinceUserActivity = (int)(sec_since_activity / 60); + } else { + state->MinutesSinceUserActivity = 0; + } +} + +/** + * Update our current view of network participation from an or_state_t object. + **/ +void +netstatus_load_from_state(const or_state_t *state, time_t now) +{ + time_t last_activity; + if (state->Dormant == -1) { // Initial setup. + if (get_options()->DormantOnFirstStartup) { + last_activity = 0; + participating_on_network = false; + } else { + // Start up as active, treat activity as happening now. + last_activity = now; + participating_on_network = true; + } + } else if (state->Dormant) { + last_activity = 0; + participating_on_network = false; + } else { + last_activity = now - 60 * state->MinutesSinceUserActivity; + participating_on_network = true; + } + if (get_options()->DormantCanceledByStartup) { + last_activity = now; + participating_on_network = true; + } + reset_user_activity(last_activity); +} + +/** + * Adjust the time at which the user was last active by <b>seconds_diff</b> + * in response to a clock jump. + */ +void +netstatus_note_clock_jumped(time_t seconds_diff) +{ + time_t last_active = get_last_user_activity_time(); + if (last_active) + reset_user_activity(last_active + seconds_diff); +} diff --git a/src/core/mainloop/netstatus.h b/src/core/mainloop/netstatus.h index ae8547b30d..aba631e2fb 100644 --- a/src/core/mainloop/netstatus.h +++ b/src/core/mainloop/netstatus.h @@ -10,4 +10,15 @@ int net_is_disabled(void); int net_is_completely_disabled(void); +void note_user_activity(time_t now); +void reset_user_activity(time_t now); +time_t get_last_user_activity_time(void); + +void set_network_participation(bool participation); +bool is_participating_on_network(void); + +void netstatus_flush_to_state(or_state_t *state, time_t now); +void netstatus_load_from_state(const or_state_t *state, time_t now); +void netstatus_note_clock_jumped(time_t seconds_diff); + #endif diff --git a/src/core/mainloop/periodic.c b/src/core/mainloop/periodic.c index 2651bbbc89..5c01213d17 100644 --- a/src/core/mainloop/periodic.c +++ b/src/core/mainloop/periodic.c @@ -45,10 +45,6 @@ periodic_event_dispatch(mainloop_event_t *ev, void *data) periodic_event_item_t *event = data; tor_assert(ev == event->ev); - if (BUG(!periodic_event_is_enabled(event))) { - return; - } - time_t now = time(NULL); update_current_time(now); const or_options_t *options = get_options(); @@ -57,7 +53,7 @@ periodic_event_dispatch(mainloop_event_t *ev, void *data) int next_interval = 0; if (!periodic_event_is_enabled(event)) { - /* The event got disabled from inside its callback; no need to + /* The event got disabled from inside its callback, or before: no need to * reschedule. */ return; } @@ -177,3 +173,19 @@ periodic_event_disable(periodic_event_item_t *event) mainloop_event_cancel(event->ev); event->enabled = 0; } + +/** + * Disable an event, then schedule it to run once. + * Do nothing if the event was already disabled. + */ +void +periodic_event_schedule_and_disable(periodic_event_item_t *event) +{ + tor_assert(event); + if (!periodic_event_is_enabled(event)) + return; + + periodic_event_disable(event); + + mainloop_event_activate(event->ev); +} diff --git a/src/core/mainloop/periodic.h b/src/core/mainloop/periodic.h index e49fafd174..344fc9ad25 100644 --- a/src/core/mainloop/periodic.h +++ b/src/core/mainloop/periodic.h @@ -15,6 +15,10 @@ #define PERIODIC_EVENT_ROLE_BRIDGEAUTH (1U << 4) #define PERIODIC_EVENT_ROLE_HS_SERVICE (1U << 5) #define PERIODIC_EVENT_ROLE_DIRSERVER (1U << 6) +#define PERIODIC_EVENT_ROLE_CONTROLEV (1U << 7) + +#define PERIODIC_EVENT_ROLE_NET_PARTICIPANT (1U << 8) +#define PERIODIC_EVENT_ROLE_ALL (1U << 9) /* Helper macro to make it a bit less annoying to defined groups of roles that * are often used. */ @@ -25,10 +29,6 @@ /* Authorities that is both bridge and directory. */ #define PERIODIC_EVENT_ROLE_AUTHORITIES \ (PERIODIC_EVENT_ROLE_BRIDGEAUTH | PERIODIC_EVENT_ROLE_DIRAUTH) -/* All roles. */ -#define PERIODIC_EVENT_ROLE_ALL \ - (PERIODIC_EVENT_ROLE_AUTHORITIES | PERIODIC_EVENT_ROLE_CLIENT | \ - PERIODIC_EVENT_ROLE_HS_SERVICE | PERIODIC_EVENT_ROLE_ROUTER) /* * Event flags which can change the behavior of an event. @@ -39,6 +39,11 @@ * the net_is_disabled() check. */ #define PERIODIC_EVENT_FLAG_NEED_NET (1U << 0) +/* Indicate that if the event is enabled, it needs to be run once before + * it becomes disabled. + */ +#define PERIODIC_EVENT_FLAG_RUN_ON_DISABLE (1U << 1) + /** Callback function for a periodic event to take action. The return value * influences the next time the function will get called. Return * PERIODIC_EVENT_NO_UPDATE to not update <b>last_action_time</b> and be polled @@ -83,6 +88,6 @@ void periodic_event_destroy(periodic_event_item_t *event); void periodic_event_reschedule(periodic_event_item_t *event); void periodic_event_enable(periodic_event_item_t *event); void periodic_event_disable(periodic_event_item_t *event); +void periodic_event_schedule_and_disable(periodic_event_item_t *event); #endif /* !defined(TOR_PERIODIC_H) */ - diff --git a/src/core/or/channeltls.c b/src/core/or/channeltls.c index 4db283d20e..d508c91988 100644 --- a/src/core/or/channeltls.c +++ b/src/core/or/channeltls.c @@ -59,6 +59,7 @@ #include "feature/nodelist/torcert.h" #include "feature/nodelist/networkstatus.h" #include "trunnel/channelpadding_negotiation.h" +#include "trunnel/netinfo.h" #include "core/or/channelpadding.h" #include "core/or/cell_st.h" @@ -949,7 +950,6 @@ channel_tls_listener_describe_transport_method(channel_listener_t *chan_l) void channel_tls_handle_state_change_on_orconn(channel_tls_t *chan, or_connection_t *conn, - uint8_t old_state, uint8_t state) { channel_t *base_chan; @@ -958,8 +958,6 @@ channel_tls_handle_state_change_on_orconn(channel_tls_t *chan, tor_assert(conn); tor_assert(conn->chan == chan); tor_assert(chan->conn == conn); - /* Shut the compiler up without triggering -Wtautological-compare */ - (void)old_state; base_chan = TLS_CHAN_TO_BASE(chan); @@ -1645,6 +1643,35 @@ channel_tls_process_padding_negotiate_cell(cell_t *cell, channel_tls_t *chan) } /** + * Convert <b>netinfo_addr</b> into corresponding <b>tor_addr</b>. + * Return 0 on success; on failure, return -1 and log a warning. + */ +static int +tor_addr_from_netinfo_addr(tor_addr_t *tor_addr, + const netinfo_addr_t *netinfo_addr) { + tor_assert(tor_addr); + tor_assert(netinfo_addr); + + uint8_t type = netinfo_addr_get_addr_type(netinfo_addr); + uint8_t len = netinfo_addr_get_len(netinfo_addr); + + if (type == NETINFO_ADDR_TYPE_IPV4 && len == 4) { + uint32_t ipv4 = netinfo_addr_get_addr_ipv4(netinfo_addr); + tor_addr_from_ipv4h(tor_addr, ipv4); + } else if (type == NETINFO_ADDR_TYPE_IPV6 && len == 16) { + const uint8_t *ipv6_bytes = netinfo_addr_getconstarray_addr_ipv6( + netinfo_addr); + tor_addr_from_ipv6_bytes(tor_addr, (const char *)ipv6_bytes); + } else { + log_fn(LOG_PROTOCOL_WARN, LD_OR, "Cannot read address from NETINFO " + "- wrong type/length."); + return -1; + } + + return 0; +} + +/** * Helper: compute the absolute value of a time_t. * * (we need this because labs() doesn't always work for time_t, since @@ -1668,8 +1695,6 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan) time_t timestamp; uint8_t my_addr_type; uint8_t my_addr_len; - const uint8_t *my_addr_ptr; - const uint8_t *cp, *end; uint8_t n_other_addrs; time_t now = time(NULL); const routerinfo_t *me = router_get_my_routerinfo(); @@ -1740,38 +1765,48 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan) } /* Decode the cell. */ - timestamp = ntohl(get_uint32(cell->payload)); - const time_t sent_versions_at = - chan->conn->handshake_state->sent_versions_at; - if (now > sent_versions_at && (now - sent_versions_at) < 180) { - /* If we have gotten the NETINFO cell reasonably soon after having - * sent our VERSIONS cell, maybe we can learn skew information from it. */ - apparent_skew = now - timestamp; + netinfo_cell_t *netinfo_cell = NULL; + + ssize_t parsed = netinfo_cell_parse(&netinfo_cell, cell->payload, + CELL_PAYLOAD_SIZE); + + if (parsed < 0) { + log_fn(LOG_PROTOCOL_WARN, LD_OR, + "Failed to parse NETINFO cell - closing connection."); + connection_or_close_for_error(chan->conn, 0); + return; } - my_addr_type = (uint8_t) cell->payload[4]; - my_addr_len = (uint8_t) cell->payload[5]; - my_addr_ptr = (uint8_t*) cell->payload + 6; - end = cell->payload + CELL_PAYLOAD_SIZE; - cp = cell->payload + 6 + my_addr_len; + timestamp = netinfo_cell_get_timestamp(netinfo_cell); + const netinfo_addr_t *my_addr = + netinfo_cell_getconst_other_addr(netinfo_cell); + + my_addr_type = netinfo_addr_get_addr_type(my_addr); + my_addr_len = netinfo_addr_get_len(my_addr); + + if ((now - chan->conn->handshake_state->sent_versions_at) < 180) { + apparent_skew = now - timestamp; + } /* We used to check: * if (my_addr_len >= CELL_PAYLOAD_SIZE - 6) { * * This is actually never going to happen, since my_addr_len is at most 255, * and CELL_PAYLOAD_LEN - 6 is 503. So we know that cp is < end. */ - if (my_addr_type == RESOLVED_TYPE_IPV4 && my_addr_len == 4) { - tor_addr_from_ipv4n(&my_apparent_addr, get_uint32(my_addr_ptr)); + if (tor_addr_from_netinfo_addr(&my_apparent_addr, my_addr) == -1) { + connection_or_close_for_error(chan->conn, 0); + netinfo_cell_free(netinfo_cell); + return; + } + if (my_addr_type == NETINFO_ADDR_TYPE_IPV4 && my_addr_len == 4) { if (!get_options()->BridgeRelay && me && - get_uint32(my_addr_ptr) == htonl(me->addr)) { + tor_addr_eq_ipv4h(&my_apparent_addr, me->addr)) { TLS_CHAN_TO_BASE(chan)->is_canonical_to_peer = 1; } - - } else if (my_addr_type == RESOLVED_TYPE_IPV6 && my_addr_len == 16) { - tor_addr_from_ipv6_bytes(&my_apparent_addr, (const char *) my_addr_ptr); - + } else if (my_addr_type == NETINFO_ADDR_TYPE_IPV6 && + my_addr_len == 16) { if (!get_options()->BridgeRelay && me && !tor_addr_is_null(&me->ipv6_addr) && tor_addr_eq(&my_apparent_addr, &me->ipv6_addr)) { @@ -1779,18 +1814,20 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan) } } - n_other_addrs = (uint8_t) *cp++; - while (n_other_addrs && cp < end-2) { + n_other_addrs = netinfo_cell_get_n_my_addrs(netinfo_cell); + for (uint8_t i = 0; i < n_other_addrs; i++) { /* Consider all the other addresses; if any matches, this connection is * "canonical." */ + + const netinfo_addr_t *netinfo_addr = + netinfo_cell_getconst_my_addrs(netinfo_cell, i); + tor_addr_t addr; - const uint8_t *next = - decode_address_from_payload(&addr, cp, (int)(end-cp)); - if (next == NULL) { + + if (tor_addr_from_netinfo_addr(&addr, netinfo_addr) == -1) { log_fn(LOG_PROTOCOL_WARN, LD_OR, - "Bad address in netinfo cell; closing connection."); - connection_or_close_for_error(chan->conn, 0); - return; + "Bad address in netinfo cell; Skipping."); + continue; } /* A relay can connect from anywhere and be canonical, so * long as it tells you from where it came. This may sound a bit @@ -1803,10 +1840,10 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan) connection_or_set_canonical(chan->conn, 1); break; } - cp = next; - --n_other_addrs; } + netinfo_cell_free(netinfo_cell); + if (me && !TLS_CHAN_TO_BASE(chan)->is_canonical_to_peer && channel_is_canonical(TLS_CHAN_TO_BASE(chan))) { const char *descr = diff --git a/src/core/or/channeltls.h b/src/core/or/channeltls.h index a2648ff537..634a2a00e9 100644 --- a/src/core/or/channeltls.h +++ b/src/core/or/channeltls.h @@ -49,7 +49,6 @@ channel_tls_t * channel_tls_from_base(channel_t *chan); void channel_tls_handle_cell(cell_t *cell, or_connection_t *conn); void channel_tls_handle_state_change_on_orconn(channel_tls_t *chan, or_connection_t *conn, - uint8_t old_state, uint8_t state); void channel_tls_handle_var_cell(var_cell_t *var_cell, or_connection_t *conn); diff --git a/src/core/or/circuit_st.h b/src/core/or/circuit_st.h index d4339ff50d..af343f082e 100644 --- a/src/core/or/circuit_st.h +++ b/src/core/or/circuit_st.h @@ -12,6 +12,11 @@ #include "core/or/cell_queue_st.h" struct hs_token_t; +struct circpad_machine_spec_t; +struct circpad_machine_state_t; + +/** Number of padding state machines on a circuit. */ +#define CIRCPAD_MAX_MACHINES (2) /** "magic" value for an origin_circuit_t */ #define ORIGIN_CIRCUIT_MAGIC 0x35315243u @@ -177,6 +182,27 @@ struct circuit_t { /** Hashtable node: used to look up the circuit by its HS token using the HS circuitmap. */ HT_ENTRY(circuit_t) hs_circuitmap_node; + + /** Adaptive Padding state machines: these are immutable. The state machines + * that come from the consensus are saved to a global structure, to avoid + * per-circuit allocations. This merely points to the global copy in + * origin_padding_machines or relay_padding_machines that should never + * change or get deallocated. + * + * Each element of this array corresponds to a different padding machine, + * and we can have up to CIRCPAD_MAX_MACHINES such machines. */ + const struct circpad_machine_spec_t *padding_machine[CIRCPAD_MAX_MACHINES]; + + /** Adaptive Padding machine info for above machines. This is the + * per-circuit mutable information, such as the current state and + * histogram token counts. Some of it is optional (aka NULL). + * If a machine is being shut down, these indexes can be NULL + * without the corresponding padding_machine being NULL, while we + * wait for the other end to respond to our shutdown request. + * + * Each element of this array corresponds to a different padding machine, + * and we can have up to CIRCPAD_MAX_MACHINES such machines. */ + struct circpad_machine_state_t *padding_info[CIRCPAD_MAX_MACHINES]; }; #endif diff --git a/src/core/or/circuitbuild.c b/src/core/or/circuitbuild.c index 21369fb538..3ec1e01f11 100644 --- a/src/core/or/circuitbuild.c +++ b/src/core/or/circuitbuild.c @@ -26,6 +26,7 @@ **/ #define CIRCUITBUILD_PRIVATE +#define OCIRC_EVENT_PRIVATE #include "core/or/or.h" #include "app/config/config.h" @@ -42,10 +43,12 @@ #include "core/or/circuitlist.h" #include "core/or/circuitstats.h" #include "core/or/circuituse.h" +#include "core/or/circuitpadding.h" #include "core/or/command.h" #include "core/or/connection_edge.h" #include "core/or/connection_or.h" #include "core/or/onion.h" +#include "core/or/ocirc_event.h" #include "core/or/policies.h" #include "core/or/relay.h" #include "feature/client/bridges.h" @@ -492,7 +495,7 @@ circuit_establish_circuit(uint8_t purpose, extend_info_t *exit_ei, int flags) return NULL; } - control_event_circuit_status(circ, CIRC_EVENT_LAUNCHED, 0); + circuit_event_status(circ, CIRC_EVENT_LAUNCHED, 0); if ((err_reason = circuit_handle_first_hop(circ)) < 0) { circuit_mark_for_close(TO_CIRCUIT(circ), -err_reason); @@ -508,6 +511,28 @@ origin_circuit_get_guard_state(origin_circuit_t *circ) return circ->guard_state; } +/** + * Helper function to publish a channel association message + * + * circuit_handle_first_hop() calls this to notify subscribers about a + * channel launch event, which associates a circuit with a channel. + * This doesn't always correspond to an assignment of the circuit's + * n_chan field, because that seems to be only for fully-open + * channels. + **/ +static void +circuit_chan_publish(const origin_circuit_t *circ, const channel_t *chan) +{ + ocirc_event_msg_t msg; + + msg.type = OCIRC_MSGTYPE_CHAN; + msg.u.chan.gid = circ->global_identifier; + msg.u.chan.chan = chan->global_identifier; + msg.u.chan.onehop = circ->build_state->onehop_tunnel; + + ocirc_event_publish(&msg); +} + /** Start establishing the first hop of our circuit. Figure out what * OR we should connect to, and if necessary start the connection to * it. If we're already connected, then send the 'create' cell. @@ -559,8 +584,6 @@ circuit_handle_first_hop(origin_circuit_t *circ) circ->base_.n_hop = extend_info_dup(firsthop->extend_info); if (should_launch) { - if (circ->build_state->onehop_tunnel) - control_event_bootstrap(BOOTSTRAP_STATUS_CONN_DIR, 0); n_chan = channel_connect_for_circuit( &firsthop->extend_info->addr, firsthop->extend_info->port, @@ -570,6 +593,7 @@ circuit_handle_first_hop(origin_circuit_t *circ) log_info(LD_CIRC,"connect to firsthop failed. Closing."); return -END_CIRC_REASON_CONNECTFAILED; } + circuit_chan_publish(circ, n_chan); } log_debug(LD_CIRC,"connecting in progress (or finished). Good."); @@ -581,6 +605,7 @@ circuit_handle_first_hop(origin_circuit_t *circ) } else { /* it's already open. use it. */ tor_assert(!circ->base_.n_hop); circ->base_.n_chan = n_chan; + circuit_chan_publish(circ, n_chan); log_debug(LD_CIRC,"Conn open. Delivering first onion skin."); if ((err_reason = circuit_send_next_onion_skin(circ)) < 0) { log_info(LD_CIRC,"circuit_send_next_onion_skin failed."); @@ -926,12 +951,15 @@ circuit_send_next_onion_skin(origin_circuit_t *circ) crypt_path_t *hop = onion_next_hop_in_cpath(circ->cpath); circuit_build_times_handle_completed_hop(circ); + circpad_machine_event_circ_added_hop(circ); + if (hop) { /* Case two: we're on a hop after the first. */ return circuit_send_intermediate_onion_skin(circ, hop); } /* Case three: the circuit is finished. Do housekeeping tasks on it. */ + circpad_machine_event_circ_built(circ); return circuit_build_no_more_hops(circ); } @@ -1416,7 +1444,7 @@ circuit_finish_handshake(origin_circuit_t *circ, hop->state = CPATH_STATE_OPEN; log_info(LD_CIRC,"Finished building circuit hop:"); circuit_log_path(LOG_INFO,LD_CIRC,circ); - control_event_circuit_status(circ, CIRC_EVENT_EXTENDED, 0); + circuit_event_status(circ, CIRC_EVENT_EXTENDED, 0); return 0; } @@ -1657,22 +1685,25 @@ route_len_for_purpose(uint8_t purpose, extend_info_t *exit_ei) STATIC int new_route_len(uint8_t purpose, extend_info_t *exit_ei, smartlist_t *nodes) { - int num_acceptable_routers; int routelen; tor_assert(nodes); routelen = route_len_for_purpose(purpose, exit_ei); - num_acceptable_routers = count_acceptable_nodes(nodes); + int num_acceptable_direct = count_acceptable_nodes(nodes, 1); + int num_acceptable_indirect = count_acceptable_nodes(nodes, 0); - log_debug(LD_CIRC,"Chosen route length %d (%d/%d routers suitable).", - routelen, num_acceptable_routers, smartlist_len(nodes)); + log_debug(LD_CIRC,"Chosen route length %d (%d direct and %d indirect " + "routers suitable).", routelen, num_acceptable_direct, + num_acceptable_indirect); - if (num_acceptable_routers < routelen) { + if (num_acceptable_direct < 1 || num_acceptable_indirect < routelen - 1) { log_info(LD_CIRC, - "Not enough acceptable routers (%d/%d). Discarding this circuit.", - num_acceptable_routers, routelen); + "Not enough acceptable routers (%d/%d direct and %d/%d " + "indirect routers suitable). Discarding this circuit.", + num_acceptable_direct, routelen, + num_acceptable_indirect, routelen); return -1; } @@ -2314,7 +2345,7 @@ circuit_extend_to_new_exit(origin_circuit_t *circ, extend_info_t *exit_ei) * particular router. See bug #25885.) */ MOCK_IMPL(STATIC int, -count_acceptable_nodes, (smartlist_t *nodes)) +count_acceptable_nodes, (smartlist_t *nodes, int direct)) { int num=0; @@ -2328,7 +2359,7 @@ count_acceptable_nodes, (smartlist_t *nodes)) if (! node->is_valid) // log_debug(LD_CIRC,"Nope, the directory says %d is not valid.",i); continue; - if (! node_has_any_descriptor(node)) + if (! node_has_preferred_descriptor(node, direct)) continue; /* The node has a descriptor, so we can just check the ntor key directly */ if (!node_has_curve25519_onion_key(node)) @@ -2579,7 +2610,24 @@ choose_good_middle_server(uint8_t purpose, return choice; } - choice = router_choose_random_node(excluded, options->ExcludeNodes, flags); + if (options->MiddleNodes) { + smartlist_t *sl = smartlist_new(); + routerset_get_all_nodes(sl, options->MiddleNodes, + options->ExcludeNodes, 1); + + smartlist_subtract(sl, excluded); + + choice = node_sl_choose_by_bandwidth(sl, WEIGHT_FOR_MID); + smartlist_free(sl); + if (choice) { + log_fn(LOG_INFO, LD_CIRC, "Chose fixed middle node: %s", + hex_str(choice->identity, DIGEST_LEN)); + } else { + log_fn(LOG_NOTICE, LD_CIRC, "Restricted middle not available"); + } + } else { + choice = router_choose_random_node(excluded, options->ExcludeNodes, flags); + } smartlist_free(excluded); return choice; } diff --git a/src/core/or/circuitbuild.h b/src/core/or/circuitbuild.h index 969f5b5cc3..b19bc41235 100644 --- a/src/core/or/circuitbuild.h +++ b/src/core/or/circuitbuild.h @@ -84,7 +84,8 @@ void circuit_upgrade_circuits_from_guard_wait(void); STATIC circid_t get_unique_circ_id_by_chan(channel_t *chan); STATIC int new_route_len(uint8_t purpose, extend_info_t *exit_ei, smartlist_t *nodes); -MOCK_DECL(STATIC int, count_acceptable_nodes, (smartlist_t *nodes)); +MOCK_DECL(STATIC int, count_acceptable_nodes, (smartlist_t *nodes, + int direct)); STATIC int onion_extend_cpath(origin_circuit_t *circ); diff --git a/src/core/or/circuitlist.c b/src/core/or/circuitlist.c index 145004c71d..6b5f30e418 100644 --- a/src/core/or/circuitlist.c +++ b/src/core/or/circuitlist.c @@ -51,6 +51,7 @@ * logic, which was originally circuit-focused. **/ #define CIRCUITLIST_PRIVATE +#define OCIRC_EVENT_PRIVATE #include "lib/cc/torint.h" /* TOR_PRIuSZ */ #include "core/or/or.h" @@ -61,6 +62,7 @@ #include "core/or/circuitlist.h" #include "core/or/circuituse.h" #include "core/or/circuitstats.h" +#include "core/or/circuitpadding.h" #include "core/mainloop/connection.h" #include "app/config/config.h" #include "core/or/connection_edge.h" @@ -94,7 +96,10 @@ #include "lib/compress/compress_lzma.h" #include "lib/compress/compress_zlib.h" #include "lib/compress/compress_zstd.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" + +#define OCIRC_EVENT_PRIVATE +#include "core/or/ocirc_event.h" #include "ht.h" @@ -481,6 +486,56 @@ circuit_set_n_circid_chan(circuit_t *circ, circid_t id, } } +/** + * Helper function to publish a message about events on an origin circuit + * + * Publishes a message to subscribers of origin circuit events, and + * sends the control event. + **/ +int +circuit_event_status(origin_circuit_t *circ, circuit_status_event_t tp, + int reason_code) +{ + ocirc_event_msg_t msg; + + tor_assert(circ); + + msg.type = OCIRC_MSGTYPE_CEVENT; + msg.u.cevent.gid = circ->global_identifier; + msg.u.cevent.evtype = tp; + msg.u.cevent.reason = reason_code; + msg.u.cevent.onehop = circ->build_state->onehop_tunnel; + + ocirc_event_publish(&msg); + return control_event_circuit_status(circ, tp, reason_code); +} + +/** + * Helper function to publish a state change message + * + * circuit_set_state() calls this to notify subscribers about a change + * of the state of an origin circuit. + **/ +static void +circuit_state_publish(const circuit_t *circ) +{ + ocirc_event_msg_t msg; + const origin_circuit_t *ocirc; + + if (!CIRCUIT_IS_ORIGIN(circ)) + return; + ocirc = CONST_TO_ORIGIN_CIRCUIT(circ); + /* Only inbound OR circuits can be in this state, not origin circuits. */ + tor_assert(circ->state != CIRCUIT_STATE_ONIONSKIN_PENDING); + + msg.type = OCIRC_MSGTYPE_STATE; + msg.u.state.gid = ocirc->global_identifier; + msg.u.state.state = circ->state; + msg.u.state.onehop = ocirc->build_state->onehop_tunnel; + + ocirc_event_publish(&msg); +} + /** Change the state of <b>circ</b> to <b>state</b>, adding it to or removing * it from lists as appropriate. */ void @@ -510,6 +565,7 @@ circuit_set_state(circuit_t *circ, uint8_t state) if (state == CIRCUIT_STATE_GUARD_WAIT || state == CIRCUIT_STATE_OPEN) tor_assert(!circ->n_chan_create_cell); circ->state = state; + circuit_state_publish(circ); } /** Append to <b>out</b> all circuits in state CHAN_WAIT waiting for @@ -1176,6 +1232,9 @@ circuit_free_(circuit_t *circ) CIRCUIT_IS_ORIGIN(circ) ? TO_ORIGIN_CIRCUIT(circ)->global_identifier : 0); + /* Free any circuit padding structures */ + circpad_circuit_free_all_machineinfos(circ); + if (should_free) { memwipe(mem, 0xAA, memlen); /* poison memory */ tor_free(mem); @@ -2270,7 +2329,7 @@ circuit_about_to_free(circuit_t *circ) smartlist_remove(circuits_pending_other_guards, circ); } if (CIRCUIT_IS_ORIGIN(circ)) { - control_event_circuit_status(TO_ORIGIN_CIRCUIT(circ), + circuit_event_status(TO_ORIGIN_CIRCUIT(circ), (circ->state == CIRCUIT_STATE_OPEN || circ->state == CIRCUIT_STATE_GUARD_WAIT) ? CIRC_EVENT_CLOSED:CIRC_EVENT_FAILED, diff --git a/src/core/or/circuitlist.h b/src/core/or/circuitlist.h index b87c6a3667..f34f4ed6b7 100644 --- a/src/core/or/circuitlist.h +++ b/src/core/or/circuitlist.h @@ -14,6 +14,7 @@ #include "lib/testsupport/testsupport.h" #include "feature/hs/hs_ident.h" +#include "core/or/ocirc_event.h" /** Circuit state: I'm the origin, still haven't done all my handshakes. */ #define CIRCUIT_STATE_BUILDING 0 @@ -184,6 +185,8 @@ void channel_mark_circid_unusable(channel_t *chan, circid_t id); void channel_mark_circid_usable(channel_t *chan, circid_t id); time_t circuit_id_when_marked_unusable_on_channel(circid_t circ_id, channel_t *chan); +int circuit_event_status(origin_circuit_t *circ, circuit_status_event_t tp, + int reason_code); void circuit_set_state(circuit_t *circ, uint8_t state); void circuit_close_all_marked(void); int32_t circuit_initial_package_window(void); diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c new file mode 100644 index 0000000000..aa38b0ffc3 --- /dev/null +++ b/src/core/or/circuitpadding.c @@ -0,0 +1,2589 @@ +/* Copyright (c) 2017 The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file circuitpadding.c + * \brief Circuit-level padding implementation + * + * \details + * + * This file implements Tor proposal 254 "Padding Negotiation" which is heavily + * inspired by the paper "Toward an Efficient Website Fingerprinting Defense" + * by M. Juarez, M. Imani, M. Perry, C. Diaz, M. Wright. + * + * In particular the code in this file describes mechanisms for clients to + * negotiate various types of circuit-level padding from relays. + * + * Each padding type is described by a state machine (circpad_machine_spec_t), + * which is also referred as a "padding machine" in this file. Currently, + * these state machines are hardcoded in the source code (e.g. see + * circpad_circ_client_machine_init()), but in the future we will be able to + * serialize them in the torrc or the consensus. + * + * As specified by prop#254, clients can negotiate padding with relays by using + * PADDING_NEGOTIATE cells. After successful padding negotiation, padding + * machines are assigned to the circuit in their mutable form as a + * circpad_machine_state_t. + * + * Each state of a padding state machine can be either: + * - A histogram that specifies inter-arrival padding delays. + * - Or a parametrized probability distribution that specifies inter-arrival + * delays (see circpad_distribution_type_t). + * + * Padding machines start from the START state and finish with the END + * state. They can transition between states using the events in + * circpad_event_t. + * + * When a padding machine reaches the END state, it gets wiped from the circuit + * so that other padding machines can take over if needed (see + * circpad_machine_spec_transitioned_to_end()). + **/ + +#define CIRCUITPADDING_PRIVATE + +#include <math.h> +#include "lib/math/fp.h" +#include "lib/math/prob_distr.h" +#include "core/or/or.h" +#include "core/or/circuitpadding.h" +#include "core/or/circuitlist.h" +#include "core/or/circuituse.h" +#include "core/or/relay.h" +#include "feature/stats/rephist.h" +#include "feature/nodelist/networkstatus.h" + +#include "core/or/channel.h" + +#include "lib/time/compat_time.h" +#include "lib/defs/time.h" +#include "lib/crypt_ops/crypto_rand.h" + +#include "core/or/crypt_path_st.h" +#include "core/or/circuit_st.h" +#include "core/or/origin_circuit_st.h" +#include "core/or/or_circuit_st.h" +#include "feature/nodelist/routerstatus_st.h" +#include "feature/nodelist/node_st.h" +#include "core/or/cell_st.h" +#include "core/or/extend_info_st.h" +#include "core/crypto/relay_crypto.h" +#include "feature/nodelist/nodelist.h" + +#include "app/config/config.h" + +static inline circpad_purpose_mask_t circpad_circ_purpose_to_mask(uint8_t + circ_purpose); +static inline circpad_circuit_state_t circpad_circuit_state( + origin_circuit_t *circ); +static void circpad_setup_machine_on_circ(circuit_t *on_circ, + const circpad_machine_spec_t *machine); +static double circpad_distribution_sample(circpad_distribution_t dist); + +/** Cached consensus params */ +static uint8_t circpad_global_max_padding_percent; +static uint16_t circpad_global_allowed_cells; +static uint16_t circpad_max_circ_queued_cells; + +/** Global cell counts, for rate limiting */ +static uint64_t circpad_global_padding_sent; +static uint64_t circpad_global_nonpadding_sent; + +/** This is the list of circpad_machine_spec_t's parsed from consensus and + * torrc that have origin_side == 1 (ie: are for client side). + * + * The machines in this smartlist are considered immutable and they are used + * as-is by circuits so they should not change or get deallocated in Tor's + * runtime and as long as circuits are alive. */ +STATIC smartlist_t *origin_padding_machines = NULL; + +/** This is the list of circpad_machine_spec_t's parsed from consensus and + * torrc that have origin_side == 0 (ie: are for relay side). + * + * The machines in this smartlist are considered immutable and they are used + * as-is by circuits so they should not change or get deallocated in Tor's + * runtime and as long as circuits are alive. */ +STATIC smartlist_t *relay_padding_machines = NULL; + +/** Loop over the current padding state machines using <b>loop_var</b> as the + * loop variable. */ +#define FOR_EACH_CIRCUIT_MACHINE_BEGIN(loop_var) \ + STMT_BEGIN \ + for (int loop_var = 0; loop_var < CIRCPAD_MAX_MACHINES; loop_var++) { +#define FOR_EACH_CIRCUIT_MACHINE_END } STMT_END ; + +/** Loop over the current active padding state machines using <b>loop_var</b> + * as the loop variable. If a machine is not active, skip it. */ +#define FOR_EACH_ACTIVE_CIRCUIT_MACHINE_BEGIN(loop_var, circ) \ + FOR_EACH_CIRCUIT_MACHINE_BEGIN(loop_var) \ + if (!(circ)->padding_info[loop_var]) \ + continue; +#define FOR_EACH_ACTIVE_CIRCUIT_MACHINE_END } STMT_END ; + +/** + * Return a human-readable description for a circuit padding state. + */ +static const char * +circpad_state_to_string(circpad_statenum_t state) +{ + const char *descr; + + switch (state) { + case CIRCPAD_STATE_START: + descr = "START"; + break; + case CIRCPAD_STATE_BURST: + descr = "BURST"; + break; + case CIRCPAD_STATE_GAP: + descr = "GAP"; + break; + case CIRCPAD_STATE_END: + descr = "END"; + break; + default: + descr = "CUSTOM"; // XXX: Just return # in static char buf? + } + + return descr; +} + +/** + * Free the machineinfo at an index + */ +static void +circpad_circuit_machineinfo_free_idx(circuit_t *circ, int idx) +{ + if (circ->padding_info[idx]) { + tor_free(circ->padding_info[idx]->histogram); + timer_free(circ->padding_info[idx]->padding_timer); + tor_free(circ->padding_info[idx]); + } +} + +/** + * Free all the machineinfos in <b>circ</b> that match <b>machine_num</b>. + * + * Returns true if any machineinfos with that number were freed. + * False otherwise. */ +static int +free_circ_machineinfos_with_machine_num(circuit_t *circ, int machine_num) +{ + int found = 0; + FOR_EACH_CIRCUIT_MACHINE_BEGIN(i) { + if (circ->padding_machine[i] && + circ->padding_machine[i]->machine_num == machine_num) { + circpad_circuit_machineinfo_free_idx(circ, i); + circ->padding_machine[i] = NULL; + found = 1; + } + } FOR_EACH_CIRCUIT_MACHINE_END; + + return found; +} + +/** + * Free all padding machines and mutable info associated with circuit + */ +void +circpad_circuit_free_all_machineinfos(circuit_t *circ) +{ + FOR_EACH_CIRCUIT_MACHINE_BEGIN(i) { + circpad_circuit_machineinfo_free_idx(circ, i); + } FOR_EACH_CIRCUIT_MACHINE_END; +} + +/** + * Allocate a new mutable machineinfo structure. + */ +STATIC circpad_machine_state_t * +circpad_circuit_machineinfo_new(circuit_t *on_circ, int machine_index) +{ + circpad_machine_state_t *mi = + tor_malloc_zero(sizeof(circpad_machine_state_t)); + mi->machine_index = machine_index; + mi->on_circ = on_circ; + + return mi; +} + +/** + * Return the circpad_state_t for the current state based on the + * mutable info. + * + * This function returns NULL when the machine is in the end state or in an + * invalid state. + */ +STATIC const circpad_state_t * +circpad_machine_current_state(const circpad_machine_state_t *mi) +{ + const circpad_machine_spec_t *machine = CIRCPAD_GET_MACHINE(mi); + + if (mi->current_state == CIRCPAD_STATE_END) { + return NULL; + } else if (BUG(mi->current_state >= machine->num_states)) { + log_fn(LOG_WARN,LD_CIRC, + "Invalid circuit padding state %d", + mi->current_state); + + return NULL; + } + + return &machine->states[mi->current_state]; +} + +/** + * Calculate the lower bound of a histogram bin. The upper bound + * is obtained by calling this function with bin+1, and subtracting 1. + * + * The 0th bin has a special value -- it only represents start_usec. + * This is so we can specify a probability on 0-delay values. + * + * After bin 0, bins are exponentially spaced, so that each subsequent + * bin is twice as large as the previous. This is done so that higher + * time resolution is given to lower time values. + * + * The infinity bin is a the last bin in the array (histogram_len-1). + * It has a usec value of CIRCPAD_DELAY_INFINITE (UINT32_MAX). + */ +STATIC circpad_delay_t +circpad_histogram_bin_to_usec(const circpad_machine_state_t *mi, + circpad_hist_index_t bin) +{ + const circpad_state_t *state = circpad_machine_current_state(mi); + circpad_delay_t start_usec; + + /* Our state should have been checked to be non-null by the caller + * (circpad_machine_remove_token()) */ + if (BUG(state == NULL)) { + return CIRCPAD_DELAY_INFINITE; + } + + if (state->use_rtt_estimate) + start_usec = mi->rtt_estimate_usec+state->start_usec; + else + start_usec = state->start_usec; + + if (bin >= CIRCPAD_INFINITY_BIN(state)) + return CIRCPAD_DELAY_INFINITE; + + if (bin == 0) + return start_usec; + + if (bin == 1) + return start_usec+1; + + /* The bin widths double every index, so that we can have more resolution + * for lower time values in the histogram. */ + const circpad_time_t bin_width_exponent = + 1 << (CIRCPAD_INFINITY_BIN(state) - bin); + return (circpad_delay_t)MIN(start_usec + + state->range_usec/bin_width_exponent, + CIRCPAD_DELAY_INFINITE); +} + +/** Return the midpoint of the histogram bin <b>bin_index</b>. */ +static circpad_delay_t +circpad_get_histogram_bin_midpoint(const circpad_machine_state_t *mi, + int bin_index) +{ + circpad_delay_t left_bound = circpad_histogram_bin_to_usec(mi, bin_index); + circpad_delay_t right_bound = + circpad_histogram_bin_to_usec(mi, bin_index+1)-1; + + return left_bound + (right_bound - left_bound)/2; +} + +/** + * Return the bin that contains the usec argument. + * "Contains" is defined as us in [lower, upper). + * + * This function will never return the infinity bin (histogram_len-1), + * in order to simplify the rest of the code. + * + * This means that technically the last bin (histogram_len-2) + * has range [start_usec+range_usec, CIRCPAD_DELAY_INFINITE]. + */ +STATIC circpad_hist_index_t +circpad_histogram_usec_to_bin(const circpad_machine_state_t *mi, + circpad_delay_t usec) +{ + const circpad_state_t *state = circpad_machine_current_state(mi); + circpad_delay_t start_usec; + int32_t bin; /* Larger than return type to properly clamp overflow */ + + /* Our state should have been checked to be non-null by the caller + * (circpad_machine_remove_token()) */ + if (BUG(state == NULL)) { + return 0; + } + + if (state->use_rtt_estimate) + start_usec = mi->rtt_estimate_usec+state->start_usec; + else + start_usec = state->start_usec; + + /* The first bin (#0) has zero width and starts (and ends) at start_usec. */ + if (usec <= start_usec) + return 0; + + if (usec == start_usec+1) + return 1; + + const circpad_time_t histogram_range_usec = state->range_usec; + /* We need to find the bin corresponding to our position in the range. + * Since bins are exponentially spaced in powers of two, we need to + * take the log2 of our position in histogram_range_usec. However, + * since tor_log2() returns the floor(log2(u64)), we have to adjust + * it to behave like ceil(log2(u64)). This is verified in our tests + * to properly invert the operation done in + * circpad_histogram_bin_to_usec(). */ + bin = CIRCPAD_INFINITY_BIN(state) - + tor_log2(2*histogram_range_usec/(usec-start_usec+1)); + + /* Clamp the return value to account for timevals before the start + * of bin 0, or after the last bin. Don't return the infinity bin + * index. */ + bin = MIN(MAX(bin, 1), CIRCPAD_INFINITY_BIN(state)-1); + return bin; +} + +/** + * This function frees any token bins allocated from a previous state + * + * Called after a state transition, or if the bins are empty. + */ +STATIC void +circpad_machine_setup_tokens(circpad_machine_state_t *mi) +{ + const circpad_state_t *state = circpad_machine_current_state(mi); + + /* If this state doesn't exist, or doesn't have token removal, + * free any previous state's histogram, and bail */ + if (!state || state->token_removal == CIRCPAD_TOKEN_REMOVAL_NONE) { + if (mi->histogram) { + tor_free(mi->histogram); + mi->histogram = NULL; + mi->histogram_len = 0; + } + return; + } + + /* Try to avoid re-mallocing if we don't really need to */ + if (!mi->histogram || (mi->histogram + && mi->histogram_len != state->histogram_len)) { + tor_free(mi->histogram); // null ok + mi->histogram = tor_malloc_zero(sizeof(circpad_hist_token_t) + *state->histogram_len); + } + mi->histogram_len = state->histogram_len; + + memcpy(mi->histogram, state->histogram, + sizeof(circpad_hist_token_t)*state->histogram_len); +} + +/** + * Choose a length for this state (in cells), if specified. + */ +static void +circpad_choose_state_length(circpad_machine_state_t *mi) +{ + const circpad_state_t *state = circpad_machine_current_state(mi); + double length; + + if (!state || state->length_dist.type == CIRCPAD_DIST_NONE) { + mi->state_length = CIRCPAD_STATE_LENGTH_INFINITE; + return; + } + + length = circpad_distribution_sample(state->length_dist); + length = MAX(0, length); + length += state->start_length; + length = MIN(length, state->max_length); + + mi->state_length = clamp_double_to_int64(length); +} + +/** + * Sample a value from our iat_dist, and clamp it safely + * to circpad_delay_t. + */ +static circpad_delay_t +circpad_distribution_sample_iat_delay(const circpad_state_t *state, + circpad_delay_t start_usec) +{ + double val = circpad_distribution_sample(state->iat_dist); + /* These comparisons are safe, because the output is in the range + * [0, 2**32), and double has a precision of 53 bits. */ + val = MAX(0, val); + val = MIN(val, state->range_usec); + + /* This addition is exact: val is at most 2**32-1, start_usec + * is at most 2**32-1, and doubles have a precision of 53 bits. */ + val += start_usec; + + /* Clamp the distribution at infinite delay val */ + return (circpad_delay_t)MIN(tor_llround(val), CIRCPAD_DELAY_INFINITE); +} + +/** + * Sample an expected time-until-next-packet delay from the histogram. + * + * The bin is chosen with probability proportional to the number + * of tokens in each bin, and then a time value is chosen uniformly from + * that bin's [start,end) time range. + */ +STATIC circpad_delay_t +circpad_machine_sample_delay(circpad_machine_state_t *mi) +{ + const circpad_state_t *state = circpad_machine_current_state(mi); + const circpad_hist_token_t *histogram = NULL; + circpad_hist_index_t curr_bin = 0; + circpad_delay_t bin_start, bin_end; + circpad_delay_t start_usec; + /* These three must all be larger than circpad_hist_token_t, because + * we sum several circpad_hist_token_t values across the histogram */ + uint64_t curr_weight = 0; + uint64_t histogram_total_tokens = 0; + uint64_t bin_choice; + + tor_assert(state); + + if (state->use_rtt_estimate) + start_usec = mi->rtt_estimate_usec+state->start_usec; + else + start_usec = state->start_usec; + + if (state->iat_dist.type != CIRCPAD_DIST_NONE) { + /* Sample from a fixed IAT distribution and return */ + return circpad_distribution_sample_iat_delay(state, start_usec); + } else if (state->token_removal != CIRCPAD_TOKEN_REMOVAL_NONE) { + /* We have a mutable histogram. Do basic sanity check and apply: */ + if (BUG(!mi->histogram) || + BUG(mi->histogram_len != state->histogram_len)) { + return CIRCPAD_DELAY_INFINITE; + } + + histogram = mi->histogram; + for (circpad_hist_index_t b = 0; b < state->histogram_len; b++) + histogram_total_tokens += histogram[b]; + } else { + /* We have a histogram, but it's immutable */ + histogram = state->histogram; + histogram_total_tokens = state->histogram_total_tokens; + } + + bin_choice = crypto_rand_uint64(histogram_total_tokens); + + /* Skip all the initial zero bins */ + while (!histogram[curr_bin]) { + curr_bin++; + } + curr_weight = histogram[curr_bin]; + + // TODO: This is not constant-time. Pretty sure we don't + // really need it to be, though. + while (curr_weight < bin_choice) { + curr_bin++; + /* It should be impossible to run past the end of the histogram */ + if (BUG(curr_bin >= state->histogram_len)) { + return CIRCPAD_DELAY_INFINITE; + } + curr_weight += histogram[curr_bin]; + } + + /* Do some basic checking of the current bin we are in */ + if (BUG(curr_bin >= state->histogram_len) || + BUG(histogram[curr_bin] == 0)) { + return CIRCPAD_DELAY_INFINITE; + } + + // Store this index to remove the token upon callback. + if (state->token_removal != CIRCPAD_TOKEN_REMOVAL_NONE) { + mi->chosen_bin = curr_bin; + } + + if (curr_bin >= CIRCPAD_INFINITY_BIN(state)) { + if (state->token_removal != CIRCPAD_TOKEN_REMOVAL_NONE && + mi->histogram[curr_bin] > 0) { + mi->histogram[curr_bin]--; + } + + // Infinity: Don't send a padding packet. Wait for a real packet + // and then see if our bins are empty or what else we should do. + return CIRCPAD_DELAY_INFINITE; + } + + tor_assert(curr_bin < CIRCPAD_INFINITY_BIN(state)); + + bin_start = circpad_histogram_bin_to_usec(mi, curr_bin); + /* We don't need to reduct 1 from the upper bound because the random range + * function below samples from [bin_start, bin_end) */ + bin_end = circpad_histogram_bin_to_usec(mi, curr_bin+1); + + /* Truncate the high bin in case it's the infinity bin: + * Don't actually schedule an "infinite"-1 delay */ + bin_end = MIN(bin_end, start_usec+state->range_usec); + + // Sample uniformly between histogram[i] to histogram[i+1]-1, + // but no need to sample if they are the same timeval (aka bin 0 or bin 1). + if (bin_end <= bin_start+1) + return bin_start; + else + return (circpad_delay_t)crypto_rand_uint64_range(bin_start, bin_end); +} + +/** + * Sample a value from the specified probability distribution. + * + * This performs inverse transform sampling + * (https://en.wikipedia.org/wiki/Inverse_transform_sampling). + * + * XXX: These formulas were taken verbatim. Need a floating wizard + * to check them for catastropic cancellation and other issues (teor?). + * Also: is 32bits of double from [0.0,1.0) enough? + */ +static double +circpad_distribution_sample(circpad_distribution_t dist) +{ + log_fn(LOG_DEBUG,LD_CIRC, "Sampling delay with distribution %d", + dist.type); + + switch (dist.type) { + case CIRCPAD_DIST_NONE: + { + /* We should not get in here like this */ + tor_assert_nonfatal_unreached(); + return 0; + } + case CIRCPAD_DIST_UNIFORM: + { + // param2 is upper bound, param1 is lower + const struct uniform my_uniform = { + .base = UNIFORM(my_uniform), + .a = dist.param1, + .b = dist.param2, + }; + return dist_sample(&my_uniform.base); + } + case CIRCPAD_DIST_LOGISTIC: + { + /* param1 is Mu, param2 is sigma. */ + const struct logistic my_logistic = { + .base = LOGISTIC(my_logistic), + .mu = dist.param1, + .sigma = dist.param2, + }; + return dist_sample(&my_logistic.base); + } + case CIRCPAD_DIST_LOG_LOGISTIC: + { + /* param1 is Alpha, param2 is 1.0/Beta */ + const struct log_logistic my_log_logistic = { + .base = LOG_LOGISTIC(my_log_logistic), + .alpha = dist.param1, + .beta = dist.param2, + }; + return dist_sample(&my_log_logistic.base); + } + case CIRCPAD_DIST_GEOMETRIC: + { + /* param1 is 'p' (success probability) */ + const struct geometric my_geometric = { + .base = GEOMETRIC(my_geometric), + .p = dist.param1, + }; + return dist_sample(&my_geometric.base); + } + case CIRCPAD_DIST_WEIBULL: + { + /* param1 is k, param2 is Lambda */ + const struct weibull my_weibull = { + .base = WEIBULL(my_weibull), + .k = dist.param1, + .lambda = dist.param2, + }; + return dist_sample(&my_weibull.base); + } + case CIRCPAD_DIST_PARETO: + { + /* param1 is sigma, param2 is xi, no more params for mu so we use 0 */ + const struct genpareto my_genpareto = { + .base = GENPARETO(my_genpareto), + .mu = 0, + .sigma = dist.param1, + .xi = dist.param2, + }; + return dist_sample(&my_genpareto.base); + } + } + + tor_assert_nonfatal_unreached(); + return 0; +} + +/** + * Find the index of the first bin whose upper bound is + * greater than the target, and that has tokens remaining. + */ +static circpad_hist_index_t +circpad_machine_first_higher_index(const circpad_machine_state_t *mi, + circpad_delay_t target_bin_usec) +{ + circpad_hist_index_t bin = circpad_histogram_usec_to_bin(mi, + target_bin_usec); + + /* Don't remove from the infinity bin */ + for (; bin < CIRCPAD_INFINITY_BIN(mi); bin++) { + if (mi->histogram[bin] && + circpad_histogram_bin_to_usec(mi, bin+1) > target_bin_usec) { + return bin; + } + } + + return mi->histogram_len; +} + +/** + * Find the index of the first bin whose lower bound is lower or equal to + * <b>target_bin_usec</b>, and that still has tokens remaining. + */ +static circpad_hist_index_t +circpad_machine_first_lower_index(const circpad_machine_state_t *mi, + circpad_delay_t target_bin_usec) +{ + circpad_hist_index_t bin = circpad_histogram_usec_to_bin(mi, + target_bin_usec); + + for (; bin >= 0; bin--) { + if (mi->histogram[bin] && + circpad_histogram_bin_to_usec(mi, bin) <= target_bin_usec) { + return bin; + } + } + + return -1; +} + +/** + * Remove a token from the first non-empty bin whose upper bound is + * greater than the target. + */ +STATIC void +circpad_machine_remove_higher_token(circpad_machine_state_t *mi, + circpad_delay_t target_bin_usec) +{ + /* We need to remove the token from the first bin + * whose upper bound is greater than the target, and that + * has tokens remaining. */ + circpad_hist_index_t bin = circpad_machine_first_higher_index(mi, + target_bin_usec); + + if (bin >= 0 && bin < CIRCPAD_INFINITY_BIN(mi)) { + if (!BUG(mi->histogram[bin] == 0)) { + mi->histogram[bin]--; + } + } +} + +/** + * Remove a token from the first non-empty bin whose upper bound is + * lower than the target. + */ +STATIC void +circpad_machine_remove_lower_token(circpad_machine_state_t *mi, + circpad_delay_t target_bin_usec) +{ + circpad_hist_index_t bin = circpad_machine_first_lower_index(mi, + target_bin_usec); + + if (bin >= 0 && bin < CIRCPAD_INFINITY_BIN(mi)) { + if (!BUG(mi->histogram[bin] == 0)) { + mi->histogram[bin]--; + } + } +} + +/* Helper macro: Ensure that the bin has tokens available, and BUG out of the + * function if it's not the case. */ +#define ENSURE_BIN_CAPACITY(bin_index) \ + if (BUG(mi->histogram[bin_index] == 0)) { \ + return; \ + } + +/** + * Remove a token from the closest non-empty bin to the target. + * + * If use_usec is true, measure "closest" in terms of the next closest bin + * midpoint. + * + * If it is false, use bin index distance only. + */ +STATIC void +circpad_machine_remove_closest_token(circpad_machine_state_t *mi, + circpad_delay_t target_bin_usec, + bool use_usec) +{ + circpad_hist_index_t lower, higher, current; + circpad_hist_index_t bin_to_remove = -1; + + lower = circpad_machine_first_lower_index(mi, target_bin_usec); + higher = circpad_machine_first_higher_index(mi, target_bin_usec); + current = circpad_histogram_usec_to_bin(mi, target_bin_usec); + + /* Sanity check the results */ + if (BUG(lower > current) || BUG(higher < current)) { + return; + } + + /* Take care of edge cases first */ + if (higher == mi->histogram_len && lower == -1) { + /* All bins are empty */ + return; + } else if (higher == mi->histogram_len) { + /* All higher bins are empty */ + ENSURE_BIN_CAPACITY(lower); + mi->histogram[lower]--; + return; + } else if (lower == -1) { + /* All lower bins are empty */ + ENSURE_BIN_CAPACITY(higher); + mi->histogram[higher]--; + return; + } + + /* Now handle the intermediate cases */ + if (use_usec) { + /* Find the closest bin midpoint to the target */ + circpad_delay_t lower_usec = circpad_get_histogram_bin_midpoint(mi, lower); + circpad_delay_t higher_usec = + circpad_get_histogram_bin_midpoint(mi, higher); + + if (target_bin_usec < lower_usec) { + // Lower bin is closer + ENSURE_BIN_CAPACITY(lower); + bin_to_remove = lower; + } else if (target_bin_usec > higher_usec) { + // Higher bin is closer + ENSURE_BIN_CAPACITY(higher); + bin_to_remove = higher; + } else if (target_bin_usec-lower_usec > higher_usec-target_bin_usec) { + // Higher bin is closer + ENSURE_BIN_CAPACITY(higher); + bin_to_remove = higher; + } else { + // Lower bin is closer + ENSURE_BIN_CAPACITY(lower); + bin_to_remove = lower; + } + mi->histogram[bin_to_remove]--; + log_debug(LD_GENERAL, "Removing token from bin %d", bin_to_remove); + return; + } else { + if (current - lower > higher - current) { + // Higher bin is closer + ENSURE_BIN_CAPACITY(higher); + mi->histogram[higher]--; + return; + } else { + // Lower bin is closer + ENSURE_BIN_CAPACITY(lower); + mi->histogram[lower]--; + return; + } + } +} + +#undef ENSURE_BIN_CAPACITY + +/** + * Remove a token from the exact bin corresponding to the target. + * + * If it is empty, do nothing. + */ +static void +circpad_machine_remove_exact(circpad_machine_state_t *mi, + circpad_delay_t target_bin_usec) +{ + circpad_hist_index_t bin = circpad_histogram_usec_to_bin(mi, + target_bin_usec); + + if (mi->histogram[bin] > 0) + mi->histogram[bin]--; +} + +/** + * Check our state's cell limit count and tokens. + * + * Returns 1 if either limits are hit and we decide to change states, + * otherwise returns 0. + */ +static circpad_decision_t +check_machine_token_supply(circpad_machine_state_t *mi) +{ + uint32_t histogram_total_tokens = 0; + + /* Check if bins empty. This requires summing up the current mutable + * machineinfo histogram token total and checking if it is zero. + * Machineinfo does not keep a running token count. We're assuming the + * extra space is not worth this short loop iteration. + * + * We also do not count infinity bin in histogram totals. + */ + if (mi->histogram_len && mi->histogram) { + for (circpad_hist_index_t b = 0; b < CIRCPAD_INFINITY_BIN(mi); b++) + histogram_total_tokens += mi->histogram[b]; + + /* If we change state, we're done */ + if (histogram_total_tokens == 0) { + if (circpad_internal_event_bins_empty(mi) == CIRCPAD_STATE_CHANGED) + return CIRCPAD_STATE_CHANGED; + } + } + + if (mi->state_length == 0) { + return circpad_internal_event_state_length_up(mi); + } + + return CIRCPAD_STATE_UNCHANGED; +} + +/** + * Remove a token from the bin corresponding to the delta since + * last packet. If that bin is empty, choose a token based on + * the specified removal strategy in the state machine. + * + * This function also updates and checks rate limit and state + * limit counters. + * + * Returns 1 if we transition states, 0 otherwise. + */ +STATIC circpad_decision_t +circpad_machine_remove_token(circpad_machine_state_t *mi) +{ + const circpad_state_t *state = NULL; + circpad_time_t current_time; + circpad_delay_t target_bin_usec; + + /* Update non-padding counts for rate limiting: We scale at UINT16_MAX + * because we only use this for a percentile limit of 2 sig figs, and + * space is scare in the machineinfo struct. */ + mi->nonpadding_sent++; + if (mi->nonpadding_sent == UINT16_MAX) { + mi->padding_sent /= 2; + mi->nonpadding_sent /= 2; + } + + /* Dont remove any tokens if there was no padding scheduled */ + if (!mi->padding_scheduled_at_usec) { + return CIRCPAD_STATE_UNCHANGED; + } + + state = circpad_machine_current_state(mi); + current_time = monotime_absolute_usec(); + + /* If we have scheduled padding some time in the future, we want to see what + bin we are in at the current time */ + target_bin_usec = (circpad_delay_t) + MIN((current_time - mi->padding_scheduled_at_usec), + CIRCPAD_DELAY_INFINITE-1); + + /* We are treating this non-padding cell as a padding cell, so we cancel + padding timer, if present. */ + mi->padding_scheduled_at_usec = 0; + if (mi->is_padding_timer_scheduled) { + mi->is_padding_timer_scheduled = 0; + timer_disable(mi->padding_timer); + } + + /* If we are not in a padding state (like start or end), we're done */ + if (!state) + return CIRCPAD_STATE_UNCHANGED; + + /* If we're enforcing a state length on non-padding packets, + * decrement it */ + if (mi->state_length != CIRCPAD_STATE_LENGTH_INFINITE && + state->length_includes_nonpadding && + mi->state_length > 0) { + mi->state_length--; + } + + /* Perform the specified token removal strategy */ + switch (state->token_removal) { + case CIRCPAD_TOKEN_REMOVAL_NONE: + break; + case CIRCPAD_TOKEN_REMOVAL_CLOSEST_USEC: + circpad_machine_remove_closest_token(mi, target_bin_usec, 1); + break; + case CIRCPAD_TOKEN_REMOVAL_CLOSEST: + circpad_machine_remove_closest_token(mi, target_bin_usec, 0); + break; + case CIRCPAD_TOKEN_REMOVAL_LOWER: + circpad_machine_remove_lower_token(mi, target_bin_usec); + break; + case CIRCPAD_TOKEN_REMOVAL_HIGHER: + circpad_machine_remove_higher_token(mi, target_bin_usec); + break; + case CIRCPAD_TOKEN_REMOVAL_EXACT: + circpad_machine_remove_exact(mi, target_bin_usec); + break; + } + + /* Check our token and state length limits */ + return check_machine_token_supply(mi); +} + +/** + * Send a relay command with a relay cell payload on a circuit to + * the particular hopnum. + * + * Hopnum starts at 1 (1=guard, 2=middle, 3=exit, etc). + * + * Payload may be null. + * + * Returns negative on error, 0 on success. + */ +MOCK_IMPL(STATIC signed_error_t, +circpad_send_command_to_hop,(origin_circuit_t *circ, uint8_t hopnum, + uint8_t relay_command, const uint8_t *payload, + ssize_t payload_len)) +{ + crypt_path_t *target_hop = circuit_get_cpath_hop(circ, hopnum); + signed_error_t ret; + + /* Check that the cpath has the target hop */ + if (!target_hop) { + log_fn(LOG_WARN, LD_BUG, "Padding circuit %u has %d hops, not %d", + circ->global_identifier, circuit_get_cpath_len(circ), hopnum); + return -1; + } + + /* Check that the target hop is opened */ + if (target_hop->state != CPATH_STATE_OPEN) { + log_fn(LOG_WARN,LD_CIRC, + "Padding circuit %u has %d hops, not %d", + circ->global_identifier, + circuit_get_cpath_opened_len(circ), hopnum); + return -1; + } + + /* Send the drop command to the second hop */ + ret = relay_send_command_from_edge(0, TO_CIRCUIT(circ), relay_command, + (const char*)payload, payload_len, + target_hop); + return ret; +} + +/** + * Callback helper to send a padding cell. + * + * This helper is called after our histogram-sampled delay period passes + * without another packet being sent first. If a packet is sent before this + * callback happens, it is canceled. So when we're called here, send padding + * right away. + * + * If sending this padding cell forced us to transition states return + * CIRCPAD_STATE_CHANGED. Otherwise return CIRCPAD_STATE_UNCHANGED. + */ +circpad_decision_t +circpad_send_padding_cell_for_callback(circpad_machine_state_t *mi) +{ + circuit_t *circ = mi->on_circ; + int machine_idx = mi->machine_index; + mi->padding_scheduled_at_usec = 0; + circpad_statenum_t state = mi->current_state; + + // Make sure circuit didn't close on us + if (mi->on_circ->marked_for_close) { + log_fn(LOG_INFO,LD_CIRC, + "Padding callback on a circuit marked for close. Ignoring."); + return CIRCPAD_STATE_CHANGED; + } + + /* If it's a histogram, reduce the token count */ + if (mi->histogram && mi->histogram_len) { + /* Basic sanity check on the histogram before removing anything */ + if (BUG(mi->chosen_bin >= mi->histogram_len) || + BUG(mi->histogram[mi->chosen_bin] == 0)) { + return CIRCPAD_STATE_CHANGED; + } + + mi->histogram[mi->chosen_bin]--; + } + + /* If we have a valid state length bound, consider it */ + if (mi->state_length != CIRCPAD_STATE_LENGTH_INFINITE && + !BUG(mi->state_length <= 0)) { + mi->state_length--; + } + + /* + * Update non-padding counts for rate limiting: We scale at UINT16_MAX + * because we only use this for a percentile limit of 2 sig figs, and + * space is scare in the machineinfo struct. + */ + mi->padding_sent++; + if (mi->padding_sent == UINT16_MAX) { + mi->padding_sent /= 2; + mi->nonpadding_sent /= 2; + } + circpad_global_padding_sent++; + + if (CIRCUIT_IS_ORIGIN(mi->on_circ)) { + circpad_send_command_to_hop(TO_ORIGIN_CIRCUIT(mi->on_circ), + CIRCPAD_GET_MACHINE(mi)->target_hopnum, + RELAY_COMMAND_DROP, NULL, 0); + log_fn(LOG_INFO,LD_CIRC, "Callback: Sending padding to origin circuit %u.", + TO_ORIGIN_CIRCUIT(mi->on_circ)->global_identifier); + } else { + // If we're a non-origin circ, we can just send from here as if we're the + // edge. + if (TO_OR_CIRCUIT(circ)->p_chan_cells.n <= circpad_max_circ_queued_cells) { + log_fn(LOG_INFO,LD_CIRC, + "Callback: Sending padding to non-origin circuit."); + relay_send_command_from_edge(0, mi->on_circ, RELAY_COMMAND_DROP, NULL, + 0, NULL); + } else { + static ratelim_t cell_lim = RATELIM_INIT(600); + log_fn_ratelim(&cell_lim,LOG_NOTICE,LD_CIRC, + "Too many cells (%d) in circ queue to send padding.", + TO_OR_CIRCUIT(circ)->p_chan_cells.n); + } + } + + rep_hist_padding_count_write(PADDING_TYPE_DROP); + /* This is a padding cell sent from the client or from the middle node, + * (because it's invoked from circuitpadding.c) */ + circpad_cell_event_padding_sent(circ); + + /* The circpad_cell_event_padding_sent() could cause us to transition. + * Check that we still have a padding machineinfo, and then check our token + * supply. */ + if (circ->padding_info[machine_idx] != NULL) { + if (state != circ->padding_info[machine_idx]->current_state) + return CIRCPAD_STATE_CHANGED; + else + return check_machine_token_supply(circ->padding_info[machine_idx]); + } else { + return CIRCPAD_STATE_CHANGED; + } +} + +/** + * Tor-timer compatible callback that tells us to send a padding cell. + * + * Timers are associated with circpad_machine_state_t's. When the machineinfo + * is freed on a circuit, the timers are cancelled. Since the lifetime + * of machineinfo is always longer than the timers, handles are not + * needed. + */ +static void +circpad_send_padding_callback(tor_timer_t *timer, void *args, + const struct monotime_t *time) +{ + circpad_machine_state_t *mi = ((circpad_machine_state_t*)args); + (void)timer; (void)time; + + if (mi && mi->on_circ) { + assert_circuit_ok(mi->on_circ); + circpad_send_padding_cell_for_callback(mi); + } else { + // This shouldn't happen (represents a timer leak) + log_fn(LOG_WARN,LD_CIRC, + "Circuit closed while waiting for padding timer."); + tor_fragile_assert(); + } + + // TODO-MP-AP: Unify this counter with channelpadding for rephist stats + //total_timers_pending--; +} + +/** + * Cache our consensus parameters upon consensus update. + */ +void +circpad_new_consensus_params(const networkstatus_t *ns) +{ + circpad_global_allowed_cells = + networkstatus_get_param(ns, "circpad_global_allowed_cells", + 0, 0, UINT16_MAX-1); + + circpad_global_max_padding_percent = + networkstatus_get_param(ns, "circpad_global_max_padding_pct", + 0, 0, 100); + + circpad_max_circ_queued_cells = + networkstatus_get_param(ns, "circpad_max_circ_queued_cells", + CIRCWINDOW_START_MAX, 0, 50*CIRCWINDOW_START_MAX); +} + +/** + * Check this machine against its padding limits, as well as global + * consensus limits. + * + * We have two limits: a percent and a cell count. The cell count + * limit must be reached before the percent is enforced (this is to + * optionally allow very light padding of things like circuit setup + * while there is no other traffic on the circuit). + * + * TODO: Don't apply limits to machines form torrc. + * + * Returns 1 if limits are set and we've hit them. Otherwise returns 0. + */ +STATIC bool +circpad_machine_reached_padding_limit(circpad_machine_state_t *mi) +{ + const circpad_machine_spec_t *machine = CIRCPAD_GET_MACHINE(mi); + + /* If machine_padding_pct is non-zero, and we've sent more + * than the allowed count of padding cells, then check our + * percent limits for this machine. */ + if (machine->max_padding_percent && + mi->padding_sent >= machine->allowed_padding_count) { + uint32_t total_cells = mi->padding_sent + mi->nonpadding_sent; + + /* Check the percent */ + if ((100*(uint32_t)mi->padding_sent) / total_cells > + machine->max_padding_percent) { + return 1; // limit is reached. Stop. + } + } + + /* If circpad_max_global_padding_pct is non-zero, and we've + * sent more than the global padding cell limit, then check our + * gloabl tor process percentage limit on padding. */ + if (circpad_global_max_padding_percent && + circpad_global_padding_sent >= circpad_global_allowed_cells) { + uint64_t total_cells = circpad_global_padding_sent + + circpad_global_nonpadding_sent; + + /* Check the percent */ + if ((100*circpad_global_padding_sent) / total_cells > + circpad_global_max_padding_percent) { + return 1; // global limit reached. Stop. + } + } + + return 0; // All good! +} + +/** + * Schedule the next padding time according to the machineinfo on a + * circuit. + * + * The histograms represent inter-packet-delay. Whenever you get an packet + * event you should be scheduling your next timer (after cancelling any old + * ones and updating tokens accordingly). + * + * Returns 1 if we decide to transition states (due to infinity bin), + * 0 otherwise. + */ +MOCK_IMPL(circpad_decision_t, +circpad_machine_schedule_padding,(circpad_machine_state_t *mi)) +{ + circpad_delay_t in_usec = 0; + struct timeval timeout; + tor_assert(mi); + + // Don't pad in end (but also don't cancel any previously + // scheduled padding either). + if (mi->current_state == CIRCPAD_STATE_END) { + log_fn(LOG_INFO, LD_CIRC, "Padding end state"); + return CIRCPAD_STATE_UNCHANGED; + } + + /* Check our padding limits */ + if (circpad_machine_reached_padding_limit(mi)) { + if (CIRCUIT_IS_ORIGIN(mi->on_circ)) { + log_fn(LOG_INFO, LD_CIRC, + "Padding machine has reached padding limit on circuit %u", + TO_ORIGIN_CIRCUIT(mi->on_circ)->global_identifier); + } else { + log_fn(LOG_INFO, LD_CIRC, + "Padding machine has reached padding limit on circuit %"PRIu64 + ", %d", + mi->on_circ->n_chan ? mi->on_circ->n_chan->global_identifier : 0, + mi->on_circ->n_circ_id); + } + return CIRCPAD_STATE_UNCHANGED; + } + + if (mi->is_padding_timer_scheduled) { + /* Cancel current timer (if any) */ + timer_disable(mi->padding_timer); + mi->is_padding_timer_scheduled = 0; + } + + /* in_usec = in microseconds */ + in_usec = circpad_machine_sample_delay(mi); + mi->padding_scheduled_at_usec = monotime_absolute_usec(); + log_fn(LOG_INFO,LD_CIRC,"\tPadding in %u usec", in_usec); + + // Don't schedule if we have infinite delay. + if (in_usec == CIRCPAD_DELAY_INFINITE) { + return circpad_internal_event_infinity(mi); + } + + if (mi->state_length == 0) { + /* If we're at length 0, that means we hit 0 after sending + * a cell earlier, and emitted an event for it, but + * for whatever reason we did not decide to change states then. + * So maybe the machine is waiting for bins empty, or for an + * infinity event later? That would be a strange machine, + * but there's no reason to make it impossible. */ + return CIRCPAD_STATE_UNCHANGED; + } + + if (in_usec <= 0) { + return circpad_send_padding_cell_for_callback(mi); + } + + timeout.tv_sec = in_usec/TOR_USEC_PER_SEC; + timeout.tv_usec = (in_usec%TOR_USEC_PER_SEC); + + log_fn(LOG_INFO, LD_CIRC, "\tPadding in %u sec, %u usec", + (unsigned)timeout.tv_sec, (unsigned)timeout.tv_usec); + + if (mi->padding_timer) { + timer_set_cb(mi->padding_timer, circpad_send_padding_callback, mi); + } else { + mi->padding_timer = + timer_new(circpad_send_padding_callback, mi); + } + timer_schedule(mi->padding_timer, &timeout); + mi->is_padding_timer_scheduled = 1; + + // TODO-MP-AP: Unify with channelpadding counter + //rep_hist_padding_count_timers(++total_timers_pending); + + return CIRCPAD_STATE_UNCHANGED; +} + +/** + * If the machine transitioned to the END state, we need + * to check to see if it wants us to shut it down immediately. + * If it does, then we need to send the appropate negotation commands + * depending on which side it is. + * + * After this function is called, mi may point to freed memory. Do + * not access it. + */ +static void +circpad_machine_spec_transitioned_to_end(circpad_machine_state_t *mi) +{ + const circpad_machine_spec_t *machine = CIRCPAD_GET_MACHINE(mi); + + /* + * We allow machines to shut down and delete themselves as opposed + * to just going back to START or waiting forever in END so that + * we can handle the case where this machine started while it was + * the only machine that matched conditions, but *since* then more + * "higher ranking" machines now match the conditions, and would + * be given a chance to take precidence over this one in + * circpad_add_matching_machines(). + * + * Returning to START or waiting forever in END would not give those + * other machines a chance to be launched, where as shutting down + * here does. + */ + if (machine->should_negotiate_end) { + circuit_t *on_circ = mi->on_circ; + if (machine->is_origin_side) { + /* We free the machine info here so that we can be replaced + * by a different machine. But we must leave the padding_machine + * in place to wait for the negotiated response */ + circpad_circuit_machineinfo_free_idx(on_circ, + machine->machine_index); + circpad_negotiate_padding(TO_ORIGIN_CIRCUIT(on_circ), + machine->machine_num, + machine->target_hopnum, + CIRCPAD_COMMAND_STOP); + } else { + circpad_circuit_machineinfo_free_idx(on_circ, + machine->machine_index); + circpad_padding_negotiated(on_circ, + machine->machine_num, + CIRCPAD_COMMAND_STOP, + CIRCPAD_RESPONSE_OK); + on_circ->padding_machine[machine->machine_index] = NULL; + } + } +} + +/** + * Generic state transition function for padding state machines. + * + * Given an event and our mutable machine info, decide if/how to + * transition to a different state, and perform actions accordingly. + * + * Returns 1 if we transition states, 0 otherwise. + */ +MOCK_IMPL(circpad_decision_t, +circpad_machine_spec_transition,(circpad_machine_state_t *mi, + circpad_event_t event)) +{ + const circpad_state_t *state = + circpad_machine_current_state(mi); + + /* If state is null we are in the end state. */ + if (!state) { + /* If we in end state we don't pad no matter what. */ + return CIRCPAD_STATE_UNCHANGED; + } + + /* Check if this event is ignored or causes a cancel */ + if (state->next_state[event] == CIRCPAD_STATE_IGNORE) { + return CIRCPAD_STATE_UNCHANGED; + } else if (state->next_state[event] == CIRCPAD_STATE_CANCEL) { + /* Check cancel events and cancel any pending padding */ + mi->padding_scheduled_at_usec = 0; + if (mi->is_padding_timer_scheduled) { + mi->is_padding_timer_scheduled = 0; + /* Cancel current timer (if any) */ + timer_disable(mi->padding_timer); + } + return CIRCPAD_STATE_UNCHANGED; + } else { + circpad_statenum_t s = state->next_state[event]; + /* See if we need to transition to any other states based on this event. + * Whenever a transition happens, even to our own state, we schedule + * padding. + * + * So if a state only wants to schedule padding for an event, it specifies + * a transition to itself. All non-specified events are ignored. + */ + log_fn(LOG_INFO, LD_CIRC, + "Circpad machine %d transitioning from %s to %s", + mi->machine_index, circpad_state_to_string(mi->current_state), + circpad_state_to_string(s)); + + /* If this is not the same state, switch and init tokens, + * otherwise just reschedule padding. */ + if (mi->current_state != s) { + mi->current_state = s; + circpad_machine_setup_tokens(mi); + circpad_choose_state_length(mi); + + /* If we transition to the end state, check to see + * if this machine wants to be shut down at end */ + if (s == CIRCPAD_STATE_END) { + circpad_machine_spec_transitioned_to_end(mi); + /* We transitioned but we don't pad in end. Also, mi + * may be freed. Returning STATE_CHANGED prevents us + * from accessing it in any callers of this function. */ + return CIRCPAD_STATE_CHANGED; + } + + /* We transitioned to a new state, schedule padding */ + circpad_machine_schedule_padding(mi); + return CIRCPAD_STATE_CHANGED; + } + + /* We transitioned back to the same state. Schedule padding, + * and inform if that causes a state transition. */ + return circpad_machine_schedule_padding(mi); + } + + return CIRCPAD_STATE_UNCHANGED; +} + +/** + * Estimate the circuit RTT from the current middle hop out to the + * end of the circuit. + * + * We estimate RTT by calculating the time between "receive" and + * "send" at a middle hop. This is because we "receive" a cell + * from the origin, and then relay it towards the exit before a + * response comes back. It is that response time from the exit side + * that we want to measure, so that we can make use of it for synthetic + * response delays. + */ +static void +circpad_estimate_circ_rtt_on_received(circuit_t *circ, + circpad_machine_state_t *mi) +{ + /* Origin circuits don't estimate RTT. They could do it easily enough, + * but they have no reason to use it in any delay calculations. */ + if (CIRCUIT_IS_ORIGIN(circ) || mi->stop_rtt_update) + return; + + /* If we already have a last receieved packet time, that means we + * did not get a response before this packet. The RTT estimate + * only makes sense if we do not have multiple packets on the + * wire, so stop estimating if this is the second packet + * back to back. However, for the first set of back-to-back + * packets, we can wait until the very first response comes back + * to us, to measure that RTT (for the response to optimistic + * data, for example). Hence stop_rtt_update is only checked + * in this received side function, and not in send side below. + */ + if (mi->last_received_time_usec) { + /* We also allow multiple back-to-back packets if the circuit is not + * opened, to handle var cells. + * XXX: Will this work with out var cell plans? Maybe not, + * since we're opened at the middle hop as soon as we process + * one var extend2 :/ */ + if (circ->state == CIRCUIT_STATE_OPEN) { + log_fn(LOG_INFO, LD_CIRC, + "Stopping padding RTT estimation on circuit (%"PRIu64 + ", %d) after two back to back packets. Current RTT: %d", + circ->n_chan ? circ->n_chan->global_identifier : 0, + circ->n_circ_id, mi->rtt_estimate_usec); + mi->stop_rtt_update = 1; + } + } else { + mi->last_received_time_usec = monotime_absolute_usec(); + } +} + +/** + * Handles the "send" side of RTT calculation at middle nodes. + * + * This function calculates the RTT from the middle to the end + * of the circuit by subtracting the last received cell timestamp + * from the current time. It allows back-to-back cells until + * the circuit is opened, to allow for var cell handshakes. + * XXX: Check our var cell plans to make sure this will work. + */ +static void +circpad_estimate_circ_rtt_on_send(circuit_t *circ, + circpad_machine_state_t *mi) +{ + /* Origin circuits don't estimate RTT. They could do it easily enough, + * but they have no reason to use it in any delay calculations. */ + if (CIRCUIT_IS_ORIGIN(circ)) + return; + + /* If last_received_time_usec is non-zero, we are waiting for a response + * from the exit side. Calculate the time delta and use it as RTT. */ + if (mi->last_received_time_usec) { + circpad_time_t rtt_time = monotime_absolute_usec() - + mi->last_received_time_usec; + + /* Reset the last RTT packet time, so we can tell if two cells + * arrive back to back */ + mi->last_received_time_usec = 0; + + /* Use INT32_MAX to ensure the addition doesn't overflow */ + if (rtt_time >= INT32_MAX) { + log_fn(LOG_WARN,LD_CIRC, + "Circuit padding RTT estimate overflowed: %"PRIu64 + " vs %"PRIu64, monotime_absolute_usec(), + mi->last_received_time_usec); + return; + } + + /* If the old RTT estimate is lower than this one, use this one, because + * the circuit is getting longer. If this estimate is somehow + * faster than the previous, then maybe that was network jitter, or a + * bad monotonic clock source (so our ratchet returned a zero delta). + * In that case, average them. */ + if (mi->rtt_estimate_usec < (circpad_delay_t)rtt_time) { + mi->rtt_estimate_usec = (circpad_delay_t)rtt_time; + } else { + mi->rtt_estimate_usec += (circpad_delay_t)rtt_time; + mi->rtt_estimate_usec /= 2; + } + } else if (circ->state == CIRCUIT_STATE_OPEN) { + /* If last_received_time_usec is zero, then we have gotten two cells back + * to back. Stop estimating RTT in this case. Note that we only + * stop RTT update if the circuit is opened, to allow for RTT estimates + * of var cells during circ setup. */ + mi->stop_rtt_update = 1; + + if (!mi->rtt_estimate_usec) { + log_fn(LOG_NOTICE, LD_CIRC, + "Got two cells back to back on a circuit before estimating RTT."); + } + } +} + +/** + * A "non-padding" cell has been sent from this endpoint. React + * according to any padding state machines on the circuit. + * + * For origin circuits, this means we sent a cell into the network. + * For middle relay circuits, this means we sent a cell towards the + * origin. + */ +void +circpad_cell_event_nonpadding_sent(circuit_t *on_circ) +{ + /* Update global cell count */ + circpad_global_nonpadding_sent++; + + /* If there are no machines then this loop should not iterate */ + FOR_EACH_ACTIVE_CIRCUIT_MACHINE_BEGIN(i, on_circ) { + /* First, update any RTT estimate */ + circpad_estimate_circ_rtt_on_send(on_circ, on_circ->padding_info[i]); + + /* Remove a token: this is the idea of adaptive padding, since we have an + * ideal distribution that we want our distribution to look like. */ + if (!circpad_machine_remove_token(on_circ->padding_info[i])) { + /* If removing a token did not cause a transition, check if + * non-padding sent event should */ + circpad_machine_spec_transition(on_circ->padding_info[i], + CIRCPAD_EVENT_NONPADDING_SENT); + } + } FOR_EACH_ACTIVE_CIRCUIT_MACHINE_END; +} + +/** + * A "non-padding" cell has been received by this endpoint. React + * according to any padding state machines on the circuit. + * + * For origin circuits, this means we read a cell from the network. + * For middle relay circuits, this means we received a cell from the + * origin. + */ +void +circpad_cell_event_nonpadding_received(circuit_t *on_circ) +{ + FOR_EACH_ACTIVE_CIRCUIT_MACHINE_BEGIN(i, on_circ) { + /* First, update any RTT estimate */ + circpad_estimate_circ_rtt_on_received(on_circ, on_circ->padding_info[i]); + + circpad_machine_spec_transition(on_circ->padding_info[i], + CIRCPAD_EVENT_NONPADDING_RECV); + } FOR_EACH_ACTIVE_CIRCUIT_MACHINE_END; +} + +/** + * A padding cell has been sent from this endpoint. React + * according to any padding state machines on the circuit. + * + * For origin circuits, this means we sent a cell into the network. + * For middle relay circuits, this means we sent a cell towards the + * origin. + */ +void +circpad_cell_event_padding_sent(circuit_t *on_circ) +{ + FOR_EACH_ACTIVE_CIRCUIT_MACHINE_BEGIN(i, on_circ) { + circpad_machine_spec_transition(on_circ->padding_info[i], + CIRCPAD_EVENT_PADDING_SENT); + } FOR_EACH_ACTIVE_CIRCUIT_MACHINE_END; +} + +/** + * A padding cell has been received by this endpoint. React + * according to any padding state machines on the circuit. + * + * For origin circuits, this means we read a cell from the network. + * For middle relay circuits, this means we received a cell from the + * origin. + */ +void +circpad_cell_event_padding_received(circuit_t *on_circ) +{ + /* identical to padding sent */ + FOR_EACH_ACTIVE_CIRCUIT_MACHINE_BEGIN(i, on_circ) { + circpad_machine_spec_transition(on_circ->padding_info[i], + CIRCPAD_EVENT_PADDING_RECV); + } FOR_EACH_ACTIVE_CIRCUIT_MACHINE_END; +} + +/** + * An "infinite" delay has ben chosen from one of our histograms. + * + * "Infinite" delays mean don't send padding -- but they can also + * mean transition to another state depending on the state machine + * definitions. Check the rules and react accordingly. + * + * Return 1 if we decide to transition, 0 otherwise. + */ +circpad_decision_t +circpad_internal_event_infinity(circpad_machine_state_t *mi) +{ + return circpad_machine_spec_transition(mi, CIRCPAD_EVENT_INFINITY); +} + +/** + * All of the bins of our current state's histogram's are empty. + * + * Check to see if this means transition to another state, and if + * not, refill the tokens. + * + * Return 1 if we decide to transition, 0 otherwise. + */ +circpad_decision_t +circpad_internal_event_bins_empty(circpad_machine_state_t *mi) +{ + if (circpad_machine_spec_transition(mi, CIRCPAD_EVENT_BINS_EMPTY) + == CIRCPAD_STATE_CHANGED) { + return CIRCPAD_STATE_CHANGED; + } else { + /* If we dont transition, then we refill the tokens */ + circpad_machine_setup_tokens(mi); + return CIRCPAD_STATE_UNCHANGED; + } +} + +/** + * This state has used up its cell count. Emit the event and + * see if we transition. + * + * Return 1 if we decide to transition, 0 otherwise. + */ +circpad_decision_t +circpad_internal_event_state_length_up(circpad_machine_state_t *mi) +{ + return circpad_machine_spec_transition(mi, CIRCPAD_EVENT_LENGTH_COUNT); +} + +/** + * Returns true if the circuit matches the conditions. + */ +static inline bool +circpad_machine_conditions_met(origin_circuit_t *circ, + const circpad_machine_spec_t *machine) +{ + if (!(circpad_circ_purpose_to_mask(TO_CIRCUIT(circ)->purpose) + & machine->conditions.purpose_mask)) + return 0; + + if (machine->conditions.requires_vanguards) { + const or_options_t *options = get_options(); + + /* Pinned middles are effectively vanguards */ + if (!(options->HSLayer2Nodes || options->HSLayer3Nodes)) + return 0; + } + + /* We check for any bits set in the circuit state mask so that machines + * can say any of the following through their state bitmask: + * "I want to apply to circuits with either streams or no streams"; OR + * "I only want to apply to circuits with streams"; OR + * "I only want to apply to circuits without streams". */ + if (!(circpad_circuit_state(circ) & machine->conditions.state_mask)) + return 0; + + if (circuit_get_cpath_opened_len(circ) < machine->conditions.min_hops) + return 0; + + return 1; +} + +/** + * Returns a minimized representation of the circuit state. + * + * The padding code only cares if the circuit is building, + * opened, used for streams, and/or still has relay early cells. + * This returns a bitmask of all state properities that apply to + * this circuit. + */ +static inline +circpad_circuit_state_t +circpad_circuit_state(origin_circuit_t *circ) +{ + circpad_circuit_state_t retmask = 0; + + if (circ->p_streams) + retmask |= CIRCPAD_CIRC_STREAMS; + else + retmask |= CIRCPAD_CIRC_NO_STREAMS; + + /* We use has_opened to prevent cannibialized circs from flapping. */ + if (circ->has_opened) + retmask |= CIRCPAD_CIRC_OPENED; + else + retmask |= CIRCPAD_CIRC_BUILDING; + + if (circ->remaining_relay_early_cells > 0) + retmask |= CIRCPAD_CIRC_HAS_RELAY_EARLY; + else + retmask |= CIRCPAD_CIRC_HAS_NO_RELAY_EARLY; + + return retmask; +} + +/** + * Convert a normal circuit purpose into a bitmask that we can + * use for determining matching circuits. + */ +static inline +circpad_purpose_mask_t +circpad_circ_purpose_to_mask(uint8_t circ_purpose) +{ + /* Treat OR circ purposes as ignored. They should not be passed here*/ + if (BUG(circ_purpose <= CIRCUIT_PURPOSE_OR_MAX_)) { + return 0; + } + + /* Treat new client circuit purposes as "OMG ITS EVERYTHING". + * This also should not happen */ + if (BUG(circ_purpose - CIRCUIT_PURPOSE_OR_MAX_ - 1 > 32)) { + return CIRCPAD_PURPOSE_ALL; + } + + /* Convert the purpose to a bit position */ + return 1 << (circ_purpose - CIRCUIT_PURPOSE_OR_MAX_ - 1); +} + +/** + * Shut down any machines whose conditions no longer match + * the current circuit. + */ +static void +circpad_shutdown_old_machines(origin_circuit_t *on_circ) +{ + circuit_t *circ = TO_CIRCUIT(on_circ); + + FOR_EACH_ACTIVE_CIRCUIT_MACHINE_BEGIN(i, circ) { + if (!circpad_machine_conditions_met(on_circ, + circ->padding_machine[i])) { + // Clear machineinfo (frees timers) + circpad_circuit_machineinfo_free_idx(circ, i); + // Send padding negotiate stop + circpad_negotiate_padding(on_circ, + circ->padding_machine[i]->machine_num, + circ->padding_machine[i]->target_hopnum, + CIRCPAD_COMMAND_STOP); + } + } FOR_EACH_ACTIVE_CIRCUIT_MACHINE_END; +} + +/** + * Negotiate new machines that would apply to this circuit. + * + * This function checks to see if we have any free machine indexes, + * and for each free machine index, it initializes the most recently + * added origin-side padding machine that matches the target machine + * index and circuit conditions, and negotiates it with the appropriate + * middle relay. + */ +static void +circpad_add_matching_machines(origin_circuit_t *on_circ) +{ + circuit_t *circ = TO_CIRCUIT(on_circ); + +#ifdef TOR_UNIT_TESTS + /* Tests don't have to init our padding machines */ + if (!origin_padding_machines) + return; +#endif + + /* If padding negotiation failed before, do not try again */ + if (on_circ->padding_negotiation_failed) + return; + + FOR_EACH_CIRCUIT_MACHINE_BEGIN(i) { + /* If there is a padding machine info, this index is occupied. + * No need to check conditions for this index. */ + if (circ->padding_info[i]) + continue; + + /* We have a free machine index. Check the origin padding + * machines in reverse order, so that more recently added + * machines take priority over older ones. */ + SMARTLIST_FOREACH_REVERSE_BEGIN(origin_padding_machines, + circpad_machine_spec_t *, + machine) { + /* Machine definitions have a specific target machine index. + * This is so event ordering is deterministic with respect + * to which machine gets events first when there are two + * machines installed on a circuit. Make sure we only + * add this machine if its target machine index is free. */ + if (machine->machine_index == i && + circpad_machine_conditions_met(on_circ, machine)) { + + // We can only replace this machine if the target hopnum + // is the same, otherwise we'll get invalid data + if (circ->padding_machine[i]) { + if (circ->padding_machine[i]->target_hopnum != + machine->target_hopnum) + continue; + /* Replace it. (Don't free - is global). */ + circ->padding_machine[i] = NULL; + } + + /* Set up the machine immediately so that the slot is occupied. + * We will tear it down on error return, or if there is an error + * response from the relay. */ + circpad_setup_machine_on_circ(circ, machine); + if (circpad_negotiate_padding(on_circ, machine->machine_num, + machine->target_hopnum, + CIRCPAD_COMMAND_START) < 0) { + circpad_circuit_machineinfo_free_idx(circ, i); + circ->padding_machine[i] = NULL; + on_circ->padding_negotiation_failed = 1; + } else { + /* Success. Don't try any more machines */ + return; + } + } + } SMARTLIST_FOREACH_END(machine); + } FOR_EACH_CIRCUIT_MACHINE_END; +} + +/** + * Event that tells us we added a hop to an origin circuit. + * + * This event is used to decide if we should create a padding machine + * on a circuit. + */ +void +circpad_machine_event_circ_added_hop(origin_circuit_t *on_circ) +{ + /* Since our padding conditions do not specify a max_hops, + * all we can do is add machines here */ + circpad_add_matching_machines(on_circ); +} + +/** + * Event that tells us that an origin circuit is now built. + * + * Shut down any machines that only applied to un-built circuits. + * Activate any new ones. + */ +void +circpad_machine_event_circ_built(origin_circuit_t *circ) +{ + circpad_shutdown_old_machines(circ); + circpad_add_matching_machines(circ); +} + +/** + * Circpad purpose changed event. + * + * Shut down any machines that don't apply to our circ purpose. + * Activate any new ones that do. + */ +void +circpad_machine_event_circ_purpose_changed(origin_circuit_t *circ) +{ + circpad_shutdown_old_machines(circ); + circpad_add_matching_machines(circ); +} + +/** + * Event that tells us that an origin circuit is out of RELAY_EARLY + * cells. + * + * Shut down any machines that only applied to RELAY_EARLY circuits. + * Activate any new ones. + */ +void +circpad_machine_event_circ_has_no_relay_early(origin_circuit_t *circ) +{ + circpad_shutdown_old_machines(circ); + circpad_add_matching_machines(circ); +} + +/** + * Streams attached event. + * + * Called from link_apconn_to_circ() and handle_hs_exit_conn() + * + * Shut down any machines that only applied to machines without + * streams. Activate any new ones. + */ +void +circpad_machine_event_circ_has_streams(origin_circuit_t *circ) +{ + circpad_shutdown_old_machines(circ); + circpad_add_matching_machines(circ); +} + +/** + * Streams detached event. + * + * Called from circuit_detach_stream() + * + * Shut down any machines that only applied to machines without + * streams. Activate any new ones. + */ +void +circpad_machine_event_circ_has_no_streams(origin_circuit_t *circ) +{ + circpad_shutdown_old_machines(circ); + circpad_add_matching_machines(circ); +} + +/** + * Verify that padding is coming from the expected hop. + * + * Returns true if from_hop matches the target hop from + * one of our padding machines. + * + * Returns false if we're not an origin circuit, or if from_hop + * does not match one of the padding machines. + */ +bool +circpad_padding_is_from_expected_hop(circuit_t *circ, + crypt_path_t *from_hop) +{ + crypt_path_t *target_hop = NULL; + if (!CIRCUIT_IS_ORIGIN(circ)) + return 0; + + FOR_EACH_CIRCUIT_MACHINE_BEGIN(i) { + /* We have to check padding_machine and not padding_info/active + * machines here because padding may arrive after we shut down a + * machine. The info is gone, but the padding_machine waits + * for the padding_negotiated response to come back. */ + if (!circ->padding_machine[i]) + continue; + + target_hop = circuit_get_cpath_hop(TO_ORIGIN_CIRCUIT(circ), + circ->padding_machine[i]->target_hopnum); + + if (target_hop == from_hop) + return 1; + } FOR_EACH_CIRCUIT_MACHINE_END; + + return 0; +} + +/** + * Deliver circpad events for an "unrecognized cell". + * + * Unrecognized cells are sent to relays and are forwarded + * onto the next hop of their circuits. Unrecognized cells + * are by definition not padding. We need to tell relay-side + * state machines that a non-padding cell was sent or received, + * depending on the direction, so they can update their histograms + * and decide to pad or not. + */ +void +circpad_deliver_unrecognized_cell_events(circuit_t *circ, + cell_direction_t dir) +{ + // We should never see unrecognized cells at origin. + // Our caller emits a warn when this happens. + if (CIRCUIT_IS_ORIGIN(circ)) { + return; + } + + if (dir == CELL_DIRECTION_OUT) { + /* When direction is out (away from origin), then we received non-padding + cell coming from the origin to us. */ + circpad_cell_event_nonpadding_received(circ); + } else if (dir == CELL_DIRECTION_IN) { + /* It's in and not origin, so the cell is going away from us. + * So we are relaying a non-padding cell towards the origin. */ + circpad_cell_event_nonpadding_sent(circ); + } +} + +/** + * Deliver circpad events for "recognized" relay cells. + * + * Recognized cells are destined for this hop, either client or middle. + * Check if this is a padding cell or not, and send the appropiate + * received event. + */ +void +circpad_deliver_recognized_relay_cell_events(circuit_t *circ, + uint8_t relay_command, + crypt_path_t *layer_hint) +{ + /* Padding negotiate cells are ignored by the state machines + * for simplicity. */ + if (relay_command == RELAY_COMMAND_PADDING_NEGOTIATE || + relay_command == RELAY_COMMAND_PADDING_NEGOTIATED) { + return; + } + + if (relay_command == RELAY_COMMAND_DROP) { + rep_hist_padding_count_read(PADDING_TYPE_DROP); + + if (CIRCUIT_IS_ORIGIN(circ)) { + if (circpad_padding_is_from_expected_hop(circ, layer_hint)) { + circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), 0); + } else { + /* This is unexpected padding. Ignore it for now. */ + return; + } + } + + /* The cell should be recognized by now, which means that we are on the + destination, which means that we received a padding cell. We might be + the client or the Middle node, still, because leaky-pipe. */ + circpad_cell_event_padding_received(circ); + log_fn(LOG_INFO, LD_CIRC, "Got padding cell on %s circuit %u.", + CIRCUIT_IS_ORIGIN(circ) ? "origin" : "non-origin", + CIRCUIT_IS_ORIGIN(circ) ? + TO_ORIGIN_CIRCUIT(circ)->global_identifier : 0); + } else { + /* We received a non-padding cell on the edge */ + circpad_cell_event_nonpadding_received(circ); + } +} + +/** + * Deliver circpad events for relay cells sent from us. + * + * If this is a padding cell, update our padding stats + * and deliver the event. Otherwise just deliver the event. + */ +void +circpad_deliver_sent_relay_cell_events(circuit_t *circ, + uint8_t relay_command) +{ + /* Padding negotiate cells are ignored by the state machines + * for simplicity. */ + if (relay_command == RELAY_COMMAND_PADDING_NEGOTIATE || + relay_command == RELAY_COMMAND_PADDING_NEGOTIATED) { + return; + } + + /* RELAY_COMMAND_DROP is the multi-hop (aka circuit-level) padding cell in + * tor. (CELL_PADDING is a channel-level padding cell, which is not relayed + * or processed here) */ + if (relay_command == RELAY_COMMAND_DROP) { + /* Optimization: The event for RELAY_COMMAND_DROP is sent directly + * from circpad_send_padding_cell_for_callback(). This is to avoid + * putting a cell_t and a relay_header_t on the stack repeatedly + * if we decide to send a long train of padidng cells back-to-back + * with 0 delay. So we do nothing here. */ + return; + } else { + /* This is a non-padding cell sent from the client or from + * this node. */ + circpad_cell_event_nonpadding_sent(circ); + } +} + +/** + * Initialize the states array for a circpad machine. + */ +void +circpad_machine_states_init(circpad_machine_spec_t *machine, + circpad_statenum_t num_states) +{ + if (BUG(num_states > CIRCPAD_MAX_MACHINE_STATES)) { + num_states = CIRCPAD_MAX_MACHINE_STATES; + } + + machine->num_states = num_states; + machine->states = tor_malloc_zero(sizeof(circpad_state_t)*num_states); + + /* Initialize the default next state for all events to + * "ignore" -- if events aren't specified, they are ignored. */ + for (circpad_statenum_t s = 0; s < num_states; s++) { + for (int e = 0; e < CIRCPAD_NUM_EVENTS; e++) { + machine->states[s].next_state[e] = CIRCPAD_STATE_IGNORE; + } + } +} + +static void +circpad_setup_machine_on_circ(circuit_t *on_circ, + const circpad_machine_spec_t *machine) +{ + if (CIRCUIT_IS_ORIGIN(on_circ) && !machine->is_origin_side) { + log_fn(LOG_WARN, LD_BUG, + "Can't set up non-origin machine on origin circuit!"); + return; + } + + if (!CIRCUIT_IS_ORIGIN(on_circ) && machine->is_origin_side) { + log_fn(LOG_WARN, LD_BUG, + "Can't set up origin machine on non-origin circuit!"); + return; + } + + tor_assert_nonfatal(on_circ->padding_machine[machine->machine_index] + == NULL); + tor_assert_nonfatal(on_circ->padding_info[machine->machine_index] == NULL); + + on_circ->padding_info[machine->machine_index] = + circpad_circuit_machineinfo_new(on_circ, machine->machine_index); + on_circ->padding_machine[machine->machine_index] = machine; +} + +/* These padding machines are only used for tests pending #28634. */ +#ifdef TOR_UNIT_TESTS +static void +circpad_circ_client_machine_init(void) +{ + circpad_machine_spec_t *circ_client_machine + = tor_malloc_zero(sizeof(circpad_machine_spec_t)); + + // XXX: Better conditions for merge.. Or disable this machine in + // merge? + circ_client_machine->conditions.min_hops = 2; + circ_client_machine->conditions.state_mask = + CIRCPAD_CIRC_BUILDING|CIRCPAD_CIRC_OPENED|CIRCPAD_CIRC_HAS_RELAY_EARLY; + circ_client_machine->conditions.purpose_mask = CIRCPAD_PURPOSE_ALL; + + circ_client_machine->target_hopnum = 2; + circ_client_machine->is_origin_side = 1; + + /* Start, gap, burst */ + circpad_machine_states_init(circ_client_machine, 3); + + circ_client_machine->states[CIRCPAD_STATE_START]. + next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_BURST; + + circ_client_machine->states[CIRCPAD_STATE_BURST]. + next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_BURST; + circ_client_machine->states[CIRCPAD_STATE_BURST]. + next_state[CIRCPAD_EVENT_PADDING_RECV] = CIRCPAD_STATE_BURST; + + /* If we are in burst state, and we send a non-padding cell, then we cancel + the timer for the next padding cell: + We dont want to send fake extends when actual extends are going on */ + circ_client_machine->states[CIRCPAD_STATE_BURST]. + next_state[CIRCPAD_EVENT_NONPADDING_SENT] = CIRCPAD_STATE_CANCEL; + + circ_client_machine->states[CIRCPAD_STATE_BURST]. + next_state[CIRCPAD_EVENT_BINS_EMPTY] = CIRCPAD_STATE_END; + + circ_client_machine->states[CIRCPAD_STATE_BURST].token_removal = + CIRCPAD_TOKEN_REMOVAL_CLOSEST; + + // FIXME: Tune this histogram + circ_client_machine->states[CIRCPAD_STATE_BURST].histogram_len = 2; + circ_client_machine->states[CIRCPAD_STATE_BURST].start_usec = 500; + circ_client_machine->states[CIRCPAD_STATE_BURST].range_usec = 1000000; + /* We have 5 tokens in the histogram, which means that all circuits will look + * like they have 7 hops (since we start this machine after the second hop, + * and tokens are decremented for any valid hops, and fake extends are + * used after that -- 2+5==7). */ + circ_client_machine->states[CIRCPAD_STATE_BURST].histogram[0] = 5; + circ_client_machine->states[CIRCPAD_STATE_BURST].histogram_total_tokens = 5; + + circ_client_machine->machine_num = smartlist_len(origin_padding_machines); + smartlist_add(origin_padding_machines, circ_client_machine); +} + +static void +circpad_circ_responder_machine_init(void) +{ + circpad_machine_spec_t *circ_responder_machine + = tor_malloc_zero(sizeof(circpad_machine_spec_t)); + + /* Shut down the machine after we've sent enough packets */ + circ_responder_machine->should_negotiate_end = 1; + + /* The relay-side doesn't care what hopnum it is, but for consistency, + * let's match the client */ + circ_responder_machine->target_hopnum = 2; + circ_responder_machine->is_origin_side = 0; + + /* Start, gap, burst */ + circpad_machine_states_init(circ_responder_machine, 3); + + /* This is the settings of the state machine. In the future we are gonna + serialize this into the consensus or the torrc */ + + /* We transition to the burst state on padding receive and on non-padding + * recieve */ + circ_responder_machine->states[CIRCPAD_STATE_START]. + next_state[CIRCPAD_EVENT_PADDING_RECV] = CIRCPAD_STATE_BURST; + circ_responder_machine->states[CIRCPAD_STATE_START]. + next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_BURST; + + /* Inside the burst state we _stay_ in the burst state when a non-padding + * is sent */ + circ_responder_machine->states[CIRCPAD_STATE_BURST]. + next_state[CIRCPAD_EVENT_NONPADDING_SENT] = CIRCPAD_STATE_BURST; + + /* Inside the burst state we transition to the gap state when we receive a + * padding cell */ + circ_responder_machine->states[CIRCPAD_STATE_BURST]. + next_state[CIRCPAD_EVENT_PADDING_RECV] = CIRCPAD_STATE_GAP; + + /* These describe the padding charasteristics when in burst state */ + + /* use_rtt_estimate tries to estimate how long padding cells take to go from + C->M, and uses that as what as the base of the histogram */ + circ_responder_machine->states[CIRCPAD_STATE_BURST].use_rtt_estimate = 1; + /* The histogram is 2 bins: an empty one, and infinity */ + circ_responder_machine->states[CIRCPAD_STATE_BURST].histogram_len = 2; + circ_responder_machine->states[CIRCPAD_STATE_BURST].start_usec = 5000; + circ_responder_machine->states[CIRCPAD_STATE_BURST].range_usec = 1000000; + /* During burst state we wait forever for padding to arrive. + + We are waiting for a padding cell from the client to come in, so that we + respond, and we immitate how extend looks like */ + circ_responder_machine->states[CIRCPAD_STATE_BURST].histogram[0] = 0; + // Only infinity bin: + circ_responder_machine->states[CIRCPAD_STATE_BURST].histogram[1] = 1; + circ_responder_machine->states[CIRCPAD_STATE_BURST]. + histogram_total_tokens = 1; + + /* From the gap state, we _stay_ in the gap state, when we receive padding + * or non padding */ + circ_responder_machine->states[CIRCPAD_STATE_GAP]. + next_state[CIRCPAD_EVENT_PADDING_RECV] = CIRCPAD_STATE_GAP; + circ_responder_machine->states[CIRCPAD_STATE_GAP]. + next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_GAP; + + /* And from the gap state, we go to the end, when the bins are empty or a + * non-padding cell is sent */ + circ_responder_machine->states[CIRCPAD_STATE_GAP]. + next_state[CIRCPAD_EVENT_BINS_EMPTY] = CIRCPAD_STATE_END; + circ_responder_machine->states[CIRCPAD_STATE_GAP]. + next_state[CIRCPAD_EVENT_NONPADDING_SENT] = CIRCPAD_STATE_END; + + // FIXME: Tune this histogram + + /* The gap state is the delay you wait after you receive a padding cell + before you send a padding response */ + circ_responder_machine->states[CIRCPAD_STATE_GAP].use_rtt_estimate = 1; + circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram_len = 6; + circ_responder_machine->states[CIRCPAD_STATE_GAP].start_usec = 5000; + circ_responder_machine->states[CIRCPAD_STATE_GAP].range_usec = 1000000; + circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram[0] = 0; + circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram[1] = 1; + circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram[2] = 2; + circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram[3] = 2; + circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram[4] = 1; + /* Total number of tokens */ + circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram_total_tokens = 6; + circ_responder_machine->states[CIRCPAD_STATE_GAP].token_removal = + CIRCPAD_TOKEN_REMOVAL_CLOSEST_USEC; + + circ_responder_machine->machine_num = smartlist_len(relay_padding_machines); + smartlist_add(relay_padding_machines, circ_responder_machine); +} +#endif + +/** + * Initialize all of our padding machines. + * + * This is called at startup. It sets up some global machines, and then + * loads some from torrc, and from the tor consensus. + */ +void +circpad_machines_init(void) +{ + tor_assert_nonfatal(origin_padding_machines == NULL); + tor_assert_nonfatal(relay_padding_machines == NULL); + + origin_padding_machines = smartlist_new(); + relay_padding_machines = smartlist_new(); + + // TODO: Parse machines from consensus and torrc +#ifdef TOR_UNIT_TESTS + circpad_circ_client_machine_init(); + circpad_circ_responder_machine_init(); +#endif +} + +/** + * Free our padding machines + */ +void +circpad_machines_free(void) +{ + if (origin_padding_machines) { + SMARTLIST_FOREACH(origin_padding_machines, + circpad_machine_spec_t *, + m, tor_free(m->states); tor_free(m)); + smartlist_free(origin_padding_machines); + } + + if (relay_padding_machines) { + SMARTLIST_FOREACH(relay_padding_machines, + circpad_machine_spec_t *, + m, tor_free(m->states); tor_free(m)); + smartlist_free(relay_padding_machines); + } +} + +/** + * Check the Protover info to see if a node supports padding. + */ +static bool +circpad_node_supports_padding(const node_t *node) +{ + if (node->rs) { + log_fn(LOG_INFO, LD_CIRC, "Checking padding: %s", + node->rs->pv.supports_padding ? "supported" : "unsupported"); + return node->rs->pv.supports_padding; + } + + log_fn(LOG_INFO, LD_CIRC, "Empty routerstatus in padding check"); + return 0; +} + +/** + * Get a node_t for the nth hop in our circuit, starting from 1. + * + * Returns node_t from the consensus for that hop, if it is opened. + * Otherwise returns NULL. + */ +static const node_t * +circuit_get_nth_node(origin_circuit_t *circ, int hop) +{ + crypt_path_t *iter = circuit_get_cpath_hop(circ, hop); + + if (!iter || iter->state != CPATH_STATE_OPEN) + return NULL; + + return node_get_by_id(iter->extend_info->identity_digest); +} + +/** + * Return true if a particular circuit supports padding + * at the desired hop. + */ +static bool +circpad_circuit_supports_padding(origin_circuit_t *circ, + int target_hopnum) +{ + const node_t *hop; + + if (!(hop = circuit_get_nth_node(circ, target_hopnum))) { + return 0; + } + + return circpad_node_supports_padding(hop); +} + +/** + * Try to negotiate padding. + * + * Returns -1 on error, 0 on success. + */ +signed_error_t +circpad_negotiate_padding(origin_circuit_t *circ, + circpad_machine_num_t machine, + uint8_t target_hopnum, + uint8_t command) +{ + circpad_negotiate_t type; + cell_t cell; + ssize_t len; + + /* Check that the target hop lists support for padding in + * its ProtoVer fields */ + if (!circpad_circuit_supports_padding(circ, target_hopnum)) { + return -1; + } + + memset(&cell, 0, sizeof(cell_t)); + memset(&type, 0, sizeof(circpad_negotiate_t)); + // This gets reset to RELAY_EARLY appropriately by + // relay_send_command_from_edge_. At least, it looks that way. + // QQQ-MP-AP: Verify that. + cell.command = CELL_RELAY; + + circpad_negotiate_set_command(&type, command); + circpad_negotiate_set_version(&type, 0); + circpad_negotiate_set_machine_type(&type, machine); + + if ((len = circpad_negotiate_encode(cell.payload, CELL_PAYLOAD_SIZE, + &type)) < 0) + return -1; + + log_fn(LOG_INFO,LD_CIRC, "Negotiating padding on circuit %u", + circ->global_identifier); + + return circpad_send_command_to_hop(circ, target_hopnum, + RELAY_COMMAND_PADDING_NEGOTIATE, + cell.payload, len); +} + +/** + * Try to negotiate padding. + * + * Returns 1 if successful (or already set up), 0 otherwise. + */ +bool +circpad_padding_negotiated(circuit_t *circ, + circpad_machine_num_t machine, + uint8_t command, + uint8_t response) +{ + circpad_negotiated_t type; + cell_t cell; + ssize_t len; + + memset(&cell, 0, sizeof(cell_t)); + memset(&type, 0, sizeof(circpad_negotiated_t)); + // This gets reset to RELAY_EARLY appropriately by + // relay_send_command_from_edge_. At least, it looks that way. + // QQQ-MP-AP: Verify that. + cell.command = CELL_RELAY; + + circpad_negotiated_set_command(&type, command); + circpad_negotiated_set_response(&type, response); + circpad_negotiated_set_version(&type, 0); + circpad_negotiated_set_machine_type(&type, machine); + + if ((len = circpad_negotiated_encode(cell.payload, CELL_PAYLOAD_SIZE, + &type)) < 0) + return 0; + + /* Use relay_send because we're from the middle to the origin. We don't + * need to specify a target hop or layer_hint. */ + return relay_send_command_from_edge(0, circ, + RELAY_COMMAND_PADDING_NEGOTIATED, + (void*)cell.payload, + (size_t)len, NULL) == 0; +} + +/** + * Parse and react to a padding_negotiate cell. + * + * This is called at the middle node upon receipt of the client's choice of + * state machine, so that it can use the requested state machine index, if + * it is available. + * + * Returns -1 on error, 0 on success. + */ +signed_error_t +circpad_handle_padding_negotiate(circuit_t *circ, cell_t *cell) +{ + int retval = 0; + circpad_negotiate_t *negotiate; + + if (CIRCUIT_IS_ORIGIN(circ)) { + log_fn(LOG_PROTOCOL_WARN, LD_CIRC, + "Padding negotiate cell unsupported at origin."); + return -1; + } + + if (circpad_negotiate_parse(&negotiate, cell->payload+RELAY_HEADER_SIZE, + CELL_PAYLOAD_SIZE-RELAY_HEADER_SIZE) < 0) { + log_fn(LOG_PROTOCOL_WARN, LD_CIRC, + "Received malformed PADDING_NEGOTIATE cell; dropping."); + return -1; + } + + if (negotiate->command == CIRCPAD_COMMAND_STOP) { + /* Free the machine corresponding to this machine type */ + if (free_circ_machineinfos_with_machine_num(circ, + negotiate->machine_type)) { + log_info(LD_CIRC, "Received STOP command for machine %u", + negotiate->machine_type); + goto done; + } + log_fn(LOG_PROTOCOL_WARN, LD_CIRC, + "Received circuit padding stop command for unknown machine."); + goto err; + } else if (negotiate->command == CIRCPAD_COMMAND_START) { + SMARTLIST_FOREACH_BEGIN(relay_padding_machines, + const circpad_machine_spec_t *, m) { + if (m->machine_num == negotiate->machine_type) { + circpad_setup_machine_on_circ(circ, m); + goto done; + } + } SMARTLIST_FOREACH_END(m); + } + + err: + retval = -1; + + done: + circpad_padding_negotiated(circ, negotiate->machine_type, + negotiate->command, + (retval == 0) ? CIRCPAD_RESPONSE_OK : CIRCPAD_RESPONSE_ERR); + circpad_negotiate_free(negotiate); + + return retval; +} + +/** + * Parse and react to a padding_negotiated cell. + * + * This is called at the origin upon receipt of the middle's response + * to our choice of state machine. + * + * Returns -1 on error, 0 on success. + */ +signed_error_t +circpad_handle_padding_negotiated(circuit_t *circ, cell_t *cell, + crypt_path_t *layer_hint) +{ + circpad_negotiated_t *negotiated; + + if (!CIRCUIT_IS_ORIGIN(circ)) { + log_fn(LOG_PROTOCOL_WARN, LD_CIRC, + "Padding negotiated cell unsupported at non-origin."); + return -1; + } + + /* Verify this came from the expected hop */ + if (!circpad_padding_is_from_expected_hop(circ, layer_hint)) { + log_fn(LOG_WARN, LD_CIRC, + "Padding negotiated cell from wrong hop!"); + return -1; + } + + if (circpad_negotiated_parse(&negotiated, cell->payload+RELAY_HEADER_SIZE, + CELL_PAYLOAD_SIZE-RELAY_HEADER_SIZE) < 0) { + log_fn(LOG_PROTOCOL_WARN, LD_CIRC, + "Received malformed PADDING_NEGOTIATED cell; " + "dropping."); + return -1; + } + + if (negotiated->command == CIRCPAD_COMMAND_STOP) { + /* There may not be a padding_info here if we shut down the + * machine in circpad_shutdown_old_machines(). Or, if + * circpad_add_matching_matchines() added a new machine, + * there may be a padding_machine for a different machine num + * than this response. */ + free_circ_machineinfos_with_machine_num(circ, negotiated->machine_type); + } else if (negotiated->command == CIRCPAD_COMMAND_START && + negotiated->response == CIRCPAD_RESPONSE_ERR) { + // This can happen due to consensus drift.. free the machines + // and be sad + free_circ_machineinfos_with_machine_num(circ, negotiated->machine_type); + TO_ORIGIN_CIRCUIT(circ)->padding_negotiation_failed = 1; + log_fn(LOG_PROTOCOL_WARN, LD_CIRC, + "Middle node did not accept our padding request."); + } + + circpad_negotiated_free(negotiated); + return 0; +} + +/* Serialization */ +// TODO: Should we use keyword=value here? Are there helpers for that? +#if 0 +static void +circpad_state_serialize(const circpad_state_t *state, + smartlist_t *chunks) +{ + smartlist_add_asprintf(chunks, " %u", state->histogram[0]); + for (int i = 1; i < state->histogram_len; i++) { + smartlist_add_asprintf(chunks, ",%u", + state->histogram[i]); + } + + smartlist_add_asprintf(chunks, " 0x%x", + state->transition_cancel_events); + + for (int i = 0; i < CIRCPAD_NUM_STATES; i++) { + smartlist_add_asprintf(chunks, ",0x%x", + state->transition_events[i]); + } + + smartlist_add_asprintf(chunks, " %u %u", + state->use_rtt_estimate, + state->token_removal); +} + +char * +circpad_machine_spec_to_string(const circpad_machine_spec_t *machine) +{ + smartlist_t *chunks = smartlist_new(); + char *out; + (void)machine; + + circpad_state_serialize(&machine->start, chunks); + circpad_state_serialize(&machine->gap, chunks); + circpad_state_serialize(&machine->burst, chunks); + + out = smartlist_join_strings(chunks, "", 0, NULL); + + SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp)); + smartlist_free(chunks); + return out; +} + +// XXX: Writeme +const circpad_machine_spec_t * +circpad_string_to_machine(const char *str) +{ + (void)str; + return NULL; +} + +#endif diff --git a/src/core/or/circuitpadding.h b/src/core/or/circuitpadding.h new file mode 100644 index 0000000000..92fd4fc2d5 --- /dev/null +++ b/src/core/or/circuitpadding.h @@ -0,0 +1,696 @@ +/* + * Copyright (c) 2017-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file circuitpadding.h + * \brief Header file for circuitpadding.c. + **/ + +#ifndef TOR_CIRCUITPADDING_H +#define TOR_CIRCUITPADDING_H + +#include "src/trunnel/circpad_negotiation.h" +#include "lib/evloop/timers.h" + +struct circuit_t; +struct origin_circuit_t; +struct cell_t; + +/** + * Signed error return with the specific property that negative + * values mean error codes of various semantics, 0 means success, + * and positive values are unused. + * + * XXX: Tor uses this concept a lot but just calls it int. Should we move + * this somewhere centralized? Where? + */ +typedef int signed_error_t; + +/** + * These constants specify the types of events that can cause + * transitions between state machine states. + * + * Note that SENT and RECV are relative to this endpoint. For + * relays, SENT means packets destined towards the client and + * RECV means packets destined towards the relay. On the client, + * SENT means packets destined towards the relay, where as RECV + * means packets destined towards the client. + */ +typedef enum { + /* A non-padding cell was received. */ + CIRCPAD_EVENT_NONPADDING_RECV = 0, + /* A non-padding cell was sent. */ + CIRCPAD_EVENT_NONPADDING_SENT = 1, + /* A padding cell (RELAY_COMMAND_DROP) was sent. */ + CIRCPAD_EVENT_PADDING_SENT = 2, + /* A padding cell was received. */ + CIRCPAD_EVENT_PADDING_RECV = 3, + /* We tried to schedule padding but we ended up picking the infinity bin + * which means that padding was delayed infinitely */ + CIRCPAD_EVENT_INFINITY = 4, + /* All histogram bins are empty (we are out of tokens) */ + CIRCPAD_EVENT_BINS_EMPTY = 5, + /* just a counter of the events above */ + CIRCPAD_EVENT_LENGTH_COUNT = 6 +} circpad_event_t; +#define CIRCPAD_NUM_EVENTS ((int)CIRCPAD_EVENT_LENGTH_COUNT+1) + +/** Boolean type that says if we decided to transition states or not */ +typedef enum { + CIRCPAD_STATE_UNCHANGED = 0, + CIRCPAD_STATE_CHANGED = 1 +} circpad_decision_t; + +/** The type for the things in histogram bins (aka tokens) */ +typedef uint32_t circpad_hist_token_t; + +/** The type for histogram indexes (needs to be negative for errors) */ +typedef int8_t circpad_hist_index_t; + +/** The type for absolute time, from monotime_absolute_usec() */ +typedef uint64_t circpad_time_t; + +/** The type for timer delays, in microseconds */ +typedef uint32_t circpad_delay_t; + +/** + * An infinite padding cell delay means don't schedule any padding -- + * simply wait until a different event triggers a transition. + * + * This means that the maximum delay we can scedule is UINT32_MAX-1 + * microseconds, or about 4300 seconds (1.25 hours). + * XXX: Is this enough if we want to simulate light, intermittent + * activity on an onion service? + */ +#define CIRCPAD_DELAY_INFINITE (UINT32_MAX) + +/** + * Macro to clarify when we're checking the infinity bin. + * + * Works with either circpad_state_t or circpad_machine_state_t + */ +#define CIRCPAD_INFINITY_BIN(mi) ((mi)->histogram_len-1) + +/** + * These constants form a bitfield that specifies when a state machine + * should be applied to a circuit. + * + * If any of these elements is set, then the circuit will be tested against + * that specific condition. If an element is unset, then we don't test it. + * (E.g. If neither NO_STREAMS or STREAMS are set, then we will not care + * whether a circuit has streams attached when we apply a state machine) + * + * The helper function circpad_circuit_state() converts circuit state + * flags into this more compact representation. + */ +typedef enum { + /* Only apply machine if the circuit is still building */ + CIRCPAD_CIRC_BUILDING = 1<<0, + /* Only apply machine if the circuit is open */ + CIRCPAD_CIRC_OPENED = 1<<1, + /* Only apply machine if the circuit has no attached streams */ + CIRCPAD_CIRC_NO_STREAMS = 1<<2, + /* Only apply machine if the circuit has attached streams */ + CIRCPAD_CIRC_STREAMS = 1<<3, + /* Only apply machine if the circuit still allows RELAY_EARLY cells */ + CIRCPAD_CIRC_HAS_RELAY_EARLY = 1<<4, + /* Only apply machine if the circuit has depleted its RELAY_EARLY cells + * allowance. */ + CIRCPAD_CIRC_HAS_NO_RELAY_EARLY = 1<<5 +} circpad_circuit_state_t; + +/** Bitmask that says "apply this machine to all states" */ +#define CIRCPAD_STATE_ALL \ + (CIRCPAD_CIRC_BUILDING|CIRCPAD_CIRC_OPENED| \ + CIRCPAD_CIRC_STREAMS|CIRCPAD_CIRC_NO_STREAMS| \ + CIRCPAD_CIRC_HAS_RELAY_EARLY|CIRCPAD_CIRC_HAS_NO_RELAY_EARLY) + +/** + * A compact circuit purpose bitfield mask that allows us to compactly + * specify which circuit purposes a machine should apply to. + * + * The helper function circpad_circ_purpose_to_mask() converts circuit + * purposes into bit positions in this bitmask. + */ +typedef uint32_t circpad_purpose_mask_t; + +/** Bitmask that says "apply this machine to all purposes". */ +#define CIRCPAD_PURPOSE_ALL (0xFFFFFFFF) + +/** + * This type specifies all of the conditions that must be met before + * a client decides to initiate padding on a circuit. + * + * A circuit must satisfy every sub-field in this type in order + * to be considered to match the conditions. + */ +typedef struct circpad_machine_conditions_t { + /** Only apply the machine *if* the circuit has at least this many hops */ + unsigned min_hops : 3; + + /** Only apply the machine *if* vanguards are enabled */ + unsigned requires_vanguards : 1; + + /** Only apply the machine *if* the circuit's state matches any of + * the bits set in this bitmask. */ + circpad_circuit_state_t state_mask; + + /** Only apply a machine *if* the circuit's purpose matches one + * of the bits set in this bitmask */ + circpad_purpose_mask_t purpose_mask; + +} circpad_machine_conditions_t; + +/** + * Token removal strategy options. + * + * The WTF-PAD histograms are meant to specify a target distribution to shape + * traffic towards. This is accomplished by removing tokens from the histogram + * when either padding or non-padding cells are sent. + * + * When we see a non-padding cell at a particular time since the last cell, you + * remove a token from the corresponding delay bin. These flags specify + * which bin to choose if that bin is already empty. + */ +typedef enum { + /** Don't remove any tokens */ + CIRCPAD_TOKEN_REMOVAL_NONE = 0, + /** + * Remove from the first non-zero higher bin index when current is zero. + * This is the recommended strategy from the Adaptive Padding paper. */ + CIRCPAD_TOKEN_REMOVAL_HIGHER = 1, + /** Remove from the first non-zero lower bin index when current is empty. */ + CIRCPAD_TOKEN_REMOVAL_LOWER = 2, + /** Remove from the closest non-zero bin index when current is empty. */ + CIRCPAD_TOKEN_REMOVAL_CLOSEST = 3, + /** Remove from the closest bin by time value (since bins are + * exponentially spaced). */ + CIRCPAD_TOKEN_REMOVAL_CLOSEST_USEC = 4, + /** Only remove from the exact bin corresponding to this delay. If + * the bin is 0, simply do nothing. Don't pick another bin. */ + CIRCPAD_TOKEN_REMOVAL_EXACT = 5 +} circpad_removal_t; + +/** + * Distribution types supported by circpad_distribution_sample(). + * + * These can be used instead of histograms for the inter-packet + * timing distribution, or to specify a distribution on the number + * of cells that can be sent while in a specific state of the state + * machine. */ +typedef enum { + CIRCPAD_DIST_NONE = 0, + CIRCPAD_DIST_UNIFORM = 1, + CIRCPAD_DIST_LOGISTIC = 2, + CIRCPAD_DIST_LOG_LOGISTIC = 3, + CIRCPAD_DIST_GEOMETRIC = 4, + CIRCPAD_DIST_WEIBULL = 5, + CIRCPAD_DIST_PARETO = 6 +} circpad_distribution_type_t; + +/** + * Distribution information. + * + * This type specifies a specific distribution above, as well as + * up to two parameters for that distribution. The specific + * per-distribution meaning of these parameters is specified + * in circpad_distribution_sample(). + */ +typedef struct circpad_distribution_t { + circpad_distribution_type_t type; + double param1; + double param2; +} circpad_distribution_t; + +/** State number type. Represents current state of state machine. */ +typedef uint16_t circpad_statenum_t; +#define CIRCPAD_STATENUM_MAX (UINT16_MAX) + +/** A histogram is used to sample padding delays given a machine state. This + * constant defines the maximum histogram width (i.e. the max number of bins) + * + * Each histogram bin is twice as large as the previous. Two exceptions: The + * first bin has zero width (which means that minimum delay is applied to the + * next padding cell), and the last bin (infinity bin) has infinite width + * (which means that the next padding cell will be delayed infinitely). */ +#define CIRCPAD_MAX_HISTOGRAM_LEN (sizeof(circpad_delay_t)*8 + 1) + +/** + * A state of a padding state machine. The information here are immutable and + * represent the initial form of the state; it does not get updated as things + * happen. The mutable information that gets updated in runtime are carried in + * a circpad_machine_state_t. + * + * This struct describes the histograms and parameters of a single + * state in the adaptive padding machine. Instances of this struct + * exist in global circpad machine definitions that come from torrc + * or the consensus. + */ +typedef struct circpad_state_t { + /** If a histogram is used for this state, this specifies the number of bins + * of this histogram. Histograms must have at least 2 bins. + * + * If a delay probability distribution is used for this state, this is set + * to 0. */ + circpad_hist_index_t histogram_len; + /** The histogram itself: an array of uint16s of tokens, whose + * widths are exponentially spaced, in microseconds */ + circpad_hist_token_t histogram[CIRCPAD_MAX_HISTOGRAM_LEN]; + /** Total number of tokens in this histogram. This is a constant and is *not* + * decremented every time we spend a token. It's used for initializing and + * refilling the histogram. */ + uint32_t histogram_total_tokens; + + /** Minimum padding delay of this state in microseconds. + * + * If histograms are used, this is the left (and right) bound of the first + * bin (since it has zero width). + * + * If a delay probability distribution is used, this represents the minimum + * delay we can sample from the distribution. + */ + circpad_delay_t start_usec; + + /** If histograms are used, this is the width of the whole histogram in + * microseconds, and it's used to calculate individual bin width. + * + * If a delay probability distribution is used, this is used as the max + * delay we can sample from the distribution. + */ + circpad_delay_t range_usec; + + /** + * Represents a delay probability distribution (aka IAT distribution). It's a + * parametrized way of encoding inter-packet delay information in + * microseconds. It can be used instead of histograms. + * + * If it is used, token_removal below must be set to + * CIRCPAD_TOKEN_REMOVAL_NONE. + * + * Start_usec, range_sec, and rtt_estimates are still applied to the + * results of sampling from this distribution (range_sec is used as a max). + */ + circpad_distribution_t iat_dist; + + /** + * The length dist is a parameterized way of encoding how long this + * state machine runs in terms of sent padding cells or all + * sent cells. Values are sampled from this distribution, clamped + * to max_len, and then start_len is added to that value. + * + * It may be specified instead of or in addition to + * the infinity bins and bins empty conditions. */ + circpad_distribution_t length_dist; + /** A minimum length value, added to the output of length_dist */ + uint16_t start_length; + /** A cap on the length value that can be sampled from the length_dist */ + uint64_t max_length; + + /** Should we decrement length when we see a nonpadding packet? + * XXX: Are there any machines that actually want to set this to 0? There may + * not be. OTOH, it's only a bit.. */ + unsigned length_includes_nonpadding : 1; + + /** + * This is an array that specifies the next state to transition to upon + * receipt an event matching the indicated array index. + * + * This aborts our scheduled packet and switches to the state + * corresponding to the index of the array. Tokens are filled upon + * this transition. + * + * States are allowed to transition to themselves, which means re-schedule + * a new padding timer. They are also allowed to temporarily "transition" + * to the "IGNORE" and "CANCEL" pseudo-states. See #defines below + * for details on state behavior and meaning. + */ + circpad_statenum_t next_state[CIRCPAD_NUM_EVENTS]; + + /** + * If true, estimate the RTT from this relay to the exit/website and add that + * to start_usec for use as the histogram bin 0 start delay. + * + * Right now this is only supported for relay-side state machines. + */ + unsigned use_rtt_estimate : 1; + + /** This specifies the token removal strategy to use upon padding and + * non-padding activity. */ + circpad_removal_t token_removal; +} circpad_state_t; + +/** + * The start state for this machine. + * + * In the original WTF-PAD, this is only used for transition to/from + * the burst state. All other fields are not used. But to simplify the + * code we've made it a first-class state. This has no performance + * consequences, but may make naive serialization of the state machine + * large, if we're not careful about how we represent empty fields. + */ +#define CIRCPAD_STATE_START 0 + +/** + * The burst state for this machine. + * + * In the original Adaptive Padding algorithm and in WTF-PAD + * (https://www.freehaven.net/anonbib/cache/ShWa-Timing06.pdf and + * https://www.cs.kau.se/pulls/hot/thebasketcase-wtfpad/), the burst + * state serves to detect bursts in traffic. This is done by using longer + * delays in its histogram, which represent the expected delays between + * bursts of packets in the target stream. If this delay expires without a + * real packet being sent, the burst state sends a padding packet and then + * immediately transitions to the gap state, which is used to generate + * a synthetic padding packet train. In this implementation, this transition + * needs to be explicitly specified in the burst state's transition events. + * + * Because of this flexibility, other padding mechanisms can transition + * between these two states arbitrarily, to encode other dynamics of + * target traffic. + */ +#define CIRCPAD_STATE_BURST 1 + +/** + * The gap state for this machine. + * + * In the original Adaptive Padding algorithm and in WTF-PAD, the gap + * state serves to simulate an artificial packet train composed of padding + * packets. It does this by specifying much lower inter-packet delays than + * the burst state, and transitioning back to itself after padding is sent + * if these timers expire before real traffic is sent. If real traffic is + * sent, it transitions back to the burst state. + * + * Again, in this implementation, these transitions must be specified + * explicitly, and other transitions are also permitted. + */ +#define CIRCPAD_STATE_GAP 2 + +/** + * End is a pseudo-state that causes the machine to go completely + * idle, and optionally get torn down (depending on the + * value of circpad_machine_spec_t.should_negotiate_end) + * + * End MUST NOT occupy a slot in the machine state array. + */ +#define CIRCPAD_STATE_END CIRCPAD_STATENUM_MAX + +/** + * "Ignore" is a pseudo-state that means "do not react to this + * event". + * + * "Ignore" MUST NOT occupy a slot in the machine state array. + */ +#define CIRCPAD_STATE_IGNORE (CIRCPAD_STATENUM_MAX-1) + +/** + * "Cancel" is a pseudo-state that means "cancel pending timers, + * but remain in your current state". + * + * Cancel MUST NOT occupy a slot in the machine state array. + */ +#define CIRCPAD_STATE_CANCEL (CIRCPAD_STATENUM_MAX-2) + +/** + * Since we have 3 pseudo-states, the max state array length is + * up to one less than cancel's statenum. + */ +#define CIRCPAD_MAX_MACHINE_STATES (CIRCPAD_STATE_CANCEL-1) + +/** + * Mutable padding machine info. + * + * This structure contains mutable information about a padding + * machine. The mutable information must be kept separate because + * it exists per-circuit, where as the machines themselves are global. + * This separation is done to conserve space in the circuit structure. + * + * This is the per-circuit state that changes regarding the global state + * machine. Some parts of it are optional (ie NULL). + * + * XXX: Play with layout to minimize space on x64 Linux (most common relay). + */ +typedef struct circpad_machine_state_t { + /** The callback pointer for the padding callbacks. + * + * These timers stick around the machineinfo until the machineinfo's circuit + * is closed, at which point the timer is cancelled. For this reason it's + * safe to assume that the machineinfo exists if this timer gets + * triggered. */ + tor_timer_t *padding_timer; + + /** The circuit for this machine */ + struct circuit_t *on_circ; + + /** A mutable copy of the histogram for the current state. + * NULL if remove_tokens is false for that state */ + circpad_hist_token_t *histogram; + /** Length of the above histogram. + * XXX: This field *could* be removed at the expense of added + * complexity+overhead for reaching back into the immutable machine + * state every time we need to inspect the histogram. It's only a byte, + * though, so it seemed worth it. + */ + circpad_hist_index_t histogram_len; + /** Remove token from this index upon sending padding */ + circpad_hist_index_t chosen_bin; + + /** Stop padding/transition if this many cells sent */ + uint64_t state_length; +#define CIRCPAD_STATE_LENGTH_INFINITE UINT64_MAX + + /** A scaled count of padding packets sent, used to limit padding overhead. + * When this reaches UINT16_MAX, we cut it and nonpadding_sent in half. */ + uint16_t padding_sent; + /** A scaled count of non-padding packets sent, used to limit padding + * overhead. When this reaches UINT16_MAX, we cut it and padding_sent in + * half. */ + uint16_t nonpadding_sent; + + /** + * EWMA estimate of the RTT of the circuit from this hop + * to the exit end, in microseconds. */ + circpad_delay_t rtt_estimate_usec; + + /** + * The last time we got an event relevant to estimating + * the RTT. Monotonic time in microseconds since system + * start. + */ + circpad_time_t last_received_time_usec; + + /** + * The time at which we scheduled a non-padding packet, + * or selected an infinite delay. + * + * Monotonic time in microseconds since system start. + * This is 0 if we haven't chosen a padding delay. + */ + circpad_time_t padding_scheduled_at_usec; + + /** What state is this machine in? */ + circpad_statenum_t current_state; + + /** + * True if we have scheduled a timer for padding. + * + * This is 1 if a timer is pending. It is 0 if + * no timer is scheduled. (It can be 0 even when + * padding_was_scheduled_at_usec is non-zero). + */ + unsigned is_padding_timer_scheduled : 1; + + /** + * If this is true, we have seen full duplex behavior. + * Stop updating the RTT. + */ + unsigned stop_rtt_update : 1; + +/** Max number of padding machines on each circuit. If changed, + * also ensure the machine_index bitwith supports the new size. */ +#define CIRCPAD_MAX_MACHINES (2) + /** Which padding machine index was this for. + * (make sure changes to the bitwidth can support the + * CIRCPAD_MAX_MACHINES define). */ + unsigned machine_index : 1; + +} circpad_machine_state_t; + +/** Helper macro to get an actual state machine from a machineinfo */ +#define CIRCPAD_GET_MACHINE(machineinfo) \ + ((machineinfo)->on_circ->padding_machine[(machineinfo)->machine_index]) + +/** + * This specifies a particular padding machine to use after negotiation. + * + * The constants for machine_num_t are in trunnel. + * We want to be able to define extra numbers in the consensus/torrc, though. + */ +typedef uint8_t circpad_machine_num_t; + +/** Global state machine structure from the consensus */ +typedef struct circpad_machine_spec_t { + /** Global machine number */ + circpad_machine_num_t machine_num; + + /** Which machine index slot should this machine go into in + * the array on the circuit_t */ + unsigned machine_index : 1; + + /** Send a padding negotiate to shut down machine at end state? */ + unsigned should_negotiate_end : 1; + + // These next three fields are origin machine-only... + /** Origin side or relay side */ + unsigned is_origin_side : 1; + + /** Which hop in the circuit should we send padding to/from? + * 1-indexed (ie: hop #1 is guard, #2 middle, #3 exit). */ + unsigned target_hopnum : 3; + + /** This machine only kills fascists if the following conditions are met. */ + circpad_machine_conditions_t conditions; + + /** How many padding cells can be sent before we apply overhead limits? + * XXX: Note that we can only allow up to 64k of padding cells on an + * otherwise quiet circuit. Is this enough? It's 33MB. */ + uint16_t allowed_padding_count; + + /** Padding percent cap: Stop padding if we exceed this percent overhead. + * 0 means no limit. Overhead is defined as percent of total traffic, so + * that we can use 0..100 here. This is the same definition as used in + * Prop#265. */ + uint8_t max_padding_percent; + + /** State array: indexed by circpad_statenum_t */ + circpad_state_t *states; + + /** + * Number of states this machine has (ie: length of the states array). + * XXX: This field is not needed other than for safety. */ + circpad_statenum_t num_states; +} circpad_machine_spec_t; + +void circpad_new_consensus_params(const networkstatus_t *ns); + +/** + * The following are event call-in points that are of interest to + * the state machines. They are called during cell processing. */ +void circpad_deliver_unrecognized_cell_events(struct circuit_t *circ, + cell_direction_t dir); +void circpad_deliver_sent_relay_cell_events(struct circuit_t *circ, + uint8_t relay_command); +void circpad_deliver_recognized_relay_cell_events(struct circuit_t *circ, + uint8_t relay_command, + crypt_path_t *layer_hint); + +/** Cell events are delivered by the above delivery functions */ +void circpad_cell_event_nonpadding_sent(struct circuit_t *on_circ); +void circpad_cell_event_nonpadding_received(struct circuit_t *on_circ); +void circpad_cell_event_padding_sent(struct circuit_t *on_circ); +void circpad_cell_event_padding_received(struct circuit_t *on_circ); + +/** Internal events are events the machines send to themselves */ +circpad_decision_t +circpad_internal_event_infinity(circpad_machine_state_t *mi); +circpad_decision_t +circpad_internal_event_bins_empty(circpad_machine_state_t *); +circpad_decision_t circpad_internal_event_state_length_up( + circpad_machine_state_t *); + +/** Machine creation events are events that cause us to set up or + * tear down padding state machines. */ +void circpad_machine_event_circ_added_hop(struct origin_circuit_t *on_circ); +void circpad_machine_event_circ_built(struct origin_circuit_t *circ); +void circpad_machine_event_circ_purpose_changed(struct origin_circuit_t *circ); +void circpad_machine_event_circ_has_streams(struct origin_circuit_t *circ); +void circpad_machine_event_circ_has_no_streams(struct origin_circuit_t *circ); +void +circpad_machine_event_circ_has_no_relay_early(struct origin_circuit_t *circ); + +void circpad_machines_init(void); +void circpad_machines_free(void); + +void circpad_machine_states_init(circpad_machine_spec_t *machine, + circpad_statenum_t num_states); + +void circpad_circuit_free_all_machineinfos(struct circuit_t *circ); + +bool circpad_padding_is_from_expected_hop(struct circuit_t *circ, + crypt_path_t *from_hop); + +/** Serializaton functions for writing to/from torrc and consensus */ +char *circpad_machine_spec_to_string(const circpad_machine_spec_t *machine); +const circpad_machine_spec_t *circpad_string_to_machine(const char *str); + +/* Padding negotiation between client and middle */ +signed_error_t circpad_handle_padding_negotiate(struct circuit_t *circ, + struct cell_t *cell); +signed_error_t circpad_handle_padding_negotiated(struct circuit_t *circ, + struct cell_t *cell, + crypt_path_t *layer_hint); +signed_error_t circpad_negotiate_padding(struct origin_circuit_t *circ, + circpad_machine_num_t machine, + uint8_t target_hopnum, + uint8_t command); +bool circpad_padding_negotiated(struct circuit_t *circ, + circpad_machine_num_t machine, + uint8_t command, + uint8_t response); + +MOCK_DECL(circpad_decision_t, +circpad_machine_schedule_padding,(circpad_machine_state_t *)); + +MOCK_DECL(circpad_decision_t, +circpad_machine_spec_transition, (circpad_machine_state_t *mi, + circpad_event_t event)); + +circpad_decision_t circpad_send_padding_cell_for_callback( + circpad_machine_state_t *mi); + +#ifdef CIRCUITPADDING_PRIVATE +STATIC circpad_delay_t +circpad_machine_sample_delay(circpad_machine_state_t *mi); + +STATIC bool +circpad_machine_reached_padding_limit(circpad_machine_state_t *mi); + +STATIC +circpad_decision_t circpad_machine_remove_token(circpad_machine_state_t *mi); + +STATIC circpad_delay_t +circpad_histogram_bin_to_usec(const circpad_machine_state_t *mi, + circpad_hist_index_t bin); + +STATIC const circpad_state_t * +circpad_machine_current_state(const circpad_machine_state_t *mi); + +STATIC circpad_hist_index_t circpad_histogram_usec_to_bin( + const circpad_machine_state_t *mi, + circpad_delay_t us); + +STATIC circpad_machine_state_t *circpad_circuit_machineinfo_new( + struct circuit_t *on_circ, + int machine_index); +STATIC void circpad_machine_remove_higher_token(circpad_machine_state_t *mi, + circpad_delay_t target_bin_us); +STATIC void circpad_machine_remove_lower_token(circpad_machine_state_t *mi, + circpad_delay_t target_bin_us); +STATIC void circpad_machine_remove_closest_token(circpad_machine_state_t *mi, + circpad_delay_t target_bin_us, + bool use_usec); +STATIC void circpad_machine_setup_tokens(circpad_machine_state_t *mi); + +MOCK_DECL(STATIC signed_error_t, +circpad_send_command_to_hop,(struct origin_circuit_t *circ, uint8_t hopnum, + uint8_t relay_command, const uint8_t *payload, + ssize_t payload_len)); + +#ifdef TOR_UNIT_TESTS +extern smartlist_t *origin_padding_machines; +extern smartlist_t *relay_padding_machines; +#endif + +#endif + +#endif diff --git a/src/core/or/circuitstats.c b/src/core/or/circuitstats.c index 2cde21fa1f..c6ea2fff97 100644 --- a/src/core/or/circuitstats.c +++ b/src/core/or/circuitstats.c @@ -639,9 +639,9 @@ circuit_build_times_rewind_history(circuit_build_times_t *cbt, int n) void circuit_build_times_mark_circ_as_measurement_only(origin_circuit_t *circ) { - control_event_circuit_status(circ, - CIRC_EVENT_FAILED, - END_CIRC_REASON_TIMEOUT); + circuit_event_status(circ, + CIRC_EVENT_FAILED, + END_CIRC_REASON_TIMEOUT); circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT); /* Record this event to check for too many timeouts diff --git a/src/core/or/circuituse.c b/src/core/or/circuituse.c index 000a7c36da..3e604adcfc 100644 --- a/src/core/or/circuituse.c +++ b/src/core/or/circuituse.c @@ -35,6 +35,7 @@ #include "core/or/circuitlist.h" #include "core/or/circuitstats.h" #include "core/or/circuituse.h" +#include "core/or/circuitpadding.h" #include "core/or/connection_edge.h" #include "core/or/policies.h" #include "feature/client/addressmap.h" @@ -1425,6 +1426,11 @@ circuit_detach_stream(circuit_t *circ, edge_connection_t *conn) if (circ->purpose == CIRCUIT_PURPOSE_S_REND_JOINED) { hs_dec_rdv_stream_counter(origin_circ); } + + /* If there are no more streams on this circ, tell circpad */ + if (!origin_circ->p_streams) + circpad_machine_event_circ_has_no_streams(origin_circ); + return; } } else { @@ -1674,7 +1680,7 @@ circuit_testing_failed(origin_circuit_t *circ, int at_last_hop) void circuit_has_opened(origin_circuit_t *circ) { - control_event_circuit_status(circ, CIRC_EVENT_BUILT, 0); + circuit_event_status(circ, CIRC_EVENT_BUILT, 0); /* Remember that this circuit has finished building. Now if we start * it building again later (e.g. by extending it), we will know not @@ -2596,6 +2602,12 @@ link_apconn_to_circ(entry_connection_t *apconn, origin_circuit_t *circ, /* add it into the linked list of streams on this circuit */ log_debug(LD_APP|LD_CIRC, "attaching new conn to circ. n_circ_id %u.", (unsigned)circ->base_.n_circ_id); + + /* If this is the first stream on this circuit, tell circpad + * that streams are attached */ + if (!circ->p_streams) + circpad_machine_event_circ_has_streams(circ); + /* reset it, so we can measure circ timeouts */ ENTRY_TO_CONN(apconn)->timestamp_last_read_allowed = time(NULL); ENTRY_TO_EDGE_CONN(apconn)->next_stream = circ->p_streams; @@ -3080,6 +3092,8 @@ circuit_change_purpose(circuit_t *circ, uint8_t new_purpose) if (CIRCUIT_IS_ORIGIN(circ)) { control_event_circuit_purpose_changed(TO_ORIGIN_CIRCUIT(circ), old_purpose); + + circpad_machine_event_circ_purpose_changed(TO_ORIGIN_CIRCUIT(circ)); } } diff --git a/src/core/or/connection_edge.c b/src/core/or/connection_edge.c index 90991107dc..20c48bcedd 100644 --- a/src/core/or/connection_edge.c +++ b/src/core/or/connection_edge.c @@ -62,10 +62,12 @@ #include "app/config/config.h" #include "core/mainloop/connection.h" #include "core/mainloop/mainloop.h" +#include "core/mainloop/netstatus.h" #include "core/or/channel.h" #include "core/or/circuitbuild.h" #include "core/or/circuitlist.h" #include "core/or/circuituse.h" +#include "core/or/circuitpadding.h" #include "core/or/connection_edge.h" #include "core/or/connection_or.h" #include "core/or/policies.h" @@ -97,7 +99,7 @@ #include "feature/rend/rendservice.h" #include "feature/stats/predict_ports.h" #include "feature/stats/rephist.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/crypt_ops/crypto_util.h" #include "core/or/cell_st.h" @@ -300,6 +302,11 @@ connection_edge_process_inbuf(edge_connection_t *conn, int package_partial) } return 0; case AP_CONN_STATE_OPEN: + if (! conn->base_.linked) { + note_user_activity(approx_time()); + } + + /* falls through. */ case EXIT_CONN_STATE_OPEN: if (connection_edge_package_raw_inbuf(conn, package_partial, NULL) < 0) { /* (We already sent an end cell if possible) */ @@ -754,6 +761,11 @@ connection_edge_flushed_some(edge_connection_t *conn) { switch (conn->base_.state) { case AP_CONN_STATE_OPEN: + if (! conn->base_.linked) { + note_user_activity(approx_time()); + } + + /* falls through. */ case EXIT_CONN_STATE_OPEN: connection_edge_consider_sending_sendme(conn); break; @@ -3706,6 +3718,10 @@ handle_hs_exit_conn(circuit_t *circ, edge_connection_t *conn) /* Link the circuit and the connection crypt path. */ conn->cpath_layer = origin_circ->cpath->prev; + /* If this is the first stream on this circuit, tell circpad */ + if (!origin_circ->p_streams) + circpad_machine_event_circ_has_streams(origin_circ); + /* Add it into the linked list of p_streams on this circuit */ conn->next_stream = origin_circ->p_streams; origin_circ->p_streams = conn; diff --git a/src/core/or/connection_or.c b/src/core/or/connection_or.c index 67157d8d0b..debf482cb3 100644 --- a/src/core/or/connection_or.c +++ b/src/core/or/connection_or.c @@ -22,13 +22,14 @@ **/ #include "core/or/or.h" #include "feature/client/bridges.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" /* * Define this so we get channel internal functions, since we're implementing * part of a subclass (channel_tls_t). */ #define TOR_CHANNEL_INTERNAL_ #define CONNECTION_OR_PRIVATE +#define ORCONN_EVENT_PRIVATE #include "core/or/channel.h" #include "core/or/channeltls.h" #include "core/or/circuitbuild.h" @@ -46,6 +47,7 @@ #include "lib/geoip/geoip.h" #include "core/mainloop/mainloop.h" #include "trunnel/link_handshake.h" +#include "trunnel/netinfo.h" #include "feature/nodelist/microdesc.h" #include "feature/nodelist/networkstatus.h" #include "feature/nodelist/nodelist.h" @@ -78,6 +80,8 @@ #include "lib/tls/tortls.h" #include "lib/tls/x509.h" +#include "core/or/orconn_event.h" + static int connection_tls_finish_handshake(or_connection_t *conn); static int connection_or_launch_v3_or_handshake(or_connection_t *conn); static int connection_or_process_cells_from_inbuf(or_connection_t *conn); @@ -400,6 +404,57 @@ connection_or_report_broken_states(int severity, int domain) smartlist_free(items); } +/** + * Helper function to publish an OR connection status event + * + * Publishes a messages to subscribers of ORCONN messages, and sends + * the control event. + **/ +void +connection_or_event_status(or_connection_t *conn, or_conn_status_event_t tp, + int reason) +{ + orconn_event_msg_t msg; + + msg.type = ORCONN_MSGTYPE_STATUS; + msg.u.status.gid = conn->base_.global_identifier; + msg.u.status.status = tp; + msg.u.status.reason = reason; + orconn_event_publish(&msg); + control_event_or_conn_status(conn, tp, reason); +} + +/** + * Helper function to publish a state change message + * + * connection_or_change_state() calls this to notify subscribers about + * a change of an OR connection state. + **/ +static void +connection_or_state_publish(const or_connection_t *conn, uint8_t state) +{ + orconn_event_msg_t msg; + + msg.type = ORCONN_MSGTYPE_STATE; + msg.u.state.gid = conn->base_.global_identifier; + if (conn->is_pt) { + /* Do extra decoding because conn->proxy_type indicates the proxy + * protocol that tor uses to talk with the transport plugin, + * instead of PROXY_PLUGGABLE. */ + tor_assert_nonfatal(conn->proxy_type != PROXY_NONE); + msg.u.state.proxy_type = PROXY_PLUGGABLE; + } else { + msg.u.state.proxy_type = conn->proxy_type; + } + msg.u.state.state = state; + if (conn->chan) { + msg.u.state.chan = TLS_CHAN_TO_BASE(conn->chan)->global_identifier; + } else { + msg.u.state.chan = 0; + } + orconn_event_publish(&msg); +} + /** Call this to change or_connection_t states, so the owning channel_tls_t can * be notified. */ @@ -407,16 +462,13 @@ connection_or_report_broken_states(int severity, int domain) static void connection_or_change_state(or_connection_t *conn, uint8_t state) { - uint8_t old_state; - tor_assert(conn); - old_state = conn->base_.state; conn->base_.state = state; + connection_or_state_publish(conn, state); if (conn->chan) - channel_tls_handle_state_change_on_orconn(conn->chan, conn, - old_state, state); + channel_tls_handle_state_change_on_orconn(conn->chan, conn, state); } /** Return the number of circuits using an or_connection_t; this used to @@ -707,8 +759,6 @@ connection_or_finished_connecting(or_connection_t *or_conn) log_debug(LD_HANDSHAKE,"OR connect() to router at %s:%u finished.", conn->address,conn->port); - control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE, 0); - control_event_boot_first_orconn(); if (proxy_type != PROXY_NONE) { /* start proxy handshake */ @@ -758,8 +808,8 @@ connection_or_about_to_close(or_connection_t *or_conn) entry_guard_chan_failed(TLS_CHAN_TO_BASE(or_conn->chan)); if (conn->state >= OR_CONN_STATE_TLS_HANDSHAKING) { int reason = tls_error_to_orconn_end_reason(or_conn->tls_error); - control_event_or_conn_status(or_conn, OR_CONN_EVENT_FAILED, - reason); + connection_or_event_status(or_conn, OR_CONN_EVENT_FAILED, + reason); if (!authdir_mode_tests_reachability(options)) control_event_bootstrap_prob_or( orconn_end_reason_to_control_string(reason), @@ -769,10 +819,10 @@ connection_or_about_to_close(or_connection_t *or_conn) } else if (conn->hold_open_until_flushed) { /* We only set hold_open_until_flushed when we're intentionally * closing a connection. */ - control_event_or_conn_status(or_conn, OR_CONN_EVENT_CLOSED, + connection_or_event_status(or_conn, OR_CONN_EVENT_CLOSED, tls_error_to_orconn_end_reason(or_conn->tls_error)); } else if (!tor_digest_is_zero(or_conn->identity_digest)) { - control_event_or_conn_status(or_conn, OR_CONN_EVENT_CLOSED, + connection_or_event_status(or_conn, OR_CONN_EVENT_CLOSED, tls_error_to_orconn_end_reason(or_conn->tls_error)); } } @@ -1364,7 +1414,7 @@ void connection_or_connect_failed(or_connection_t *conn, int reason, const char *msg) { - control_event_or_conn_status(conn, OR_CONN_EVENT_FAILED, reason); + connection_or_event_status(conn, OR_CONN_EVENT_FAILED, reason); if (!authdir_mode_tests_reachability(get_options())) control_event_bootstrap_prob_or(msg, reason, conn); note_or_connect_failed(conn); @@ -1430,7 +1480,7 @@ connection_or_connect, (const tor_addr_t *_addr, uint16_t port, int r; tor_addr_t proxy_addr; uint16_t proxy_port; - int proxy_type; + int proxy_type, is_pt = 0; tor_assert(_addr); tor_assert(id_digest); @@ -1471,21 +1521,27 @@ connection_or_connect, (const tor_addr_t *_addr, uint16_t port, return NULL; } - connection_or_change_state(conn, OR_CONN_STATE_CONNECTING); - control_event_or_conn_status(conn, OR_CONN_EVENT_LAUNCHED, 0); - conn->is_outgoing = 1; /* If we are using a proxy server, find it and use it. */ - r = get_proxy_addrport(&proxy_addr, &proxy_port, &proxy_type, TO_CONN(conn)); + r = get_proxy_addrport(&proxy_addr, &proxy_port, &proxy_type, &is_pt, + TO_CONN(conn)); if (r == 0) { conn->proxy_type = proxy_type; if (proxy_type != PROXY_NONE) { tor_addr_copy(&addr, &proxy_addr); port = proxy_port; conn->base_.proxy_state = PROXY_INFANT; + conn->is_pt = is_pt; } + connection_or_change_state(conn, OR_CONN_STATE_CONNECTING); + connection_or_event_status(conn, OR_CONN_EVENT_LAUNCHED, 0); } else { + /* This duplication of state change calls is necessary in case we + * run into an error condition below */ + connection_or_change_state(conn, OR_CONN_STATE_CONNECTING); + connection_or_event_status(conn, OR_CONN_EVENT_LAUNCHED, 0); + /* get_proxy_addrport() might fail if we have a Bridge line that references a transport, but no ClientTransportPlugin lines defining its transport proxy. If this is the case, let's try to @@ -1981,8 +2037,8 @@ connection_or_client_learned_peer_id(or_connection_t *conn, /* Tell the new guard API about the channel failure */ entry_guard_chan_failed(TLS_CHAN_TO_BASE(conn->chan)); - control_event_or_conn_status(conn, OR_CONN_EVENT_FAILED, - END_OR_CONN_REASON_OR_IDENTITY); + connection_or_event_status(conn, OR_CONN_EVENT_FAILED, + END_OR_CONN_REASON_OR_IDENTITY); if (!authdir_mode_tests_reachability(options)) control_event_bootstrap_prob_or( "Unexpected identity in router certificate", @@ -2222,7 +2278,7 @@ int connection_or_set_state_open(or_connection_t *conn) { connection_or_change_state(conn, OR_CONN_STATE_OPEN); - control_event_or_conn_status(conn, OR_CONN_EVENT_CONNECTED, 0); + connection_or_event_status(conn, OR_CONN_EVENT_CONNECTED, 0); /* Link protocol 3 appeared in Tor 0.2.3.6-alpha, so any connection * that uses an earlier link protocol should not be treated as a relay. */ @@ -2428,6 +2484,31 @@ connection_or_send_versions(or_connection_t *conn, int v3_plus) return 0; } +static netinfo_addr_t * +netinfo_addr_from_tor_addr(const tor_addr_t *tor_addr) +{ + sa_family_t addr_family = tor_addr_family(tor_addr); + + if (BUG(addr_family != AF_INET && addr_family != AF_INET6)) + return NULL; + + netinfo_addr_t *netinfo_addr = netinfo_addr_new(); + + if (addr_family == AF_INET) { + netinfo_addr_set_addr_type(netinfo_addr, NETINFO_ADDR_TYPE_IPV4); + netinfo_addr_set_len(netinfo_addr, 4); + netinfo_addr_set_addr_ipv4(netinfo_addr, tor_addr_to_ipv4h(tor_addr)); + } else if (addr_family == AF_INET6) { + netinfo_addr_set_addr_type(netinfo_addr, NETINFO_ADDR_TYPE_IPV6); + netinfo_addr_set_len(netinfo_addr, 16); + uint8_t *ipv6_buf = netinfo_addr_getarray_addr_ipv6(netinfo_addr); + const uint8_t *in6_addr = tor_addr_to_in6_addr8(tor_addr); + memcpy(ipv6_buf, in6_addr, 16); + } + + return netinfo_addr; +} + /** Send a NETINFO cell on <b>conn</b>, telling the other server what we know * about their address, our address, and the current time. */ MOCK_IMPL(int, @@ -2436,8 +2517,7 @@ connection_or_send_netinfo,(or_connection_t *conn)) cell_t cell; time_t now = time(NULL); const routerinfo_t *me; - int len; - uint8_t *out; + int r = -1; tor_assert(conn->handshake_state); @@ -2450,20 +2530,21 @@ connection_or_send_netinfo,(or_connection_t *conn)) memset(&cell, 0, sizeof(cell_t)); cell.command = CELL_NETINFO; + netinfo_cell_t *netinfo_cell = netinfo_cell_new(); + /* Timestamp, if we're a relay. */ if (public_server_mode(get_options()) || ! conn->is_outgoing) - set_uint32(cell.payload, htonl((uint32_t)now)); + netinfo_cell_set_timestamp(netinfo_cell, (uint32_t)now); /* Their address. */ - out = cell.payload + 4; + const tor_addr_t *remote_tor_addr = + !tor_addr_is_null(&conn->real_addr) ? &conn->real_addr : &conn->base_.addr; /* We use &conn->real_addr below, unless it hasn't yet been set. If it * hasn't yet been set, we know that base_.addr hasn't been tampered with * yet either. */ - len = append_address_to_payload(out, !tor_addr_is_null(&conn->real_addr) - ? &conn->real_addr : &conn->base_.addr); - if (len<0) - return -1; - out += len; + netinfo_addr_t *their_addr = netinfo_addr_from_tor_addr(remote_tor_addr); + + netinfo_cell_set_other_addr(netinfo_cell, their_addr); /* My address -- only include it if I'm a public relay, or if I'm a * bridge and this is an incoming connection. If I'm a bridge and this @@ -2471,28 +2552,42 @@ connection_or_send_netinfo,(or_connection_t *conn)) if ((public_server_mode(get_options()) || !conn->is_outgoing) && (me = router_get_my_routerinfo())) { tor_addr_t my_addr; - *out++ = 1 + !tor_addr_is_null(&me->ipv6_addr); - tor_addr_from_ipv4h(&my_addr, me->addr); - len = append_address_to_payload(out, &my_addr); - if (len < 0) - return -1; - out += len; + + uint8_t n_my_addrs = 1 + !tor_addr_is_null(&me->ipv6_addr); + netinfo_cell_set_n_my_addrs(netinfo_cell, n_my_addrs); + + netinfo_cell_add_my_addrs(netinfo_cell, + netinfo_addr_from_tor_addr(&my_addr)); if (!tor_addr_is_null(&me->ipv6_addr)) { - len = append_address_to_payload(out, &me->ipv6_addr); - if (len < 0) - return -1; + netinfo_cell_add_my_addrs(netinfo_cell, + netinfo_addr_from_tor_addr(&me->ipv6_addr)); } - } else { - *out = 0; + } + + const char *errmsg = NULL; + if ((errmsg = netinfo_cell_check(netinfo_cell))) { + log_warn(LD_OR, "Failed to validate NETINFO cell with error: %s", + errmsg); + goto cleanup; + } + + if (netinfo_cell_encode(cell.payload, CELL_PAYLOAD_SIZE, + netinfo_cell) < 0) { + log_warn(LD_OR, "Failed generating NETINFO cell"); + goto cleanup; } conn->handshake_state->digest_sent_data = 0; conn->handshake_state->sent_netinfo = 1; connection_or_write_cell_to_buf(&cell, conn); - return 0; + r = 0; + cleanup: + netinfo_cell_free(netinfo_cell); + + return r; } /** Helper used to add an encoded certs to a cert cell */ diff --git a/src/core/or/connection_or.h b/src/core/or/connection_or.h index 817bcdd317..272f536b83 100644 --- a/src/core/or/connection_or.h +++ b/src/core/or/connection_or.h @@ -17,32 +17,7 @@ struct ed25519_keypair_t; or_connection_t *TO_OR_CONN(connection_t *); -#define OR_CONN_STATE_MIN_ 1 -/** State for a connection to an OR: waiting for connect() to finish. */ -#define OR_CONN_STATE_CONNECTING 1 -/** State for a connection to an OR: waiting for proxy handshake to complete */ -#define OR_CONN_STATE_PROXY_HANDSHAKING 2 -/** State for an OR connection client: SSL is handshaking, not done - * yet. */ -#define OR_CONN_STATE_TLS_HANDSHAKING 3 -/** State for a connection to an OR: We're doing a second SSL handshake for - * renegotiation purposes. (V2 handshake only.) */ -#define OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING 4 -/** State for a connection at an OR: We're waiting for the client to - * renegotiate (to indicate a v2 handshake) or send a versions cell (to - * indicate a v3 handshake) */ -#define OR_CONN_STATE_TLS_SERVER_RENEGOTIATING 5 -/** State for an OR connection: We're done with our SSL handshake, we've done - * renegotiation, but we haven't yet negotiated link protocol versions and - * sent a netinfo cell. */ -#define OR_CONN_STATE_OR_HANDSHAKING_V2 6 -/** State for an OR connection: We're done with our SSL handshake, but we - * haven't yet negotiated link protocol versions, done a V3 handshake, and - * sent a netinfo cell. */ -#define OR_CONN_STATE_OR_HANDSHAKING_V3 7 -/** State for an OR connection: Ready to send/receive cells. */ -#define OR_CONN_STATE_OPEN 8 -#define OR_CONN_STATE_MAX_ 8 +#include "core/or/orconn_event.h" void connection_or_clear_identity(or_connection_t *conn); void connection_or_clear_identity_map(void); @@ -81,6 +56,9 @@ MOCK_DECL(void,connection_or_close_for_error, void connection_or_report_broken_states(int severity, int domain); +void connection_or_event_status(or_connection_t *conn, + or_conn_status_event_t tp, int reason); + MOCK_DECL(int,connection_tls_start_handshake,(or_connection_t *conn, int receiving)); int connection_tls_continue_handshake(or_connection_t *conn); diff --git a/src/core/or/ocirc_event.c b/src/core/or/ocirc_event.c new file mode 100644 index 0000000000..4a6fc748c9 --- /dev/null +++ b/src/core/or/ocirc_event.c @@ -0,0 +1,84 @@ +/* Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file ocirc_event.c + * \brief Publish state change messages for origin circuits + * + * Implements a basic publish-subscribe framework for messages about + * the state of origin circuits. The publisher calls the subscriber + * callback functions synchronously. + * + * Although the synchronous calls might not simplify the call graph, + * this approach improves data isolation because the publisher doesn't + * need knowledge about the internals of subscribing subsystems. It + * also avoids race conditions that might occur in asynchronous + * frameworks. + **/ + +#include "core/or/or.h" + +#define OCIRC_EVENT_PRIVATE + +#include "core/or/cpath_build_state_st.h" +#include "core/or/ocirc_event.h" +#include "core/or/ocirc_event_sys.h" +#include "core/or/origin_circuit_st.h" +#include "lib/subsys/subsys.h" + +/** List of subscribers */ +static smartlist_t *ocirc_event_rcvrs; + +/** Initialize subscriber list */ +static int +ocirc_event_init(void) +{ + ocirc_event_rcvrs = smartlist_new(); + return 0; +} + +/** Free subscriber list */ +static void +ocirc_event_fini(void) +{ + smartlist_free(ocirc_event_rcvrs); +} + +/** + * Subscribe to messages about origin circuit events + * + * Register a callback function to receive messages about origin + * circuits. The publisher calls this function synchronously. + **/ +void +ocirc_event_subscribe(ocirc_event_rcvr_t fn) +{ + tor_assert(fn); + /* Don't duplicate subscriptions. */ + if (smartlist_contains(ocirc_event_rcvrs, fn)) + return; + + smartlist_add(ocirc_event_rcvrs, fn); +} + +/** + * Publish a message about OR connection events + * + * This calls the subscriber receiver function synchronously. + **/ +void +ocirc_event_publish(const ocirc_event_msg_t *msg) +{ + SMARTLIST_FOREACH_BEGIN(ocirc_event_rcvrs, ocirc_event_rcvr_t, fn) { + tor_assert(fn); + (*fn)(msg); + } SMARTLIST_FOREACH_END(fn); +} + +const subsys_fns_t sys_ocirc_event = { + .name = "ocirc_event", + .supported = true, + .level = -32, + .initialize = ocirc_event_init, + .shutdown = ocirc_event_fini, +}; diff --git a/src/core/or/ocirc_event.h b/src/core/or/ocirc_event.h new file mode 100644 index 0000000000..0b125c2898 --- /dev/null +++ b/src/core/or/ocirc_event.h @@ -0,0 +1,89 @@ +/* Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file ocirc_event.h + * \brief Header file for ocirc_event.c + **/ + +#ifndef TOR_OCIRC_EVENT_H +#define TOR_OCIRC_EVENT_H + +#include <stdbool.h> + +#include "lib/cc/torint.h" + +/** Used to indicate the type of a circuit event passed to the controller. + * The various types are defined in control-spec.txt */ +typedef enum circuit_status_event_t { + CIRC_EVENT_LAUNCHED = 0, + CIRC_EVENT_BUILT = 1, + CIRC_EVENT_EXTENDED = 2, + CIRC_EVENT_FAILED = 3, + CIRC_EVENT_CLOSED = 4, +} circuit_status_event_t; + +/** Message for origin circuit state update */ +typedef struct ocirc_state_msg_t { + uint32_t gid; /**< global ID (only origin circuits have them) */ + int state; /**< new circuit state */ + bool onehop; /**< one-hop circuit? */ +} ocirc_state_msg_t; + +/** + * Message when a channel gets associated to a circuit. + * + * This doesn't always correspond to something in circuitbuild.c + * setting the n_chan field in the circuit. For some reason, if + * circuit_handle_first_hop() launches a new circuit, it doesn't set + * the n_chan field. + */ +typedef struct ocirc_chan_msg_t { + uint32_t gid; /**< global ID */ + uint64_t chan; /**< channel ID */ + bool onehop; /**< one-hop circuit? */ +} ocirc_chan_msg_t; + +/** + * Message for origin circuit status event + * + * This contains information that ends up in CIRC control protocol events. + */ +typedef struct ocirc_cevent_msg_t { + uint32_t gid; /**< global ID */ + int evtype; /**< event type */ + int reason; /**< reason */ + bool onehop; /**< one-hop circuit? */ +} ocirc_cevent_msg_t; + +/** Discriminant values for origin circuit event message */ +typedef enum ocirc_msgtype_t { + OCIRC_MSGTYPE_STATE, + OCIRC_MSGTYPE_CHAN, + OCIRC_MSGTYPE_CEVENT, +} ocirc_msgtype_t; + +/** Discriminated union for the actual message */ +typedef struct ocirc_event_msg_t { + int type; + union { + ocirc_state_msg_t state; + ocirc_chan_msg_t chan; + ocirc_cevent_msg_t cevent; + } u; +} ocirc_event_msg_t; + +/** + * Receiver function pointer for origin circuit subscribers + * + * This function gets called synchronously by the publisher. + **/ +typedef void (*ocirc_event_rcvr_t)(const ocirc_event_msg_t *); + +void ocirc_event_subscribe(ocirc_event_rcvr_t fn); + +#ifdef OCIRC_EVENT_PRIVATE +void ocirc_event_publish(const ocirc_event_msg_t *msg); +#endif + +#endif /* defined(TOR_OCIRC_EVENT_STATE_H) */ diff --git a/src/core/or/ocirc_event_sys.h b/src/core/or/ocirc_event_sys.h new file mode 100644 index 0000000000..9d4bfe5333 --- /dev/null +++ b/src/core/or/ocirc_event_sys.h @@ -0,0 +1,13 @@ +/* Copyright (c) 2007-2019, The Tor Project, Inc. */ + +/** + * \file ocirc_event_sys.h + * \brief Declare subsystem object for the origin circuit event module. + **/ + +#ifndef TOR_OCIRC_EVENT_SYS_H +#define TOR_OCIRC_EVENT_SYS_H + +extern const struct subsys_fns_t sys_ocirc_event; + +#endif /* defined(TOR_OCIRC_EVENT_H) */ diff --git a/src/core/or/or.h b/src/core/or/or.h index 7c601e49b3..db6d089582 100644 --- a/src/core/or/or.h +++ b/src/core/or/or.h @@ -26,7 +26,7 @@ #include "lib/cc/compat_compiler.h" #include "lib/cc/torint.h" #include "lib/container/map.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/container/smartlist.h" #include "lib/crypt_ops/crypto_cipher.h" #include "lib/crypt_ops/crypto_rsa.h" @@ -97,6 +97,8 @@ struct curve25519_public_key_t; #define SIGNEWNYM 129 #define SIGCLEARDNSCACHE 130 #define SIGHEARTBEAT 131 +#define SIGACTIVE 132 +#define SIGDORMANT 133 #if (SIZEOF_CELL_T != 0) /* On Irix, stdlib.h defines a cell_t type, so we need to make sure @@ -205,6 +207,9 @@ struct curve25519_public_key_t; #define RELAY_COMMAND_RENDEZVOUS_ESTABLISHED 39 #define RELAY_COMMAND_INTRODUCE_ACK 40 +#define RELAY_COMMAND_PADDING_NEGOTIATE 41 +#define RELAY_COMMAND_PADDING_NEGOTIATED 42 + /* Reasons why an OR connection is closed. */ #define END_OR_CONN_REASON_DONE 1 #define END_OR_CONN_REASON_REFUSED 2 /* connection refused */ @@ -834,6 +839,10 @@ typedef struct protover_summary_flags_t { * service rendezvous point supporting version 3 as seen in proposal 224. * This requires HSRend=2. */ unsigned int supports_v3_rendezvous_point: 1; + + /** True iff this router has a protocol list that allows clients to + * negotiate link-level padding. Requires Padding>=1. */ + unsigned int supports_padding : 1; } protover_summary_flags_t; typedef struct routerinfo_t routerinfo_t; diff --git a/src/core/or/or_connection_st.h b/src/core/or/or_connection_st.h index d5db5e8694..a5ce844bff 100644 --- a/src/core/or/or_connection_st.h +++ b/src/core/or/or_connection_st.h @@ -67,6 +67,8 @@ struct or_connection_t { * geoip cache and handled by the DoS mitigation subsystem. We use this to * insure we have a coherent count of concurrent connection. */ unsigned int tracked_for_dos_mitigation : 1; + /** True iff this connection is using a pluggable transport */ + unsigned int is_pt : 1; uint16_t link_proto; /**< What protocol version are we using? 0 for * "none negotiated yet." */ diff --git a/src/core/or/orconn_event.c b/src/core/or/orconn_event.c new file mode 100644 index 0000000000..9fb34bd1ff --- /dev/null +++ b/src/core/or/orconn_event.c @@ -0,0 +1,81 @@ +/* Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file orconn_event.c + * \brief Publish state change messages for OR connections + * + * Implements a basic publish-subscribe framework for messages about + * the state of OR connections. The publisher calls the subscriber + * callback functions synchronously. + * + * Although the synchronous calls might not simplify the call graph, + * this approach improves data isolation because the publisher doesn't + * need knowledge about the internals of subscribing subsystems. It + * also avoids race conditions that might occur in asynchronous + * frameworks. + **/ + +#include "core/or/or.h" +#include "lib/subsys/subsys.h" + +#define ORCONN_EVENT_PRIVATE +#include "core/or/orconn_event.h" +#include "core/or/orconn_event_sys.h" + +/** List of subscribers */ +static smartlist_t *orconn_event_rcvrs; + +/** Initialize subscriber list */ +static int +orconn_event_init(void) +{ + orconn_event_rcvrs = smartlist_new(); + return 0; +} + +/** Free subscriber list */ +static void +orconn_event_fini(void) +{ + smartlist_free(orconn_event_rcvrs); +} + +/** + * Subscribe to messages about OR connection events + * + * Register a callback function to receive messages about ORCONNs. + * The publisher calls this function synchronously. + **/ +void +orconn_event_subscribe(orconn_event_rcvr_t fn) +{ + tor_assert(fn); + /* Don't duplicate subscriptions. */ + if (smartlist_contains(orconn_event_rcvrs, fn)) + return; + + smartlist_add(orconn_event_rcvrs, fn); +} + +/** + * Publish a message about OR connection events + * + * This calls the subscriber receiver function synchronously. + **/ +void +orconn_event_publish(const orconn_event_msg_t *msg) +{ + SMARTLIST_FOREACH_BEGIN(orconn_event_rcvrs, orconn_event_rcvr_t, fn) { + tor_assert(fn); + (*fn)(msg); + } SMARTLIST_FOREACH_END(fn); +} + +const subsys_fns_t sys_orconn_event = { + .name = "orconn_event", + .supported = true, + .level = -33, + .initialize = orconn_event_init, + .shutdown = orconn_event_fini, +}; diff --git a/src/core/or/orconn_event.h b/src/core/or/orconn_event.h new file mode 100644 index 0000000000..80289d53e6 --- /dev/null +++ b/src/core/or/orconn_event.h @@ -0,0 +1,120 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file orconn_event.h + * \brief Header file for orconn_event.c + * + * The OR_CONN_STATE_* symbols are here to make it easier for + * subscribers to make decisions based on the messages that they + * receive. + **/ + +#ifndef TOR_ORCONN_EVENT_H +#define TOR_ORCONN_EVENT_H + +/** + * @name States of OR connections + * + * These must be in a partial ordering such that usually no OR + * connection will transition from a higher-numbered state to a + * lower-numbered one. Code such as bto_update_best() depends on this + * ordering to determine the best state it's seen so far. + * @{ */ +#define OR_CONN_STATE_MIN_ 1 +/** State for a connection to an OR: waiting for connect() to finish. */ +#define OR_CONN_STATE_CONNECTING 1 +/** State for a connection to an OR: waiting for proxy handshake to complete */ +#define OR_CONN_STATE_PROXY_HANDSHAKING 2 +/** State for an OR connection client: SSL is handshaking, not done + * yet. */ +#define OR_CONN_STATE_TLS_HANDSHAKING 3 +/** State for a connection to an OR: We're doing a second SSL handshake for + * renegotiation purposes. (V2 handshake only.) */ +#define OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING 4 +/** State for a connection at an OR: We're waiting for the client to + * renegotiate (to indicate a v2 handshake) or send a versions cell (to + * indicate a v3 handshake) */ +#define OR_CONN_STATE_TLS_SERVER_RENEGOTIATING 5 +/** State for an OR connection: We're done with our SSL handshake, we've done + * renegotiation, but we haven't yet negotiated link protocol versions and + * sent a netinfo cell. */ +#define OR_CONN_STATE_OR_HANDSHAKING_V2 6 +/** State for an OR connection: We're done with our SSL handshake, but we + * haven't yet negotiated link protocol versions, done a V3 handshake, and + * sent a netinfo cell. */ +#define OR_CONN_STATE_OR_HANDSHAKING_V3 7 +/** State for an OR connection: Ready to send/receive cells. */ +#define OR_CONN_STATE_OPEN 8 +#define OR_CONN_STATE_MAX_ 8 +/** @} */ + +/** Used to indicate the type of an OR connection event passed to the + * controller. The various types are defined in control-spec.txt */ +typedef enum or_conn_status_event_t { + OR_CONN_EVENT_LAUNCHED = 0, + OR_CONN_EVENT_CONNECTED = 1, + OR_CONN_EVENT_FAILED = 2, + OR_CONN_EVENT_CLOSED = 3, + OR_CONN_EVENT_NEW = 4, +} or_conn_status_event_t; + +/** Discriminant values for orconn event message */ +typedef enum orconn_msgtype_t { + ORCONN_MSGTYPE_STATE, + ORCONN_MSGTYPE_STATUS, +} orconn_msgtype_t; + +/** + * Message for orconn state update + * + * This contains information about internal state changes of + * or_connection_t objects. The chan and proxy_type fields are + * additional information that a subscriber may need to make + * decisions. + **/ +typedef struct orconn_state_msg_t { + uint64_t gid; /**< connection's global ID */ + uint64_t chan; /**< associated channel ID */ + int proxy_type; /**< connection's proxy type */ + uint8_t state; /**< new connection state */ +} orconn_state_msg_t; + +/** + * Message for orconn status event + * + * This contains information that ends up in ORCONN control protocol + * events. + **/ +typedef struct orconn_status_msg_t { + uint64_t gid; /**< connection's global ID */ + int status; /**< or_conn_status_event_t */ + int reason; /**< reason */ +} orconn_status_msg_t; + +/** Discriminated union for the actual message */ +typedef struct orconn_event_msg_t { + int type; + union { + orconn_state_msg_t state; + orconn_status_msg_t status; + } u; +} orconn_event_msg_t; + +/** + * Receiver function pointer for OR subscribers + * + * This function gets called synchronously by the publisher. + **/ +typedef void (*orconn_event_rcvr_t)(const orconn_event_msg_t *); + +void orconn_event_subscribe(orconn_event_rcvr_t); + +#ifdef ORCONN_EVENT_PRIVATE +void orconn_event_publish(const orconn_event_msg_t *); +#endif + +#endif /* defined(TOR_ORCONN_EVENT_H) */ diff --git a/src/core/or/orconn_event_sys.h b/src/core/or/orconn_event_sys.h new file mode 100644 index 0000000000..bfb0a3ac4a --- /dev/null +++ b/src/core/or/orconn_event_sys.h @@ -0,0 +1,12 @@ +/* Copyright (c) 2007-2019, The Tor Project, Inc. */ + +/** + * \file orconn_event_sys.h + * \brief Declare subsystem object for the OR connection event module. + **/ +#ifndef TOR_ORCONN_EVENT_SYS_H +#define TOR_ORCONN_EVENT_SYS_H + +extern const struct subsys_fns_t sys_orconn_event; + +#endif /* defined(TOR_ORCONN_SYS_H) */ diff --git a/src/core/or/origin_circuit_st.h b/src/core/or/origin_circuit_st.h index f55416db14..daa5f41dad 100644 --- a/src/core/or/origin_circuit_st.h +++ b/src/core/or/origin_circuit_st.h @@ -161,6 +161,10 @@ struct origin_circuit_t { * connections to this circuit. */ unsigned int unusable_for_new_conns : 1; + /* If this flag is set (due to padding negotiation failure), we should + * not try to negotiate further circuit padding. */ + unsigned padding_negotiation_failed : 1; + /** * Tristate variable to guard against pathbias miscounting * due to circuit purpose transitions changing the decision diff --git a/src/core/or/policies.c b/src/core/or/policies.c index 3ed282d785..a6d66d36de 100644 --- a/src/core/or/policies.c +++ b/src/core/or/policies.c @@ -29,6 +29,7 @@ #include "feature/relay/routermode.h" #include "lib/geoip/geoip.h" #include "ht.h" +#include "lib/crypt_ops/crypto_rand.h" #include "lib/encoding/confline.h" #include "core/or/addr_policy_st.h" @@ -461,7 +462,8 @@ fascist_firewall_use_ipv6(const or_options_t *options) * ClientPreferIPv6DirPort is deprecated, but check it anyway. */ return (options->ClientUseIPv6 == 1 || options->ClientUseIPv4 == 0 || options->ClientPreferIPv6ORPort == 1 || - options->ClientPreferIPv6DirPort == 1 || options->UseBridges == 1); + options->ClientPreferIPv6DirPort == 1 || options->UseBridges == 1 || + options->ClientAutoIPv6ORPort == 1); } /** Do we prefer to connect to IPv6, ignoring ClientPreferIPv6ORPort and @@ -488,6 +490,15 @@ fascist_firewall_prefer_ipv6_impl(const or_options_t *options) return -1; } +/* Choose whether we prefer IPv4 or IPv6 by randomly choosing an address + * family. Return 0 for IPv4, and 1 for IPv6. */ +MOCK_IMPL(int, +fascist_firewall_rand_prefer_ipv6_addr, (void)) +{ + /* TODO: Check for failures, and infer our preference based on this. */ + return crypto_rand_int(2); +} + /** Do we prefer to connect to IPv6 ORPorts? * Use node_ipv6_or_preferred() whenever possible: it supports bridge client * per-node IPv6 preferences. @@ -502,7 +513,10 @@ fascist_firewall_prefer_ipv6_orport(const or_options_t *options) } /* We can use both IPv4 and IPv6 - which do we prefer? */ - if (options->ClientPreferIPv6ORPort == 1) { + if (options->ClientAutoIPv6ORPort == 1) { + /* If ClientAutoIPv6ORPort is 1, we prefer IPv4 or IPv6 at random. */ + return fascist_firewall_rand_prefer_ipv6_addr(); + } else if (options->ClientPreferIPv6ORPort == 1) { return 1; } @@ -2706,7 +2720,7 @@ parse_short_policy(const char *summary) int is_accept; int n_entries; short_policy_entry_t entries[MAX_EXITPOLICY_SUMMARY_LEN]; /* overkill */ - const char *next; + char *next; if (!strcmpstart(summary, "accept ")) { is_accept = 1; @@ -2721,57 +2735,56 @@ parse_short_policy(const char *summary) n_entries = 0; for ( ; *summary; summary = next) { - const char *comma = strchr(summary, ','); - unsigned low, high; - char dummy; - char ent_buf[32]; - size_t len; - - next = comma ? comma+1 : strchr(summary, '\0'); - len = comma ? (size_t)(comma - summary) : strlen(summary); - if (n_entries == MAX_EXITPOLICY_SUMMARY_LEN) { log_fn(LOG_PROTOCOL_WARN, LD_DIR, "Impossibly long policy summary %s", escaped(orig_summary)); return NULL; } - if (! TOR_ISDIGIT(*summary) || len > (sizeof(ent_buf)-1)) { - /* unrecognized entry format. skip it. */ - continue; - } - if (len < 1) { - /* empty; skip it. */ - /* XXX This happens to be unreachable, since if len==0, then *summary is - * ',' or '\0', and the TOR_ISDIGIT test above would have failed. */ - continue; + unsigned low, high; + int ok; + low = (unsigned) tor_parse_ulong(summary, 10, 1, 65535, &ok, &next); + if (!ok) { + if (! TOR_ISDIGIT(*summary) || *summary == ',') { + /* Unrecognized format: skip it. */ + goto skip_ent; + } else { + goto bad_ent; + } } - memcpy(ent_buf, summary, len); - ent_buf[len] = '\0'; + switch (*next) { + case ',': + ++next; + /* fall through */ + case '\0': + high = low; + break; + case '-': + high = (unsigned) tor_parse_ulong(next+1, 10, low, 65535, &ok, &next); + if (!ok) + goto bad_ent; - if (tor_sscanf(ent_buf, "%u-%u%c", &low, &high, &dummy) == 2) { - if (low<1 || low>65535 || high<1 || high>65535 || low>high) { - log_fn(LOG_PROTOCOL_WARN, LD_DIR, - "Found bad entry in policy summary %s", escaped(orig_summary)); - return NULL; - } - } else if (tor_sscanf(ent_buf, "%u%c", &low, &dummy) == 1) { - if (low<1 || low>65535) { - log_fn(LOG_PROTOCOL_WARN, LD_DIR, - "Found bad entry in policy summary %s", escaped(orig_summary)); - return NULL; - } - high = low; - } else { - log_fn(LOG_PROTOCOL_WARN, LD_DIR,"Found bad entry in policy summary %s", - escaped(orig_summary)); - return NULL; + if (*next == ',') + ++next; + else if (*next != '\0') + goto bad_ent; + + break; + default: + goto bad_ent; } entries[n_entries].min_port = low; entries[n_entries].max_port = high; n_entries++; + + continue; + skip_ent: + next = strchr(next, ','); + if (!next) + break; + ++next; } if (n_entries == 0) { @@ -2792,6 +2805,11 @@ parse_short_policy(const char *summary) result->n_entries = n_entries; memcpy(result->entries, entries, sizeof(short_policy_entry_t)*n_entries); return result; + + bad_ent: + log_fn(LOG_PROTOCOL_WARN, LD_DIR,"Found bad entry in policy summary %s", + escaped(orig_summary)); + return NULL; } /** Write <b>policy</b> back out into a string. */ diff --git a/src/core/or/policies.h b/src/core/or/policies.h index 2c38de362f..324c1c2dd1 100644 --- a/src/core/or/policies.h +++ b/src/core/or/policies.h @@ -70,6 +70,7 @@ typedef struct short_policy_t { int firewall_is_fascist_or(void); int firewall_is_fascist_dir(void); int fascist_firewall_use_ipv6(const or_options_t *options); +MOCK_DECL(int, fascist_firewall_rand_prefer_ipv6_addr, (void)); int fascist_firewall_prefer_ipv6_orport(const or_options_t *options); int fascist_firewall_prefer_ipv6_dirport(const or_options_t *options); diff --git a/src/core/or/protover.c b/src/core/or/protover.c index 17979d04ea..53709ad002 100644 --- a/src/core/or/protover.c +++ b/src/core/or/protover.c @@ -39,6 +39,9 @@ static int protocol_list_contains(const smartlist_t *protos, static const struct { protocol_type_t protover_type; const char *name; +/* If you add a new protocol here, you probably also want to add + * parsing for it in routerstatus_parse_entry_from_string() so that + * it is set in routerstatus_t */ } PROTOCOL_NAMES[] = { { PRT_LINK, "Link" }, { PRT_LINKAUTH, "LinkAuth" }, @@ -49,6 +52,7 @@ static const struct { { PRT_HSREND, "HSRend" }, { PRT_DESC, "Desc" }, { PRT_MICRODESC, "Microdesc"}, + { PRT_PADDING, "Padding"}, { PRT_CONS, "Cons" } }; @@ -396,7 +400,8 @@ protover_get_supported_protocols(void) "LinkAuth=3 " #endif "Microdesc=1-2 " - "Relay=1-2"; + "Relay=1-2 " + "Padding=1"; } /** The protocols from protover_get_supported_protocols(), as parsed into a diff --git a/src/core/or/protover.h b/src/core/or/protover.h index 7e181ba97a..567b94a168 100644 --- a/src/core/or/protover.h +++ b/src/core/or/protover.h @@ -33,16 +33,17 @@ struct smartlist_t; /// C_RUST_COUPLED: src/rust/protover/ffi.rs `translate_to_rust` /// C_RUST_COUPLED: src/rust/protover/protover.rs `Proto` typedef enum protocol_type_t { - PRT_LINK, - PRT_LINKAUTH, - PRT_RELAY, - PRT_DIRCACHE, - PRT_HSDIR, - PRT_HSINTRO, - PRT_HSREND, - PRT_DESC, - PRT_MICRODESC, - PRT_CONS, + PRT_LINK = 0, + PRT_LINKAUTH = 1, + PRT_RELAY = 2, + PRT_DIRCACHE = 3, + PRT_HSDIR = 4, + PRT_HSINTRO = 5, + PRT_HSREND = 6, + PRT_DESC = 7, + PRT_MICRODESC = 8, + PRT_CONS = 9, + PRT_PADDING = 10, } protocol_type_t; bool protover_contains_long_protocol_names(const char *s); diff --git a/src/core/or/relay.c b/src/core/or/relay.c index dc88a6f649..706a6e05cb 100644 --- a/src/core/or/relay.c +++ b/src/core/or/relay.c @@ -49,12 +49,13 @@ #include "core/or/or.h" #include "feature/client/addressmap.h" #include "lib/err/backtrace.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/or/channel.h" #include "feature/client/circpathbias.h" #include "core/or/circuitbuild.h" #include "core/or/circuitlist.h" #include "core/or/circuituse.h" +#include "core/or/circuitpadding.h" #include "lib/compress/compress.h" #include "app/config/config.h" #include "core/mainloop/connection.h" @@ -80,7 +81,6 @@ #include "feature/nodelist/describe.h" #include "feature/nodelist/routerlist.h" #include "core/or/scheduler.h" -#include "feature/stats/rephist.h" #include "core/or/cell_st.h" #include "core/or/cell_queue_st.h" @@ -293,7 +293,9 @@ circuit_receive_relay_cell(cell_t *cell, circuit_t *circ, return 0; } - /* not recognized. pass it on. */ + /* not recognized. inform circpad and pass it on. */ + circpad_deliver_unrecognized_cell_events(circ, cell_direction); + if (cell_direction == CELL_DIRECTION_OUT) { cell->circ_id = circ->n_circ_id; /* switch it */ chan = circ->n_chan; @@ -353,11 +355,11 @@ circuit_receive_relay_cell(cell_t *cell, circuit_t *circ, * - Encrypt it to the right layer * - Append it to the appropriate cell_queue on <b>circ</b>. */ -static int -circuit_package_relay_cell(cell_t *cell, circuit_t *circ, +MOCK_IMPL(int, +circuit_package_relay_cell, (cell_t *cell, circuit_t *circ, cell_direction_t cell_direction, crypt_path_t *layer_hint, streamid_t on_stream, - const char *filename, int lineno) + const char *filename, int lineno)) { channel_t *chan; /* where to send the cell */ @@ -524,6 +526,8 @@ relay_command_to_string(uint8_t command) case RELAY_COMMAND_INTRODUCE_ACK: return "INTRODUCE_ACK"; case RELAY_COMMAND_EXTEND2: return "EXTEND2"; case RELAY_COMMAND_EXTENDED2: return "EXTENDED2"; + case RELAY_COMMAND_PADDING_NEGOTIATE: return "PADDING_NEGOTIATE"; + case RELAY_COMMAND_PADDING_NEGOTIATED: return "PADDING_NEGOTIATED"; default: tor_snprintf(buf, sizeof(buf), "Unrecognized relay command %u", (unsigned)command); @@ -577,8 +581,8 @@ relay_send_command_from_edge_,(streamid_t stream_id, circuit_t *circ, log_debug(LD_OR,"delivering %d cell %s.", relay_command, cell_direction == CELL_DIRECTION_OUT ? "forward" : "backward"); - if (relay_command == RELAY_COMMAND_DROP) - rep_hist_padding_count_write(PADDING_TYPE_DROP); + /* Tell circpad we're sending a relay cell */ + circpad_deliver_sent_relay_cell_events(circ, relay_command); /* If we are sending an END cell and this circuit is used for a tunneled * directory request, advance its state. */ @@ -602,7 +606,9 @@ relay_send_command_from_edge_,(streamid_t stream_id, circuit_t *circ, * one of them. Don't worry about the conn protocol version: * append_cell_to_circuit_queue will fix it up. */ cell.command = CELL_RELAY_EARLY; - --origin_circ->remaining_relay_early_cells; + /* If we're out of relay early cells, tell circpad */ + if (--origin_circ->remaining_relay_early_cells == 0) + circpad_machine_event_circ_has_no_relay_early(origin_circ); log_debug(LD_OR, "Sending a RELAY_EARLY cell; %d remaining.", (int)origin_circ->remaining_relay_early_cells); /* Memorize the command that is sent as RELAY_EARLY cell; helps debug @@ -1481,9 +1487,11 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, } } + /* Tell circpad that we've recieved a recognized cell */ + circpad_deliver_recognized_relay_cell_events(circ, rh.command, layer_hint); + /* either conn is NULL, in which case we've got a control cell, or else * conn points to the recognized stream. */ - if (conn && !connection_state_is_open(TO_CONN(conn))) { if (conn->base_.type == CONN_TYPE_EXIT && (conn->base_.state == EXIT_CONN_STATE_CONNECTING || @@ -1504,8 +1512,14 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, switch (rh.command) { case RELAY_COMMAND_DROP: - rep_hist_padding_count_read(PADDING_TYPE_DROP); -// log_info(domain,"Got a relay-level padding cell. Dropping."); + /* Already examined in circpad_deliver_recognized_relay_cell_events */ + return 0; + case RELAY_COMMAND_PADDING_NEGOTIATE: + circpad_handle_padding_negotiate(circ, cell); + return 0; + case RELAY_COMMAND_PADDING_NEGOTIATED: + if (circpad_handle_padding_negotiated(circ, cell, layer_hint) == 0) + circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh.length); return 0; case RELAY_COMMAND_BEGIN: case RELAY_COMMAND_BEGIN_DIR: diff --git a/src/core/or/relay.h b/src/core/or/relay.h index 7cc3c43e43..044f6be156 100644 --- a/src/core/or/relay.h +++ b/src/core/or/relay.h @@ -78,6 +78,11 @@ void destroy_cell_queue_append(destroy_cell_queue_t *queue, void channel_unlink_all_circuits(channel_t *chan, smartlist_t *detached_out); MOCK_DECL(int, channel_flush_from_first_active_circuit, (channel_t *chan, int max)); +MOCK_DECL(int, circuit_package_relay_cell, (cell_t *cell, circuit_t *circ, + cell_direction_t cell_direction, + crypt_path_t *layer_hint, streamid_t on_stream, + const char *filename, int lineno)); + void update_circuit_on_cmux_(circuit_t *circ, cell_direction_t direction, const char *file, int lineno); #define update_circuit_on_cmux(circ, direction) \ diff --git a/src/core/or/scheduler.c b/src/core/or/scheduler.c index f85201b7d5..ee22a38142 100644 --- a/src/core/or/scheduler.c +++ b/src/core/or/scheduler.c @@ -9,7 +9,7 @@ #define SCHEDULER_KIST_PRIVATE #include "core/or/scheduler.h" #include "core/mainloop/mainloop.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #define TOR_CHANNEL_INTERNAL_ #include "core/or/channeltls.h" #include "lib/evloop/compat_libevent.h" diff --git a/src/core/or/scheduler_kist.c b/src/core/or/scheduler_kist.c index 79ecb0bc7e..1ec1c49c73 100644 --- a/src/core/or/scheduler_kist.c +++ b/src/core/or/scheduler_kist.c @@ -4,7 +4,7 @@ #define SCHEDULER_KIST_PRIVATE #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "app/config/config.h" #include "core/mainloop/connection.h" #include "feature/nodelist/networkstatus.h" @@ -724,7 +724,7 @@ kist_scheduler_run(void) SMARTLIST_FOREACH_BEGIN(to_readd, channel_t *, readd_chan) { scheduler_set_channel_state(readd_chan, SCHED_CHAN_PENDING); if (!smartlist_contains(cp, readd_chan)) { - if (!SCHED_BUG(chan->sched_heap_idx != -1, chan)) { + if (!SCHED_BUG(readd_chan->sched_heap_idx != -1, readd_chan)) { /* XXXX Note that the check above is in theory redundant with * the smartlist_contains check. But let's make sure we're * not messing anything up, and leave them both for now. */ diff --git a/src/core/or/versions.c b/src/core/or/versions.c index 33273a5294..2a572d4704 100644 --- a/src/core/or/versions.c +++ b/src/core/or/versions.c @@ -16,6 +16,25 @@ #include "core/or/tor_version_st.h" +/** + * Return the approximate date when this release came out, or was + * scheduled to come out, according to the APPROX_RELEASE_DATE set in + * configure.ac + **/ +time_t +tor_get_approx_release_date(void) +{ + char tbuf[ISO_TIME_LEN+1]; + tor_snprintf(tbuf, sizeof(tbuf), + "%s 00:00:00", APPROX_RELEASE_DATE); + time_t result = 0; + int r = parse_iso_time(tbuf, &result); + if (BUG(r < 0)) { + result = 0; + } + return result; +} + /** Return VS_RECOMMENDED if <b>myversion</b> is contained in * <b>versionlist</b>. Else, return VS_EMPTY if versionlist has no * entries. Else, return VS_OLD if every member of @@ -377,6 +396,66 @@ sort_version_list(smartlist_t *versions, int remove_duplicates) smartlist_uniq(versions, compare_tor_version_str_ptr_, tor_free_); } +/** If there are more than this many entries, we're probably under + * some kind of weird DoS. */ +static const int MAX_PROTOVER_SUMMARY_MAP_LEN = 1024; + +/** + * Map from protover string to protover_summary_flags_t. + */ +static strmap_t *protover_summary_map = NULL; + +/** + * Helper. Given a non-NULL protover string <b>protocols</b>, set <b>out</b> + * to its summary, and memoize the result in <b>protover_summary_map</b>. + */ +static void +memoize_protover_summary(protover_summary_flags_t *out, + const char *protocols) +{ + if (!protover_summary_map) + protover_summary_map = strmap_new(); + + if (strmap_size(protover_summary_map) >= MAX_PROTOVER_SUMMARY_MAP_LEN) { + protover_summary_cache_free_all(); + tor_assert(protover_summary_map == NULL); + protover_summary_map = strmap_new(); + } + + const protover_summary_flags_t *cached = + strmap_get(protover_summary_map, protocols); + + if (cached != NULL) { + /* We found a cached entry; no need to parse this one. */ + memcpy(out, cached, sizeof(protover_summary_flags_t)); + tor_assert(out->protocols_known); + return; + } + + memset(out, 0, sizeof(*out)); + out->protocols_known = 1; + out->supports_extend2_cells = + protocol_list_supports_protocol(protocols, PRT_RELAY, 2); + out->supports_ed25519_link_handshake_compat = + protocol_list_supports_protocol(protocols, PRT_LINKAUTH, 3); + out->supports_ed25519_link_handshake_any = + protocol_list_supports_protocol_or_later(protocols, PRT_LINKAUTH, 3); + out->supports_ed25519_hs_intro = + protocol_list_supports_protocol(protocols, PRT_HSINTRO, 4); + out->supports_v3_hsdir = + protocol_list_supports_protocol(protocols, PRT_HSDIR, + PROTOVER_HSDIR_V3); + out->supports_v3_rendezvous_point = + protocol_list_supports_protocol(protocols, PRT_HSREND, + PROTOVER_HS_RENDEZVOUS_POINT_V3); + out->supports_padding = + protocol_list_supports_protocol(protocols, PRT_PADDING, 1); + + protover_summary_flags_t *new_cached = tor_memdup(out, sizeof(*out)); + cached = strmap_set(protover_summary_map, protocols, new_cached); + tor_assert(!cached); +} + /** Summarize the protocols listed in <b>protocols</b> into <b>out</b>, * falling back or correcting them based on <b>version</b> as appropriate. */ @@ -388,21 +467,7 @@ summarize_protover_flags(protover_summary_flags_t *out, tor_assert(out); memset(out, 0, sizeof(*out)); if (protocols) { - out->protocols_known = 1; - out->supports_extend2_cells = - protocol_list_supports_protocol(protocols, PRT_RELAY, 2); - out->supports_ed25519_link_handshake_compat = - protocol_list_supports_protocol(protocols, PRT_LINKAUTH, 3); - out->supports_ed25519_link_handshake_any = - protocol_list_supports_protocol_or_later(protocols, PRT_LINKAUTH, 3); - out->supports_ed25519_hs_intro = - protocol_list_supports_protocol(protocols, PRT_HSINTRO, 4); - out->supports_v3_hsdir = - protocol_list_supports_protocol(protocols, PRT_HSDIR, - PROTOVER_HSDIR_V3); - out->supports_v3_rendezvous_point = - protocol_list_supports_protocol(protocols, PRT_HSREND, - PROTOVER_HS_RENDEZVOUS_POINT_V3); + memoize_protover_summary(out, protocols); } if (version && !strcmpstart(version, "Tor ")) { if (!out->protocols_known) { @@ -420,3 +485,13 @@ summarize_protover_flags(protover_summary_flags_t *out, } } } + +/** + * Free all space held in the protover_summary_map. + */ +void +protover_summary_cache_free_all(void) +{ + strmap_free(protover_summary_map, tor_free_); + protover_summary_map = NULL; +} diff --git a/src/core/or/versions.h b/src/core/or/versions.h index 22f3be176f..9aa7a0db87 100644 --- a/src/core/or/versions.h +++ b/src/core/or/versions.h @@ -26,6 +26,8 @@ typedef enum version_status_t { VS_UNKNOWN, /**< We have no idea. */ } version_status_t; +time_t tor_get_approx_release_date(void); + version_status_t tor_version_is_obsolete(const char *myversion, const char *versionlist); int tor_version_parse_platform(const char *platform, @@ -41,4 +43,6 @@ void summarize_protover_flags(protover_summary_flags_t *out, const char *protocols, const char *version); +void protover_summary_cache_free_all(void); + #endif /* !defined(TOR_VERSIONS_H) */ diff --git a/src/core/proto/proto_cell.c b/src/core/proto/proto_cell.c index 0442e2c6ee..697fed29e1 100644 --- a/src/core/proto/proto_cell.c +++ b/src/core/proto/proto_cell.c @@ -5,7 +5,7 @@ /* See LICENSE for licensing information */ #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/proto/proto_cell.h" #include "core/or/connection_or.h" diff --git a/src/core/proto/proto_control0.c b/src/core/proto/proto_control0.c index 21fa328f02..d741f28f09 100644 --- a/src/core/proto/proto_control0.c +++ b/src/core/proto/proto_control0.c @@ -5,7 +5,7 @@ /* See LICENSE for licensing information */ #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/proto/proto_control0.h" /** Return 1 iff buf looks more like it has an (obsolete) v0 controller diff --git a/src/core/proto/proto_ext_or.c b/src/core/proto/proto_ext_or.c index edbc51b10c..4213bc14dd 100644 --- a/src/core/proto/proto_ext_or.c +++ b/src/core/proto/proto_ext_or.c @@ -5,7 +5,7 @@ /* See LICENSE for licensing information */ #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "feature/relay/ext_orport.h" #include "core/proto/proto_ext_or.h" diff --git a/src/core/proto/proto_http.c b/src/core/proto/proto_http.c index 5c86fc4979..88c59ef561 100644 --- a/src/core/proto/proto_http.c +++ b/src/core/proto/proto_http.c @@ -6,7 +6,7 @@ #define PROTO_HTTP_PRIVATE #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/proto/proto_http.h" /** Return true if <b>cmd</b> looks like a HTTP (proxy) request. */ diff --git a/src/core/proto/proto_socks.c b/src/core/proto/proto_socks.c index 8b78ed44c2..ac0c9e911b 100644 --- a/src/core/proto/proto_socks.c +++ b/src/core/proto/proto_socks.c @@ -6,7 +6,7 @@ #include "core/or/or.h" #include "feature/client/addressmap.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/mainloop/connection.h" #include "feature/control/control.h" #include "app/config/config.h" diff --git a/src/ext/.may_include b/src/ext/.may_include new file mode 100644 index 0000000000..1eafff2eeb --- /dev/null +++ b/src/ext/.may_include @@ -0,0 +1,10 @@ + +orconfig.h + +lib/err/*.h +lib/cc/*.h + +tinytest*.h +ext/siphash.h +ext/byteorder.h +ext/tor_readpassphrase.h
\ No newline at end of file diff --git a/src/ext/csiphash.c b/src/ext/csiphash.c index a6a9846db4..af8559a476 100644 --- a/src/ext/csiphash.c +++ b/src/ext/csiphash.c @@ -30,12 +30,12 @@ */ #include "lib/cc/torint.h" -#include "lib/log/util_bug.h" +#include "lib/err/torerr.h" -#include "siphash.h" +#include "ext/siphash.h" #include <string.h> #include <stdlib.h> -#include "byteorder.h" +#include "ext/byteorder.h" #define ROTATE(x, b) (uint64_t)( ((x) << (b)) | ( (x) >> (64 - (b))) ) @@ -112,13 +112,13 @@ static int the_siphash_key_is_set = 0; static struct sipkey the_siphash_key; uint64_t siphash24g(const void *src, unsigned long src_sz) { - tor_assert(the_siphash_key_is_set); + raw_assert(the_siphash_key_is_set); return siphash24(src, src_sz, &the_siphash_key); } void siphash_set_global_key(const struct sipkey *key) { - tor_assert(! the_siphash_key_is_set); + raw_assert(! the_siphash_key_is_set); the_siphash_key.k0 = key->k0; the_siphash_key.k1 = key->k1; the_siphash_key_is_set = 1; diff --git a/src/ext/readpassphrase.c b/src/ext/readpassphrase.c index e0df05d7b7..16611af1e2 100644 --- a/src/ext/readpassphrase.c +++ b/src/ext/readpassphrase.c @@ -30,7 +30,7 @@ #include <signal.h> #include <ctype.h> #include <fcntl.h> -#include "tor_readpassphrase.h" +#include "ext/tor_readpassphrase.h" #include <errno.h> #include <string.h> #include <unistd.h> diff --git a/src/ext/timeouts/.may_include b/src/ext/timeouts/.may_include new file mode 100644 index 0000000000..42f44befd4 --- /dev/null +++ b/src/ext/timeouts/.may_include @@ -0,0 +1,6 @@ +orconfig.h + +ext/tor_queue.h +timeout-bitops.c +timeout-debug.h +timeout.h diff --git a/src/ext/timeouts/timeout.c b/src/ext/timeouts/timeout.c index d4b514d2c5..07d06772c5 100644 --- a/src/ext/timeouts/timeout.c +++ b/src/ext/timeouts/timeout.c @@ -38,7 +38,7 @@ #include <errno.h> /* errno */ -#include "tor_queue.h" /* TAILQ(3) */ +#include "ext/tor_queue.h" /* TAILQ(3) */ #include "timeout.h" @@ -531,7 +531,7 @@ static timeout_t timeouts_int(struct timeouts *T) { timeout = MIN(_timeout, timeout); } - relmask <<= WHEEL_BIT; + relmask <<= WHEEL_BIT; relmask |= WHEEL_MASK; } @@ -751,4 +751,3 @@ TIMEOUT_PUBLIC int timeout_v_abi(void) { TIMEOUT_PUBLIC int timeout_v_api(void) { return TIMEOUT_V_API; } /* timeout_version() */ - diff --git a/src/ext/timeouts/timeout.h b/src/ext/timeouts/timeout.h index b35874e153..1ed309fd08 100644 --- a/src/ext/timeouts/timeout.h +++ b/src/ext/timeouts/timeout.h @@ -31,7 +31,7 @@ #include <inttypes.h> /* PRIu64 PRIx64 PRIX64 uint64_t */ -#include "tor_queue.h" /* TAILQ(3) */ +#include "ext/tor_queue.h" /* TAILQ(3) */ /* @@ -147,7 +147,7 @@ TIMEOUT_PUBLIC struct timeout *timeout_init(struct timeout *, int); #ifndef TIMEOUT_DISABLE_RELATIVE_ACCESS TIMEOUT_PUBLIC bool timeout_pending(struct timeout *); /* true if on timing wheel, false otherwise */ - + TIMEOUT_PUBLIC bool timeout_expired(struct timeout *); /* true if on expired queue, false otherwise */ diff --git a/src/feature/client/bridges.c b/src/feature/client/bridges.c index 626c5efcae..05f89ad36c 100644 --- a/src/feature/client/bridges.c +++ b/src/feature/client/bridges.c @@ -844,7 +844,8 @@ rewrite_node_address_for_bridge(const bridge_info_t *bridge, node_t *node) } } - if (options->ClientPreferIPv6ORPort == -1) { + if (options->ClientPreferIPv6ORPort == -1 || + options->ClientAutoIPv6ORPort == 0) { /* Mark which address to use based on which bridge_t we got. */ node->ipv6_preferred = (tor_addr_family(&bridge->addr) == AF_INET6 && !tor_addr_is_null(&node->ri->ipv6_addr)); diff --git a/src/feature/client/dnsserv.c b/src/feature/client/dnsserv.c index cf593cfb68..44e0caaafa 100644 --- a/src/feature/client/dnsserv.c +++ b/src/feature/client/dnsserv.c @@ -28,6 +28,7 @@ #include "core/or/connection_edge.h" #include "feature/control/control.h" #include "core/mainloop/mainloop.h" +#include "core/mainloop/netstatus.h" #include "core/or/policies.h" #include "feature/control/control_connection_st.h" @@ -213,6 +214,9 @@ dnsserv_launch_request(const char *name, int reverse, edge_connection_t *conn; char *q_name; + /* Launching a request for a user counts as user activity. */ + note_user_activity(approx_time()); + /* Make a new dummy AP connection, and attach the request to it. */ entry_conn = entry_connection_new(CONN_TYPE_AP, AF_INET); entry_conn->entry_cfg.dns_request = 1; diff --git a/src/feature/client/transports.c b/src/feature/client/transports.c index 0d1342c87b..f34d4af480 100644 --- a/src/feature/client/transports.c +++ b/src/feature/client/transports.c @@ -101,11 +101,13 @@ #include "core/or/connection_or.h" #include "feature/relay/ext_orport.h" #include "feature/control/control.h" +#include "lib/encoding/confline.h" +#include "lib/encoding/kvline.h" +#include "lib/process/process.h" #include "lib/process/env.h" -#include "lib/process/subprocess.h" -static process_environment_t * +static smartlist_t * create_managed_proxy_environment(const managed_proxy_t *mp); static inline int proxy_configuration_finished(const managed_proxy_t *mp); @@ -127,6 +129,8 @@ static void parse_method_error(const char *line, int is_server_method); #define PROTO_SMETHODS_DONE "SMETHODS DONE" #define PROTO_PROXY_DONE "PROXY DONE" #define PROTO_PROXY_ERROR "PROXY-ERROR" +#define PROTO_LOG "LOG" +#define PROTO_STATUS "STATUS" /** The first and only supported - at the moment - configuration protocol version. */ @@ -490,8 +494,8 @@ proxy_prepare_for_restart(managed_proxy_t *mp) tor_assert(mp->conf_state == PT_PROTO_COMPLETED); /* destroy the process handle and terminate the process. */ - tor_process_handle_destroy(mp->process_handle, 1); - mp->process_handle = NULL; + process_set_data(mp->process, NULL); + process_terminate(mp->process); /* destroy all its registered transports, since we will no longer use them. */ @@ -520,34 +524,35 @@ proxy_prepare_for_restart(managed_proxy_t *mp) static int launch_managed_proxy(managed_proxy_t *mp) { - int retval; - - process_environment_t *env = create_managed_proxy_environment(mp); - -#ifdef _WIN32 - /* Passing NULL as lpApplicationName makes Windows search for the .exe */ - retval = tor_spawn_background(NULL, - (const char **)mp->argv, - env, - &mp->process_handle); -#else /* !(defined(_WIN32)) */ - retval = tor_spawn_background(mp->argv[0], - (const char **)mp->argv, - env, - &mp->process_handle); -#endif /* defined(_WIN32) */ - - process_environment_free(env); - - if (retval == PROCESS_STATUS_ERROR) { - log_warn(LD_GENERAL, "Managed proxy at '%s' failed at launch.", + tor_assert(mp); + + smartlist_t *env = create_managed_proxy_environment(mp); + + /* Configure our process. */ + process_set_data(mp->process, mp); + process_set_stdout_read_callback(mp->process, managed_proxy_stdout_callback); + process_set_stderr_read_callback(mp->process, managed_proxy_stderr_callback); + process_set_exit_callback(mp->process, managed_proxy_exit_callback); + process_set_protocol(mp->process, PROCESS_PROTOCOL_LINE); + process_reset_environment(mp->process, env); + + /* Cleanup our env. */ + SMARTLIST_FOREACH(env, char *, x, tor_free(x)); + smartlist_free(env); + + /* Skip the argv[0] as we get that from process_new(argv[0]). */ + for (int i = 1; mp->argv[i] != NULL; ++i) + process_append_argument(mp->process, mp->argv[i]); + + if (process_exec(mp->process) != PROCESS_STATUS_RUNNING) { + log_warn(LD_CONFIG, "Managed proxy at '%s' failed at launch.", mp->argv[0]); return -1; } - log_info(LD_CONFIG, "Managed proxy at '%s' has spawned with PID '%d'.", - mp->argv[0], tor_process_get_pid(mp->process_handle)); - + log_info(LD_CONFIG, + "Managed proxy at '%s' has spawned with PID '%" PRIu64 "'.", + mp->argv[0], process_get_pid(mp->process)); mp->conf_state = PT_PROTO_LAUNCHED; return 0; @@ -615,10 +620,6 @@ pt_configure_remaining_proxies(void) STATIC int configure_proxy(managed_proxy_t *mp) { - int configuration_finished = 0; - smartlist_t *proxy_output = NULL; - enum stream_status stream_status = 0; - /* if we haven't launched the proxy yet, do it now */ if (mp->conf_state == PT_PROTO_INFANT) { if (launch_managed_proxy(mp) < 0) { /* launch fail */ @@ -629,45 +630,8 @@ configure_proxy(managed_proxy_t *mp) } tor_assert(mp->conf_state != PT_PROTO_INFANT); - tor_assert(mp->process_handle); - - proxy_output = - tor_get_lines_from_handle(tor_process_get_stdout_pipe(mp->process_handle), - &stream_status); - if (!proxy_output) { /* failed to get input from proxy */ - if (stream_status != IO_STREAM_EAGAIN) { /* bad stream status! */ - mp->conf_state = PT_PROTO_BROKEN; - log_warn(LD_GENERAL, "The communication stream of managed proxy '%s' " - "is '%s'. Most probably the managed proxy stopped running. " - "This might be a bug of the managed proxy, a bug of Tor, or " - "a misconfiguration. Please enable logging on your managed " - "proxy and check the logs for errors.", - mp->argv[0], stream_status_to_string(stream_status)); - } - - goto done; - } - - /* Handle lines. */ - SMARTLIST_FOREACH_BEGIN(proxy_output, const char *, line) { - handle_proxy_line(line, mp); - if (proxy_configuration_finished(mp)) - goto done; - } SMARTLIST_FOREACH_END(line); - - done: - /* if the proxy finished configuring, exit the loop. */ - if (proxy_configuration_finished(mp)) { - handle_finished_proxy(mp); - configuration_finished = 1; - } - - if (proxy_output) { - SMARTLIST_FOREACH(proxy_output, char *, cp, tor_free(cp)); - smartlist_free(proxy_output); - } - - return configuration_finished; + tor_assert(mp->process); + return mp->conf_state == PT_PROTO_COMPLETED; } /** Register server managed proxy <b>mp</b> transports to state */ @@ -748,8 +712,14 @@ managed_proxy_destroy(managed_proxy_t *mp, /* free the outgoing proxy URI */ tor_free(mp->proxy_uri); - tor_process_handle_destroy(mp->process_handle, also_terminate_process); - mp->process_handle = NULL; + /* do we want to terminate our process if it's still running? */ + if (also_terminate_process && mp->process) { + /* Note that we do not call process_free(mp->process) here because we let + * the exit handler in managed_proxy_exit_callback() return `true` which + * makes the process subsystem deallocate the process_t. */ + process_set_data(mp->process, NULL); + process_terminate(mp->process); + } tor_free(mp); } @@ -945,21 +915,16 @@ handle_proxy_line(const char *line, managed_proxy_t *mp) parse_proxy_error(line); goto err; - } else if (!strcmpstart(line, SPAWN_ERROR_MESSAGE)) { - /* managed proxy launch failed: parse error message to learn why. */ - int retval, child_state, saved_errno; - retval = tor_sscanf(line, SPAWN_ERROR_MESSAGE "%x/%x", - &child_state, &saved_errno); - if (retval == 2) { - log_warn(LD_GENERAL, - "Could not launch managed proxy executable at '%s' ('%s').", - mp->argv[0], strerror(saved_errno)); - } else { /* failed to parse error message */ - log_warn(LD_GENERAL,"Could not launch managed proxy executable at '%s'.", - mp->argv[0]); - } - mp->conf_state = PT_PROTO_FAILED_LAUNCH; + /* We check for the additional " " after the PROTO_LOG * PROTO_STATUS + * string to make sure we can later extend this big if/else-if table with + * something that begins with "LOG" without having to get the order right. + * */ + } else if (!strcmpstart(line, PROTO_LOG " ")) { + parse_log_line(line, mp); + return; + } else if (!strcmpstart(line, PROTO_STATUS " ")) { + parse_status_line(line, mp); return; } @@ -1182,6 +1147,121 @@ parse_proxy_error(const char *line) line+strlen(PROTO_PROXY_ERROR)+1); } +/** Parses a LOG <b>line</b> and emit log events accordingly. */ +STATIC void +parse_log_line(const char *line, managed_proxy_t *mp) +{ + tor_assert(line); + tor_assert(mp); + + config_line_t *values = NULL; + char *log_message = NULL; + + if (strlen(line) < (strlen(PROTO_LOG) + 1)) { + log_warn(LD_PT, "Managed proxy sent us a %s line " + "with missing argument.", PROTO_LOG); + goto done; + } + + const char *data = line + strlen(PROTO_LOG) + 1; + values = kvline_parse(data, KV_QUOTED); + + if (! values) { + log_warn(LD_PT, "Managed proxy \"%s\" wrote an invalid LOG message: %s", + mp->argv[0], data); + goto done; + } + + const config_line_t *severity = config_line_find(values, "SEVERITY"); + const config_line_t *message = config_line_find(values, "MESSAGE"); + + /* Check if we got a message. */ + if (! message) { + log_warn(LD_PT, "Managed proxy \"%s\" wrote a LOG line without " + "MESSAGE: %s", mp->argv[0], escaped(data)); + goto done; + } + + /* Check if severity is there and whether it's valid. */ + if (! severity) { + log_warn(LD_PT, "Managed proxy \"%s\" wrote a LOG line without " + "SEVERITY: %s", mp->argv[0], escaped(data)); + goto done; + } + + int log_severity = managed_proxy_severity_parse(severity->value); + + if (log_severity == -1) { + log_warn(LD_PT, "Managed proxy \"%s\" wrote a LOG line with an " + "invalid severity level: %s", + mp->argv[0], severity->value); + goto done; + } + + tor_log(log_severity, LD_PT, "Managed proxy \"%s\": %s", + mp->argv[0], message->value); + + /* Prepend the PT name. */ + config_line_prepend(&values, "PT", mp->argv[0]); + log_message = kvline_encode(values, KV_QUOTED); + + /* Emit control port event. */ + control_event_pt_log(log_message); + + done: + config_free_lines(values); + tor_free(log_message); +} + +/** Parses a STATUS <b>line</b> and emit control events accordingly. */ +STATIC void +parse_status_line(const char *line, managed_proxy_t *mp) +{ + tor_assert(line); + tor_assert(mp); + + config_line_t *values = NULL; + char *status_message = NULL; + + if (strlen(line) < (strlen(PROTO_STATUS) + 1)) { + log_warn(LD_PT, "Managed proxy sent us a %s line " + "with missing argument.", PROTO_STATUS); + goto done; + } + + const char *data = line + strlen(PROTO_STATUS) + 1; + + values = kvline_parse(data, KV_QUOTED); + + if (! values) { + log_warn(LD_PT, "Managed proxy \"%s\" wrote an invalid " + "STATUS message: %s", mp->argv[0], escaped(data)); + goto done; + } + + /* We check if we received the TRANSPORT parameter, which is the only + * *required* value. */ + const config_line_t *type = config_line_find(values, "TRANSPORT"); + + if (! type) { + log_warn(LD_PT, "Managed proxy \"%s\" wrote a STATUS line without " + "TRANSPORT: %s", mp->argv[0], escaped(data)); + goto done; + } + + /* Prepend the PT name. */ + config_line_prepend(&values, "PT", mp->argv[0]); + status_message = kvline_encode(values, KV_QUOTED); + + /* We have checked that TRANSPORT is there, we can now emit the STATUS event + * via the control port. */ + control_event_pt_status(status_message); + + done: + config_free_lines(values); + tor_free(status_message); +} + /** Return a newly allocated string that tor should place in * TOR_PT_SERVER_TRANSPORT_OPTIONS while configuring the server * manged proxy in <b>mp</b>. Return NULL if no such options are found. */ @@ -1257,7 +1337,7 @@ get_bindaddr_for_server_proxy(const managed_proxy_t *mp) /** Return a newly allocated process_environment_t * for <b>mp</b>'s * process. */ -static process_environment_t * +static smartlist_t * create_managed_proxy_environment(const managed_proxy_t *mp) { const or_options_t *options = get_options(); @@ -1272,8 +1352,6 @@ create_managed_proxy_environment(const managed_proxy_t *mp) /* The final environment to be passed to mp. */ smartlist_t *merged_env_vars = get_current_process_environment_variables(); - process_environment_t *env; - { char *state_tmp = get_datadir_fname("pt_state/"); /* XXX temp */ smartlist_add_asprintf(envs, "TOR_PT_STATE_LOCATION=%s", state_tmp); @@ -1366,14 +1444,9 @@ create_managed_proxy_environment(const managed_proxy_t *mp) tor_free_, 1); } SMARTLIST_FOREACH_END(env_var); - env = process_environment_make(merged_env_vars); - smartlist_free(envs); - SMARTLIST_FOREACH(merged_env_vars, void *, x, tor_free(x)); - smartlist_free(merged_env_vars); - - return env; + return merged_env_vars; } /** Create and return a new managed proxy for <b>transport</b> using @@ -1392,6 +1465,7 @@ managed_proxy_create(const smartlist_t *with_transport_list, mp->argv = proxy_argv; mp->transports = smartlist_new(); mp->proxy_uri = get_pt_proxy_uri(); + mp->process = process_new(proxy_argv[0]); mp->transports_to_launch = smartlist_new(); SMARTLIST_FOREACH(with_transport_list, const char *, transport, @@ -1736,3 +1810,93 @@ tor_escape_str_for_pt_args(const char *string, const char *chars_to_escape) return new_string; } + +/** Callback function that is called when our PT process have data on its + * stdout. Our process can be found in <b>process</b>, the data can be found in + * <b>line</b> and the length of our line is given in <b>size</b>. */ +STATIC void +managed_proxy_stdout_callback(process_t *process, + const char *line, + size_t size) +{ + tor_assert(process); + tor_assert(line); + + (void)size; + + managed_proxy_t *mp = process_get_data(process); + + if (mp == NULL) + return; + + handle_proxy_line(line, mp); + + if (proxy_configuration_finished(mp)) + handle_finished_proxy(mp); +} + +/** Callback function that is called when our PT process have data on its + * stderr. Our process can be found in <b>process</b>, the data can be found in + * <b>line</b> and the length of our line is given in <b>size</b>. */ +STATIC void +managed_proxy_stderr_callback(process_t *process, + const char *line, + size_t size) +{ + tor_assert(process); + tor_assert(line); + + (void)size; + + managed_proxy_t *mp = process_get_data(process); + + if (BUG(mp == NULL)) + return; + + log_warn(LD_PT, "Managed proxy at '%s' reported: %s", mp->argv[0], line); +} + +/** Callback function that is called when our PT process terminates. The + * process exit code can be found in <b>exit_code</b> and our process can be + * found in <b>process</b>. Returns true iff we want the process subsystem to + * free our process_t handle for us. */ +STATIC bool +managed_proxy_exit_callback(process_t *process, process_exit_code_t exit_code) +{ + tor_assert(process); + + log_warn(LD_PT, + "Pluggable Transport process terminated with status code %" PRIu64, + exit_code); + + /* Returning true here means that the process subsystem will take care of + * calling process_free() on our process_t. */ + return true; +} + +/** Returns a valid integer log severity level from <b>severity</b> that + * is compatible with Tor's logging functions. Returns <b>-1</b> on + * error. */ +STATIC int +managed_proxy_severity_parse(const char *severity) +{ + tor_assert(severity); + + /* Slightly different than log.c's parse_log_level :-( */ + if (! strcmp(severity, "debug")) + return LOG_DEBUG; + + if (! strcmp(severity, "info")) + return LOG_INFO; + + if (! strcmp(severity, "notice")) + return LOG_NOTICE; + + if (! strcmp(severity, "warning")) + return LOG_WARN; + + if (! strcmp(severity, "error")) + return LOG_ERR; + + return -1; +} diff --git a/src/feature/client/transports.h b/src/feature/client/transports.h index e2fa45828f..900dd9288e 100644 --- a/src/feature/client/transports.h +++ b/src/feature/client/transports.h @@ -11,6 +11,8 @@ #ifndef TOR_TRANSPORTS_H #define TOR_TRANSPORTS_H +#include "lib/process/process.h" + /** Represents a pluggable transport used by a bridge. */ typedef struct transport_t { /** SOCKS version: One of PROXY_SOCKS4, PROXY_SOCKS5. */ @@ -81,7 +83,7 @@ enum pt_proto_state { PT_PROTO_FAILED_LAUNCH /* failed while launching */ }; -struct process_handle_t; +struct process_t; /** Structure containing information of a managed proxy. */ typedef struct { @@ -94,10 +96,8 @@ typedef struct { int is_server; /* is it a server proxy? */ - /* A pointer to the process handle of this managed proxy. */ - struct process_handle_t *process_handle; - - int pid; /* The Process ID this managed proxy is using. */ + /* A pointer to the process of this managed proxy. */ + struct process_t *process; /** Boolean: We are re-parsing our config, and we are going to * remove this managed proxy if we don't find it any transport @@ -128,6 +128,8 @@ STATIC int parse_version(const char *line, managed_proxy_t *mp); STATIC void parse_env_error(const char *line); STATIC void parse_proxy_error(const char *line); STATIC void handle_proxy_line(const char *line, managed_proxy_t *mp); +STATIC void parse_log_line(const char *line, managed_proxy_t *mp); +STATIC void parse_status_line(const char *line, managed_proxy_t *mp); STATIC char *get_transport_options_for_server_proxy(const managed_proxy_t *mp); STATIC void managed_proxy_destroy(managed_proxy_t *mp, @@ -142,6 +144,12 @@ STATIC char* get_pt_proxy_uri(void); STATIC void free_execve_args(char **arg); +STATIC void managed_proxy_stdout_callback(process_t *, const char *, size_t); +STATIC void managed_proxy_stderr_callback(process_t *, const char *, size_t); +STATIC bool managed_proxy_exit_callback(process_t *, process_exit_code_t); + +STATIC int managed_proxy_severity_parse(const char *); + #endif /* defined(PT_PRIVATE) */ #endif /* !defined(TOR_TRANSPORTS_H) */ diff --git a/src/feature/control/btrack.c b/src/feature/control/btrack.c new file mode 100644 index 0000000000..d3d12cb2b7 --- /dev/null +++ b/src/feature/control/btrack.c @@ -0,0 +1,53 @@ +/* Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file btrack.c + * \brief Bootstrap trackers + * + * Initializes and shuts down the specific bootstrap trackers. These + * trackers help the reporting of bootstrap progress by maintaining + * state information about various subsystems within tor. When the + * correct state changes happen, these trackers emit controller + * events. + * + * These trackers avoid referring directly to the internals of state + * objects of other subsystems. + * + * btrack_circuit.c contains the tracker for origin circuits. + * + * btrack_orconn.c contains the tracker for OR connections. + * + * Eventually there will be a tracker for directory downloads as well. + **/ + +#include "feature/control/btrack_circuit.h" +#include "feature/control/btrack_orconn.h" +#include "feature/control/btrack_sys.h" +#include "lib/subsys/subsys.h" + +static int +btrack_init(void) +{ + if (btrack_orconn_init()) + return -1; + if (btrack_circ_init()) + return -1; + + return 0; +} + +static void +btrack_fini(void) +{ + btrack_orconn_fini(); + btrack_circ_fini(); +} + +const subsys_fns_t sys_btrack = { + .name = "btrack", + .supported = true, + .level = -30, + .initialize = btrack_init, + .shutdown = btrack_fini, +}; diff --git a/src/feature/control/btrack_circuit.c b/src/feature/control/btrack_circuit.c new file mode 100644 index 0000000000..dcee9e460e --- /dev/null +++ b/src/feature/control/btrack_circuit.c @@ -0,0 +1,164 @@ +/* Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file btrack_circuit.c + * \brief Bootstrap tracker for origin circuits + * + * Track state changes of origin circuits, as published by the circuit + * subsystem. + **/ + +#include "core/or/or.h" + +#include "core/or/ocirc_event.h" + +#include "feature/control/btrack_circuit.h" +#include "feature/control/control.h" +#include "lib/log/log.h" + +/** Pair of a best origin circuit GID with its state or status */ +typedef struct btc_best_t { + uint32_t gid; + int val; +} btc_best_t; + +/** GID and state of the best origin circuit we've seen so far */ +static btc_best_t best_any_state = { 0, -1 }; +/** GID and state of the best application circuit we've seen so far */ +static btc_best_t best_ap_state = { 0, -1 }; +/** GID and status of the best origin circuit we've seen so far */ +static btc_best_t best_any_evtype = { 0, -1 }; +/** GID and status of the best application circuit we've seen so far */ +static btc_best_t best_ap_evtype = { 0, -1 }; + +/** Reset cached "best" values */ +static void +btc_reset_bests(void) +{ + best_any_state.gid = best_ap_state.gid = 0; + best_any_state.val = best_ap_state.val = -1; + best_any_evtype.gid = best_ap_state.gid = 0; + best_any_evtype.val = best_ap_evtype.val = -1; +} + +/** True if @a state is a "better" origin circuit state than @a best->val */ +static bool +btc_state_better(int state, const btc_best_t *best) +{ + return state > best->val; +} + +/** + * Definine an ordering on circuit status events + * + * The CIRC_EVENT_ constants aren't sorted in a useful order, so this + * array helps to decode them. This approach depends on the statuses + * being nonnegative and dense. + **/ +static int circ_event_order[] = { + [CIRC_EVENT_FAILED] = -1, + [CIRC_EVENT_CLOSED] = -1, + [CIRC_EVENT_LAUNCHED] = 1, + [CIRC_EVENT_EXTENDED] = 2, + [CIRC_EVENT_BUILT] = 3, +}; +#define N_CIRC_EVENT_ORDER \ + (sizeof(circ_event_order) / sizeof(circ_event_order[0])) + +/** True if @a state is a "better" origin circuit event status than @a + best->val */ +static bool +btc_evtype_better(int state, const btc_best_t *best) +{ + if (state < 0) + return false; + if (best->val < 0) + return true; + + tor_assert(state >= 0 && (unsigned)state < N_CIRC_EVENT_ORDER); + tor_assert(best->val >= 0 && (unsigned)best->val < N_CIRC_EVENT_ORDER); + return circ_event_order[state] > circ_event_order[best->val]; +} + +static bool +btc_update_state(const ocirc_state_msg_t *msg, btc_best_t *best, + const char *type) +{ + if (btc_state_better(msg->state, best)) { + log_info(LD_BTRACK, "CIRC BEST_%s state %d->%d gid=%"PRIu32, type, + best->val, msg->state, msg->gid); + best->gid = msg->gid; + best->val = msg->state; + return true; + } + return false; +} + +static bool +btc_update_evtype(const ocirc_cevent_msg_t *msg, btc_best_t *best, + const char *type) +{ + if (btc_evtype_better(msg->evtype, best)) { + log_info(LD_BTRACK, "CIRC BEST_%s evtype %d->%d gid=%"PRIu32, type, + best->val, msg->evtype, msg->gid); + best->gid = msg->gid; + best->val = msg->evtype; + return true; + } + return false; +} + +static void +btc_state_rcvr(const ocirc_state_msg_t *msg) +{ + log_debug(LD_BTRACK, "CIRC gid=%"PRIu32" state=%d onehop=%d", + msg->gid, msg->state, msg->onehop); + + btc_update_state(msg, &best_any_state, "ANY"); + if (msg->onehop) + return; + btc_update_state(msg, &best_ap_state, "AP"); +} + +static void +btc_cevent_rcvr(const ocirc_cevent_msg_t *msg) +{ + log_debug(LD_BTRACK, "CIRC gid=%"PRIu32" evtype=%d reason=%d onehop=%d", + msg->gid, msg->evtype, msg->reason, msg->onehop); + + btc_update_evtype(msg, &best_any_evtype, "ANY"); + if (msg->onehop) + return; + btc_update_evtype(msg, &best_ap_evtype, "AP"); +} + +static void +btc_event_rcvr(const ocirc_event_msg_t *msg) +{ + switch (msg->type) { + case OCIRC_MSGTYPE_STATE: + return btc_state_rcvr(&msg->u.state); + case OCIRC_MSGTYPE_CHAN: + log_debug(LD_BTRACK, "CIRC gid=%"PRIu32" chan=%"PRIu64" onehop=%d", + msg->u.chan.gid, msg->u.chan.chan, msg->u.chan.onehop); + break; + case OCIRC_MSGTYPE_CEVENT: + return btc_cevent_rcvr(&msg->u.cevent); + default: + break; + } +} + +int +btrack_circ_init(void) +{ + ocirc_event_subscribe(btc_event_rcvr); + return 0; +} + +void +btrack_circ_fini(void) +{ + btc_reset_bests(); +} diff --git a/src/feature/control/btrack_circuit.h b/src/feature/control/btrack_circuit.h new file mode 100644 index 0000000000..c40822f1f1 --- /dev/null +++ b/src/feature/control/btrack_circuit.h @@ -0,0 +1,15 @@ +/* Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file btrack_circuit.h + * \brief Header file for btrack_circuit.c + **/ + +#ifndef TOR_BTRACK_CIRCUIT_H +#define TOR_BTRACK_CIRCUIT_H + +int btrack_circ_init(void); +void btrack_circ_fini(void); + +#endif /* defined(TOR_BTRACK_CIRCUIT_H) */ diff --git a/src/feature/control/btrack_orconn.c b/src/feature/control/btrack_orconn.c new file mode 100644 index 0000000000..93ebe8d9cc --- /dev/null +++ b/src/feature/control/btrack_orconn.c @@ -0,0 +1,206 @@ +/* Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file btrack_orconn.c + * \brief Bootstrap tracker for OR connections + * + * Track state changes of OR connections, as published by the + * connection subsystem. Also track circuit launch events, because + * they're one of the few ways to discover the association between a + * channel (and OR connection) and a circuit. + * + * We track all OR connections that we receive events for, whether or + * not they're carrying origin circuits. (An OR connection might + * carry origin circuits only after we first find out about that + * connection.) + * + * All origin ORCONN events update the "any" state variables, while + * only application ORCONN events update the "ap" state variables (and + * also update the "any") variables. + * + * We do this because we want to report the first increments of + * connection progress as the earliest bootstrap phases. This results + * in a better user experience because failures here translate into + * zero or very small amounts of displayed progress, instead of + * progress stuck near completion. The first connection to a relay + * might be a one-hop circuit for directory lookups, or it might be a + * connection for an application circuit because we already have + * enough directory info to build an application circuit. + * + * We call functions in btrack_orconn_cevent.c to generate the actual + * controller events, because some of the state decoding we need to do + * is complicated. + **/ + +#include <stdbool.h> + +#include "core/or/or.h" + +#define BTRACK_ORCONN_PRIVATE + +#include "core/or/ocirc_event.h" +#include "core/or/orconn_event.h" +#include "feature/control/btrack_orconn.h" +#include "feature/control/btrack_orconn_cevent.h" +#include "feature/control/btrack_orconn_maps.h" +#include "lib/log/log.h" + +/** Pair of a best ORCONN GID and with its state */ +typedef struct bto_best_t { + uint64_t gid; + int state; +} bto_best_t; + +/** GID and state of the best ORCONN we've seen so far */ +static bto_best_t best_any = { 0, -1 }; +/** GID and state of the best application circuit ORCONN we've seen so far */ +static bto_best_t best_ap = { 0, -1 }; + +/** + * Update a cached state of a best ORCONN progress we've seen so far. + * + * Return true if the new state is better than the old. + **/ +static bool +bto_update_best(const bt_orconn_t *bto, bto_best_t *best, const char *type) +{ + if (bto->state < best->state) + return false; + /* Update even if we won't change best->state, because it's more + * recent information that a particular connection transitioned to + * that state. */ + best->gid = bto->gid; + if (bto->state > best->state) { + log_info(LD_BTRACK, "ORCONN BEST_%s state %d->%d gid=%"PRIu64, type, + best->state, bto->state, bto->gid); + best->state = bto->state; + return true; + } + return false; +} + +/** + * Update cached states of best ORCONN progress we've seen + * + * Only update the application ORCONN state if we know it's carrying + * an application circuit. + **/ +static void +bto_update_bests(const bt_orconn_t *bto) +{ + tor_assert(bto->is_orig); + + if (bto_update_best(bto, &best_any, "ANY")) + bto_cevent_anyconn(bto); + if (!bto->is_onehop && bto_update_best(bto, &best_ap, "AP")) + bto_cevent_apconn(bto); +} + +/** Reset cached "best" values */ +static void +bto_reset_bests(void) +{ + best_any.gid = best_ap.gid = 0; + best_any.state = best_ap.state = -1; +} + +/** + * Update cached states of ORCONNs from the incoming message. This + * message comes from code in connection_or.c. + **/ +static void +bto_state_rcvr(const orconn_state_msg_t *msg) +{ + bt_orconn_t *bto; + + bto = bto_find_or_new(msg->gid, msg->chan); + log_debug(LD_BTRACK, "ORCONN gid=%"PRIu64" chan=%"PRIu64 + " proxy_type=%d state=%d", + msg->gid, msg->chan, msg->proxy_type, msg->state); + bto->proxy_type = msg->proxy_type; + bto->state = msg->state; + if (bto->is_orig) + bto_update_bests(bto); +} + +/** + * Delete a cached ORCONN state if we get an incoming message saying + * the ORCONN is failed or closed. This message comes from code in + * control.c. + **/ +static void +bto_status_rcvr(const orconn_status_msg_t *msg) +{ + switch (msg->status) { + case OR_CONN_EVENT_FAILED: + case OR_CONN_EVENT_CLOSED: + log_info(LD_BTRACK, "ORCONN DELETE gid=%"PRIu64" status=%d reason=%d", + msg->gid, msg->status, msg->reason); + return bto_delete(msg->gid); + default: + break; + } +} + +/** Dispatch to individual ORCONN message handlers */ +static void +bto_event_rcvr(const orconn_event_msg_t *msg) +{ + switch (msg->type) { + case ORCONN_MSGTYPE_STATE: + return bto_state_rcvr(&msg->u.state); + case ORCONN_MSGTYPE_STATUS: + return bto_status_rcvr(&msg->u.status); + default: + tor_assert(false); + } +} + +/** + * Create or update a cached ORCONN state for a newly launched + * connection, including whether it's launched by an origin circuit + * and whether it's a one-hop circuit. + **/ +static void +bto_chan_rcvr(const ocirc_event_msg_t *msg) +{ + bt_orconn_t *bto; + + /* Ignore other kinds of origin circuit events; we don't need them */ + if (msg->type != OCIRC_MSGTYPE_CHAN) + return; + + bto = bto_find_or_new(0, msg->u.chan.chan); + if (!bto->is_orig || (bto->is_onehop && !msg->u.chan.onehop)) { + log_debug(LD_BTRACK, "ORCONN LAUNCH chan=%"PRIu64" onehop=%d", + msg->u.chan.chan, msg->u.chan.onehop); + } + bto->is_orig = true; + if (!msg->u.chan.onehop) + bto->is_onehop = false; + bto_update_bests(bto); +} + +/** + * Initialize the hash maps and subscribe to ORCONN and origin + * circuit events. + **/ +int +btrack_orconn_init(void) +{ + bto_init_maps(); + orconn_event_subscribe(bto_event_rcvr); + ocirc_event_subscribe(bto_chan_rcvr); + + return 0; +} + +/** Clear the hash maps and reset the "best" states */ +void +btrack_orconn_fini(void) +{ + bto_clear_maps(); + bto_reset_bests(); + bto_cevent_reset(); +} diff --git a/src/feature/control/btrack_orconn.h b/src/feature/control/btrack_orconn.h new file mode 100644 index 0000000000..6ab4892a78 --- /dev/null +++ b/src/feature/control/btrack_orconn.h @@ -0,0 +1,38 @@ +/* Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file btrack_orconn.h + * \brief Header file for btrack_orconn.c + **/ + +#ifndef TOR_BTRACK_ORCONN_H +#define TOR_BTRACK_ORCONN_H + +#ifdef BTRACK_ORCONN_PRIVATE + +#include "ht.h" + +/** + * Structure for tracking OR connection states + * + * This gets linked into two hash maps: one with connection IDs, and + * another with channel IDs. + **/ +typedef struct bt_orconn_t { + HT_ENTRY(bt_orconn_t) node; /**< Hash map entry indexed by gid */ + HT_ENTRY(bt_orconn_t) chan_node; /**< Hash map entry indexed by channel ID */ + uint64_t gid; /**< Global ID of this ORCONN */ + uint64_t chan; /**< Channel ID, if known */ + int proxy_type; /**< Proxy type */ + uint8_t state; /**< State of this ORCONN */ + bool is_orig; /**< Does this carry an origin circuit? */ + bool is_onehop; /**< Is this for a one-hop circuit? */ +} bt_orconn_t; + +#endif /* defined(BTRACK_ORCONN_PRIVATE) */ + +int btrack_orconn_init(void); +void btrack_orconn_fini(void); + +#endif /* defined(TOR_BTRACK_ORCONN_H) */ diff --git a/src/feature/control/btrack_orconn_cevent.c b/src/feature/control/btrack_orconn_cevent.c new file mode 100644 index 0000000000..ee142f2873 --- /dev/null +++ b/src/feature/control/btrack_orconn_cevent.c @@ -0,0 +1,159 @@ +/* Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file btrack_orconn_cevent.c + * \brief Emit bootstrap status events for OR connections + * + * We do some decoding of the raw OR_CONN_STATE_* values. For + * example, OR_CONN_STATE_CONNECTING means the first TCP connect() + * completing, regardless of whether it's directly to a relay instead + * of a proxy or a PT. + **/ + +#include <stdbool.h> + +#include "core/or/or.h" + +#define BTRACK_ORCONN_PRIVATE + +#include "core/or/orconn_event.h" +#include "feature/control/btrack_orconn.h" +#include "feature/control/btrack_orconn_cevent.h" +#include "feature/control/control.h" + +/** + * Have we completed our first OR connection? + * + * Block display of application circuit progress until we do, to avoid + * some misleading behavior of jumping to high progress. + **/ +static bool bto_first_orconn = false; + +/** Is the ORCONN using a pluggable transport? */ +static bool +using_pt(const bt_orconn_t *bto) +{ + return bto->proxy_type == PROXY_PLUGGABLE; +} + +/** Is the ORCONN using a non-PT proxy? */ +static bool +using_proxy(const bt_orconn_t *bto) +{ + switch (bto->proxy_type) { + case PROXY_CONNECT: + case PROXY_SOCKS4: + case PROXY_SOCKS5: + return true; + default: + return false; + } +} + +/** + * Emit control events when we have updated our idea of the best state + * that any OR connection has reached. + * + * Do some decoding of the ORCONN states depending on whether a PT or + * a proxy is in use. + **/ +void +bto_cevent_anyconn(const bt_orconn_t *bto) +{ + switch (bto->state) { + case OR_CONN_STATE_CONNECTING: + /* Exactly what kind of thing we're connecting to isn't + * information we directly get from the states in connection_or.c, + * so decode it here. */ + if (using_pt(bto)) + control_event_bootstrap(BOOTSTRAP_STATUS_CONN_PT, 0); + else if (using_proxy(bto)) + control_event_bootstrap(BOOTSTRAP_STATUS_CONN_PROXY, 0); + else + control_event_bootstrap(BOOTSTRAP_STATUS_CONN, 0); + break; + case OR_CONN_STATE_PROXY_HANDSHAKING: + /* Similarly, starting a proxy handshake means the TCP connect() + * succeeded to the proxy. Let's be specific about what kind of + * proxy. */ + if (using_pt(bto)) + control_event_bootstrap(BOOTSTRAP_STATUS_CONN_DONE_PT, 0); + else if (using_proxy(bto)) + control_event_bootstrap(BOOTSTRAP_STATUS_CONN_DONE_PROXY, 0); + break; + case OR_CONN_STATE_TLS_HANDSHAKING: + control_event_bootstrap(BOOTSTRAP_STATUS_CONN_DONE, 0); + break; + case OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING: + case OR_CONN_STATE_OR_HANDSHAKING_V2: + case OR_CONN_STATE_OR_HANDSHAKING_V3: + control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE, 0); + break; + case OR_CONN_STATE_OPEN: + control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE_DONE, 0); + /* Unblock directory progress display */ + control_event_boot_first_orconn(); + /* Unblock apconn progress display */ + bto_first_orconn = true; + break; + default: + break; + } +} + +/** + * Emit control events when we have updated our idea of the best state + * that any application circuit OR connection has reached. + * + * Do some decoding of the ORCONN states depending on whether a PT or + * a proxy is in use. + **/ +void +bto_cevent_apconn(const bt_orconn_t *bto) +{ + if (!bto_first_orconn) + return; + + switch (bto->state) { + case OR_CONN_STATE_CONNECTING: + /* Exactly what kind of thing we're connecting to isn't + * information we directly get from the states in connection_or.c, + * so decode it here. */ + if (using_pt(bto)) + control_event_bootstrap(BOOTSTRAP_STATUS_AP_CONN_PT, 0); + else if (using_proxy(bto)) + control_event_bootstrap(BOOTSTRAP_STATUS_AP_CONN_PROXY, 0); + else + control_event_bootstrap(BOOTSTRAP_STATUS_AP_CONN, 0); + break; + case OR_CONN_STATE_PROXY_HANDSHAKING: + /* Similarly, starting a proxy handshake means the TCP connect() + * succeeded to the proxy. Let's be specific about what kind of + * proxy. */ + if (using_pt(bto)) + control_event_bootstrap(BOOTSTRAP_STATUS_AP_CONN_DONE_PT, 0); + else if (using_proxy(bto)) + control_event_bootstrap(BOOTSTRAP_STATUS_AP_CONN_DONE_PROXY, 0); + break; + case OR_CONN_STATE_TLS_HANDSHAKING: + control_event_bootstrap(BOOTSTRAP_STATUS_AP_CONN_DONE, 0); + break; + case OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING: + case OR_CONN_STATE_OR_HANDSHAKING_V2: + case OR_CONN_STATE_OR_HANDSHAKING_V3: + control_event_bootstrap(BOOTSTRAP_STATUS_AP_HANDSHAKE, 0); + break; + case OR_CONN_STATE_OPEN: + control_event_bootstrap(BOOTSTRAP_STATUS_AP_HANDSHAKE_DONE, 0); + default: + break; + } +} + +/** Forget that we completed our first OR connection */ +void +bto_cevent_reset(void) +{ + bto_first_orconn = false; +} diff --git a/src/feature/control/btrack_orconn_cevent.h b/src/feature/control/btrack_orconn_cevent.h new file mode 100644 index 0000000000..f9d24633aa --- /dev/null +++ b/src/feature/control/btrack_orconn_cevent.h @@ -0,0 +1,17 @@ +/* Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file btrack_orconn_cevent.h + * \brief Header file for btrack_orconn_cevent.c + **/ + +#ifndef TOR_BTRACK_ORCONN_CEVENT_H + +#include "feature/control/btrack_orconn.h" + +void bto_cevent_anyconn(const bt_orconn_t *); +void bto_cevent_apconn(const bt_orconn_t *); +void bto_cevent_reset(void); + +#endif /* defined(TOR_BTRACK_ORCONN_CEVENT_H) */ diff --git a/src/feature/control/btrack_orconn_maps.c b/src/feature/control/btrack_orconn_maps.c new file mode 100644 index 0000000000..e64bd3f0fe --- /dev/null +++ b/src/feature/control/btrack_orconn_maps.c @@ -0,0 +1,223 @@ +/* Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file btrack_orconn_maps.c + * \brief Hash map implementation for btrack_orconn.c + * + * These functions manipulate the hash maps that contain bt_orconn + * objects. + **/ + +#include <stdbool.h> + +#include "core/or/or.h" + +#include "ht.h" +#include "siphash.h" + +#define BTRACK_ORCONN_PRIVATE + +#include "feature/control/btrack_orconn.h" +#include "feature/control/btrack_orconn_maps.h" +#include "lib/log/log.h" + +static inline unsigned int +bto_gid_hash_(bt_orconn_t *elm) +{ + return (unsigned)siphash24g(&elm->gid, sizeof(elm->gid)); +} + +static inline int +bto_gid_eq_(bt_orconn_t *a, bt_orconn_t *b) +{ + return a->gid == b->gid; +} + +static inline unsigned int +bto_chan_hash_(bt_orconn_t *elm) +{ + return (unsigned)siphash24g(&elm->chan, sizeof(elm->chan)); +} + +static inline int +bto_chan_eq_(bt_orconn_t *a, bt_orconn_t *b) +{ + return a->chan == b->chan; +} + +HT_HEAD(bto_gid_ht, bt_orconn_t); +HT_PROTOTYPE(bto_gid_ht, bt_orconn_t, node, bto_gid_hash_, bto_gid_eq_) +HT_GENERATE2(bto_gid_ht, bt_orconn_t, node, + bto_gid_hash_, bto_gid_eq_, 0.6, + tor_reallocarray_, tor_free_) +static struct bto_gid_ht *bto_gid_map; + +HT_HEAD(bto_chan_ht, bt_orconn_t); +HT_PROTOTYPE(bto_chan_ht, bt_orconn_t, chan_node, bto_chan_hash_, bto_chan_eq_) +HT_GENERATE2(bto_chan_ht, bt_orconn_t, chan_node, + bto_chan_hash_, bto_chan_eq_, 0.6, + tor_reallocarray_, tor_free_) +static struct bto_chan_ht *bto_chan_map; + +/** Clear the GID hash map, freeing any bt_orconn_t objects that become + * unreferenced */ +static void +bto_gid_clear_map(void) +{ + bt_orconn_t **elt, **next, *c; + + for (elt = HT_START(bto_gid_ht, bto_gid_map); + elt; + elt = next) { + c = *elt; + next = HT_NEXT_RMV(bto_gid_ht, bto_gid_map, elt); + + c->gid = 0; + /* Don't delete if chan ID isn't zero: it's still in the chan hash map */ + if (!c->chan) + tor_free(c); + } + HT_CLEAR(bto_gid_ht, bto_gid_map); + tor_free(bto_gid_map); +} + +/** Clear the chan ID hash map, freeing any bt_orconn_t objects that + * become unreferenced */ +static void +bto_chan_clear_map(void) +{ + bt_orconn_t **elt, **next, *c; + + for (elt = HT_START(bto_chan_ht, bto_chan_map); + elt; + elt = next) { + c = *elt; + next = HT_NEXT_RMV(bto_chan_ht, bto_chan_map, elt); + + c->chan = 0; + /* Don't delete if GID isn't zero, it's still in the GID hash map */ + if (!c->gid) + tor_free(c); + } + HT_CLEAR(bto_chan_ht, bto_chan_map); + tor_free(bto_chan_map); +} + +/** Delete a bt_orconn from the hash maps by GID */ +void +bto_delete(uint64_t gid) +{ + bt_orconn_t key, *bto; + + key.gid = gid; + key.chan = 0; + bto = HT_FIND(bto_gid_ht, bto_gid_map, &key); + if (!bto) { + /* The orconn might be unregistered because it's an EXT_OR_CONN? */ + log_debug(LD_BTRACK, "tried to delete unregistered ORCONN gid=%"PRIu64, + gid); + return; + } + HT_REMOVE(bto_gid_ht, bto_gid_map, &key); + if (bto->chan) { + key.chan = bto->chan; + HT_REMOVE(bto_chan_ht, bto_chan_map, &key); + } + tor_free(bto); +} + +/** + * Helper for bto_find_or_new(). + * + * Update GID and chan ID of an existing bt_orconn object if needed, + * given a search key previously used within bto_find_or_new(). + **/ +static bt_orconn_t * +bto_update(bt_orconn_t *bto, const bt_orconn_t *key) +{ + /* ORCONN GIDs shouldn't change once assigned */ + tor_assert(!bto->gid || !key->gid || bto->gid == key->gid); + if (!bto->gid && key->gid) { + /* Got a gid when we didn't already have one; insert into gid map */ + log_debug(LD_BTRACK, "ORCONN chan=%"PRIu64" newgid=%"PRIu64, key->chan, + key->gid); + bto->gid = key->gid; + HT_INSERT(bto_gid_ht, bto_gid_map, bto); + } + /* association of ORCONN with channel shouldn't change */ + tor_assert(!bto->chan || !key->chan || bto->chan == key->chan); + if (!bto->chan && key->chan) { + /* Got a chan when we didn't already have one; insert into chan map */ + log_debug(LD_BTRACK, "ORCONN gid=%"PRIu64" newchan=%"PRIu64, + bto->gid, key->chan); + bto->chan = key->chan; + HT_INSERT(bto_chan_ht, bto_chan_map, bto); + } + return bto; +} + +/** Helper for bto_find_or_new() */ +static bt_orconn_t * +bto_new(const bt_orconn_t *key) +{ + struct bt_orconn_t *bto = tor_malloc(sizeof(*bto)); + + bto->gid = key->gid; + bto->chan = key->chan; + bto->state = 0; + bto->proxy_type = 0; + bto->is_orig = false; + bto->is_onehop = true; + + if (bto->gid) + HT_INSERT(bto_gid_ht, bto_gid_map, bto); + if (bto->chan) + HT_INSERT(bto_chan_ht, bto_chan_map, bto); + + return bto; +} + +/** + * Insert a new bt_orconn with the given GID and chan ID, or update + * the GID and chan ID if one already exists. + * + * Return the found or allocated bt_orconn. + **/ +bt_orconn_t * +bto_find_or_new(uint64_t gid, uint64_t chan) +{ + bt_orconn_t key, *bto = NULL; + + tor_assert(gid || chan); + key.gid = gid; + key.chan = chan; + if (key.gid) + bto = HT_FIND(bto_gid_ht, bto_gid_map, &key); + if (!bto && key.chan) { + /* Not found by GID; look up by chan ID */ + bto = HT_FIND(bto_chan_ht, bto_chan_map, &key); + } + if (bto) + return bto_update(bto, &key); + else + return bto_new(&key); +} + +/** Initialize the hash maps */ +void +bto_init_maps(void) +{ + bto_gid_map = tor_malloc(sizeof(*bto_gid_map)); + HT_INIT(bto_gid_ht, bto_gid_map); + bto_chan_map = tor_malloc(sizeof(*bto_chan_map)); + HT_INIT(bto_chan_ht, bto_chan_map); +} + +/** Clear the hash maps, freeing all associated storage */ +void +bto_clear_maps(void) +{ + bto_gid_clear_map(); + bto_chan_clear_map(); +} diff --git a/src/feature/control/btrack_orconn_maps.h b/src/feature/control/btrack_orconn_maps.h new file mode 100644 index 0000000000..3ead40984c --- /dev/null +++ b/src/feature/control/btrack_orconn_maps.h @@ -0,0 +1,17 @@ +/* Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file btrack_orconn_maps.h + * \brief Header file for btrack_orconn_maps.c + **/ + +#ifndef TOR_BTRACK_ORCONN_MAPS_H + +void bto_delete(uint64_t); +bt_orconn_t *bto_find_or_new(uint64_t, uint64_t); + +void bto_init_maps(void); +void bto_clear_maps(void); + +#endif /* defined(TOR_BTRACK_ORCONN_MAPS_H) */ diff --git a/src/feature/control/btrack_sys.h b/src/feature/control/btrack_sys.h new file mode 100644 index 0000000000..fad35b41db --- /dev/null +++ b/src/feature/control/btrack_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file btrack_sys.h + * \brief Declare subsystem object for the bootstrap tracker susbystem. + **/ + +#ifndef TOR_BTRACK_SYS_H +#define TOR_BTRACK_SYS_H + +extern const struct subsys_fns_t sys_btrack; + +#endif /* defined(TOR_BTRACK_SYS_H) */ diff --git a/src/feature/control/control.c b/src/feature/control/control.c index cc7ecff2ff..6f8cd8f0aa 100644 --- a/src/feature/control/control.c +++ b/src/feature/control/control.c @@ -34,6 +34,7 @@ **/ #define CONTROL_PRIVATE +#define OCIRC_EVENT_PRIVATE #include "core/or/or.h" #include "app/config/config.h" @@ -50,6 +51,7 @@ #include "core/or/command.h" #include "core/or/connection_edge.h" #include "core/or/connection_or.h" +#include "core/or/ocirc_event.h" #include "core/or/policies.h" #include "core/or/reasons.h" #include "core/or/versions.h" @@ -87,11 +89,12 @@ #include "feature/rend/rendservice.h" #include "feature/stats/geoip_stats.h" #include "feature/stats/predict_ports.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/crypt_ops/crypto_rand.h" #include "lib/crypt_ops/crypto_util.h" #include "lib/encoding/confline.h" #include "lib/evloop/compat_libevent.h" +#include "lib/version/torversion.h" #include "feature/dircache/cached_dir_st.h" #include "feature/control/control_connection_st.h" @@ -178,13 +181,6 @@ static uint8_t *authentication_cookie = NULL; */ static smartlist_t *detached_onion_services = NULL; -/** A sufficiently large size to record the last bootstrap phase string. */ -#define BOOTSTRAP_MSG_LEN 1024 - -/** What was the last bootstrap phase message we sent? We keep track - * of this so we can respond to getinfo status/bootstrap-phase queries. */ -static char last_sent_bootstrap_message[BOOTSTRAP_MSG_LEN]; - static void connection_printf_to_buf(control_connection_t *conn, const char *format, ...) CHECK_PRINTF(2,3); @@ -367,7 +363,7 @@ control_update_global_event_mask(void) control_get_bytes_rw_last_sec(&r, &w); } if (any_old_per_sec_events != control_any_per_second_event_enabled()) { - reschedule_per_second_timer(); + rescan_periodic_events(get_options()); } #undef NEWLY_ENABLED @@ -1680,6 +1676,8 @@ static const struct signal_t signal_table[] = { { SIGNEWNYM, "NEWNYM" }, { SIGCLEARDNSCACHE, "CLEARDNSCACHE"}, { SIGHEARTBEAT, "HEARTBEAT"}, + { SIGACTIVE, "ACTIVE" }, + { SIGDORMANT, "DORMANT" }, { 0, NULL }, }; @@ -1745,6 +1743,26 @@ handle_control_takeownership(control_connection_t *conn, uint32_t len, return 0; } +/** Called when we get a DROPOWNERSHIP command. Mark this connection + * as a non-owning connection, so that we will not exit if the connection + * closes. */ +static int +handle_control_dropownership(control_connection_t *conn, uint32_t len, + const char *body) +{ + (void)len; + (void)body; + + conn->is_owning_control_connection = 0; + + log_info(LD_CONTROL, "Control connection %d has dropped ownership of this " + "Tor instance.", + (int)(conn->base_.s)); + + send_control_done(conn); + return 0; +} + /** Return true iff <b>addr</b> is unusable as a mapaddress target because of * containing funny characters. */ static int @@ -2352,7 +2370,11 @@ getinfo_helper_dir(control_connection_t *control_conn, *answer = tor_strdup(consensus->dir); } if (!*answer) { /* try loading it from disk */ - *answer = networkstatus_read_cached_consensus("ns"); + tor_mmap_t *mapped = networkstatus_map_cached_consensus("ns"); + if (mapped) { + *answer = tor_memdup_nulterm(mapped->data, mapped->size); + tor_munmap_file(mapped); + } if (!*answer) { /* generate an error */ *errmsg = "Could not open cached consensus. " "Make sure FetchUselessDescriptors is set to 1."; @@ -3037,7 +3059,7 @@ getinfo_helper_events(control_connection_t *control_conn, check_whether_orport_reachable(options) ? 1 : 0, check_whether_dirport_reachable(options) ? 1 : 0); } else if (!strcmp(question, "status/bootstrap-phase")) { - *answer = tor_strdup(last_sent_bootstrap_message); + *answer = control_event_boot_last_msg(); } else if (!strcmpstart(question, "status/version/")) { int is_server = server_mode(options); networkstatus_t *c = networkstatus_get_latest_consensus(); @@ -3067,11 +3089,6 @@ getinfo_helper_events(control_connection_t *control_conn, case VS_UNKNOWN: *answer = tor_strdup("unknown"); break; default: tor_fragile_assert(); } - } else if (!strcmp(question, "status/version/num-versioning") || - !strcmp(question, "status/version/num-concurring")) { - tor_asprintf(answer, "%d", get_n_authorities(V3_DIRINFO)); - log_warn(LD_GENERAL, "%s is deprecated; it no longer gives useful " - "information", question); } } else if (!strcmp(question, "status/clients-seen")) { char *bridge_stats = geoip_get_bridge_stats_controller(time(NULL)); @@ -3364,10 +3381,6 @@ static const getinfo_item_t getinfo_items[] = { "A fresh relay/ei descriptor pair for Tor's current state. Not stored."), DOC("status/version/recommended", "List of currently recommended versions."), DOC("status/version/current", "Status of the current version."), - DOC("status/version/num-versioning", "Number of versioning authorities."), - DOC("status/version/num-concurring", - "Number of versioning authorities agreeing on the status of the " - "current version"), ITEM("address", misc, "IP address of this Tor host, if we can guess it."), ITEM("traffic/read", misc,"Bytes read since the process was started."), ITEM("traffic/written", misc, @@ -3749,7 +3762,7 @@ handle_control_extendcircuit(control_connection_t *conn, uint32_t len, connection_printf_to_buf(conn, "250 EXTENDED %lu\r\n", (unsigned long)circ->global_identifier); if (zero_circ) /* send a 'launched' event, for completeness */ - control_event_circuit_status(circ, CIRC_EVENT_LAUNCHED, 0); + circuit_event_status(circ, CIRC_EVENT_LAUNCHED, 0); done: SMARTLIST_FOREACH(router_nicknames, char *, n, tor_free(n)); smartlist_free(router_nicknames); @@ -5548,6 +5561,9 @@ connection_control_process_inbuf(control_connection_t *conn) } else if (!strcasecmp(conn->incoming_cmd, "TAKEOWNERSHIP")) { if (handle_control_takeownership(conn, cmd_data_len, args)) return -1; + } else if (!strcasecmp(conn->incoming_cmd, "DROPOWNERSHIP")) { + if (handle_control_dropownership(conn, cmd_data_len, args)) + return -1; } else if (!strcasecmp(conn->incoming_cmd, "MAPADDRESS")) { if (handle_control_mapaddress(conn, cmd_data_len, args)) return -1; @@ -5625,6 +5641,7 @@ control_event_circuit_status(origin_circuit_t *circ, circuit_status_event_t tp, { const char *status; char reasons[64] = ""; + if (!EVENT_IS_INTERESTING(EVENT_CIRCUIT_STATUS)) return 0; tor_assert(circ); @@ -7008,361 +7025,6 @@ monitor_owning_controller_process(const char *process_spec) } } -/** Convert the name of a bootstrapping phase <b>s</b> into strings - * <b>tag</b> and <b>summary</b> suitable for display by the controller. */ -static int -bootstrap_status_to_string(bootstrap_status_t s, const char **tag, - const char **summary) -{ - switch (s) { - case BOOTSTRAP_STATUS_UNDEF: - *tag = "undef"; - *summary = "Undefined"; - break; - case BOOTSTRAP_STATUS_STARTING: - *tag = "starting"; - *summary = "Starting"; - break; - case BOOTSTRAP_STATUS_CONN_DIR: - *tag = "conn_dir"; - *summary = "Connecting to directory server"; - break; - case BOOTSTRAP_STATUS_HANDSHAKE: - *tag = "status_handshake"; - *summary = "Finishing handshake"; - break; - case BOOTSTRAP_STATUS_HANDSHAKE_DIR: - *tag = "handshake_dir"; - *summary = "Finishing handshake with directory server"; - break; - case BOOTSTRAP_STATUS_ONEHOP_CREATE: - *tag = "onehop_create"; - *summary = "Establishing an encrypted directory connection"; - break; - case BOOTSTRAP_STATUS_REQUESTING_STATUS: - *tag = "requesting_status"; - *summary = "Asking for networkstatus consensus"; - break; - case BOOTSTRAP_STATUS_LOADING_STATUS: - *tag = "loading_status"; - *summary = "Loading networkstatus consensus"; - break; - case BOOTSTRAP_STATUS_LOADING_KEYS: - *tag = "loading_keys"; - *summary = "Loading authority key certs"; - break; - case BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS: - *tag = "requesting_descriptors"; - /* XXXX this appears to incorrectly report internal on most loads */ - *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ? - "Asking for relay descriptors for internal paths" : - "Asking for relay descriptors"; - break; - /* If we're sure there are no exits in the consensus, - * inform the controller by adding "internal" - * to the status summaries. - * (We only check this while loading descriptors, - * so we may not know in the earlier stages.) - * But if there are exits, we can't be sure whether - * we're creating internal or exit paths/circuits. - * XXXX Or should be use different tags or statuses - * for internal and exit/all? */ - case BOOTSTRAP_STATUS_LOADING_DESCRIPTORS: - *tag = "loading_descriptors"; - *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ? - "Loading relay descriptors for internal paths" : - "Loading relay descriptors"; - break; - case BOOTSTRAP_STATUS_CONN_OR: - *tag = "conn_or"; - *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ? - "Connecting to the Tor network internally" : - "Connecting to the Tor network"; - break; - case BOOTSTRAP_STATUS_HANDSHAKE_OR: - *tag = "handshake_or"; - *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ? - "Finishing handshake with first hop of internal circuit" : - "Finishing handshake with first hop"; - break; - case BOOTSTRAP_STATUS_CIRCUIT_CREATE: - *tag = "circuit_create"; - *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ? - "Establishing an internal Tor circuit" : - "Establishing a Tor circuit"; - break; - case BOOTSTRAP_STATUS_DONE: - *tag = "done"; - *summary = "Done"; - break; - default: -// log_warn(LD_BUG, "Unrecognized bootstrap status code %d", s); - *tag = *summary = "unknown"; - return -1; - } - return 0; -} - -/** What percentage through the bootstrap process are we? We remember - * this so we can avoid sending redundant bootstrap status events, and - * so we can guess context for the bootstrap messages which are - * ambiguous. It starts at 'undef', but gets set to 'starting' while - * Tor initializes. */ -static int bootstrap_percent = BOOTSTRAP_STATUS_UNDEF; - -/** Like bootstrap_percent, but only takes on the enumerated values in - * bootstrap_status_t. - */ -static int bootstrap_phase = BOOTSTRAP_STATUS_UNDEF; - -/** As bootstrap_percent, but holds the bootstrapping level at which we last - * logged a NOTICE-level message. We use this, plus BOOTSTRAP_PCT_INCREMENT, - * to avoid flooding the log with a new message every time we get a few more - * microdescriptors */ -static int notice_bootstrap_percent = 0; - -/** How many problems have we had getting to the next bootstrapping phase? - * These include failure to establish a connection to a Tor relay, - * failures to finish the TLS handshake, failures to validate the - * consensus document, etc. */ -static int bootstrap_problems = 0; - -/** We only tell the controller once we've hit a threshold of problems - * for the current phase. */ -#define BOOTSTRAP_PROBLEM_THRESHOLD 10 - -/** When our bootstrapping progress level changes, but our bootstrapping - * status has not advanced, we only log at NOTICE when we have made at least - * this much progress. - */ -#define BOOTSTRAP_PCT_INCREMENT 5 - -/** Do the actual logging and notifications for - * control_event_bootstrap(). Doesn't change any state beyond that. - */ -static void -control_event_bootstrap_core(int loglevel, bootstrap_status_t status, - int progress) -{ - char buf[BOOTSTRAP_MSG_LEN]; - const char *tag, *summary; - - bootstrap_status_to_string(status, &tag, &summary); - /* Locally reset status if there's incremental progress */ - if (progress) - status = progress; - - tor_log(loglevel, LD_CONTROL, - "Bootstrapped %d%%: %s", status, summary); - tor_snprintf(buf, sizeof(buf), - "BOOTSTRAP PROGRESS=%d TAG=%s SUMMARY=\"%s\"", - status, tag, summary); - tor_snprintf(last_sent_bootstrap_message, - sizeof(last_sent_bootstrap_message), - "NOTICE %s", buf); - control_event_client_status(LOG_NOTICE, "%s", buf); -} - -/** Called when Tor has made progress at bootstrapping its directory - * information and initial circuits. - * - * <b>status</b> is the new status, that is, what task we will be doing - * next. <b>progress</b> is zero if we just started this task, else it - * represents progress on the task. - */ -void -control_event_bootstrap(bootstrap_status_t status, int progress) -{ - int loglevel = LOG_NOTICE; - - if (bootstrap_percent == BOOTSTRAP_STATUS_DONE) - return; /* already bootstrapped; nothing to be done here. */ - - /* special case for handshaking status, since our TLS handshaking code - * can't distinguish what the connection is going to be for. */ - if (status == BOOTSTRAP_STATUS_HANDSHAKE) { - if (bootstrap_percent < BOOTSTRAP_STATUS_CONN_OR) { - status = BOOTSTRAP_STATUS_HANDSHAKE_DIR; - } else { - status = BOOTSTRAP_STATUS_HANDSHAKE_OR; - } - } - - if (status <= bootstrap_percent) { - /* If there's no new progress, return early. */ - if (!progress || progress <= bootstrap_percent) - return; - /* Log at INFO if not enough progress happened. */ - if (progress < notice_bootstrap_percent + BOOTSTRAP_PCT_INCREMENT) - loglevel = LOG_INFO; - } - - control_event_bootstrap_core(loglevel, status, progress); - - if (status > bootstrap_percent) { - bootstrap_phase = status; /* new milestone reached */ - bootstrap_percent = status; - } - if (progress > bootstrap_percent) { - /* incremental progress within a milestone */ - bootstrap_percent = progress; - bootstrap_problems = 0; /* Progress! Reset our problem counter. */ - } - if (loglevel == LOG_NOTICE && - bootstrap_percent > notice_bootstrap_percent) { - /* Remember that we gave a notice at this level. */ - notice_bootstrap_percent = bootstrap_percent; - } -} - -/** Flag whether we've opened an OR_CONN yet */ -static int bootstrap_first_orconn = 0; - -/** Like bootstrap_phase, but for (possibly deferred) directory progress */ -static int bootstrap_dir_phase = BOOTSTRAP_STATUS_UNDEF; - -/** Like bootstrap_problems, but for (possibly deferred) directory progress */ -static int bootstrap_dir_progress = BOOTSTRAP_STATUS_UNDEF; - -/** Defer directory info bootstrap events until we have successfully - * completed our first connection to a router. */ -void -control_event_boot_dir(bootstrap_status_t status, int progress) -{ - if (status > bootstrap_dir_progress) { - bootstrap_dir_progress = status; - bootstrap_dir_phase = status; - } - if (progress && progress >= bootstrap_dir_progress) { - bootstrap_dir_progress = progress; - } - - /* Don't report unless we have successfully opened at least one OR_CONN */ - if (!bootstrap_first_orconn) - return; - - control_event_bootstrap(status, progress); -} - -/** Set a flag to allow reporting of directory bootstrap progress. - * (Code that reports completion of an OR_CONN calls this.) Also, - * report directory progress so far. */ -void -control_event_boot_first_orconn(void) -{ - bootstrap_first_orconn = 1; - control_event_bootstrap(bootstrap_dir_phase, bootstrap_dir_progress); -} - -/** Called when Tor has failed to make bootstrapping progress in a way - * that indicates a problem. <b>warn</b> gives a human-readable hint - * as to why, and <b>reason</b> provides a controller-facing short - * tag. <b>conn</b> is the connection that caused this problem and - * can be NULL if a connection cannot be easily identified. - */ -void -control_event_bootstrap_problem(const char *warn, const char *reason, - const connection_t *conn, int dowarn) -{ - int status = bootstrap_percent; - const char *tag = "", *summary = ""; - char buf[BOOTSTRAP_MSG_LEN]; - const char *recommendation = "ignore"; - int severity; - char *or_id = NULL, *hostaddr = NULL; - or_connection_t *or_conn = NULL; - - /* bootstrap_percent must not be in "undefined" state here. */ - tor_assert(status >= 0); - - if (bootstrap_percent == 100) - return; /* already bootstrapped; nothing to be done here. */ - - bootstrap_problems++; - - if (bootstrap_problems >= BOOTSTRAP_PROBLEM_THRESHOLD) - dowarn = 1; - - /* Don't warn about our bootstrapping status if we are hibernating or - * shutting down. */ - if (we_are_hibernating()) - dowarn = 0; - - tor_assert(bootstrap_status_to_string(bootstrap_phase, &tag, &summary) == 0); - - severity = dowarn ? LOG_WARN : LOG_INFO; - - if (dowarn) - recommendation = "warn"; - - if (conn && conn->type == CONN_TYPE_OR) { - /* XXX TO_OR_CONN can't deal with const */ - or_conn = TO_OR_CONN((connection_t *)conn); - or_id = tor_strdup(hex_str(or_conn->identity_digest, DIGEST_LEN)); - } else { - or_id = tor_strdup("?"); - } - - if (conn) - tor_asprintf(&hostaddr, "%s:%d", conn->address, (int)conn->port); - else - hostaddr = tor_strdup("?"); - - log_fn(severity, - LD_CONTROL, "Problem bootstrapping. Stuck at %d%%: %s. (%s; %s; " - "count %d; recommendation %s; host %s at %s)", - status, summary, warn, reason, - bootstrap_problems, recommendation, - or_id, hostaddr); - - connection_or_report_broken_states(severity, LD_HANDSHAKE); - - tor_snprintf(buf, sizeof(buf), - "BOOTSTRAP PROGRESS=%d TAG=%s SUMMARY=\"%s\" WARNING=\"%s\" REASON=%s " - "COUNT=%d RECOMMENDATION=%s HOSTID=\"%s\" HOSTADDR=\"%s\"", - bootstrap_percent, tag, summary, warn, reason, bootstrap_problems, - recommendation, - or_id, hostaddr); - - tor_snprintf(last_sent_bootstrap_message, - sizeof(last_sent_bootstrap_message), - "WARN %s", buf); - control_event_client_status(LOG_WARN, "%s", buf); - - tor_free(hostaddr); - tor_free(or_id); -} - -/** Called when Tor has failed to make bootstrapping progress in a way - * that indicates a problem. <b>warn</b> gives a hint as to why, and - * <b>reason</b> provides an "or_conn_end_reason" tag. <b>or_conn</b> - * is the connection that caused this problem. - */ -MOCK_IMPL(void, -control_event_bootstrap_prob_or, (const char *warn, int reason, - or_connection_t *or_conn)) -{ - int dowarn = 0; - - if (or_conn->have_noted_bootstrap_problem) - return; - - or_conn->have_noted_bootstrap_problem = 1; - - if (reason == END_OR_CONN_REASON_NO_ROUTE) - dowarn = 1; - - /* If we are using bridges and all our OR connections are now - closed, it means that we totally failed to connect to our - bridges. Throw a warning. */ - if (get_options()->UseBridges && !any_other_active_or_conns(or_conn)) - dowarn = 1; - - control_event_bootstrap_problem(warn, - orconn_end_reason_to_control_string(reason), - TO_CONN(or_conn), dowarn); -} - /** We just generated a new summary of which countries we've seen clients * from recently. Send a copy to the controller in case it wants to * display it for the user. */ @@ -7388,6 +7050,26 @@ control_event_transport_launched(const char *mode, const char *transport_name, mode, transport_name, fmt_addr(addr), port); } +/** A pluggable transport called <b>pt_name</b> has emitted a log message + * found in <b>message</b> at <b>severity</b> log level. */ +void +control_event_pt_log(const char *log) +{ + send_control_event(EVENT_PT_LOG, + "650 PT_LOG %s\r\n", + log); +} + +/** A pluggable transport has emitted a STATUS message found in + * <b>status</b>. */ +void +control_event_pt_status(const char *status) +{ + send_control_event(EVENT_PT_STATUS, + "650 PT_STATUS %s\r\n", + status); +} + /** Convert rendezvous auth type to string for HS_DESC control events */ const char * @@ -7879,17 +7561,10 @@ control_free_all(void) mainloop_event_free(flush_queued_events_event); flush_queued_events_event = NULL; } - bootstrap_percent = BOOTSTRAP_STATUS_UNDEF; - bootstrap_phase = BOOTSTRAP_STATUS_UNDEF; - notice_bootstrap_percent = 0; - bootstrap_problems = 0; - bootstrap_first_orconn = 0; - bootstrap_dir_progress = BOOTSTRAP_STATUS_UNDEF; - bootstrap_dir_phase = BOOTSTRAP_STATUS_UNDEF; + control_event_bootstrap_reset(); authentication_cookie_is_set = 0; global_event_mask = 0; disable_log_messages = 0; - memset(last_sent_bootstrap_message, 0, sizeof(last_sent_bootstrap_message)); } #ifdef TOR_UNIT_TESTS diff --git a/src/feature/control/control.h b/src/feature/control/control.h index a09c1cd11b..b2ab4c1997 100644 --- a/src/feature/control/control.h +++ b/src/feature/control/control.h @@ -12,15 +12,7 @@ #ifndef TOR_CONTROL_H #define TOR_CONTROL_H -/** Used to indicate the type of a circuit event passed to the controller. - * The various types are defined in control-spec.txt */ -typedef enum circuit_status_event_t { - CIRC_EVENT_LAUNCHED = 0, - CIRC_EVENT_BUILT = 1, - CIRC_EVENT_EXTENDED = 2, - CIRC_EVENT_FAILED = 3, - CIRC_EVENT_CLOSED = 4, -} circuit_status_event_t; +#include "core/or/ocirc_event.h" /** Used to indicate the type of a CIRC_MINOR event passed to the controller. * The various types are defined in control-spec.txt . */ @@ -29,6 +21,8 @@ typedef enum circuit_status_minor_event_t { CIRC_MINOR_EVENT_CANNIBALIZED, } circuit_status_minor_event_t; +#include "core/or/orconn_event.h" + /** Used to indicate the type of a stream event passed to the controller. * The various types are defined in control-spec.txt */ typedef enum stream_status_event_t { @@ -43,16 +37,6 @@ typedef enum stream_status_event_t { STREAM_EVENT_REMAP = 8 } stream_status_event_t; -/** Used to indicate the type of an OR connection event passed to the - * controller. The various types are defined in control-spec.txt */ -typedef enum or_conn_status_event_t { - OR_CONN_EVENT_LAUNCHED = 0, - OR_CONN_EVENT_CONNECTED = 1, - OR_CONN_EVENT_FAILED = 2, - OR_CONN_EVENT_CLOSED = 3, - OR_CONN_EVENT_NEW = 4, -} or_conn_status_event_t; - /** Used to indicate the type of a buildtime event */ typedef enum buildtimeout_set_event_t { BUILDTIMEOUT_SET_EVENT_COMPUTED = 0, @@ -67,18 +51,42 @@ typedef enum buildtimeout_set_event_t { typedef enum { BOOTSTRAP_STATUS_UNDEF=-1, BOOTSTRAP_STATUS_STARTING=0, - BOOTSTRAP_STATUS_CONN_DIR=5, - BOOTSTRAP_STATUS_HANDSHAKE=-2, - BOOTSTRAP_STATUS_HANDSHAKE_DIR=10, - BOOTSTRAP_STATUS_ONEHOP_CREATE=15, - BOOTSTRAP_STATUS_REQUESTING_STATUS=20, - BOOTSTRAP_STATUS_LOADING_STATUS=25, + + /* Initial connection to any relay */ + + BOOTSTRAP_STATUS_CONN_PT=1, + BOOTSTRAP_STATUS_CONN_DONE_PT=2, + BOOTSTRAP_STATUS_CONN_PROXY=3, + BOOTSTRAP_STATUS_CONN_DONE_PROXY=4, + BOOTSTRAP_STATUS_CONN=5, + BOOTSTRAP_STATUS_CONN_DONE=10, + BOOTSTRAP_STATUS_HANDSHAKE=14, + BOOTSTRAP_STATUS_HANDSHAKE_DONE=15, + + /* Loading directory info */ + + BOOTSTRAP_STATUS_ONEHOP_CREATE=20, + BOOTSTRAP_STATUS_REQUESTING_STATUS=25, + BOOTSTRAP_STATUS_LOADING_STATUS=30, BOOTSTRAP_STATUS_LOADING_KEYS=40, BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS=45, BOOTSTRAP_STATUS_LOADING_DESCRIPTORS=50, - BOOTSTRAP_STATUS_CONN_OR=80, - BOOTSTRAP_STATUS_HANDSHAKE_OR=85, - BOOTSTRAP_STATUS_CIRCUIT_CREATE=90, + BOOTSTRAP_STATUS_ENOUGH_DIRINFO=75, + + /* Connecting to a relay for AP circuits */ + + BOOTSTRAP_STATUS_AP_CONN_PT=76, + BOOTSTRAP_STATUS_AP_CONN_DONE_PT=77, + BOOTSTRAP_STATUS_AP_CONN_PROXY=78, + BOOTSTRAP_STATUS_AP_CONN_DONE_PROXY=79, + BOOTSTRAP_STATUS_AP_CONN=80, + BOOTSTRAP_STATUS_AP_CONN_DONE=85, + BOOTSTRAP_STATUS_AP_HANDSHAKE=89, + BOOTSTRAP_STATUS_AP_HANDSHAKE_DONE=90, + + /* Creating AP circuits */ + + BOOTSTRAP_STATUS_CIRCUIT_CREATE=95, BOOTSTRAP_STATUS_DONE=100 } bootstrap_status_t; @@ -200,11 +208,15 @@ void control_event_boot_dir(bootstrap_status_t status, int progress); void control_event_boot_first_orconn(void); void control_event_bootstrap_problem(const char *warn, const char *reason, const connection_t *conn, int dowarn); +char *control_event_boot_last_msg(void); +void control_event_bootstrap_reset(void); void control_event_clients_seen(const char *controller_str); void control_event_transport_launched(const char *mode, const char *transport_name, tor_addr_t *addr, uint16_t port); +void control_event_pt_log(const char *log); +void control_event_pt_status(const char *status); const char *rend_auth_type_to_string(rend_auth_type_t auth_type); MOCK_DECL(const char *, node_describe_longname_by_id,(const char *id_digest)); void control_event_hs_descriptor_requested(const char *onion_address, @@ -293,7 +305,9 @@ void control_free_all(void); #define EVENT_HS_DESC 0x0021 #define EVENT_HS_DESC_CONTENT 0x0022 #define EVENT_NETWORK_LIVENESS 0x0023 -#define EVENT_MAX_ 0x0023 +#define EVENT_PT_LOG 0x0024 +#define EVENT_PT_STATUS 0x0025 +#define EVENT_MAX_ 0x0025 /* sizeof(control_connection_t.event_mask) in bits, currently a uint64_t */ #define EVENT_CAPACITY_ 0x0040 diff --git a/src/feature/control/control_bootstrap.c b/src/feature/control/control_bootstrap.c new file mode 100644 index 0000000000..8153d7595a --- /dev/null +++ b/src/feature/control/control_bootstrap.c @@ -0,0 +1,383 @@ +/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file control_bootstrap.c + * \brief Provide bootstrap progress events for the control port. + */ +#include "core/or/or.h" + +#include "app/config/config.h" +#include "core/mainloop/connection.h" +#include "core/or/connection_or.h" +#include "core/or/connection_st.h" +#include "core/or/or_connection_st.h" +#include "core/or/reasons.h" +#include "feature/control/control.h" +#include "feature/hibernate/hibernate.h" +#include "lib/malloc/malloc.h" + +/** A sufficiently large size to record the last bootstrap phase string. */ +#define BOOTSTRAP_MSG_LEN 1024 + +/** What was the last bootstrap phase message we sent? We keep track + * of this so we can respond to getinfo status/bootstrap-phase queries. */ +static char last_sent_bootstrap_message[BOOTSTRAP_MSG_LEN]; + +/** Table to convert bootstrap statuses to strings. */ +static const struct { + bootstrap_status_t status; + const char *tag; + const char *summary; +} boot_to_str_tab[] = { + { BOOTSTRAP_STATUS_UNDEF, "undef", "Undefined" }, + { BOOTSTRAP_STATUS_STARTING, "starting", "Starting" }, + + /* Initial connection to any relay */ + + { BOOTSTRAP_STATUS_CONN_PT, "conn_pt", "Connecting to pluggable transport" }, + { BOOTSTRAP_STATUS_CONN_DONE_PT, "conn_done_pt", + "Connected to pluggable transport" }, + { BOOTSTRAP_STATUS_CONN_PROXY, "conn_proxy", "Connecting to proxy" }, + { BOOTSTRAP_STATUS_CONN_DONE_PROXY, "conn_done_proxy", + "Connected to proxy" }, + { BOOTSTRAP_STATUS_CONN, "conn", "Connecting to a relay" }, + { BOOTSTRAP_STATUS_CONN_DONE, "conn_done", "Connected to a relay" }, + { BOOTSTRAP_STATUS_HANDSHAKE, "handshake", + "Handshaking with a relay" }, + { BOOTSTRAP_STATUS_HANDSHAKE_DONE, "handshake_done", + "Handshake with a relay done" }, + + /* Loading directory info */ + + { BOOTSTRAP_STATUS_ONEHOP_CREATE, "onehop_create", + "Establishing an encrypted directory connection" }, + { BOOTSTRAP_STATUS_REQUESTING_STATUS, "requesting_status", + "Asking for networkstatus consensus" }, + { BOOTSTRAP_STATUS_LOADING_STATUS, "loading_status", + "Loading networkstatus consensus" }, + { BOOTSTRAP_STATUS_LOADING_KEYS, "loading_keys", + "Loading authority key certs" }, + { BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS, "requesting_descriptors", + "Asking for relay descriptors" }, + { BOOTSTRAP_STATUS_LOADING_DESCRIPTORS, "loading_descriptors", + "Loading relay descriptors" }, + { BOOTSTRAP_STATUS_ENOUGH_DIRINFO, "enough_dirinfo", + "Loaded enough directory info to build circuits" }, + + /* Connecting to a relay for AP circuits */ + + { BOOTSTRAP_STATUS_AP_CONN_PT, "ap_conn_pt", + "Connecting to pluggable transport to build circuits" }, + { BOOTSTRAP_STATUS_AP_CONN_DONE_PT, "ap_conn_done_pt", + "Connected to pluggable transport to build circuits" }, + { BOOTSTRAP_STATUS_AP_CONN_PROXY, "ap_conn_proxy", + "Connecting to proxy to build circuits" }, + { BOOTSTRAP_STATUS_AP_CONN_DONE_PROXY, "ap_conn_done_proxy", + "Connected to proxy to build circuits" }, + { BOOTSTRAP_STATUS_AP_CONN, "ap_conn", + "Connecting to a relay to build circuits" }, + { BOOTSTRAP_STATUS_AP_CONN_DONE, "ap_conn_done", + "Connected to a relay to build circuits" }, + { BOOTSTRAP_STATUS_AP_HANDSHAKE, "ap_handshake", + "Finishing handshake with a relay to build circuits" }, + { BOOTSTRAP_STATUS_AP_HANDSHAKE_DONE, "ap_handshake_done", + "Handshake finished with a relay to build circuits" }, + + /* Creating AP circuits */ + + { BOOTSTRAP_STATUS_CIRCUIT_CREATE, "circuit_create", + "Establishing a Tor circuit" }, + { BOOTSTRAP_STATUS_DONE, "done", "Done" }, +}; +#define N_BOOT_TO_STR (sizeof(boot_to_str_tab)/sizeof(boot_to_str_tab[0])) + +/** Convert the name of a bootstrapping phase <b>s</b> into strings + * <b>tag</b> and <b>summary</b> suitable for display by the controller. */ +static int +bootstrap_status_to_string(bootstrap_status_t s, const char **tag, + const char **summary) +{ + for (size_t i = 0; i < N_BOOT_TO_STR; i++) { + if (s == boot_to_str_tab[i].status) { + *tag = boot_to_str_tab[i].tag; + *summary = boot_to_str_tab[i].summary; + return 0; + } + } + + *tag = *summary = "unknown"; + return -1; +} + +/** What percentage through the bootstrap process are we? We remember + * this so we can avoid sending redundant bootstrap status events, and + * so we can guess context for the bootstrap messages which are + * ambiguous. It starts at 'undef', but gets set to 'starting' while + * Tor initializes. */ +static int bootstrap_percent = BOOTSTRAP_STATUS_UNDEF; + +/** Like bootstrap_percent, but only takes on the enumerated values in + * bootstrap_status_t. + */ +static int bootstrap_phase = BOOTSTRAP_STATUS_UNDEF; + +/** As bootstrap_percent, but holds the bootstrapping level at which we last + * logged a NOTICE-level message. We use this, plus BOOTSTRAP_PCT_INCREMENT, + * to avoid flooding the log with a new message every time we get a few more + * microdescriptors */ +static int notice_bootstrap_percent = 0; + +/** How many problems have we had getting to the next bootstrapping phase? + * These include failure to establish a connection to a Tor relay, + * failures to finish the TLS handshake, failures to validate the + * consensus document, etc. */ +static int bootstrap_problems = 0; + +/** We only tell the controller once we've hit a threshold of problems + * for the current phase. */ +#define BOOTSTRAP_PROBLEM_THRESHOLD 10 + +/** When our bootstrapping progress level changes, but our bootstrapping + * status has not advanced, we only log at NOTICE when we have made at least + * this much progress. + */ +#define BOOTSTRAP_PCT_INCREMENT 5 + +/** Do the actual logging and notifications for + * control_event_bootstrap(). Doesn't change any state beyond that. + */ +static void +control_event_bootstrap_core(int loglevel, bootstrap_status_t status, + int progress) +{ + char buf[BOOTSTRAP_MSG_LEN]; + const char *tag, *summary; + + bootstrap_status_to_string(status, &tag, &summary); + /* Locally reset status if there's incremental progress */ + if (progress) + status = progress; + + tor_log(loglevel, LD_CONTROL, + "Bootstrapped %d%% (%s): %s", status, tag, summary); + tor_snprintf(buf, sizeof(buf), + "BOOTSTRAP PROGRESS=%d TAG=%s SUMMARY=\"%s\"", + status, tag, summary); + tor_snprintf(last_sent_bootstrap_message, + sizeof(last_sent_bootstrap_message), + "NOTICE %s", buf); + control_event_client_status(LOG_NOTICE, "%s", buf); +} + +/** Called when Tor has made progress at bootstrapping its directory + * information and initial circuits. + * + * <b>status</b> is the new status, that is, what task we will be doing + * next. <b>progress</b> is zero if we just started this task, else it + * represents progress on the task. + */ +void +control_event_bootstrap(bootstrap_status_t status, int progress) +{ + int loglevel = LOG_NOTICE; + + if (bootstrap_percent == BOOTSTRAP_STATUS_DONE) + return; /* already bootstrapped; nothing to be done here. */ + + if (status <= bootstrap_percent) { + /* If there's no new progress, return early. */ + if (!progress || progress <= bootstrap_percent) + return; + /* Log at INFO if not enough progress happened. */ + if (progress < notice_bootstrap_percent + BOOTSTRAP_PCT_INCREMENT) + loglevel = LOG_INFO; + } + + control_event_bootstrap_core(loglevel, status, progress); + + if (status > bootstrap_percent) { + bootstrap_phase = status; /* new milestone reached */ + bootstrap_percent = status; + } + if (progress > bootstrap_percent) { + /* incremental progress within a milestone */ + bootstrap_percent = progress; + bootstrap_problems = 0; /* Progress! Reset our problem counter. */ + } + if (loglevel == LOG_NOTICE && + bootstrap_percent > notice_bootstrap_percent) { + /* Remember that we gave a notice at this level. */ + notice_bootstrap_percent = bootstrap_percent; + } +} + +/** Flag whether we've opened an OR_CONN yet */ +static int bootstrap_first_orconn = 0; + +/** Like bootstrap_phase, but for (possibly deferred) directory progress */ +static int bootstrap_dir_phase = BOOTSTRAP_STATUS_UNDEF; + +/** Like bootstrap_problems, but for (possibly deferred) directory progress */ +static int bootstrap_dir_progress = BOOTSTRAP_STATUS_UNDEF; + +/** Defer directory info bootstrap events until we have successfully + * completed our first connection to a router. */ +void +control_event_boot_dir(bootstrap_status_t status, int progress) +{ + if (status > bootstrap_dir_progress) { + bootstrap_dir_progress = status; + bootstrap_dir_phase = status; + } + if (progress && progress >= bootstrap_dir_progress) { + bootstrap_dir_progress = progress; + } + + /* Don't report unless we have successfully opened at least one OR_CONN */ + if (!bootstrap_first_orconn) + return; + + control_event_bootstrap(status, progress); +} + +/** Set a flag to allow reporting of directory bootstrap progress. + * (Code that reports completion of an OR_CONN calls this.) Also, + * report directory progress so far. */ +void +control_event_boot_first_orconn(void) +{ + bootstrap_first_orconn = 1; + control_event_bootstrap(bootstrap_dir_phase, bootstrap_dir_progress); +} + +/** Called when Tor has failed to make bootstrapping progress in a way + * that indicates a problem. <b>warn</b> gives a human-readable hint + * as to why, and <b>reason</b> provides a controller-facing short + * tag. <b>conn</b> is the connection that caused this problem and + * can be NULL if a connection cannot be easily identified. + */ +void +control_event_bootstrap_problem(const char *warn, const char *reason, + const connection_t *conn, int dowarn) +{ + int status = bootstrap_percent; + const char *tag = "", *summary = ""; + char buf[BOOTSTRAP_MSG_LEN]; + const char *recommendation = "ignore"; + int severity; + char *or_id = NULL, *hostaddr = NULL; + or_connection_t *or_conn = NULL; + + /* bootstrap_percent must not be in "undefined" state here. */ + tor_assert(status >= 0); + + if (bootstrap_percent == 100) + return; /* already bootstrapped; nothing to be done here. */ + + bootstrap_problems++; + + if (bootstrap_problems >= BOOTSTRAP_PROBLEM_THRESHOLD) + dowarn = 1; + + /* Don't warn about our bootstrapping status if we are hibernating or + * shutting down. */ + if (we_are_hibernating()) + dowarn = 0; + + tor_assert(bootstrap_status_to_string(bootstrap_phase, &tag, &summary) == 0); + + severity = dowarn ? LOG_WARN : LOG_INFO; + + if (dowarn) + recommendation = "warn"; + + if (conn && conn->type == CONN_TYPE_OR) { + /* XXX TO_OR_CONN can't deal with const */ + or_conn = TO_OR_CONN((connection_t *)conn); + or_id = tor_strdup(hex_str(or_conn->identity_digest, DIGEST_LEN)); + } else { + or_id = tor_strdup("?"); + } + + if (conn) + tor_asprintf(&hostaddr, "%s:%d", conn->address, (int)conn->port); + else + hostaddr = tor_strdup("?"); + + log_fn(severity, + LD_CONTROL, "Problem bootstrapping. Stuck at %d%% (%s): %s. (%s; %s; " + "count %d; recommendation %s; host %s at %s)", + status, tag, summary, warn, reason, + bootstrap_problems, recommendation, + or_id, hostaddr); + + connection_or_report_broken_states(severity, LD_HANDSHAKE); + + tor_snprintf(buf, sizeof(buf), + "BOOTSTRAP PROGRESS=%d TAG=%s SUMMARY=\"%s\" WARNING=\"%s\" REASON=%s " + "COUNT=%d RECOMMENDATION=%s HOSTID=\"%s\" HOSTADDR=\"%s\"", + bootstrap_percent, tag, summary, warn, reason, bootstrap_problems, + recommendation, + or_id, hostaddr); + + tor_snprintf(last_sent_bootstrap_message, + sizeof(last_sent_bootstrap_message), + "WARN %s", buf); + control_event_client_status(LOG_WARN, "%s", buf); + + tor_free(hostaddr); + tor_free(or_id); +} + +/** Called when Tor has failed to make bootstrapping progress in a way + * that indicates a problem. <b>warn</b> gives a hint as to why, and + * <b>reason</b> provides an "or_conn_end_reason" tag. <b>or_conn</b> + * is the connection that caused this problem. + */ +MOCK_IMPL(void, +control_event_bootstrap_prob_or, (const char *warn, int reason, + or_connection_t *or_conn)) +{ + int dowarn = 0; + + if (or_conn->have_noted_bootstrap_problem) + return; + + or_conn->have_noted_bootstrap_problem = 1; + + if (reason == END_OR_CONN_REASON_NO_ROUTE) + dowarn = 1; + + /* If we are using bridges and all our OR connections are now + closed, it means that we totally failed to connect to our + bridges. Throw a warning. */ + if (get_options()->UseBridges && !any_other_active_or_conns(or_conn)) + dowarn = 1; + + control_event_bootstrap_problem(warn, + orconn_end_reason_to_control_string(reason), + TO_CONN(or_conn), dowarn); +} + +/** Return a copy of the last sent bootstrap message. */ +char * +control_event_boot_last_msg(void) +{ + return tor_strdup(last_sent_bootstrap_message); +} + +/** Reset bootstrap tracking state. */ +void +control_event_bootstrap_reset(void) +{ + bootstrap_percent = BOOTSTRAP_STATUS_UNDEF; + bootstrap_phase = BOOTSTRAP_STATUS_UNDEF; + notice_bootstrap_percent = 0; + bootstrap_problems = 0; + bootstrap_first_orconn = 0; + bootstrap_dir_progress = BOOTSTRAP_STATUS_UNDEF; + bootstrap_dir_phase = BOOTSTRAP_STATUS_UNDEF; + memset(last_sent_bootstrap_message, 0, sizeof(last_sent_bootstrap_message)); +} diff --git a/src/feature/dirauth/bwauth.c b/src/feature/dirauth/bwauth.c index 12f9399e9f..1cfd8119df 100644 --- a/src/feature/dirauth/bwauth.c +++ b/src/feature/dirauth/bwauth.c @@ -20,6 +20,7 @@ #include "feature/nodelist/routerinfo_st.h" #include "feature/nodelist/vote_routerstatus_st.h" +#include "lib/crypt_ops/crypto_format.h" #include "lib/encoding/keyval.h" /** Total number of routers with measured bandwidth; this is set by @@ -205,7 +206,8 @@ dirserv_get_credible_bandwidth_kb(const routerinfo_t *ri) int dirserv_read_measured_bandwidths(const char *from_file, smartlist_t *routerstatuses, - smartlist_t *bw_file_headers) + smartlist_t *bw_file_headers, + uint8_t *digest_out) { FILE *fp = tor_fopen_cloexec(from_file, "r"); int applied_lines = 0; @@ -219,6 +221,7 @@ dirserv_read_measured_bandwidths(const char *from_file, int rv = -1; char *line = NULL; size_t n = 0; + crypto_digest_t *digest = crypto_digest256_new(DIGEST_SHA256); /* Initialise line, so that we can't possibly run off the end. */ @@ -233,11 +236,14 @@ dirserv_read_measured_bandwidths(const char *from_file, log_warn(LD_DIRSERV, "Empty bandwidth file"); goto err; } + /* If the line could be gotten, add it to the digest */ + crypto_digest_add_bytes(digest, (const char *) line, strlen(line)); if (!strlen(line) || line[strlen(line)-1] != '\n') { log_warn(LD_DIRSERV, "Long or truncated time in bandwidth file: %s", escaped(line)); - goto err; + /* Continue adding lines to the digest. */ + goto continue_digest; } line[strlen(line)-1] = '\0'; @@ -245,14 +251,14 @@ dirserv_read_measured_bandwidths(const char *from_file, if (!ok) { log_warn(LD_DIRSERV, "Non-integer time in bandwidth file: %s", escaped(line)); - goto err; + goto continue_digest; } - now = time(NULL); + now = approx_time(); if ((now - file_time) > MAX_MEASUREMENT_AGE) { log_warn(LD_DIRSERV, "Bandwidth measurement file stale. Age: %u", (unsigned)(time(NULL) - file_time)); - goto err; + goto continue_digest; } /* If timestamp was correct and bw_file_headers is not NULL, @@ -267,6 +273,7 @@ dirserv_read_measured_bandwidths(const char *from_file, while (!feof(fp)) { measured_bw_line_t parsed_line; if (tor_getline(&line, &n, fp) >= 0) { + crypto_digest_add_bytes(digest, (const char *) line, strlen(line)); if (measured_bw_line_parse(&parsed_line, line, line_is_after_headers) != -1) { /* This condition will be true when the first complete valid bw line @@ -305,6 +312,14 @@ dirserv_read_measured_bandwidths(const char *from_file, "Applied %d measurements.", applied_lines); rv = 0; + continue_digest: + /* Continue parsing lines to return the digest of the Bandwidth File. */ + while (!feof(fp)) { + if (tor_getline(&line, &n, fp) >= 0) { + crypto_digest_add_bytes(digest, (const char *) line, strlen(line)); + } + } + err: if (line) { // we need to raw_free this buffer because we got it from tor_getdelim() @@ -312,6 +327,9 @@ dirserv_read_measured_bandwidths(const char *from_file, } if (fp) fclose(fp); + if (digest_out) + crypto_digest_get_digest(digest, (char *) digest_out, DIGEST256_LEN); + crypto_digest_free(digest); return rv; } diff --git a/src/feature/dirauth/bwauth.h b/src/feature/dirauth/bwauth.h index 4507728458..8b7acc4a1c 100644 --- a/src/feature/dirauth/bwauth.h +++ b/src/feature/dirauth/bwauth.h @@ -21,8 +21,8 @@ int dirserv_read_measured_bandwidths(const char *from_file, smartlist_t *routerstatuses, - smartlist_t *bw_file_headers); - + smartlist_t *bw_file_headers, + uint8_t *digest_out); int dirserv_query_measured_bw_cache_kb(const char *node_id, long *bw_out, time_t *as_of_out); diff --git a/src/feature/dirauth/dirvote.c b/src/feature/dirauth/dirvote.c index af8b3dc207..755b99bae2 100644 --- a/src/feature/dirauth/dirvote.c +++ b/src/feature/dirauth/dirvote.c @@ -28,6 +28,7 @@ #include "feature/nodelist/fmt_routerstatus.h" #include "feature/nodelist/microdesc.h" #include "feature/nodelist/networkstatus.h" +#include "feature/nodelist/nodefamily.h" #include "feature/nodelist/nodelist.h" #include "feature/nodelist/routerlist.h" #include "feature/relay/router.h" @@ -60,6 +61,9 @@ #include "lib/encoding/confline.h" #include "lib/crypt_ops/crypto_format.h" +/* Algorithm to use for the bandwidth file digest. */ +#define DIGEST_ALG_BW_FILE DIGEST_SHA256 + /** * \file dirvote.c * \brief Functions to compute directory consensus, and schedule voting. @@ -268,6 +272,7 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key, char *flag_thresholds = dirserv_get_flag_thresholds_line(); char *params; char *bw_headers_line = NULL; + char *bw_file_digest = NULL; authority_cert_t *cert = v3_ns->cert; char *methods = make_consensus_method_list(MIN_SUPPORTED_CONSENSUS_METHOD, @@ -307,6 +312,27 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key, tor_free(bw_file_headers); } + /* Create bandwidth-file-digest if applicable. + * v3_ns->b64_digest_bw_file will contain the digest when V3BandwidthsFile + * is configured and the bandwidth file could be read, even if it was not + * parseable. + */ + if (!tor_digest256_is_zero((const char *)v3_ns->bw_file_digest256)) { + /* Encode the digest. */ + char b64_digest_bw_file[BASE64_DIGEST256_LEN+1] = {0}; + digest256_to_base64(b64_digest_bw_file, + (const char *)v3_ns->bw_file_digest256); + /* "bandwidth-file-digest" 1*(SP algorithm "=" digest) NL */ + char *digest_algo_b64_digest_bw_file = NULL; + tor_asprintf(&digest_algo_b64_digest_bw_file, "%s=%s", + crypto_digest_algorithm_get_name(DIGEST_ALG_BW_FILE), + b64_digest_bw_file); + /* No need for tor_strdup(""), format_line_if_present does it. */ + bw_file_digest = format_line_if_present( + "bandwidth-file-digest", digest_algo_b64_digest_bw_file); + tor_free(digest_algo_b64_digest_bw_file); + } + smartlist_add_asprintf(chunks, "network-status-version 3\n" "vote-status %s\n" @@ -323,6 +349,7 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key, "flag-thresholds %s\n" "params %s\n" "%s" /* bandwidth file headers */ + "%s" /* bandwidth file digest */ "dir-source %s %s %s %s %d %d\n" "contact %s\n" "%s" /* shared randomness information */ @@ -339,6 +366,7 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key, flag_thresholds, params, bw_headers_line ? bw_headers_line : "", + bw_file_digest ? bw_file_digest: "", voter->nickname, fingerprint, voter->address, fmt_addr32(addr), voter->dir_port, voter->or_port, voter->contact, @@ -351,6 +379,7 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key, tor_free(methods); tor_free(shared_random_vote_str); tor_free(bw_headers_line); + tor_free(bw_file_digest); if (!tor_digest_is_zero(voter->legacy_id_digest)) { char fpbuf[HEX_DIGEST_LEN+1]; @@ -412,7 +441,8 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key, { networkstatus_t *v; - if (!(v = networkstatus_parse_vote_from_string(status, NULL, + if (!(v = networkstatus_parse_vote_from_string(status, strlen(status), + NULL, v3_ns->type))) { log_err(LD_BUG,"Generated a networkstatus %s we couldn't parse: " "<<%s>>", @@ -2409,7 +2439,8 @@ networkstatus_compute_consensus(smartlist_t *votes, { networkstatus_t *c; - if (!(c = networkstatus_parse_vote_from_string(result, NULL, + if (!(c = networkstatus_parse_vote_from_string(result, strlen(result), + NULL, NS_TYPE_CONSENSUS))) { log_err(LD_BUG, "Generated a networkstatus consensus we couldn't " "parse."); @@ -3132,7 +3163,8 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out) *msg_out = NULL; again: - vote = networkstatus_parse_vote_from_string(vote_body, &end_of_vote, + vote = networkstatus_parse_vote_from_string(vote_body, strlen(vote_body), + &end_of_vote, NS_TYPE_VOTE); if (!end_of_vote) end_of_vote = vote_body + strlen(vote_body); @@ -3390,7 +3422,9 @@ dirvote_compute_consensuses(void) flavor_name); continue; } - consensus = networkstatus_parse_vote_from_string(consensus_body, NULL, + consensus = networkstatus_parse_vote_from_string(consensus_body, + strlen(consensus_body), + NULL, NS_TYPE_CONSENSUS); if (!consensus) { log_warn(LD_DIR, "Couldn't parse %s consensus we generated!", @@ -3529,7 +3563,7 @@ dirvote_add_signatures_to_pending_consensus( * just in case we break detached signature processing at some point. */ { networkstatus_t *v = networkstatus_parse_vote_from_string( - pc->body, NULL, + pc->body, strlen(pc->body), NULL, NS_TYPE_CONSENSUS); tor_assert(v); networkstatus_vote_free(v); @@ -3654,7 +3688,9 @@ dirvote_publish_consensus(void) continue; } - if (networkstatus_set_current_consensus(pending->body, name, 0, NULL)) + if (networkstatus_set_current_consensus(pending->body, + strlen(pending->body), + name, 0, NULL)) log_warn(LD_DIR, "Error publishing %s consensus", name); else log_notice(LD_DIR, "Published %s consensus", name); @@ -3791,8 +3827,16 @@ dirvote_create_microdescriptor(const routerinfo_t *ri, int consensus_method) smartlist_add_asprintf(chunks, "a %s\n", fmt_addrport(&ri->ipv6_addr, ri->ipv6_orport)); - if (family) - smartlist_add_asprintf(chunks, "family %s\n", family); + if (family) { + if (consensus_method < MIN_METHOD_FOR_CANONICAL_FAMILIES_IN_MICRODESCS) { + smartlist_add_asprintf(chunks, "family %s\n", family); + } else { + const uint8_t *id = (const uint8_t *)ri->cache_info.identity_digest; + char *canonical_family = nodefamily_canonicalize(family, id, 0); + smartlist_add_asprintf(chunks, "family %s\n", canonical_family); + tor_free(canonical_family); + } + } if (summary && strcmp(summary, "reject 1-65535")) smartlist_add_asprintf(chunks, "p %s\n", summary); @@ -3890,7 +3934,10 @@ static const struct consensus_method_range_t { int high; } microdesc_consensus_methods[] = { {MIN_SUPPORTED_CONSENSUS_METHOD, MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC - 1}, - {MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC, MAX_SUPPORTED_CONSENSUS_METHOD}, + {MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC, + MIN_METHOD_FOR_CANONICAL_FAMILIES_IN_MICRODESCS - 1}, + {MIN_METHOD_FOR_CANONICAL_FAMILIES_IN_MICRODESCS, + MAX_SUPPORTED_CONSENSUS_METHOD}, {-1, -1} }; @@ -4364,6 +4411,23 @@ clear_status_flags_on_sybil(routerstatus_t *rs) * forget to add it to this clause. */ } +/** Space-separated list of all the flags that we will always vote on. */ +const char DIRVOTE_UNIVERSAL_FLAGS[] = + "Authority " + "Exit " + "Fast " + "Guard " + "HSDir " + "Stable " + "StaleDesc " + "V2Dir " + "Valid"; +/** Space-separated list of all flags that we may or may not vote on, + * depending on our configuration. */ +const char DIRVOTE_OPTIONAL_FLAGS[] = + "BadExit " + "Running"; + /** Return a new networkstatus_t* containing our current opinion. (For v3 * authorities) */ networkstatus_t * @@ -4388,6 +4452,7 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, const int vote_on_reachability = running_long_enough_to_decide_unreachable(); smartlist_t *microdescriptors = NULL; smartlist_t *bw_file_headers = NULL; + uint8_t bw_file_digest256[DIGEST256_LEN] = {0}; tor_assert(private_key); tor_assert(cert); @@ -4425,7 +4490,8 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, * set_routerstatus_from_routerinfo() see up-to-date bandwidth info. */ if (options->V3BandwidthsFile) { - dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL, NULL); + dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL, NULL, + NULL); } else { /* * No bandwidths file; clear the measured bandwidth cache in case we had @@ -4530,7 +4596,9 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, /* Only set bw_file_headers when V3BandwidthsFile is configured */ bw_file_headers = smartlist_new(); dirserv_read_measured_bandwidths(options->V3BandwidthsFile, - routerstatuses, bw_file_headers); + routerstatuses, bw_file_headers, + bw_file_digest256); + } else { /* * No bandwidths file; clear the measured bandwidth cache in case we had @@ -4612,7 +4680,7 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, v3_out->known_flags = smartlist_new(); smartlist_split_string(v3_out->known_flags, - "Authority Exit Fast Guard Stable V2Dir Valid HSDir", + DIRVOTE_UNIVERSAL_FLAGS, 0, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); if (vote_on_reachability) smartlist_add_strdup(v3_out->known_flags, "Running"); @@ -4627,6 +4695,7 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, smartlist_sort_strings(v3_out->net_params); } v3_out->bw_file_headers = bw_file_headers; + memcpy(v3_out->bw_file_digest256, bw_file_digest256, DIGEST256_LEN); voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t)); voter->nickname = tor_strdup(options->Nickname); diff --git a/src/feature/dirauth/dirvote.h b/src/feature/dirauth/dirvote.h index 02d88d19d1..f9de5ebc41 100644 --- a/src/feature/dirauth/dirvote.h +++ b/src/feature/dirauth/dirvote.h @@ -57,7 +57,7 @@ #define MIN_SUPPORTED_CONSENSUS_METHOD 25 /** The highest consensus method that we currently support. */ -#define MAX_SUPPORTED_CONSENSUS_METHOD 28 +#define MAX_SUPPORTED_CONSENSUS_METHOD 29 /** Lowest consensus method where authorities vote on required/recommended * protocols. */ @@ -79,6 +79,12 @@ * addresses. See #23828 and #20916. */ #define MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC 28 +/** + * Lowest consensus method where microdescriptor lines are put in canonical + * form for improved compressibility and ease of storage. See proposal 298. + **/ +#define MIN_METHOD_FOR_CANONICAL_FAMILIES_IN_MICRODESCS 29 + /** Default bandwidth to clip unmeasured bandwidths to using method >= * MIN_METHOD_TO_CLIP_UNMEASURED_BW. (This is not a consensus method; do not * get confused with the above macros.) */ @@ -92,6 +98,9 @@ /** Maximum size of a line in a vote. */ #define MAX_BW_FILE_HEADERS_LINE_LEN 1024 +extern const char DIRVOTE_UNIVERSAL_FLAGS[]; +extern const char DIRVOTE_OPTIONAL_FLAGS[]; + /* * Public API. Used outside of the dirauth subsystem. * diff --git a/src/feature/dirauth/process_descs.c b/src/feature/dirauth/process_descs.c index 21b8e239ec..656922233e 100644 --- a/src/feature/dirauth/process_descs.c +++ b/src/feature/dirauth/process_descs.c @@ -519,7 +519,8 @@ WRA_MORE_SEVERE(was_router_added_t a, was_router_added_t b) /** As for dirserv_add_descriptor(), but accepts multiple documents, and * returns the most severe error that occurred for any one of them. */ was_router_added_t -dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose, +dirserv_add_multiple_descriptors(const char *desc, size_t desclen, + uint8_t purpose, const char *source, const char **msg) { @@ -536,6 +537,11 @@ dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose, r=ROUTER_ADDED_SUCCESSFULLY; /*Least severe return value. */ + if (!string_is_utf8_no_bom(desc, desclen)) { + *msg = "descriptor(s) or extrainfo(s) not valid UTF-8 or had BOM."; + return ROUTER_AUTHDIR_REJECTS; + } + format_iso_time(time_buf, now); if (tor_snprintf(annotation_buf, sizeof(annotation_buf), "@uploaded-at %s\n" @@ -552,7 +558,7 @@ dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose, s = desc; list = smartlist_new(); - if (!router_parse_list_from_string(&s, NULL, list, SAVED_NOWHERE, 0, 0, + if (!router_parse_list_from_string(&s, s+desclen, list, SAVED_NOWHERE, 0, 0, annotation_buf, NULL)) { SMARTLIST_FOREACH(list, routerinfo_t *, ri, { msg_out = NULL; @@ -568,7 +574,7 @@ dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose, smartlist_clear(list); s = desc; - if (!router_parse_list_from_string(&s, NULL, list, SAVED_NOWHERE, 1, 0, + if (!router_parse_list_from_string(&s, s+desclen, list, SAVED_NOWHERE, 1, 0, NULL, NULL)) { SMARTLIST_FOREACH(list, extrainfo_t *, ei, { msg_out = NULL; diff --git a/src/feature/dirauth/process_descs.h b/src/feature/dirauth/process_descs.h index ae2d6ad25d..510e54f813 100644 --- a/src/feature/dirauth/process_descs.h +++ b/src/feature/dirauth/process_descs.h @@ -17,7 +17,8 @@ void dirserv_free_fingerprint_list(void); int dirserv_add_own_fingerprint(crypto_pk_t *pk); enum was_router_added_t dirserv_add_multiple_descriptors( - const char *desc, uint8_t purpose, + const char *desc, size_t desclen, + uint8_t purpose, const char *source, const char **msg); enum was_router_added_t dirserv_add_descriptor(routerinfo_t *ri, diff --git a/src/feature/dirauth/shared_random.c b/src/feature/dirauth/shared_random.c index 34b2283250..137c49800f 100644 --- a/src/feature/dirauth/shared_random.c +++ b/src/feature/dirauth/shared_random.c @@ -120,8 +120,8 @@ static const char sr_flag_ns_str[] = "shared-rand-participate"; static int32_t num_srv_agreements_from_vote; /* Return a heap allocated copy of the SRV <b>orig</b>. */ -STATIC sr_srv_t * -srv_dup(const sr_srv_t *orig) +sr_srv_t * +sr_srv_dup(const sr_srv_t *orig) { sr_srv_t *duplicate = NULL; @@ -1253,8 +1253,8 @@ sr_act_post_consensus(const networkstatus_t *consensus) * decided by the majority. */ sr_state_unset_fresh_srv(); /* Set the SR values from the given consensus. */ - sr_state_set_previous_srv(srv_dup(consensus->sr_info.previous_srv)); - sr_state_set_current_srv(srv_dup(consensus->sr_info.current_srv)); + sr_state_set_previous_srv(sr_srv_dup(consensus->sr_info.previous_srv)); + sr_state_set_current_srv(sr_srv_dup(consensus->sr_info.current_srv)); } /* Prepare our state so that it's ready for the next voting period. */ diff --git a/src/feature/dirauth/shared_random.h b/src/feature/dirauth/shared_random.h index 25d95ebbc7..0b45ad1ed7 100644 --- a/src/feature/dirauth/shared_random.h +++ b/src/feature/dirauth/shared_random.h @@ -154,6 +154,7 @@ const char *sr_commit_get_rsa_fpr(const sr_commit_t *commit) void sr_compute_srv(void); sr_commit_t *sr_generate_our_commit(time_t timestamp, const authority_cert_t *my_rsa_cert); +sr_srv_t *sr_srv_dup(const sr_srv_t *orig); #ifdef SHARED_RANDOM_PRIVATE @@ -172,7 +173,6 @@ STATIC sr_srv_t *get_majority_srv_from_votes(const smartlist_t *votes, int current); STATIC void save_commit_to_state(sr_commit_t *commit); -STATIC sr_srv_t *srv_dup(const sr_srv_t *orig); STATIC int commitments_are_the_same(const sr_commit_t *commit_one, const sr_commit_t *commit_two); STATIC int commit_is_authoritative(const sr_commit_t *commit, diff --git a/src/feature/dirauth/shared_random_state.c b/src/feature/dirauth/shared_random_state.c index b3e4a4ef92..a7b7480edd 100644 --- a/src/feature/dirauth/shared_random_state.c +++ b/src/feature/dirauth/shared_random_state.c @@ -22,6 +22,7 @@ #include "feature/dirauth/shared_random_state.h" #include "feature/dircommon/voting_schedule.h" #include "lib/encoding/confline.h" +#include "lib/version/torversion.h" #include "app/config/or_state_st.h" @@ -101,6 +102,8 @@ static const config_format_t state_format = { &state_extra_var, }; +static void state_query_del_(sr_state_object_t obj_type, void *data); + /* Return a string representation of a protocol phase. */ STATIC const char * get_phase_str(sr_phase_t phase) @@ -833,6 +836,9 @@ state_query_get_commit(const char *rsa_fpr) static void * state_query_get_(sr_state_object_t obj_type, const void *data) { + if (BUG(!sr_state)) + return NULL; + void *obj = NULL; switch (obj_type) { @@ -861,23 +867,44 @@ state_query_get_(sr_state_object_t obj_type, const void *data) } /* Helper function: This handles the PUT state action using an - * <b>obj_type</b> and <b>data</b> needed for the action. */ + * <b>obj_type</b> and <b>data</b> needed for the action. + * PUT frees the previous data before replacing it, if needed. */ static void state_query_put_(sr_state_object_t obj_type, void *data) { + if (BUG(!sr_state)) + return; + switch (obj_type) { case SR_STATE_OBJ_COMMIT: { sr_commit_t *commit = data; tor_assert(commit); + /* commit_add_to_state() frees the old commit, if there is one */ commit_add_to_state(commit, sr_state); break; } case SR_STATE_OBJ_CURSRV: - sr_state->current_srv = (sr_srv_t *) data; + /* Check if the new pointer is the same as the old one: if it is, it's + * probably a bug. The caller may have confused current and previous, + * or they may have forgotten to sr_srv_dup(). + * Putting NULL multiple times is allowed. */ + if (!BUG(data && sr_state->current_srv == (sr_srv_t *) data)) { + /* We own the old SRV, so we need to free it. */ + state_query_del_(SR_STATE_OBJ_CURSRV, NULL); + sr_state->current_srv = (sr_srv_t *) data; + } break; case SR_STATE_OBJ_PREVSRV: - sr_state->previous_srv = (sr_srv_t *) data; + /* Check if the new pointer is the same as the old one: if it is, it's + * probably a bug. The caller may have confused current and previous, + * or they may have forgotten to sr_srv_dup(). + * Putting NULL multiple times is allowed. */ + if (!BUG(data && sr_state->previous_srv == (sr_srv_t *) data)) { + /* We own the old SRV, so we need to free it. */ + state_query_del_(SR_STATE_OBJ_PREVSRV, NULL); + sr_state->previous_srv = (sr_srv_t *) data; + } break; case SR_STATE_OBJ_VALID_AFTER: sr_state->valid_after = *((time_t *) data); @@ -897,6 +924,9 @@ state_query_put_(sr_state_object_t obj_type, void *data) static void state_query_del_all_(sr_state_object_t obj_type) { + if (BUG(!sr_state)) + return; + switch (obj_type) { case SR_STATE_OBJ_COMMIT: { @@ -925,6 +955,9 @@ state_query_del_(sr_state_object_t obj_type, void *data) { (void) data; + if (BUG(!sr_state)) + return; + switch (obj_type) { case SR_STATE_OBJ_PREVSRV: tor_free(sr_state->previous_srv); @@ -999,16 +1032,16 @@ state_del_previous_srv(void) state_query(SR_STATE_ACTION_DEL, SR_STATE_OBJ_PREVSRV, NULL, NULL); } -/* Rotate SRV value by freeing the previous value, assigning the current - * value to the previous one and nullifying the current one. */ +/* Rotate SRV value by setting the previous SRV to the current SRV, and + * clearing the current SRV. */ STATIC void state_rotate_srv(void) { /* First delete previous SRV from the state. Object will be freed. */ state_del_previous_srv(); - /* Set previous SRV with the current one. */ - sr_state_set_previous_srv(sr_state_get_current_srv()); - /* Nullify the current srv. */ + /* Set previous SRV to a copy of the current one. */ + sr_state_set_previous_srv(sr_srv_dup(sr_state_get_current_srv())); + /* Free and NULL the current srv. */ sr_state_set_current_srv(NULL); } @@ -1029,7 +1062,9 @@ sr_state_get_phase(void) return *(sr_phase_t *) ptr; } -/* Return the previous SRV value from our state. Value CAN be NULL. */ +/* Return the previous SRV value from our state. Value CAN be NULL. + * The state object owns the SRV, so the calling code should not free the SRV. + * Use sr_srv_dup() if you want to keep a copy of the SRV. */ const sr_srv_t * sr_state_get_previous_srv(void) { @@ -1048,7 +1083,9 @@ sr_state_set_previous_srv(const sr_srv_t *srv) NULL); } -/* Return the current SRV value from our state. Value CAN be NULL. */ +/* Return the current SRV value from our state. Value CAN be NULL. + * The state object owns the SRV, so the calling code should not free the SRV. + * Use sr_srv_dup() if you want to keep a copy of the SRV. */ const sr_srv_t * sr_state_get_current_srv(void) { diff --git a/src/feature/dirauth/voteflags.c b/src/feature/dirauth/voteflags.c index 54c70b989a..4f7593a3e1 100644 --- a/src/feature/dirauth/voteflags.c +++ b/src/feature/dirauth/voteflags.c @@ -95,7 +95,7 @@ real_uptime(const routerinfo_t *router, time_t now) */ static int dirserv_thinks_router_is_unreliable(time_t now, - routerinfo_t *router, + const routerinfo_t *router, int need_uptime, int need_capacity) { if (need_uptime) { @@ -541,7 +541,7 @@ dirserv_set_router_is_running(routerinfo_t *router, time_t now) void set_routerstatus_from_routerinfo(routerstatus_t *rs, node_t *node, - routerinfo_t *ri, + const routerinfo_t *ri, time_t now, int listbadexits) { @@ -593,6 +593,10 @@ set_routerstatus_from_routerinfo(routerstatus_t *rs, rs->or_port = ri->or_port; rs->dir_port = ri->dir_port; rs->is_v2_dir = ri->supports_tunnelled_dir_requests; + + rs->is_staledesc = + (ri->cache_info.published_on + DESC_IS_STALE_INTERVAL) < now; + if (options->AuthDirHasIPv6Connectivity == 1 && !tor_addr_is_null(&ri->ipv6_addr) && node->last_reachable6 >= now - REACHABLE_TIMEOUT) { diff --git a/src/feature/dirauth/voteflags.h b/src/feature/dirauth/voteflags.h index aa7b6ed082..cca6f53746 100644 --- a/src/feature/dirauth/voteflags.h +++ b/src/feature/dirauth/voteflags.h @@ -19,12 +19,16 @@ int running_long_enough_to_decide_unreachable(void); void set_routerstatus_from_routerinfo(routerstatus_t *rs, node_t *node, - routerinfo_t *ri, time_t now, + const routerinfo_t *ri, + time_t now, int listbadexits); void dirserv_compute_performance_thresholds(digestmap_t *omit_as_sybil); #ifdef VOTEFLAGS_PRIVATE +/** Any descriptor older than this age causes the authorities to set the + * StaleDesc flag. */ +#define DESC_IS_STALE_INTERVAL (18*60*60) STATIC void dirserv_set_routerstatus_testing(routerstatus_t *rs); #endif diff --git a/src/feature/dircache/consdiffmgr.c b/src/feature/dircache/consdiffmgr.c index 025361fa60..6b16307e3c 100644 --- a/src/feature/dircache/consdiffmgr.c +++ b/src/feature/dircache/consdiffmgr.c @@ -189,6 +189,7 @@ static consdiff_cfg_t consdiff_cfg = { static int consdiffmgr_ensure_space_for_files(int n); static int consensus_queue_compression_work(const char *consensus, + size_t consensus_len, const networkstatus_t *as_parsed); static int consensus_diff_queue_diff_work(consensus_cache_entry_t *diff_from, consensus_cache_entry_t *diff_to); @@ -509,8 +510,25 @@ get_max_age_to_cache(void) MAX_MAX_AGE_TO_CACHE); } +#ifdef TOR_UNIT_TESTS +/** As consdiffmgr_add_consensus, but requires a nul-terminated input. For + * testing. */ +int +consdiffmgr_add_consensus_nulterm(const char *consensus, + const networkstatus_t *as_parsed) +{ + size_t len = strlen(consensus); + /* make a non-nul-terminated copy so that we can have a better chance + * of catching errors. */ + char *ctmp = tor_memdup(consensus, len); + int r = consdiffmgr_add_consensus(ctmp, len, as_parsed); + tor_free(ctmp); + return r; +} +#endif + /** - * Given a string containing a networkstatus consensus, and the results of + * Given a buffer containing a networkstatus consensus, and the results of * having parsed that consensus, add that consensus to the cache if it is not * already present and not too old. Create new consensus diffs from or to * that consensus as appropriate. @@ -519,6 +537,7 @@ get_max_age_to_cache(void) */ int consdiffmgr_add_consensus(const char *consensus, + size_t consensus_len, const networkstatus_t *as_parsed) { if (BUG(consensus == NULL) || BUG(as_parsed == NULL)) @@ -544,7 +563,7 @@ consdiffmgr_add_consensus(const char *consensus, } /* We don't have it. Add it to the cache. */ - return consensus_queue_compression_work(consensus, as_parsed); + return consensus_queue_compression_work(consensus, consensus_len, as_parsed); } /** @@ -1387,19 +1406,21 @@ typedef struct consensus_diff_worker_job_t { } consensus_diff_worker_job_t; /** Given a consensus_cache_entry_t, check whether it has a label claiming - * that it was compressed. If so, uncompress its contents into <b>out</b> and - * set <b>outlen</b> to hold their size. If not, just copy the body into - * <b>out</b> and set <b>outlen</b> to its length. Return 0 on success, - * -1 on failure. - * - * In all cases, the output is nul-terminated. */ + * that it was compressed. If so, uncompress its contents into *<b>out</b> and + * set <b>outlen</b> to hold their size, and set *<b>owned_out</b> to a pointer + * that the caller will need to free. If not, just set *<b>out</b> and + * <b>outlen</b> to its extent in memory. Return 0 on success, -1 on failure. + **/ STATIC int -uncompress_or_copy(char **out, size_t *outlen, - consensus_cache_entry_t *ent) +uncompress_or_set_ptr(const char **out, size_t *outlen, + char **owned_out, + consensus_cache_entry_t *ent) { const uint8_t *body; size_t bodylen; + *owned_out = NULL; + if (consensus_cache_entry_get_body(ent, &body, &bodylen) < 0) return -1; @@ -1410,8 +1431,17 @@ uncompress_or_copy(char **out, size_t *outlen, if (lv_compression) method = compression_method_get_by_name(lv_compression); - return tor_uncompress(out, outlen, (const char *)body, bodylen, + int rv; + if (method == NO_METHOD) { + *out = (const char *)body; + *outlen = bodylen; + rv = 0; + } else { + rv = tor_uncompress(owned_out, outlen, (const char *)body, bodylen, method, 1, LOG_WARN); + *out = *owned_out; + } + return rv; } /** @@ -1478,16 +1508,17 @@ consensus_diff_worker_threadfn(void *state_, void *work_) char *consensus_diff; { - char *diff_from_nt = NULL, *diff_to_nt = NULL; + const char *diff_from_nt = NULL, *diff_to_nt = NULL; + char *owned1 = NULL, *owned2 = NULL; size_t diff_from_nt_len, diff_to_nt_len; - if (uncompress_or_copy(&diff_from_nt, &diff_from_nt_len, - job->diff_from) < 0) { + if (uncompress_or_set_ptr(&diff_from_nt, &diff_from_nt_len, &owned1, + job->diff_from) < 0) { return WQ_RPL_REPLY; } - if (uncompress_or_copy(&diff_to_nt, &diff_to_nt_len, - job->diff_to) < 0) { - tor_free(diff_from_nt); + if (uncompress_or_set_ptr(&diff_to_nt, &diff_to_nt_len, &owned2, + job->diff_to) < 0) { + tor_free(owned1); return WQ_RPL_REPLY; } tor_assert(diff_from_nt); @@ -1496,9 +1527,12 @@ consensus_diff_worker_threadfn(void *state_, void *work_) // XXXX ugh; this is going to calculate the SHA3 of both its // XXXX inputs again, even though we already have that. Maybe it's time // XXXX to change the API here? - consensus_diff = consensus_diff_generate(diff_from_nt, diff_to_nt); - tor_free(diff_from_nt); - tor_free(diff_to_nt); + consensus_diff = consensus_diff_generate(diff_from_nt, + diff_from_nt_len, + diff_to_nt, + diff_to_nt_len); + tor_free(owned1); + tor_free(owned2); } if (!consensus_diff) { /* Couldn't generate consensus; we'll leave the reply blank. */ @@ -1746,8 +1780,8 @@ consensus_compress_worker_threadfn(void *state_, void *work_) (const uint8_t *)consensus, bodylen); { const char *start, *end; - if (router_get_networkstatus_v3_signed_boundaries(consensus, - &start, &end) < 0) { + if (router_get_networkstatus_v3_signed_boundaries(consensus, bodylen, + &start, &end) < 0) { start = consensus; end = consensus+bodylen; } @@ -1811,14 +1845,15 @@ static int background_compression = 0; */ static int consensus_queue_compression_work(const char *consensus, + size_t consensus_len, const networkstatus_t *as_parsed) { tor_assert(consensus); tor_assert(as_parsed); consensus_compress_worker_job_t *job = tor_malloc_zero(sizeof(*job)); - job->consensus = tor_strdup(consensus); - job->consensus_len = strlen(consensus); + job->consensus = tor_memdup_nulterm(consensus, consensus_len); + job->consensus_len = strlen(job->consensus); job->flavor = as_parsed->flavor; char va_str[ISO_TIME_LEN+1]; diff --git a/src/feature/dircache/consdiffmgr.h b/src/feature/dircache/consdiffmgr.h index 39e8fa31cb..b1b3323b6c 100644 --- a/src/feature/dircache/consdiffmgr.h +++ b/src/feature/dircache/consdiffmgr.h @@ -22,6 +22,7 @@ typedef struct consdiff_cfg_t { struct consensus_cache_entry_t; // from conscache.h int consdiffmgr_add_consensus(const char *consensus, + size_t consensus_len, const networkstatus_t *as_parsed); consdiff_status_t consdiffmgr_find_consensus( @@ -68,8 +69,14 @@ STATIC consensus_cache_entry_t *cdm_cache_lookup_consensus( STATIC int cdm_entry_get_sha3_value(uint8_t *digest_out, consensus_cache_entry_t *ent, const char *label); -STATIC int uncompress_or_copy(char **out, size_t *outlen, - consensus_cache_entry_t *ent); +STATIC int uncompress_or_set_ptr(const char **out, size_t *outlen, + char **owned_out, + consensus_cache_entry_t *ent); #endif /* defined(CONSDIFFMGR_PRIVATE) */ +#ifdef TOR_UNIT_TESTS +int consdiffmgr_add_consensus_nulterm(const char *consensus, + const networkstatus_t *as_parsed); +#endif + #endif /* !defined(TOR_CONSDIFFMGR_H) */ diff --git a/src/feature/dircache/dircache.c b/src/feature/dircache/dircache.c index e8cb284165..eece1e6503 100644 --- a/src/feature/dircache/dircache.c +++ b/src/feature/dircache/dircache.c @@ -49,7 +49,8 @@ #define ROUTERDESC_BY_DIGEST_CACHE_LIFETIME (48*60*60) #define ROBOTS_CACHE_LIFETIME (24*60*60) #define MICRODESC_CACHE_LIFETIME (48*60*60) - +/* Bandwidth files change every hour. */ +#define BANDWIDTH_CACHE_LIFETIME (30*60) /** Parse an HTTP request string <b>headers</b> of the form * \verbatim * "\%s [http[s]://]\%s HTTP/1..." @@ -123,7 +124,7 @@ write_http_response_header_impl(dir_connection_t *conn, ssize_t length, long cache_lifetime) { char date[RFC1123_TIME_LEN+1]; - time_t now = time(NULL); + time_t now = approx_time(); buf_t *buf = buf_new_with_capacity(1024); tor_assert(conn); @@ -166,22 +167,16 @@ write_http_response_header_impl(dir_connection_t *conn, ssize_t length, buf_free(buf); } -/** As write_http_response_header_impl, but sets encoding and content-typed - * based on whether the response will be <b>compressed</b> or not. */ +/** As write_http_response_header_impl, but translates method into + * encoding */ static void write_http_response_headers(dir_connection_t *conn, ssize_t length, compress_method_t method, const char *extra_headers, long cache_lifetime) { - const char *methodname = compression_method_get_name(method); - const char *doctype; - if (method == NO_METHOD) - doctype = "text/plain"; - else - doctype = "application/octet-stream"; write_http_response_header_impl(conn, length, - doctype, - methodname, + "text/plain", + compression_method_get_name(method), extra_headers, cache_lifetime); } @@ -357,12 +352,15 @@ static int handle_get_robots(dir_connection_t *conn, const get_handler_args_t *args); static int handle_get_networkstatus_bridges(dir_connection_t *conn, const get_handler_args_t *args); +static int handle_get_next_bandwidth(dir_connection_t *conn, + const get_handler_args_t *args); /** Table for handling GET requests. */ static const url_table_ent_t url_table[] = { { "/tor/", 0, handle_get_frontpage }, { "/tor/status-vote/current/consensus", 1, handle_get_current_consensus }, { "/tor/status-vote/current/", 1, handle_get_status_vote }, + { "/tor/status-vote/next/bandwidth", 0, handle_get_next_bandwidth }, { "/tor/status-vote/next/", 1, handle_get_status_vote }, { "/tor/micro/d/", 1, handle_get_microdesc }, { "/tor/server/", 1, handle_get_descriptor }, @@ -495,28 +493,47 @@ handle_get_frontpage(dir_connection_t *conn, const get_handler_args_t *args) } /** Warn that the cached consensus <b>consensus</b> of type - * <b>flavor</b> is too old and will not be served to clients. Rate-limit the - * warning to avoid logging an entry on every request. + * <b>flavor</b> too new or too old, based on <b>is_too_new</b>, + * and will not be served to clients. Rate-limit the warning to avoid logging + * an entry on every request. */ static void -warn_consensus_is_too_old(const struct consensus_cache_entry_t *consensus, - const char *flavor, time_t now) +warn_consensus_is_not_reasonably_live( + const struct consensus_cache_entry_t *consensus, + const char *flavor, time_t now, bool is_too_new) { -#define TOO_OLD_WARNING_INTERVAL (60*60) - static ratelim_t warned = RATELIM_INIT(TOO_OLD_WARNING_INTERVAL); +#define NOT_REASONABLY_LIVE_WARNING_INTERVAL (60*60) + static ratelim_t warned[2] = { RATELIM_INIT( + NOT_REASONABLY_LIVE_WARNING_INTERVAL), + RATELIM_INIT( + NOT_REASONABLY_LIVE_WARNING_INTERVAL) }; char timestamp[ISO_TIME_LEN+1]; - time_t valid_until; - char *dupes; - - if (consensus_cache_entry_get_valid_until(consensus, &valid_until)) - return; + /* valid_after if is_too_new, valid_until if !is_too_new */ + time_t valid_time = 0; + char *dupes = NULL; - if ((dupes = rate_limit_log(&warned, now))) { - format_local_iso_time(timestamp, valid_until); - log_warn(LD_DIRSERV, "Our %s%sconsensus is too old, so we will not " - "serve it to clients. It was valid until %s local time and we " - "continued to serve it for up to 24 hours after it expired.%s", - flavor ? flavor : "", flavor ? " " : "", timestamp, dupes); + if (is_too_new) { + if (consensus_cache_entry_get_valid_after(consensus, &valid_time)) + return; + dupes = rate_limit_log(&warned[1], now); + } else { + if (consensus_cache_entry_get_valid_until(consensus, &valid_time)) + return; + dupes = rate_limit_log(&warned[0], now); + } + + if (dupes) { + format_local_iso_time(timestamp, valid_time); + log_warn(LD_DIRSERV, "Our %s%sconsensus is too %s, so we will not " + "serve it to clients. It was valid %s %s local time and we " + "continued to serve it for up to 24 hours %s.%s", + flavor ? flavor : "", + flavor ? " " : "", + is_too_new ? "new" : "old", + is_too_new ? "after" : "until", + timestamp, + is_too_new ? "before it was valid" : "after it expired", + dupes); tor_free(dupes); } } @@ -859,7 +876,6 @@ handle_get_current_consensus(dir_connection_t *conn, if (req.diff_only && !cached_consensus) { write_short_http_response(conn, 404, "No such diff available"); - // XXXX warn_consensus_is_too_old(v, req.flavor, now); geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND); goto done; } @@ -870,19 +886,30 @@ handle_get_current_consensus(dir_connection_t *conn, &compression_used); } - time_t fresh_until, valid_until; - int have_fresh_until = 0, have_valid_until = 0; + time_t valid_after, fresh_until, valid_until; + int have_valid_after = 0, have_fresh_until = 0, have_valid_until = 0; if (cached_consensus) { + have_valid_after = + !consensus_cache_entry_get_valid_after(cached_consensus, &valid_after); have_fresh_until = !consensus_cache_entry_get_fresh_until(cached_consensus, &fresh_until); have_valid_until = !consensus_cache_entry_get_valid_until(cached_consensus, &valid_until); } - if (cached_consensus && have_valid_until && + if (cached_consensus && have_valid_after && + !networkstatus_valid_after_is_reasonably_live(valid_after, now)) { + write_short_http_response(conn, 404, "Consensus is too new"); + warn_consensus_is_not_reasonably_live(cached_consensus, req.flavor, now, + 1); + geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND); + goto done; + } else if ( + cached_consensus && have_valid_until && !networkstatus_valid_until_is_reasonably_live(valid_until, now)) { write_short_http_response(conn, 404, "Consensus is too old"); - warn_consensus_is_too_old(cached_consensus, req.flavor, now); + warn_consensus_is_not_reasonably_live(cached_consensus, req.flavor, now, + 0); geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND); goto done; } @@ -1438,6 +1465,39 @@ handle_get_networkstatus_bridges(dir_connection_t *conn, return 0; } +/** Helper function for GET the bandwidth file used for the next vote */ +static int +handle_get_next_bandwidth(dir_connection_t *conn, + const get_handler_args_t *args) +{ + log_debug(LD_DIR, "Getting next bandwidth."); + const or_options_t *options = get_options(); + const compress_method_t compress_method = + find_best_compression_method(args->compression_supported, 1); + + if (options->V3BandwidthsFile) { + char *bandwidth = read_file_to_str(options->V3BandwidthsFile, + RFTS_IGNORE_MISSING, NULL); + if (bandwidth != NULL) { + ssize_t len = strlen(bandwidth); + write_http_response_header(conn, compress_method != NO_METHOD ? -1 : len, + compress_method, BANDWIDTH_CACHE_LIFETIME); + if (compress_method != NO_METHOD) { + conn->compress_state = tor_compress_new(1, compress_method, + choose_compression_level(len/2)); + log_debug(LD_DIR, "Compressing bandwidth file."); + } else { + log_debug(LD_DIR, "Not compressing bandwidth file."); + } + connection_dir_buf_add((const char*)bandwidth, len, conn, 1); + tor_free(bandwidth); + return 0; + } + } + write_short_http_response(conn, 404, "Not found"); + return 0; +} + /** Helper function for GET robots.txt or /tor/robots.txt */ static int handle_get_robots(dir_connection_t *conn, const get_handler_args_t *args) @@ -1608,8 +1668,8 @@ directory_handle_command_post,(dir_connection_t *conn, const char *headers, const char *msg = "[None]"; uint8_t purpose = authdir_mode_bridge(options) ? ROUTER_PURPOSE_BRIDGE : ROUTER_PURPOSE_GENERAL; - was_router_added_t r = dirserv_add_multiple_descriptors(body, purpose, - conn->base_.address, &msg); + was_router_added_t r = dirserv_add_multiple_descriptors(body, body_len, + purpose, conn->base_.address, &msg); tor_assert(msg); if (r == ROUTER_ADDED_SUCCESSFULLY) { diff --git a/src/feature/dircache/dirserv.c b/src/feature/dircache/dirserv.c index 213c490314..4be6836fe1 100644 --- a/src/feature/dircache/dirserv.c +++ b/src/feature/dircache/dirserv.c @@ -234,6 +234,7 @@ free_cached_dir_(void *_d) * validation is performed. */ void dirserv_set_cached_consensus_networkstatus(const char *networkstatus, + size_t networkstatus_len, const char *flavor_name, const common_digests_t *digests, const uint8_t *sha3_as_signed, @@ -244,7 +245,9 @@ dirserv_set_cached_consensus_networkstatus(const char *networkstatus, if (!cached_consensuses) cached_consensuses = strmap_new(); - new_networkstatus = new_cached_dir(tor_strdup(networkstatus), published); + new_networkstatus = + new_cached_dir(tor_memdup_nulterm(networkstatus, networkstatus_len), + published); memcpy(&new_networkstatus->digests, digests, sizeof(common_digests_t)); memcpy(&new_networkstatus->digest_sha3_as_signed, sha3_as_signed, DIGEST256_LEN); diff --git a/src/feature/dircache/dirserv.h b/src/feature/dircache/dirserv.h index 890b10fd80..7f944459da 100644 --- a/src/feature/dircache/dirserv.h +++ b/src/feature/dircache/dirserv.h @@ -84,6 +84,7 @@ int directory_too_idle_to_fetch_descriptors(const or_options_t *options, cached_dir_t *dirserv_get_consensus(const char *flavor_name); void dirserv_set_cached_consensus_networkstatus(const char *consensus, + size_t consensus_len, const char *flavor_name, const common_digests_t *digests, const uint8_t *sha3_as_signed, diff --git a/src/feature/dirclient/dirclient.c b/src/feature/dirclient/dirclient.c index 0fd1a47017..70b6a20028 100644 --- a/src/feature/dirclient/dirclient.c +++ b/src/feature/dirclient/dirclient.c @@ -2205,24 +2205,31 @@ handle_response_fetch_consensus(dir_connection_t *conn, if (looks_like_a_consensus_diff(body, body_len)) { /* First find our previous consensus. Maybe it's in ram, maybe not. */ cached_dir_t *cd = dirserv_get_consensus(flavname); - const char *consensus_body; - char *owned_consensus = NULL; + const char *consensus_body = NULL; + size_t consensus_body_len; + tor_mmap_t *mapped_consensus = NULL; if (cd) { consensus_body = cd->dir; + consensus_body_len = cd->dir_len; } else { - owned_consensus = networkstatus_read_cached_consensus(flavname); - consensus_body = owned_consensus; + mapped_consensus = networkstatus_map_cached_consensus(flavname); + if (mapped_consensus) { + consensus_body = mapped_consensus->data; + consensus_body_len = mapped_consensus->size; + } } if (!consensus_body) { log_warn(LD_DIR, "Received a consensus diff, but we can't find " "any %s-flavored consensus in our current cache.",flavname); + tor_munmap_file(mapped_consensus); networkstatus_consensus_download_failed(0, flavname); // XXXX if this happens too much, see below return -1; } - new_consensus = consensus_diff_apply(consensus_body, body); - tor_free(owned_consensus); + new_consensus = consensus_diff_apply(consensus_body, consensus_body_len, + body, body_len); + tor_munmap_file(mapped_consensus); if (new_consensus == NULL) { log_warn(LD_DIR, "Could not apply consensus diff received from server " "'%s:%d'", conn->base_.address, conn->base_.port); @@ -2244,7 +2251,9 @@ handle_response_fetch_consensus(dir_connection_t *conn, sourcename = "downloaded"; } - if ((r=networkstatus_set_current_consensus(consensus, flavname, 0, + if ((r=networkstatus_set_current_consensus(consensus, + strlen(consensus), + flavname, 0, conn->identity_digest))<0) { log_fn(r<-1?LOG_WARN:LOG_INFO, LD_DIR, "Unable to load %s consensus directory %s from " diff --git a/src/feature/dircommon/consdiff.c b/src/feature/dircommon/consdiff.c index d0f7594ce3..8e93953f73 100644 --- a/src/feature/dircommon/consdiff.c +++ b/src/feature/dircommon/consdiff.c @@ -101,11 +101,11 @@ smartlist_add_linecpy(smartlist_t *lst, memarea_t *area, const char *s) /* This is a separate, mockable function so that we can override it when * fuzzing. */ MOCK_IMPL(STATIC int, -consensus_compute_digest,(const char *cons, +consensus_compute_digest,(const char *cons, size_t len, consensus_digest_t *digest_out)) { int r = crypto_digest256((char*)digest_out->sha3_256, - cons, strlen(cons), DIGEST_SHA3_256); + cons, len, DIGEST_SHA3_256); return r; } @@ -114,11 +114,11 @@ consensus_compute_digest,(const char *cons, /* This is a separate, mockable function so that we can override it when * fuzzing. */ MOCK_IMPL(STATIC int, -consensus_compute_digest_as_signed,(const char *cons, +consensus_compute_digest_as_signed,(const char *cons, size_t len, consensus_digest_t *digest_out)) { return router_get_networkstatus_v3_sha3_as_signed(digest_out->sha3_256, - cons); + cons, len); } /** Return true iff <b>d1</b> and <b>d2</b> contain the same digest */ @@ -1229,7 +1229,8 @@ consdiff_apply_diff(const smartlist_t *cons1, cons2_str = consensus_join_lines(cons2); consensus_digest_t cons2_digests; - if (consensus_compute_digest(cons2_str, &cons2_digests) < 0) { + if (consensus_compute_digest(cons2_str, strlen(cons2_str), + &cons2_digests) < 0) { /* LCOV_EXCL_START -- digest can't fail */ log_warn(LD_CONSDIFF, "Could not compute digests of the consensus " "resulting from applying a consensus diff."); @@ -1283,12 +1284,13 @@ consdiff_apply_diff(const smartlist_t *cons1, * generated cdlines will become invalid. */ STATIC int -consensus_split_lines(smartlist_t *out, const char *s, memarea_t *area) +consensus_split_lines(smartlist_t *out, + const char *s, size_t len, + memarea_t *area) { - const char *end_of_str = s + strlen(s); - tor_assert(*end_of_str == '\0'); + const char *end_of_str = s + len; - while (*s) { + while (s < end_of_str) { const char *eol = memchr(s, '\n', end_of_str - s); if (!eol) { /* File doesn't end with newline. */ @@ -1334,25 +1336,25 @@ consensus_join_lines(const smartlist_t *inp) * success, retun a newly allocated string containing that diff. On failure, * return NULL. */ char * -consensus_diff_generate(const char *cons1, - const char *cons2) +consensus_diff_generate(const char *cons1, size_t cons1len, + const char *cons2, size_t cons2len) { consensus_digest_t d1, d2; smartlist_t *lines1 = NULL, *lines2 = NULL, *result_lines = NULL; int r1, r2; char *result = NULL; - r1 = consensus_compute_digest_as_signed(cons1, &d1); - r2 = consensus_compute_digest(cons2, &d2); + r1 = consensus_compute_digest_as_signed(cons1, cons1len, &d1); + r2 = consensus_compute_digest(cons2, cons2len, &d2); if (BUG(r1 < 0 || r2 < 0)) return NULL; // LCOV_EXCL_LINE memarea_t *area = memarea_new(); lines1 = smartlist_new(); lines2 = smartlist_new(); - if (consensus_split_lines(lines1, cons1, area) < 0) + if (consensus_split_lines(lines1, cons1, cons1len, area) < 0) goto done; - if (consensus_split_lines(lines2, cons2, area) < 0) + if (consensus_split_lines(lines2, cons2, cons2len, area) < 0) goto done; result_lines = consdiff_gen_diff(lines1, lines2, &d1, &d2, area); @@ -1375,7 +1377,9 @@ consensus_diff_generate(const char *cons1, * consensus. On failure, return NULL. */ char * consensus_diff_apply(const char *consensus, - const char *diff) + size_t consensus_len, + const char *diff, + size_t diff_len) { consensus_digest_t d1; smartlist_t *lines1 = NULL, *lines2 = NULL; @@ -1383,15 +1387,15 @@ consensus_diff_apply(const char *consensus, char *result = NULL; memarea_t *area = memarea_new(); - r1 = consensus_compute_digest_as_signed(consensus, &d1); + r1 = consensus_compute_digest_as_signed(consensus, consensus_len, &d1); if (BUG(r1 < 0)) goto done; lines1 = smartlist_new(); lines2 = smartlist_new(); - if (consensus_split_lines(lines1, consensus, area) < 0) + if (consensus_split_lines(lines1, consensus, consensus_len, area) < 0) goto done; - if (consensus_split_lines(lines2, diff, area) < 0) + if (consensus_split_lines(lines2, diff, diff_len, area) < 0) goto done; result = consdiff_apply_diff(lines1, lines2, &d1); diff --git a/src/feature/dircommon/consdiff.h b/src/feature/dircommon/consdiff.h index 98217e6d46..b63fcb2cc6 100644 --- a/src/feature/dircommon/consdiff.h +++ b/src/feature/dircommon/consdiff.h @@ -7,10 +7,10 @@ #include "core/or/or.h" -char *consensus_diff_generate(const char *cons1, - const char *cons2); -char *consensus_diff_apply(const char *consensus, - const char *diff); +char *consensus_diff_generate(const char *cons1, size_t cons1len, + const char *cons2, size_t cons2len); +char *consensus_diff_apply(const char *consensus, size_t consensus_len, + const char *diff, size_t diff_len); int looks_like_a_consensus_diff(const char *document, size_t len); @@ -78,7 +78,8 @@ STATIC int smartlist_slice_string_pos(const smartlist_slice_t *slice, STATIC void set_changed(bitarray_t *changed1, bitarray_t *changed2, const smartlist_slice_t *slice1, const smartlist_slice_t *slice2); -STATIC int consensus_split_lines(smartlist_t *out, const char *s, +STATIC int consensus_split_lines(smartlist_t *out, + const char *s, size_t len, struct memarea_t *area); STATIC void smartlist_add_linecpy(smartlist_t *lst, struct memarea_t *area, const char *s); @@ -86,10 +87,10 @@ STATIC int lines_eq(const cdline_t *a, const cdline_t *b); STATIC int line_str_eq(const cdline_t *a, const char *b); MOCK_DECL(STATIC int, - consensus_compute_digest,(const char *cons, + consensus_compute_digest,(const char *cons, size_t len, consensus_digest_t *digest_out)); MOCK_DECL(STATIC int, - consensus_compute_digest_as_signed,(const char *cons, + consensus_compute_digest_as_signed,(const char *cons, size_t len, consensus_digest_t *digest_out)); MOCK_DECL(STATIC int, consensus_digest_eq,(const uint8_t *d1, diff --git a/src/feature/dirparse/authcert_parse.c b/src/feature/dirparse/authcert_parse.c index 1680bdbf30..8ba5a53981 100644 --- a/src/feature/dirparse/authcert_parse.c +++ b/src/feature/dirparse/authcert_parse.c @@ -24,7 +24,8 @@ static token_rule_t dir_key_certificate_table[] = { /** Parse a key certificate from <b>s</b>; point <b>end-of-string</b> to * the first character after the certificate. */ authority_cert_t * -authority_cert_parse_from_string(const char *s, const char **end_of_string) +authority_cert_parse_from_string(const char *s, size_t maxlen, + const char **end_of_string) { /** Reject any certificate at least this big; it is probably an overflow, an * attack, a bug, or some other nonsense. */ @@ -35,24 +36,25 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) char digest[DIGEST_LEN]; directory_token_t *tok; char fp_declared[DIGEST_LEN]; - char *eos; + const char *eos; size_t len; int found; memarea_t *area = NULL; + const char *end_of_s = s + maxlen; const char *s_dup = s; - s = eat_whitespace(s); - eos = strstr(s, "\ndir-key-certification"); + s = eat_whitespace_eos(s, end_of_s); + eos = tor_memstr(s, end_of_s - s, "\ndir-key-certification"); if (! eos) { log_warn(LD_DIR, "No signature found on key certificate"); return NULL; } - eos = strstr(eos, "\n-----END SIGNATURE-----\n"); + eos = tor_memstr(eos, end_of_s - eos, "\n-----END SIGNATURE-----\n"); if (! eos) { log_warn(LD_DIR, "No end-of-signature found on key certificate"); return NULL; } - eos = strchr(eos+2, '\n'); + eos = memchr(eos+2, '\n', end_of_s - (eos+2)); tor_assert(eos); ++eos; len = eos - s; @@ -69,7 +71,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) log_warn(LD_DIR, "Error tokenizing key certificate"); goto err; } - if (router_get_hash_impl(s, strlen(s), digest, "dir-key-certificate-version", + if (router_get_hash_impl(s, eos - s, digest, "dir-key-certificate-version", "\ndir-key-certification", '\n', DIGEST_SHA1) < 0) goto err; tok = smartlist_get(tokens, 0); diff --git a/src/feature/dirparse/authcert_parse.h b/src/feature/dirparse/authcert_parse.h index ca475ad0e3..800631c3de 100644 --- a/src/feature/dirparse/authcert_parse.h +++ b/src/feature/dirparse/authcert_parse.h @@ -13,6 +13,7 @@ #define TOR_AUTHCERT_PARSE_H authority_cert_t *authority_cert_parse_from_string(const char *s, + size_t maxlen, const char **end_of_string); #endif /* !defined(TOR_AUTHCERT_PARSE_H) */ diff --git a/src/feature/dirparse/microdesc_parse.c b/src/feature/dirparse/microdesc_parse.c index 5a75af3994..22cc1e272e 100644 --- a/src/feature/dirparse/microdesc_parse.c +++ b/src/feature/dirparse/microdesc_parse.c @@ -18,6 +18,7 @@ #include "feature/dirparse/routerparse.h" #include "feature/nodelist/microdesc.h" #include "feature/nodelist/nickname.h" +#include "feature/nodelist/nodefamily.h" #include "feature/relay/router.h" #include "lib/crypt_ops/crypto_curve25519.h" #include "lib/crypt_ops/crypto_ed25519.h" @@ -32,7 +33,7 @@ static token_rule_t microdesc_token_table[] = { T01("ntor-onion-key", K_ONION_KEY_NTOR, GE(1), NO_OBJ ), T0N("id", K_ID, GE(2), NO_OBJ ), T0N("a", K_A, GE(1), NO_OBJ ), - T01("family", K_FAMILY, ARGS, NO_OBJ ), + T01("family", K_FAMILY, CONCAT_ARGS, NO_OBJ ), T01("p", K_P, CONCAT_ARGS, NO_OBJ ), T01("p6", K_P6, CONCAT_ARGS, NO_OBJ ), A01("@last-listed", A_LAST_LISTED, CONCAT_ARGS, NO_OBJ ), @@ -159,7 +160,22 @@ microdescs_parse_from_string(const char *s, const char *eos, if (tokenize_string(area, s, start_of_next_microdesc, tokens, microdesc_token_table, flags)) { - log_warn(LD_DIR, "Unparseable microdescriptor"); + const char *location; + switch (where) { + case SAVED_NOWHERE: + location = "download or generated string"; + break; + case SAVED_IN_CACHE: + location = "cache"; + break; + case SAVED_IN_JOURNAL: + location = "journal"; + break; + default: + location = "unknown location"; + break; + } + log_warn(LD_DIR, "Unparseable microdescriptor found in %s", location); goto next; } @@ -176,8 +192,8 @@ microdescs_parse_from_string(const char *s, const char *eos, "Relay's onion key had invalid exponent."); goto next; } - router_set_rsa_onion_pkey(tok->key, &md->onion_pkey, - &md->onion_pkey_len); + md->onion_pkey = tor_memdup(tok->object_body, tok->object_size); + md->onion_pkey_len = tok->object_size; crypto_pk_free(tok->key); if ((tok = find_opt_by_keyword(tokens, K_ONION_KEY_NTOR))) { @@ -222,16 +238,9 @@ microdescs_parse_from_string(const char *s, const char *eos, } if ((tok = find_opt_by_keyword(tokens, K_FAMILY))) { - int i; - md->family = smartlist_new(); - for (i=0;i<tok->n_args;++i) { - if (!is_legal_nickname_or_hexdigest(tok->args[i])) { - log_warn(LD_DIR, "Illegal nickname %s in family line", - escaped(tok->args[i])); - goto next; - } - smartlist_add_strdup(md->family, tok->args[i]); - } + md->family = nodefamily_parse(tok->args[0], + NULL, + NF_WARN_MALFORMED); } if ((tok = find_opt_by_keyword(tokens, K_P))) { diff --git a/src/feature/dirparse/ns_parse.c b/src/feature/dirparse/ns_parse.c index 109eebeb66..d653a59826 100644 --- a/src/feature/dirparse/ns_parse.c +++ b/src/feature/dirparse/ns_parse.c @@ -151,10 +151,11 @@ static token_rule_t networkstatus_vote_footer_token_table[] = { * -1. */ int router_get_networkstatus_v3_signed_boundaries(const char *s, + size_t len, const char **start_out, const char **end_out) { - return router_get_hash_impl_helper(s, strlen(s), + return router_get_hash_impl_helper(s, len, "network-status-version", "\ndirectory-signature", ' ', LOG_INFO, @@ -166,12 +167,13 @@ router_get_networkstatus_v3_signed_boundaries(const char *s, * signed portion can be identified. Return 0 on success, -1 on failure. */ int router_get_networkstatus_v3_sha3_as_signed(uint8_t *digest_out, - const char *s) + const char *s, size_t len) { const char *start, *end; - if (router_get_networkstatus_v3_signed_boundaries(s, &start, &end) < 0) { + if (router_get_networkstatus_v3_signed_boundaries(s, len, + &start, &end) < 0) { start = s; - end = s + strlen(s); + end = s + len; } tor_assert(start); tor_assert(end); @@ -182,9 +184,10 @@ router_get_networkstatus_v3_sha3_as_signed(uint8_t *digest_out, /** Set <b>digests</b> to all the digests of the consensus document in * <b>s</b> */ int -router_get_networkstatus_v3_hashes(const char *s, common_digests_t *digests) +router_get_networkstatus_v3_hashes(const char *s, size_t len, + common_digests_t *digests) { - return router_get_hashes_impl(s,strlen(s),digests, + return router_get_hashes_impl(s, len, digests, "network-status-version", "\ndirectory-signature", ' '); @@ -195,13 +198,13 @@ router_get_networkstatus_v3_hashes(const char *s, common_digests_t *digests) * return the start of the directory footer, or the next directory signature. * If none is found, return the end of the string. */ static inline const char * -find_start_of_next_routerstatus(const char *s) +find_start_of_next_routerstatus(const char *s, const char *s_eos) { const char *eos, *footer, *sig; - if ((eos = strstr(s, "\nr "))) + if ((eos = tor_memstr(s, s_eos - s, "\nr "))) ++eos; else - eos = s + strlen(s); + eos = s_eos; footer = tor_memstr(s, eos-s, "\ndirectory-footer"); sig = tor_memstr(s, eos-s, "\ndirectory-signature"); @@ -289,7 +292,8 @@ routerstatus_parse_guardfraction(const char *guardfraction_str, **/ STATIC routerstatus_t * routerstatus_parse_entry_from_string(memarea_t *area, - const char **s, smartlist_t *tokens, + const char **s, const char *s_eos, + smartlist_t *tokens, networkstatus_t *vote, vote_routerstatus_t *vote_rs, int consensus_method, @@ -308,7 +312,7 @@ routerstatus_parse_entry_from_string(memarea_t *area, flav = FLAV_NS; tor_assert(flav == FLAV_NS || flav == FLAV_MICRODESC); - eos = find_start_of_next_routerstatus(*s); + eos = find_start_of_next_routerstatus(*s, s_eos); if (tokenize_string(area,*s, eos, tokens, rtrstatus_token_table,0)) { log_warn(LD_DIR, "Error tokenizing router status"); @@ -430,6 +434,8 @@ routerstatus_parse_entry_from_string(memarea_t *area, rs->is_hs_dir = 1; } else if (!strcmp(tok->args[i], "V2Dir")) { rs->is_v2_dir = 1; + } else if (!strcmp(tok->args[i], "StaleDesc")) { + rs->is_staledesc = 1; } } /* These are implied true by having been included in a consensus made @@ -1051,7 +1057,9 @@ extract_shared_random_srvs(networkstatus_t *ns, smartlist_t *tokens) /** Parse a v3 networkstatus vote, opinion, or consensus (depending on * ns_type), from <b>s</b>, and return the result. Return NULL on failure. */ networkstatus_t * -networkstatus_parse_vote_from_string(const char *s, const char **eos_out, +networkstatus_parse_vote_from_string(const char *s, + size_t s_len, + const char **eos_out, networkstatus_type_t ns_type) { smartlist_t *tokens = smartlist_new(); @@ -1067,20 +1075,22 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, memarea_t *area = NULL, *rs_area = NULL; consensus_flavor_t flav = FLAV_NS; char *last_kwd=NULL; + const char *eos = s + s_len; tor_assert(s); if (eos_out) *eos_out = NULL; - if (router_get_networkstatus_v3_hashes(s, &ns_digests) || - router_get_networkstatus_v3_sha3_as_signed(sha3_as_signed, s)<0) { + if (router_get_networkstatus_v3_hashes(s, s_len, &ns_digests) || + router_get_networkstatus_v3_sha3_as_signed(sha3_as_signed, + s, s_len)<0) { log_warn(LD_DIR, "Unable to compute digest of network-status"); goto err; } area = memarea_new(); - end_of_header = find_start_of_next_routerstatus(s); + end_of_header = find_start_of_next_routerstatus(s, eos); if (tokenize_string(area, s, end_of_header, tokens, (ns_type == NS_TYPE_CONSENSUS) ? networkstatus_consensus_token_table : @@ -1111,10 +1121,12 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, if (ns_type != NS_TYPE_CONSENSUS) { const char *end_of_cert = NULL; - if (!(cert = strstr(s, "\ndir-key-certificate-version"))) + if (!(cert = tor_memstr(s, end_of_header - s, + "\ndir-key-certificate-version"))) goto err; ++cert; - ns->cert = authority_cert_parse_from_string(cert, &end_of_cert); + ns->cert = authority_cert_parse_from_string(cert, end_of_header - cert, + &end_of_cert); if (!ns->cert || !end_of_cert || end_of_cert > end_of_header) goto err; } @@ -1424,10 +1436,10 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, s = end_of_header; ns->routerstatus_list = smartlist_new(); - while (!strcmpstart(s, "r ")) { + while (eos - s >= 2 && fast_memeq(s, "r ", 2)) { if (ns->type != NS_TYPE_CONSENSUS) { vote_routerstatus_t *rs = tor_malloc_zero(sizeof(vote_routerstatus_t)); - if (routerstatus_parse_entry_from_string(rs_area, &s, rs_tokens, ns, + if (routerstatus_parse_entry_from_string(rs_area, &s, eos, rs_tokens, ns, rs, 0, 0)) { smartlist_add(ns->routerstatus_list, rs); } else { @@ -1435,7 +1447,8 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, } } else { routerstatus_t *rs; - if ((rs = routerstatus_parse_entry_from_string(rs_area, &s, rs_tokens, + if ((rs = routerstatus_parse_entry_from_string(rs_area, &s, eos, + rs_tokens, NULL, NULL, ns->consensus_method, flav))) { @@ -1480,10 +1493,10 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, /* Parse footer; check signature. */ footer_tokens = smartlist_new(); - if ((end_of_footer = strstr(s, "\nnetwork-status-version "))) + if ((end_of_footer = tor_memstr(s, eos-s, "\nnetwork-status-version "))) ++end_of_footer; else - end_of_footer = s + strlen(s); + end_of_footer = eos; if (tokenize_string(area,s, end_of_footer, footer_tokens, networkstatus_vote_footer_token_table, 0)) { log_warn(LD_DIR, "Error tokenizing network-status vote footer."); diff --git a/src/feature/dirparse/ns_parse.h b/src/feature/dirparse/ns_parse.h index 10a6f9cefc..dedfa6fc88 100644 --- a/src/feature/dirparse/ns_parse.h +++ b/src/feature/dirparse/ns_parse.h @@ -12,18 +12,19 @@ #ifndef TOR_NS_PARSE_H #define TOR_NS_PARSE_H -int router_get_networkstatus_v3_hashes(const char *s, +int router_get_networkstatus_v3_hashes(const char *s, size_t len, common_digests_t *digests); -int router_get_networkstatus_v3_signed_boundaries(const char *s, +int router_get_networkstatus_v3_signed_boundaries(const char *s, size_t len, const char **start_out, const char **end_out); int router_get_networkstatus_v3_sha3_as_signed(uint8_t *digest_out, - const char *s); + const char *s, size_t len); int compare_vote_routerstatus_entries(const void **_a, const void **_b); int networkstatus_verify_bw_weights(networkstatus_t *ns, int); enum networkstatus_type_t; networkstatus_t *networkstatus_parse_vote_from_string(const char *s, + size_t len, const char **eos_out, enum networkstatus_type_t ns_type); @@ -35,7 +36,8 @@ STATIC int routerstatus_parse_guardfraction(const char *guardfraction_str, struct memarea_t; STATIC routerstatus_t *routerstatus_parse_entry_from_string( struct memarea_t *area, - const char **s, smartlist_t *tokens, + const char **s, const char *eos, + smartlist_t *tokens, networkstatus_t *vote, vote_routerstatus_t *vote_rs, int consensus_method, diff --git a/src/feature/dirparse/parsecommon.c b/src/feature/dirparse/parsecommon.c index 632f5adff0..036a51689c 100644 --- a/src/feature/dirparse/parsecommon.c +++ b/src/feature/dirparse/parsecommon.c @@ -15,6 +15,7 @@ #include "lib/string/printf.h" #include "lib/memarea/memarea.h" #include "lib/crypt_ops/crypto_rsa.h" +#include "lib/ctime/di_ops.h" #include <string.h> @@ -169,7 +170,6 @@ get_token_arguments(memarea_t *area, directory_token_t *tok, char *cp = mem; int j = 0; char *args[MAX_ARGS]; - memset(args, 0, sizeof(args)); while (*cp) { if (j == MAX_ARGS) return -1; @@ -251,6 +251,16 @@ token_check_object(memarea_t *area, const char *kwd, return tok; } +/** Return true iff the <b>memlen</b>-byte chunk of memory at + * <b>memlen</b> is the same length as <b>token</b>, and their + * contents are equal. */ +static bool +mem_eq_token(const void *mem, size_t memlen, const char *token) +{ + size_t len = strlen(token); + return memlen == len && fast_memeq(mem, token, len); +} + /** Helper function: read the next token from *s, advance *s to the end of the * token, and return the parsed token. Parse *<b>s</b> according to the list * of tokens in <b>table</b>. @@ -266,7 +276,7 @@ get_next_token(memarea_t *area, * attack, a bug, or some other nonsense. */ #define MAX_LINE_LENGTH (128*1024) - const char *next, *eol, *obstart; + const char *next, *eol; size_t obname_len; int i; directory_token_t *tok; @@ -290,7 +300,7 @@ get_next_token(memarea_t *area, next = find_whitespace_eos(*s, eol); - if (!strcmp_len(*s, "opt", next-*s)) { + if (mem_eq_token(*s, next-*s, "opt")) { /* Skip past an "opt" at the start of the line. */ *s = eat_whitespace_eos_no_nl(next, eol); next = find_whitespace_eos(*s, eol); @@ -301,7 +311,7 @@ get_next_token(memarea_t *area, /* Search the table for the appropriate entry. (I tried a binary search * instead, but it wasn't any faster.) */ for (i = 0; table[i].t ; ++i) { - if (!strcmp_len(*s, table[i].t, next-*s)) { + if (mem_eq_token(*s, next-*s, table[i].t)) { /* We've found the keyword. */ kwd = table[i].t; tok->tp = table[i].v; @@ -352,9 +362,8 @@ get_next_token(memarea_t *area, if (!eol || eol-*s<11 || strcmpstart(*s, "-----BEGIN ")) /* No object. */ goto check_object; - obstart = *s; /* Set obstart to start of object spec */ if (eol - *s <= 16 || memchr(*s+11,'\0',eol-*s-16) || /* no short lines, */ - strcmp_len(eol-5, "-----", 5) || /* nuls or invalid endings */ + !mem_eq_token(eol-5, 5, "-----") || /* nuls or invalid endings */ (eol-*s) > MAX_UNPARSED_OBJECT_SIZE) { /* name too long */ RET_ERR("Malformed object: bad begin line"); } @@ -373,8 +382,8 @@ get_next_token(memarea_t *area, eol = eos; /* Validate the ending tag, which should be 9 + NAME + 5 + eol */ if ((size_t)(eol-next) != 9+obname_len+5 || - strcmp_len(next+9, tok->object_type, obname_len) || - strcmp_len(eol-5, "-----", 5)) { + !mem_eq_token(next+9, obname_len, tok->object_type) || + !mem_eq_token(eol-5, 5, "-----")) { tor_snprintf(ebuf, sizeof(ebuf), "Malformed object: mismatched end tag %s", tok->object_type); ebuf[sizeof(ebuf)-1] = '\0'; @@ -383,22 +392,26 @@ get_next_token(memarea_t *area, if (next - *s > MAX_UNPARSED_OBJECT_SIZE) RET_ERR("Couldn't parse object: missing footer or object much too big."); - if (!strcmp(tok->object_type, "RSA PUBLIC KEY")) { /* If it's a public key */ - tok->key = crypto_pk_new(); - if (crypto_pk_read_public_key_from_string(tok->key, obstart, eol-obstart)) - RET_ERR("Couldn't parse public key."); - } else if (!strcmp(tok->object_type, "RSA PRIVATE KEY")) { /* private key */ - tok->key = crypto_pk_new(); - if (crypto_pk_read_private_key_from_string(tok->key, obstart, eol-obstart)) - RET_ERR("Couldn't parse private key."); - } else { /* If it's something else, try to base64-decode it */ + { int r; - tok->object_body = ALLOC(next-*s); /* really, this is too much RAM. */ - r = base64_decode(tok->object_body, next-*s, *s, next-*s); + size_t maxsize = base64_decode_maxsize(next-*s); + tok->object_body = ALLOC(maxsize); + r = base64_decode(tok->object_body, maxsize, *s, next-*s); if (r<0) RET_ERR("Malformed object: bad base64-encoded data"); tok->object_size = r; } + + if (!strcmp(tok->object_type, "RSA PUBLIC KEY")) { /* If it's a public key */ + tok->key = crypto_pk_asn1_decode(tok->object_body, tok->object_size); + if (! tok->key) + RET_ERR("Couldn't parse public key."); + } else if (!strcmp(tok->object_type, "RSA PRIVATE KEY")) { /* private key */ + tok->key = crypto_pk_asn1_decode_private(tok->object_body, + tok->object_size); + if (! tok->key) + RET_ERR("Couldn't parse private key."); + } *s = eol; check_object: diff --git a/src/feature/dirparse/routerparse.c b/src/feature/dirparse/routerparse.c index e44fbf77f9..f78c46f186 100644 --- a/src/feature/dirparse/routerparse.c +++ b/src/feature/dirparse/routerparse.c @@ -591,8 +591,8 @@ router_parse_entry_from_string(const char *s, const char *end, "Relay's onion key had invalid exponent."); goto err; } - router_set_rsa_onion_pkey(tok->key, &router->onion_pkey, - &router->onion_pkey_len); + router->onion_pkey = tor_memdup(tok->object_body, tok->object_size); + router->onion_pkey_len = tok->object_size; crypto_pk_free(tok->key); if ((tok = find_opt_by_keyword(tokens, K_ONION_KEY_NTOR))) { diff --git a/src/feature/hibernate/hibernate.c b/src/feature/hibernate/hibernate.c index 09932c97ac..70c2b4f69f 100644 --- a/src/feature/hibernate/hibernate.c +++ b/src/feature/hibernate/hibernate.c @@ -37,6 +37,7 @@ hibernating, phase 2: #include "core/or/connection_or.h" #include "feature/control/control.h" #include "lib/crypt_ops/crypto_rand.h" +#include "lib/defs/time.h" #include "feature/hibernate/hibernate.h" #include "core/mainloop/mainloop.h" #include "feature/relay/router.h" @@ -66,8 +67,9 @@ static hibernate_state_t hibernate_state = HIBERNATE_STATE_INITIAL; /** If are hibernating, when do we plan to wake up? Set to 0 if we * aren't hibernating. */ static time_t hibernate_end_time = 0; -/** If we are shutting down, when do we plan finally exit? Set to 0 if - * we aren't shutting down. */ +/** If we are shutting down, when do we plan finally exit? Set to 0 if we + * aren't shutting down. (This is obsolete; scheduled shutdowns are supposed + * to happen from mainloop_schedule_shutdown() now.) */ static time_t shutdown_time = 0; /** A timed event that we'll use when it's time to wake up from @@ -831,8 +833,6 @@ hibernate_soft_limit_reached(void) return get_accounting_bytes() >= soft_limit; } -#define TOR_USEC_PER_SEC (1000000) - /** Called when we get a SIGINT, or when bandwidth soft limit is * reached. Puts us into "loose hibernation": we don't accept new * connections, but we continue handling old ones. */ @@ -867,7 +867,13 @@ hibernate_begin(hibernate_state_t new_state, time_t now) log_notice(LD_GENERAL,"Interrupt: we have stopped accepting new " "connections, and will shut down in %d seconds. Interrupt " "again to exit now.", options->ShutdownWaitLength); - shutdown_time = time(NULL) + options->ShutdownWaitLength; + /* We add an arbitrary delay here so that even if something goes wrong + * with the mainloop shutdown code, we can still shutdown from + * consider_hibernation() if we call it... but so that the + * mainloop_schedule_shutdown() mechanism will be the first one called. + */ + shutdown_time = time(NULL) + options->ShutdownWaitLength + 5; + mainloop_schedule_shutdown(options->ShutdownWaitLength); #ifdef HAVE_SYSTEMD /* tell systemd that we may need more than the default 90 seconds to shut * down so they don't kill us. add some extra time to actually finish @@ -1096,11 +1102,12 @@ consider_hibernation(time_t now) hibernate_state_t prev_state = hibernate_state; /* If we're in 'exiting' mode, then we just shut down after the interval - * elapses. */ + * elapses. The mainloop was supposed to catch this via + * mainloop_schedule_shutdown(), but apparently it didn't. */ if (hibernate_state == HIBERNATE_STATE_EXITING) { tor_assert(shutdown_time); if (shutdown_time <= now) { - log_notice(LD_GENERAL, "Clean shutdown finished. Exiting."); + log_notice(LD_BUG, "Mainloop did not catch shutdown event; exiting."); tor_shutdown_event_loop_and_exit(0); } return; /* if exiting soon, don't worry about bandwidth limits */ @@ -1112,7 +1119,7 @@ consider_hibernation(time_t now) if (hibernate_end_time > now && accounting_enabled) { /* If we're hibernating, don't wake up until it's time, regardless of * whether we're in a new interval. */ - return ; + return; } else { hibernate_end_time_elapsed(now); } @@ -1240,8 +1247,6 @@ on_hibernate_state_change(hibernate_state_t prev_state) if (prev_state != HIBERNATE_STATE_INITIAL) { rescan_periodic_events(get_options()); } - - reschedule_per_second_timer(); } /** Free all resources held by the accounting module */ diff --git a/src/feature/hs/hs_descriptor.c b/src/feature/hs/hs_descriptor.c index b6abf14a11..b09d50e010 100644 --- a/src/feature/hs/hs_descriptor.c +++ b/src/feature/hs/hs_descriptor.c @@ -1400,6 +1400,50 @@ encrypted_data_length_is_valid(size_t len) return 0; } +/* Build the KEYS component for the authorized client computation. The format + * of the construction is: + * + * SECRET_SEED = x25519(sk, pk) + * KEYS = KDF(subcredential | SECRET_SEED, 40) + * + * Set the <b>keys_out</b> argument to point to the buffer containing the KEYS, + * and return the buffer's length. The caller should wipe and free its content + * once done with it. This function can't fail. */ +static size_t +build_descriptor_cookie_keys(const uint8_t *subcredential, + size_t subcredential_len, + const curve25519_secret_key_t *sk, + const curve25519_public_key_t *pk, + uint8_t **keys_out) +{ + uint8_t secret_seed[CURVE25519_OUTPUT_LEN]; + uint8_t *keystream; + size_t keystream_len = HS_DESC_CLIENT_ID_LEN + HS_DESC_COOKIE_KEY_LEN; + crypto_xof_t *xof; + + tor_assert(subcredential); + tor_assert(sk); + tor_assert(pk); + tor_assert(keys_out); + + keystream = tor_malloc_zero(keystream_len); + + /* Calculate x25519(sk, pk) to get the secret seed. */ + curve25519_handshake(secret_seed, sk, pk); + + /* Calculate KEYS = KDF(subcredential | SECRET_SEED, 40) */ + xof = crypto_xof_new(); + crypto_xof_add_bytes(xof, subcredential, subcredential_len); + crypto_xof_add_bytes(xof, secret_seed, sizeof(secret_seed)); + crypto_xof_squeeze_bytes(xof, keystream, keystream_len); + crypto_xof_free(xof); + + memwipe(secret_seed, 0, sizeof(secret_seed)); + + *keys_out = keystream; + return keystream_len; +} + /* Decrypt the descriptor cookie given the descriptor, the auth client, * and the client secret key. On sucess, return 0 and a newly allocated * descriptor cookie descriptor_cookie_out. On error or if the client id @@ -1412,12 +1456,11 @@ decrypt_descriptor_cookie(const hs_descriptor_t *desc, uint8_t **descriptor_cookie_out) { int ret = -1; - uint8_t secret_seed[CURVE25519_OUTPUT_LEN]; - uint8_t keystream[HS_DESC_CLIENT_ID_LEN + HS_DESC_COOKIE_KEY_LEN]; - uint8_t *cookie_key = NULL; + uint8_t *keystream = NULL; + size_t keystream_length = 0; uint8_t *descriptor_cookie = NULL; + const uint8_t *cookie_key = NULL; crypto_cipher_t *cipher = NULL; - crypto_xof_t *xof = NULL; tor_assert(desc); tor_assert(client); @@ -1429,16 +1472,13 @@ decrypt_descriptor_cookie(const hs_descriptor_t *desc, sizeof(*client_auth_sk))); tor_assert(!tor_mem_is_zero((char *) desc->subcredential, DIGEST256_LEN)); - /* Calculate x25519(client_x, hs_Y) */ - curve25519_handshake(secret_seed, client_auth_sk, - &desc->superencrypted_data.auth_ephemeral_pubkey); - - /* Calculate KEYS = KDF(subcredential | SECRET_SEED, 40) */ - xof = crypto_xof_new(); - crypto_xof_add_bytes(xof, desc->subcredential, DIGEST256_LEN); - crypto_xof_add_bytes(xof, secret_seed, sizeof(secret_seed)); - crypto_xof_squeeze_bytes(xof, keystream, sizeof(keystream)); - crypto_xof_free(xof); + /* Get the KEYS component to derive the CLIENT-ID and COOKIE-KEY. */ + keystream_length = + build_descriptor_cookie_keys(desc->subcredential, DIGEST256_LEN, + client_auth_sk, + &desc->superencrypted_data.auth_ephemeral_pubkey, + &keystream); + tor_assert(keystream_length > 0); /* If the client id of auth client is not the same as the calculcated * client id, it means that this auth client is invaild according to the @@ -1464,8 +1504,8 @@ decrypt_descriptor_cookie(const hs_descriptor_t *desc, if (cipher) { crypto_cipher_free(cipher); } - memwipe(secret_seed, 0, sizeof(secret_seed)); - memwipe(keystream, 0, sizeof(keystream)); + memwipe(keystream, 0, keystream_length); + tor_free(keystream); return ret; } @@ -2878,11 +2918,10 @@ hs_desc_build_authorized_client(const uint8_t *subcredential, const uint8_t *descriptor_cookie, hs_desc_authorized_client_t *client_out) { - uint8_t secret_seed[CURVE25519_OUTPUT_LEN]; - uint8_t keystream[HS_DESC_CLIENT_ID_LEN + HS_DESC_COOKIE_KEY_LEN]; - uint8_t *cookie_key; + uint8_t *keystream = NULL; + size_t keystream_length = 0; + const uint8_t *cookie_key; crypto_cipher_t *cipher; - crypto_xof_t *xof; tor_assert(client_auth_pk); tor_assert(auth_ephemeral_sk); @@ -2898,18 +2937,14 @@ hs_desc_build_authorized_client(const uint8_t *subcredential, tor_assert(!tor_mem_is_zero((char *) subcredential, DIGEST256_LEN)); - /* Calculate x25519(hs_y, client_X) */ - curve25519_handshake(secret_seed, - auth_ephemeral_sk, - client_auth_pk); - - /* Calculate KEYS = KDF(subcredential | SECRET_SEED, 40) */ - xof = crypto_xof_new(); - crypto_xof_add_bytes(xof, subcredential, DIGEST256_LEN); - crypto_xof_add_bytes(xof, secret_seed, sizeof(secret_seed)); - crypto_xof_squeeze_bytes(xof, keystream, sizeof(keystream)); - crypto_xof_free(xof); + /* Get the KEYS part so we can derive the CLIENT-ID and COOKIE-KEY. */ + keystream_length = + build_descriptor_cookie_keys(subcredential, DIGEST256_LEN, + auth_ephemeral_sk, client_auth_pk, + &keystream); + tor_assert(keystream_length > 0); + /* Extract the CLIENT-ID and COOKIE-KEY from the KEYS. */ memcpy(client_out->client_id, keystream, HS_DESC_CLIENT_ID_LEN); cookie_key = keystream + HS_DESC_CLIENT_ID_LEN; @@ -2924,8 +2959,8 @@ hs_desc_build_authorized_client(const uint8_t *subcredential, (const char *) descriptor_cookie, HS_DESC_DESCRIPTOR_COOKIE_LEN); - memwipe(secret_seed, 0, sizeof(secret_seed)); - memwipe(keystream, 0, sizeof(keystream)); + memwipe(keystream, 0, keystream_length); + tor_free(keystream); crypto_cipher_free(cipher); } diff --git a/src/feature/hs/hs_service.c b/src/feature/hs/hs_service.c index 7e150599fc..160fb87397 100644 --- a/src/feature/hs/hs_service.c +++ b/src/feature/hs/hs_service.c @@ -2956,8 +2956,8 @@ set_descriptor_revision_counter(hs_service_descriptor_t *hs_desc, time_t now, /* The OPE module returns CRYPTO_OPE_ERROR in case of errors. */ tor_assert_nonfatal(rev_counter < CRYPTO_OPE_ERROR); - log_info(LD_REND, "Encrypted revision counter %d to %ld", - (int) seconds_since_start_of_srv, (long int) rev_counter); + log_info(LD_REND, "Encrypted revision counter %d to %" PRIu64, + (int) seconds_since_start_of_srv, rev_counter); hs_desc->desc->plaintext_data.revision_counter = rev_counter; } @@ -3694,8 +3694,8 @@ hs_service_lookup_current_desc(const ed25519_public_key_t *pk) } /* Return the number of service we have configured and usable. */ -unsigned int -hs_service_get_num_services(void) +MOCK_IMPL(unsigned int, +hs_service_get_num_services,(void)) { if (hs_service_map == NULL) { return 0; diff --git a/src/feature/hs/hs_service.h b/src/feature/hs/hs_service.h index 5f43233ea1..ec53f2f23b 100644 --- a/src/feature/hs/hs_service.h +++ b/src/feature/hs/hs_service.h @@ -310,7 +310,7 @@ hs_service_t *hs_service_new(const or_options_t *options); void hs_service_free_(hs_service_t *service); #define hs_service_free(s) FREE_AND_NULL(hs_service_t, hs_service_free_, (s)) -unsigned int hs_service_get_num_services(void); +MOCK_DECL(unsigned int, hs_service_get_num_services,(void)); void hs_service_stage_services(const smartlist_t *service_list); int hs_service_load_all_keys(void); int hs_service_get_version_from_key(const hs_service_t *service); diff --git a/src/feature/nodelist/authcert.c b/src/feature/nodelist/authcert.c index 7a065662a7..9fc3b62525 100644 --- a/src/feature/nodelist/authcert.c +++ b/src/feature/nodelist/authcert.c @@ -380,7 +380,8 @@ trusted_dirs_load_certs_from_string(const char *contents, int source, int added_trusted_cert = 0; for (s = contents; *s; s = eos) { - authority_cert_t *cert = authority_cert_parse_from_string(s, &eos); + authority_cert_t *cert = authority_cert_parse_from_string(s, strlen(s), + &eos); cert_list_t *cl; if (!cert) { failure_code = -1; diff --git a/src/feature/nodelist/fmt_routerstatus.c b/src/feature/nodelist/fmt_routerstatus.c index 75cab7a0af..8c9212e05c 100644 --- a/src/feature/nodelist/fmt_routerstatus.c +++ b/src/feature/nodelist/fmt_routerstatus.c @@ -135,7 +135,7 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version, goto done; smartlist_add_asprintf(chunks, - "s%s%s%s%s%s%s%s%s%s%s\n", + "s%s%s%s%s%s%s%s%s%s%s%s\n", /* These must stay in alphabetical order. */ rs->is_authority?" Authority":"", rs->is_bad_exit?" BadExit":"", @@ -145,6 +145,7 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version, rs->is_hs_dir?" HSDir":"", rs->is_flagged_running?" Running":"", rs->is_stable?" Stable":"", + rs->is_staledesc?" StaleDesc":"", rs->is_v2_dir?" V2Dir":"", rs->is_valid?" Valid":""); diff --git a/src/feature/nodelist/microdesc.c b/src/feature/nodelist/microdesc.c index dafaabb5e5..36922561a0 100644 --- a/src/feature/nodelist/microdesc.c +++ b/src/feature/nodelist/microdesc.c @@ -23,6 +23,7 @@ #include "feature/nodelist/dirlist.h" #include "feature/nodelist/microdesc.h" #include "feature/nodelist/networkstatus.h" +#include "feature/nodelist/nodefamily.h" #include "feature/nodelist/nodelist.h" #include "feature/nodelist/routerlist.h" #include "feature/relay/router.h" @@ -69,6 +70,8 @@ struct microdesc_cache_t { }; static microdesc_cache_t *get_microdesc_cache_noload(void); +static void warn_if_nul_found(const char *inp, size_t len, int64_t offset, + const char *activity); /** Helper: computes a hash of <b>md</b> to place it in a hash table. */ static inline unsigned int @@ -110,8 +113,9 @@ microdesc_note_outdated_dirserver(const char *relay_digest) /* If we have a reasonably live consensus, then most of our dirservers should * still be caching all the microdescriptors in it. Reasonably live - * consensuses are up to a day old. But microdescriptors expire 7 days after - * the last consensus that referenced them. */ + * consensuses are up to a day old (or a day in the future). But + * microdescriptors expire 7 days after the last consensus that referenced + * them. */ if (!networkstatus_get_reasonably_live_consensus(approx_time(), FLAV_MICRODESC)) { return; @@ -221,6 +225,8 @@ dump_microdescriptor(int fd, microdesc_t *md, size_t *annotation_len_out) } md->off = tor_fd_getpos(fd); + warn_if_nul_found(md->body, md->bodylen, (int64_t) md->off, + "dumping a microdescriptor"); written = write_all_to_fd(fd, md->body, md->bodylen); if (written != (ssize_t)md->bodylen) { written = written < 0 ? 0 : written; @@ -480,6 +486,27 @@ microdesc_cache_clear(microdesc_cache_t *cache) cache->bytes_dropped = 0; } +static void +warn_if_nul_found(const char *inp, size_t len, int64_t offset, + const char *activity) +{ + const char *nul_found = memchr(inp, 0, len); + if (BUG(nul_found)) { + log_warn(LD_BUG, "Found unexpected NUL while %s, offset %"PRId64 + "at position %"TOR_PRIuSZ"/%"TOR_PRIuSZ".", + activity, offset, (nul_found - inp), len); + const char *start_excerpt_at, *eos = inp + len; + if ((nul_found - inp) >= 16) + start_excerpt_at = nul_found - 16; + else + start_excerpt_at = inp; + size_t excerpt_len = MIN(32, eos - start_excerpt_at); + char tmp[65]; + base16_encode(tmp, sizeof(tmp), start_excerpt_at, excerpt_len); + log_warn(LD_BUG, " surrounding string: %s", tmp); + } +} + /** Reload the contents of <b>cache</b> from disk. If it is empty, load it * for the first time. Return 0 on success, -1 on failure. */ int @@ -497,6 +524,7 @@ microdesc_cache_reload(microdesc_cache_t *cache) mm = cache->cache_content = tor_mmap_file(cache->cache_fname); if (mm) { + warn_if_nul_found(mm->data, mm->size, 0, "scanning microdesc cache"); added = microdescs_add_to_cache(cache, mm->data, mm->data+mm->size, SAVED_IN_CACHE, 0, -1, NULL); if (added) { @@ -509,6 +537,8 @@ microdesc_cache_reload(microdesc_cache_t *cache) RFTS_IGNORE_MISSING, &st); if (journal_content) { cache->journal_len = (size_t) st.st_size; + warn_if_nul_found(journal_content, cache->journal_len, 0, + "reading microdesc journal"); added = microdescs_add_to_cache(cache, journal_content, journal_content+st.st_size, SAVED_IN_JOURNAL, 0, -1, NULL); @@ -544,8 +574,8 @@ microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force) size_t bytes_dropped = 0; time_t now = time(NULL); - /* If we don't know a live consensus, don't believe last_listed values: we - * might be starting up after being down for a while. */ + /* If we don't know a reasonably live consensus, don't believe last_listed + * values: we might be starting up after being down for a while. */ if (! force && ! networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC)) return; @@ -884,10 +914,7 @@ microdesc_free_(microdesc_t *md, const char *fname, int lineno) if (md->body && md->saved_location != SAVED_IN_CACHE) tor_free(md->body); - if (md->family) { - SMARTLIST_FOREACH(md->family, char *, cp, tor_free(cp)); - smartlist_free(md->family); - } + nodefamily_free(md->family); short_policy_free(md->exit_policy); short_policy_free(md->ipv6_exit_policy); @@ -973,6 +1000,7 @@ update_microdesc_downloads(time_t now) if (directory_too_idle_to_fetch_descriptors(options, now)) return; + /* Give up if we don't have a reasonably live consensus. */ consensus = networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC); if (!consensus) return; diff --git a/src/feature/nodelist/microdesc_st.h b/src/feature/nodelist/microdesc_st.h index bb8b23d664..367e6a3ef6 100644 --- a/src/feature/nodelist/microdesc_st.h +++ b/src/feature/nodelist/microdesc_st.h @@ -9,6 +9,7 @@ struct curve25519_public_key_t; struct ed25519_public_key_t; +struct nodefamily_t; struct short_policy_t; /** A microdescriptor is the smallest amount of information needed to build a @@ -69,8 +70,8 @@ struct microdesc_t { tor_addr_t ipv6_addr; /** As routerinfo_t.ipv6_orport */ uint16_t ipv6_orport; - /** As routerinfo_t.family */ - smartlist_t *family; + /** As routerinfo_t.family, with readable members parsed. */ + struct nodefamily_t *family; /** IPv4 exit policy summary */ struct short_policy_t *exit_policy; /** IPv6 exit policy summary */ diff --git a/src/feature/nodelist/networkstatus.c b/src/feature/nodelist/networkstatus.c index c74acd8b74..24e3b212f0 100644 --- a/src/feature/nodelist/networkstatus.c +++ b/src/feature/nodelist/networkstatus.c @@ -44,6 +44,7 @@ #include "core/mainloop/netstatus.h" #include "core/or/channel.h" #include "core/or/channelpadding.h" +#include "core/or/circuitpadding.h" #include "core/or/circuitmux.h" #include "core/or/circuitmux_ewma.h" #include "core/or/circuitstats.h" @@ -116,8 +117,6 @@ STATIC networkstatus_t *current_md_consensus = NULL; typedef struct consensus_waiting_for_certs_t { /** The consensus itself. */ networkstatus_t *consensus; - /** The encoded version of the consensus, nul-terminated. */ - char *body; /** When did we set the current value of consensus_waiting_for_certs? If * this is too recent, we shouldn't try to fetch a new consensus for a * little while, to give ourselves time to get certificates for this one. */ @@ -179,6 +178,10 @@ static void update_consensus_bootstrap_multiple_downloads( static int networkstatus_check_required_protocols(const networkstatus_t *ns, int client_mode, char **warning_out); +static int reload_consensus_from_file(const char *fname, + const char *flavor, + unsigned flags, + const char *source_dir); /** Forget that we've warned about anything networkstatus-related, so we will * give fresh warnings if the same behavior happens again. */ @@ -210,14 +213,11 @@ networkstatus_reset_download_failures(void) download_status_reset(&consensus_bootstrap_dl_status[i]); } -/** - * Read and and return the cached consensus of type <b>flavorname</b>. If - * <b>unverified</b> is true, get the one we haven't verified. Return NULL if - * the file isn't there. */ +/** Return the filename used to cache the consensus of a given flavor */ static char * -networkstatus_read_cached_consensus_impl(int flav, - const char *flavorname, - int unverified_consensus) +networkstatus_get_cache_fname(int flav, + const char *flavorname, + int unverified_consensus) { char buf[128]; const char *prefix; @@ -232,21 +232,35 @@ networkstatus_read_cached_consensus_impl(int flav, tor_snprintf(buf, sizeof(buf), "%s-%s-consensus", prefix, flavorname); } - char *filename = get_cachedir_fname(buf); - char *result = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL); + return get_cachedir_fname(buf); +} + +/** + * Read and and return the cached consensus of type <b>flavorname</b>. If + * <b>unverified</b> is false, get the one we haven't verified. Return NULL if + * the file isn't there. */ +static tor_mmap_t * +networkstatus_map_cached_consensus_impl(int flav, + const char *flavorname, + int unverified_consensus) +{ + char *filename = networkstatus_get_cache_fname(flav, + flavorname, + unverified_consensus); + tor_mmap_t *result = tor_mmap_file(filename); tor_free(filename); return result; } -/** Return a new string containing the current cached consensus of flavor - * <b>flavorname</b>. */ -char * -networkstatus_read_cached_consensus(const char *flavorname) - { +/** Map the file containing the current cached consensus of flavor + * <b>flavorname</b> */ +tor_mmap_t * +networkstatus_map_cached_consensus(const char *flavorname) +{ int flav = networkstatus_parse_flavor_name(flavorname); if (flav < 0) return NULL; - return networkstatus_read_cached_consensus_impl(flav, flavorname, 0); + return networkstatus_map_cached_consensus_impl(flav, flavorname, 0); } /** Read every cached v3 consensus networkstatus from the disk. */ @@ -259,25 +273,15 @@ router_reload_consensus_networkstatus(void) /* FFFF Suppress warnings if cached consensus is bad? */ for (flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) { const char *flavor = networkstatus_get_flavor_name(flav); - char *s = networkstatus_read_cached_consensus_impl(flav, flavor, 0); - if (s) { - if (networkstatus_set_current_consensus(s, flavor, flags, NULL) < -1) { - log_warn(LD_FS, "Couldn't load consensus %s networkstatus from cache", - flavor); - } - tor_free(s); - } + char *fname = networkstatus_get_cache_fname(flav, flavor, 0); + reload_consensus_from_file(fname, flavor, flags, NULL); + tor_free(fname); - s = networkstatus_read_cached_consensus_impl(flav, flavor, 1); - if (s) { - if (networkstatus_set_current_consensus(s, flavor, - flags | NSSET_WAS_WAITING_FOR_CERTS, - NULL)) { - log_info(LD_FS, "Couldn't load unverified consensus %s networkstatus " - "from cache", flavor); - } - tor_free(s); - } + fname = networkstatus_get_cache_fname(flav, flavor, 1); + reload_consensus_from_file(fname, flavor, + flags | NSSET_WAS_WAITING_FOR_CERTS, + NULL); + tor_free(fname); } update_certificate_downloads(time(NULL)); @@ -713,8 +717,8 @@ networkstatus_vote_find_mutable_entry(networkstatus_t *ns, const char *digest) /** Return the entry in <b>ns</b> for the identity digest <b>digest</b>, or * NULL if none was found. */ -const routerstatus_t * -networkstatus_vote_find_entry(networkstatus_t *ns, const char *digest) +MOCK_IMPL(const routerstatus_t *, +networkstatus_vote_find_entry,(networkstatus_t *ns, const char *digest)) { return networkstatus_vote_find_mutable_entry(ns, digest); } @@ -1377,7 +1381,7 @@ networkstatus_get_dl_status_by_flavor_running,(consensus_flavor_t flavor)) } /** Return the most recent consensus that we have downloaded, or NULL if we - * don't have one. */ + * don't have one. May return future or expired consensuses. */ MOCK_IMPL(networkstatus_t *, networkstatus_get_latest_consensus,(void)) { @@ -1388,7 +1392,7 @@ networkstatus_get_latest_consensus,(void)) } /** Return the latest consensus we have whose flavor matches <b>f</b>, or NULL - * if we don't have one. */ + * if we don't have one. May return future or expired consensuses. */ MOCK_IMPL(networkstatus_t *, networkstatus_get_latest_consensus_by_flavor,(consensus_flavor_t f)) { @@ -1422,10 +1426,11 @@ networkstatus_is_live(const networkstatus_t *ns, time_t now) return (ns->valid_after <= now && now <= ns->valid_until); } -/** Determine if <b>consensus</b> is valid or expired recently enough that - * we can still use it. +/** Determine if <b>consensus</b> is valid, or expired recently enough, or not + * too far in the future, so that we can still use it. * - * Return 1 if the consensus is reasonably live, or 0 if it is too old. + * Return 1 if the consensus is reasonably live, or 0 if it is too old or + * too new. */ int networkstatus_consensus_reasonably_live(const networkstatus_t *consensus, @@ -1434,29 +1439,42 @@ networkstatus_consensus_reasonably_live(const networkstatus_t *consensus, if (BUG(!consensus)) return 0; - return networkstatus_valid_until_is_reasonably_live(consensus->valid_until, + return networkstatus_valid_after_is_reasonably_live(consensus->valid_after, + now) && + networkstatus_valid_until_is_reasonably_live(consensus->valid_until, now); } +#define REASONABLY_LIVE_TIME (24*60*60) + +/** As networkstatus_consensus_reasonably_live, but takes a valid_after + * time, and checks to see if it is in the past, or not too far in the future. + */ +int +networkstatus_valid_after_is_reasonably_live(time_t valid_after, + time_t now) +{ + return (now >= valid_after - REASONABLY_LIVE_TIME); +} + /** As networkstatus_consensus_reasonably_live, but takes a valid_until - * time rather than an entire consensus. */ + * time, and checks to see if it is in the future, or not too far in the past. + */ int networkstatus_valid_until_is_reasonably_live(time_t valid_until, time_t now) { -#define REASONABLY_LIVE_TIME (24*60*60) return (now <= valid_until + REASONABLY_LIVE_TIME); } /** As networkstatus_get_live_consensus(), but is way more tolerant of expired - * consensuses. */ + * and future consensuses. */ MOCK_IMPL(networkstatus_t *, networkstatus_get_reasonably_live_consensus,(time_t now, int flavor)) { networkstatus_t *consensus = networkstatus_get_latest_consensus_by_flavor(flavor); if (consensus && - consensus->valid_after <= now && networkstatus_consensus_reasonably_live(consensus, now)) return consensus; else @@ -1725,6 +1743,44 @@ networkstatus_set_current_consensus_from_ns(networkstatus_t *c, #endif /* defined(TOR_UNIT_TESTS) */ /** + * Helper: Read the current consensus of type <b>flavor</b> from + * <b>fname</b>. Flags and return values are as for + * networkstatus_set_current_consensus(). + **/ +static int +reload_consensus_from_file(const char *fname, + const char *flavor, + unsigned flags, + const char *source_dir) +{ + tor_mmap_t *map = tor_mmap_file(fname); + if (!map) + return 0; + + int rv = networkstatus_set_current_consensus(map->data, map->size, + flavor, flags, source_dir); +#ifdef _WIN32 + if (rv < 0 && tor_memstr(map->data, map->size, "\r\n")) { + log_notice(LD_GENERAL, "Looks like the above failures are probably " + "because of a CRLF in consensus file %s; falling back to " + "read_file_to_string. Nothing to worry about: this file " + "was probably saved by an earlier version of Tor.", + escaped(fname)); + char *content = read_file_to_str(fname, RFTS_IGNORE_MISSING, NULL); + rv = networkstatus_set_current_consensus(content, strlen(content), + flavor, flags, source_dir); + tor_free(content); + } +#endif + if (rv < -1) { + log_warn(LD_GENERAL, "Couldn't set consensus from cache file %s", + escaped(fname)); + } + tor_munmap_file(map); + return rv; +} + +/** * Helper for handle_missing_protocol_warning: handles either the * client case (if <b>is_client</b> is set) or the server case otherwise. */ @@ -1841,6 +1897,7 @@ warn_early_consensus(const networkstatus_t *c, const char *flavor, */ int networkstatus_set_current_consensus(const char *consensus, + size_t consensus_len, const char *flavor, unsigned flags, const char *source_dir) @@ -1869,7 +1926,9 @@ networkstatus_set_current_consensus(const char *consensus, } /* Make sure it's parseable. */ - c = networkstatus_parse_vote_from_string(consensus, NULL, NS_TYPE_CONSENSUS); + c = networkstatus_parse_vote_from_string(consensus, + consensus_len, + NULL, NS_TYPE_CONSENSUS); if (!c) { log_warn(LD_DIR, "Unable to parse networkstatus consensus"); result = -2; @@ -1957,14 +2016,12 @@ networkstatus_set_current_consensus(const char *consensus, c->valid_after > current_valid_after) { waiting = &consensus_waiting_for_certs[flav]; networkstatus_vote_free(waiting->consensus); - tor_free(waiting->body); waiting->consensus = c; free_consensus = 0; - waiting->body = tor_strdup(consensus); waiting->set_at = now; waiting->dl_failed = 0; if (!from_cache) { - write_str_to_file(unverified_fname, consensus, 0); + write_bytes_to_file(unverified_fname, consensus, consensus_len, 1); } if (dl_certs) authority_certs_fetch_missing(c, now, source_dir); @@ -1976,9 +2033,9 @@ networkstatus_set_current_consensus(const char *consensus, * latest consensus. */ if (was_waiting_for_certs && from_cache) if (unlink(unverified_fname) != 0) { - log_warn(LD_FS, - "Failed to unlink %s: %s", - unverified_fname, strerror(errno)); + log_debug(LD_FS, + "Failed to unlink %s: %s", + unverified_fname, strerror(errno)); } } goto done; @@ -1991,9 +2048,9 @@ networkstatus_set_current_consensus(const char *consensus, } if (was_waiting_for_certs && (r < -1) && from_cache) { if (unlink(unverified_fname) != 0) { - log_warn(LD_FS, - "Failed to unlink %s: %s", - unverified_fname, strerror(errno)); + log_debug(LD_FS, + "Failed to unlink %s: %s", + unverified_fname, strerror(errno)); } } goto done; @@ -2055,16 +2112,12 @@ networkstatus_set_current_consensus(const char *consensus, waiting->consensus->valid_after <= c->valid_after) { networkstatus_vote_free(waiting->consensus); waiting->consensus = NULL; - if (consensus != waiting->body) - tor_free(waiting->body); - else - waiting->body = NULL; waiting->set_at = 0; waiting->dl_failed = 0; if (unlink(unverified_fname) != 0) { - log_warn(LD_FS, - "Failed to unlink %s: %s", - unverified_fname, strerror(errno)); + log_debug(LD_FS, + "Failed to unlink %s: %s", + unverified_fname, strerror(errno)); } } @@ -2082,7 +2135,6 @@ networkstatus_set_current_consensus(const char *consensus, nodelist_set_consensus(c); - /* XXXXNM Microdescs: needs a non-ns variant. ???? NM*/ update_consensus_networkstatus_fetch_time(now); /* Change the cell EWMA settings */ @@ -2095,6 +2147,7 @@ networkstatus_set_current_consensus(const char *consensus, circuit_build_times_new_consensus_params( get_circuit_build_times_mutable(), c); channelpadding_new_consensus_params(c); + circpad_new_consensus_params(c); } /* Reset the failure count only if this consensus is actually valid. */ @@ -2108,17 +2161,18 @@ networkstatus_set_current_consensus(const char *consensus, if (we_want_to_fetch_flavor(options, flav)) { if (dir_server_mode(get_options())) { dirserv_set_cached_consensus_networkstatus(consensus, + consensus_len, flavor, &c->digests, c->digest_sha3_as_signed, c->valid_after); - consdiffmgr_add_consensus(consensus, c); + consdiffmgr_add_consensus(consensus, consensus_len, c); } } if (!from_cache) { - write_str_to_file(consensus_fname, consensus, 0); + write_bytes_to_file(consensus_fname, consensus, consensus_len, 1); } warn_early_consensus(c, flavor, now); @@ -2154,14 +2208,10 @@ networkstatus_note_certs_arrived(const char *source_dir) if (!waiting->consensus) continue; if (networkstatus_check_consensus_signature(waiting->consensus, 0)>=0) { - char *waiting_body = waiting->body; - if (!networkstatus_set_current_consensus( - waiting_body, - flavor_name, - NSSET_WAS_WAITING_FOR_CERTS, - source_dir)) { - tor_free(waiting_body); - } + char *fname = networkstatus_get_cache_fname(i, flavor_name, 1); + reload_consensus_from_file(fname, flavor_name, + NSSET_WAS_WAITING_FOR_CERTS, source_dir); + tor_free(fname); } } } @@ -2389,7 +2439,9 @@ networkstatus_dump_bridge_status_to_file(time_t now) published, thresholds, fingerprint_line ? fingerprint_line : "", status); fname = get_datadir_fname("networkstatus-bridges"); - write_str_to_file(fname,published_thresholds_and_status,0); + if (write_str_to_file(fname,published_thresholds_and_status,0)<0) { + log_warn(LD_DIRSERV, "Unable to write networkstatus-bridges file."); + } tor_free(thresholds); tor_free(published_thresholds_and_status); tor_free(fname); @@ -2668,6 +2720,9 @@ networkstatus_check_required_protocols(const networkstatus_t *ns, const char *required, *recommended; char *missing = NULL; + const bool consensus_postdates_this_release = + ns->valid_after >= tor_get_approx_release_date(); + tor_assert(warning_out); if (client_mode) { @@ -2685,7 +2740,7 @@ networkstatus_check_required_protocols(const networkstatus_t *ns, "%s on the Tor network. The missing protocols are: %s", func, missing); tor_free(missing); - return 1; + return consensus_postdates_this_release ? 1 : 0; } if (! protover_all_supported(recommended, &missing)) { @@ -2718,6 +2773,5 @@ networkstatus_free_all(void) networkstatus_vote_free(waiting->consensus); waiting->consensus = NULL; } - tor_free(waiting->body); } } diff --git a/src/feature/nodelist/networkstatus.h b/src/feature/nodelist/networkstatus.h index 9e7b0f1bb0..8269fc6182 100644 --- a/src/feature/nodelist/networkstatus.h +++ b/src/feature/nodelist/networkstatus.h @@ -16,7 +16,7 @@ void networkstatus_reset_warnings(void); void networkstatus_reset_download_failures(void); -char *networkstatus_read_cached_consensus(const char *flavorname); +tor_mmap_t *networkstatus_map_cached_consensus(const char *flavorname); int router_reload_consensus_networkstatus(void); void routerstatus_free_(routerstatus_t *rs); #define routerstatus_free(rs) \ @@ -40,8 +40,9 @@ int compare_digest_to_routerstatus_entry(const void *_key, const void **_member); int compare_digest_to_vote_routerstatus_entry(const void *_key, const void **_member); -const routerstatus_t *networkstatus_vote_find_entry(networkstatus_t *ns, - const char *digest); +MOCK_DECL(const routerstatus_t *,networkstatus_vote_find_entry,( + networkstatus_t *ns, + const char *digest)); routerstatus_t *networkstatus_vote_find_mutable_entry(networkstatus_t *ns, const char *digest); int networkstatus_vote_find_entry_idx(networkstatus_t *ns, @@ -87,6 +88,8 @@ MOCK_DECL(networkstatus_t *, networkstatus_get_live_consensus,(time_t now)); int networkstatus_is_live(const networkstatus_t *ns, time_t now); int networkstatus_consensus_reasonably_live(const networkstatus_t *consensus, time_t now); +int networkstatus_valid_after_is_reasonably_live(time_t valid_after, + time_t now); int networkstatus_valid_until_is_reasonably_live(time_t valid_until, time_t now); MOCK_DECL(networkstatus_t *,networkstatus_get_reasonably_live_consensus, @@ -106,6 +109,7 @@ int networkstatus_consensus_has_ipv6(const or_options_t* options); #define NSSET_ACCEPT_OBSOLETE 8 #define NSSET_REQUIRE_FLAVOR 16 int networkstatus_set_current_consensus(const char *consensus, + size_t consensus_len, const char *flavor, unsigned flags, const char *source_dir); @@ -157,4 +161,3 @@ extern networkstatus_t *current_md_consensus; #endif /* defined(NETWORKSTATUS_PRIVATE) */ #endif /* !defined(TOR_NETWORKSTATUS_H) */ - diff --git a/src/feature/nodelist/networkstatus_st.h b/src/feature/nodelist/networkstatus_st.h index 6160f12361..5c1eea3259 100644 --- a/src/feature/nodelist/networkstatus_st.h +++ b/src/feature/nodelist/networkstatus_st.h @@ -99,6 +99,9 @@ struct networkstatus_t { /** List of key=value strings from the headers of the bandwidth list file */ smartlist_t *bw_file_headers; + + /** A SHA256 digest of the bandwidth file used in a vote. */ + uint8_t bw_file_digest256[DIGEST256_LEN]; }; #endif diff --git a/src/feature/nodelist/nodefamily.c b/src/feature/nodelist/nodefamily.c new file mode 100644 index 0000000000..2ec9d5fa40 --- /dev/null +++ b/src/feature/nodelist/nodefamily.c @@ -0,0 +1,416 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file nodefamily.c + * \brief Code to manipulate encoded, reference-counted node families. We + * use these tricks to save space, since these families would otherwise + * require a large number of tiny allocations. + **/ + +#include "core/or/or.h" +#include "feature/nodelist/nickname.h" +#include "feature/nodelist/nodefamily.h" +#include "feature/nodelist/nodefamily_st.h" +#include "feature/nodelist/nodelist.h" +#include "feature/relay/router.h" +#include "feature/nodelist/routerlist.h" + +#include "ht.h" +#include "siphash.h" + +#include "lib/container/smartlist.h" +#include "lib/ctime/di_ops.h" +#include "lib/defs/digest_sizes.h" +#include "lib/log/util_bug.h" + +#include <stdlib.h> +#include <string.h> + +/** + * Allocate and return a blank node family with space to hold <b>n_members</b> + * members. + */ +static nodefamily_t * +nodefamily_alloc(int n_members) +{ + size_t alloc_len = offsetof(nodefamily_t, family_members) + + NODEFAMILY_ARRAY_SIZE(n_members); + nodefamily_t *nf = tor_malloc_zero(alloc_len); + nf->n_members = n_members; + return nf; +} + +/** + * Hashtable hash implementation. + */ +static inline unsigned int +nodefamily_hash(const nodefamily_t *nf) +{ + return (unsigned) siphash24g(nf->family_members, + NODEFAMILY_ARRAY_SIZE(nf->n_members)); +} + +/** + * Hashtable equality implementation. + */ +static inline unsigned int +nodefamily_eq(const nodefamily_t *a, const nodefamily_t *b) +{ + return (a->n_members == b->n_members) && + fast_memeq(a->family_members, b->family_members, + NODEFAMILY_ARRAY_SIZE(a->n_members)); +} + +static HT_HEAD(nodefamily_map, nodefamily_t) the_node_families + = HT_INITIALIZER(); + +HT_PROTOTYPE(nodefamily_map, nodefamily_t, ht_ent, nodefamily_hash, + nodefamily_eq) +HT_GENERATE2(nodefamily_map, nodefamily_t, ht_ent, nodefamily_hash, + node_family_eq, 0.6, tor_reallocarray_, tor_free_) + +/** + * Parse the family declaration in <b>s</b>, returning the canonical + * <b>nodefamily_t</b> for its members. Return NULL on error. + * + * If <b>rsa_id_self</b> is provided, it is a DIGEST_LEN-byte digest + * for the router that declared this family: insert it into the + * family declaration if it is not there already. + * + * If NF_WARN_MALFORMED is set in <b>flags</b>, warn about any + * elements that we can't parse. (By default, we log at info.) + * + * If NF_REJECT_MALFORMED is set in <b>flags</b>, treat any unparseable + * elements as an error. (By default, we simply omit them.) + **/ +nodefamily_t * +nodefamily_parse(const char *s, const uint8_t *rsa_id_self, + unsigned flags) +{ + smartlist_t *sl = smartlist_new(); + smartlist_split_string(sl, s, NULL, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + nodefamily_t *result = nodefamily_from_members(sl, rsa_id_self, flags, NULL); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_free(sl); + return result; +} + +/** + * Canonicalize the family list <b>s</b>, returning a newly allocated string. + * + * The canonicalization rules are fully specified in dir-spec.txt, but, + * briefly: $hexid entries are put in caps, $hexid[=~]foo entries are + * truncated, nicknames are put into lowercase, unrecognized entries are left + * alone, and everything is sorted. + **/ +char * +nodefamily_canonicalize(const char *s, const uint8_t *rsa_id_self, + unsigned flags) +{ + smartlist_t *sl = smartlist_new(); + smartlist_t *result_members = smartlist_new(); + smartlist_split_string(sl, s, NULL, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + nodefamily_t *nf = nodefamily_from_members(sl, rsa_id_self, flags, + result_members); + + char *formatted = nodefamily_format(nf); + smartlist_split_string(result_members, formatted, NULL, + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + smartlist_sort_strings(result_members); + char *combined = smartlist_join_strings(result_members, " ", 0, NULL); + + nodefamily_free(nf); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_free(sl); + SMARTLIST_FOREACH(result_members, char *, cp, tor_free(cp)); + smartlist_free(result_members); + tor_free(formatted); + + return combined; +} + +/** + * qsort helper for encoded nodefamily elements. + **/ +static int +compare_members(const void *a, const void *b) +{ + return fast_memcmp(a, b, NODEFAMILY_MEMBER_LEN); +} + +/** + * Parse the member strings in <b>members</b>, returning their canonical + * <b>nodefamily_t</b>. Return NULL on error. + * + * If <b>rsa_id_self</b> is provided, it is a DIGEST_LEN-byte digest + * for the router that declared this family: insert it into the + * family declaration if it is not there already. + * + * The <b>flags</b> element is interpreted as in nodefamily_parse(). + * + * If <b>unrecognized</b> is provided, fill it copies of any unrecognized + * members. (Note that malformed $hexids are not considered unrecognized.) + **/ +nodefamily_t * +nodefamily_from_members(const smartlist_t *members, + const uint8_t *rsa_id_self, + unsigned flags, + smartlist_t *unrecognized_out) +{ + const int n_self = rsa_id_self ? 1 : 0; + int n_bad_elements = 0; + int n_members = smartlist_len(members) + n_self; + nodefamily_t *tmp = nodefamily_alloc(n_members); + uint8_t *ptr = NODEFAMILY_MEMBER_PTR(tmp, 0); + + SMARTLIST_FOREACH_BEGIN(members, const char *, cp) { + bool bad_element = true; + if (is_legal_nickname(cp)) { + ptr[0] = NODEFAMILY_BY_NICKNAME; + tor_assert(strlen(cp) < DIGEST_LEN); // guaranteed by is_legal_nickname + memcpy(ptr+1, cp, strlen(cp)); + tor_strlower((char*) ptr+1); + bad_element = false; + } else if (is_legal_hexdigest(cp)) { + char digest_buf[DIGEST_LEN]; + char nn_buf[MAX_NICKNAME_LEN+1]; + char nn_char=0; + if (hex_digest_nickname_decode(cp, digest_buf, &nn_char, nn_buf)==0) { + bad_element = false; + ptr[0] = NODEFAMILY_BY_RSA_ID; + memcpy(ptr+1, digest_buf, DIGEST_LEN); + } + } else { + if (unrecognized_out) + smartlist_add_strdup(unrecognized_out, cp); + } + + if (bad_element) { + const int severity = (flags & NF_WARN_MALFORMED) ? LOG_WARN : LOG_INFO; + log_fn(severity, LD_GENERAL, + "Bad element %s while parsing a node family.", + escaped(cp)); + ++n_bad_elements; + } else { + ptr += NODEFAMILY_MEMBER_LEN; + } + } SMARTLIST_FOREACH_END(cp); + + if (n_bad_elements && (flags & NF_REJECT_MALFORMED)) + goto err; + + if (rsa_id_self) { + /* Add self. */ + ptr[0] = NODEFAMILY_BY_RSA_ID; + memcpy(ptr+1, rsa_id_self, DIGEST_LEN); + } + + n_members -= n_bad_elements; + + /* Sort tmp into canonical order. */ + qsort(tmp->family_members, n_members, NODEFAMILY_MEMBER_LEN, + compare_members); + + /* Remove duplicates. */ + int i; + for (i = 0; i < n_members-1; ++i) { + uint8_t *thisptr = NODEFAMILY_MEMBER_PTR(tmp, i); + uint8_t *nextptr = NODEFAMILY_MEMBER_PTR(tmp, i+1); + if (fast_memeq(thisptr, nextptr, NODEFAMILY_MEMBER_LEN)) { + memmove(thisptr, nextptr, (n_members-i-1)*NODEFAMILY_MEMBER_LEN); + --n_members; + --i; + } + } + int n_members_alloc = tmp->n_members; + tmp->n_members = n_members; + + /* See if we already allocated this family. */ + nodefamily_t *found = HT_FIND(nodefamily_map, &the_node_families, tmp); + if (found) { + /* If we did, great: incref it and return it. */ + ++found->refcnt; + tor_free(tmp); + return found; + } else { + /* If not, insert it into the hashtable. */ + if (n_members_alloc != n_members) { + /* Compact the family if needed */ + nodefamily_t *tmp2 = nodefamily_alloc(n_members); + memcpy(tmp2->family_members, tmp->family_members, + n_members * NODEFAMILY_MEMBER_LEN); + tor_free(tmp); + tmp = tmp2; + } + + tmp->refcnt = 1; + HT_INSERT(nodefamily_map, &the_node_families, tmp); + return tmp; + } + + err: + tor_free(tmp); + return NULL; +} + +/** + * Drop our reference to <b>family</b>, freeing it if there are no more + * references. + */ +void +nodefamily_free_(nodefamily_t *family) +{ + if (family == NULL) + return; + + --family->refcnt; + + if (family->refcnt == 0) { + HT_REMOVE(nodefamily_map, &the_node_families, family); + tor_free(family); + } +} + +/** + * Return true iff <b>family</b> contains the SHA1 RSA1024 identity + * <b>rsa_id</b>. + */ +bool +nodefamily_contains_rsa_id(const nodefamily_t *family, + const uint8_t *rsa_id) +{ + if (family == NULL) + return false; + + unsigned i; + for (i = 0; i < family->n_members; ++i) { + const uint8_t *ptr = NODEFAMILY_MEMBER_PTR(family, i); + if (ptr[0] == NODEFAMILY_BY_RSA_ID && + fast_memeq(ptr+1, rsa_id, DIGEST_LEN)) { + return true; + } + } + return false; +} + +/** + * Return true iff <b>family</b> contains the nickname <b>name</b>. + */ +bool +nodefamily_contains_nickname(const nodefamily_t *family, + const char *name) +{ + if (family == NULL) + return false; + + unsigned i; + for (i = 0; i < family->n_members; ++i) { + const uint8_t *ptr = NODEFAMILY_MEMBER_PTR(family, i); + // note that the strcasecmp() is safe because there is always at least one + // NUL in the encoded nickname, because all legal nicknames are less than + // DIGEST_LEN bytes long. + if (ptr[0] == NODEFAMILY_BY_NICKNAME && !strcasecmp((char*)ptr+1, name)) { + return true; + } + } + return false; +} + +/** + * Return true if <b>family</b> contains the nickname or the RSA ID for + * <b>node</b> + **/ +bool +nodefamily_contains_node(const nodefamily_t *family, + const node_t *node) +{ + return + nodefamily_contains_nickname(family, node_get_nickname(node)) + || + nodefamily_contains_rsa_id(family, node_get_rsa_id_digest(node)); +} + +/** + * Look up every entry in <b>family</b>, and add add the corresponding + * node_t to <b>out</b>. + **/ +void +nodefamily_add_nodes_to_smartlist(const nodefamily_t *family, + smartlist_t *out) +{ + if (!family) + return; + + unsigned i; + for (i = 0; i < family->n_members; ++i) { + const uint8_t *ptr = NODEFAMILY_MEMBER_PTR(family, i); + const node_t *node = NULL; + switch (ptr[0]) { + case NODEFAMILY_BY_NICKNAME: + node = node_get_by_nickname((char*)ptr+1, NNF_NO_WARN_UNNAMED); + break; + case NODEFAMILY_BY_RSA_ID: + node = node_get_by_id((char*)ptr+1); + break; + default: + /* LCOV_EXCL_START */ + tor_assert_nonfatal_unreached(); + break; + /* LCOV_EXCL_STOP */ + } + if (node) + smartlist_add(out, (void *)node); + } +} + +/** + * Encode <b>family</b> as a space-separated string. + */ +char * +nodefamily_format(const nodefamily_t *family) +{ + if (!family) + return tor_strdup(""); + + unsigned i; + smartlist_t *sl = smartlist_new(); + for (i = 0; i < family->n_members; ++i) { + const uint8_t *ptr = NODEFAMILY_MEMBER_PTR(family, i); + switch (ptr[0]) { + case NODEFAMILY_BY_NICKNAME: + smartlist_add_strdup(sl, (char*)ptr+1); + break; + case NODEFAMILY_BY_RSA_ID: { + char buf[HEX_DIGEST_LEN+2]; + buf[0]='$'; + base16_encode(buf+1, sizeof(buf)-1, (char*)ptr+1, DIGEST_LEN); + tor_strupper(buf); + smartlist_add_strdup(sl, buf); + break; + } + default: + /* LCOV_EXCL_START */ + tor_assert_nonfatal_unreached(); + break; + /* LCOV_EXCL_STOP */ + } + } + + char *result = smartlist_join_strings(sl, " ", 0, NULL); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_free(sl); + return result; +} + +/** + * Free all storage held in the nodefamily map. + **/ +void +nodefamily_free_all(void) +{ + HT_CLEAR(nodefamily_map, &the_node_families); +} diff --git a/src/feature/nodelist/nodefamily.h b/src/feature/nodelist/nodefamily.h new file mode 100644 index 0000000000..bc5dafce03 --- /dev/null +++ b/src/feature/nodelist/nodefamily.h @@ -0,0 +1,50 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file nodefamily.h + * \brief Header file for nodefamily.c. + **/ + +#ifndef TOR_NODEFAMILY_H +#define TOR_NODEFAMILY_H + +#include "lib/malloc/malloc.h" +#include <stdbool.h> + +typedef struct nodefamily_t nodefamily_t; +struct node_t; +struct smartlist_t; + +#define NF_WARN_MALFORMED (1u<<0) +#define NF_REJECT_MALFORMED (1u<<1) + +nodefamily_t *nodefamily_parse(const char *s, + const uint8_t *rsa_id_self, + unsigned flags); +nodefamily_t *nodefamily_from_members(const struct smartlist_t *members, + const uint8_t *rsa_id_self, + unsigned flags, + smartlist_t *unrecognized_out); +void nodefamily_free_(nodefamily_t *family); +#define nodefamily_free(family) \ + FREE_AND_NULL(nodefamily_t, nodefamily_free_, (family)) + +bool nodefamily_contains_rsa_id(const nodefamily_t *family, + const uint8_t *rsa_id); +bool nodefamily_contains_nickname(const nodefamily_t *family, + const char *name); +bool nodefamily_contains_node(const nodefamily_t *family, + const struct node_t *node); +void nodefamily_add_nodes_to_smartlist(const nodefamily_t *family, + struct smartlist_t *out); +char *nodefamily_format(const nodefamily_t *family); +char *nodefamily_canonicalize(const char *s, const uint8_t *rsa_id_self, + unsigned flags); + +void nodefamily_free_all(void); + +#endif diff --git a/src/feature/nodelist/nodefamily_st.h b/src/feature/nodelist/nodefamily_st.h new file mode 100644 index 0000000000..be533da824 --- /dev/null +++ b/src/feature/nodelist/nodefamily_st.h @@ -0,0 +1,48 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_NODEFAMILY_ST_H +#define TOR_NODEFAMILY_ST_H + +#include "orconfig.h" +#include "ht.h" + +struct nodefamily_t { + /** Entry for this nodefamily_t within the hashtable. */ + HT_ENTRY(nodefamily_t) ht_ent; + /** Reference count. (The hashtable is not treated as a reference */ + uint32_t refcnt; + /** Number of items encoded in <b>family_members</b>. */ + uint32_t n_members; + /* A byte-array encoding the members of this family. We encode each member + * as one byte to indicate whether it's a nickname or a fingerprint, plus + * DIGEST_LEN bytes of data. The entries are lexically sorted. + */ + uint8_t family_members[FLEXIBLE_ARRAY_MEMBER]; +}; + +#define NODEFAMILY_MEMBER_LEN (1+DIGEST_LEN) + +/** Tag byte, indicates that the following bytes are a RSA1024 SHA1 ID. + */ +#define NODEFAMILY_BY_RSA_ID 0 +/** Tag byte, indicates that the following bytes are a NUL-padded nickname. + */ +#define NODEFAMILY_BY_NICKNAME 1 + +/** + * Number of bytes to allocate in the array for a nodefamily_t with N members. + **/ +#define NODEFAMILY_ARRAY_SIZE(n) \ + ((n) * NODEFAMILY_MEMBER_LEN) + +/** + * Pointer to the i'th member of <b>nf</b>, as encoded. + */ +#define NODEFAMILY_MEMBER_PTR(nf, i) \ + (&((nf)->family_members[(i) * NODEFAMILY_MEMBER_LEN])) + +#endif diff --git a/src/feature/nodelist/nodelist.c b/src/feature/nodelist/nodelist.c index 99d7f746a8..8b02dd9c66 100644 --- a/src/feature/nodelist/nodelist.c +++ b/src/feature/nodelist/nodelist.c @@ -59,6 +59,7 @@ #include "feature/nodelist/microdesc.h" #include "feature/nodelist/networkstatus.h" #include "feature/nodelist/node_select.h" +#include "feature/nodelist/nodefamily.h" #include "feature/nodelist/nodelist.h" #include "feature/nodelist/routerlist.h" #include "feature/nodelist/routerset.h" @@ -1105,7 +1106,7 @@ node_ed25519_id_matches(const node_t *node, const ed25519_public_key_t *id) /** Dummy object that should be unreturnable. Used to ensure that * node_get_protover_summary_flags() always returns non-NULL. */ static const protover_summary_flags_t zero_protover_flags = { - 0,0,0,0,0,0,0 + 0,0,0,0,0,0,0,0 }; /** Return the protover_summary_flags for a given node. */ @@ -1503,19 +1504,6 @@ node_is_me(const node_t *node) return router_digest_is_me(node->identity); } -/** Return <b>node</b> declared family (as a list of names), or NULL if - * the node didn't declare a family. */ -const smartlist_t * -node_get_declared_family(const node_t *node) -{ - if (node->ri && node->ri->declared_family) - return node->ri->declared_family; - else if (node->md && node->md->family) - return node->md->family; - else - return NULL; -} - /* Does this node have a valid IPv6 address? * Prefer node_has_ipv6_orport() or node_has_ipv6_dirport() for * checking specific ports. */ @@ -1866,6 +1854,9 @@ int addrs_in_same_network_family(const tor_addr_t *a1, const tor_addr_t *a2) { + if (tor_addr_is_null(a1) || tor_addr_is_null(a2)) + return 0; + switch (tor_addr_family(a1)) { case AF_INET: return 0 == tor_addr_compare_masked(a1, a2, 16, CMP_SEMANTIC); @@ -1881,7 +1872,7 @@ addrs_in_same_network_family(const tor_addr_t *a1, * (case-insensitive), or if <b>node's</b> identity key digest * matches a hexadecimal value stored in <b>nickname</b>. Return * false otherwise. */ -static int +STATIC int node_nickname_matches(const node_t *node, const char *nickname) { const char *n = node_get_nickname(node); @@ -1893,7 +1884,7 @@ node_nickname_matches(const node_t *node, const char *nickname) } /** Return true iff <b>node</b> is named by some nickname in <b>lst</b>. */ -static inline int +STATIC int node_in_nickname_smartlist(const smartlist_t *lst, const node_t *node) { if (!lst) return 0; @@ -1904,6 +1895,61 @@ node_in_nickname_smartlist(const smartlist_t *lst, const node_t *node) return 0; } +/** Return true iff n1's declared family contains n2. */ +STATIC int +node_family_contains(const node_t *n1, const node_t *n2) +{ + if (n1->ri && n1->ri->declared_family) { + return node_in_nickname_smartlist(n1->ri->declared_family, n2); + } else if (n1->md) { + return nodefamily_contains_node(n1->md->family, n2); + } else { + return 0; + } +} + +/** + * Return true iff <b>node</b> has declared a nonempty family. + **/ +STATIC bool +node_has_declared_family(const node_t *node) +{ + if (node->ri && node->ri->declared_family && + smartlist_len(node->ri->declared_family)) { + return true; + } + + if (node->md && node->md->family) { + return true; + } + + return false; +} + +/** + * Add to <b>out</b> every node_t that is listed by <b>node</b> as being in + * its family. (Note that these nodes are not in node's family unless they + * also agree that node is in their family.) + **/ +STATIC void +node_lookup_declared_family(smartlist_t *out, const node_t *node) +{ + if (node->ri && node->ri->declared_family && + smartlist_len(node->ri->declared_family)) { + SMARTLIST_FOREACH_BEGIN(node->ri->declared_family, const char *, name) { + const node_t *n2 = node_get_by_nickname(name, NNF_NO_WARN_UNNAMED); + if (n2) { + smartlist_add(out, (node_t *)n2); + } + } SMARTLIST_FOREACH_END(name); + return; + } + + if (node->md && node->md->family) { + nodefamily_add_nodes_to_smartlist(node->md->family, out); + } +} + /** Return true iff r1 and r2 are in the same family, but not the same * router. */ int @@ -1916,19 +1962,20 @@ nodes_in_same_family(const node_t *node1, const node_t *node2) tor_addr_t a1, a2; node_get_addr(node1, &a1); node_get_addr(node2, &a2); - if (addrs_in_same_network_family(&a1, &a2)) + + tor_addr_port_t ap6_1, ap6_2; + node_get_pref_ipv6_orport(node1, &ap6_1); + node_get_pref_ipv6_orport(node2, &ap6_2); + + if (addrs_in_same_network_family(&a1, &a2) || + addrs_in_same_network_family(&ap6_1.addr, &ap6_2.addr)) return 1; } /* Are they in the same family because the agree they are? */ - { - const smartlist_t *f1, *f2; - f1 = node_get_declared_family(node1); - f2 = node_get_declared_family(node2); - if (f1 && f2 && - node_in_nickname_smartlist(f1, node2) && - node_in_nickname_smartlist(f2, node1)) - return 1; + if (node_family_contains(node1, node2) && + node_family_contains(node2, node1)) { + return 1; } /* Are they in the same family because the user says they are? */ @@ -1956,13 +2003,10 @@ void nodelist_add_node_and_family(smartlist_t *sl, const node_t *node) { const smartlist_t *all_nodes = nodelist_get_list(); - const smartlist_t *declared_family; const or_options_t *options = get_options(); tor_assert(node); - declared_family = node_get_declared_family(node); - /* Let's make sure that we have the node itself, if it's a real node. */ { const node_t *real_node = node_get_by_id(node->identity); @@ -1973,35 +2017,35 @@ nodelist_add_node_and_family(smartlist_t *sl, const node_t *node) /* First, add any nodes with similar network addresses. */ if (options->EnforceDistinctSubnets) { tor_addr_t node_addr; + tor_addr_port_t node_ap6; node_get_addr(node, &node_addr); + node_get_pref_ipv6_orport(node, &node_ap6); SMARTLIST_FOREACH_BEGIN(all_nodes, const node_t *, node2) { tor_addr_t a; + tor_addr_port_t ap6; node_get_addr(node2, &a); - if (addrs_in_same_network_family(&a, &node_addr)) + node_get_pref_ipv6_orport(node2, &ap6); + if (addrs_in_same_network_family(&a, &node_addr) || + addrs_in_same_network_family(&ap6.addr, &node_ap6.addr)) smartlist_add(sl, (void*)node2); } SMARTLIST_FOREACH_END(node2); } - /* Now, add all nodes in the declared_family of this node, if they + /* Now, add all nodes in the declared family of this node, if they * also declare this node to be in their family. */ - if (declared_family) { + if (node_has_declared_family(node)) { + smartlist_t *declared_family = smartlist_new(); + node_lookup_declared_family(declared_family, node); + /* Add every r such that router declares familyness with node, and node * declares familyhood with router. */ - SMARTLIST_FOREACH_BEGIN(declared_family, const char *, name) { - const node_t *node2; - const smartlist_t *family2; - if (!(node2 = node_get_by_nickname(name, NNF_NO_WARN_UNNAMED))) - continue; - if (!(family2 = node_get_declared_family(node2))) - continue; - SMARTLIST_FOREACH_BEGIN(family2, const char *, name2) { - if (node_nickname_matches(node, name2)) { - smartlist_add(sl, (void*)node2); - break; - } - } SMARTLIST_FOREACH_END(name2); - } SMARTLIST_FOREACH_END(name); + SMARTLIST_FOREACH_BEGIN(declared_family, const node_t *, node2) { + if (node_family_contains(node2, node)) { + smartlist_add(sl, (void*)node2); + } + } SMARTLIST_FOREACH_END(node2); + smartlist_free(declared_family); } /* If the user declared any families locally, honor those too. */ @@ -2306,7 +2350,7 @@ compute_frac_paths_available(const networkstatus_t *consensus, const int authdir = authdir_mode_v3(options); count_usable_descriptors(num_present_out, num_usable_out, - mid, consensus, now, NULL, + mid, consensus, now, options->MiddleNodes, USABLE_DESCRIPTOR_ALL); log_debug(LD_NET, "%s: %d present, %d usable", @@ -2508,7 +2552,7 @@ count_loading_descriptors_progress(void) if (fraction > 1.0) return 0; /* it's not the number of descriptors holding us back */ return BOOTSTRAP_STATUS_LOADING_DESCRIPTORS + (int) - (fraction*(BOOTSTRAP_STATUS_CONN_OR-1 - + (fraction*(BOOTSTRAP_STATUS_ENOUGH_DIRINFO-1 - BOOTSTRAP_STATUS_LOADING_DESCRIPTORS)); } @@ -2595,7 +2639,7 @@ update_router_have_minimum_dir_info(void) /* If paths have just become available in this update. */ if (res && !have_min_dir_info) { control_event_client_status(LOG_NOTICE, "ENOUGH_DIR_INFO"); - control_event_boot_dir(BOOTSTRAP_STATUS_CONN_OR, 0); + control_event_boot_dir(BOOTSTRAP_STATUS_ENOUGH_DIRINFO, 0); log_info(LD_DIR, "We now have enough directory information to build circuits."); } diff --git a/src/feature/nodelist/nodelist.h b/src/feature/nodelist/nodelist.h index c430f497d5..3420959618 100644 --- a/src/feature/nodelist/nodelist.h +++ b/src/feature/nodelist/nodelist.h @@ -68,7 +68,6 @@ const char *node_get_platform(const node_t *node); uint32_t node_get_prim_addr_ipv4h(const node_t *node); void node_get_address_string(const node_t *node, char *cp, size_t len); long node_get_declared_uptime(const node_t *node); -const smartlist_t *node_get_declared_family(const node_t *node); const struct ed25519_public_key_t *node_get_ed25519_id(const node_t *node); int node_ed25519_id_matches(const node_t *node, const struct ed25519_public_key_t *id); @@ -155,10 +154,16 @@ int count_loading_descriptors_progress(void); #ifdef NODELIST_PRIVATE +STATIC int node_nickname_matches(const node_t *node, const char *nickname); +STATIC int node_in_nickname_smartlist(const smartlist_t *lst, + const node_t *node); +STATIC int node_family_contains(const node_t *n1, const node_t *n2); +STATIC bool node_has_declared_family(const node_t *node); +STATIC void node_lookup_declared_family(smartlist_t *out, const node_t *node); + #ifdef TOR_UNIT_TESTS -STATIC void -node_set_hsdir_index(node_t *node, const networkstatus_t *ns); +STATIC void node_set_hsdir_index(node_t *node, const networkstatus_t *ns); #endif /* defined(TOR_UNIT_TESTS) */ diff --git a/src/feature/nodelist/routerlist.c b/src/feature/nodelist/routerlist.c index e48675aada..b33dca67e3 100644 --- a/src/feature/nodelist/routerlist.c +++ b/src/feature/nodelist/routerlist.c @@ -160,7 +160,7 @@ static time_t last_descriptor_download_attempted = 0; * * From time to time, we replace "cached-descriptors" with a new file * containing only the live, non-superseded descriptors, and clear - * cached-routers.new. + * cached-descriptors.new. * * On startup, we read both files. */ @@ -3223,6 +3223,8 @@ refresh_all_country_info(void) routerset_refresh_countries(options->EntryNodes); if (options->ExitNodes) routerset_refresh_countries(options->ExitNodes); + if (options->MiddleNodes) + routerset_refresh_countries(options->MiddleNodes); if (options->ExcludeNodes) routerset_refresh_countries(options->ExcludeNodes); if (options->ExcludeExitNodes) diff --git a/src/feature/nodelist/routerstatus_st.h b/src/feature/nodelist/routerstatus_st.h index 288edf5982..8d91b45e11 100644 --- a/src/feature/nodelist/routerstatus_st.h +++ b/src/feature/nodelist/routerstatus_st.h @@ -47,6 +47,8 @@ struct routerstatus_t { unsigned int is_v2_dir:1; /** True iff this router publishes an open DirPort * or it claims to accept tunnelled dir requests. */ + unsigned int is_staledesc:1; /** True iff the authorities think this router + * should upload a new descriptor soon. */ unsigned int has_bandwidth:1; /**< The vote/consensus had bw info */ unsigned int has_exitsummary:1; /**< The vote/consensus had exit summaries */ diff --git a/src/feature/relay/dns.c b/src/feature/relay/dns.c index cc9f4cf490..feaf18e764 100644 --- a/src/feature/relay/dns.c +++ b/src/feature/relay/dns.c @@ -1360,6 +1360,42 @@ evdns_err_is_transient(int err) } } +/** + * Return number of configured nameservers in <b>the_evdns_base</b>. + */ +size_t +number_of_configured_nameservers(void) +{ + return evdns_base_count_nameservers(the_evdns_base); +} + +#ifdef HAVE_EVDNS_BASE_GET_NAMESERVER_ADDR +/** + * Return address of configured nameserver in <b>the_evdns_base</b> + * at index <b>idx</b>. + */ +tor_addr_t * +configured_nameserver_address(const size_t idx) +{ + struct sockaddr_storage sa; + ev_socklen_t sa_len = sizeof(sa); + + if (evdns_base_get_nameserver_addr(the_evdns_base, (int)idx, + (struct sockaddr *)&sa, + sa_len) > 0) { + tor_addr_t *tor_addr = tor_malloc(sizeof(tor_addr_t)); + if (tor_addr_from_sockaddr(tor_addr, + (const struct sockaddr *)&sa, + NULL) == 0) { + return tor_addr; + } + tor_free(tor_addr); + } + + return NULL; +} +#endif + /** Configure eventdns nameservers if force is true, or if the configuration * has changed since the last time we called this function, or if we failed on * our last attempt. On Unix, this reads from /etc/resolv.conf or @@ -1391,16 +1427,23 @@ configure_nameservers(int force) evdns_set_log_fn(evdns_log_cb); if (conf_fname) { log_debug(LD_FS, "stat()ing %s", conf_fname); - if (stat(sandbox_intern_string(conf_fname), &st)) { + int missing_resolv_conf = 0; + int stat_res = stat(sandbox_intern_string(conf_fname), &st); + + if (stat_res) { log_warn(LD_EXIT, "Unable to stat resolver configuration in '%s': %s", conf_fname, strerror(errno)); - goto err; - } - if (!force && resolv_conf_fname && !strcmp(conf_fname,resolv_conf_fname) + missing_resolv_conf = 1; + } else if (!force && resolv_conf_fname && + !strcmp(conf_fname,resolv_conf_fname) && st.st_mtime == resolv_conf_mtime) { log_info(LD_EXIT, "No change to '%s'", conf_fname); return 0; } + + if (stat_res == 0 && st.st_size == 0) + missing_resolv_conf = 1; + if (nameservers_configured) { evdns_base_search_clear(the_evdns_base); evdns_base_clear_nameservers_and_suspend(the_evdns_base); @@ -1413,20 +1456,34 @@ configure_nameservers(int force) sandbox_intern_string("/etc/hosts")); } #endif /* defined(DNS_OPTION_HOSTSFILE) && defined(USE_LIBSECCOMP) */ - log_info(LD_EXIT, "Parsing resolver configuration in '%s'", conf_fname); - if ((r = evdns_base_resolv_conf_parse(the_evdns_base, flags, - sandbox_intern_string(conf_fname)))) { - log_warn(LD_EXIT, "Unable to parse '%s', or no nameservers in '%s' (%d)", - conf_fname, conf_fname, r); - goto err; - } - if (evdns_base_count_nameservers(the_evdns_base) == 0) { - log_warn(LD_EXIT, "Unable to find any nameservers in '%s'.", conf_fname); - goto err; + + if (!missing_resolv_conf) { + log_info(LD_EXIT, "Parsing resolver configuration in '%s'", conf_fname); + if ((r = evdns_base_resolv_conf_parse(the_evdns_base, flags, + sandbox_intern_string(conf_fname)))) { + log_warn(LD_EXIT, "Unable to parse '%s', or no nameservers " + "in '%s' (%d)", conf_fname, conf_fname, r); + + if (r != 6) // "r = 6" means "no DNS servers were in resolv.conf" - + goto err; // in which case we expect libevent to add 127.0.0.1 as + // fallback. + } + if (evdns_base_count_nameservers(the_evdns_base) == 0) { + log_warn(LD_EXIT, "Unable to find any nameservers in '%s'.", + conf_fname); + } + + tor_free(resolv_conf_fname); + resolv_conf_fname = tor_strdup(conf_fname); + resolv_conf_mtime = st.st_mtime; + } else { + log_warn(LD_EXIT, "Could not read your DNS config from '%s' - " + "please investigate your DNS configuration. " + "This is possibly a problem. Meanwhile, falling" + " back to local DNS at 127.0.0.1.", conf_fname); + evdns_base_nameserver_ip_add(the_evdns_base, "127.0.0.1"); } - tor_free(resolv_conf_fname); - resolv_conf_fname = tor_strdup(conf_fname); - resolv_conf_mtime = st.st_mtime; + if (nameservers_configured) evdns_base_resume(the_evdns_base); } diff --git a/src/feature/relay/dns.h b/src/feature/relay/dns.h index e4474cdf43..7b2a31a311 100644 --- a/src/feature/relay/dns.h +++ b/src/feature/relay/dns.h @@ -45,6 +45,11 @@ size_t dns_cache_handle_oom(time_t now, size_t min_remove_bytes); #ifdef DNS_PRIVATE #include "feature/relay/dns_structs.h" +size_t number_of_configured_nameservers(void); +#ifdef HAVE_EVDNS_BASE_GET_NAMESERVER_ADDR +tor_addr_t *configured_nameserver_address(const size_t idx); +#endif + MOCK_DECL(STATIC int,dns_resolve_impl,(edge_connection_t *exitconn, int is_resolve,or_circuit_t *oncirc, char **hostname_out, int *made_connection_pending_out, cached_resolve_t **resolve_out)); diff --git a/src/feature/relay/ext_orport.c b/src/feature/relay/ext_orport.c index 56c5bb96f5..8589efb48d 100644 --- a/src/feature/relay/ext_orport.c +++ b/src/feature/relay/ext_orport.c @@ -90,7 +90,7 @@ connection_ext_or_transition(or_connection_t *conn) conn->base_.type = CONN_TYPE_OR; TO_CONN(conn)->state = 0; // set the state to a neutral value - control_event_or_conn_status(conn, OR_CONN_EVENT_NEW, 0); + connection_or_event_status(conn, OR_CONN_EVENT_NEW, 0); connection_tls_start_handshake(conn, 1); } diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c index 1dbaf2ed66..12ee1dd6e2 100644 --- a/src/feature/relay/router.c +++ b/src/feature/relay/router.c @@ -30,6 +30,7 @@ #include "feature/nodelist/dirlist.h" #include "feature/nodelist/networkstatus.h" #include "feature/nodelist/nickname.h" +#include "feature/nodelist/nodefamily.h" #include "feature/nodelist/nodelist.h" #include "feature/nodelist/routerlist.h" #include "feature/nodelist/torcert.h" @@ -49,6 +50,7 @@ #include "lib/encoding/confline.h" #include "lib/osinfo/uname.h" #include "lib/tls/tortls.h" +#include "lib/version/torversion.h" #include "feature/dirauth/authmode.h" @@ -58,6 +60,7 @@ #include "feature/dircommon/dir_connection_st.h" #include "feature/nodelist/authority_cert_st.h" #include "feature/nodelist/extrainfo_st.h" +#include "feature/nodelist/networkstatus_st.h" #include "feature/nodelist/node_st.h" #include "feature/nodelist/routerinfo_st.h" #include "feature/nodelist/routerstatus_st.h" @@ -337,6 +340,16 @@ set_server_identity_key(crypto_pk_t *k) } } +#ifdef TOR_UNIT_TESTS +/** Testing only -- set the server's RSA identity digest to + * be <b>digest</b> */ +void +set_server_identity_key_digest_testing(const uint8_t *digest) +{ + memcpy(server_identitykey_digest, digest, DIGEST_LEN); +} +#endif + /** Make sure that we have set up our identity keys to match or not match as * appropriate, and die with an assertion if we have not. */ static void @@ -634,7 +647,7 @@ load_authority_keyset(int legacy, crypto_pk_t **key_out, fname); goto done; } - parsed = authority_cert_parse_from_string(cert, &eos); + parsed = authority_cert_parse_from_string(cert, strlen(cert), &eos); if (!parsed) { log_warn(LD_DIR, "Unable to parse certificate in %s", fname); goto done; @@ -1467,9 +1480,9 @@ static extrainfo_t *desc_extrainfo = NULL; static const char *desc_gen_reason = "uninitialized reason"; /** Since when has our descriptor been "clean"? 0 if we need to regenerate it * now. */ -static time_t desc_clean_since = 0; +STATIC time_t desc_clean_since = 0; /** Why did we mark the descriptor dirty? */ -static const char *desc_dirty_reason = "Tor just started"; +STATIC const char *desc_dirty_reason = "Tor just started"; /** Boolean: do we need to regenerate the above? */ static int desc_needs_upload = 0; @@ -1688,10 +1701,6 @@ router_get_descriptor_gen_reason(void) return desc_gen_reason; } -/** A list of nicknames that we've warned about including in our family - * declaration verbatim rather than as digests. */ -static smartlist_t *warned_nonexistent_family = NULL; - static int router_guess_address_from_dir_headers(uint32_t *guess); /** Make a current best guess at our address, either because @@ -1804,6 +1813,132 @@ router_check_descriptor_address_consistency(uint32_t ipv4h_desc_addr) CONN_TYPE_DIR_LISTENER); } +/** A list of nicknames that we've warned about including in our family, + * for one reason or another. */ +static smartlist_t *warned_family = NULL; + +/** + * Return a new smartlist containing the family members configured in + * <b>options</b>. Warn about invalid or missing entries. Return NULL + * if this relay should not declare a family. + **/ +STATIC smartlist_t * +get_my_declared_family(const or_options_t *options) +{ + if (!options->MyFamily) + return NULL; + + if (options->BridgeRelay) + return NULL; + + if (!warned_family) + warned_family = smartlist_new(); + + smartlist_t *declared_family = smartlist_new(); + config_line_t *family; + + /* First we try to get the whole family in the form of hexdigests. */ + for (family = options->MyFamily; family; family = family->next) { + char *name = family->value; + const node_t *member; + if (options->Nickname && !strcasecmp(name, options->Nickname)) + continue; /* Don't list ourself by nickname, that's redundant */ + else + member = node_get_by_nickname(name, 0); + + if (!member) { + /* This node doesn't seem to exist, so warn about it if it is not + * a hexdigest. */ + int is_legal = is_legal_nickname_or_hexdigest(name); + if (!smartlist_contains_string(warned_family, name) && + !is_legal_hexdigest(name)) { + if (is_legal) + log_warn(LD_CONFIG, + "There is a router named %s in my declared family, but " + "I have no descriptor for it. I'll use the nickname " + "as is, but this may confuse clients. Please list it " + "by identity digest instead.", escaped(name)); + else + log_warn(LD_CONFIG, "There is a router named %s in my declared " + "family, but that isn't a legal digest or nickname. " + "Skipping it.", escaped(name)); + smartlist_add_strdup(warned_family, name); + } + if (is_legal) { + smartlist_add_strdup(declared_family, name); + } + } else { + /* List the node by digest. */ + char *fp = tor_malloc(HEX_DIGEST_LEN+2); + fp[0] = '$'; + base16_encode(fp+1,HEX_DIGEST_LEN+1, + member->identity, DIGEST_LEN); + smartlist_add(declared_family, fp); + + if (! is_legal_hexdigest(name) && + !smartlist_contains_string(warned_family, name)) { + /* Warn if this node was not specified by hexdigest. */ + log_warn(LD_CONFIG, "There is a router named %s in my declared " + "family, but it wasn't listed by digest. Please consider " + "saying %s instead, if that's what you meant.", + escaped(name), fp); + smartlist_add_strdup(warned_family, name); + } + } + } + + /* Now declared_family should have the closest we can come to the + * identities that the user wanted. + * + * Unlike older versions of Tor, we _do_ include our own identity: this + * helps microdescriptor compression, and helps in-memory compression + * on clients. */ + nodefamily_t *nf = nodefamily_from_members(declared_family, + router_get_my_id_digest(), + NF_WARN_MALFORMED, + NULL); + SMARTLIST_FOREACH(declared_family, char *, s, tor_free(s)); + smartlist_free(declared_family); + if (!nf) { + return NULL; + } + + char *s = nodefamily_format(nf); + nodefamily_free(nf); + + smartlist_t *result = smartlist_new(); + smartlist_split_string(result, s, NULL, + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + tor_free(s); + + if (smartlist_len(result) == 1) { + /* This is a one-element list containing only ourself; instead return + * nothing */ + const char *singleton = smartlist_get(result, 0); + bool is_me = false; + if (singleton[0] == '$') { + char d[DIGEST_LEN]; + int n = base16_decode(d, sizeof(d), singleton+1, strlen(singleton+1)); + if (n == DIGEST_LEN && + fast_memeq(d, router_get_my_id_digest(), DIGEST_LEN)) { + is_me = true; + } + } + if (!is_me) { + // LCOV_EXCL_START + log_warn(LD_BUG, "Found a singleton family list with an element " + "that wasn't us! Element was %s", escaped(singleton)); + // LCOV_EXCL_STOP + } else { + SMARTLIST_FOREACH(result, char *, cp, tor_free(cp)); + smartlist_free(result); + return NULL; + } + } + + return result; +} + /** Build a fresh routerinfo, signed server descriptor, and extra-info document * for this OR. Set r to the generated routerinfo, e to the generated * extra-info document. Return 0 on success, -1 on temporary error. Failure to @@ -1918,54 +2053,7 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e) tor_free(p_tmp); } - if (options->MyFamily && ! options->BridgeRelay) { - if (!warned_nonexistent_family) - warned_nonexistent_family = smartlist_new(); - ri->declared_family = smartlist_new(); - config_line_t *family; - for (family = options->MyFamily; family; family = family->next) { - char *name = family->value; - const node_t *member; - if (!strcasecmp(name, options->Nickname)) - continue; /* Don't list ourself, that's redundant */ - else - member = node_get_by_nickname(name, 0); - if (!member) { - int is_legal = is_legal_nickname_or_hexdigest(name); - if (!smartlist_contains_string(warned_nonexistent_family, name) && - !is_legal_hexdigest(name)) { - if (is_legal) - log_warn(LD_CONFIG, - "I have no descriptor for the router named \"%s\" in my " - "declared family; I'll use the nickname as is, but " - "this may confuse clients.", name); - else - log_warn(LD_CONFIG, "There is a router named \"%s\" in my " - "declared family, but that isn't a legal nickname. " - "Skipping it.", escaped(name)); - smartlist_add_strdup(warned_nonexistent_family, name); - } - if (is_legal) { - smartlist_add_strdup(ri->declared_family, name); - } - } else if (router_digest_is_me(member->identity)) { - /* Don't list ourself in our own family; that's redundant */ - /* XXX shouldn't be possible */ - } else { - char *fp = tor_malloc(HEX_DIGEST_LEN+2); - fp[0] = '$'; - base16_encode(fp+1,HEX_DIGEST_LEN+1, - member->identity, DIGEST_LEN); - smartlist_add(ri->declared_family, fp); - if (smartlist_contains_string(warned_nonexistent_family, name)) - smartlist_string_remove(warned_nonexistent_family, name); - } - } - - /* remove duplicates from the list */ - smartlist_sort_strings(ri->declared_family); - smartlist_uniq_strings(ri->declared_family); - } + ri->declared_family = get_my_declared_family(options); /* Now generate the extrainfo. */ ei = tor_malloc_zero(sizeof(extrainfo_t)); @@ -2131,7 +2219,9 @@ mark_my_descriptor_dirty_if_too_old(time_t now) /* Now we see whether we want to be retrying frequently or no. The * rule here is that we'll retry frequently if we aren't listed in the * live consensus we have, or if the publication time of the - * descriptor listed for us in the consensus is very old. */ + * descriptor listed for us in the consensus is very old, or if the + * consensus lists us as "stale" and we haven't regenerated since the + * consensus was published. */ ns = networkstatus_get_live_consensus(now); if (ns) { rs = networkstatus_vote_find_entry(ns, server_identitykey_digest); @@ -2139,6 +2229,8 @@ mark_my_descriptor_dirty_if_too_old(time_t now) retry_fast_reason = "not listed in consensus"; else if (rs->published_on < slow_cutoff) retry_fast_reason = "version listed in consensus is quite old"; + else if (rs->is_staledesc && ns->valid_after > desc_clean_since) + retry_fast_reason = "listed as stale in consensus"; } if (retry_fast_reason && desc_clean_since < fast_cutoff) @@ -3055,9 +3147,9 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo, void router_reset_warnings(void) { - if (warned_nonexistent_family) { - SMARTLIST_FOREACH(warned_nonexistent_family, char *, cp, tor_free(cp)); - smartlist_clear(warned_nonexistent_family); + if (warned_family) { + SMARTLIST_FOREACH(warned_family, char *, cp, tor_free(cp)); + smartlist_clear(warned_family); } } @@ -3081,11 +3173,12 @@ router_free_all(void) memwipe(&curve25519_onion_key, 0, sizeof(curve25519_onion_key)); memwipe(&last_curve25519_onion_key, 0, sizeof(last_curve25519_onion_key)); - if (warned_nonexistent_family) { - SMARTLIST_FOREACH(warned_nonexistent_family, char *, cp, tor_free(cp)); - smartlist_free(warned_nonexistent_family); + if (warned_family) { + SMARTLIST_FOREACH(warned_family, char *, cp, tor_free(cp)); + smartlist_free(warned_family); } } + /* From the given RSA key object, convert it to ASN-1 encoded format and set * the newly allocated object in onion_pkey_out. The length of the key is set * in onion_pkey_len_out. */ diff --git a/src/feature/relay/router.h b/src/feature/relay/router.h index bd6a8a012e..60bc857ceb 100644 --- a/src/feature/relay/router.h +++ b/src/feature/relay/router.h @@ -117,6 +117,14 @@ void router_free_all(void); /* Used only by router.c and test.c */ STATIC void get_platform_str(char *platform, size_t len); STATIC int router_write_fingerprint(int hashed); +STATIC smartlist_t *get_my_declared_family(const or_options_t *options); + +#ifdef TOR_UNIT_TESTS +extern time_t desc_clean_since; +extern const char *desc_dirty_reason; +void set_server_identity_key_digest_testing(const uint8_t *digest); +#endif + #endif #endif /* !defined(TOR_ROUTER_H) */ diff --git a/src/feature/rend/rendcache.c b/src/feature/rend/rendcache.c index 1c3badaff3..fadfb43883 100644 --- a/src/feature/rend/rendcache.c +++ b/src/feature/rend/rendcache.c @@ -45,7 +45,7 @@ STATIC digestmap_t *rend_cache_v2_dir = NULL; * looked up in this cache and if present, it is discarded from the fetched * descriptor. At the end, all IP(s) in the cache, for a specific service * ID, that were NOT present in the descriptor are removed from this cache. - * Which means that if at least one IP was not in this cache, thus usuable, + * Which means that if at least one IP was not in this cache, thus usable, * it's considered a new descriptor so we keep it. Else, if all IPs were in * this cache, we discard the descriptor as it's considered unusable. * diff --git a/src/feature/rend/rendmid.c b/src/feature/rend/rendmid.c index 3ba48f8858..421e0f2139 100644 --- a/src/feature/rend/rendmid.c +++ b/src/feature/rend/rendmid.c @@ -237,8 +237,8 @@ rend_mid_establish_rendezvous(or_circuit_t *circ, const uint8_t *request, goto err; } - /* Check if we are configured to accept established rendezvous cells from - * client or in other words Tor2Web clients. */ + /* Check if we are configured to defend ourselves from clients that + * attempt to establish rendezvous points directly to us. */ if (channel_is_client(circ->p_chan) && dos_should_refuse_single_hop_client()) { /* Note it down for the heartbeat log purposes. */ diff --git a/src/feature/stats/geoip_stats.c b/src/feature/stats/geoip_stats.c index a54b589eb6..5119da19a0 100644 --- a/src/feature/stats/geoip_stats.c +++ b/src/feature/stats/geoip_stats.c @@ -30,7 +30,7 @@ #include "core/or/or.h" #include "ht.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "app/config/config.h" #include "feature/control/control.h" #include "feature/client/dnsserv.h" diff --git a/src/include.am b/src/include.am index d2f83da814..9070a69a03 100644 --- a/src/include.am +++ b/src/include.am @@ -1,5 +1,6 @@ include src/ext/include.am include src/lib/arch/include.am +include src/lib/buf/include.am include src/lib/err/include.am include src/lib/cc/include.am include src/lib/ctime/include.am @@ -25,6 +26,7 @@ include src/lib/osinfo/include.am include src/lib/process/include.am include src/lib/sandbox/include.am include src/lib/string/include.am +include src/lib/subsys/include.am include src/lib/smartlist_core/include.am include src/lib/term/include.am include src/lib/testsupport/include.am @@ -32,6 +34,7 @@ include src/lib/thread/include.am include src/lib/time/include.am include src/lib/tls/include.am include src/lib/trace/include.am +include src/lib/version/include.am include src/lib/wallclock/include.am include src/trunnel/include.am diff --git a/src/lib/buf/.may_include b/src/lib/buf/.may_include new file mode 100644 index 0000000000..c4be73bce2 --- /dev/null +++ b/src/lib/buf/.may_include @@ -0,0 +1,10 @@ +orconfig.h + +lib/buf/*.h +lib/cc/*.h +lib/ctime/*.h +lib/malloc/*.h +lib/testsupport/*.h +lib/log/*.h +lib/string/*.h +lib/time/*.h diff --git a/src/lib/container/buffers.c b/src/lib/buf/buffers.c index 67887f2f30..88a25b8470 100644 --- a/src/lib/container/buffers.c +++ b/src/lib/buf/buffers.c @@ -25,7 +25,7 @@ #define BUFFERS_PRIVATE #include "orconfig.h" #include <stddef.h> -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/cc/torint.h" #include "lib/log/log.h" #include "lib/log/util_bug.h" diff --git a/src/lib/container/buffers.h b/src/lib/buf/buffers.h index c103b93a82..c103b93a82 100644 --- a/src/lib/container/buffers.h +++ b/src/lib/buf/buffers.h diff --git a/src/lib/buf/include.am b/src/lib/buf/include.am new file mode 100644 index 0000000000..3338c3dbdb --- /dev/null +++ b/src/lib/buf/include.am @@ -0,0 +1,17 @@ + +noinst_LIBRARIES += src/lib/libtor-buf.a + +if UNITTESTS_ENABLED +noinst_LIBRARIES += src/lib/libtor-buf-testing.a +endif + +src_lib_libtor_buf_a_SOURCES = \ + src/lib/buf/buffers.c + +src_lib_libtor_buf_testing_a_SOURCES = \ + $(src_lib_libtor_buf_a_SOURCES) +src_lib_libtor_buf_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) +src_lib_libtor_buf_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) + +noinst_HEADERS += \ + src/lib/buf/buffers.h diff --git a/src/lib/cc/.may_include b/src/lib/cc/.may_include index 2b06e8519c..fa1478ce46 100644 --- a/src/lib/cc/.may_include +++ b/src/lib/cc/.may_include @@ -1 +1,2 @@ orconfig.h +lib/cc/*.h
\ No newline at end of file diff --git a/src/lib/cc/ctassert.h b/src/lib/cc/ctassert.h new file mode 100644 index 0000000000..e42976360f --- /dev/null +++ b/src/lib/cc/ctassert.h @@ -0,0 +1,53 @@ +/* Copyright (c) 2018 The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file ctassert.h + * + * \brief Compile-time assertions: CTASSERT(expression). + */ + +#ifndef TOR_CTASSERT_H +#define TOR_CTASSERT_H + +#include "lib/cc/compat_compiler.h" + +/** + * CTASSERT(expression) + * + * Trigger a compiler error if expression is false. + */ +#if __STDC_VERSION__ >= 201112L + +/* If C11 is available, just use _Static_assert. */ +#define CTASSERT(x) _Static_assert((x), #x) + +#else + +/* + * If C11 is not available, expand __COUNTER__, or __INCLUDE_LEVEL__ + * and __LINE__, or just __LINE__, with an intermediate preprocessor + * macro CTASSERT_EXPN, and then use CTASSERT_DECL to paste the + * expansions together into a unique name. + * + * We use this name as a typedef of an array type with a positive + * length if the assertion is true, and a negative length of the + * assertion is false, which is invalid and hence triggers a compiler + * error. + */ +#if defined(__COUNTER__) +#define CTASSERT(x) CTASSERT_EXPN((x), c, __COUNTER__) +#elif defined(__INCLUDE_LEVEL__) +#define CTASSERT(x) CTASSERT_EXPN((x), __INCLUDE_LEVEL__, __LINE__) +#else +/* hope it's unique enough */ +#define CTASSERT(x) CTASSERT_EXPN((x), l, __LINE__) +#endif + +#define CTASSERT_EXPN(x, a, b) CTASSERT_DECL(x, a, b) +#define CTASSERT_DECL(x, a, b) \ + typedef char tor_ctassert_##a##_##b[(x) ? 1 : -1] ATTR_UNUSED + +#endif + +#endif /* !defined(TOR_CTASSERT_H) */ diff --git a/src/lib/cc/include.am b/src/lib/cc/include.am index 2ae90f97dd..52cf8a9f72 100644 --- a/src/lib/cc/include.am +++ b/src/lib/cc/include.am @@ -1,4 +1,5 @@ noinst_HEADERS += \ src/lib/cc/compat_compiler.h \ + src/lib/cc/ctassert.h \ src/lib/cc/torint.h diff --git a/src/lib/compress/.may_include b/src/lib/compress/.may_include index 68fe9f1c54..6cd80086e6 100644 --- a/src/lib/compress/.may_include +++ b/src/lib/compress/.may_include @@ -1,5 +1,6 @@ orconfig.h lib/arch/*.h +lib/buf/*.h lib/cc/*.h lib/compress/*.h lib/container/*.h @@ -8,5 +9,6 @@ lib/intmath/*.h lib/log/*.h lib/malloc/*.h lib/string/*.h +lib/subsys/*.h lib/testsupport/*.h lib/thread/*.h diff --git a/src/lib/compress/compress.c b/src/lib/compress/compress.c index 95fd73bb32..51591410a2 100644 --- a/src/lib/compress/compress.c +++ b/src/lib/compress/compress.c @@ -29,10 +29,12 @@ #include "lib/compress/compress.h" #include "lib/compress/compress_lzma.h" #include "lib/compress/compress_none.h" +#include "lib/compress/compress_sys.h" #include "lib/compress/compress_zlib.h" #include "lib/compress/compress_zstd.h" #include "lib/intmath/cmp.h" #include "lib/malloc/malloc.h" +#include "lib/subsys/subsys.h" #include "lib/thread/threads.h" /** Total number of bytes allocated for compression state overhead. */ @@ -660,7 +662,7 @@ tor_compress_state_size(const tor_compress_state_t *state) } /** Initialize all compression modules. */ -void +int tor_compress_init(void) { atomic_counter_init(&total_compress_allocation); @@ -668,6 +670,8 @@ tor_compress_init(void) tor_zlib_init(); tor_lzma_init(); tor_zstd_init(); + + return 0; } /** Warn if we had any problems while setting up our compression libraries. @@ -677,5 +681,20 @@ tor_compress_init(void) void tor_compress_log_init_warnings(void) { + // XXXX can we move this into tor_compress_init() after all? log.c queues + // XXXX log messages at startup. tor_zstd_warn_if_version_mismatched(); } + +static int +subsys_compress_initialize(void) +{ + return tor_compress_init(); +} + +const subsys_fns_t sys_compress = { + .name = "compress", + .supported = true, + .level = -70, + .initialize = subsys_compress_initialize, +}; diff --git a/src/lib/compress/compress.h b/src/lib/compress/compress.h index 5f16a2ab27..8cea4ead60 100644 --- a/src/lib/compress/compress.h +++ b/src/lib/compress/compress.h @@ -89,7 +89,7 @@ void tor_compress_free_(tor_compress_state_t *state); size_t tor_compress_state_size(const tor_compress_state_t *state); -void tor_compress_init(void); +int tor_compress_init(void); void tor_compress_log_init_warnings(void); struct buf_t; diff --git a/src/lib/compress/compress_buf.c b/src/lib/compress/compress_buf.c index 198128b261..2e704466f2 100644 --- a/src/lib/compress/compress_buf.c +++ b/src/lib/compress/compress_buf.c @@ -11,7 +11,7 @@ #define BUFFERS_PRIVATE #include "lib/cc/compat_compiler.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/compress/compress.h" #include "lib/log/util_bug.h" diff --git a/src/lib/compress/compress_sys.h b/src/lib/compress/compress_sys.h new file mode 100644 index 0000000000..6181072315 --- /dev/null +++ b/src/lib/compress/compress_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_sys.h + * \brief Declare subsystem object for the compress module + **/ + +#ifndef TOR_COMPRESS_SYS_H +#define TOR_COMPRESS_SYS_H + +extern const struct subsys_fns_t sys_compress; + +#endif /* !defined(TOR_COMPRESS_SYS_H) */ diff --git a/src/lib/compress/include.am b/src/lib/compress/include.am index 75c9032bd2..b952779578 100644 --- a/src/lib/compress/include.am +++ b/src/lib/compress/include.am @@ -22,5 +22,6 @@ noinst_HEADERS += \ src/lib/compress/compress.h \ src/lib/compress/compress_lzma.h \ src/lib/compress/compress_none.h \ + src/lib/compress/compress_sys.h \ src/lib/compress/compress_zlib.h \ src/lib/compress/compress_zstd.h diff --git a/src/lib/container/.may_include b/src/lib/container/.may_include index 90de5eda40..81507527d3 100644 --- a/src/lib/container/.may_include +++ b/src/lib/container/.may_include @@ -7,12 +7,9 @@ lib/malloc/*.h lib/err/*.h lib/smartlist_core/*.h lib/string/*.h -lib/testsupport/testsupport.h +lib/testsupport/*.h lib/intmath/*.h lib/log/*.h -# XXXX I am unsure about this one. It's only here for buffers.c -lib/time/*.h - -ht.h -siphash.h +ext/ht.h +ext/siphash.h diff --git a/src/lib/container/bloomfilt.c b/src/lib/container/bloomfilt.c index 9aa9b1ee56..8c61db81d6 100644 --- a/src/lib/container/bloomfilt.c +++ b/src/lib/container/bloomfilt.c @@ -14,7 +14,7 @@ #include "lib/container/bloomfilt.h" #include "lib/intmath/bits.h" #include "lib/log/util_bug.h" -#include "siphash.h" +#include "ext/siphash.h" /** How many bloom-filter bits we set per address. This is twice the * BLOOMFILT_N_HASHES value, since we split the siphash output into two 32-bit diff --git a/src/lib/container/include.am b/src/lib/container/include.am index e6492098b5..032e4033da 100644 --- a/src/lib/container/include.am +++ b/src/lib/container/include.am @@ -7,7 +7,6 @@ endif src_lib_libtor_container_a_SOURCES = \ src/lib/container/bloomfilt.c \ - src/lib/container/buffers.c \ src/lib/container/map.c \ src/lib/container/order.c \ src/lib/container/smartlist.c @@ -20,7 +19,6 @@ src_lib_libtor_container_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) noinst_HEADERS += \ src/lib/container/bitarray.h \ src/lib/container/bloomfilt.h \ - src/lib/container/buffers.h \ src/lib/container/handles.h \ src/lib/container/map.h \ src/lib/container/order.h \ diff --git a/src/lib/container/map.c b/src/lib/container/map.c index d213ad50bf..fde33d6ace 100644 --- a/src/lib/container/map.c +++ b/src/lib/container/map.c @@ -21,7 +21,7 @@ #include <stdlib.h> #include <string.h> -#include "ht.h" +#include "ext/ht.h" /** Helper: Declare an entry type and a map type to implement a mapping using * ht.h. The map type will be called <b>maptype</b>. The key part of each diff --git a/src/lib/container/map.h b/src/lib/container/map.h index a2d1b01d12..d61b1ec18f 100644 --- a/src/lib/container/map.h +++ b/src/lib/container/map.h @@ -15,7 +15,7 @@ #include "lib/testsupport/testsupport.h" #include "lib/cc/torint.h" -#include "siphash.h" +#include "ext/siphash.h" #define DECLARE_MAP_FNS(maptype, keytype, prefix) \ typedef struct maptype maptype; \ diff --git a/src/lib/crypt_ops/.may_include b/src/lib/crypt_ops/.may_include index a0fa4ec05c..0739699686 100644 --- a/src/lib/crypt_ops/.may_include +++ b/src/lib/crypt_ops/.may_include @@ -12,7 +12,8 @@ lib/malloc/*.h lib/intmath/*.h lib/sandbox/*.h lib/string/*.h -lib/testsupport/testsupport.h +lib/subsys/*.h +lib/testsupport/*.h lib/thread/*.h lib/log/*.h @@ -21,4 +22,4 @@ trunnel/pwbox.h keccak-tiny/*.h ed25519/*.h -siphash.h +ext/siphash.h diff --git a/src/lib/crypt_ops/crypto_init.c b/src/lib/crypt_ops/crypto_init.c index 329c264af6..4040085c76 100644 --- a/src/lib/crypt_ops/crypto_init.c +++ b/src/lib/crypt_ops/crypto_init.c @@ -20,8 +20,11 @@ #include "lib/crypt_ops/crypto_openssl_mgt.h" #include "lib/crypt_ops/crypto_nss_mgt.h" #include "lib/crypt_ops/crypto_rand.h" +#include "lib/crypt_ops/crypto_sys.h" -#include "siphash.h" +#include "lib/subsys/subsys.h" + +#include "ext/siphash.h" /** Boolean: has our crypto library been initialized? (early phase) */ static int crypto_early_initialized_ = 0; @@ -202,3 +205,47 @@ tor_is_using_nss(void) return 0; #endif } + +static int +subsys_crypto_initialize(void) +{ + if (crypto_early_init() < 0) + return -1; + crypto_dh_init(); + return 0; +} + +static void +subsys_crypto_shutdown(void) +{ + crypto_global_cleanup(); +} + +static void +subsys_crypto_prefork(void) +{ + crypto_prefork(); +} + +static void +subsys_crypto_postfork(void) +{ + crypto_postfork(); +} + +static void +subsys_crypto_thread_cleanup(void) +{ + crypto_thread_cleanup(); +} + +const struct subsys_fns_t sys_crypto = { + .name = "crypto", + .supported = true, + .level = -60, + .initialize = subsys_crypto_initialize, + .shutdown = subsys_crypto_shutdown, + .prefork = subsys_crypto_prefork, + .postfork = subsys_crypto_postfork, + .thread_cleanup = subsys_crypto_thread_cleanup, +}; diff --git a/src/lib/crypt_ops/crypto_rand.c b/src/lib/crypt_ops/crypto_rand.c index 915fe0870d..0b1cb96c1b 100644 --- a/src/lib/crypt_ops/crypto_rand.c +++ b/src/lib/crypt_ops/crypto_rand.c @@ -11,7 +11,6 @@ * number generators, and working with randomness. **/ -#ifndef CRYPTO_RAND_PRIVATE #define CRYPTO_RAND_PRIVATE #include "lib/crypt_ops/crypto_rand.h" @@ -530,111 +529,14 @@ crypto_rand_unmocked(char *to, size_t n) } /** - * Return a pseudorandom integer, chosen uniformly from the values - * between 0 and <b>max</b>-1 inclusive. <b>max</b> must be between 1 and - * INT_MAX+1, inclusive. + * Draw an unsigned 32-bit integer uniformly at random. */ -int -crypto_rand_int(unsigned int max) -{ - unsigned int val; - unsigned int cutoff; - tor_assert(max <= ((unsigned int)INT_MAX)+1); - tor_assert(max > 0); /* don't div by 0 */ - - /* We ignore any values that are >= 'cutoff,' to avoid biasing the - * distribution with clipping at the upper end of unsigned int's - * range. - */ - cutoff = UINT_MAX - (UINT_MAX%max); - while (1) { - crypto_rand((char*)&val, sizeof(val)); - if (val < cutoff) - return val % max; - } -} - -/** - * Return a pseudorandom integer, chosen uniformly from the values i such - * that min <= i < max. - * - * <b>min</b> MUST be in range [0, <b>max</b>). - * <b>max</b> MUST be in range (min, INT_MAX]. - **/ -int -crypto_rand_int_range(unsigned int min, unsigned int max) +uint32_t +crypto_rand_u32(void) { - tor_assert(min < max); - tor_assert(max <= INT_MAX); - - /* The overflow is avoided here because crypto_rand_int() returns a value - * between 0 and (max - min) inclusive. */ - return min + crypto_rand_int(max - min); -} - -/** - * As crypto_rand_int_range, but supports uint64_t. - **/ -uint64_t -crypto_rand_uint64_range(uint64_t min, uint64_t max) -{ - tor_assert(min < max); - return min + crypto_rand_uint64(max - min); -} - -/** - * As crypto_rand_int_range, but supports time_t. - **/ -time_t -crypto_rand_time_range(time_t min, time_t max) -{ - tor_assert(min < max); - return min + (time_t)crypto_rand_uint64(max - min); -} - -/** - * Return a pseudorandom 64-bit integer, chosen uniformly from the values - * between 0 and <b>max</b>-1 inclusive. - **/ -uint64_t -crypto_rand_uint64(uint64_t max) -{ - uint64_t val; - uint64_t cutoff; - tor_assert(max < UINT64_MAX); - tor_assert(max > 0); /* don't div by 0 */ - - /* We ignore any values that are >= 'cutoff,' to avoid biasing the - * distribution with clipping at the upper end of unsigned int's - * range. - */ - cutoff = UINT64_MAX - (UINT64_MAX%max); - while (1) { - crypto_rand((char*)&val, sizeof(val)); - if (val < cutoff) - return val % max; - } -} - -/** - * Return a pseudorandom double d, chosen uniformly from the range - * 0.0 <= d < 1.0. - **/ -double -crypto_rand_double(void) -{ - /* We just use an unsigned int here; we don't really care about getting - * more than 32 bits of resolution */ - unsigned int u; - crypto_rand((char*)&u, sizeof(u)); -#if SIZEOF_INT == 4 -#define UINT_MAX_AS_DOUBLE 4294967296.0 -#elif SIZEOF_INT == 8 -#define UINT_MAX_AS_DOUBLE 1.8446744073709552e+19 -#else -#error SIZEOF_INT is neither 4 nor 8 -#endif /* SIZEOF_INT == 4 || ... */ - return ((double)u) / UINT_MAX_AS_DOUBLE; + uint32_t rand; + crypto_rand((void*)&rand, sizeof(rand)); + return rand; } /** @@ -727,5 +629,3 @@ crypto_force_rand_ssleay(void) #endif return 0; } - -#endif /* !defined(CRYPTO_RAND_PRIVATE) */ diff --git a/src/lib/crypt_ops/crypto_rand.h b/src/lib/crypt_ops/crypto_rand.h index 86fa20faa3..8a81a4acdc 100644 --- a/src/lib/crypt_ops/crypto_rand.h +++ b/src/lib/crypt_ops/crypto_rand.h @@ -16,6 +16,7 @@ #include "lib/cc/compat_compiler.h" #include "lib/cc/torint.h" #include "lib/testsupport/testsupport.h" +#include "lib/malloc/malloc.h" /* random numbers */ int crypto_seed_rng(void) ATTR_WUR; @@ -24,9 +25,11 @@ void crypto_rand_unmocked(char *to, size_t n); void crypto_strongest_rand(uint8_t *out, size_t out_len); MOCK_DECL(void,crypto_strongest_rand_,(uint8_t *out, size_t out_len)); int crypto_rand_int(unsigned int max); +unsigned crypto_rand_uint(unsigned limit); int crypto_rand_int_range(unsigned int min, unsigned int max); uint64_t crypto_rand_uint64_range(uint64_t min, uint64_t max); time_t crypto_rand_time_range(time_t min, time_t max); +uint32_t crypto_rand_u32(void); uint64_t crypto_rand_uint64(uint64_t max); double crypto_rand_double(void); struct tor_weak_rng_t; @@ -40,6 +43,36 @@ void *smartlist_choose(const struct smartlist_t *sl); void smartlist_shuffle(struct smartlist_t *sl); int crypto_force_rand_ssleay(void); +/** + * A fast PRNG, for use when the PRNG provided by our crypto library isn't + * fast enough. This one _should_ be cryptographically strong, but + * has seen less auditing than the PRNGs in OpenSSL and NSS. Use with + * caution. + * + * Note that this object is NOT thread-safe. If you need a thread-safe + * prng, use crypto_rand(), or wrap this in a mutex. + **/ +typedef struct crypto_fast_rng_t crypto_fast_rng_t; +/** + * Number of bytes used to seed a crypto_rand_fast_t. + **/ +crypto_fast_rng_t *crypto_fast_rng_new(void); +#define CRYPTO_FAST_RNG_SEED_LEN 48 +crypto_fast_rng_t *crypto_fast_rng_new_from_seed(const uint8_t *seed); +void crypto_fast_rng_getbytes(crypto_fast_rng_t *rng, uint8_t *out, size_t n); +void crypto_fast_rng_free_(crypto_fast_rng_t *); +#define crypto_fast_rng_free(c) \ + FREE_AND_NULL(crypto_fast_rng_t, crypto_fast_rng_free_, (c)) + +unsigned crypto_fast_rng_get_uint(crypto_fast_rng_t *rng, unsigned limit); +uint64_t crypto_fast_rng_get_uint64(crypto_fast_rng_t *rng, uint64_t limit); +double crypto_fast_rng_get_double(crypto_fast_rng_t *rng); + +#if defined(TOR_UNIT_TESTS) +/* Used for white-box testing */ +size_t crypto_fast_rng_get_bytes_used_per_stream(void); +#endif + #ifdef CRYPTO_RAND_PRIVATE STATIC int crypto_strongest_rand_raw(uint8_t *out, size_t out_len); diff --git a/src/lib/crypt_ops/crypto_rand_fast.c b/src/lib/crypt_ops/crypto_rand_fast.c new file mode 100644 index 0000000000..34e763bf51 --- /dev/null +++ b/src/lib/crypt_ops/crypto_rand_fast.c @@ -0,0 +1,263 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file crypto_rand_fast.c + * + * \brief A fast strong PRNG for use when our underlying cryptographic + * library's PRNG isn't fast enough. + **/ + +/* This library is currently implemented to use the same implementation + * technique as libottery, using AES-CTR-256 as our underlying stream cipher. + * It's backtracking-resistant immediately, and prediction-resistant after + * a while. + * + * Here's how it works: + * + * We generate pseudorandom bytes using AES-CTR-256. We generate BUFLEN bytes + * at a time. When we do this, we keep the first SEED_LEN bytes as the key + * and the IV for our next invocation of AES_CTR, and yield the remaining + * BUFLEN - SEED_LEN bytes to the user as they invoke the PRNG. As we yield + * bytes to the user, we clear them from the buffer. + * + * After we have refilled the buffer RESEED_AFTER times, we mix in an + * additional SEED_LEN bytes from our strong PRNG into the seed. + * + * If the user ever asks for a huge number of bytes at once, we pull SEED_LEN + * bytes from the PRNG and use them with our stream cipher to fill the user's + * request. + */ + +#define CRYPTO_RAND_FAST_PRIVATE + +#include "lib/crypt_ops/crypto_rand.h" +#include "lib/crypt_ops/crypto_cipher.h" +#include "lib/crypt_ops/crypto_digest.h" +#include "lib/crypt_ops/crypto_util.h" +#include "lib/intmath/cmp.h" +#include "lib/cc/ctassert.h" +#include "lib/malloc/map_anon.h" + +#include "lib/log/util_bug.h" + +#include <string.h> + +/* Alias for CRYPTO_FAST_RNG_SEED_LEN to make our code shorter. + */ +#define SEED_LEN (CRYPTO_FAST_RNG_SEED_LEN) + +/* The amount of space that we mmap for a crypto_fast_rng_t. + */ +#define MAPLEN 4096 + +/* The number of random bytes that we can yield to the user after each + * time we fill a crypto_fast_rng_t's buffer. + */ +#define BUFLEN (MAPLEN - 2*sizeof(uint16_t) - SEED_LEN) + +/* The number of buffer refills after which we should fetch more + * entropy from crypto_strongest_rand(). + */ +#define RESEED_AFTER 16 + +/* The length of the stream cipher key we will use for the PRNG, in bytes. + */ +#define KEY_LEN (CRYPTO_FAST_RNG_SEED_LEN - CIPHER_IV_LEN) +/* The length of the stream cipher key we will use for the PRNG, in bits. + */ +#define KEY_BITS (KEY_LEN * 8) + +/* Make sure that we have a key length we can actually use with AES. */ +CTASSERT(KEY_BITS == 128 || KEY_BITS == 192 || KEY_BITS == 256); + +struct crypto_fast_rng_t { + /** How many more fills does this buffer have before we should mix + * in the output of crypto_rand()? */ + uint16_t n_till_reseed; + /** How many bytes are remaining in cbuf.bytes? */ + uint16_t bytes_left; + struct cbuf { + /** The seed (key and IV) that we will use the next time that we refill + * cbuf. */ + uint8_t seed[SEED_LEN]; + /** + * Bytes that we are yielding to the user. The next byte to be + * yielded is at bytes[BUFLEN-bytes_left]; all other bytes in this + * array are set to zero. + */ + uint8_t bytes[BUFLEN]; + } buf; +}; + +/* alignof(uint8_t) should be 1, so there shouldn't be any padding in cbuf. + */ +CTASSERT(sizeof(struct cbuf) == BUFLEN+SEED_LEN); +/* We're trying to fit all of the RNG state into a nice mmapable chunk. + */ +CTASSERT(sizeof(crypto_fast_rng_t) <= MAPLEN); + +/** + * Initialize and return a new fast PRNG, using a strong random seed. + * + * Note that this object is NOT thread-safe. If you need a thread-safe + * prng, use crypto_rand(), or wrap this in a mutex. + **/ +crypto_fast_rng_t * +crypto_fast_rng_new(void) +{ + uint8_t seed[SEED_LEN]; + crypto_strongest_rand(seed, sizeof(seed)); + crypto_fast_rng_t *result = crypto_fast_rng_new_from_seed(seed); + memwipe(seed, 0, sizeof(seed)); + return result; +} + +/** + * Initialize and return a new fast PRNG, using a seed value specified + * in <b>seed</b>. This value must be CRYPTO_FAST_RNG_SEED_LEN bytes + * long. + * + * Note that this object is NOT thread-safe. If you need a thread-safe + * prng, use crypto_rand(), or wrap this in a mutex. + **/ +crypto_fast_rng_t * +crypto_fast_rng_new_from_seed(const uint8_t *seed) +{ + /* We try to allocate this object as securely as we can, to avoid + * having it get dumped, swapped, or shared after fork. + */ + crypto_fast_rng_t *result = tor_mmap_anonymous(sizeof(*result), + ANONMAP_PRIVATE | ANONMAP_NOINHERIT); + + memcpy(result->buf.seed, seed, SEED_LEN); + /* Causes an immediate refill once the user asks for data. */ + result->bytes_left = 0; + result->n_till_reseed = RESEED_AFTER; + return result; +} + +/** + * Helper: create a crypto_cipher_t object from SEED_LEN bytes of + * input. The first KEY_LEN bytes are used as the stream cipher's key, + * and the remaining CIPHER_IV_LEN bytes are used as its IV. + **/ +static inline crypto_cipher_t * +cipher_from_seed(const uint8_t *seed) +{ + return crypto_cipher_new_with_iv_and_bits(seed, seed+KEY_LEN, KEY_BITS); +} + +/** + * Helper: refill the seed bytes and output buffer of <b>rng</b>, using + * the input seed bytes as input (key and IV) for the stream cipher. + * + * If the n_till_reseed counter has reached zero, mix more random bytes into + * the seed before refilling the buffer. + **/ +static void +crypto_fast_rng_refill(crypto_fast_rng_t *rng) +{ + if (rng->n_till_reseed-- == 0) { + /* It's time to reseed the RNG. We'll do this by using our XOF to mix the + * old value for the seed with some additional bytes from + * crypto_strongest_rand(). */ + crypto_xof_t *xof = crypto_xof_new(); + crypto_xof_add_bytes(xof, rng->buf.seed, SEED_LEN); + { + uint8_t seedbuf[SEED_LEN]; + crypto_strongest_rand(seedbuf, SEED_LEN); + crypto_xof_add_bytes(xof, seedbuf, SEED_LEN); + memwipe(seedbuf, 0, SEED_LEN); + } + crypto_xof_squeeze_bytes(xof, rng->buf.seed, SEED_LEN); + crypto_xof_free(xof); + + rng->n_till_reseed = RESEED_AFTER; + } + /* Now fill rng->buf with output from our stream cipher, initialized from + * that seed value. */ + crypto_cipher_t *c = cipher_from_seed(rng->buf.seed); + memset(&rng->buf, 0, sizeof(rng->buf)); + crypto_cipher_crypt_inplace(c, (char*)&rng->buf, sizeof(rng->buf)); + crypto_cipher_free(c); + + rng->bytes_left = sizeof(rng->buf.bytes); +} + +/** + * Release all storage held by <b>rng</b>. + **/ +void +crypto_fast_rng_free_(crypto_fast_rng_t *rng) +{ + if (!rng) + return; + memwipe(rng, 0, sizeof(*rng)); + tor_munmap_anonymous(rng, sizeof(*rng)); +} + +/** + * Helper: extract bytes from the PRNG, refilling it as necessary. Does not + * optimize the case when the user has asked for a huge output. + **/ +static void +crypto_fast_rng_getbytes_impl(crypto_fast_rng_t *rng, uint8_t *out, + const size_t n) +{ + size_t bytes_to_yield = n; + + while (bytes_to_yield) { + if (rng->bytes_left == 0) + crypto_fast_rng_refill(rng); + + const size_t to_copy = MIN(rng->bytes_left, bytes_to_yield); + + tor_assert(sizeof(rng->buf.bytes) >= rng->bytes_left); + uint8_t *copy_from = rng->buf.bytes + + (sizeof(rng->buf.bytes) - rng->bytes_left); + memcpy(out, copy_from, to_copy); + memset(copy_from, 0, to_copy); + + out += to_copy; + bytes_to_yield -= to_copy; + rng->bytes_left -= to_copy; + } +} + +/** + * Extract <b>n</b> bytes from <b>rng</b> into the buffer at <b>out</b>. + **/ +void +crypto_fast_rng_getbytes(crypto_fast_rng_t *rng, uint8_t *out, size_t n) +{ + if (PREDICT_UNLIKELY(n > BUFLEN)) { + /* The user has asked for a lot of output; generate it from a stream + * cipher seeded by the PRNG rather than by pulling it out of the PRNG + * directly. + */ + uint8_t seed[SEED_LEN]; + crypto_fast_rng_getbytes_impl(rng, seed, SEED_LEN); + crypto_cipher_t *c = cipher_from_seed(seed); + memset(out, 0, n); + crypto_cipher_crypt_inplace(c, (char*)out, n); + crypto_cipher_free(c); + memwipe(seed, 0, sizeof(seed)); + return; + } + + crypto_fast_rng_getbytes_impl(rng, out, n); +} + +#if defined(TOR_UNIT_TESTS) +/** for white-box testing: return the number of bytes that are returned from + * the user for each invocation of the stream cipher in this RNG. */ +size_t +crypto_fast_rng_get_bytes_used_per_stream(void) +{ + return BUFLEN; +} +#endif diff --git a/src/lib/crypt_ops/crypto_rand_numeric.c b/src/lib/crypt_ops/crypto_rand_numeric.c new file mode 100644 index 0000000000..d02c5cdcfa --- /dev/null +++ b/src/lib/crypt_ops/crypto_rand_numeric.c @@ -0,0 +1,166 @@ +/** + * \file crypto_rand_numeric.c + * + * \brief Functions for retrieving uniformly distributed numbers + * from our PRNGs. + **/ + +#include "lib/crypt_ops/crypto_rand.h" +#include "lib/log/util_bug.h" + +/** + * Implementation macro: yields code that returns a uniform unbiased + * random number between 0 and limit. "type" is the type of the number to + * return; "maxval" is the largest possible value of "type"; and "fill_stmt" + * is a code snippet that fills an object named "val" with random bits. + **/ +#define IMPLEMENT_RAND_UNSIGNED(type, maxval, limit, fill_stmt) \ + do { \ + type val; \ + type cutoff; \ + tor_assert((limit) > 0); \ + \ + /* We ignore any values that are >= 'cutoff,' to avoid biasing */ \ + /* the distribution with clipping at the upper end of the type's */ \ + /* range. */ \ + cutoff = (maxval) - ((maxval)%(limit)); \ + while (1) { \ + fill_stmt; \ + if (val < cutoff) \ + return val % (limit); \ + } \ + } while (0) + +/** + * Return a pseudorandom integer chosen uniformly from the values between 0 + * and <b>limit</b>-1 inclusive. limit must be strictly between 0 and + * UINT_MAX. */ +unsigned +crypto_rand_uint(unsigned limit) +{ + tor_assert(limit < UINT_MAX); + IMPLEMENT_RAND_UNSIGNED(unsigned, UINT_MAX, limit, + crypto_rand((char*)&val, sizeof(val))); +} + +/** + * Return a pseudorandom integer, chosen uniformly from the values + * between 0 and <b>max</b>-1 inclusive. <b>max</b> must be between 1 and + * INT_MAX+1, inclusive. + */ +int +crypto_rand_int(unsigned int max) +{ + /* We can't use IMPLEMENT_RAND_UNSIGNED directly, since we're trying + * to return a signed type. Instead we make sure that the range is + * reasonable for a nonnegative int, use crypto_rand_uint(), and cast. + */ + tor_assert(max <= ((unsigned int)INT_MAX)+1); + + return (int)crypto_rand_uint(max); +} + +/** + * Return a pseudorandom integer, chosen uniformly from the values i such + * that min <= i < max. + * + * <b>min</b> MUST be in range [0, <b>max</b>). + * <b>max</b> MUST be in range (min, INT_MAX]. + **/ +int +crypto_rand_int_range(unsigned int min, unsigned int max) +{ + tor_assert(min < max); + tor_assert(max <= INT_MAX); + + /* The overflow is avoided here because crypto_rand_int() returns a value + * between 0 and (max - min) inclusive. */ + return min + crypto_rand_int(max - min); +} + +/** + * As crypto_rand_int_range, but supports uint64_t. + **/ +uint64_t +crypto_rand_uint64_range(uint64_t min, uint64_t max) +{ + tor_assert(min < max); + return min + crypto_rand_uint64(max - min); +} + +/** + * As crypto_rand_int_range, but supports time_t. + **/ +time_t +crypto_rand_time_range(time_t min, time_t max) +{ + tor_assert(min < max); + return min + (time_t)crypto_rand_uint64(max - min); +} + +/** + * Return a pseudorandom 64-bit integer, chosen uniformly from the values + * between 0 and <b>max</b>-1 inclusive. + **/ +uint64_t +crypto_rand_uint64(uint64_t max) +{ + tor_assert(max < UINT64_MAX); + IMPLEMENT_RAND_UNSIGNED(uint64_t, UINT64_MAX, max, + crypto_rand((char*)&val, sizeof(val))); +} + +#if SIZEOF_INT == 4 +#define UINT_MAX_AS_DOUBLE 4294967296.0 +#elif SIZEOF_INT == 8 +#define UINT_MAX_AS_DOUBLE 1.8446744073709552e+19 +#else +#error SIZEOF_INT is neither 4 nor 8 +#endif /* SIZEOF_INT == 4 || ... */ + +/** + * Return a pseudorandom double d, chosen uniformly from the range + * 0.0 <= d < 1.0. + **/ +double +crypto_rand_double(void) +{ + /* We just use an unsigned int here; we don't really care about getting + * more than 32 bits of resolution */ + unsigned int u; + crypto_rand((char*)&u, sizeof(u)); + return ((double)u) / UINT_MAX_AS_DOUBLE; +} + +/** + * As crypto_rand_uint, but extract the result from a crypto_fast_rng_t + */ +unsigned +crypto_fast_rng_get_uint(crypto_fast_rng_t *rng, unsigned limit) +{ + tor_assert(limit < UINT_MAX); + IMPLEMENT_RAND_UNSIGNED(unsigned, UINT_MAX, limit, + crypto_fast_rng_getbytes(rng, (void*)&val, sizeof(val))); +} + +/** + * As crypto_rand_uint64, but extract the result from a crypto_fast_rng_t. + */ +uint64_t +crypto_fast_rng_get_uint64(crypto_fast_rng_t *rng, uint64_t limit) +{ + tor_assert(limit < UINT64_MAX); + IMPLEMENT_RAND_UNSIGNED(uint64_t, UINT64_MAX, limit, + crypto_fast_rng_getbytes(rng, (void*)&val, sizeof(val))); +} + +/** + * As crypto_rand_, but extract the result from a crypto_fast_rng_t. + */ +double +crypto_fast_rng_get_double(crypto_fast_rng_t *rng) +{ + unsigned int u; + crypto_fast_rng_getbytes(rng, (void*)&u, sizeof(u)); + return ((double)u) / UINT_MAX_AS_DOUBLE; +} diff --git a/src/lib/crypt_ops/crypto_sys.h b/src/lib/crypt_ops/crypto_sys.h new file mode 100644 index 0000000000..894243b175 --- /dev/null +++ b/src/lib/crypt_ops/crypto_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file log_crypto.h + * \brief Declare subsystem object for the crypto module. + **/ + +#ifndef TOR_CRYPTO_SYS_H +#define TOR_CRYPTO_SYS_H + +extern const struct subsys_fns_t sys_crypto; + +#endif /* !defined(TOR_CRYPTO_SYS_H) */ diff --git a/src/lib/crypt_ops/digestset.c b/src/lib/crypt_ops/digestset.c index 0dba64d595..c931b58369 100644 --- a/src/lib/crypt_ops/digestset.c +++ b/src/lib/crypt_ops/digestset.c @@ -11,7 +11,7 @@ #include "lib/crypt_ops/crypto_rand.h" #include "lib/defs/digest_sizes.h" #include "lib/crypt_ops/digestset.h" -#include "siphash.h" +#include "ext/siphash.h" /* Wrap our hash function to have the signature that the bloom filter * needs. */ diff --git a/src/lib/crypt_ops/include.am b/src/lib/crypt_ops/include.am index 1022096fdc..4730440143 100644 --- a/src/lib/crypt_ops/include.am +++ b/src/lib/crypt_ops/include.am @@ -17,6 +17,8 @@ src_lib_libtor_crypt_ops_a_SOURCES = \ src/lib/crypt_ops/crypto_ope.c \ src/lib/crypt_ops/crypto_pwbox.c \ src/lib/crypt_ops/crypto_rand.c \ + src/lib/crypt_ops/crypto_rand_fast.c \ + src/lib/crypt_ops/crypto_rand_numeric.c \ src/lib/crypt_ops/crypto_rsa.c \ src/lib/crypt_ops/crypto_s2k.c \ src/lib/crypt_ops/crypto_util.c \ @@ -66,5 +68,6 @@ noinst_HEADERS += \ src/lib/crypt_ops/crypto_rand.h \ src/lib/crypt_ops/crypto_rsa.h \ src/lib/crypt_ops/crypto_s2k.h \ + src/lib/crypt_ops/crypto_sys.h \ src/lib/crypt_ops/crypto_util.h \ src/lib/crypt_ops/digestset.h diff --git a/src/lib/defs/include.am b/src/lib/defs/include.am index 48ee7f29fc..6a7f9114ea 100644 --- a/src/lib/defs/include.am +++ b/src/lib/defs/include.am @@ -2,4 +2,5 @@ noinst_HEADERS += \ src/lib/defs/dh_sizes.h \ src/lib/defs/digest_sizes.h \ + src/lib/defs/time.h \ src/lib/defs/x25519_sizes.h diff --git a/src/lib/defs/time.h b/src/lib/defs/time.h new file mode 100644 index 0000000000..c25f5022c5 --- /dev/null +++ b/src/lib/defs/time.h @@ -0,0 +1,23 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_TIME_DEFS_H +#define TOR_TIME_DEFS_H + +/** + * \file time.h + * + * \brief Definitions for timing-related constants. + **/ + +/** How many microseconds per second */ +#define TOR_USEC_PER_SEC (1000000) +/** How many nanoseconds per microsecond */ +#define TOR_NSEC_PER_USEC (1000) +/* How many nanoseconds per millisecond */ +#define TOR_NSEC_PER_MSEC (1000*1000) + +#endif diff --git a/src/lib/encoding/.may_include b/src/lib/encoding/.may_include index 7c2ef36929..c9bf4b1786 100644 --- a/src/lib/encoding/.may_include +++ b/src/lib/encoding/.may_include @@ -1,5 +1,6 @@ orconfig.h lib/cc/*.h +lib/container/*.h lib/ctime/*.h lib/encoding/*.h lib/intmath/*.h diff --git a/src/lib/encoding/binascii.c b/src/lib/encoding/binascii.c index bd063440d6..de4d1648bb 100644 --- a/src/lib/encoding/binascii.c +++ b/src/lib/encoding/binascii.c @@ -179,6 +179,18 @@ base64_encode_size(size_t srclen, int flags) return enclen; } +/** Return an upper bound on the number of bytes that might be needed to hold + * the data from decoding the base64 string <b>srclen</b>. This is only an + * upper bound, since some part of the base64 string might be padding or + * space. */ +size_t +base64_decode_maxsize(size_t srclen) +{ + tor_assert(srclen < INT_MAX / 3); + + return CEIL_DIV(srclen * 3, 4); +} + /** Internal table mapping 6 bit values to the Base64 alphabet. */ static const char base64_encode_table[64] = { 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', diff --git a/src/lib/encoding/binascii.h b/src/lib/encoding/binascii.h index 7e3cc04f09..44998bb85b 100644 --- a/src/lib/encoding/binascii.h +++ b/src/lib/encoding/binascii.h @@ -42,6 +42,7 @@ const char *hex_str(const char *from, size_t fromlen); #define BASE64_ENCODE_MULTILINE 1 size_t base64_encode_size(size_t srclen, int flags); +size_t base64_decode_maxsize(size_t srclen); int base64_encode(char *dest, size_t destlen, const char *src, size_t srclen, int flags); int base64_decode(char *dest, size_t destlen, const char *src, size_t srclen); diff --git a/src/lib/encoding/include.am b/src/lib/encoding/include.am index 2d2aa3988a..83e9211b6f 100644 --- a/src/lib/encoding/include.am +++ b/src/lib/encoding/include.am @@ -9,6 +9,7 @@ src_lib_libtor_encoding_a_SOURCES = \ src/lib/encoding/confline.c \ src/lib/encoding/cstring.c \ src/lib/encoding/keyval.c \ + src/lib/encoding/kvline.c \ src/lib/encoding/pem.c \ src/lib/encoding/time_fmt.c @@ -22,5 +23,6 @@ noinst_HEADERS += \ src/lib/encoding/confline.h \ src/lib/encoding/cstring.h \ src/lib/encoding/keyval.h \ + src/lib/encoding/kvline.h \ src/lib/encoding/pem.h \ src/lib/encoding/time_fmt.h diff --git a/src/lib/encoding/kvline.c b/src/lib/encoding/kvline.c new file mode 100644 index 0000000000..307adc3f12 --- /dev/null +++ b/src/lib/encoding/kvline.c @@ -0,0 +1,239 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file kvline.c + * + * \brief Manipulating lines of key-value pairs. + **/ + +#include "orconfig.h" + +#include "lib/container/smartlist.h" +#include "lib/encoding/confline.h" +#include "lib/encoding/cstring.h" +#include "lib/encoding/kvline.h" +#include "lib/malloc/malloc.h" +#include "lib/string/compat_ctype.h" +#include "lib/string/printf.h" +#include "lib/string/util_string.h" +#include "lib/log/escape.h" +#include "lib/log/util_bug.h" + +#include <stdbool.h> +#include <stddef.h> +#include <string.h> + +/** Return true iff we need to quote and escape the string <b>s</b> to encode + * it. */ +static bool +needs_escape(const char *s, bool as_keyless_val) +{ + if (as_keyless_val && *s == 0) + return true; + + for (; *s; ++s) { + if (*s >= 127 || TOR_ISSPACE(*s) || ! TOR_ISPRINT(*s) || + *s == '\'' || *s == '\"') { + return true; + } + } + return false; +} + +/** + * Return true iff the key in <b>line</b> is not set. + **/ +static bool +line_has_no_key(const config_line_t *line) +{ + return line->key == NULL || strlen(line->key) == 0; +} + +/** + * Return true iff the all the lines in <b>line</b> can be encoded + * using <b>flags</b>. + **/ +static bool +kvline_can_encode_lines(const config_line_t *line, unsigned flags) +{ + for ( ; line; line = line->next) { + const bool keyless = line_has_no_key(line); + if (keyless) { + if (! (flags & KV_OMIT_KEYS)) { + /* If KV_OMIT_KEYS is not set, we can't encode a line with no key. */ + return false; + } + if (strchr(line->value, '=') && !( flags & KV_QUOTED)) { + /* We can't have a keyless value with = without quoting it. */ + return false; + } + } + + if (needs_escape(line->value, keyless) && ! (flags & KV_QUOTED)) { + /* If KV_QUOTED is false, we can't encode a value that needs quotes. */ + return false; + } + if (line->key && strlen(line->key) && + (needs_escape(line->key, false) || strchr(line->key, '='))) { + /* We can't handle keys that need quoting. */ + return false; + } + } + return true; +} + +/** + * Encode a linked list of lines in <b>line</b> as a series of 'Key=Value' + * pairs, using the provided <b>flags</b> to encode it. Return a newly + * allocated string on success, or NULL on failure. + * + * If KV_QUOTED is set in <b>flags</b>, then all values that contain + * spaces or unusual characters are escaped and quoted. Otherwise, such + * values are not allowed. + * + * If KV_OMIT_KEYS is set in <b>flags</b>, then pairs with empty keys are + * allowed, and are encoded as 'Value'. Otherwise, such pairs are not + * allowed. + */ +char * +kvline_encode(const config_line_t *line, + unsigned flags) +{ + if (!kvline_can_encode_lines(line, flags)) + return NULL; + + smartlist_t *elements = smartlist_new(); + + for (; line; line = line->next) { + + const char *k = ""; + const char *eq = "="; + const char *v = ""; + const bool keyless = line_has_no_key(line); + bool esc = needs_escape(line->value, keyless); + char *tmp = NULL; + + if (! keyless) { + k = line->key; + } else { + eq = ""; + if (strchr(line->value, '=')) { + esc = true; + } + } + + if (esc) { + tmp = esc_for_log(line->value); + v = tmp; + } else { + v = line->value; + } + + smartlist_add_asprintf(elements, "%s%s%s", k, eq, v); + tor_free(tmp); + } + + char *result = smartlist_join_strings(elements, " ", 0, NULL); + + SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp)); + smartlist_free(elements); + + return result; +} + +/** + * Decode a <b>line</b> containing a series of space-separated 'Key=Value' + * pairs, using the provided <b>flags</b> to decode it. Return a newly + * allocated list of pairs on success, or NULL on failure. + * + * If KV_QUOTED is set in <b>flags</b>, then (double-)quoted values are + * allowed. Otherwise, such values are not allowed. + * + * If KV_OMIT_KEYS is set in <b>flags</b>, then values without keys are + * allowed. Otherwise, such values are not allowed. + */ +config_line_t * +kvline_parse(const char *line, unsigned flags) +{ + const char *cp = line, *cplast = NULL; + bool omit_keys = (flags & KV_OMIT_KEYS) != 0; + bool quoted = (flags & KV_QUOTED) != 0; + + config_line_t *result = NULL; + config_line_t **next_line = &result; + + char *key = NULL; + char *val = NULL; + + while (*cp) { + key = val = NULL; + { + size_t idx = strspn(cp, " \t\r\v\n"); + cp += idx; + } + if (BUG(cp == cplast)) { + /* If we didn't parse anything, this code is broken. */ + goto err; // LCOV_EXCL_LINE + } + cplast = cp; + if (! *cp) + break; /* End of string; we're done. */ + + /* Possible formats are K=V, K="V", V, and "V", depending on flags. */ + + /* Find the key. */ + if (*cp != '\"') { + size_t idx = strcspn(cp, " \t\r\v\n="); + + if (cp[idx] == '=') { + key = tor_memdup_nulterm(cp, idx); + cp += idx + 1; + } else { + if (!omit_keys) + goto err; + } + } + + if (*cp == '\"') { + /* The type is "V". */ + if (!quoted) + goto err; + size_t len=0; + cp = unescape_string(cp, &val, &len); + if (cp == NULL || len != strlen(val)) { + // The string contains a NUL or is badly coded. + goto err; + } + } else { + size_t idx = strcspn(cp, " \t\r\v\n"); + val = tor_memdup_nulterm(cp, idx); + cp += idx; + } + + if (key && strlen(key) == 0) { + /* We don't allow empty keys. */ + goto err; + } + + *next_line = tor_malloc_zero(sizeof(config_line_t)); + (*next_line)->key = key ? key : tor_strdup(""); + (*next_line)->value = val; + next_line = &(*next_line)->next; + key = val = NULL; + } + + if (!kvline_can_encode_lines(result, flags)) { + goto err; + } + return result; + + err: + tor_free(key); + tor_free(val); + config_free_lines(result); + return NULL; +} diff --git a/src/lib/encoding/kvline.h b/src/lib/encoding/kvline.h new file mode 100644 index 0000000000..4eed30a223 --- /dev/null +++ b/src/lib/encoding/kvline.h @@ -0,0 +1,24 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file kvline.h + * + * \brief Header for kvline.c + **/ + +#ifndef TOR_KVLINE_H +#define TOR_KVLINE_H + +struct config_line_t; + +#define KV_QUOTED (1u<<0) +#define KV_OMIT_KEYS (1u<<1) + +struct config_line_t *kvline_parse(const char *line, unsigned flags); +char *kvline_encode(const struct config_line_t *line, unsigned flags); + +#endif /* !defined(TOR_KVLINE_H) */ diff --git a/src/lib/err/.may_include b/src/lib/err/.may_include index 48cc0ef088..daa1b6e4ca 100644 --- a/src/lib/err/.may_include +++ b/src/lib/err/.may_include @@ -1,3 +1,5 @@ orconfig.h lib/cc/*.h lib/err/*.h +lib/subsys/*.h +lib/version/*.h
\ No newline at end of file diff --git a/src/lib/err/include.am b/src/lib/err/include.am index f2a409c51e..43adcd2694 100644 --- a/src/lib/err/include.am +++ b/src/lib/err/include.am @@ -6,8 +6,9 @@ noinst_LIBRARIES += src/lib/libtor-err-testing.a endif src_lib_libtor_err_a_SOURCES = \ - src/lib/err/backtrace.c \ - src/lib/err/torerr.c + src/lib/err/backtrace.c \ + src/lib/err/torerr.c \ + src/lib/err/torerr_sys.c src_lib_libtor_err_testing_a_SOURCES = \ $(src_lib_libtor_err_a_SOURCES) @@ -16,4 +17,5 @@ src_lib_libtor_err_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) noinst_HEADERS += \ src/lib/err/backtrace.h \ - src/lib/err/torerr.h + src/lib/err/torerr.h \ + src/lib/err/torerr_sys.h diff --git a/src/lib/err/torerr.c b/src/lib/err/torerr.c index 6b5224273a..1a246e995c 100644 --- a/src/lib/err/torerr.c +++ b/src/lib/err/torerr.c @@ -123,6 +123,16 @@ tor_log_set_sigsafe_err_fds(const int *fds, int n) } /** + * Reset the list of emergency error fds to its default. + */ +void +tor_log_reset_sigsafe_err_fds(void) +{ + int fds[] = { STDERR_FILENO }; + tor_log_set_sigsafe_err_fds(fds, 1); +} + +/** * Set the granularity (in ms) to use when reporting fatal errors outside * the logging system. */ diff --git a/src/lib/err/torerr.h b/src/lib/err/torerr.h index 6ae91fbe85..0badaf7c6d 100644 --- a/src/lib/err/torerr.h +++ b/src/lib/err/torerr.h @@ -39,6 +39,7 @@ void tor_raw_assertion_failed_msg_(const char *file, int line, void tor_log_err_sigsafe(const char *m, ...); int tor_log_get_sigsafe_err_fds(const int **out); void tor_log_set_sigsafe_err_fds(const int *fds, int n); +void tor_log_reset_sigsafe_err_fds(void); void tor_log_sigsafe_err_set_granularity(int ms); int format_hex_number_sigsafe(unsigned long x, char *buf, int max_len); diff --git a/src/lib/err/torerr_sys.c b/src/lib/err/torerr_sys.c new file mode 100644 index 0000000000..3ab1b3c4e1 --- /dev/null +++ b/src/lib/err/torerr_sys.c @@ -0,0 +1,40 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file torerr_sys.c + * \brief Subsystem object for the error handling subsystem. + **/ + +#include "orconfig.h" +#include "lib/err/backtrace.h" +#include "lib/err/torerr.h" +#include "lib/err/torerr_sys.h" +#include "lib/subsys/subsys.h" +#include "lib/version/torversion.h" + +#include <stddef.h> + +static int +subsys_torerr_initialize(void) +{ + if (configure_backtrace_handler(get_version()) < 0) + return -1; + tor_log_reset_sigsafe_err_fds(); + + return 0; +} +static void +subsys_torerr_shutdown(void) +{ + tor_log_reset_sigsafe_err_fds(); + clean_up_backtrace_handler(); +} + +const subsys_fns_t sys_torerr = { + .name = "err", + .level = -100, + .supported = true, + .initialize = subsys_torerr_initialize, + .shutdown = subsys_torerr_shutdown +}; diff --git a/src/lib/err/torerr_sys.h b/src/lib/err/torerr_sys.h new file mode 100644 index 0000000000..c947695689 --- /dev/null +++ b/src/lib/err/torerr_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file torerr_sys.h + * \brief Declare subsystem object for torerr.c + **/ + +#ifndef TOR_TORERR_SYS_H +#define TOR_TORERR_SYS_H + +extern const struct subsys_fns_t sys_torerr; + +#endif /* !defined(TOR_TORERR_SYS_H) */ diff --git a/src/lib/evloop/.may_include b/src/lib/evloop/.may_include index 30af508914..273de7bb94 100644 --- a/src/lib/evloop/.may_include +++ b/src/lib/evloop/.may_include @@ -12,5 +12,5 @@ lib/testsupport/*.h lib/thread/*.h lib/time/*.h -src/ext/timeouts/timeout.c -tor_queue.h
\ No newline at end of file +ext/timeouts/timeout.c +ext/tor_queue.h
\ No newline at end of file diff --git a/src/lib/evloop/timers.c b/src/lib/evloop/timers.c index e46d2635a8..4b2a96ef7d 100644 --- a/src/lib/evloop/timers.c +++ b/src/lib/evloop/timers.c @@ -80,7 +80,8 @@ struct timeout_cb { * use 32-bit math. */ #define WHEEL_BIT 5 #endif -#include "src/ext/timeouts/timeout.c" + +#include "ext/timeouts/timeout.c" static struct timeouts *global_timeouts = NULL; static struct mainloop_event_t *global_timer_event = NULL; diff --git a/src/lib/evloop/workqueue.c b/src/lib/evloop/workqueue.c index 931f65e710..b36a02da5e 100644 --- a/src/lib/evloop/workqueue.c +++ b/src/lib/evloop/workqueue.c @@ -15,7 +15,7 @@ * * The main thread informs the worker threads of pending work by using a * condition variable. The workers inform the main process of completed work - * by using an alert_sockets_t object, as implemented in compat_threads.c. + * by using an alert_sockets_t object, as implemented in net/alertsock.c. * * The main thread can also queue an "update" that will be handled by all the * workers. This is useful for updating state that all the workers share. @@ -36,7 +36,7 @@ #include "lib/net/socket.h" #include "lib/thread/threads.h" -#include "tor_queue.h" +#include "ext/tor_queue.h" #include <event2/event.h> #include <string.h> @@ -622,8 +622,8 @@ reply_event_cb(evutil_socket_t sock, short events, void *arg) tp->reply_cb(tp); } -/** Register the threadpool <b>tp</b>'s reply queue with the libevent - * mainloop of <b>base</b>. If <b>tp</b> is provided, it is run after +/** Register the threadpool <b>tp</b>'s reply queue with Tor's global + * libevent mainloop. If <b>cb</b> is provided, it is run after * each time there is work to process from the reply queue. Return 0 on * success, -1 on failure. */ diff --git a/src/lib/evloop/workqueue.h b/src/lib/evloop/workqueue.h index 333a3f6dde..d0ee8f2be2 100644 --- a/src/lib/evloop/workqueue.h +++ b/src/lib/evloop/workqueue.h @@ -63,7 +63,6 @@ replyqueue_t *threadpool_get_replyqueue(threadpool_t *tp); replyqueue_t *replyqueue_new(uint32_t alertsocks_flags); void replyqueue_process(replyqueue_t *queue); -struct event_base; int threadpool_register_reply_event(threadpool_t *tp, void (*cb)(threadpool_t *tp)); diff --git a/src/lib/fdio/fdio.c b/src/lib/fdio/fdio.c index 6c87af791d..078af6a9ba 100644 --- a/src/lib/fdio/fdio.c +++ b/src/lib/fdio/fdio.c @@ -17,12 +17,16 @@ #ifdef _WIN32 #include <windows.h> #endif +#ifdef HAVE_SYS_TYPES_H +#include <sys/types.h> +#endif #include "lib/fdio/fdio.h" #include "lib/cc/torint.h" #include "lib/err/torerr.h" #include <stdlib.h> +#include <stdio.h> /** @{ */ /** Some old versions of Unix didn't define constants for these values, diff --git a/src/lib/fs/.may_include b/src/lib/fs/.may_include index b1e49fc891..c192e6181c 100644 --- a/src/lib/fs/.may_include +++ b/src/lib/fs/.may_include @@ -13,4 +13,4 @@ lib/malloc/*.h lib/memarea/*.h lib/sandbox/*.h lib/string/*.h -lib/testsupport/testsupport.h +lib/testsupport/*.h diff --git a/src/lib/intmath/cmp.h b/src/lib/intmath/cmp.h index d0b0e8b954..67a738861b 100644 --- a/src/lib/intmath/cmp.h +++ b/src/lib/intmath/cmp.h @@ -36,4 +36,7 @@ ((v) > (max)) ? (max) : \ (v) ) +/** Give the absolute value of <b>x</b>, independent of its type. */ +#define ABS(x) ( ((x)<0) ? -(x) : (x) ) + #endif /* !defined(TOR_INTMATH_CMP_H) */ diff --git a/src/lib/log/.may_include b/src/lib/log/.may_include index 852173aab3..11c87f0a0d 100644 --- a/src/lib/log/.may_include +++ b/src/lib/log/.may_include @@ -9,7 +9,7 @@ lib/lock/*.h lib/log/*.h lib/malloc/*.h lib/string/*.h +lib/subsys/*.h lib/testsupport/*.h +lib/version/*.h lib/wallclock/*.h - -micro-revision.i
\ No newline at end of file diff --git a/src/lib/log/include.am b/src/lib/log/include.am index 4a6c9b3686..9d3dbe3104 100644 --- a/src/lib/log/include.am +++ b/src/lib/log/include.am @@ -7,9 +7,9 @@ endif src_lib_libtor_log_a_SOURCES = \ src/lib/log/escape.c \ - src/lib/log/git_revision.c \ src/lib/log/ratelim.c \ src/lib/log/log.c \ + src/lib/log/log_sys.c \ src/lib/log/util_bug.c if WIN32 @@ -21,16 +21,10 @@ src_lib_libtor_log_testing_a_SOURCES = \ src_lib_libtor_log_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_log_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) -# Declare that these object files depend on micro-revision.i. Without this -# rule, we could try to build them before micro-revision.i was created. -src/lib/log/git_revision.$(OBJEXT) \ - src/lib/log/src_lib_libtor_log_testing_a-git_revision.$(OBJEXT): \ - micro-revision.i - noinst_HEADERS += \ src/lib/log/escape.h \ - src/lib/log/git_revision.h \ src/lib/log/ratelim.h \ src/lib/log/log.h \ + src/lib/log/log_sys.h \ src/lib/log/util_bug.h \ src/lib/log/win32err.h diff --git a/src/lib/log/log.c b/src/lib/log/log.c index a9ad38fb25..d21d8d1d41 100644 --- a/src/lib/log/log.c +++ b/src/lib/log/log.c @@ -32,7 +32,8 @@ #define LOG_PRIVATE #include "lib/log/log.h" -#include "lib/log/git_revision.h" +#include "lib/log/log_sys.h" +#include "lib/version/git_revision.h" #include "lib/log/ratelim.h" #include "lib/lock/compat_mutex.h" #include "lib/smartlist_core/smartlist_core.h" @@ -1267,7 +1268,7 @@ static const char *domain_list[] = { "GENERAL", "CRYPTO", "NET", "CONFIG", "FS", "PROTOCOL", "MM", "HTTP", "APP", "CONTROL", "CIRC", "REND", "BUG", "DIR", "DIRSERV", "OR", "EDGE", "ACCT", "HIST", "HANDSHAKE", "HEARTBEAT", "CHANNEL", - "SCHED", "GUARD", "CONSDIFF", "DOS", NULL + "SCHED", "GUARD", "CONSDIFF", "DOS", "PROCESS", "PT", "BTRACK", NULL }; /** Return a bitmask for the log domain for which <b>domain</b> is the name, diff --git a/src/lib/log/log.h b/src/lib/log/log.h index d7a5070610..dbc1c47021 100644 --- a/src/lib/log/log.h +++ b/src/lib/log/log.h @@ -107,8 +107,14 @@ #define LD_CONSDIFF (1u<<24) /** Denial of Service mitigation. */ #define LD_DOS (1u<<25) +/** Processes */ +#define LD_PROCESS (1u<<26) +/** Pluggable Transports. */ +#define LD_PT (1u<<27) +/** Bootstrap tracker. */ +#define LD_BTRACK (1u<<28) /** Number of logging domains in the code. */ -#define N_LOGGING_DOMAINS 26 +#define N_LOGGING_DOMAINS 29 /** This log message is not safe to send to a callback-based logger * immediately. Used as a flag, not a log domain. */ diff --git a/src/lib/log/log_sys.c b/src/lib/log/log_sys.c new file mode 100644 index 0000000000..d1080f2264 --- /dev/null +++ b/src/lib/log/log_sys.c @@ -0,0 +1,35 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file log_sys.c + * \brief Setup and tear down the logging module. + **/ + +#include "orconfig.h" +#include "lib/subsys/subsys.h" +#include "lib/log/escape.h" +#include "lib/log/log.h" +#include "lib/log/log_sys.h" + +static int +subsys_logging_initialize(void) +{ + init_logging(0); + return 0; +} + +static void +subsys_logging_shutdown(void) +{ + logs_free_all(); + escaped(NULL); +} + +const subsys_fns_t sys_logging = { + .name = "log", + .supported = true, + .level = -90, + .initialize = subsys_logging_initialize, + .shutdown = subsys_logging_shutdown, +}; diff --git a/src/lib/log/log_sys.h b/src/lib/log/log_sys.h new file mode 100644 index 0000000000..7043253066 --- /dev/null +++ b/src/lib/log/log_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file log_sys.h + * \brief Declare subsystem object for the logging module. + **/ + +#ifndef TOR_LOG_SYS_H +#define TOR_LOG_SYS_H + +extern const struct subsys_fns_t sys_logging; + +#endif /* !defined(TOR_LOG_SYS_H) */ diff --git a/src/lib/malloc/.may_include b/src/lib/malloc/.may_include index cc62bb1013..7686bf862a 100644 --- a/src/lib/malloc/.may_include +++ b/src/lib/malloc/.may_include @@ -3,4 +3,4 @@ orconfig.h lib/cc/*.h lib/err/*.h lib/malloc/*.h -lib/testsupport/testsupport.h +lib/testsupport/*.h diff --git a/src/lib/malloc/include.am b/src/lib/malloc/include.am index 502cc1c6b7..95d96168e1 100644 --- a/src/lib/malloc/include.am +++ b/src/lib/malloc/include.am @@ -6,7 +6,8 @@ noinst_LIBRARIES += src/lib/libtor-malloc-testing.a endif src_lib_libtor_malloc_a_SOURCES = \ - src/lib/malloc/malloc.c + src/lib/malloc/malloc.c \ + src/lib/malloc/map_anon.c if USE_OPENBSD_MALLOC src_lib_libtor_malloc_a_SOURCES += src/ext/OpenBSD_malloc_Linux.c @@ -18,4 +19,5 @@ src_lib_libtor_malloc_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_malloc_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) noinst_HEADERS += \ - src/lib/malloc/malloc.h + src/lib/malloc/malloc.h \ + src/lib/malloc/map_anon.h diff --git a/src/lib/malloc/map_anon.c b/src/lib/malloc/map_anon.c new file mode 100644 index 0000000000..2430f7ad11 --- /dev/null +++ b/src/lib/malloc/map_anon.c @@ -0,0 +1,217 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file map_anon.c + * \brief Manage anonymous mappings. + **/ + +#include "orconfig.h" +#include "lib/malloc/map_anon.h" +#include "lib/malloc/malloc.h" +#include "lib/err/torerr.h" + +#ifdef HAVE_SYS_MMAN_H +#include <sys/mman.h> +#endif +#ifdef HAVE_SYS_TYPES_H +#include <sys/types.h> +#endif +#ifdef HAVE_MACH_VM_INHERIT_H +#include <mach/vm_inherit.h> +#endif + +#ifdef _WIN32 +#include <windows.h> +#endif + +/** + * Macro to get the high bytes of a size_t, if there are high bytes. + * Windows needs this; other operating systems define a size_t that does + * what it should. + */ +#if SIZEOF_SIZE_T > 4 +#define HIGH_SIZE_T_BYTES(sz) ((sz) >> 32) +#else +#define HIGH_SIZE_T_BYTES(sz) (0) +#endif + +/* Here we define a MINHERIT macro that is minherit() or madvise(), depending + * on what we actually want. + * + * If there's a flag that sets pages to zero after fork, we define FLAG_ZERO + * to be that flag. If there's a flag unmaps pages after fork, we define + * FLAG_NOINHERIT to be that flag. + */ +#if defined(HAVE_MINHERIT) +#define MINHERIT minherit + +#ifdef INHERIT_ZERO +#define FLAG_ZERO INHERIT_ZERO +#elif defined(MAP_INHERIT_ZERO) +#define FLAG_ZERO MAP_INHERIT_ZERO +#endif +#ifdef INHERIT_NONE +#define FLAG_NOINHERIT INHERIT_NONE +#elif defined(VM_INHERIT_NONE) +#define FLAG_NOINHERIT VM_INHERIT_NONE +#elif defined(MAP_INHERIT_NONE) +#define FLAG_NOINHERIT MAP_INHERIT_NONE +#endif + +#elif defined(HAVE_MADVISE) + +#define MINHERIT madvise + +#ifdef MADV_WIPEONFORK +#define FLAG_ZERO MADV_WIPEONFORK +#endif +#ifdef MADV_DONTFORK +#define FLAG_NOINHERIT MADV_DONTFORK +#endif + +#endif + +/** + * Helper: try to prevent the <b>sz</b> bytes at <b>mem</b> from being swapped + * to disk. Return 0 on success or if the facility is not available on this + * OS; return -1 on failure. + */ +static int +lock_mem(void *mem, size_t sz) +{ +#ifdef _WIN32 + return VirtualLock(mem, sz) ? 0 : -1; +#elif defined(HAVE_MLOCK) + return mlock(mem, sz); +#else + (void) mem; + (void) sz; + + return 0; +#endif +} + +/** + * Helper: try to prevent the <b>sz</b> bytes at <b>mem</b> from appearing in + * a core dump. Return 0 on success or if the facility is not available on + * this OS; return -1 on failure. + */ +static int +nodump_mem(void *mem, size_t sz) +{ +#if defined(MADV_DONTDUMP) + return madvise(mem, sz, MADV_DONTDUMP); +#else + (void) mem; + (void) sz; + return 0; +#endif +} + +/** + * Helper: try to prevent the <b>sz</b> bytes at <b>mem</b> from being + * accessible in child processes -- ideally by having them set to 0 after a + * fork, and if that doesn't work, by having them unmapped after a fork. + * Return 0 on success or if the facility is not available on this OS; return + * -1 on failure. + */ +static int +noinherit_mem(void *mem, size_t sz) +{ +#ifdef FLAG_ZERO + int r = MINHERIT(mem, sz, FLAG_ZERO); + if (r == 0) + return 0; +#endif +#ifdef FLAG_NOINHERIT + return MINHERIT(mem, sz, FLAG_NOINHERIT); +#else + (void)mem; + (void)sz; + return 0; +#endif +} + +/** + * Return a new anonymous memory mapping that holds <b>sz</b> bytes. + * + * Memory mappings are unlike the results from malloc() in that they are + * handled separately by the operating system, and as such can have different + * kernel-level flags set on them. + * + * The "flags" argument may be zero or more of ANONMAP_PRIVATE and + * ANONMAP_NOINHERIT. + * + * Memory returned from this function must be released with + * tor_munmap_anonymous(). + * + * [Note: OS people use the word "anonymous" here to mean that the memory + * isn't associated with any file. This has *nothing* to do with the kind of + * anonymity that Tor is trying to provide.] + */ +void * +tor_mmap_anonymous(size_t sz, unsigned flags) +{ + void *ptr; +#if defined(_WIN32) + HANDLE mapping = CreateFileMapping(INVALID_HANDLE_VALUE, + NULL, /*attributes*/ + PAGE_READWRITE, + HIGH_SIZE_T_BYTES(sz), + sz & 0xffffffff, + NULL /* name */); + raw_assert(mapping != NULL); + ptr = MapViewOfFile(mapping, FILE_MAP_WRITE, + 0, 0, /* Offset */ + 0 /* Extend to end of mapping */); + raw_assert(ptr); + CloseHandle(mapping); /* mapped view holds a reference */ +#elif defined(HAVE_SYS_MMAN_H) + ptr = mmap(NULL, sz, + PROT_READ|PROT_WRITE, + MAP_ANON|MAP_PRIVATE, + -1, 0); + raw_assert(ptr != MAP_FAILED); + raw_assert(ptr != NULL); +#else + ptr = tor_malloc_zero(sz); +#endif + + if (flags & ANONMAP_PRIVATE) { + int lock_result = lock_mem(ptr, sz); + raw_assert(lock_result == 0); + int nodump_result = nodump_mem(ptr, sz); + raw_assert(nodump_result == 0); + } + + if (flags & ANONMAP_NOINHERIT) { + int noinherit_result = noinherit_mem(ptr, sz); + raw_assert(noinherit_result == 0); + } + + return ptr; +} + +/** + * Release <b>sz</b> bytes of memory that were previously mapped at + * <b>mapping</b> by tor_mmap_anonymous(). + **/ +void +tor_munmap_anonymous(void *mapping, size_t sz) +{ + if (!mapping) + return; + +#if defined(_WIN32) + (void)sz; + UnmapViewOfFile(mapping); +#elif defined(HAVE_SYS_MMAN_H) + munmap(mapping, sz); +#else + (void)sz; + tor_free(mapping); +#endif +} diff --git a/src/lib/malloc/map_anon.h b/src/lib/malloc/map_anon.h new file mode 100644 index 0000000000..cc5797e4ec --- /dev/null +++ b/src/lib/malloc/map_anon.h @@ -0,0 +1,37 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file map_anon.h + * \brief Headers for map_anon.c + **/ + +#ifndef TOR_MAP_ANON_H +#define TOR_MAP_ANON_H + +#include "lib/malloc/malloc.h" +#include <stddef.h> + +/** + * When this flag is specified, try to prevent the mapping from being + * swapped or dumped. + * + * In some operating systems, this flag is not implemented. + */ +#define ANONMAP_PRIVATE (1u<<0) +/** + * When this flag is specified, try to prevent the mapping from being + * inherited after a fork(). In some operating systems, trying to access it + * afterwards will cause its contents to be zero. In others, trying to access + * it afterwards will cause a crash. + * + * In some operating systems, this flag is not implemented at all. + */ +#define ANONMAP_NOINHERIT (1u<<1) + +void *tor_mmap_anonymous(size_t sz, unsigned flags); +void tor_munmap_anonymous(void *mapping, size_t sz); + +#endif /* !defined(TOR_MAP_ANON_H) */ diff --git a/src/lib/math/.may_include b/src/lib/math/.may_include index 1fd26864dc..f8bc264a5f 100644 --- a/src/lib/math/.may_include +++ b/src/lib/math/.may_include @@ -3,3 +3,5 @@ orconfig.h lib/cc/*.h lib/log/*.h lib/math/*.h +lib/testsupport/*.h +lib/crypt_ops/*.h diff --git a/src/lib/math/fp.c b/src/lib/math/fp.c index eafad358c3..616e4f15c0 100644 --- a/src/lib/math/fp.c +++ b/src/lib/math/fp.c @@ -121,3 +121,23 @@ ENABLE_GCC_WARNING(double-promotion) ENABLE_GCC_WARNING(float-conversion) #endif } + +/* isinf() wrapper for tor */ +int +tor_isinf(double x) +{ + /* Same as above, work around the "double promotion" warnings */ +#ifdef PROBLEMATIC_FLOAT_CONVERSION_WARNING +DISABLE_GCC_WARNING(float-conversion) +#endif +#ifdef PROBLEMATIC_DOUBLE_PROMOTION_WARNING +DISABLE_GCC_WARNING(double-promotion) +#endif + return isinf(x); +#ifdef PROBLEMATIC_DOUBLE_PROMOTION_WARNING +ENABLE_GCC_WARNING(double-promotion) +#endif +#ifdef PROBLEMATIC_FLOAT_CONVERSION_WARNING +ENABLE_GCC_WARNING(float-conversion) +#endif +} diff --git a/src/lib/math/fp.h b/src/lib/math/fp.h index 6f07152e92..cb24649e6c 100644 --- a/src/lib/math/fp.h +++ b/src/lib/math/fp.h @@ -19,5 +19,6 @@ double tor_mathlog(double d) ATTR_CONST; long tor_lround(double d) ATTR_CONST; int64_t tor_llround(double d) ATTR_CONST; int64_t clamp_double_to_int64(double number); +int tor_isinf(double x); #endif diff --git a/src/lib/math/include.am b/src/lib/math/include.am index b088b3f3cc..6d65ce90a7 100644 --- a/src/lib/math/include.am +++ b/src/lib/math/include.am @@ -7,7 +7,8 @@ endif src_lib_libtor_math_a_SOURCES = \ src/lib/math/fp.c \ - src/lib/math/laplace.c + src/lib/math/laplace.c \ + src/lib/math/prob_distr.c src_lib_libtor_math_testing_a_SOURCES = \ @@ -17,4 +18,5 @@ src_lib_libtor_math_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) noinst_HEADERS += \ src/lib/math/fp.h \ - src/lib/math/laplace.h + src/lib/math/laplace.h \ + src/lib/math/prob_distr.h diff --git a/src/lib/math/prob_distr.c b/src/lib/math/prob_distr.c new file mode 100644 index 0000000000..c952dadc06 --- /dev/null +++ b/src/lib/math/prob_distr.c @@ -0,0 +1,1717 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file prob_distr.c + * + * \brief + * Implements various probability distributions. + * Almost all code is courtesy of Riastradh. + * + * \details + * Here are some details that might help you understand this file: + * + * - Throughout this file, `eps' means the largest relative error of a + * correctly rounded floating-point operation, which in binary64 + * floating-point arithmetic is 2^-53. Here the relative error of a + * true value x from a computed value y is |x - y|/|x|. This + * definition of epsilon is conventional for numerical analysts when + * writing error analyses. (If your libm doesn't provide correctly + * rounded exp and log, their relative error is usually below 2*2^-53 + * and probably closer to 1.1*2^-53 instead.) + * + * The C constant DBL_EPSILON is actually twice this, and should + * perhaps rather be named ulp(1) -- that is, it is the distance from + * 1 to the next greater floating-point number, which is usually of + * more interest to programmers and hardware engineers. + * + * Since this file is concerned mainly with error bounds rather than + * with low-level bit-hacking of floating-point numbers, we adopt the + * numerical analysts' definition in the comments, though we do use + * DBL_EPSILON in a handful of places where it is convenient to use + * some function of eps = DBL_EPSILON/2 in a case analysis. + * + * - In various functions (e.g. sample_log_logistic()) we jump through hoops so + * that we can use reals closer to 0 than closer to 1, since we achieve much + * greater accuracy for floating point numbers near 0. In particular, we can + * represent differences as small as 10^-300 for numbers near 0, but of no + * less than 10^-16 for numbers near 1. + **/ + +#define PROB_DISTR_PRIVATE + +#include "orconfig.h" + +#include "lib/math/prob_distr.h" + +#include "lib/crypt_ops/crypto_rand.h" +#include "lib/cc/ctassert.h" + +#include <float.h> +#include <math.h> +#include <stddef.h> + +/** Validators for downcasting macros below */ +#define validate_container_of(PTR, TYPE, FIELD) \ + (0 * sizeof((PTR) - &((TYPE *)(((char *)(PTR)) - \ + offsetof(TYPE, FIELD)))->FIELD)) +#define validate_const_container_of(PTR, TYPE, FIELD) \ + (0 * sizeof((PTR) - &((const TYPE *)(((const char *)(PTR)) - \ + offsetof(TYPE, FIELD)))->FIELD)) +/** Downcasting macro */ +#define container_of(PTR, TYPE, FIELD) \ + ((TYPE *)(((char *)(PTR)) - offsetof(TYPE, FIELD)) \ + + validate_container_of(PTR, TYPE, FIELD)) +/** Constified downcasting macro */ +#define const_container_of(PTR, TYPE, FIELD) \ + ((const TYPE *)(((const char *)(PTR)) - offsetof(TYPE, FIELD)) \ + + validate_const_container_of(PTR, TYPE, FIELD)) + +/** + * Count number of one bits in 32-bit word. + */ +static unsigned +bitcount32(uint32_t x) +{ + + /* Count two-bit groups. */ + x -= (x >> 1) & UINT32_C(0x55555555); + + /* Count four-bit groups. */ + x = ((x >> 2) & UINT32_C(0x33333333)) + (x & UINT32_C(0x33333333)); + + /* Count eight-bit groups. */ + x = (x + (x >> 4)) & UINT32_C(0x0f0f0f0f); + + /* Sum all eight-bit groups, and extract the sum. */ + return (x * UINT32_C(0x01010101)) >> 24; +} + +/** + * Count leading zeros in 32-bit word. + */ +static unsigned +clz32(uint32_t x) +{ + + /* Round up to a power of two. */ + x |= x >> 1; + x |= x >> 2; + x |= x >> 4; + x |= x >> 8; + x |= x >> 16; + + /* Subtract count of one bits from 32. */ + return (32 - bitcount32(x)); +} + +/* + * Some lemmas that will be used throughout this file to prove various error + * bounds: + * + * Lemma 1. If |d| <= 1/2, then 1/(1 + d) <= 2. + * + * Proof. If 0 <= d <= 1/2, then 1 + d >= 1, so that 1/(1 + d) <= 1. + * If -1/2 <= d <= 0, then 1 + d >= 1/2, so that 1/(1 + d) <= 2. QED. + * + * Lemma 2. If b = a*(1 + d)/(1 + d') for |d'| < 1/2 and nonzero a, b, + * then b = a*(1 + e) for |e| <= 2|d' - d|. + * + * Proof. |a - b|/|a| + * = |a - a*(1 + d)/(1 + d')|/|a| + * = |1 - (1 + d)/(1 + d')| + * = |(1 + d' - 1 - d)/(1 + d')| + * = |(d' - d)/(1 + d')| + * <= 2|d' - d|, by Lemma 1, + * + * QED. + * + * Lemma 3. For |d|, |d'| < 1/4, + * + * |log((1 + d)/(1 + d'))| <= 4|d - d'|. + * + * Proof. Write + * + * log((1 + d)/(1 + d')) + * = log(1 + (1 + d)/(1 + d') - 1) + * = log(1 + (1 + d - 1 - d')/(1 + d') + * = log(1 + (d - d')/(1 + d')). + * + * By Lemma 1, |(d - d')/(1 + d')| < 2|d' - d| < 1, so the Taylor + * series of log(1 + x) converges absolutely for (d - d')/(1 + d'), + * and thus we have + * + * |log(1 + (d - d')/(1 + d'))| + * = |\sum_{n=1}^\infty ((d - d')/(1 + d'))^n/n| + * <= \sum_{n=1}^\infty |(d - d')/(1 + d')|^n/n + * <= \sum_{n=1}^\infty |2(d' - d)|^n/n + * <= \sum_{n=1}^\infty |2(d' - d)|^n + * = 1/(1 - |2(d' - d)|) + * <= 4|d' - d|, + * + * QED. + * + * Lemma 4. If 1/e <= 1 + x <= e, then + * + * log(1 + (1 + d) x) = (1 + d') log(1 + x) + * + * for |d'| < 8|d|. + * + * Proof. Write + * + * log(1 + (1 + d) x) + * = log(1 + x + x*d) + * = log((1 + x) (1 + x + x*d)/(1 + x)) + * = log(1 + x) + log((1 + x + x*d)/(1 + x)) + * = log(1 + x) (1 + log((1 + x + x*d)/(1 + x))/log(1 + x)). + * + * The relative error is bounded by + * + * |log((1 + x + x*d)/(1 + x))/log(1 + x)| + * <= 4|x + x*d - x|/|log(1 + x)|, by Lemma 3, + * = 4|x*d|/|log(1 + x)| + * < 8|d|, + * + * since in this range 0 < 1 - 1/e < x/log(1 + x) <= e - 1 < 2. QED. + */ + +/** + * Compute the logistic function: f(x) = 1/(1 + e^{-x}) = e^x/(1 + e^x). + * Maps a log-odds-space probability in [-\infty, +\infty] into a direct-space + * probability in [0,1]. Inverse of logit. + * + * Ill-conditioned for large x; the identity logistic(-x) = 1 - + * logistic(x) and the function logistichalf(x) = logistic(x) - 1/2 may + * help to rearrange a computation. + * + * This implementation gives relative error bounded by 7 eps. + */ +STATIC double +logistic(double x) +{ + if (x <= log(DBL_EPSILON/2)) { + /* + * If x <= log(DBL_EPSILON/2) = log(eps), then e^x <= eps. In this case + * we will approximate the logistic() function with e^x because the + * relative error is less than eps. Here is a calculation of the + * relative error between the logistic() function and e^x and a proof + * that it's less than eps: + * + * |e^x - e^x/(1 + e^x)|/|e^x/(1 + e^x)| + * <= |1 - 1/(1 + e^x)|*|1 + e^x| + * = |e^x/(1 + e^x)|*|1 + e^x| + * = |e^x| + * <= eps. + */ + return exp(x); /* return e^x */ + } else if (x <= -log(DBL_EPSILON/2)) { + /* + * e^{-x} > 0, so 1 + e^{-x} > 1, and 0 < 1/(1 + + * e^{-x}) < 1; further, since e^{-x} < 1 + e^{-x}, we + * also have 0 < 1/(1 + e^{-x}) < 1. Thus, if exp has + * relative error d0, + has relative error d1, and / + * has relative error d2, then we get + * + * (1 + d2)/[(1 + (1 + d0) e^{-x})(1 + d1)] + * = (1 + d0)/[1 + e^{-x} + d0 e^{-x} + * + d1 + d1 e^{-x} + d0 d1 e^{-x}] + * = (1 + d0)/[(1 + e^{-x}) + * * (1 + d0 e^{-x}/(1 + e^{-x}) + * + d1/(1 + e^{-x}) + * + d0 d1 e^{-x}/(1 + e^{-x}))]. + * = (1 + d0)/[(1 + e^{-x})(1 + d')] + * = [1/(1 + e^{-x})] (1 + d0)/(1 + d') + * + * where + * + * d' = d0 e^{-x}/(1 + e^{-x}) + * + d1/(1 + e^{-x}) + * + d0 d1 e^{-x}/(1 + e^{-x}). + * + * By Lemma 2 this relative error is bounded by + * + * 2|d0 - d'| + * = 2|d0 - d0 e^{-x}/(1 + e^{-x}) + * - d1/(1 + e^{-x}) + * - d0 d1 e^{-x}/(1 + e^{-x})| + * <= 2|d0| + 2|d0 e^{-x}/(1 + e^{-x})| + * + 2|d1/(1 + e^{-x})| + * + 2|d0 d1 e^{-x}/(1 + e^{-x})| + * <= 2|d0| + 2|d0| + 2|d1| + 2|d0 d1| + * <= 4|d0| + 2|d1| + 2|d0 d1| + * <= 6 eps + 2 eps^2. + */ + return 1/(1 + exp(-x)); + } else { + /* + * e^{-x} <= eps, so the relative error of 1 from 1/(1 + * + e^{-x}) is + * + * |1/(1 + e^{-x}) - 1|/|1/(1 + e^{-x})| + * = |e^{-x}/(1 + e^{-x})|/|1/(1 + e^{-x})| + * = |e^{-x}| + * <= eps. + * + * This computation avoids an intermediate overflow + * exception, although the effect on the result is + * harmless. + * + * XXX Should maybe raise inexact here. + */ + return 1; + } +} + +/** + * Compute the logit function: log p/(1 - p). Defined on [0,1]. Maps + * a direct-space probability in [0,1] to a log-odds-space probability + * in [-\infty, +\infty]. Inverse of logistic. + * + * Ill-conditioned near 1/2 and 1; the identity logit(1 - p) = + * -logit(p) and the function logithalf(p0) = logit(1/2 + p0) may help + * to rearrange a computation for p in [1/(1 + e), 1 - 1/(1 + e)]. + * + * This implementation gives relative error bounded by 10 eps. + */ +STATIC double +logit(double p) +{ + + /* logistic(-1) <= p <= logistic(+1) */ + if (1/(1 + exp(1)) <= p && p <= 1/(1 + exp(-1))) { + /* + * For inputs near 1/2, we want to compute log1p(near + * 0) rather than log(near 1), so write this as: + * + * log(p/(1 - p)) = -log((1 - p)/p) + * = -log(1 + (1 - p)/p - 1) + * = -log(1 + (1 - p - p)/p) + * = -log(1 + (1 - 2p)/p). + * + * Since p = 2p/2 <= 1 <= 2*2p = 4p, the floating-point + * evaluation of 1 - 2p is exact; the only error arises + * from division and log1p. First, note that if + * logistic(-1) <= p <= logistic(+1), (1 - 2p)/p lies + * in the bounds of Lemma 4. + * + * If division has relative error d0 and log1p has + * relative error d1, the outcome is + * + * -(1 + d1) log(1 + (1 - 2p) (1 + d0)/p) + * = -(1 + d1) (1 + d') log(1 + (1 - 2p)/p) + * = -(1 + d1 + d' + d1 d') log(1 + (1 - 2p)/p). + * + * where |d'| < 8|d0| by Lemma 4. The relative error + * is then bounded by + * + * |d1 + d' + d1 d'| + * <= |d1| + 8|d0| + 8|d1 d0| + * <= 9 eps + 8 eps^2. + */ + return -log1p((1 - 2*p)/p); + } else { + /* + * For inputs near 0, although 1 - p may be rounded to + * 1, it doesn't matter much because the magnitude of + * the result is so much larger. For inputs near 1, we + * can compute 1 - p exactly, although the precision on + * the input is limited so we won't ever get more than + * about 700 for the output. + * + * If - has relative error d0, / has relative error d1, + * and log has relative error d2, then + * + * (1 + d2) log((1 + d0) p/[(1 - p)(1 + d1)]) + * = (1 + d2) [log(p/(1 - p)) + log((1 + d0)/(1 + d1))] + * = log(p/(1 - p)) + d2 log(p/(1 - p)) + * + (1 + d2) log((1 + d0)/(1 + d1)) + * = log(p/(1 - p))*[1 + d2 + + * + (1 + d2) log((1 + d0)/(1 + d1))/log(p/(1 - p))] + * + * Since 0 <= p < logistic(-1) or logistic(+1) < p <= + * 1, we have |log(p/(1 - p))| > 1. Hence this error + * is bounded by + * + * |d2 + (1 + d2) log((1 + d0)/(1 + d1))/log(p/(1 - p))| + * <= |d2| + |(1 + d2) log((1 + d0)/(1 + d1)) + * / log(p/(1 - p))| + * <= |d2| + |(1 + d2) log((1 + d0)/(1 + d1))| + * <= |d2| + 4|(1 + d2) (d0 - d1)|, by Lemma 3, + * <= |d2| + 4|d0 - d1 + d2 d0 - d1 d0| + * <= |d2| + 4|d0| + 4|d1| + 4|d2 d0| + 4|d1 d0| + * <= 9 eps + 8 eps^2. + */ + return log(p/(1 - p)); + } +} + +/** + * Compute the logit function, translated in input by 1/2: logithalf(p) + * = logit(1/2 + p). Defined on [-1/2, 1/2]. Inverse of logistichalf. + * + * Ill-conditioned near +/-1/2. If |p0| > 1/2 - 1/(1 + e), it may be + * better to compute 1/2 + p0 or -1/2 - p0 and to use logit instead. + * This implementation gives relative error bounded by 34 eps. + */ +STATIC double +logithalf(double p0) +{ + + if (fabs(p0) <= 0.5 - 1/(1 + exp(1))) { + /* + * logit(1/2 + p0) + * = log((1/2 + p0)/(1 - (1/2 + p0))) + * = log((1/2 + p0)/(1/2 - p0)) + * = log(1 + (1/2 + p0)/(1/2 - p0) - 1) + * = log(1 + (1/2 + p0 - (1/2 - p0))/(1/2 - p0)) + * = log(1 + (1/2 + p0 - 1/2 + p0)/(1/2 - p0)) + * = log(1 + 2 p0/(1/2 - p0)) + * + * If the error of subtraction is d0, the error of + * division is d1, and the error of log1p is d2, then + * what we compute is + * + * (1 + d2) log(1 + (1 + d1) 2 p0/[(1 + d0) (1/2 - p0)]) + * = (1 + d2) log(1 + (1 + d') 2 p0/(1/2 - p0)) + * = (1 + d2) (1 + d'') log(1 + 2 p0/(1/2 - p0)) + * = (1 + d2 + d'' + d2 d'') log(1 + 2 p0/(1/2 - p0)), + * + * where |d'| < 2|d0 - d1| <= 4 eps by Lemma 2, and + * |d''| < 8|d'| < 32 eps by Lemma 4 since + * + * 1/e <= 1 + 2*p0/(1/2 - p0) <= e + * + * when |p0| <= 1/2 - 1/(1 + e). Hence the relative + * error is bounded by + * + * |d2 + d'' + d2 d''| + * <= |d2| + |d''| + |d2 d''| + * <= |d1| + 32 |d0| + 32 |d1 d0| + * <= 33 eps + 32 eps^2. + */ + return log1p(2*p0/(0.5 - p0)); + } else { + /* + * We have a choice of computing logit(1/2 + p0) or + * -logit(1 - (1/2 + p0)) = -logit(1/2 - p0). It + * doesn't matter which way we do this: either way, + * since 1/2 p0 <= 1/2 <= 2 p0, the sum and difference + * are computed exactly. So let's do the one that + * skips the final negation. + * + * The result is + * + * (1 + d1) log((1 + d0) (1/2 + p0)/[(1 + d2) (1/2 - p0)]) + * = (1 + d1) (1 + log((1 + d0)/(1 + d2)) + * / log((1/2 + p0)/(1/2 - p0))) + * * log((1/2 + p0)/(1/2 - p0)) + * = (1 + d') log((1/2 + p0)/(1/2 - p0)) + * = (1 + d') logit(1/2 + p0) + * + * where + * + * d' = d1 + log((1 + d0)/(1 + d2))/logit(1/2 + p0) + * + d1 log((1 + d0)/(1 + d2))/logit(1/2 + p0). + * + * For |p| > 1/2 - 1/(1 + e), logit(1/2 + p0) > 1. + * Provided |d0|, |d2| < 1/4, by Lemma 3 we have + * + * |log((1 + d0)/(1 + d2))| <= 4|d0 - d2|. + * + * Hence the relative error is bounded by + * + * |d'| <= |d1| + 4|d0 - d2| + 4|d1| |d0 - d2| + * <= |d1| + 4|d0| + 4|d2| + 4|d1 d0| + 4|d1 d2| + * <= 9 eps + 8 eps^2. + */ + return log((0.5 + p0)/(0.5 - p0)); + } +} + +/* + * The following random_uniform_01 is tailored for IEEE 754 binary64 + * floating-point or smaller. It can be adapted to larger + * floating-point formats like i387 80-bit or IEEE 754 binary128, but + * it may require sampling more bits. + */ +CTASSERT(FLT_RADIX == 2); +CTASSERT(-DBL_MIN_EXP <= 1021); +CTASSERT(DBL_MANT_DIG <= 53); + +/** + * Draw a floating-point number in [0, 1] with uniform distribution. + * + * Note that the probability of returning 0 is less than 2^-1074, so + * callers need not check for it. However, callers that cannot handle + * rounding to 1 must deal with that, because it occurs with + * probability 2^-54, which is small but nonnegligible. + */ +STATIC double +random_uniform_01(void) +{ + uint32_t z, x, hi, lo; + double s; + + /* + * Draw an exponent, geometrically distributed, but give up if + * we get a run of more than 1088 zeros, which really means the + * system is broken. + */ + z = 0; + while ((x = crypto_rand_u32()) == 0) { + if (z >= 1088) + /* Your bit sampler is broken. Go home. */ + return 0; + z += 32; + } + z += clz32(x); + + /* + * Pick 32-bit halves of an odd normalized significand. + * Picking it odd breaks ties in the subsequent rounding, which + * occur only with measure zero in the uniform distribution on + * [0, 1]. + */ + hi = crypto_rand_u32() | UINT32_C(0x80000000); + lo = crypto_rand_u32() | UINT32_C(0x00000001); + + /* Round to nearest scaled significand in [2^63, 2^64]. */ + s = hi*(double)4294967296 + lo; + + /* Rescale into [1/2, 1] and apply exponent in one swell foop. */ + return s * ldexp(1, -(64 + z)); +} + +/*******************************************************************/ + +/* Functions for specific probability distributions start here: */ + +/* + * Logistic(mu, sigma) distribution, supported on (-\infty,+\infty) + * + * This is the uniform distribution on [0,1] mapped into log-odds + * space, scaled by sigma and translated by mu. + * + * pdf(x) = e^{-(x - mu)/sigma} sigma (1 + e^{-(x - mu)/sigma})^2 + * cdf(x) = 1/(1 + e^{-(x - mu)/sigma}) = logistic((x - mu)/sigma) + * sf(x) = 1 - cdf(x) = 1 - logistic((x - mu)/sigma = logistic(-(x - mu)/sigma) + * icdf(p) = mu + sigma log p/(1 - p) = mu + sigma logit(p) + * isf(p) = mu + sigma log (1 - p)/p = mu - sigma logit(p) + */ + +/** + * Compute the CDF of the Logistic(mu, sigma) distribution: the + * logistic function. Well-conditioned for negative inputs and small + * positive inputs; ill-conditioned for large positive inputs. + */ +STATIC double +cdf_logistic(double x, double mu, double sigma) +{ + return logistic((x - mu)/sigma); +} + +/** + * Compute the SF of the Logistic(mu, sigma) distribution: the logistic + * function reflected over the y axis. Well-conditioned for positive + * inputs and small negative inputs; ill-conditioned for large negative + * inputs. + */ +STATIC double +sf_logistic(double x, double mu, double sigma) +{ + return logistic(-(x - mu)/sigma); +} + +/** + * Compute the inverse of the CDF of the Logistic(mu, sigma) + * distribution: the logit function. Well-conditioned near 0; + * ill-conditioned near 1/2 and 1. + */ +STATIC double +icdf_logistic(double p, double mu, double sigma) +{ + return mu + sigma*logit(p); +} + +/** + * Compute the inverse of the SF of the Logistic(mu, sigma) + * distribution: the -logit function. Well-conditioned near 0; + * ill-conditioned near 1/2 and 1. + */ +STATIC double +isf_logistic(double p, double mu, double sigma) +{ + return mu - sigma*logit(p); +} + +/* + * LogLogistic(alpha, beta) distribution, supported on (0, +\infty). + * + * This is the uniform distribution on [0,1] mapped into odds space, + * scaled by positive alpha and shaped by positive beta. + * + * Equivalent to computing exp of a Logistic(log alpha, 1/beta) sample. + * (Name arises because the pdf has LogLogistic(x; alpha, beta) = + * Logistic(log x; log alpha, 1/beta) and mathematicians got their + * covariance contravariant.) + * + * pdf(x) = (beta/alpha) (x/alpha)^{beta - 1}/(1 + (x/alpha)^beta)^2 + * = (1/e^mu sigma) (x/e^mu)^{1/sigma - 1} / + * (1 + (x/e^mu)^{1/sigma})^2 + * cdf(x) = 1/(1 + (x/alpha)^-beta) = 1/(1 + (x/e^mu)^{-1/sigma}) + * = 1/(1 + (e^{log x}/e^mu)^{-1/sigma}) + * = 1/(1 + (e^{log x - mu})^{-1/sigma}) + * = 1/(1 + e^{-(log x - mu)/sigma}) + * = logistic((log x - mu)/sigma) + * = logistic((log x - log alpha)/(1/beta)) + * sf(x) = 1 - 1/(1 + (x/alpha)^-beta) + * = (x/alpha)^-beta/(1 + (x/alpha)^-beta) + * = 1/((x/alpha)^beta + 1) + * = 1/(1 + (x/alpha)^beta) + * icdf(p) = alpha (p/(1 - p))^{1/beta} + * = alpha e^{logit(p)/beta} + * = e^{mu + sigma logit(p)} + * isf(p) = alpha ((1 - p)/p)^{1/beta} + * = alpha e^{-logit(p)/beta} + * = e^{mu - sigma logit(p)} + */ + +/** + * Compute the CDF of the LogLogistic(alpha, beta) distribution. + * Well-conditioned for all x and alpha, and the condition number + * + * -beta/[1 + (x/alpha)^{-beta}] + * + * grows linearly with beta. + * + * Loosely, the relative error of this implementation is bounded by + * + * 4 eps + 2 eps^2 + O(beta eps), + * + * so don't bother trying this for beta anywhere near as large as + * 1/eps, around which point it levels off at 1. + */ +STATIC double +cdf_log_logistic(double x, double alpha, double beta) +{ + /* + * Let d0 be the error of x/alpha; d1, of pow; d2, of +; and + * d3, of the final quotient. The exponentiation gives + * + * ((1 + d0) x/alpha)^{-beta} + * = (x/alpha)^{-beta} (1 + d0)^{-beta} + * = (x/alpha)^{-beta} (1 + (1 + d0)^{-beta} - 1) + * = (x/alpha)^{-beta} (1 + d') + * + * where d' = (1 + d0)^{-beta} - 1. If y = (x/alpha)^{-beta}, + * the denominator is + * + * (1 + d2) (1 + (1 + d1) (1 + d') y) + * = (1 + d2) (1 + y + (d1 + d' + d1 d') y) + * = 1 + y + (1 + d2) (d1 + d' + d1 d') y + * = (1 + y) (1 + (1 + d2) (d1 + d' + d1 d') y/(1 + y)) + * = (1 + y) (1 + d''), + * + * where d'' = (1 + d2) (d1 + d' + d1 d') y/(1 + y). The + * final result is + * + * (1 + d3) / [(1 + d2) (1 + d'') (1 + y)] + * = (1 + d''') / (1 + y) + * + * for |d'''| <= 2|d3 - d''| by Lemma 2 as long as |d''| < 1/2 + * (which may not be the case for very large beta). This + * relative error is therefore bounded by + * + * |d'''| + * <= 2|d3 - d''| + * <= 2|d3| + 2|(1 + d2) (d1 + d' + d1 d') y/(1 + y)| + * <= 2|d3| + 2|(1 + d2) (d1 + d' + d1 d')| + * = 2|d3| + 2|d1 + d' + d1 d' + d2 d1 + d2 d' + d2 d1 d'| + * <= 2|d3| + 2|d1| + 2|d'| + 2|d1 d'| + 2|d2 d1| + 2|d2 d'| + * + 2|d2 d1 d'| + * <= 4 eps + 2 eps^2 + (2 + 2 eps + 2 eps^2) |d'|. + * + * Roughly, |d'| = |(1 + d0)^{-beta} - 1| grows like beta eps, + * until it levels off at 1. + */ + return 1/(1 + pow(x/alpha, -beta)); +} + +/** + * Compute the SF of the LogLogistic(alpha, beta) distribution. + * Well-conditioned for all x and alpha, and the condition number + * + * beta/[1 + (x/alpha)^beta] + * + * grows linearly with beta. + * + * Loosely, the relative error of this implementation is bounded by + * + * 4 eps + 2 eps^2 + O(beta eps) + * + * so don't bother trying this for beta anywhere near as large as + * 1/eps, beyond which point it grows unbounded. + */ +STATIC double +sf_log_logistic(double x, double alpha, double beta) +{ + /* + * The error analysis here is essentially the same as in + * cdf_log_logistic, except that rather than levelling off at + * 1, |(1 + d0)^beta - 1| grows unbounded. + */ + return 1/(1 + pow(x/alpha, beta)); +} + +/** + * Compute the inverse of the CDF of the LogLogistic(alpha, beta) + * distribution. Ill-conditioned for p near 1 and beta near 0 with + * condition number 1/[beta (1 - p)]. + */ +STATIC double +icdf_log_logistic(double p, double alpha, double beta) +{ + return alpha*pow(p/(1 - p), 1/beta); +} + +/** + * Compute the inverse of the SF of the LogLogistic(alpha, beta) + * distribution. Ill-conditioned for p near 1 and for large beta, with + * condition number -1/[beta (1 - p)]. + */ +STATIC double +isf_log_logistic(double p, double alpha, double beta) +{ + return alpha*pow((1 - p)/p, 1/beta); +} + +/* + * Weibull(lambda, k) distribution, supported on (0, +\infty). + * + * pdf(x) = (k/lambda) (x/lambda)^{k - 1} e^{-(x/lambda)^k} + * cdf(x) = 1 - e^{-(x/lambda)^k} + * icdf(p) = lambda * (-log (1 - p))^{1/k} + * sf(x) = e^{-(x/lambda)^k} + * isf(p) = lambda * (-log p)^{1/k} + */ + +/** + * Compute the CDF of the Weibull(lambda, k) distribution. + * Well-conditioned for small x and k, and for large lambda -- + * condition number + * + * -k (x/lambda)^k exp(-(x/lambda)^k)/[exp(-(x/lambda)^k) - 1] + * + * grows linearly with k, x^k, and lambda^{-k}. + */ +STATIC double +cdf_weibull(double x, double lambda, double k) +{ + return -expm1(-pow(x/lambda, k)); +} + +/** + * Compute the SF of the Weibull(lambda, k) distribution. + * Well-conditioned for small x and k, and for large lambda -- + * condition number + * + * -k (x/lambda)^k + * + * grows linearly with k, x^k, and lambda^{-k}. + */ +STATIC double +sf_weibull(double x, double lambda, double k) +{ + return exp(-pow(x/lambda, k)); +} + +/** + * Compute the inverse of the CDF of the Weibull(lambda, k) + * distribution. Ill-conditioned for p near 1, and for k near 0; + * condition number is + * + * (p/(1 - p))/(k log(1 - p)). + */ +STATIC double +icdf_weibull(double p, double lambda, double k) +{ + return lambda*pow(-log1p(-p), 1/k); +} + +/** + * Compute the inverse of the SF of the Weibull(lambda, k) + * distribution. Ill-conditioned for p near 0, and for k near 0; + * condition number is + * + * 1/(k log(p)). + */ +STATIC double +isf_weibull(double p, double lambda, double k) +{ + return lambda*pow(-log(p), 1/k); +} + +/* + * GeneralizedPareto(mu, sigma, xi), supported on (mu, +\infty) for + * nonnegative xi, or (mu, mu - sigma/xi) for negative xi. + * + * Samples: + * = mu - sigma log U, if xi = 0; + * = mu + sigma (U^{-xi} - 1)/xi = mu + sigma*expm1(-xi log U)/xi, if xi =/= 0, + * where U is uniform on (0,1]. + * = mu + sigma (e^{xi X} - 1)/xi, + * where X has standard exponential distribution. + * + * pdf(x) = sigma^{-1} (1 + xi (x - mu)/sigma)^{-(1 + 1/xi)} + * cdf(x) = 1 - (1 + xi (x - mu)/sigma)^{-1/xi} + * = 1 - e^{-log(1 + xi (x - mu)/sigma)/xi} + * --> 1 - e^{-(x - mu)/sigma} as xi --> 0 + * sf(x) = (1 + xi (x - mu)/sigma)^{-1/xi} + * --> e^{-(x - mu)/sigma} as xi --> 0 + * icdf(p) = mu + sigma*(p^{-xi} - 1)/xi + * = mu + sigma*expm1(-xi log p)/xi + * --> mu + sigma*log p as xi --> 0 + * isf(p) = mu + sigma*((1 - p)^{xi} - 1)/xi + * = mu + sigma*expm1(-xi log1p(-p))/xi + * --> mu + sigma*log1p(-p) as xi --> 0 + */ + +/** + * Compute the CDF of the GeneralizedPareto(mu, sigma, xi) + * distribution. Well-conditioned everywhere. For standard + * distribution (mu=0, sigma=1), condition number + * + * (x/(1 + x xi)) / ((1 + x xi)^{1/xi} - 1) + * + * is bounded by 1, attained only at x = 0. + */ +STATIC double +cdf_genpareto(double x, double mu, double sigma, double xi) +{ + double x_0 = (x - mu)/sigma; + + /* + * log(1 + xi x_0)/xi + * = (-1/xi) \sum_{n=1}^\infty (-xi x_0)^n/n + * = (-1/xi) (-xi x_0 + \sum_{n=2}^\infty (-xi x_0)^n/n) + * = x_0 - (1/xi) \sum_{n=2}^\infty (-xi x_0)^n/n + * = x_0 - x_0 \sum_{n=2}^\infty (-xi x_0)^{n-1}/n + * = x_0 (1 - d), + * + * where d = \sum_{n=2}^\infty (-xi x_0)^{n-1}/n. If |xi| < + * eps/4|x_0|, then + * + * |d| <= \sum_{n=2}^\infty (eps/4)^{n-1}/n + * <= \sum_{n=2}^\infty (eps/4)^{n-1} + * = \sum_{n=1}^\infty (eps/4)^n + * = (eps/4) \sum_{n=0}^\infty (eps/4)^n + * = (eps/4)/(1 - eps/4) + * < eps/2 + * + * for any 0 < eps < 2. Thus, the relative error of x_0 from + * log(1 + xi x_0)/xi is bounded by eps. + */ + if (fabs(xi) < 1e-17/x_0) + return -expm1(-x_0); + else + return -expm1(-log1p(xi*x_0)/xi); +} + +/** + * Compute the SF of the GeneralizedPareto(mu, sigma, xi) distribution. + * For standard distribution (mu=0, sigma=1), ill-conditioned for xi + * near 0; condition number + * + * -x (1 + x xi)^{(-1 - xi)/xi}/(1 + x xi)^{-1/xi} + * = -x (1 + x xi)^{-1/xi - 1}/(1 + x xi)^{-1/xi} + * = -(x/(1 + x xi)) (1 + x xi)^{-1/xi}/(1 + x xi)^{-1/xi} + * = -x/(1 + x xi) + * + * is bounded by 1/xi. + */ +STATIC double +sf_genpareto(double x, double mu, double sigma, double xi) +{ + double x_0 = (x - mu)/sigma; + + if (fabs(xi) < 1e-17/x_0) + return exp(-x_0); + else + return exp(-log1p(xi*x_0)/xi); +} + +/** + * Compute the inverse of the CDF of the GeneralizedPareto(mu, sigma, + * xi) distribution. Ill-conditioned for p near 1; condition number is + * + * xi (p/(1 - p))/(1 - (1 - p)^xi) + */ +STATIC double +icdf_genpareto(double p, double mu, double sigma, double xi) +{ + /* + * To compute f(xi) = (U^{-xi} - 1)/xi = (e^{-xi log U} - 1)/xi + * for xi near zero (note f(xi) --> -log U as xi --> 0), write + * the absolutely convergent Taylor expansion + * + * f(xi) = (1/xi)*(-xi log U + \sum_{n=2}^\infty (-xi log U)^n/n! + * = -log U + (1/xi)*\sum_{n=2}^\infty (-xi log U)^n/n! + * = -log U + \sum_{n=2}^\infty xi^{n-1} (-log U)^n/n! + * = -log U - log U \sum_{n=2}^\infty (-xi log U)^{n-1}/n! + * = -log U (1 + \sum_{n=2}^\infty (-xi log U)^{n-1}/n!). + * + * Let d = \sum_{n=2}^\infty (-xi log U)^{n-1}/n!. What do we + * lose if we discard it and use -log U as an approximation to + * f(xi)? If |xi| < eps/-4log U, then + * + * |d| <= \sum_{n=2}^\infty |xi log U|^{n-1}/n! + * <= \sum_{n=2}^\infty (eps/4)^{n-1}/n! + * <= \sum_{n=1}^\infty (eps/4)^n + * = (eps/4) \sum_{n=0}^\infty (eps/4)^n + * = (eps/4)/(1 - eps/4) + * < eps/2, + * + * for any 0 < eps < 2. Hence, as long as |xi| < eps/-2log U, + * f(xi) = -log U (1 + d) for |d| <= eps/2. |d| is the + * relative error of f(xi) from -log U; from this bound, the + * relative error of -log U from f(xi) is at most (eps/2)/(1 - + * eps/2) = eps/2 + (eps/2)^2 + (eps/2)^3 + ... < eps for 0 < + * eps < 1. Since -log U < 1000 for all U in (0, 1] in + * binary64 floating-point, we can safely cut xi off at 1e-20 < + * eps/4000 and attain <1ulp error from series truncation. + */ + if (fabs(xi) <= 1e-20) + return mu - sigma*log1p(-p); + else + return mu + sigma*expm1(-xi*log1p(-p))/xi; +} + +/** + * Compute the inverse of the SF of the GeneralizedPareto(mu, sigma, + * xi) distribution. Ill-conditioned for p near 1; conditon number is + * + * -xi/(1 - p^{-xi}) + */ +STATIC double +isf_genpareto(double p, double mu, double sigma, double xi) +{ + if (fabs(xi) <= 1e-20) + return mu - sigma*log(p); + else + return mu + sigma*expm1(-xi*log(p))/xi; +} + +/*******************************************************************/ + +/** + * Deterministic samplers, parametrized by uniform integer and (0,1] + * samples. No guarantees are made about _which_ mapping from the + * integer and (0,1] samples these use; all that is guaranteed is the + * distribution of the outputs conditioned on a uniform distribution on + * the inputs. The automatic tests in test_prob_distr.c double-check + * the particular mappings we use. + * + * Beware: Unlike random_uniform_01(), these are not guaranteed to be + * supported on all possible outputs. See Ilya Mironov, `On the + * Significance of the Least Significant Bits for Differential + * Privacy', for an example of what can go wrong if you try to use + * these to conceal information from an adversary but you expose the + * specific full-precision floating-point values. + * + * Note: None of these samplers use rejection sampling; they are all + * essentially inverse-CDF transforms with tweaks. If you were to add, + * say, a Gamma sampler with the Marsaglia-Tsang method, you would have + * to parametrize it by a potentially infinite stream of uniform (and + * perhaps normal) samples rather than a fixed number, which doesn't + * make for quite as nice automatic testing as for these. + */ + +/** + * Deterministically sample from the interval [a, b], indexed by a + * uniform random floating-point number p0 in (0, 1]. + * + * Note that even if p0 is nonzero, the result may be equal to a, if + * ulp(a)/2 is nonnegligible, e.g. if a = 1. For maximum resolution, + * arrange |a| <= |b|. + */ +STATIC double +sample_uniform_interval(double p0, double a, double b) +{ + /* + * XXX Prove that the distribution is, in fact, uniform on + * [a,b], particularly around p0 = 1, or at least has very + * small deviation from uniform, quantified appropriately + * (e.g., like in Monahan 1984, or by KL divergence). It + * almost certainly does but it would be nice to quantify the + * error. + */ + if ((a <= 0 && 0 <= b) || (b <= 0 && 0 <= a)) { + /* + * When ab < 0, (1 - t) a + t b is monotonic, since for + * a <= b it is a sum of nondecreasing functions of t, + * and for b <= a, of nonincreasing functions of t. + * Further, clearly at 0 and 1 it attains a and b, + * respectively. Hence it is bounded within [a, b]. + */ + return (1 - p0)*a + p0*b; + } else { + /* + * a + (b - a) t is monotonic -- it is obviously a + * nondecreasing function of t for a <= b. Further, it + * attains a at 0, and while it may overshoot b at 1, + * we have a + * + * Theorem. If 0 <= t < 1, then the floating-point + * evaluation of a + (b - a) t is bounded in [a, b]. + * + * Lemma 1. If 0 <= t < 1 is a floating-point number, + * then for any normal floating-point number x except + * the smallest in magnitude, |round(x*t)| < |x|. + * + * Proof. WLOG, assume x >= 0. Since the rounding + * function and t |---> x*t are nondecreasing, their + * composition t |---> round(x*t) is also + * nondecreasing, so it suffices to consider the + * largest floating-point number below 1, in particular + * t = 1 - ulp(1)/2. + * + * Case I: If x is a power of two, then the next + * floating-point number below x is x - ulp(x)/2 = x - + * x*ulp(1)/2 = x*(1 - ulp(1)/2) = x*t, so, since x*t + * is a floating-point number, multiplication is exact, + * and thus round(x*t) = x*t < x. + * + * Case II: If x is not a power of two, then the + * greatest lower bound of real numbers rounded to x is + * x - ulp(x)/2 = x - ulp(T(x))/2 = x - T(x)*ulp(1)/2, + * where T(X) is the largest power of two below x. + * Anything below this bound is rounded to a + * floating-point number smaller than x, and x*t = x*(1 + * - ulp(1)/2) = x - x*ulp(1)/2 < x - T(x)*ulp(1)/2 + * since T(x) < x, so round(x*t) < x*t < x. QED. + * + * Lemma 2. If x and y are subnormal, then round(x + + * y) = x + y. + * + * Proof. It is a matter of adding the significands, + * since if we treat subnormals as having an implicit + * zero bit before the `binary' point, their exponents + * are all the same. There is at most one carry/borrow + * bit, which can always be acommodated either in a + * subnormal, or, at largest, in the implicit one bit + * of a normal. + * + * Lemma 3. Let x and y be floating-point numbers. If + * round(x - y) is subnormal or zero, then it is equal + * to x - y. + * + * Proof. Case I (equal): round(x - y) = 0 iff x = y; + * hence if round(x - y) = 0, then round(x - y) = 0 = x + * - y. + * + * Case II (subnormal/subnormal): If x and y are both + * subnormal, this follows directly from Lemma 2. + * + * Case IIIa (normal/subnormal): If x is normal and y + * is subnormal, then x and y must share sign, or else + * x - y would be larger than x and thus rounded to + * normal. If s is the smallest normal positive + * floating-point number, |x| < 2s since by + * construction 2s - |y| is normal for all subnormal y. + * This means that x and y must have the same exponent, + * so the difference is the difference of significands, + * which is exact. + * + * Case IIIb (subnormal/normal): Same as case IIIa for + * -(y - x). + * + * Case IV (normal/normal): If x and y are both normal, + * then they must share sign, or else x - y would be + * larger than x and thus rounded to normal. Note that + * |y| < 2|x|, for if |y| >= 2|x|, then |x| - |y| <= + * -|x| but -|x| is normal like x. Also, |x|/2 < |y|: + * if |x|/2 is subnormal, it must hold because y is + * normal; if |x|/2 is normal, then |x|/2 >= s, so + * since |x| - |y| < s, + * + * |x|/2 = |x| - |x|/2 <= |x| - s <= |y|; + * + * that is, |x|/2 < |y| < 2|x|, so by the Sterbenz + * lemma, round(x - y) = x - y. QED. + * + * Proof of theorem. WLOG, assume 0 <= a <= b. Since + * round(a + round(round(b - a)*t) is nondecreasing in + * t and attains a at 0, the lower end of the bound is + * trivial; we must show the upper end of the bound + * strictly. It suffices to show this for the largest + * floating-point number below 1, namely 1 - ulp(1)/2. + * + * Case I: round(b - a) is normal. Then it is at most + * the smallest floating-point number above b - a. By + * Lemma 1, round(round(b - a)*t) < round(b - a). + * Since the inequality is strict, and since + * round(round(b - a)*t) is a floating-point number + * below round(b - a), and since there are no + * floating-point numbers between b - a and round(b - + * a), we must have round(round(b - a)*t) < b - a. + * Then since y |---> round(a + y) is nondecreasing, we + * must have + * + * round(a + round(round(b - a)*t)) + * <= round(a + (b - a)) + * = round(b) = b. + * + * Case II: round(b - a) is subnormal. In this case, + * Lemma 1 falls apart -- we are not guaranteed the + * strict inequality. However, by Lemma 3, the + * difference is exact: round(b - a) = b - a. Thus, + * + * round(a + round(round(b - a)*t)) + * <= round(a + round((b - a)*t)) + * <= round(a + (b - a)) + * = round(b) + * = b, + * + * QED. + */ + + /* p0 is restricted to [0,1], but we use >= to silence -Wfloat-equal. */ + if (p0 >= 1) + return b; + return a + (b - a)*p0; + } +} + +/** + * Deterministically sample from the standard logistic distribution, + * indexed by a uniform random 32-bit integer s and uniform random + * floating-point numbers t and p0 in (0, 1]. + */ +STATIC double +sample_logistic(uint32_t s, double t, double p0) +{ + double sign = (s & 1) ? -1 : +1; + double r; + + /* + * We carve up the interval (0, 1) into subregions to compute + * the inverse CDF precisely: + * + * A = (0, 1/(1 + e)] ---> (-\infty, -1] + * B = [1/(1 + e), 1/2] ---> [-1, 0] + * C = [1/2, 1 - 1/(1 + e)] ---> [0, 1] + * D = [1 - 1/(1 + e), 1) ---> [1, +\infty) + * + * Cases D and C are mirror images of cases A and B, + * respectively, so we choose between them by the sign chosen + * by a fair coin toss. We choose between cases A and B by a + * coin toss weighted by + * + * 2/(1 + e) = 1 - [1/2 - 1/(1 + e)]/(1/2): + * + * if it comes up heads, scale p0 into a uniform (0, 1/(1 + e)] + * sample p; if it comes up tails, scale p0 into a uniform (0, + * 1/2 - 1/(1 + e)] sample and compute the inverse CDF of p = + * 1/2 - p0. + */ + if (t <= 2/(1 + exp(1))) { + /* p uniform in (0, 1/(1 + e)], represented by p. */ + p0 /= 1 + exp(1); + r = logit(p0); + } else { + /* + * p uniform in [1/(1 + e), 1/2), actually represented + * by p0 = 1/2 - p uniform in (0, 1/2 - 1/(1 + e)], so + * that p = 1/2 - p. + */ + p0 *= 0.5 - 1/(1 + exp(1)); + r = logithalf(p0); + } + + /* + * We have chosen from the negative half of the standard + * logistic distribution, which is symmetric with the positive + * half. Now use the sign to choose uniformly between them. + */ + return sign*r; +} + +/** + * Deterministically sample from the logistic distribution scaled by + * sigma and translated by mu. + */ +static double +sample_logistic_locscale(uint32_t s, double t, double p0, double mu, + double sigma) +{ + + return mu + sigma*sample_logistic(s, t, p0); +} + +/** + * Deterministically sample from the standard log-logistic + * distribution, indexed by a uniform random 32-bit integer s and a + * uniform random floating-point number p0 in (0, 1]. + */ +STATIC double +sample_log_logistic(uint32_t s, double p0) +{ + + /* + * Carve up the interval (0, 1) into (0, 1/2] and [1/2, 1); the + * condition numbers of the icdf and the isf coincide at 1/2. + */ + p0 *= 0.5; + if ((s & 1) == 0) { + /* p = p0 in (0, 1/2] */ + return p0/(1 - p0); + } else { + /* p = 1 - p0 in [1/2, 1) */ + return (1 - p0)/p0; + } +} + +/** + * Deterministically sample from the log-logistic distribution with + * scale alpha and shape beta. + */ +static double +sample_log_logistic_scaleshape(uint32_t s, double p0, double alpha, + double beta) +{ + double x = sample_log_logistic(s, p0); + + return alpha*pow(x, 1/beta); +} + +/** + * Deterministically sample from the standard exponential distribution, + * indexed by a uniform random 32-bit integer s and a uniform random + * floating-point number p0 in (0, 1]. + */ +static double +sample_exponential(uint32_t s, double p0) +{ + /* + * We would like to evaluate log(p) for p near 0, and log1p(-p) + * for p near 1. Simply carve the interval into (0, 1/2] and + * [1/2, 1) by a fair coin toss. + */ + p0 *= 0.5; + if ((s & 1) == 0) + /* p = p0 in (0, 1/2] */ + return -log(p0); + else + /* p = 1 - p0 in [1/2, 1) */ + return -log1p(-p0); +} + +/** + * Deterministically sample from a Weibull distribution with scale + * lambda and shape k -- just an exponential with a shape parameter in + * addition to a scale parameter. (Yes, lambda really is the scale, + * _not_ the rate.) + */ +STATIC double +sample_weibull(uint32_t s, double p0, double lambda, double k) +{ + + return lambda*pow(sample_exponential(s, p0), 1/k); +} + +/** + * Deterministically sample from the generalized Pareto distribution + * with shape xi, indexed by a uniform random 32-bit integer s and a + * uniform random floating-point number p0 in (0, 1]. + */ +STATIC double +sample_genpareto(uint32_t s, double p0, double xi) +{ + double x = sample_exponential(s, p0); + + /* + * Write f(xi) = (e^{xi x} - 1)/xi for xi near zero as the + * absolutely convergent Taylor series + * + * f(x) = (1/xi) (xi x + \sum_{n=2}^\infty (xi x)^n/n!) + * = x + (1/xi) \sum_{n=2}^\inty (xi x)^n/n! + * = x + \sum_{n=2}^\infty xi^{n-1} x^n/n! + * = x + x \sum_{n=2}^\infty (xi x)^{n-1}/n! + * = x (1 + \sum_{n=2}^\infty (xi x)^{n-1}/n!). + * + * d = \sum_{n=2}^\infty (xi x)^{n-1}/n! is the relative error + * of f(x) from x. If |xi| < eps/4x, then + * + * |d| <= \sum_{n=2}^\infty |xi x|^{n-1}/n! + * <= \sum_{n=2}^\infty (eps/4)^{n-1}/n! + * <= \sum_{n=1}^\infty (eps/4) + * = (eps/4) \sum_{n=0}^\infty (eps/4)^n + * = (eps/4)/(1 - eps/4) + * < eps/2, + * + * for any 0 < eps < 2. Hence, as long as |xi| < eps/2x, f(xi) + * = x (1 + d) for |d| <= eps/2, so x = f(xi) (1 + d') for |d'| + * <= eps. What bound should we use for x? + * + * - If x is exponentially distributed, x > 200 with + * probability below e^{-200} << 2^{-256}, i.e. never. + * + * - If x is computed by -log(U) for U in (0, 1], x is + * guaranteed to be below 1000 in IEEE 754 binary64 + * floating-point. + * + * We can safely cut xi off at 1e-20 < eps/4000 and attain an + * error bounded by 0.5 ulp for this expression. + */ + return (fabs(xi) < 1e-20 ? x : expm1(xi*x)/xi); +} + +/** + * Deterministically sample from a generalized Pareto distribution with + * shape xi, scaled by sigma and translated by mu. + */ +static double +sample_genpareto_locscale(uint32_t s, double p0, double mu, double sigma, + double xi) +{ + + return mu + sigma*sample_genpareto(s, p0, xi); +} + +/** + * Deterministically sample from the geometric distribution with + * per-trial success probability p. + * + * XXX Quantify the error (KL divergence?) of this + * ceiling-of-exponential sampler from a true geometric distribution, + * which we could get by rejection sampling. Relevant papers: + * + * John F. Monahan, `Accuracy in Random Number Generation', + * Mathematics of Computation 45(172), October 1984, pp. 559--568. +*https://pdfs.semanticscholar.org/aca6/74b96da1df77b2224e8cfc5dd6d61a471632.pdf + * + * Karl Bringmann and Tobias Friedrich, `Exact and Efficient + * Generation of Geometric Random Variates and Random Graphs', in + * Proceedings of the 40th International Colloaquium on Automata, + * Languages, and Programming -- ICALP 2013, Springer LNCS 7965, + * pp.267--278. + * https://doi.org/10.1007/978-3-642-39206-1_23 + * https://people.mpi-inf.mpg.de/~kbringma/paper/2013ICALP-1.pdf + */ +static double +sample_geometric(uint32_t s, double p0, double p) +{ + double x = sample_exponential(s, p0); + + /* This is actually a check against 1, but we do >= so that the compiler + does not raise a -Wfloat-equal */ + if (p >= 1) + return 1; + + return ceil(-x/log1p(-p)); +} + +/*******************************************************************/ + +/** Public API for probability distributions: + * + * For each probability distribution we define each public functions + * (sample/cdf/sf/icdf/isf) as part of its dist_ops structure. + */ + +const char * +dist_name(const struct dist *dist) +{ + return dist->ops->name; +} + +double +dist_sample(const struct dist *dist) +{ + return dist->ops->sample(dist); +} + +double +dist_cdf(const struct dist *dist, double x) +{ + return dist->ops->cdf(dist, x); +} + +double +dist_sf(const struct dist *dist, double x) +{ + return dist->ops->sf(dist, x); +} + +double +dist_icdf(const struct dist *dist, double p) +{ + return dist->ops->icdf(dist, p); +} + +double +dist_isf(const struct dist *dist, double p) +{ + return dist->ops->isf(dist, p); +} + +/** Functions for uniform distribution */ + +static double +uniform_sample(const struct dist *dist) +{ + const struct uniform *U = const_container_of(dist, struct uniform, + base); + double p0 = random_uniform_01(); + + return sample_uniform_interval(p0, U->a, U->b); +} + +static double +uniform_cdf(const struct dist *dist, double x) +{ + const struct uniform *U = const_container_of(dist, struct uniform, + base); + + if (x < U->a) + return 0; + else if (x < U->b) + return (x - U->a)/(U->b - U->a); + else + return 1; +} + +static double +uniform_sf(const struct dist *dist, double x) +{ + const struct uniform *U = const_container_of(dist, struct uniform, + base); + + if (x > U->b) + return 0; + else if (x > U->a) + return (U->b - x)/(U->b - U->a); + else + return 1; +} + +static double +uniform_icdf(const struct dist *dist, double p) +{ + const struct uniform *U = const_container_of(dist, struct uniform, + base); + double w = U->b - U->a; + + return (p < 0.5 ? (U->a + w*p) : (U->b - w*(1 - p))); +} + +static double +uniform_isf(const struct dist *dist, double p) +{ + const struct uniform *U = const_container_of(dist, struct uniform, + base); + double w = U->b - U->a; + + return (p < 0.5 ? (U->b - w*p) : (U->a + w*(1 - p))); +} + +const struct dist_ops uniform_ops = { + .name = "uniform", + .sample = uniform_sample, + .cdf = uniform_cdf, + .sf = uniform_sf, + .icdf = uniform_icdf, + .isf = uniform_isf, +}; + +/** Functions for logistic distribution: */ + +static double +logistic_sample(const struct dist *dist) +{ + const struct logistic *L = const_container_of(dist, struct logistic, + base); + uint32_t s = crypto_rand_u32(); + double t = random_uniform_01(); + double p0 = random_uniform_01(); + + return sample_logistic_locscale(s, t, p0, L->mu, L->sigma); +} + +static double +logistic_cdf(const struct dist *dist, double x) +{ + const struct logistic *L = const_container_of(dist, struct logistic, + base); + + return cdf_logistic(x, L->mu, L->sigma); +} + +static double +logistic_sf(const struct dist *dist, double x) +{ + const struct logistic *L = const_container_of(dist, struct logistic, + base); + + return sf_logistic(x, L->mu, L->sigma); +} + +static double +logistic_icdf(const struct dist *dist, double p) +{ + const struct logistic *L = const_container_of(dist, struct logistic, + base); + + return icdf_logistic(p, L->mu, L->sigma); +} + +static double +logistic_isf(const struct dist *dist, double p) +{ + const struct logistic *L = const_container_of(dist, struct logistic, + base); + + return isf_logistic(p, L->mu, L->sigma); +} + +const struct dist_ops logistic_ops = { + .name = "logistic", + .sample = logistic_sample, + .cdf = logistic_cdf, + .sf = logistic_sf, + .icdf = logistic_icdf, + .isf = logistic_isf, +}; + +/** Functions for log-logistic distribution: */ + +static double +log_logistic_sample(const struct dist *dist) +{ + const struct log_logistic *LL = const_container_of(dist, struct + log_logistic, base); + uint32_t s = crypto_rand_u32(); + double p0 = random_uniform_01(); + + return sample_log_logistic_scaleshape(s, p0, LL->alpha, LL->beta); +} + +static double +log_logistic_cdf(const struct dist *dist, double x) +{ + const struct log_logistic *LL = const_container_of(dist, + struct log_logistic, base); + + return cdf_log_logistic(x, LL->alpha, LL->beta); +} + +static double +log_logistic_sf(const struct dist *dist, double x) +{ + const struct log_logistic *LL = const_container_of(dist, + struct log_logistic, base); + + return sf_log_logistic(x, LL->alpha, LL->beta); +} + +static double +log_logistic_icdf(const struct dist *dist, double p) +{ + const struct log_logistic *LL = const_container_of(dist, + struct log_logistic, base); + + return icdf_log_logistic(p, LL->alpha, LL->beta); +} + +static double +log_logistic_isf(const struct dist *dist, double p) +{ + const struct log_logistic *LL = const_container_of(dist, + struct log_logistic, base); + + return isf_log_logistic(p, LL->alpha, LL->beta); +} + +const struct dist_ops log_logistic_ops = { + .name = "log logistic", + .sample = log_logistic_sample, + .cdf = log_logistic_cdf, + .sf = log_logistic_sf, + .icdf = log_logistic_icdf, + .isf = log_logistic_isf, +}; + +/** Functions for Weibull distribution */ + +static double +weibull_sample(const struct dist *dist) +{ + const struct weibull *W = const_container_of(dist, struct weibull, + base); + uint32_t s = crypto_rand_u32(); + double p0 = random_uniform_01(); + + return sample_weibull(s, p0, W->lambda, W->k); +} + +static double +weibull_cdf(const struct dist *dist, double x) +{ + const struct weibull *W = const_container_of(dist, struct weibull, + base); + + return cdf_weibull(x, W->lambda, W->k); +} + +static double +weibull_sf(const struct dist *dist, double x) +{ + const struct weibull *W = const_container_of(dist, struct weibull, + base); + + return sf_weibull(x, W->lambda, W->k); +} + +static double +weibull_icdf(const struct dist *dist, double p) +{ + const struct weibull *W = const_container_of(dist, struct weibull, + base); + + return icdf_weibull(p, W->lambda, W->k); +} + +static double +weibull_isf(const struct dist *dist, double p) +{ + const struct weibull *W = const_container_of(dist, struct weibull, + base); + + return isf_weibull(p, W->lambda, W->k); +} + +const struct dist_ops weibull_ops = { + .name = "Weibull", + .sample = weibull_sample, + .cdf = weibull_cdf, + .sf = weibull_sf, + .icdf = weibull_icdf, + .isf = weibull_isf, +}; + +/** Functions for generalized Pareto distributions */ + +static double +genpareto_sample(const struct dist *dist) +{ + const struct genpareto *GP = const_container_of(dist, struct genpareto, + base); + uint32_t s = crypto_rand_u32(); + double p0 = random_uniform_01(); + + return sample_genpareto_locscale(s, p0, GP->mu, GP->sigma, GP->xi); +} + +static double +genpareto_cdf(const struct dist *dist, double x) +{ + const struct genpareto *GP = const_container_of(dist, struct genpareto, + base); + + return cdf_genpareto(x, GP->mu, GP->sigma, GP->xi); +} + +static double +genpareto_sf(const struct dist *dist, double x) +{ + const struct genpareto *GP = const_container_of(dist, struct genpareto, + base); + + return sf_genpareto(x, GP->mu, GP->sigma, GP->xi); +} + +static double +genpareto_icdf(const struct dist *dist, double p) +{ + const struct genpareto *GP = const_container_of(dist, struct genpareto, + base); + + return icdf_genpareto(p, GP->mu, GP->sigma, GP->xi); +} + +static double +genpareto_isf(const struct dist *dist, double p) +{ + const struct genpareto *GP = const_container_of(dist, struct genpareto, + base); + + return isf_genpareto(p, GP->mu, GP->sigma, GP->xi); +} + +const struct dist_ops genpareto_ops = { + .name = "generalized Pareto", + .sample = genpareto_sample, + .cdf = genpareto_cdf, + .sf = genpareto_sf, + .icdf = genpareto_icdf, + .isf = genpareto_isf, +}; + +/** Functions for geometric distribution on number of trials before success */ + +static double +geometric_sample(const struct dist *dist) +{ + const struct geometric *G = const_container_of(dist, struct geometric, base); + uint32_t s = crypto_rand_u32(); + double p0 = random_uniform_01(); + + return sample_geometric(s, p0, G->p); +} + +static double +geometric_cdf(const struct dist *dist, double x) +{ + const struct geometric *G = const_container_of(dist, struct geometric, base); + + if (x < 1) + return 0; + /* 1 - (1 - p)^floor(x) = 1 - e^{floor(x) log(1 - p)} */ + return -expm1(floor(x)*log1p(-G->p)); +} + +static double +geometric_sf(const struct dist *dist, double x) +{ + const struct geometric *G = const_container_of(dist, struct geometric, base); + + if (x < 1) + return 0; + /* (1 - p)^floor(x) = e^{ceil(x) log(1 - p)} */ + return exp(floor(x)*log1p(-G->p)); +} + +static double +geometric_icdf(const struct dist *dist, double p) +{ + const struct geometric *G = const_container_of(dist, struct geometric, base); + + return log1p(-p)/log1p(-G->p); +} + +static double +geometric_isf(const struct dist *dist, double p) +{ + const struct geometric *G = const_container_of(dist, struct geometric, base); + + return log(p)/log1p(-G->p); +} + +const struct dist_ops geometric_ops = { + .name = "geometric (1-based)", + .sample = geometric_sample, + .cdf = geometric_cdf, + .sf = geometric_sf, + .icdf = geometric_icdf, + .isf = geometric_isf, +}; diff --git a/src/lib/math/prob_distr.h b/src/lib/math/prob_distr.h new file mode 100644 index 0000000000..66acb796fd --- /dev/null +++ b/src/lib/math/prob_distr.h @@ -0,0 +1,158 @@ + +/** + * \file prob_distr.h + * + * \brief Header for prob_distr.c + **/ + +#ifndef TOR_PROB_DISTR_H +#define TOR_PROB_DISTR_H + +#include "lib/cc/compat_compiler.h" +#include "lib/cc/torint.h" +#include "lib/testsupport/testsupport.h" + +/** + * Container for distribution parameters for sampling, CDF, &c. + */ +struct dist { + const struct dist_ops *ops; +}; + +#define DIST_BASE(OPS) { .ops = (OPS) } +#define DIST_BASE_TYPED(OPS, OBJ, TYPE) \ + DIST_BASE((OPS) + 0*sizeof(&(OBJ) - (const TYPE *)&(OBJ))) + +const char *dist_name(const struct dist *); +double dist_sample(const struct dist *); +double dist_cdf(const struct dist *, double x); +double dist_sf(const struct dist *, double x); +double dist_icdf(const struct dist *, double p); +double dist_isf(const struct dist *, double p); + +struct dist_ops { + const char *name; + double (*sample)(const struct dist *); + double (*cdf)(const struct dist *, double x); + double (*sf)(const struct dist *, double x); + double (*icdf)(const struct dist *, double p); + double (*isf)(const struct dist *, double p); +}; + +/* Geometric distribution on positive number of trials before first success */ + +struct geometric { + struct dist base; + double p; /* success probability */ +}; + +extern const struct dist_ops geometric_ops; + +#define GEOMETRIC(OBJ) \ + DIST_BASE_TYPED(&geometric_ops, OBJ, struct geometric) + +/* Pareto distribution */ + +struct genpareto { + struct dist base; + double mu; + double sigma; + double xi; +}; + +extern const struct dist_ops genpareto_ops; + +#define GENPARETO(OBJ) \ + DIST_BASE_TYPED(&genpareto_ops, OBJ, struct genpareto) + +/* Weibull distribution */ + +struct weibull { + struct dist base; + double lambda; + double k; +}; + +extern const struct dist_ops weibull_ops; + +#define WEIBULL(OBJ) \ + DIST_BASE_TYPED(&weibull_ops, OBJ, struct weibull) + +/* Log-logistic distribution */ + +struct log_logistic { + struct dist base; + double alpha; + double beta; +}; + +extern const struct dist_ops log_logistic_ops; + +#define LOG_LOGISTIC(OBJ) \ + DIST_BASE_TYPED(&log_logistic_ops, OBJ, struct log_logistic) + +/* Logistic distribution */ + +struct logistic { + struct dist base; + double mu; + double sigma; +}; + +extern const struct dist_ops logistic_ops; + +#define LOGISTIC(OBJ) \ + DIST_BASE_TYPED(&logistic_ops, OBJ, struct logistic) + +/* Uniform distribution */ + +struct uniform { + struct dist base; + double a; + double b; +}; + +extern const struct dist_ops uniform_ops; + +#define UNIFORM(OBJ) \ + DIST_BASE_TYPED(&uniform_ops, OBJ, struct uniform) + +/** Only by unittests */ + +#ifdef PROB_DISTR_PRIVATE + +STATIC double logithalf(double p0); +STATIC double logit(double p); + +STATIC double random_uniform_01(void); + +STATIC double logistic(double x); +STATIC double cdf_logistic(double x, double mu, double sigma); +STATIC double sf_logistic(double x, double mu, double sigma); +STATIC double icdf_logistic(double p, double mu, double sigma); +STATIC double isf_logistic(double p, double mu, double sigma); +STATIC double sample_logistic(uint32_t s, double t, double p0); + +STATIC double cdf_log_logistic(double x, double alpha, double beta); +STATIC double sf_log_logistic(double x, double alpha, double beta); +STATIC double icdf_log_logistic(double p, double alpha, double beta); +STATIC double isf_log_logistic(double p, double alpha, double beta); +STATIC double sample_log_logistic(uint32_t s, double p0); + +STATIC double cdf_weibull(double x, double lambda, double k); +STATIC double sf_weibull(double x, double lambda, double k); +STATIC double icdf_weibull(double p, double lambda, double k); +STATIC double isf_weibull(double p, double lambda, double k); +STATIC double sample_weibull(uint32_t s, double p0, double lambda, double k); + +STATIC double sample_uniform_interval(double p0, double a, double b); + +STATIC double cdf_genpareto(double x, double mu, double sigma, double xi); +STATIC double sf_genpareto(double x, double mu, double sigma, double xi); +STATIC double icdf_genpareto(double p, double mu, double sigma, double xi); +STATIC double isf_genpareto(double p, double mu, double sigma, double xi); +STATIC double sample_genpareto(uint32_t s, double p0, double xi); + +#endif + +#endif diff --git a/src/lib/memarea/.may_include b/src/lib/memarea/.may_include index 814652a93c..a1edaf2231 100644 --- a/src/lib/memarea/.may_include +++ b/src/lib/memarea/.may_include @@ -1,7 +1,7 @@ orconfig.h lib/arch/*.h lib/cc/*.h -lib/container/*.h lib/log/*.h lib/malloc/*.h lib/memarea/*.h +lib/smartlist_core/*.h
\ No newline at end of file diff --git a/src/lib/memarea/memarea.c b/src/lib/memarea/memarea.c index 486673116c..84c73b0b95 100644 --- a/src/lib/memarea/memarea.c +++ b/src/lib/memarea/memarea.c @@ -16,7 +16,8 @@ #include "lib/arch/bytes.h" #include "lib/cc/torint.h" -#include "lib/container/smartlist.h" +#include "lib/smartlist_core/smartlist_core.h" +#include "lib/smartlist_core/smartlist_foreach.h" #include "lib/log/log.h" #include "lib/log/util_bug.h" #include "lib/malloc/malloc.h" diff --git a/src/lib/net/.may_include b/src/lib/net/.may_include index 13b209bbed..e4368f799b 100644 --- a/src/lib/net/.may_include +++ b/src/lib/net/.may_include @@ -1,8 +1,9 @@ orconfig.h -siphash.h -ht.h +ext/siphash.h +ext/ht.h lib/arch/*.h +lib/buf/*.h lib/cc/*.h lib/container/*.h lib/ctime/*.h @@ -11,5 +12,6 @@ lib/lock/*.h lib/log/*.h lib/net/*.h lib/string/*.h +lib/subsys/*.h lib/testsupport/*.h lib/malloc/*.h
\ No newline at end of file diff --git a/src/lib/net/address.c b/src/lib/net/address.c index a2d234b742..214d8aa3eb 100644 --- a/src/lib/net/address.c +++ b/src/lib/net/address.c @@ -40,6 +40,7 @@ #include "lib/net/address.h" #include "lib/net/socket.h" +#include "lib/cc/ctassert.h" #include "lib/container/smartlist.h" #include "lib/ctime/di_ops.h" #include "lib/log/log.h" @@ -52,7 +53,7 @@ #include "lib/string/printf.h" #include "lib/string/util_string.h" -#include "siphash.h" +#include "ext/siphash.h" #ifdef HAVE_SYS_TIME_H #include <sys/time.h> @@ -98,6 +99,7 @@ #if AF_UNSPEC != 0 #error We rely on AF_UNSPEC being 0. Let us know about your platform, please! #endif +CTASSERT(AF_UNSPEC == 0); /** Convert the tor_addr_t in <b>a</b>, with port in <b>port</b>, into a * sockaddr object in *<b>sa_out</b> of object size <b>len</b>. If not enough @@ -1198,14 +1200,22 @@ tor_addr_parse(tor_addr_t *addr, const char *src) int result; struct in_addr in_tmp; struct in6_addr in6_tmp; + int brackets_detected = 0; + tor_assert(addr && src); - if (src[0] == '[' && src[1]) + + size_t len = strlen(src); + + if (len && src[0] == '[' && src[len - 1] == ']') { + brackets_detected = 1; src = tmp = tor_strndup(src+1, strlen(src)-2); + } if (tor_inet_pton(AF_INET6, src, &in6_tmp) > 0) { result = AF_INET6; tor_addr_from_in6(addr, &in6_tmp); - } else if (tor_inet_pton(AF_INET, src, &in_tmp) > 0) { + } else if (!brackets_detected && + tor_inet_pton(AF_INET, src, &in_tmp) > 0) { result = AF_INET; tor_addr_from_in(addr, &in_tmp); } else { diff --git a/src/lib/net/buffers_net.c b/src/lib/net/buffers_net.c index 3eb0a033d5..cfe1a7dc26 100644 --- a/src/lib/net/buffers_net.c +++ b/src/lib/net/buffers_net.c @@ -11,7 +11,7 @@ #define BUFFERS_PRIVATE #include "lib/net/buffers_net.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/log/log.h" #include "lib/log/util_bug.h" #include "lib/net/nettypes.h" @@ -22,6 +22,10 @@ #include <stdlib.h> +#ifdef HAVE_UNISTD_H +#include <unistd.h> +#endif + #ifdef PARANOIA /** Helper: If PARANOIA is defined, assert that the buffer in local variable * <b>buf</b> is well-formed. */ @@ -30,27 +34,36 @@ #define check() STMT_NIL #endif /* defined(PARANOIA) */ -/** Read up to <b>at_most</b> bytes from the socket <b>fd</b> into +/** Read up to <b>at_most</b> bytes from the file descriptor <b>fd</b> into * <b>chunk</b> (which must be on <b>buf</b>). If we get an EOF, set - * *<b>reached_eof</b> to 1. Return -1 on error, 0 on eof or blocking, - * and the number of bytes read otherwise. */ + * *<b>reached_eof</b> to 1. Uses <b>tor_socket_recv()</b> iff <b>is_socket</b> + * is true, otherwise it uses <b>read()</b>. Return -1 on error (and sets + * *<b>error</b> to errno), 0 on eof or blocking, and the number of bytes read + * otherwise. */ static inline int read_to_chunk(buf_t *buf, chunk_t *chunk, tor_socket_t fd, size_t at_most, - int *reached_eof, int *socket_error) + int *reached_eof, int *error, bool is_socket) { ssize_t read_result; if (at_most > CHUNK_REMAINING_CAPACITY(chunk)) at_most = CHUNK_REMAINING_CAPACITY(chunk); - read_result = tor_socket_recv(fd, CHUNK_WRITE_PTR(chunk), at_most, 0); + + if (is_socket) + read_result = tor_socket_recv(fd, CHUNK_WRITE_PTR(chunk), at_most, 0); + else + read_result = read(fd, CHUNK_WRITE_PTR(chunk), at_most); if (read_result < 0) { - int e = tor_socket_errno(fd); + int e = is_socket ? tor_socket_errno(fd) : errno; + if (!ERRNO_IS_EAGAIN(e)) { /* it's a real error */ #ifdef _WIN32 if (e == WSAENOBUFS) - log_warn(LD_NET,"recv() failed: WSAENOBUFS. Not enough ram?"); + log_warn(LD_NET, "%s() failed: WSAENOBUFS. Not enough ram?", + is_socket ? "recv" : "read"); #endif - *socket_error = e; + if (error) + *error = e; return -1; } return 0; /* would block. */ @@ -68,16 +81,17 @@ read_to_chunk(buf_t *buf, chunk_t *chunk, tor_socket_t fd, size_t at_most, } } -/** Read from socket <b>s</b>, writing onto end of <b>buf</b>. Read at most - * <b>at_most</b> bytes, growing the buffer as necessary. If recv() returns 0 - * (because of EOF), set *<b>reached_eof</b> to 1 and return 0. Return -1 on - * error; else return the number of bytes read. +/** Read from file descriptor <b>fd</b>, writing onto end of <b>buf</b>. Read + * at most <b>at_most</b> bytes, growing the buffer as necessary. If recv() + * returns 0 (because of EOF), set *<b>reached_eof</b> to 1 and return 0. + * Return -1 on error; else return the number of bytes read. */ /* XXXX indicate "read blocked" somehow? */ -int -buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most, - int *reached_eof, - int *socket_error) +static int +buf_read_from_fd(buf_t *buf, int fd, size_t at_most, + int *reached_eof, + int *socket_error, + bool is_socket) { /* XXXX It's stupid to overload the return values for these functions: * "error status" and "number of bytes read" are not mutually exclusive. @@ -87,7 +101,7 @@ buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most, check(); tor_assert(reached_eof); - tor_assert(SOCKET_OK(s)); + tor_assert(SOCKET_OK(fd)); if (BUG(buf->datalen >= INT_MAX)) return -1; @@ -108,7 +122,8 @@ buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most, readlen = cap; } - r = read_to_chunk(buf, chunk, s, readlen, reached_eof, socket_error); + r = read_to_chunk(buf, chunk, fd, readlen, + reached_eof, socket_error, is_socket); check(); if (r < 0) return r; /* Error */ @@ -122,22 +137,27 @@ buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most, } /** Helper for buf_flush_to_socket(): try to write <b>sz</b> bytes from chunk - * <b>chunk</b> of buffer <b>buf</b> onto socket <b>s</b>. On success, deduct - * the bytes written from *<b>buf_flushlen</b>. Return the number of bytes - * written on success, 0 on blocking, -1 on failure. + * <b>chunk</b> of buffer <b>buf</b> onto file descriptor <b>fd</b>. On + * success, deduct the bytes written from *<b>buf_flushlen</b>. Return the + * number of bytes written on success, 0 on blocking, -1 on failure. */ static inline int -flush_chunk(tor_socket_t s, buf_t *buf, chunk_t *chunk, size_t sz, - size_t *buf_flushlen) +flush_chunk(tor_socket_t fd, buf_t *buf, chunk_t *chunk, size_t sz, + size_t *buf_flushlen, bool is_socket) { ssize_t write_result; if (sz > chunk->datalen) sz = chunk->datalen; - write_result = tor_socket_send(s, chunk->data, sz, 0); + + if (is_socket) + write_result = tor_socket_send(fd, chunk->data, sz, 0); + else + write_result = write(fd, chunk->data, sz); if (write_result < 0) { - int e = tor_socket_errno(s); + int e = is_socket ? tor_socket_errno(fd) : errno; + if (!ERRNO_IS_EAGAIN(e)) { /* it's a real error */ #ifdef _WIN32 if (e == WSAENOBUFS) @@ -155,15 +175,15 @@ flush_chunk(tor_socket_t s, buf_t *buf, chunk_t *chunk, size_t sz, } } -/** Write data from <b>buf</b> to the socket <b>s</b>. Write at most +/** Write data from <b>buf</b> to the file descriptor <b>fd</b>. Write at most * <b>sz</b> bytes, decrement *<b>buf_flushlen</b> by * the number of bytes actually written, and remove the written bytes * from the buffer. Return the number of bytes written on success, * -1 on failure. Return 0 if write() would block. */ -int -buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz, - size_t *buf_flushlen) +static int +buf_flush_to_fd(buf_t *buf, int fd, size_t sz, + size_t *buf_flushlen, bool is_socket) { /* XXXX It's stupid to overload the return values for these functions: * "error status" and "number of bytes flushed" are not mutually exclusive. @@ -171,7 +191,7 @@ buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz, int r; size_t flushed = 0; tor_assert(buf_flushlen); - tor_assert(SOCKET_OK(s)); + tor_assert(SOCKET_OK(fd)); if (BUG(*buf_flushlen > buf->datalen)) { *buf_flushlen = buf->datalen; } @@ -188,7 +208,7 @@ buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz, else flushlen0 = buf->head->datalen; - r = flush_chunk(s, buf, buf->head, flushlen0, buf_flushlen); + r = flush_chunk(fd, buf, buf->head, flushlen0, buf_flushlen, is_socket); check(); if (r < 0) return r; @@ -200,3 +220,55 @@ buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz, tor_assert(flushed < INT_MAX); return (int)flushed; } + +/** Write data from <b>buf</b> to the socket <b>s</b>. Write at most + * <b>sz</b> bytes, decrement *<b>buf_flushlen</b> by + * the number of bytes actually written, and remove the written bytes + * from the buffer. Return the number of bytes written on success, + * -1 on failure. Return 0 if write() would block. + */ +int +buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz, + size_t *buf_flushlen) +{ + return buf_flush_to_fd(buf, s, sz, buf_flushlen, true); +} + +/** Read from socket <b>s</b>, writing onto end of <b>buf</b>. Read at most + * <b>at_most</b> bytes, growing the buffer as necessary. If recv() returns 0 + * (because of EOF), set *<b>reached_eof</b> to 1 and return 0. Return -1 on + * error; else return the number of bytes read. + */ +int +buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most, + int *reached_eof, + int *socket_error) +{ + return buf_read_from_fd(buf, s, at_most, reached_eof, socket_error, true); +} + +/** Write data from <b>buf</b> to the pipe <b>fd</b>. Write at most + * <b>sz</b> bytes, decrement *<b>buf_flushlen</b> by + * the number of bytes actually written, and remove the written bytes + * from the buffer. Return the number of bytes written on success, + * -1 on failure. Return 0 if write() would block. + */ +int +buf_flush_to_pipe(buf_t *buf, int fd, size_t sz, + size_t *buf_flushlen) +{ + return buf_flush_to_fd(buf, fd, sz, buf_flushlen, false); +} + +/** Read from pipe <b>fd</b>, writing onto end of <b>buf</b>. Read at most + * <b>at_most</b> bytes, growing the buffer as necessary. If read() returns 0 + * (because of EOF), set *<b>reached_eof</b> to 1 and return 0. Return -1 on + * error; else return the number of bytes read. + */ +int +buf_read_from_pipe(buf_t *buf, int fd, size_t at_most, + int *reached_eof, + int *socket_error) +{ + return buf_read_from_fd(buf, fd, at_most, reached_eof, socket_error, false); +} diff --git a/src/lib/net/buffers_net.h b/src/lib/net/buffers_net.h index 5f69bebedf..a3a90172a1 100644 --- a/src/lib/net/buffers_net.h +++ b/src/lib/net/buffers_net.h @@ -24,4 +24,11 @@ int buf_read_from_socket(struct buf_t *buf, tor_socket_t s, size_t at_most, int buf_flush_to_socket(struct buf_t *buf, tor_socket_t s, size_t sz, size_t *buf_flushlen); +int buf_read_from_pipe(struct buf_t *buf, int fd, size_t at_most, + int *reached_eof, + int *socket_error); + +int buf_flush_to_pipe(struct buf_t *buf, int fd, size_t sz, + size_t *buf_flushlen); + #endif /* !defined(TOR_BUFFERS_H) */ diff --git a/src/lib/net/inaddr.c b/src/lib/net/inaddr.c index 1a2406ce5f..d9ae7cd562 100644 --- a/src/lib/net/inaddr.c +++ b/src/lib/net/inaddr.c @@ -168,6 +168,13 @@ tor_inet_pton(int af, const char *src, void *dst) if (af == AF_INET) { return tor_inet_aton(src, dst); } else if (af == AF_INET6) { + ssize_t len = strlen(src); + + /* Reject if src has needless trailing ':'. */ + if (len > 2 && src[len - 1] == ':' && src[len - 2] != ':') { + return 0; + } + struct in6_addr *out = dst; uint16_t words[8]; int gapPos = -1, i, setWords=0; @@ -207,7 +214,6 @@ tor_inet_pton(int af, const char *src, void *dst) return 0; if (TOR_ISXDIGIT(*src)) { char *next; - ssize_t len; long r = strtol(src, &next, 16); if (next == NULL || next == src) { /* The 'next == src' error case can happen on versions of openbsd diff --git a/src/lib/net/include.am b/src/lib/net/include.am index ff0967e786..8a88f0f2ae 100644 --- a/src/lib/net/include.am +++ b/src/lib/net/include.am @@ -11,6 +11,7 @@ src_lib_libtor_net_a_SOURCES = \ src/lib/net/buffers_net.c \ src/lib/net/gethostname.c \ src/lib/net/inaddr.c \ + src/lib/net/network_sys.c \ src/lib/net/resolve.c \ src/lib/net/socket.c \ src/lib/net/socketpair.c @@ -28,6 +29,7 @@ noinst_HEADERS += \ src/lib/net/inaddr.h \ src/lib/net/inaddr_st.h \ src/lib/net/nettypes.h \ + src/lib/net/network_sys.h \ src/lib/net/resolve.h \ src/lib/net/socket.h \ src/lib/net/socketpair.h \ diff --git a/src/lib/net/network_sys.c b/src/lib/net/network_sys.c new file mode 100644 index 0000000000..9dfdb2b45a --- /dev/null +++ b/src/lib/net/network_sys.c @@ -0,0 +1,44 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file network_sys.c + * \brief Subsystem object for networking setup. + **/ + +#include "orconfig.h" +#include "lib/subsys/subsys.h" +#include "lib/net/network_sys.h" +#include "lib/net/resolve.h" +#include "lib/net/socket.h" + +#ifdef _WIN32 +#include <winsock2.h> +#include <windows.h> +#endif + +static int +subsys_network_initialize(void) +{ + if (network_init() < 0) + return -1; + + return 0; +} + +static void +subsys_network_shutdown(void) +{ +#ifdef _WIN32 + WSACleanup(); +#endif + tor_free_getaddrinfo_cache(); +} + +const subsys_fns_t sys_network = { + .name = "network", + .level = -90, + .supported = true, + .initialize = subsys_network_initialize, + .shutdown = subsys_network_shutdown, +}; diff --git a/src/lib/net/network_sys.h b/src/lib/net/network_sys.h new file mode 100644 index 0000000000..43e62592ca --- /dev/null +++ b/src/lib/net/network_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file log_network.h + * \brief Declare subsystem object for the network module. + **/ + +#ifndef TOR_NETWORK_SYS_H +#define TOR_NETWORK_SYS_H + +extern const struct subsys_fns_t sys_network; + +#endif /* !defined(TOR_NETWORK_SYS_H) */ diff --git a/src/lib/net/resolve.c b/src/lib/net/resolve.c index 8cee29df37..49c263faa2 100644 --- a/src/lib/net/resolve.c +++ b/src/lib/net/resolve.c @@ -16,8 +16,8 @@ #include "lib/string/parse_int.h" #include "lib/string/util_string.h" -#include "siphash.h" -#include "ht.h" +#include "ext/siphash.h" +#include "ext/ht.h" #ifdef HAVE_SYS_TYPES_H #include <sys/types.h> @@ -421,4 +421,13 @@ tor_make_getaddrinfo_cache_active(void) { sandbox_getaddrinfo_is_active = 1; } +#else +void +sandbox_disable_getaddrinfo_cache(void) +{ +} +void +tor_make_getaddrinfo_cache_active(void) +{ +} #endif diff --git a/src/lib/net/resolve.h b/src/lib/net/resolve.h index 47a283c81c..0fb77f1661 100644 --- a/src/lib/net/resolve.h +++ b/src/lib/net/resolve.h @@ -42,7 +42,6 @@ int tor_getaddrinfo(const char *name, const char *servname, struct addrinfo **res); void tor_freeaddrinfo(struct addrinfo *addrinfo); void tor_free_getaddrinfo_cache(void); -void tor_make_getaddrinfo_cache_active(void); #else /* !(defined(USE_SANDBOX_GETADDRINFO)) */ #define tor_getaddrinfo(name, servname, hints, res) \ getaddrinfo((name),(servname), (hints),(res)) @@ -54,5 +53,6 @@ void tor_make_getaddrinfo_cache_active(void); #endif /* defined(USE_SANDBOX_GETADDRINFO) */ void sandbox_disable_getaddrinfo_cache(void); +void tor_make_getaddrinfo_cache_active(void); #endif diff --git a/src/lib/net/socket.c b/src/lib/net/socket.c index fba90b7506..f978deeab8 100644 --- a/src/lib/net/socket.c +++ b/src/lib/net/socket.c @@ -31,6 +31,9 @@ #endif #include <stddef.h> #include <string.h> +#ifdef __FreeBSD__ +#include <sys/sysctl.h> +#endif /** Called before we make any calls to network-related functions. * (Some operating systems require their network libraries to be @@ -60,6 +63,32 @@ network_init(void) return 0; } +/** + * Warn the user if any system network parameters should be changed. + */ +void +check_network_configuration(bool server_mode) +{ +#ifdef __FreeBSD__ + if (server_mode) { + int random_id_state; + size_t state_size = sizeof(random_id_state); + + if (sysctlbyname("net.inet.ip.random_id", &random_id_state, + &state_size, NULL, 0)) { + log_warn(LD_CONFIG, + "Failed to figure out if IP ids are randomized."); + } else if (random_id_state == 0) { + log_warn(LD_CONFIG, "Looks like IP ids are not randomized. " + "Please consider setting the net.inet.ip.random_id sysctl, " + "so your relay makes it harder to figure out how busy it is."); + } + } +#else + (void) server_mode; +#endif +} + /* When set_max_file_sockets() is called, update this with the max file * descriptor value so we can use it to check the limit when opening a new * socket. Default value is what Debian sets as the default hard limit. */ @@ -429,7 +458,9 @@ get_n_open_sockets(void) * localhost is inaccessible (for example, if the networking * stack is down). And even if it succeeds, the socket pair will not * be able to read while localhost is down later (the socket pair may - * even close, depending on OS-specific timeouts). + * even close, depending on OS-specific timeouts). The socket pair + * should work on IPv4-only, IPv6-only, and dual-stack systems, as long + * as they have the standard localhost addresses. * * Returns 0 on success and -errno on failure; do not rely on the value * of errno or WSAGetLastError(). diff --git a/src/lib/net/socket.h b/src/lib/net/socket.h index 0909619510..86ae336dfb 100644 --- a/src/lib/net/socket.h +++ b/src/lib/net/socket.h @@ -54,6 +54,7 @@ int tor_addr_from_getsockname(struct tor_addr_t *addr_out, tor_socket_t sock); int set_socket_nonblocking(tor_socket_t socket); int tor_socketpair(int family, int type, int protocol, tor_socket_t fd[2]); int network_init(void); +void check_network_configuration(bool server_mode); int get_max_sockets(void); void set_max_sockets(int); diff --git a/src/lib/net/socketpair.c b/src/lib/net/socketpair.c index 10eb749735..15c706bec7 100644 --- a/src/lib/net/socketpair.c +++ b/src/lib/net/socketpair.c @@ -105,7 +105,12 @@ sockaddr_eq(struct sockaddr *sa1, struct sockaddr *sa2) /** * Helper used to implement socketpair on systems that lack it, by * making a direct connection to localhost. - */ + * + * See tor_socketpair() for details. + * + * The direct connection defaults to IPv4, but falls back to IPv6 if + * IPv4 is not supported. + **/ int tor_ersatz_socketpair(int family, int type, int protocol, tor_socket_t fd[2]) { diff --git a/src/lib/process/.may_include b/src/lib/process/.may_include index 05414d2a96..ce1b6ecf59 100644 --- a/src/lib/process/.may_include +++ b/src/lib/process/.may_include @@ -1,17 +1,20 @@ orconfig.h +lib/buf/*.h lib/cc/*.h lib/container/*.h lib/ctime/*.h lib/err/*.h -lib/intmath/*.h +lib/evloop/*.h lib/fs/*.h +lib/intmath/*.h lib/log/*.h lib/malloc/*.h lib/net/*.h lib/process/*.h lib/string/*.h +lib/subsys/*.h lib/testsupport/*.h lib/thread/*.h -ht.h
\ No newline at end of file +ext/ht.h diff --git a/src/lib/process/include.am b/src/lib/process/include.am index c6cc3a6699..83b67bf029 100644 --- a/src/lib/process/include.am +++ b/src/lib/process/include.am @@ -9,10 +9,14 @@ src_lib_libtor_process_a_SOURCES = \ src/lib/process/daemon.c \ src/lib/process/env.c \ src/lib/process/pidfile.c \ + src/lib/process/process.c \ + src/lib/process/process_sys.c \ + src/lib/process/process_unix.c \ + src/lib/process/process_win32.c \ src/lib/process/restrict.c \ src/lib/process/setuid.c \ - src/lib/process/subprocess.c \ - src/lib/process/waitpid.c + src/lib/process/waitpid.c \ + src/lib/process/winprocess_sys.c src_lib_libtor_process_testing_a_SOURCES = \ $(src_lib_libtor_process_a_SOURCES) @@ -23,7 +27,11 @@ noinst_HEADERS += \ src/lib/process/daemon.h \ src/lib/process/env.h \ src/lib/process/pidfile.h \ + src/lib/process/process.h \ + src/lib/process/process_sys.h \ + src/lib/process/process_unix.h \ + src/lib/process/process_win32.h \ src/lib/process/restrict.h \ src/lib/process/setuid.h \ - src/lib/process/subprocess.h \ - src/lib/process/waitpid.h + src/lib/process/waitpid.h \ + src/lib/process/winprocess_sys.h diff --git a/src/lib/process/process.c b/src/lib/process/process.c new file mode 100644 index 0000000000..422942dc83 --- /dev/null +++ b/src/lib/process/process.c @@ -0,0 +1,797 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process.c + * \brief Module for working with other processes. + **/ + +#define PROCESS_PRIVATE +#include "lib/buf/buffers.h" +#include "lib/net/buffers_net.h" +#include "lib/container/smartlist.h" +#include "lib/log/log.h" +#include "lib/log/util_bug.h" +#include "lib/process/process.h" +#include "lib/process/process_unix.h" +#include "lib/process/process_win32.h" +#include "lib/process/env.h" + +#ifdef HAVE_STDDEF_H +#include <stddef.h> +#endif + +/** A list of all <b>process_t</b> instances currently allocated. */ +static smartlist_t *processes; + +/** + * Boolean. If true, then Tor may call execve or CreateProcess via + * tor_spawn_background. + **/ +static int may_spawn_background_process = 1; + +/** Structure to represent a child process. */ +struct process_t { + /** Process status. */ + process_status_t status; + + /** Which protocol is the process using? */ + process_protocol_t protocol; + + /** Which function to call when we have data ready from stdout? */ + process_read_callback_t stdout_read_callback; + + /** Which function to call when we have data ready from stderr? */ + process_read_callback_t stderr_read_callback; + + /** Which function call when our process terminated? */ + process_exit_callback_t exit_callback; + + /** Our exit code when the process have terminated. */ + process_exit_code_t exit_code; + + /** Name of the command we want to execute (for example: /bin/ls). */ + char *command; + + /** The arguments used for the new process. The format here is one argument + * per element of the smartlist_t. On Windows these arguments are combined + * together using the <b>tor_join_win_cmdline</b> function. On Unix the + * process name (argv[0]) and the trailing NULL is added automatically before + * the process is executed. */ + smartlist_t *arguments; + + /** The environment used for the new process. */ + smartlist_t *environment; + + /** Buffer to store data from stdout when it is read. */ + buf_t *stdout_buffer; + + /** Buffer to store data from stderr when it is read. */ + buf_t *stderr_buffer; + + /** Buffer to store data to stdin before it is written. */ + buf_t *stdin_buffer; + + /** Do we need to store some custom data with the process? */ + void *data; + +#ifndef _WIN32 + /** Our Unix process handle. */ + process_unix_t *unix_process; +#else + /** Our Win32 process handle. */ + process_win32_t *win32_process; +#endif +}; + +/** Convert a given process status in <b>status</b> to its string + * representation. */ +const char * +process_status_to_string(process_status_t status) +{ + switch (status) { + case PROCESS_STATUS_NOT_RUNNING: + return "not running"; + case PROCESS_STATUS_RUNNING: + return "running"; + case PROCESS_STATUS_ERROR: + return "error"; + } + + /* LCOV_EXCL_START */ + tor_assert_unreached(); + return NULL; + /* LCOV_EXCL_STOP */ +} + +/** Convert a given process protocol in <b>protocol</b> to its string + * representation. */ +const char * +process_protocol_to_string(process_protocol_t protocol) +{ + switch (protocol) { + case PROCESS_PROTOCOL_LINE: + return "Line"; + case PROCESS_PROTOCOL_RAW: + return "Raw"; + } + + /* LCOV_EXCL_START */ + tor_assert_unreached(); + return NULL; + /* LCOV_EXCL_STOP */ +} + +/** + * Turn off may_spawn_background_process, so that all future calls to + * tor_spawn_background are guaranteed to fail. + **/ +void +tor_disable_spawning_background_processes(void) +{ + may_spawn_background_process = 0; +} + +/** Initialize the Process subsystem. This function initializes the Process + * subsystem's global state. For cleaning up, <b>process_free_all()</b> should + * be called. */ +void +process_init(void) +{ + processes = smartlist_new(); + +#ifdef _WIN32 + process_win32_init(); +#endif +} + +/** Free up all resources that is handled by the Process subsystem. Note that + * this call does not terminate already running processes. */ +void +process_free_all(void) +{ +#ifdef _WIN32 + process_win32_deinit(); +#endif + + SMARTLIST_FOREACH(processes, process_t *, x, process_free(x)); + smartlist_free(processes); +} + +/** Get a list of all processes. This function returns a smartlist of + * <b>process_t</b> containing all the currently allocated processes. */ +const smartlist_t * +process_get_all_processes(void) +{ + return processes; +} + +/** Allocate and initialize a new process. This function returns a newly + * allocated and initialized process data, which can be used to configure and + * later run a subprocess of Tor. Use the various <b>process_set_*()</b> + * methods to configure it and run the process using <b>process_exec()</b>. Use + * <b>command</b> to specify the path to the command to run. You can either + * specify an absolute path to the command or relative where Tor will use the + * underlying operating system's functionality for finding the command to run. + * */ +process_t * +process_new(const char *command) +{ + tor_assert(command); + + process_t *process; + process = tor_malloc_zero(sizeof(process_t)); + + /* Set our command. */ + process->command = tor_strdup(command); + + /* By default we are not running. */ + process->status = PROCESS_STATUS_NOT_RUNNING; + + /* Prepare process environment. */ + process->arguments = smartlist_new(); + process->environment = smartlist_new(); + + /* Prepare the buffers. */ + process->stdout_buffer = buf_new(); + process->stderr_buffer = buf_new(); + process->stdin_buffer = buf_new(); + +#ifndef _WIN32 + /* Prepare our Unix process handle. */ + process->unix_process = process_unix_new(); +#else + /* Prepare our Win32 process handle. */ + process->win32_process = process_win32_new(); +#endif + + smartlist_add(processes, process); + + return process; +} + +/** Deallocate the given process in <b>process</b>. */ +void +process_free_(process_t *process) +{ + if (! process) + return; + + /* Cleanup parameters. */ + tor_free(process->command); + + /* Cleanup arguments and environment. */ + SMARTLIST_FOREACH(process->arguments, char *, x, tor_free(x)); + smartlist_free(process->arguments); + + SMARTLIST_FOREACH(process->environment, char *, x, tor_free(x)); + smartlist_free(process->environment); + + /* Cleanup the buffers. */ + buf_free(process->stdout_buffer); + buf_free(process->stderr_buffer); + buf_free(process->stdin_buffer); + +#ifndef _WIN32 + /* Cleanup our Unix process handle. */ + process_unix_free(process->unix_process); +#else + /* Cleanup our Win32 process handle. */ + process_win32_free(process->win32_process); +#endif + + smartlist_remove(processes, process); + + tor_free(process); +} + +/** Execute the given process. This function executes the given process as a + * subprocess of Tor. Returns <b>PROCESS_STATUS_RUNNING</b> upon success. */ +process_status_t +process_exec(process_t *process) +{ + tor_assert(process); + + if (BUG(may_spawn_background_process == 0)) + return PROCESS_STATUS_ERROR; + + process_status_t status = PROCESS_STATUS_NOT_RUNNING; + + log_info(LD_PROCESS, "Starting new process: %s", process->command); + +#ifndef _WIN32 + status = process_unix_exec(process); +#else + status = process_win32_exec(process); +#endif + + /* Update our state. */ + process_set_status(process, status); + + if (status != PROCESS_STATUS_RUNNING) { + log_warn(LD_PROCESS, "Failed to start process: %s", + process_get_command(process)); + } + + return status; +} + +/** Terminate the given process. Returns true on success, + * otherwise false. */ +bool +process_terminate(process_t *process) +{ + tor_assert(process); + + /* Terminating a non-running process isn't going to work. */ + if (process_get_status(process) != PROCESS_STATUS_RUNNING) + return false; + + log_debug(LD_PROCESS, "Terminating process"); + +#ifndef _WIN32 + return process_unix_terminate(process); +#else + return process_win32_terminate(process); +#endif +} + +/** Returns the unique process identifier for the given <b>process</b>. */ +process_pid_t +process_get_pid(process_t *process) +{ + tor_assert(process); + +#ifndef _WIN32 + return process_unix_get_pid(process); +#else + return process_win32_get_pid(process); +#endif +} + +/** Set the callback function for output from the child process's standard out + * handle. This function sets the callback function which is called every time + * the child process have written output to its standard out file handle. + * + * Use <b>process_set_protocol(process, PROCESS_PROTOCOL_LINE)</b> if you want + * the callback to only contain complete "\n" or "\r\n" terminated lines. */ +void +process_set_stdout_read_callback(process_t *process, + process_read_callback_t callback) +{ + tor_assert(process); + process->stdout_read_callback = callback; +} + +/** Set the callback function for output from the child process's standard + * error handle. This function sets the callback function which is called + * every time the child process have written output to its standard error file + * handle. + * + * Use <b>process_set_protocol(process, PROCESS_PROTOCOL_LINE)</b> if you want + * the callback to only contain complete "\n" or "\r\n" terminated lines. */ +void +process_set_stderr_read_callback(process_t *process, + process_read_callback_t callback) +{ + tor_assert(process); + process->stderr_read_callback = callback; +} + +/** Set the callback function for process exit notification. The + * <b>callback</b> function will be called every time your child process have + * terminated. */ +void +process_set_exit_callback(process_t *process, + process_exit_callback_t callback) +{ + tor_assert(process); + process->exit_callback = callback; +} + +/** Get the current command of the given process. */ +const char * +process_get_command(const process_t *process) +{ + tor_assert(process); + return process->command; +} + +void +process_set_protocol(process_t *process, process_protocol_t protocol) +{ + tor_assert(process); + process->protocol = protocol; +} + +/** Get the currently used protocol of the given process. */ +process_protocol_t +process_get_protocol(const process_t *process) +{ + tor_assert(process); + return process->protocol; +} + +/** Set opague pointer to data. This function allows you to store a pointer to + * your own data in the given process. Use <b>process_get_data()</b> in the + * various callback functions to retrieve the data again. + * + * Note that the given process does NOT take ownership of the data and you are + * responsible for freeing up any resources allocated by the given data. + * */ +void +process_set_data(process_t *process, void *data) +{ + tor_assert(process); + process->data = data; +} + +/** Get the opaque pointer to callback data from the given process. This + * function allows you get the data you stored with <b>process_set_data()</b> + * in the different callback functions. */ +void * +process_get_data(const process_t *process) +{ + tor_assert(process); + return process->data; +} + +/** Set the status of a given process. */ +void +process_set_status(process_t *process, process_status_t status) +{ + tor_assert(process); + process->status = status; +} + +/** Get the status of the given process. */ +process_status_t +process_get_status(const process_t *process) +{ + tor_assert(process); + return process->status; +} + +/** Append an argument to the list of arguments in the given process. */ +void +process_append_argument(process_t *process, const char *argument) +{ + tor_assert(process); + tor_assert(argument); + + smartlist_add(process->arguments, tor_strdup(argument)); +} + +/** Returns a list of arguments (excluding the command itself) from the + * given process. */ +const smartlist_t * +process_get_arguments(const process_t *process) +{ + tor_assert(process); + return process->arguments; +} + +/** Returns a newly allocated Unix style argument vector. Use <b>tor_free()</b> + * to deallocate it after use. */ +char ** +process_get_argv(const process_t *process) +{ + tor_assert(process); + + /** Generate a Unix style process argument vector from our process's + * arguments smartlist_t. */ + char **argv = NULL; + + char *filename = process->command; + const smartlist_t *arguments = process->arguments; + const size_t size = smartlist_len(arguments); + + /* Make space for the process filename as argv[0] and a trailing NULL. */ + argv = tor_malloc_zero(sizeof(char *) * (size + 2)); + + /* Set our filename as first argument. */ + argv[0] = filename; + + /* Put in the rest of the values from arguments. */ + SMARTLIST_FOREACH_BEGIN(arguments, char *, arg_val) { + tor_assert(arg_val != NULL); + + argv[arg_val_sl_idx + 1] = arg_val; + } SMARTLIST_FOREACH_END(arg_val); + + return argv; +} + +/** This function clears the internal environment and copies over every string + * from <b>env</b> as the new environment. */ +void +process_reset_environment(process_t *process, const smartlist_t *env) +{ + tor_assert(process); + tor_assert(env); + + /* Cleanup old environment. */ + SMARTLIST_FOREACH(process->environment, char *, x, tor_free(x)); + smartlist_free(process->environment); + process->environment = smartlist_new(); + + SMARTLIST_FOREACH(env, char *, x, + smartlist_add(process->environment, tor_strdup(x))); +} + +/** Set the given <b>key</b>/<b>value</b> pair as environment variable in the + * given process. */ +void +process_set_environment(process_t *process, + const char *key, + const char *value) +{ + tor_assert(process); + tor_assert(key); + tor_assert(value); + + smartlist_add_asprintf(process->environment, "%s=%s", key, value); +} + +/** Returns a newly allocated <b>process_environment_t</b> containing the + * environment variables for the given process. */ +process_environment_t * +process_get_environment(const process_t *process) +{ + tor_assert(process); + return process_environment_make(process->environment); +} + +#ifndef _WIN32 +/** Get the internal handle for the Unix backend. */ +process_unix_t * +process_get_unix_process(const process_t *process) +{ + tor_assert(process); + tor_assert(process->unix_process); + return process->unix_process; +} +#else +/** Get the internal handle for Windows backend. */ +process_win32_t * +process_get_win32_process(const process_t *process) +{ + tor_assert(process); + tor_assert(process->win32_process); + return process->win32_process; +} +#endif + +/** Write <b>size</b> bytes of <b>data</b> to the given process's standard + * input. */ +void +process_write(process_t *process, + const uint8_t *data, size_t size) +{ + tor_assert(process); + tor_assert(data); + + buf_add(process->stdin_buffer, (char *)data, size); + process_write_stdin(process, process->stdin_buffer); +} + +/** As tor_vsnprintf(), but write the data to the given process's standard + * input. */ +void +process_vprintf(process_t *process, + const char *format, va_list args) +{ + tor_assert(process); + tor_assert(format); + + int size; + char *data; + + size = tor_vasprintf(&data, format, args); + process_write(process, (uint8_t *)data, size); + tor_free(data); +} + +/** As tor_snprintf(), but write the data to the given process's standard + * input. */ +void +process_printf(process_t *process, + const char *format, ...) +{ + tor_assert(process); + tor_assert(format); + + va_list ap; + va_start(ap, format); + process_vprintf(process, format, ap); + va_end(ap); +} + +/** This function is called by the Process backend when a given process have + * data that is ready to be read from the child process's standard output + * handle. */ +void +process_notify_event_stdout(process_t *process) +{ + tor_assert(process); + + int ret; + ret = process_read_stdout(process, process->stdout_buffer); + + if (ret > 0) + process_read_data(process, + process->stdout_buffer, + process->stdout_read_callback); +} + +/** This function is called by the Process backend when a given process have + * data that is ready to be read from the child process's standard error + * handle. */ +void +process_notify_event_stderr(process_t *process) +{ + tor_assert(process); + + int ret; + ret = process_read_stderr(process, process->stderr_buffer); + + if (ret > 0) + process_read_data(process, + process->stderr_buffer, + process->stderr_read_callback); +} + +/** This function is called by the Process backend when a given process is + * allowed to begin writing data to the standard input of the child process. */ +void +process_notify_event_stdin(process_t *process) +{ + tor_assert(process); + + process_write_stdin(process, process->stdin_buffer); +} + +/** This function is called by the Process backend when a given process have + * terminated. The exit status code is passed in <b>exit_code</b>. We mark the + * process as no longer running and calls the <b>exit_callback</b> with + * information about the process termination. The given <b>process</b> is + * free'd iff the exit_callback returns true. */ +void +process_notify_event_exit(process_t *process, process_exit_code_t exit_code) +{ + tor_assert(process); + + log_debug(LD_PROCESS, + "Process terminated with exit code: %"PRIu64, exit_code); + + /* Update our state. */ + process_set_status(process, PROCESS_STATUS_NOT_RUNNING); + process->exit_code = exit_code; + + /* Call our exit callback, if it exists. */ + bool free_process_handle = false; + + /* The exit callback will tell us if we should process_free() our handle. */ + if (process->exit_callback) + free_process_handle = process->exit_callback(process, exit_code); + + if (free_process_handle) + process_free(process); +} + +/** This function is called whenever the Process backend have notified us that + * there is data to be read from its standard out handle. Returns the number of + * bytes that have been put into the given buffer. */ +MOCK_IMPL(STATIC int, process_read_stdout, (process_t *process, buf_t *buffer)) +{ + tor_assert(process); + tor_assert(buffer); + +#ifndef _WIN32 + return process_unix_read_stdout(process, buffer); +#else + return process_win32_read_stdout(process, buffer); +#endif +} + +/** This function is called whenever the Process backend have notified us that + * there is data to be read from its standard error handle. Returns the number + * of bytes that have been put into the given buffer. */ +MOCK_IMPL(STATIC int, process_read_stderr, (process_t *process, buf_t *buffer)) +{ + tor_assert(process); + tor_assert(buffer); + +#ifndef _WIN32 + return process_unix_read_stderr(process, buffer); +#else + return process_win32_read_stderr(process, buffer); +#endif +} + +/** This function calls the backend function for the given process whenever + * there is data to be written to the backends' file handles. */ +MOCK_IMPL(STATIC void, process_write_stdin, + (process_t *process, buf_t *buffer)) +{ + tor_assert(process); + tor_assert(buffer); + +#ifndef _WIN32 + process_unix_write(process, buffer); +#else + process_win32_write(process, buffer); +#endif +} + +/** This function calls the protocol handlers based on the value of + * <b>process_get_protocol(process)</b>. Currently we call + * <b>process_read_buffer()</b> for <b>PROCESS_PROTOCOL_RAW</b> and + * <b>process_read_lines()</b> for <b>PROCESS_PROTOCOL_LINE</b>. */ +STATIC void +process_read_data(process_t *process, + buf_t *buffer, + process_read_callback_t callback) +{ + tor_assert(process); + tor_assert(buffer); + + switch (process_get_protocol(process)) { + case PROCESS_PROTOCOL_RAW: + process_read_buffer(process, buffer, callback); + break; + case PROCESS_PROTOCOL_LINE: + process_read_lines(process, buffer, callback); + break; + default: + /* LCOV_EXCL_START */ + tor_assert_unreached(); + return; + /* LCOV_EXCL_STOP */ + } +} + +/** This function takes the content of the given <b>buffer</b> and passes it to + * the given <b>callback</b> function, but ensures that an additional zero byte + * is added to the end of the data such that the given callback implementation + * can threat the content as a ASCIIZ string. */ +STATIC void +process_read_buffer(process_t *process, + buf_t *buffer, + process_read_callback_t callback) +{ + tor_assert(process); + tor_assert(buffer); + + const size_t size = buf_datalen(buffer); + + /* We allocate an extra byte for the zero byte in the end. */ + char *data = tor_malloc_zero(size + 1); + + buf_get_bytes(buffer, data, size); + log_debug(LD_PROCESS, "Read data from process"); + + if (callback) + callback(process, data, size); + + tor_free(data); +} + +/** This function tries to extract complete lines from the given <b>buffer</b> + * and calls the given <b>callback</b> function whenever it has a complete + * line. Before calling <b>callback</b> we remove the trailing "\n" or "\r\n" + * from the line. If we are unable to extract a complete line we leave the data + * in the buffer for next call. */ +STATIC void +process_read_lines(process_t *process, + buf_t *buffer, + process_read_callback_t callback) +{ + tor_assert(process); + tor_assert(buffer); + + const size_t size = buf_datalen(buffer) + 1; + size_t line_size = 0; + char *data = tor_malloc_zero(size); + int ret; + + while (true) { + line_size = size; + ret = buf_get_line(buffer, data, &line_size); + + /* A complete line should always be smaller than the size of our + * buffer. */ + tor_assert(ret != -1); + + /* Remove \n from the end of the line. */ + if (line_size >= 1 && data[line_size - 1] == '\n') { + data[line_size - 1] = '\0'; + --line_size; + } + + /* Remove \r from the end of the line. */ + if (line_size >= 1 && data[line_size - 1] == '\r') { + data[line_size - 1] = '\0'; + --line_size; + } + + if (ret == 1) { + log_debug(LD_PROCESS, "Read line from process: \"%s\"", data); + + if (callback) + callback(process, data, line_size); + + /* We have read a whole line, let's see if there is more lines to read. + * */ + continue; + } + + /* No complete line for us to read. We are done for now. */ + tor_assert_nonfatal(ret == 0); + break; + } + + tor_free(data); +} diff --git a/src/lib/process/process.h b/src/lib/process/process.h new file mode 100644 index 0000000000..14069923a0 --- /dev/null +++ b/src/lib/process/process.h @@ -0,0 +1,145 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process.h + * \brief Header for process.c + **/ + +#ifndef TOR_PROCESS_H +#define TOR_PROCESS_H + +#include "orconfig.h" +#include "lib/malloc/malloc.h" +#include "lib/string/printf.h" + +/** Maximum number of bytes to write to a process' stdin. */ +#define PROCESS_MAX_WRITE (1024) + +/** Maximum number of bytes to read from a process' stdout/stderr. */ +#define PROCESS_MAX_READ (1024) + +typedef enum { + /** The process is not running. */ + PROCESS_STATUS_NOT_RUNNING, + + /** The process is running. */ + PROCESS_STATUS_RUNNING, + + /** The process is in an erroneous state. */ + PROCESS_STATUS_ERROR +} process_status_t; + +const char *process_status_to_string(process_status_t status); + +typedef enum { + /** Pass complete \n-terminated lines to the + * callback (with the \n or \r\n removed). */ + PROCESS_PROTOCOL_LINE, + + /** Pass the raw response from read() to the callback. */ + PROCESS_PROTOCOL_RAW +} process_protocol_t; + +const char *process_protocol_to_string(process_protocol_t protocol); + +void tor_disable_spawning_background_processes(void); + +struct smartlist_t; + +struct process_t; +typedef struct process_t process_t; + +typedef uint64_t process_exit_code_t; +typedef uint64_t process_pid_t; + +typedef void (*process_read_callback_t)(process_t *, + const char *, + size_t); +typedef bool +(*process_exit_callback_t)(process_t *, process_exit_code_t); + +void process_init(void); +void process_free_all(void); +const struct smartlist_t *process_get_all_processes(void); + +process_t *process_new(const char *command); +void process_free_(process_t *process); +#define process_free(s) FREE_AND_NULL(process_t, process_free_, (s)) + +process_status_t process_exec(process_t *process); +bool process_terminate(process_t *process); + +process_pid_t process_get_pid(process_t *process); + +void process_set_stdout_read_callback(process_t *, + process_read_callback_t); +void process_set_stderr_read_callback(process_t *, + process_read_callback_t); +void process_set_exit_callback(process_t *, + process_exit_callback_t); + +const char *process_get_command(const process_t *process); + +void process_append_argument(process_t *process, const char *argument); +const struct smartlist_t *process_get_arguments(const process_t *process); +char **process_get_argv(const process_t *process); + +void process_reset_environment(process_t *process, + const struct smartlist_t *env); +void process_set_environment(process_t *process, + const char *key, + const char *value); + +struct process_environment_t; +struct process_environment_t *process_get_environment(const process_t *); + +void process_set_protocol(process_t *process, process_protocol_t protocol); +process_protocol_t process_get_protocol(const process_t *process); + +void process_set_data(process_t *process, void *data); +void *process_get_data(const process_t *process); + +void process_set_status(process_t *process, process_status_t status); +process_status_t process_get_status(const process_t *process); + +#ifndef _WIN32 +struct process_unix_t; +struct process_unix_t *process_get_unix_process(const process_t *process); +#else +struct process_win32_t; +struct process_win32_t *process_get_win32_process(const process_t *process); +#endif + +void process_write(process_t *process, + const uint8_t *data, size_t size); +void process_vprintf(process_t *process, + const char *format, va_list args) CHECK_PRINTF(2, 0); +void process_printf(process_t *process, + const char *format, ...) CHECK_PRINTF(2, 3); + +void process_notify_event_stdout(process_t *process); +void process_notify_event_stderr(process_t *process); +void process_notify_event_stdin(process_t *process); +void process_notify_event_exit(process_t *process, + process_exit_code_t); + +#ifdef PROCESS_PRIVATE +MOCK_DECL(STATIC int, process_read_stdout, (process_t *, buf_t *)); +MOCK_DECL(STATIC int, process_read_stderr, (process_t *, buf_t *)); +MOCK_DECL(STATIC void, process_write_stdin, (process_t *, buf_t *)); + +STATIC void process_read_data(process_t *process, + buf_t *buffer, + process_read_callback_t callback); +STATIC void process_read_buffer(process_t *process, + buf_t *buffer, + process_read_callback_t callback); +STATIC void process_read_lines(process_t *process, + buf_t *buffer, + process_read_callback_t callback); +#endif /* defined(PROCESS_PRIVATE). */ + +#endif /* defined(TOR_PROCESS_H). */ diff --git a/src/lib/process/process_sys.c b/src/lib/process/process_sys.c new file mode 100644 index 0000000000..3c809a00e8 --- /dev/null +++ b/src/lib/process/process_sys.c @@ -0,0 +1,33 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process_sys.c + * \brief Subsystem object for process setup. + **/ + +#include "orconfig.h" +#include "lib/subsys/subsys.h" +#include "lib/process/process_sys.h" +#include "lib/process/process.h" + +static int +subsys_process_initialize(void) +{ + process_init(); + return 0; +} + +static void +subsys_process_shutdown(void) +{ + process_free_all(); +} + +const subsys_fns_t sys_process = { + .name = "process", + .level = -35, + .supported = true, + .initialize = subsys_process_initialize, + .shutdown = subsys_process_shutdown +}; diff --git a/src/lib/process/process_sys.h b/src/lib/process/process_sys.h new file mode 100644 index 0000000000..b7a116d838 --- /dev/null +++ b/src/lib/process/process_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process_sys.h + * \brief Declare subsystem object for the process module. + **/ + +#ifndef TOR_PROCESS_SYS_H +#define TOR_PROCESS_SYS_H + +extern const struct subsys_fns_t sys_process; + +#endif /* !defined(TOR_PROCESS_SYS_H) */ diff --git a/src/lib/process/process_unix.c b/src/lib/process/process_unix.c new file mode 100644 index 0000000000..b102afff44 --- /dev/null +++ b/src/lib/process/process_unix.c @@ -0,0 +1,698 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process_unix.c + * \brief Module for working with Unix processes. + **/ + +#define PROCESS_UNIX_PRIVATE +#include "lib/intmath/cmp.h" +#include "lib/buf/buffers.h" +#include "lib/net/buffers_net.h" +#include "lib/container/smartlist.h" +#include "lib/evloop/compat_libevent.h" +#include "lib/log/log.h" +#include "lib/log/util_bug.h" +#include "lib/process/process.h" +#include "lib/process/process_unix.h" +#include "lib/process/waitpid.h" +#include "lib/process/env.h" + +#include <stdio.h> + +#ifdef HAVE_STRING_H +#include <string.h> +#endif + +#ifdef HAVE_ERRNO_H +#include <errno.h> +#endif + +#ifdef HAVE_UNISTD_H +#include <unistd.h> +#endif + +#ifdef HAVE_FCNTL_H +#include <fcntl.h> +#endif + +#if defined(HAVE_SYS_PRCTL_H) && defined(__linux__) +#include <sys/prctl.h> +#endif + +#if HAVE_SIGNAL_H +#include <signal.h> +#endif + +#ifndef _WIN32 + +/** Maximum number of file descriptors, if we cannot get it via sysconf() */ +#define DEFAULT_MAX_FD 256 + +/** Internal state for Unix handles. */ +struct process_unix_handle_t { + /** Unix File Descriptor. */ + int fd; + + /** Have we reached end of file? */ + bool reached_eof; + + /** Event structure for libevent. */ + struct event *event; + + /** Are we writing? */ + bool is_writing; +}; + +/** Internal state for our Unix process. */ +struct process_unix_t { + /** Standard in handle. */ + process_unix_handle_t stdin_handle; + + /** Standard out handle. */ + process_unix_handle_t stdout_handle; + + /** Standard error handle. */ + process_unix_handle_t stderr_handle; + + /** The process identifier of our process. */ + pid_t pid; + + /** Waitpid Callback structure. */ + waitpid_callback_t *waitpid; +}; + +/** Returns a newly allocated <b>process_unix_t</b>. */ +process_unix_t * +process_unix_new(void) +{ + process_unix_t *unix_process; + unix_process = tor_malloc_zero(sizeof(process_unix_t)); + + unix_process->stdin_handle.fd = -1; + unix_process->stderr_handle.fd = -1; + unix_process->stdout_handle.fd = -1; + + return unix_process; +} + +/** Deallocates the given <b>unix_process</b>. */ +void +process_unix_free_(process_unix_t *unix_process) +{ + if (! unix_process) + return; + + /* Clean up our waitpid callback. */ + clear_waitpid_callback(unix_process->waitpid); + + /* FIXME(ahf): Refactor waitpid code? */ + unix_process->waitpid = NULL; + + /* Close all our file descriptors. */ + process_unix_close_file_descriptors(unix_process); + + tor_event_free(unix_process->stdout_handle.event); + tor_event_free(unix_process->stderr_handle.event); + tor_event_free(unix_process->stdin_handle.event); + + tor_free(unix_process); +} + +/** Executes the given process as a child process of Tor. This function is + * responsible for setting up the child process and run it. This includes + * setting up pipes for interprocess communication, initialize the waitpid + * callbacks, and finally run fork() followed by execve(). Returns + * <b>PROCESS_STATUS_RUNNING</b> upon success. */ +process_status_t +process_unix_exec(process_t *process) +{ + static int max_fd = -1; + + process_unix_t *unix_process; + pid_t pid; + int stdin_pipe[2]; + int stdout_pipe[2]; + int stderr_pipe[2]; + int retval, fd; + + unix_process = process_get_unix_process(process); + + /* Create standard in pipe. */ + retval = pipe(stdin_pipe); + + if (-1 == retval) { + log_warn(LD_PROCESS, + "Unable to create pipe for stdin " + "communication with process: %s", + strerror(errno)); + + return PROCESS_STATUS_ERROR; + } + + /* Create standard out pipe. */ + retval = pipe(stdout_pipe); + + if (-1 == retval) { + log_warn(LD_PROCESS, + "Unable to create pipe for stdout " + "communication with process: %s", + strerror(errno)); + + /** Cleanup standard in pipe. */ + close(stdin_pipe[0]); + close(stdin_pipe[1]); + + return PROCESS_STATUS_ERROR; + } + + /* Create standard error pipe. */ + retval = pipe(stderr_pipe); + + if (-1 == retval) { + log_warn(LD_PROCESS, + "Unable to create pipe for stderr " + "communication with process: %s", + strerror(errno)); + + /** Cleanup standard in pipe. */ + close(stdin_pipe[0]); + close(stdin_pipe[1]); + + /** Cleanup standard out pipe. */ + close(stdout_pipe[0]); + close(stdout_pipe[1]); + + return PROCESS_STATUS_ERROR; + } + +#ifdef _SC_OPEN_MAX + if (-1 == max_fd) { + max_fd = (int)sysconf(_SC_OPEN_MAX); + + if (max_fd == -1) { + max_fd = DEFAULT_MAX_FD; + log_warn(LD_PROCESS, + "Cannot find maximum file descriptor, assuming: %d", max_fd); + } + } +#else /* !(defined(_SC_OPEN_MAX)) */ + max_fd = DEFAULT_MAX_FD; +#endif /* defined(_SC_OPEN_MAX) */ + + pid = fork(); + + if (0 == pid) { + /* This code is running in the child process context. */ + +#if defined(HAVE_SYS_PRCTL_H) && defined(__linux__) + /* Attempt to have the kernel issue a SIGTERM if the parent + * goes away. Certain attributes of the binary being execve()ed + * will clear this during the execve() call, but it's better + * than nothing. + */ + prctl(PR_SET_PDEATHSIG, SIGTERM); +#endif /* defined(HAVE_SYS_PRCTL_H) && defined(__linux__) */ + + /* Link process stdout to the write end of the pipe. */ + retval = dup2(stdout_pipe[1], STDOUT_FILENO); + if (-1 == retval) + goto error; + + /* Link process stderr to the write end of the pipe. */ + retval = dup2(stderr_pipe[1], STDERR_FILENO); + if (-1 == retval) + goto error; + + /* Link process stdin to the read end of the pipe */ + retval = dup2(stdin_pipe[0], STDIN_FILENO); + if (-1 == retval) + goto error; + + /* Close our pipes now after they have been dup2()'ed. */ + close(stderr_pipe[0]); + close(stderr_pipe[1]); + close(stdout_pipe[0]); + close(stdout_pipe[1]); + close(stdin_pipe[0]); + close(stdin_pipe[1]); + + /* Close all other fds, including the read end of the pipe. XXX: We should + * now be doing enough FD_CLOEXEC setting to make this needless. + */ + for (fd = STDERR_FILENO + 1; fd < max_fd; fd++) + close(fd); + + /* Create the argv value for our new process. */ + char **argv = process_get_argv(process); + + /* Create the env value for our new process. */ + process_environment_t *env = process_get_environment(process); + + /* Call the requested program. */ + execve(argv[0], argv, env->unixoid_environment_block); + + /* If we made it here it is because execve failed :-( */ + tor_free(argv); + process_environment_free(env); + + error: + fprintf(stderr, "Error from child process: %s", strerror(errno)); + _exit(1); + } + + /* We are in the parent process. */ + if (-1 == pid) { + log_warn(LD_PROCESS, + "Failed to create child process: %s", strerror(errno)); + + /** Cleanup standard in pipe. */ + close(stdin_pipe[0]); + close(stdin_pipe[1]); + + /** Cleanup standard out pipe. */ + close(stdout_pipe[0]); + close(stdout_pipe[1]); + + /** Cleanup standard error pipe. */ + close(stderr_pipe[0]); + close(stderr_pipe[1]); + + return PROCESS_STATUS_ERROR; + } + + /* Register our PID. */ + unix_process->pid = pid; + + /* Setup waitpid callbacks. */ + unix_process->waitpid = set_waitpid_callback(pid, + process_unix_waitpid_callback, + process); + + /* Handle standard out. */ + unix_process->stdout_handle.fd = stdout_pipe[0]; + retval = close(stdout_pipe[1]); + + if (-1 == retval) { + log_warn(LD_PROCESS, "Failed to close write end of standard out pipe: %s", + strerror(errno)); + } + + /* Handle standard error. */ + unix_process->stderr_handle.fd = stderr_pipe[0]; + retval = close(stderr_pipe[1]); + + if (-1 == retval) { + log_warn(LD_PROCESS, + "Failed to close write end of standard error pipe: %s", + strerror(errno)); + } + + /* Handle standard in. */ + unix_process->stdin_handle.fd = stdin_pipe[1]; + retval = close(stdin_pipe[0]); + + if (-1 == retval) { + log_warn(LD_PROCESS, "Failed to close read end of standard in pipe: %s", + strerror(errno)); + } + + /* Setup our handles. */ + process_unix_setup_handle(process, + &unix_process->stdout_handle, + EV_READ|EV_PERSIST, + stdout_read_callback); + + process_unix_setup_handle(process, + &unix_process->stderr_handle, + EV_READ|EV_PERSIST, + stderr_read_callback); + + process_unix_setup_handle(process, + &unix_process->stdin_handle, + EV_WRITE|EV_PERSIST, + stdin_write_callback); + + /* Start reading from standard out and standard error. */ + process_unix_start_reading(&unix_process->stdout_handle); + process_unix_start_reading(&unix_process->stderr_handle); + + return PROCESS_STATUS_RUNNING; +} + +/** Terminate the given process. Returns true on success, otherwise false. */ +bool +process_unix_terminate(process_t *process) +{ + tor_assert(process); + + process_unix_t *unix_process = process_get_unix_process(process); + + /* All running processes should have a waitpid. */ + if (BUG(unix_process->waitpid == NULL)) + return false; + + bool success = true; + + /* Send a SIGTERM to our child process. */ + int ret; + + ret = kill(unix_process->pid, SIGTERM); + + if (ret == -1) { + log_warn(LD_PROCESS, "Unable to terminate process: %s", + strerror(errno)); + success = false; + } + + /* Close all our FD's. */ + if (! process_unix_close_file_descriptors(unix_process)) + success = false; + + return success; +} + +/** Returns the unique process identifier for the given <b>process</b>. */ +process_pid_t +process_unix_get_pid(process_t *process) +{ + tor_assert(process); + + process_unix_t *unix_process = process_get_unix_process(process); + return (process_pid_t)unix_process->pid; +} + +/** Write the given <b>buffer</b> as input to the given <b>process</b>'s + * standard input. Returns the number of bytes written. */ +int +process_unix_write(process_t *process, buf_t *buffer) +{ + tor_assert(process); + tor_assert(buffer); + + process_unix_t *unix_process = process_get_unix_process(process); + + size_t buffer_flush_len = buf_datalen(buffer); + const size_t max_to_write = MIN(PROCESS_MAX_WRITE, buffer_flush_len); + + /* If we have data to write (when buffer_flush_len > 0) and we are not + * currently getting file descriptor events from the kernel, we tell the + * kernel to start notifying us about when we can write to our file + * descriptor and return. */ + if (buffer_flush_len > 0 && ! unix_process->stdin_handle.is_writing) { + process_unix_start_writing(&unix_process->stdin_handle); + return 0; + } + + /* We don't have any data to write, but the kernel is currently notifying us + * about whether we are able to write or not. Tell the kernel to stop + * notifying us until we have data to write. */ + if (buffer_flush_len == 0 && unix_process->stdin_handle.is_writing) { + process_unix_stop_writing(&unix_process->stdin_handle); + return 0; + } + + /* We have data to write and the kernel have told us to write it. */ + return buf_flush_to_pipe(buffer, + process_get_unix_process(process)->stdin_handle.fd, + max_to_write, &buffer_flush_len); +} + +/** Read data from the given process's standard output and put it into + * <b>buffer</b>. Returns the number of bytes read. */ +int +process_unix_read_stdout(process_t *process, buf_t *buffer) +{ + tor_assert(process); + tor_assert(buffer); + + process_unix_t *unix_process = process_get_unix_process(process); + + return process_unix_read_handle(process, + &unix_process->stdout_handle, + buffer); +} + +/** Read data from the given process's standard error and put it into + * <b>buffer</b>. Returns the number of bytes read. */ +int +process_unix_read_stderr(process_t *process, buf_t *buffer) +{ + tor_assert(process); + tor_assert(buffer); + + process_unix_t *unix_process = process_get_unix_process(process); + + return process_unix_read_handle(process, + &unix_process->stderr_handle, + buffer); +} + +/** This function is called whenever libevent thinks we have data that could be + * read from the child process's standard output. We notify the Process + * subsystem, which is then responsible for calling back to us for doing the + * actual reading of the data. */ +STATIC void +stdout_read_callback(evutil_socket_t fd, short event, void *data) +{ + (void)fd; + (void)event; + + process_t *process = data; + tor_assert(process); + + process_notify_event_stdout(process); +} + +/** This function is called whenever libevent thinks we have data that could be + * read from the child process's standard error. We notify the Process + * subsystem, which is then responsible for calling back to us for doing the + * actual reading of the data. */ +STATIC void +stderr_read_callback(evutil_socket_t fd, short event, void *data) +{ + (void)fd; + (void)event; + + process_t *process = data; + tor_assert(process); + + process_notify_event_stderr(process); +} + +/** This function is called whenever libevent thinks we have data that could be + * written the child process's standard input. We notify the Process subsystem, + * which is then responsible for calling back to us for doing the actual write + * of the data. */ +STATIC void +stdin_write_callback(evutil_socket_t fd, short event, void *data) +{ + (void)fd; + (void)event; + + process_t *process = data; + tor_assert(process); + + process_notify_event_stdin(process); +} + +/** This function tells libevent that we are interested in receiving read + * events from the given <b>handle</b>. */ +STATIC void +process_unix_start_reading(process_unix_handle_t *handle) +{ + tor_assert(handle); + + if (event_add(handle->event, NULL)) + log_warn(LD_PROCESS, + "Unable to add libevent event for handle."); +} + +/** This function tells libevent that we are no longer interested in receiving + * read events from the given <b>handle</b>. */ +STATIC void +process_unix_stop_reading(process_unix_handle_t *handle) +{ + tor_assert(handle); + + if (handle->event == NULL) + return; + + if (event_del(handle->event)) + log_warn(LD_PROCESS, + "Unable to delete libevent event for handle."); +} + +/** This function tells libevent that we are interested in receiving write + * events from the given <b>handle</b>. */ +STATIC void +process_unix_start_writing(process_unix_handle_t *handle) +{ + tor_assert(handle); + + if (event_add(handle->event, NULL)) + log_warn(LD_PROCESS, + "Unable to add libevent event for handle."); + + handle->is_writing = true; +} + +/** This function tells libevent that we are no longer interested in receiving + * write events from the given <b>handle</b>. */ +STATIC void +process_unix_stop_writing(process_unix_handle_t *handle) +{ + tor_assert(handle); + + if (handle->event == NULL) + return; + + if (event_del(handle->event)) + log_warn(LD_PROCESS, + "Unable to delete libevent event for handle."); + + handle->is_writing = false; +} + +/** This function is called when the waitpid system have detected that our + * process have terminated. We disable the waitpid system and notify the + * Process subsystem that we have terminated. */ +STATIC void +process_unix_waitpid_callback(int status, void *data) +{ + tor_assert(data); + + process_t *process = data; + process_unix_t *unix_process = process_get_unix_process(process); + + /* Remove our waitpid callback. */ + clear_waitpid_callback(unix_process->waitpid); + unix_process->waitpid = NULL; + + /* Notify our process. */ + process_notify_event_exit(process, status); + + /* Make sure you don't modify the process after we have called + * process_notify_event_exit() on it, to allow users to process_free() it in + * the exit callback. */ +} + +/** This function sets the file descriptor in the <b>handle</b> as non-blocking + * and configures the libevent event structure based on the given <b>flags</b> + * to ensure that <b>callback</b> is called whenever we have events on the + * given <b>handle</b>. */ +STATIC void +process_unix_setup_handle(process_t *process, + process_unix_handle_t *handle, + short flags, + event_callback_fn callback) +{ + tor_assert(process); + tor_assert(handle); + tor_assert(callback); + + /* Put our file descriptor into non-blocking mode. */ + if (fcntl(handle->fd, F_SETFL, O_NONBLOCK) < 0) { + log_warn(LD_PROCESS, "Unable mark Unix handle as non-blocking: %s", + strerror(errno)); + } + + /* Setup libevent event. */ + handle->event = tor_event_new(tor_libevent_get_base(), + handle->fd, + flags, + callback, + process); +} + +/** This function reads data from the given <b>handle</b> and puts it into + * <b>buffer</b>. Returns the number of bytes read this way. */ +STATIC int +process_unix_read_handle(process_t *process, + process_unix_handle_t *handle, + buf_t *buffer) +{ + tor_assert(process); + tor_assert(handle); + tor_assert(buffer); + + int ret = 0; + int eof = 0; + int error = 0; + + ret = buf_read_from_pipe(buffer, + handle->fd, + PROCESS_MAX_READ, + &eof, + &error); + + if (error) + log_warn(LD_PROCESS, + "Unable to read data: %s", strerror(error)); + + if (eof) { + handle->reached_eof = true; + process_unix_stop_reading(handle); + } + + return ret; +} + +/** Close the standard in, out, and error handles of the given + * <b>unix_process</b>. */ +STATIC bool +process_unix_close_file_descriptors(process_unix_t *unix_process) +{ + tor_assert(unix_process); + + int ret; + bool success = true; + + /* Stop reading and writing before we close() our + * file descriptors. */ + if (! unix_process->stdout_handle.reached_eof) + process_unix_stop_reading(&unix_process->stdout_handle); + + if (! unix_process->stderr_handle.reached_eof) + process_unix_stop_reading(&unix_process->stderr_handle); + + if (unix_process->stdin_handle.is_writing) + process_unix_stop_writing(&unix_process->stdin_handle); + + if (unix_process->stdin_handle.fd != -1) { + ret = close(unix_process->stdin_handle.fd); + if (ret == -1) { + log_warn(LD_PROCESS, "Unable to close standard in"); + success = false; + } + + unix_process->stdin_handle.fd = -1; + } + + if (unix_process->stdout_handle.fd != -1) { + ret = close(unix_process->stdout_handle.fd); + if (ret == -1) { + log_warn(LD_PROCESS, "Unable to close standard out"); + success = false; + } + + unix_process->stdout_handle.fd = -1; + } + + if (unix_process->stderr_handle.fd != -1) { + ret = close(unix_process->stderr_handle.fd); + if (ret == -1) { + log_warn(LD_PROCESS, "Unable to close standard error"); + success = false; + } + + unix_process->stderr_handle.fd = -1; + } + + return success; +} + +#endif /* defined(_WIN32). */ diff --git a/src/lib/process/process_unix.h b/src/lib/process/process_unix.h new file mode 100644 index 0000000000..a1d8f72993 --- /dev/null +++ b/src/lib/process/process_unix.h @@ -0,0 +1,68 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process_unix.h + * \brief Header for process_unix.c + **/ + +#ifndef TOR_PROCESS_UNIX_H +#define TOR_PROCESS_UNIX_H + +#ifndef _WIN32 + +#include "orconfig.h" +#include "lib/malloc/malloc.h" + +#include <event2/event.h> + +struct process_t; + +struct process_unix_t; +typedef struct process_unix_t process_unix_t; + +process_unix_t *process_unix_new(void); +void process_unix_free_(process_unix_t *unix_process); +#define process_unix_free(s) \ + FREE_AND_NULL(process_unix_t, process_unix_free_, (s)) + +process_status_t process_unix_exec(struct process_t *process); +bool process_unix_terminate(struct process_t *process); + +process_pid_t process_unix_get_pid(struct process_t *process); + +int process_unix_write(struct process_t *process, buf_t *buffer); +int process_unix_read_stdout(struct process_t *process, buf_t *buffer); +int process_unix_read_stderr(struct process_t *process, buf_t *buffer); + +#ifdef PROCESS_UNIX_PRIVATE +struct process_unix_handle_t; +typedef struct process_unix_handle_t process_unix_handle_t; + +STATIC void stdout_read_callback(evutil_socket_t fd, short event, void *data); +STATIC void stderr_read_callback(evutil_socket_t fd, short event, void *data); +STATIC void stdin_write_callback(evutil_socket_t fd, short event, void *data); + +STATIC void process_unix_start_reading(process_unix_handle_t *); +STATIC void process_unix_stop_reading(process_unix_handle_t *); + +STATIC void process_unix_start_writing(process_unix_handle_t *); +STATIC void process_unix_stop_writing(process_unix_handle_t *); + +STATIC void process_unix_waitpid_callback(int status, void *data); + +STATIC void process_unix_setup_handle(process_t *process, + process_unix_handle_t *handle, + short flags, + event_callback_fn callback); +STATIC int process_unix_read_handle(process_t *, + process_unix_handle_t *, + buf_t *); +STATIC bool process_unix_close_file_descriptors(process_unix_t *); +#endif /* defined(PROCESS_UNIX_PRIVATE). */ + +#endif /* defined(_WIN32). */ + +#endif /* defined(TOR_PROCESS_UNIX_H). */ diff --git a/src/lib/process/process_win32.c b/src/lib/process/process_win32.c new file mode 100644 index 0000000000..1b1de6ad55 --- /dev/null +++ b/src/lib/process/process_win32.c @@ -0,0 +1,1105 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process_win32.c + * \brief Module for working with Windows processes. + **/ + +#define PROCESS_WIN32_PRIVATE +#include "lib/intmath/cmp.h" +#include "lib/buf/buffers.h" +#include "lib/net/buffers_net.h" +#include "lib/container/smartlist.h" +#include "lib/log/log.h" +#include "lib/log/util_bug.h" +#include "lib/log/win32err.h" +#include "lib/process/process.h" +#include "lib/process/process_win32.h" +#include "lib/process/env.h" + +#ifdef HAVE_SYS_TIME_H +#include <sys/time.h> +#endif + +#ifdef HAVE_STRING_H +#include <string.h> +#endif + +#ifdef _WIN32 + +/** The size of our intermediate buffers. */ +#define BUFFER_SIZE (1024) + +/** Timer that ticks once a second and calls the process_win32_timer_callback() + * function. */ +static periodic_timer_t *periodic_timer; + +/** Structure to represent the state around the pipe HANDLE. + * + * This structure is used to store state about a given HANDLE, including + * whether we have reached end of file, its intermediate buffers, and how much + * data that is available in the intermediate buffer. */ +struct process_win32_handle_t { + /** Standard out pipe handle. */ + HANDLE pipe; + + /** True iff we have reached EOF from the pipe. */ + bool reached_eof; + + /** How much data is available in buffer. */ + size_t data_available; + + /** Intermediate buffer for ReadFileEx() and WriteFileEx(). */ + char buffer[BUFFER_SIZE]; + + /** Overlapped structure for ReadFileEx() and WriteFileEx(). */ + OVERLAPPED overlapped; + + /** Are we waiting for another I/O operation to complete? */ + bool busy; +}; + +/** Structure to represent the Windows specific implementation details of this + * Process backend. + * + * This structure is attached to <b>process_t</b> (see process.h) and is + * reachable from <b>process_t</b> via the <b>process_get_win32_process()</b> + * method. */ +struct process_win32_t { + /** Standard in state. */ + process_win32_handle_t stdin_handle; + + /** Standard out state. */ + process_win32_handle_t stdout_handle; + + /** Standard error state. */ + process_win32_handle_t stderr_handle; + + /** Process Information. */ + PROCESS_INFORMATION process_information; +}; + +/** Create a new <b>process_win32_t</b>. + * + * This function constructs a new <b>process_win32_t</b> and initializes the + * default values. */ +process_win32_t * +process_win32_new(void) +{ + process_win32_t *win32_process; + win32_process = tor_malloc_zero(sizeof(process_win32_t)); + + win32_process->stdin_handle.pipe = INVALID_HANDLE_VALUE; + win32_process->stdout_handle.pipe = INVALID_HANDLE_VALUE; + win32_process->stderr_handle.pipe = INVALID_HANDLE_VALUE; + + return win32_process; +} + +/** Free a given <b>process_win32_t</b>. + * + * This function deinitializes and frees up the resources allocated for the + * given <b>process_win32_t</b>. */ +void +process_win32_free_(process_win32_t *win32_process) +{ + if (! win32_process) + return; + + /* Cleanup our handles. */ + process_win32_cleanup_handle(&win32_process->stdin_handle); + process_win32_cleanup_handle(&win32_process->stdout_handle); + process_win32_cleanup_handle(&win32_process->stderr_handle); + + tor_free(win32_process); +} + +/** Initialize the Windows backend of the Process subsystem. */ +void +process_win32_init(void) +{ + /* We don't start the periodic timer here because it makes no sense to have + * the timer running until we have some processes that benefits from the + * timer timer ticks. */ +} + +/** Deinitialize the Windows backend of the Process subsystem. */ +void +process_win32_deinit(void) +{ + /* Stop our timer, but only if it's running. */ + if (process_win32_timer_running()) + process_win32_timer_stop(); +} + +/** Execute the given process. This function is responsible for setting up + * named pipes for I/O between the child process and the Tor process. Returns + * <b>PROCESS_STATUS_RUNNING</b> upon success. */ +process_status_t +process_win32_exec(process_t *process) +{ + tor_assert(process); + + process_win32_t *win32_process = process_get_win32_process(process); + + HANDLE stdout_pipe_read = NULL; + HANDLE stdout_pipe_write = NULL; + HANDLE stderr_pipe_read = NULL; + HANDLE stderr_pipe_write = NULL; + HANDLE stdin_pipe_read = NULL; + HANDLE stdin_pipe_write = NULL; + BOOL ret = FALSE; + + /* Setup our security attributes. */ + SECURITY_ATTRIBUTES security_attributes; + security_attributes.nLength = sizeof(security_attributes); + security_attributes.bInheritHandle = TRUE; + /* FIXME: should we set explicit security attributes? + * (See Ticket #2046, comment 5) */ + security_attributes.lpSecurityDescriptor = NULL; + + /* Create our standard out pipe. */ + if (! process_win32_create_pipe(&stdout_pipe_read, + &stdout_pipe_write, + &security_attributes, + PROCESS_WIN32_PIPE_TYPE_READER)) { + return PROCESS_STATUS_ERROR; + } + + /* Create our standard error pipe. */ + if (! process_win32_create_pipe(&stderr_pipe_read, + &stderr_pipe_write, + &security_attributes, + PROCESS_WIN32_PIPE_TYPE_READER)) { + return PROCESS_STATUS_ERROR; + } + + /* Create out standard in pipe. */ + if (! process_win32_create_pipe(&stdin_pipe_read, + &stdin_pipe_write, + &security_attributes, + PROCESS_WIN32_PIPE_TYPE_WRITER)) { + return PROCESS_STATUS_ERROR; + } + + /* Configure startup info for our child process. */ + STARTUPINFOA startup_info; + + memset(&startup_info, 0, sizeof(startup_info)); + startup_info.cb = sizeof(startup_info); + startup_info.hStdError = stderr_pipe_write; + startup_info.hStdOutput = stdout_pipe_write; + startup_info.hStdInput = stdin_pipe_read; + startup_info.dwFlags |= STARTF_USESTDHANDLES; + + /* Create the env value for our new process. */ + process_environment_t *env = process_get_environment(process); + + /* Create the argv value for our new process. */ + char **argv = process_get_argv(process); + + /* Windows expects argv to be a whitespace delimited string, so join argv up + */ + char *joined_argv = tor_join_win_cmdline((const char **)argv); + + /* Create the child process */ + ret = CreateProcessA(NULL, + joined_argv, + NULL, + NULL, + TRUE, + CREATE_NO_WINDOW, + env->windows_environment_block[0] == '\0' ? + NULL : env->windows_environment_block, + NULL, + &startup_info, + &win32_process->process_information); + + tor_free(argv); + tor_free(joined_argv); + process_environment_free(env); + + if (! ret) { + log_warn(LD_PROCESS, "CreateProcessA() failed: %s", + format_win32_error(GetLastError())); + + /* Cleanup our handles. */ + CloseHandle(stdout_pipe_read); + CloseHandle(stdout_pipe_write); + CloseHandle(stderr_pipe_read); + CloseHandle(stderr_pipe_write); + CloseHandle(stdin_pipe_read); + CloseHandle(stdin_pipe_write); + + /* In the Unix backend, we do not get an error in the Tor process when a + * child process fails to spawn its target executable since we need to + * first do the fork() call in the Tor process and then the child process + * is responsible for doing the call to execve(). + * + * This means that the user of the process_exec() API must check for + * whether it returns PROCESS_STATUS_ERROR, which will rarely happen on + * Unix, but will happen for error cases on Windows where it does not + * happen on Unix. For example: when the target executable does not exist + * on the file system. + * + * To have somewhat feature compatibility between the Unix and the Windows + * backend, we here notify the process_t owner that the process have exited + * (even though it never managed to run) to ensure that the exit callback + * is executed. + */ + process_notify_event_exit(process, 0); + + return PROCESS_STATUS_ERROR; + } + + /* TODO: Should we close hProcess and hThread in + * process_handle->process_information? */ + win32_process->stdout_handle.pipe = stdout_pipe_read; + win32_process->stderr_handle.pipe = stderr_pipe_read; + win32_process->stdin_handle.pipe = stdin_pipe_write; + + /* Close our ends of the pipes that is now owned by the child process. */ + CloseHandle(stdout_pipe_write); + CloseHandle(stderr_pipe_write); + CloseHandle(stdin_pipe_read); + + /* Used by the callback functions from ReadFileEx() and WriteFileEx() such + * that we can figure out which process_t that was responsible for the event. + * + * Warning, here be dragons: + * + * MSDN says that the hEvent member of the overlapped structure is unused + * for ReadFileEx() and WriteFileEx, which allows us to store a pointer to + * our process state there. + */ + win32_process->stdout_handle.overlapped.hEvent = (HANDLE)process; + win32_process->stderr_handle.overlapped.hEvent = (HANDLE)process; + win32_process->stdin_handle.overlapped.hEvent = (HANDLE)process; + + /* Start our timer if it is not already running. */ + if (! process_win32_timer_running()) + process_win32_timer_start(); + + /* We use Windows Extended I/O functions, so our completion callbacks are + * called automatically for us when there is data to read. Because of this + * we start the read of standard out and error right away. */ + process_notify_event_stdout(process); + process_notify_event_stderr(process); + + return PROCESS_STATUS_RUNNING; +} + +/** Terminate the given process. Returns true on success, otherwise false. */ +bool +process_win32_terminate(process_t *process) +{ + tor_assert(process); + + process_win32_t *win32_process = process_get_win32_process(process); + + /* Terminate our process. */ + BOOL ret; + + ret = TerminateProcess(win32_process->process_information.hProcess, 0); + + if (! ret) { + log_warn(LD_PROCESS, "TerminateProcess() failed: %s", + format_win32_error(GetLastError())); + return false; + } + + /* Cleanup our handles. */ + process_win32_cleanup_handle(&win32_process->stdin_handle); + process_win32_cleanup_handle(&win32_process->stdout_handle); + process_win32_cleanup_handle(&win32_process->stderr_handle); + + return true; +} + +/** Returns the unique process identifier for the given <b>process</b>. */ +process_pid_t +process_win32_get_pid(process_t *process) +{ + tor_assert(process); + + process_win32_t *win32_process = process_get_win32_process(process); + return (process_pid_t)win32_process->process_information.dwProcessId; +} + +/** Schedule an async write of the data found in <b>buffer</b> for the given + * process. This function runs an async write operation of the content of + * buffer, if we are not already waiting for a pending I/O request. Returns the + * number of bytes that Windows will hopefully write for us in the background. + * */ +int +process_win32_write(struct process_t *process, buf_t *buffer) +{ + tor_assert(process); + tor_assert(buffer); + + process_win32_t *win32_process = process_get_win32_process(process); + BOOL ret = FALSE; + DWORD error_code = 0; + const size_t buffer_size = buf_datalen(buffer); + + /* Windows is still writing our buffer. */ + if (win32_process->stdin_handle.busy) + return 0; + + /* Nothing for us to do right now. */ + if (buffer_size == 0) + return 0; + + /* We have reached end of file already? */ + if (BUG(win32_process->stdin_handle.reached_eof)) + return 0; + + /* Figure out how much data we should read. */ + const size_t write_size = MIN(buffer_size, + sizeof(win32_process->stdin_handle.buffer)); + + /* Read data from the process_t buffer into our intermediate buffer. */ + buf_get_bytes(buffer, win32_process->stdin_handle.buffer, write_size); + + /* Because of the slightly weird API for WriteFileEx() we must set this to 0 + * before we call WriteFileEx() because WriteFileEx() does not reset the last + * error itself when it's succesful. See comment below after the call to + * GetLastError(). */ + SetLastError(0); + + /* Schedule our write. */ + ret = WriteFileEx(win32_process->stdin_handle.pipe, + win32_process->stdin_handle.buffer, + write_size, + &win32_process->stdin_handle.overlapped, + process_win32_stdin_write_done); + + if (! ret) { + error_code = GetLastError(); + + /* No need to log at warning level for these two. */ + if (error_code == ERROR_HANDLE_EOF || error_code == ERROR_BROKEN_PIPE) { + log_debug(LD_PROCESS, "WriteFileEx() returned EOF from pipe: %s", + format_win32_error(error_code)); + } else { + log_warn(LD_PROCESS, "WriteFileEx() failed: %s", + format_win32_error(error_code)); + } + + win32_process->stdin_handle.reached_eof = true; + return 0; + } + + /* Here be dragons: According to MSDN's documentation for WriteFileEx() we + * should check GetLastError() after a call to WriteFileEx() even though the + * `ret` return value was successful. If everything is good, GetLastError() + * returns `ERROR_SUCCESS` and nothing happens. + * + * XXX(ahf): I have not managed to trigger this code while stress-testing + * this code. */ + error_code = GetLastError(); + + if (error_code != ERROR_SUCCESS) { + /* LCOV_EXCL_START */ + log_warn(LD_PROCESS, "WriteFileEx() failed after returning success: %s", + format_win32_error(error_code)); + win32_process->stdin_handle.reached_eof = true; + return 0; + /* LCOV_EXCL_STOP */ + } + + /* This cast should be safe since our buffer can maximum be BUFFER_SIZE + * large. */ + return (int)write_size; +} + +/** This function is called from the Process subsystem whenever the Windows + * backend says it has data ready. This function also ensures that we are + * starting a new background read from the standard output of the child process + * and asks Windows to call process_win32_stdout_read_done() when that + * operation is finished. Returns the number of bytes moved into <b>buffer</b>. + * */ +int +process_win32_read_stdout(struct process_t *process, buf_t *buffer) +{ + tor_assert(process); + tor_assert(buffer); + + process_win32_t *win32_process = process_get_win32_process(process); + + return process_win32_read_from_handle(&win32_process->stdout_handle, + buffer, + process_win32_stdout_read_done); +} + +/** This function is called from the Process subsystem whenever the Windows + * backend says it has data ready. This function also ensures that we are + * starting a new background read from the standard error of the child process + * and asks Windows to call process_win32_stderr_read_done() when that + * operation is finished. Returns the number of bytes moved into <b>buffer</b>. + * */ +int +process_win32_read_stderr(struct process_t *process, buf_t *buffer) +{ + tor_assert(process); + tor_assert(buffer); + + process_win32_t *win32_process = process_get_win32_process(process); + + return process_win32_read_from_handle(&win32_process->stderr_handle, + buffer, + process_win32_stderr_read_done); +} + +/** This function is responsible for moving the Tor process into what Microsoft + * calls an "alertable" state. Once the process is in an alertable state the + * Windows kernel will notify us when our background I/O requests have finished + * and the callbacks will be executed. */ +void +process_win32_trigger_completion_callbacks(void) +{ + DWORD ret; + + /* The call to SleepEx(dwMilliseconds, dwAlertable) makes the process sleep + * for dwMilliseconds and if dwAlertable is set to TRUE it will also cause + * the process to enter alertable state, where the Windows kernel will notify + * us about completed I/O requests from ReadFileEx() and WriteFileEX(), which + * will cause our completion callbacks to be executed. + * + * This function returns 0 if the time interval expired or WAIT_IO_COMPLETION + * if one or more I/O callbacks were executed. */ + ret = SleepEx(0, TRUE); + + /* Warn us if the function returned something we did not anticipate. */ + if (ret != 0 && ret != WAIT_IO_COMPLETION) { + log_warn(LD_PROCESS, "SleepEx() returned %lu", ret); + } +} + +/** Start the periodic timer which is reponsible for checking whether processes + * are still alive and to make sure that the Tor process is periodically being + * moved into an alertable state. */ +void +process_win32_timer_start(void) +{ + /* Make sure we never start our timer if it's already running. */ + if (BUG(process_win32_timer_running())) + return; + + /* Wake up once a second. */ + static const struct timeval interval = {1, 0}; + + log_info(LD_PROCESS, "Starting Windows Process I/O timer"); + periodic_timer = periodic_timer_new(tor_libevent_get_base(), + &interval, + process_win32_timer_callback, + NULL); +} + +/** Stops the periodic timer. */ +void +process_win32_timer_stop(void) +{ + if (BUG(periodic_timer == NULL)) + return; + + log_info(LD_PROCESS, "Stopping Windows Process I/O timer"); + periodic_timer_free(periodic_timer); +} + +/** Returns true iff the periodic timer is running. */ +bool +process_win32_timer_running(void) +{ + return periodic_timer != NULL; +} + +/** This function is called whenever the periodic_timer ticks. The function is + * responsible for moving the Tor process into an alertable state once a second + * and checking for whether our child processes have terminated since the last + * tick. */ +STATIC void +process_win32_timer_callback(periodic_timer_t *timer, void *data) +{ + tor_assert(timer == periodic_timer); + tor_assert(data == NULL); + + /* Move the process into an alertable state. */ + process_win32_trigger_completion_callbacks(); + + /* Check if our processes are still alive. */ + + /* Since the call to process_win32_timer_test_process() might call + * process_notify_event_exit() which again might call process_free() which + * updates the list of processes returned by process_get_all_processes() it + * is important here that we make sure to not touch the list of processes if + * the call to process_win32_timer_test_process() returns true. */ + bool done; + + do { + const smartlist_t *processes = process_get_all_processes(); + done = true; + + SMARTLIST_FOREACH_BEGIN(processes, process_t *, process) { + /* If process_win32_timer_test_process() returns true, it means that + * smartlist_remove() might have been called on the list returned by + * process_get_all_processes(). We start the loop over again until we + * have a succesful run over the entire list where the list was not + * modified. */ + if (process_win32_timer_test_process(process)) { + done = false; + break; + } + } SMARTLIST_FOREACH_END(process); + } while (! done); +} + +/** Test whether a given process is still alive. Notify the Process subsystem + * if our process have died. Returns true iff the given process have + * terminated. */ +STATIC bool +process_win32_timer_test_process(process_t *process) +{ + tor_assert(process); + + /* No need to look at processes that don't claim they are running. */ + if (process_get_status(process) != PROCESS_STATUS_RUNNING) + return false; + + process_win32_t *win32_process = process_get_win32_process(process); + BOOL ret = FALSE; + DWORD exit_code = 0; + + /* Sometimes the Windows kernel wont give us the EOF/Broken Pipe error + * message until some time after the process have actually terminated. We + * make sure that our ReadFileEx() calls for the process have *all* returned + * and both standard out and error have been marked as EOF before we try to + * see if the process terminated. + * + * This ensures that we *never* call the exit callback of the `process_t`, + * which potentially ends up calling `process_free()` on our `process_t`, + * before all data have been received from the process. + * + * We do NOT have a check here for whether standard in reached EOF since + * standard in's WriteFileEx() function is only called on-demand when we have + * something to write and is thus usually not awaiting to finish any + * operations. If we WriteFileEx() to a file that has terminated we'll simply + * get an error from ReadFileEx() or its completion routine and move on with + * life. */ + if (! win32_process->stdout_handle.reached_eof) + return false; + + if (! win32_process->stderr_handle.reached_eof) + return false; + + /* We start by testing whether our process is still running. */ + ret = GetExitCodeProcess(win32_process->process_information.hProcess, + &exit_code); + + if (! ret) { + log_warn(LD_PROCESS, "GetExitCodeProcess() failed: %s", + format_win32_error(GetLastError())); + return false; + } + + /* Notify our process_t that our process have terminated. Since our + * exit_callback might decide to process_free() our process handle it is very + * important that we do not touch the process_t after the call to + * process_notify_event_exit(). */ + if (exit_code != STILL_ACTIVE) { + process_notify_event_exit(process, exit_code); + return true; + } + + return false; +} + +/** Create a new overlapped named pipe. This function creates a new connected, + * named, pipe in <b>*read_pipe</b> and <b>*write_pipe</b> if the function is + * succesful. Returns true on sucess, false on failure. */ +STATIC bool +process_win32_create_pipe(HANDLE *read_pipe, + HANDLE *write_pipe, + SECURITY_ATTRIBUTES *attributes, + process_win32_pipe_type_t pipe_type) +{ + tor_assert(read_pipe); + tor_assert(write_pipe); + tor_assert(attributes); + + BOOL ret = FALSE; + + /* Buffer size. */ + const size_t size = 4096; + + /* Our additional read/write modes that depends on which pipe type we are + * creating. */ + DWORD read_mode = 0; + DWORD write_mode = 0; + + /* Generate the unique pipe name. */ + char pipe_name[MAX_PATH]; + static DWORD process_id = 0; + static DWORD counter = 0; + + if (process_id == 0) + process_id = GetCurrentProcessId(); + + tor_snprintf(pipe_name, sizeof(pipe_name), + "\\\\.\\Pipe\\Tor-Process-Pipe-%lu-%lu", + process_id, counter++); + + /* Only one of our handles can be overlapped. */ + switch (pipe_type) { + case PROCESS_WIN32_PIPE_TYPE_READER: + read_mode = FILE_FLAG_OVERLAPPED; + break; + case PROCESS_WIN32_PIPE_TYPE_WRITER: + write_mode = FILE_FLAG_OVERLAPPED; + break; + default: + /* LCOV_EXCL_START */ + tor_assert_nonfatal_unreached_once(); + /* LCOV_EXCL_STOP */ + } + + /* Setup our read and write handles. */ + HANDLE read_handle; + HANDLE write_handle; + + /* Create our named pipe. */ + read_handle = CreateNamedPipeA(pipe_name, + (PIPE_ACCESS_INBOUND|read_mode), + (PIPE_TYPE_BYTE|PIPE_WAIT), + 1, + size, + size, + 1000, + attributes); + + if (read_handle == INVALID_HANDLE_VALUE) { + log_warn(LD_PROCESS, "CreateNamedPipeA() failed: %s", + format_win32_error(GetLastError())); + return false; + } + + /* Create our file in the pipe namespace. */ + write_handle = CreateFileA(pipe_name, + GENERIC_WRITE, + 0, + attributes, + OPEN_EXISTING, + (FILE_ATTRIBUTE_NORMAL|write_mode), + NULL); + + if (write_handle == INVALID_HANDLE_VALUE) { + log_warn(LD_PROCESS, "CreateFileA() failed: %s", + format_win32_error(GetLastError())); + + CloseHandle(read_handle); + + return false; + } + + /* Set the inherit flag for our pipe. */ + switch (pipe_type) { + case PROCESS_WIN32_PIPE_TYPE_READER: + ret = SetHandleInformation(read_handle, HANDLE_FLAG_INHERIT, 0); + break; + case PROCESS_WIN32_PIPE_TYPE_WRITER: + ret = SetHandleInformation(write_handle, HANDLE_FLAG_INHERIT, 0); + break; + default: + /* LCOV_EXCL_START */ + tor_assert_nonfatal_unreached_once(); + /* LCOV_EXCL_STOP */ + } + + if (! ret) { + log_warn(LD_PROCESS, "SetHandleInformation() failed: %s", + format_win32_error(GetLastError())); + + CloseHandle(read_handle); + CloseHandle(write_handle); + + return false; + } + + /* Everything is good. */ + *read_pipe = read_handle; + *write_pipe = write_handle; + + return true; +} + +/** Cleanup a given <b>handle</b>. */ +STATIC void +process_win32_cleanup_handle(process_win32_handle_t *handle) +{ + tor_assert(handle); + +#if 0 + BOOL ret; + DWORD error_code; + + /* Cancel any pending I/O requests: This means that instead of getting + * ERROR_BROKEN_PIPE we get ERROR_OPERATION_ABORTED, but it doesn't seem + * like this is needed. */ + ret = CancelIo(handle->pipe); + + if (! ret) { + error_code = GetLastError(); + + /* There was no pending I/O requests for our handle. */ + if (error_code != ERROR_NOT_FOUND) { + log_warn(LD_PROCESS, "CancelIo() failed: %s", + format_win32_error(error_code)); + } + } +#endif + + /* Close our handle. */ + if (handle->pipe != INVALID_HANDLE_VALUE) { + CloseHandle(handle->pipe); + handle->pipe = INVALID_HANDLE_VALUE; + handle->reached_eof = true; + } +} + +/** This function is called when ReadFileEx() completes its background read + * from the child process's standard output. We notify the Process subsystem if + * there is data available for it to read from us. */ +STATIC VOID WINAPI +process_win32_stdout_read_done(DWORD error_code, + DWORD byte_count, + LPOVERLAPPED overlapped) +{ + tor_assert(overlapped); + tor_assert(overlapped->hEvent); + + /* Extract our process_t from the hEvent member of OVERLAPPED. */ + process_t *process = (process_t *)overlapped->hEvent; + process_win32_t *win32_process = process_get_win32_process(process); + + if (process_win32_handle_read_completion(&win32_process->stdout_handle, + error_code, + byte_count)) { + /* Schedule our next read. */ + process_notify_event_stdout(process); + } +} + +/** This function is called when ReadFileEx() completes its background read + * from the child process's standard error. We notify the Process subsystem if + * there is data available for it to read from us. */ +STATIC VOID WINAPI +process_win32_stderr_read_done(DWORD error_code, + DWORD byte_count, + LPOVERLAPPED overlapped) +{ + tor_assert(overlapped); + tor_assert(overlapped->hEvent); + + /* Extract our process_t from the hEvent member of OVERLAPPED. */ + process_t *process = (process_t *)overlapped->hEvent; + process_win32_t *win32_process = process_get_win32_process(process); + + if (process_win32_handle_read_completion(&win32_process->stderr_handle, + error_code, + byte_count)) { + /* Schedule our next read. */ + process_notify_event_stderr(process); + } +} + +/** This function is called when WriteFileEx() completes its background write + * to the child process's standard input. We notify the Process subsystem that + * it can write data to us again. */ +STATIC VOID WINAPI +process_win32_stdin_write_done(DWORD error_code, + DWORD byte_count, + LPOVERLAPPED overlapped) +{ + tor_assert(overlapped); + tor_assert(overlapped->hEvent); + + (void)byte_count; + + process_t *process = (process_t *)overlapped->hEvent; + process_win32_t *win32_process = process_get_win32_process(process); + + /* Mark our handle as not having any outstanding I/O requests. */ + win32_process->stdin_handle.busy = false; + + /* Check if we have been asked to write to the handle that have been marked + * as having reached EOF. */ + if (BUG(win32_process->stdin_handle.reached_eof)) + return; + + if (error_code == 0) { + /** Our data have been succesfully written. Clear our state and schedule + * the next write. */ + win32_process->stdin_handle.data_available = 0; + memset(win32_process->stdin_handle.buffer, 0, + sizeof(win32_process->stdin_handle.buffer)); + + /* Schedule the next write. */ + process_notify_event_stdin(process); + } else if (error_code == ERROR_HANDLE_EOF || + error_code == ERROR_BROKEN_PIPE) { + /* Our WriteFileEx() call was succesful, but we reached the end of our + * file. We mark our handle as having reached EOF and returns. */ + tor_assert(byte_count == 0); + + win32_process->stdin_handle.reached_eof = true; + } else { + /* An error happened: We warn the user and mark our handle as having + * reached EOF */ + log_warn(LD_PROCESS, + "Error in I/O completion routine from WriteFileEx(): %s", + format_win32_error(error_code)); + win32_process->stdin_handle.reached_eof = true; + } +} + +/** This function reads data from the given <b>handle</b>'s internal buffer and + * moves it into the given <b>buffer</b>. Additionally, we start the next + * ReadFileEx() background operation with the given <b>callback</b> as + * completion callback. Returns the number of bytes written to the buffer. */ +STATIC int +process_win32_read_from_handle(process_win32_handle_t *handle, + buf_t *buffer, + LPOVERLAPPED_COMPLETION_ROUTINE callback) +{ + tor_assert(handle); + tor_assert(buffer); + tor_assert(callback); + + BOOL ret = FALSE; + int bytes_available = 0; + DWORD error_code = 0; + + /* We already have a request to read data that isn't complete yet. */ + if (BUG(handle->busy)) + return 0; + + /* Check if we have been asked to read from a handle that have already told + * us that we have reached the end of the file. */ + if (BUG(handle->reached_eof)) + return 0; + + /* This cast should be safe since our buffer can be at maximum up to + * BUFFER_SIZE in size. */ + bytes_available = (int)handle->data_available; + + if (handle->data_available > 0) { + /* Read data from our intermediate buffer into the process_t buffer. */ + buf_add(buffer, handle->buffer, handle->data_available); + + /* Reset our read state. */ + handle->data_available = 0; + memset(handle->buffer, 0, sizeof(handle->buffer)); + } + + /* Because of the slightly weird API for ReadFileEx() we must set this to 0 + * before we call ReadFileEx() because ReadFileEx() does not reset the last + * error itself when it's succesful. See comment below after the call to + * GetLastError(). */ + SetLastError(0); + + /* Ask the Windows kernel to read data from our pipe into our buffer and call + * the callback function when it is done. */ + ret = ReadFileEx(handle->pipe, + handle->buffer, + sizeof(handle->buffer), + &handle->overlapped, + callback); + + if (! ret) { + error_code = GetLastError(); + + /* No need to log at warning level for these two. */ + if (error_code == ERROR_HANDLE_EOF || error_code == ERROR_BROKEN_PIPE) { + log_debug(LD_PROCESS, "ReadFileEx() returned EOF from pipe: %s", + format_win32_error(error_code)); + } else { + log_warn(LD_PROCESS, "ReadFileEx() failed: %s", + format_win32_error(error_code)); + } + + handle->reached_eof = true; + return bytes_available; + } + + /* Here be dragons: According to MSDN's documentation for ReadFileEx() we + * should check GetLastError() after a call to ReadFileEx() even though the + * `ret` return value was successful. If everything is good, GetLastError() + * returns `ERROR_SUCCESS` and nothing happens. + * + * XXX(ahf): I have not managed to trigger this code while stress-testing + * this code. */ + error_code = GetLastError(); + + if (error_code != ERROR_SUCCESS) { + /* LCOV_EXCL_START */ + log_warn(LD_PROCESS, "ReadFileEx() failed after returning success: %s", + format_win32_error(error_code)); + handle->reached_eof = true; + return bytes_available; + /* LCOV_EXCL_STOP */ + } + + /* We mark our handle as having a pending I/O request. */ + handle->busy = true; + + return bytes_available; +} + +/** This function checks the callback values from ReadFileEx() in + * <b>error_code</b> and <b>byte_count</b> if we have read data. Returns true + * iff our caller should request more data from ReadFileEx(). */ +STATIC bool +process_win32_handle_read_completion(process_win32_handle_t *handle, + DWORD error_code, + DWORD byte_count) +{ + tor_assert(handle); + + /* Mark our handle as not having any outstanding I/O requests. */ + handle->busy = false; + + if (error_code == 0) { + /* Our ReadFileEx() call was succesful and there is data for us. */ + + /* This cast should be safe since byte_count should never be larger than + * BUFFER_SIZE. */ + tor_assert(byte_count <= BUFFER_SIZE); + handle->data_available = (size_t)byte_count; + + /* Tell our caller to schedule the next read. */ + return true; + } else if (error_code == ERROR_HANDLE_EOF || + error_code == ERROR_BROKEN_PIPE) { + /* Our ReadFileEx() finished, but we reached the end of our file. We mark + * our handle as having reached EOF and returns. */ + tor_assert(byte_count == 0); + + handle->reached_eof = true; + } else { + /* An error happened: We warn the user and mark our handle as having + * reached EOF */ + log_warn(LD_PROCESS, + "Error in I/O completion routine from ReadFileEx(): %s", + format_win32_error(error_code)); + + handle->reached_eof = true; + } + + /* Our caller should NOT schedule the next read. */ + return false; +} + +/** Format a single argument for being put on a Windows command line. + * Returns a newly allocated string */ +STATIC char * +format_win_cmdline_argument(const char *arg) +{ + char *formatted_arg; + char need_quotes; + const char *c; + int i; + int bs_counter = 0; + /* Backslash we can point to when one is inserted into the string */ + const char backslash = '\\'; + + /* Smartlist of *char */ + smartlist_t *arg_chars; + arg_chars = smartlist_new(); + + /* Quote string if it contains whitespace or is empty */ + need_quotes = (strchr(arg, ' ') || strchr(arg, '\t') || '\0' == arg[0]); + + /* Build up smartlist of *chars */ + for (c=arg; *c != '\0'; c++) { + if ('"' == *c) { + /* Double up backslashes preceding a quote */ + for (i=0; i<(bs_counter*2); i++) + smartlist_add(arg_chars, (void*)&backslash); + bs_counter = 0; + /* Escape the quote */ + smartlist_add(arg_chars, (void*)&backslash); + smartlist_add(arg_chars, (void*)c); + } else if ('\\' == *c) { + /* Count backslashes until we know whether to double up */ + bs_counter++; + } else { + /* Don't double up slashes preceding a non-quote */ + for (i=0; i<bs_counter; i++) + smartlist_add(arg_chars, (void*)&backslash); + bs_counter = 0; + smartlist_add(arg_chars, (void*)c); + } + } + /* Don't double up trailing backslashes */ + for (i=0; i<bs_counter; i++) + smartlist_add(arg_chars, (void*)&backslash); + + /* Allocate space for argument, quotes (if needed), and terminator */ + const size_t formatted_arg_len = smartlist_len(arg_chars) + + (need_quotes ? 2 : 0) + 1; + formatted_arg = tor_malloc_zero(formatted_arg_len); + + /* Add leading quote */ + i=0; + if (need_quotes) + formatted_arg[i++] = '"'; + + /* Add characters */ + SMARTLIST_FOREACH(arg_chars, char*, ch, + { + formatted_arg[i++] = *ch; + }); + + /* Add trailing quote */ + if (need_quotes) + formatted_arg[i++] = '"'; + formatted_arg[i] = '\0'; + + smartlist_free(arg_chars); + return formatted_arg; +} + +/** Format a command line for use on Windows, which takes the command as a + * string rather than string array. Follows the rules from "Parsing C++ + * Command-Line Arguments" in MSDN. Algorithm based on list2cmdline in the + * Python subprocess module. Returns a newly allocated string */ +STATIC char * +tor_join_win_cmdline(const char *argv[]) +{ + smartlist_t *argv_list; + char *joined_argv; + int i; + + /* Format each argument and put the result in a smartlist */ + argv_list = smartlist_new(); + for (i=0; argv[i] != NULL; i++) { + smartlist_add(argv_list, (void *)format_win_cmdline_argument(argv[i])); + } + + /* Join the arguments with whitespace */ + joined_argv = smartlist_join_strings(argv_list, " ", 0, NULL); + + /* Free the newly allocated arguments, and the smartlist */ + SMARTLIST_FOREACH(argv_list, char *, arg, + { + tor_free(arg); + }); + smartlist_free(argv_list); + + return joined_argv; +} + +#endif /* ! defined(_WIN32). */ diff --git a/src/lib/process/process_win32.h b/src/lib/process/process_win32.h new file mode 100644 index 0000000000..d79dde157e --- /dev/null +++ b/src/lib/process/process_win32.h @@ -0,0 +1,97 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process_win32.h + * \brief Header for process_win32.c + **/ + +#ifndef TOR_PROCESS_WIN32_H +#define TOR_PROCESS_WIN32_H + +#ifdef _WIN32 + +#include "orconfig.h" +#include "lib/malloc/malloc.h" +#include "lib/evloop/compat_libevent.h" + +#include <windows.h> + +struct process_t; + +struct process_win32_t; +typedef struct process_win32_t process_win32_t; + +process_win32_t *process_win32_new(void); +void process_win32_free_(process_win32_t *win32_process); +#define process_win32_free(s) \ + FREE_AND_NULL(process_win32_t, process_win32_free_, (s)) + +void process_win32_init(void); +void process_win32_deinit(void); + +process_status_t process_win32_exec(struct process_t *process); +bool process_win32_terminate(struct process_t *process); + +process_pid_t process_win32_get_pid(struct process_t *process); + +int process_win32_write(struct process_t *process, buf_t *buffer); +int process_win32_read_stdout(struct process_t *process, buf_t *buffer); +int process_win32_read_stderr(struct process_t *process, buf_t *buffer); + +void process_win32_trigger_completion_callbacks(void); + +/* Timer handling. */ +void process_win32_timer_start(void); +void process_win32_timer_stop(void); +bool process_win32_timer_running(void); + +#ifdef PROCESS_WIN32_PRIVATE +STATIC void process_win32_timer_callback(periodic_timer_t *, void *); +STATIC bool process_win32_timer_test_process(process_t *); + +/* I/O pipe handling. */ +struct process_win32_handle_t; +typedef struct process_win32_handle_t process_win32_handle_t; + +typedef enum process_win32_pipe_type_t { + /** This pipe is used for reading. */ + PROCESS_WIN32_PIPE_TYPE_READER, + + /** This pipe is used for writing. */ + PROCESS_WIN32_PIPE_TYPE_WRITER +} process_win32_pipe_type_t; + +STATIC bool process_win32_create_pipe(HANDLE *, + HANDLE *, + SECURITY_ATTRIBUTES *, + process_win32_pipe_type_t); + +STATIC void process_win32_cleanup_handle(process_win32_handle_t *handle); + +STATIC VOID WINAPI process_win32_stdout_read_done(DWORD, + DWORD, + LPOVERLAPPED); +STATIC VOID WINAPI process_win32_stderr_read_done(DWORD, + DWORD, + LPOVERLAPPED); +STATIC VOID WINAPI process_win32_stdin_write_done(DWORD, + DWORD, + LPOVERLAPPED); + +STATIC int process_win32_read_from_handle(process_win32_handle_t *, + buf_t *, + LPOVERLAPPED_COMPLETION_ROUTINE); +STATIC bool process_win32_handle_read_completion(process_win32_handle_t *, + DWORD, + DWORD); + +STATIC char *format_win_cmdline_argument(const char *arg); +STATIC char *tor_join_win_cmdline(const char *argv[]); +#endif /* defined(PROCESS_WIN32_PRIVATE). */ + +#endif /* ! defined(_WIN32). */ + +#endif /* defined(TOR_PROCESS_WIN32_H). */ diff --git a/src/lib/process/subprocess.c b/src/lib/process/subprocess.c deleted file mode 100644 index f4429d7f76..0000000000 --- a/src/lib/process/subprocess.c +++ /dev/null @@ -1,1236 +0,0 @@ -/* Copyright (c) 2003, Roger Dingledine - * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ -/* See LICENSE for licensing information */ - -/** - * \file subprocess.c - * \brief Launch and monitor other processes. - **/ - -#define SUBPROCESS_PRIVATE -#include "lib/process/subprocess.h" - -#include "lib/container/smartlist.h" -#include "lib/err/torerr.h" -#include "lib/log/log.h" -#include "lib/log/util_bug.h" -#include "lib/log/win32err.h" -#include "lib/malloc/malloc.h" -#include "lib/process/env.h" -#include "lib/process/waitpid.h" -#include "lib/string/compat_ctype.h" - -#ifdef HAVE_SYS_TYPES_H -#include <sys/types.h> -#endif -#ifdef HAVE_SYS_PRCTL_H -#include <sys/prctl.h> -#endif -#ifdef HAVE_UNISTD_H -#include <unistd.h> -#endif -#ifdef HAVE_SIGNAL_H -#include <signal.h> -#endif -#ifdef HAVE_FCNTL_H -#include <fcntl.h> -#endif -#ifdef HAVE_SYS_WAIT_H -#include <sys/wait.h> -#endif -#include <errno.h> -#include <string.h> - -/** Format a single argument for being put on a Windows command line. - * Returns a newly allocated string */ -static char * -format_win_cmdline_argument(const char *arg) -{ - char *formatted_arg; - char need_quotes; - const char *c; - int i; - int bs_counter = 0; - /* Backslash we can point to when one is inserted into the string */ - const char backslash = '\\'; - - /* Smartlist of *char */ - smartlist_t *arg_chars; - arg_chars = smartlist_new(); - - /* Quote string if it contains whitespace or is empty */ - need_quotes = (strchr(arg, ' ') || strchr(arg, '\t') || '\0' == arg[0]); - - /* Build up smartlist of *chars */ - for (c=arg; *c != '\0'; c++) { - if ('"' == *c) { - /* Double up backslashes preceding a quote */ - for (i=0; i<(bs_counter*2); i++) - smartlist_add(arg_chars, (void*)&backslash); - bs_counter = 0; - /* Escape the quote */ - smartlist_add(arg_chars, (void*)&backslash); - smartlist_add(arg_chars, (void*)c); - } else if ('\\' == *c) { - /* Count backslashes until we know whether to double up */ - bs_counter++; - } else { - /* Don't double up slashes preceding a non-quote */ - for (i=0; i<bs_counter; i++) - smartlist_add(arg_chars, (void*)&backslash); - bs_counter = 0; - smartlist_add(arg_chars, (void*)c); - } - } - /* Don't double up trailing backslashes */ - for (i=0; i<bs_counter; i++) - smartlist_add(arg_chars, (void*)&backslash); - - /* Allocate space for argument, quotes (if needed), and terminator */ - const size_t formatted_arg_len = smartlist_len(arg_chars) + - (need_quotes ? 2 : 0) + 1; - formatted_arg = tor_malloc_zero(formatted_arg_len); - - /* Add leading quote */ - i=0; - if (need_quotes) - formatted_arg[i++] = '"'; - - /* Add characters */ - SMARTLIST_FOREACH(arg_chars, char*, ch, - { - formatted_arg[i++] = *ch; - }); - - /* Add trailing quote */ - if (need_quotes) - formatted_arg[i++] = '"'; - formatted_arg[i] = '\0'; - - smartlist_free(arg_chars); - return formatted_arg; -} - -/** Format a command line for use on Windows, which takes the command as a - * string rather than string array. Follows the rules from "Parsing C++ - * Command-Line Arguments" in MSDN. Algorithm based on list2cmdline in the - * Python subprocess module. Returns a newly allocated string */ -char * -tor_join_win_cmdline(const char *argv[]) -{ - smartlist_t *argv_list; - char *joined_argv; - int i; - - /* Format each argument and put the result in a smartlist */ - argv_list = smartlist_new(); - for (i=0; argv[i] != NULL; i++) { - smartlist_add(argv_list, (void *)format_win_cmdline_argument(argv[i])); - } - - /* Join the arguments with whitespace */ - joined_argv = smartlist_join_strings(argv_list, " ", 0, NULL); - - /* Free the newly allocated arguments, and the smartlist */ - SMARTLIST_FOREACH(argv_list, char *, arg, - { - tor_free(arg); - }); - smartlist_free(argv_list); - - return joined_argv; -} - -#ifndef _WIN32 -/** Format <b>child_state</b> and <b>saved_errno</b> as a hex string placed in - * <b>hex_errno</b>. Called between fork and _exit, so must be signal-handler - * safe. - * - * <b>hex_errno</b> must have at least HEX_ERRNO_SIZE+1 bytes available. - * - * The format of <b>hex_errno</b> is: "CHILD_STATE/ERRNO\n", left-padded - * with spaces. CHILD_STATE indicates where - * in the process of starting the child process did the failure occur (see - * CHILD_STATE_* macros for definition), and SAVED_ERRNO is the value of - * errno when the failure occurred. - * - * On success return the number of characters added to hex_errno, not counting - * the terminating NUL; return -1 on error. - */ -STATIC int -format_helper_exit_status(unsigned char child_state, int saved_errno, - char *hex_errno) -{ - unsigned int unsigned_errno; - int written, left; - char *cur; - size_t i; - int res = -1; - - /* Fill hex_errno with spaces, and a trailing newline (memset may - not be signal handler safe, so we can't use it) */ - for (i = 0; i < (HEX_ERRNO_SIZE - 1); i++) - hex_errno[i] = ' '; - hex_errno[HEX_ERRNO_SIZE - 1] = '\n'; - - /* Convert errno to be unsigned for hex conversion */ - if (saved_errno < 0) { - // Avoid overflow on the cast to unsigned int when result is INT_MIN - // by adding 1 to the signed int negative value, - // then, after it has been negated and cast to unsigned, - // adding the original 1 back (the double-addition is intentional). - // Otherwise, the cast to signed could cause a temporary int - // to equal INT_MAX + 1, which is undefined. - unsigned_errno = ((unsigned int) -(saved_errno + 1)) + 1; - } else { - unsigned_errno = (unsigned int) saved_errno; - } - - /* - * Count how many chars of space we have left, and keep a pointer into the - * current point in the buffer. - */ - left = HEX_ERRNO_SIZE+1; - cur = hex_errno; - - /* Emit child_state */ - written = format_hex_number_sigsafe(child_state, cur, left); - - if (written <= 0) - goto err; - - /* Adjust left and cur */ - left -= written; - cur += written; - if (left <= 0) - goto err; - - /* Now the '/' */ - *cur = '/'; - - /* Adjust left and cur */ - ++cur; - --left; - if (left <= 0) - goto err; - - /* Need minus? */ - if (saved_errno < 0) { - *cur = '-'; - ++cur; - --left; - if (left <= 0) - goto err; - } - - /* Emit unsigned_errno */ - written = format_hex_number_sigsafe(unsigned_errno, cur, left); - - if (written <= 0) - goto err; - - /* Adjust left and cur */ - left -= written; - cur += written; - - /* Check that we have enough space left for a newline and a NUL */ - if (left <= 1) - goto err; - - /* Emit the newline and NUL */ - *cur++ = '\n'; - *cur++ = '\0'; - - res = (int)(cur - hex_errno - 1); - - goto done; - - err: - /* - * In error exit, just write a '\0' in the first char so whatever called - * this at least won't fall off the end. - */ - *hex_errno = '\0'; - - done: - return res; -} -#endif /* !defined(_WIN32) */ - -/* Maximum number of file descriptors, if we cannot get it via sysconf() */ -#define DEFAULT_MAX_FD 256 - -/** Terminate the process of <b>process_handle</b>, if that process has not - * already exited. - * - * Return 0 if we succeeded in terminating the process (or if the process - * already exited), and -1 if we tried to kill the process but failed. - * - * Based on code originally borrowed from Python's os.kill. */ -int -tor_terminate_process(process_handle_t *process_handle) -{ -#ifdef _WIN32 - if (tor_get_exit_code(process_handle, 0, NULL) == PROCESS_EXIT_RUNNING) { - HANDLE handle = process_handle->pid.hProcess; - - if (!TerminateProcess(handle, 0)) - return -1; - else - return 0; - } -#else /* !(defined(_WIN32)) */ - if (process_handle->waitpid_cb) { - /* We haven't got a waitpid yet, so we can just kill off the process. */ - return kill(process_handle->pid, SIGTERM); - } -#endif /* defined(_WIN32) */ - - return 0; /* We didn't need to kill the process, so report success */ -} - -/** Return the Process ID of <b>process_handle</b>. */ -int -tor_process_get_pid(process_handle_t *process_handle) -{ -#ifdef _WIN32 - return (int) process_handle->pid.dwProcessId; -#else - return (int) process_handle->pid; -#endif -} - -#ifdef _WIN32 -HANDLE -tor_process_get_stdout_pipe(process_handle_t *process_handle) -{ - return process_handle->stdout_pipe; -} -#else /* !(defined(_WIN32)) */ -/* DOCDOC tor_process_get_stdout_pipe */ -int -tor_process_get_stdout_pipe(process_handle_t *process_handle) -{ - return process_handle->stdout_pipe; -} -#endif /* defined(_WIN32) */ - -/* DOCDOC process_handle_new */ -static process_handle_t * -process_handle_new(void) -{ - process_handle_t *out = tor_malloc_zero(sizeof(process_handle_t)); - -#ifdef _WIN32 - out->stdin_pipe = INVALID_HANDLE_VALUE; - out->stdout_pipe = INVALID_HANDLE_VALUE; - out->stderr_pipe = INVALID_HANDLE_VALUE; -#else - out->stdin_pipe = -1; - out->stdout_pipe = -1; - out->stderr_pipe = -1; -#endif /* defined(_WIN32) */ - - return out; -} - -#ifndef _WIN32 -/** Invoked when a process that we've launched via tor_spawn_background() has - * been found to have terminated. - */ -static void -process_handle_waitpid_cb(int status, void *arg) -{ - process_handle_t *process_handle = arg; - - process_handle->waitpid_exit_status = status; - clear_waitpid_callback(process_handle->waitpid_cb); - if (process_handle->status == PROCESS_STATUS_RUNNING) - process_handle->status = PROCESS_STATUS_NOTRUNNING; - process_handle->waitpid_cb = 0; -} -#endif /* !defined(_WIN32) */ - -/** - * @name child-process states - * - * Each of these values represents a possible state that a child process can - * be in. They're used to determine what to say when telling the parent how - * far along we were before failure. - * - * @{ - */ -#define CHILD_STATE_INIT 0 -#define CHILD_STATE_PIPE 1 -#define CHILD_STATE_MAXFD 2 -#define CHILD_STATE_FORK 3 -#define CHILD_STATE_DUPOUT 4 -#define CHILD_STATE_DUPERR 5 -#define CHILD_STATE_DUPIN 6 -#define CHILD_STATE_CLOSEFD 7 -#define CHILD_STATE_EXEC 8 -#define CHILD_STATE_FAILEXEC 9 -/** @} */ -/** - * Boolean. If true, then Tor may call execve or CreateProcess via - * tor_spawn_background. - **/ -static int may_spawn_background_process = 1; -/** - * Turn off may_spawn_background_process, so that all future calls to - * tor_spawn_background are guaranteed to fail. - **/ -void -tor_disable_spawning_background_processes(void) -{ - may_spawn_background_process = 0; -} -/** Start a program in the background. If <b>filename</b> contains a '/', then - * it will be treated as an absolute or relative path. Otherwise, on - * non-Windows systems, the system path will be searched for <b>filename</b>. - * On Windows, only the current directory will be searched. Here, to search the - * system path (as well as the application directory, current working - * directory, and system directories), set filename to NULL. - * - * The strings in <b>argv</b> will be passed as the command line arguments of - * the child program (following convention, argv[0] should normally be the - * filename of the executable, and this must be the case if <b>filename</b> is - * NULL). The last element of argv must be NULL. A handle to the child process - * will be returned in process_handle (which must be non-NULL). Read - * process_handle.status to find out if the process was successfully launched. - * For convenience, process_handle.status is returned by this function. - * - * Some parts of this code are based on the POSIX subprocess module from - * Python, and example code from - * http://msdn.microsoft.com/en-us/library/ms682499%28v=vs.85%29.aspx. - */ -int -tor_spawn_background(const char *const filename, const char **argv, - process_environment_t *env, - process_handle_t **process_handle_out) -{ - if (BUG(may_spawn_background_process == 0)) { - /* We should never reach this point if we're forbidden to spawn - * processes. Instead we should have caught the attempt earlier. */ - return PROCESS_STATUS_ERROR; - } - -#ifdef _WIN32 - HANDLE stdout_pipe_read = NULL; - HANDLE stdout_pipe_write = NULL; - HANDLE stderr_pipe_read = NULL; - HANDLE stderr_pipe_write = NULL; - HANDLE stdin_pipe_read = NULL; - HANDLE stdin_pipe_write = NULL; - process_handle_t *process_handle; - int status; - - STARTUPINFOA siStartInfo; - BOOL retval = FALSE; - - SECURITY_ATTRIBUTES saAttr; - char *joined_argv; - - saAttr.nLength = sizeof(SECURITY_ATTRIBUTES); - saAttr.bInheritHandle = TRUE; - /* TODO: should we set explicit security attributes? (#2046, comment 5) */ - saAttr.lpSecurityDescriptor = NULL; - - /* Assume failure to start process */ - status = PROCESS_STATUS_ERROR; - - /* Set up pipe for stdout */ - if (!CreatePipe(&stdout_pipe_read, &stdout_pipe_write, &saAttr, 0)) { - log_warn(LD_GENERAL, - "Failed to create pipe for stdout communication with child process: %s", - format_win32_error(GetLastError())); - return status; - } - if (!SetHandleInformation(stdout_pipe_read, HANDLE_FLAG_INHERIT, 0)) { - log_warn(LD_GENERAL, - "Failed to configure pipe for stdout communication with child " - "process: %s", format_win32_error(GetLastError())); - return status; - } - - /* Set up pipe for stderr */ - if (!CreatePipe(&stderr_pipe_read, &stderr_pipe_write, &saAttr, 0)) { - log_warn(LD_GENERAL, - "Failed to create pipe for stderr communication with child process: %s", - format_win32_error(GetLastError())); - return status; - } - if (!SetHandleInformation(stderr_pipe_read, HANDLE_FLAG_INHERIT, 0)) { - log_warn(LD_GENERAL, - "Failed to configure pipe for stderr communication with child " - "process: %s", format_win32_error(GetLastError())); - return status; - } - - /* Set up pipe for stdin */ - if (!CreatePipe(&stdin_pipe_read, &stdin_pipe_write, &saAttr, 0)) { - log_warn(LD_GENERAL, - "Failed to create pipe for stdin communication with child process: %s", - format_win32_error(GetLastError())); - return status; - } - if (!SetHandleInformation(stdin_pipe_write, HANDLE_FLAG_INHERIT, 0)) { - log_warn(LD_GENERAL, - "Failed to configure pipe for stdin communication with child " - "process: %s", format_win32_error(GetLastError())); - return status; - } - - /* Create the child process */ - - /* Windows expects argv to be a whitespace delimited string, so join argv up - */ - joined_argv = tor_join_win_cmdline(argv); - - process_handle = process_handle_new(); - process_handle->status = status; - - ZeroMemory(&(process_handle->pid), sizeof(PROCESS_INFORMATION)); - ZeroMemory(&siStartInfo, sizeof(STARTUPINFO)); - siStartInfo.cb = sizeof(STARTUPINFO); - siStartInfo.hStdError = stderr_pipe_write; - siStartInfo.hStdOutput = stdout_pipe_write; - siStartInfo.hStdInput = stdin_pipe_read; - siStartInfo.dwFlags |= STARTF_USESTDHANDLES; - - /* Create the child process */ - - retval = CreateProcessA(filename, // module name - joined_argv, // command line - /* TODO: should we set explicit security attributes? (#2046, comment 5) */ - NULL, // process security attributes - NULL, // primary thread security attributes - TRUE, // handles are inherited - /*(TODO: set CREATE_NEW CONSOLE/PROCESS_GROUP to make GetExitCodeProcess() - * work?) */ - CREATE_NO_WINDOW, // creation flags - (env==NULL) ? NULL : env->windows_environment_block, - NULL, // use parent's current directory - &siStartInfo, // STARTUPINFO pointer - &(process_handle->pid)); // receives PROCESS_INFORMATION - - tor_free(joined_argv); - - if (!retval) { - log_warn(LD_GENERAL, - "Failed to create child process %s: %s", filename?filename:argv[0], - format_win32_error(GetLastError())); - tor_free(process_handle); - } else { - /* TODO: Close hProcess and hThread in process_handle->pid? */ - process_handle->stdout_pipe = stdout_pipe_read; - process_handle->stderr_pipe = stderr_pipe_read; - process_handle->stdin_pipe = stdin_pipe_write; - status = process_handle->status = PROCESS_STATUS_RUNNING; - } - - /* TODO: Close pipes on exit */ - *process_handle_out = process_handle; - return status; -#else /* !(defined(_WIN32)) */ - pid_t pid; - int stdout_pipe[2]; - int stderr_pipe[2]; - int stdin_pipe[2]; - int fd, retval; - process_handle_t *process_handle; - int status; - - const char *error_message = SPAWN_ERROR_MESSAGE; - size_t error_message_length; - - /* Represents where in the process of spawning the program is; - this is used for printing out the error message */ - unsigned char child_state = CHILD_STATE_INIT; - - char hex_errno[HEX_ERRNO_SIZE + 2]; /* + 1 should be sufficient actually */ - - static int max_fd = -1; - - status = PROCESS_STATUS_ERROR; - - /* We do the strlen here because strlen() is not signal handler safe, - and we are not allowed to use unsafe functions between fork and exec */ - error_message_length = strlen(error_message); - - // child_state = CHILD_STATE_PIPE; - - /* Set up pipe for redirecting stdout, stderr, and stdin of child */ - retval = pipe(stdout_pipe); - if (-1 == retval) { - log_warn(LD_GENERAL, - "Failed to set up pipe for stdout communication with child process: %s", - strerror(errno)); - return status; - } - - retval = pipe(stderr_pipe); - if (-1 == retval) { - log_warn(LD_GENERAL, - "Failed to set up pipe for stderr communication with child process: %s", - strerror(errno)); - - close(stdout_pipe[0]); - close(stdout_pipe[1]); - - return status; - } - - retval = pipe(stdin_pipe); - if (-1 == retval) { - log_warn(LD_GENERAL, - "Failed to set up pipe for stdin communication with child process: %s", - strerror(errno)); - - close(stdout_pipe[0]); - close(stdout_pipe[1]); - close(stderr_pipe[0]); - close(stderr_pipe[1]); - - return status; - } - - // child_state = CHILD_STATE_MAXFD; - -#ifdef _SC_OPEN_MAX - if (-1 == max_fd) { - max_fd = (int) sysconf(_SC_OPEN_MAX); - if (max_fd == -1) { - max_fd = DEFAULT_MAX_FD; - log_warn(LD_GENERAL, - "Cannot find maximum file descriptor, assuming %d", max_fd); - } - } -#else /* !(defined(_SC_OPEN_MAX)) */ - max_fd = DEFAULT_MAX_FD; -#endif /* defined(_SC_OPEN_MAX) */ - - // child_state = CHILD_STATE_FORK; - - pid = fork(); - if (0 == pid) { - /* In child */ - -#if defined(HAVE_SYS_PRCTL_H) && defined(__linux__) - /* Attempt to have the kernel issue a SIGTERM if the parent - * goes away. Certain attributes of the binary being execve()ed - * will clear this during the execve() call, but it's better - * than nothing. - */ - prctl(PR_SET_PDEATHSIG, SIGTERM); -#endif /* defined(HAVE_SYS_PRCTL_H) && defined(__linux__) */ - - child_state = CHILD_STATE_DUPOUT; - - /* Link child stdout to the write end of the pipe */ - retval = dup2(stdout_pipe[1], STDOUT_FILENO); - if (-1 == retval) - goto error; - - child_state = CHILD_STATE_DUPERR; - - /* Link child stderr to the write end of the pipe */ - retval = dup2(stderr_pipe[1], STDERR_FILENO); - if (-1 == retval) - goto error; - - child_state = CHILD_STATE_DUPIN; - - /* Link child stdin to the read end of the pipe */ - retval = dup2(stdin_pipe[0], STDIN_FILENO); - if (-1 == retval) - goto error; - - // child_state = CHILD_STATE_CLOSEFD; - - close(stderr_pipe[0]); - close(stderr_pipe[1]); - close(stdout_pipe[0]); - close(stdout_pipe[1]); - close(stdin_pipe[0]); - close(stdin_pipe[1]); - - /* Close all other fds, including the read end of the pipe */ - /* XXX: We should now be doing enough FD_CLOEXEC setting to make - * this needless. */ - for (fd = STDERR_FILENO + 1; fd < max_fd; fd++) { - close(fd); - } - - // child_state = CHILD_STATE_EXEC; - - /* Call the requested program. We need the cast because - execvp doesn't define argv as const, even though it - does not modify the arguments */ - if (env) - execve(filename, (char *const *) argv, env->unixoid_environment_block); - else { - static char *new_env[] = { NULL }; - execve(filename, (char *const *) argv, new_env); - } - - /* If we got here, the exec or open(/dev/null) failed */ - - child_state = CHILD_STATE_FAILEXEC; - - error: - { - /* XXX: are we leaking fds from the pipe? */ - int n, err=0; - ssize_t nbytes; - - n = format_helper_exit_status(child_state, errno, hex_errno); - - if (n >= 0) { - /* Write the error message. GCC requires that we check the return - value, but there is nothing we can do if it fails */ - /* TODO: Don't use STDOUT, use a pipe set up just for this purpose */ - nbytes = write(STDOUT_FILENO, error_message, error_message_length); - err = (nbytes < 0); - nbytes = write(STDOUT_FILENO, hex_errno, n); - err += (nbytes < 0); - } - - _exit(err?254:255); // exit ok: in child. - } - - /* Never reached, but avoids compiler warning */ - return status; // LCOV_EXCL_LINE - } - - /* In parent */ - - if (-1 == pid) { - log_warn(LD_GENERAL, "Failed to fork child process: %s", strerror(errno)); - close(stdin_pipe[0]); - close(stdin_pipe[1]); - close(stdout_pipe[0]); - close(stdout_pipe[1]); - close(stderr_pipe[0]); - close(stderr_pipe[1]); - return status; - } - - process_handle = process_handle_new(); - process_handle->status = status; - process_handle->pid = pid; - - /* TODO: If the child process forked but failed to exec, waitpid it */ - - /* Return read end of the pipes to caller, and close write end */ - process_handle->stdout_pipe = stdout_pipe[0]; - retval = close(stdout_pipe[1]); - - if (-1 == retval) { - log_warn(LD_GENERAL, - "Failed to close write end of stdout pipe in parent process: %s", - strerror(errno)); - } - - process_handle->waitpid_cb = set_waitpid_callback(pid, - process_handle_waitpid_cb, - process_handle); - - process_handle->stderr_pipe = stderr_pipe[0]; - retval = close(stderr_pipe[1]); - - if (-1 == retval) { - log_warn(LD_GENERAL, - "Failed to close write end of stderr pipe in parent process: %s", - strerror(errno)); - } - - /* Return write end of the stdin pipe to caller, and close the read end */ - process_handle->stdin_pipe = stdin_pipe[1]; - retval = close(stdin_pipe[0]); - - if (-1 == retval) { - log_warn(LD_GENERAL, - "Failed to close read end of stdin pipe in parent process: %s", - strerror(errno)); - } - - status = process_handle->status = PROCESS_STATUS_RUNNING; - /* Set stdin/stdout/stderr pipes to be non-blocking */ - if (fcntl(process_handle->stdout_pipe, F_SETFL, O_NONBLOCK) < 0 || - fcntl(process_handle->stderr_pipe, F_SETFL, O_NONBLOCK) < 0 || - fcntl(process_handle->stdin_pipe, F_SETFL, O_NONBLOCK) < 0) { - log_warn(LD_GENERAL, "Failed to set stderror/stdout/stdin pipes " - "nonblocking in parent process: %s", strerror(errno)); - } - - *process_handle_out = process_handle; - return status; -#endif /* defined(_WIN32) */ -} - -/** Destroy all resources allocated by the process handle in - * <b>process_handle</b>. - * If <b>also_terminate_process</b> is true, also terminate the - * process of the process handle. */ -MOCK_IMPL(void, -tor_process_handle_destroy,(process_handle_t *process_handle, - int also_terminate_process)) -{ - if (!process_handle) - return; - - if (also_terminate_process) { - if (tor_terminate_process(process_handle) < 0) { - const char *errstr = -#ifdef _WIN32 - format_win32_error(GetLastError()); -#else - strerror(errno); -#endif - log_notice(LD_GENERAL, "Failed to terminate process with " - "PID '%d' ('%s').", tor_process_get_pid(process_handle), - errstr); - } else { - log_info(LD_GENERAL, "Terminated process with PID '%d'.", - tor_process_get_pid(process_handle)); - } - } - - process_handle->status = PROCESS_STATUS_NOTRUNNING; - -#ifdef _WIN32 - if (process_handle->stdout_pipe) - CloseHandle(process_handle->stdout_pipe); - - if (process_handle->stderr_pipe) - CloseHandle(process_handle->stderr_pipe); - - if (process_handle->stdin_pipe) - CloseHandle(process_handle->stdin_pipe); -#else /* !(defined(_WIN32)) */ - close(process_handle->stdout_pipe); - close(process_handle->stderr_pipe); - close(process_handle->stdin_pipe); - - clear_waitpid_callback(process_handle->waitpid_cb); -#endif /* defined(_WIN32) */ - - memset(process_handle, 0x0f, sizeof(process_handle_t)); - tor_free(process_handle); -} - -/** Get the exit code of a process specified by <b>process_handle</b> and store - * it in <b>exit_code</b>, if set to a non-NULL value. If <b>block</b> is set - * to true, the call will block until the process has exited. Otherwise if - * the process is still running, the function will return - * PROCESS_EXIT_RUNNING, and exit_code will be left unchanged. Returns - * PROCESS_EXIT_EXITED if the process did exit. If there is a failure, - * PROCESS_EXIT_ERROR will be returned and the contents of exit_code (if - * non-NULL) will be undefined. N.B. Under *nix operating systems, this will - * probably not work in Tor, because waitpid() is called in main.c to reap any - * terminated child processes.*/ -int -tor_get_exit_code(process_handle_t *process_handle, - int block, int *exit_code) -{ -#ifdef _WIN32 - DWORD retval; - BOOL success; - - if (block) { - /* Wait for the process to exit */ - retval = WaitForSingleObject(process_handle->pid.hProcess, INFINITE); - if (retval != WAIT_OBJECT_0) { - log_warn(LD_GENERAL, "WaitForSingleObject() failed (%d): %s", - (int)retval, format_win32_error(GetLastError())); - return PROCESS_EXIT_ERROR; - } - } else { - retval = WaitForSingleObject(process_handle->pid.hProcess, 0); - if (WAIT_TIMEOUT == retval) { - /* Process has not exited */ - return PROCESS_EXIT_RUNNING; - } else if (retval != WAIT_OBJECT_0) { - log_warn(LD_GENERAL, "WaitForSingleObject() failed (%d): %s", - (int)retval, format_win32_error(GetLastError())); - return PROCESS_EXIT_ERROR; - } - } - - if (exit_code != NULL) { - success = GetExitCodeProcess(process_handle->pid.hProcess, - (PDWORD)exit_code); - if (!success) { - log_warn(LD_GENERAL, "GetExitCodeProcess() failed: %s", - format_win32_error(GetLastError())); - return PROCESS_EXIT_ERROR; - } - } -#else /* !(defined(_WIN32)) */ - int stat_loc; - int retval; - - if (process_handle->waitpid_cb) { - /* We haven't processed a SIGCHLD yet. */ - retval = waitpid(process_handle->pid, &stat_loc, block?0:WNOHANG); - if (retval == process_handle->pid) { - clear_waitpid_callback(process_handle->waitpid_cb); - process_handle->waitpid_cb = NULL; - process_handle->waitpid_exit_status = stat_loc; - } - } else { - /* We already got a SIGCHLD for this process, and handled it. */ - retval = process_handle->pid; - stat_loc = process_handle->waitpid_exit_status; - } - - if (!block && 0 == retval) { - /* Process has not exited */ - return PROCESS_EXIT_RUNNING; - } else if (retval != process_handle->pid) { - log_warn(LD_GENERAL, "waitpid() failed for PID %d: %s", - (int)process_handle->pid, strerror(errno)); - return PROCESS_EXIT_ERROR; - } - - if (!WIFEXITED(stat_loc)) { - log_warn(LD_GENERAL, "Process %d did not exit normally", - (int)process_handle->pid); - return PROCESS_EXIT_ERROR; - } - - if (exit_code != NULL) - *exit_code = WEXITSTATUS(stat_loc); -#endif /* defined(_WIN32) */ - - return PROCESS_EXIT_EXITED; -} - -#ifdef _WIN32 -/** Read from a handle <b>h</b> into <b>buf</b>, up to <b>count</b> bytes. If - * <b>hProcess</b> is NULL, the function will return immediately if there is - * nothing more to read. Otherwise <b>hProcess</b> should be set to the handle - * to the process owning the <b>h</b>. In this case, the function will exit - * only once the process has exited, or <b>count</b> bytes are read. Returns - * the number of bytes read, or -1 on error. */ -ssize_t -tor_read_all_handle(HANDLE h, char *buf, size_t count, - const process_handle_t *process) -{ - size_t numread = 0; - BOOL retval; - DWORD byte_count; - BOOL process_exited = FALSE; - - if (count > SIZE_T_CEILING || count > SSIZE_MAX) - return -1; - - while (numread < count) { - /* Check if there is anything to read */ - retval = PeekNamedPipe(h, NULL, 0, NULL, &byte_count, NULL); - if (!retval) { - log_warn(LD_GENERAL, - "Failed to peek from handle: %s", - format_win32_error(GetLastError())); - return -1; - } else if (0 == byte_count) { - /* Nothing available: process exited or it is busy */ - - /* Exit if we don't know whether the process is running */ - if (NULL == process) - break; - - /* The process exited and there's nothing left to read from it */ - if (process_exited) - break; - - /* If process is not running, check for output one more time in case - it wrote something after the peek was performed. Otherwise keep on - waiting for output */ - tor_assert(process != NULL); - byte_count = WaitForSingleObject(process->pid.hProcess, 0); - if (WAIT_TIMEOUT != byte_count) - process_exited = TRUE; - - continue; - } - - /* There is data to read; read it */ - retval = ReadFile(h, buf+numread, count-numread, &byte_count, NULL); - tor_assert(byte_count + numread <= count); - if (!retval) { - log_warn(LD_GENERAL, "Failed to read from handle: %s", - format_win32_error(GetLastError())); - return -1; - } else if (0 == byte_count) { - /* End of file */ - break; - } - numread += byte_count; - } - return (ssize_t)numread; -} -#else /* !(defined(_WIN32)) */ -/** Read from a handle <b>fd</b> into <b>buf</b>, up to <b>count</b> bytes. If - * <b>process</b> is NULL, the function will return immediately if there is - * nothing more to read. Otherwise data will be read until end of file, or - * <b>count</b> bytes are read. Returns the number of bytes read, or -1 on - * error. Sets <b>eof</b> to true if <b>eof</b> is not NULL and the end of the - * file has been reached. */ -ssize_t -tor_read_all_handle(int fd, char *buf, size_t count, - const process_handle_t *process, - int *eof) -{ - size_t numread = 0; - ssize_t result; - - if (eof) - *eof = 0; - - if (count > SIZE_T_CEILING || count > SSIZE_MAX) - return -1; - - while (numread < count) { - result = read(fd, buf+numread, count-numread); - - if (result == 0) { - log_debug(LD_GENERAL, "read() reached end of file"); - if (eof) - *eof = 1; - break; - } else if (result < 0 && errno == EAGAIN) { - if (process) - continue; - else - break; - } else if (result < 0) { - log_warn(LD_GENERAL, "read() failed: %s", strerror(errno)); - return -1; - } - - numread += result; - } - - log_debug(LD_GENERAL, "read() read %d bytes from handle", (int)numread); - return (ssize_t)numread; -} -#endif /* defined(_WIN32) */ - -/** Read from stdout of a process until the process exits. */ -ssize_t -tor_read_all_from_process_stdout(const process_handle_t *process_handle, - char *buf, size_t count) -{ -#ifdef _WIN32 - return tor_read_all_handle(process_handle->stdout_pipe, buf, count, - process_handle); -#else - return tor_read_all_handle(process_handle->stdout_pipe, buf, count, - process_handle, NULL); -#endif /* defined(_WIN32) */ -} - -/** Read from stdout of a process until the process exits. */ -ssize_t -tor_read_all_from_process_stderr(const process_handle_t *process_handle, - char *buf, size_t count) -{ -#ifdef _WIN32 - return tor_read_all_handle(process_handle->stderr_pipe, buf, count, - process_handle); -#else - return tor_read_all_handle(process_handle->stderr_pipe, buf, count, - process_handle, NULL); -#endif /* defined(_WIN32) */ -} - -/** Return a string corresponding to <b>stream_status</b>. */ -const char * -stream_status_to_string(enum stream_status stream_status) -{ - switch (stream_status) { - case IO_STREAM_OKAY: - return "okay"; - case IO_STREAM_EAGAIN: - return "temporarily unavailable"; - case IO_STREAM_TERM: - return "terminated"; - case IO_STREAM_CLOSED: - return "closed"; - default: - tor_fragile_assert(); - return "unknown"; - } -} - -/** Split buf into lines, and add to smartlist. The buffer <b>buf</b> will be - * modified. The resulting smartlist will consist of pointers to buf, so there - * is no need to free the contents of sl. <b>buf</b> must be a NUL-terminated - * string. <b>len</b> should be set to the length of the buffer excluding the - * NUL. Non-printable characters (including NUL) will be replaced with "." */ -int -tor_split_lines(smartlist_t *sl, char *buf, int len) -{ - /* Index in buf of the start of the current line */ - int start = 0; - /* Index in buf of the current character being processed */ - int cur = 0; - /* Are we currently in a line */ - char in_line = 0; - - /* Loop over string */ - while (cur < len) { - /* Loop until end of line or end of string */ - for (; cur < len; cur++) { - if (in_line) { - if ('\r' == buf[cur] || '\n' == buf[cur]) { - /* End of line */ - buf[cur] = '\0'; - /* Point cur to the next line */ - cur++; - /* Line starts at start and ends with a nul */ - break; - } else { - if (!TOR_ISPRINT(buf[cur])) - buf[cur] = '.'; - } - } else { - if ('\r' == buf[cur] || '\n' == buf[cur]) { - /* Skip leading vertical space */ - ; - } else { - in_line = 1; - start = cur; - if (!TOR_ISPRINT(buf[cur])) - buf[cur] = '.'; - } - } - } - /* We are at the end of the line or end of string. If in_line is true there - * is a line which starts at buf+start and ends at a NUL. cur points to - * the character after the NUL. */ - if (in_line) - smartlist_add(sl, (void *)(buf+start)); - in_line = 0; - } - return smartlist_len(sl); -} - -#ifdef _WIN32 - -/** Return a smartlist containing lines outputted from - * <b>handle</b>. Return NULL on error, and set - * <b>stream_status_out</b> appropriately. */ -MOCK_IMPL(smartlist_t *, -tor_get_lines_from_handle, (HANDLE *handle, - enum stream_status *stream_status_out)) -{ - int pos; - char stdout_buf[600] = {0}; - smartlist_t *lines = NULL; - - tor_assert(stream_status_out); - - *stream_status_out = IO_STREAM_TERM; - - pos = tor_read_all_handle(handle, stdout_buf, sizeof(stdout_buf) - 1, NULL); - if (pos < 0) { - *stream_status_out = IO_STREAM_TERM; - return NULL; - } - if (pos == 0) { - *stream_status_out = IO_STREAM_EAGAIN; - return NULL; - } - - /* End with a null even if there isn't a \r\n at the end */ - /* TODO: What if this is a partial line? */ - stdout_buf[pos] = '\0'; - - /* Split up the buffer */ - lines = smartlist_new(); - tor_split_lines(lines, stdout_buf, pos); - - /* Currently 'lines' is populated with strings residing on the - stack. Replace them with their exact copies on the heap: */ - SMARTLIST_FOREACH(lines, char *, line, - SMARTLIST_REPLACE_CURRENT(lines, line, tor_strdup(line))); - - *stream_status_out = IO_STREAM_OKAY; - - return lines; -} - -#else /* !(defined(_WIN32)) */ - -/** Return a smartlist containing lines outputted from - * <b>fd</b>. Return NULL on error, and set - * <b>stream_status_out</b> appropriately. */ -MOCK_IMPL(smartlist_t *, -tor_get_lines_from_handle, (int fd, enum stream_status *stream_status_out)) -{ - enum stream_status stream_status; - char stdout_buf[400]; - smartlist_t *lines = NULL; - - while (1) { - memset(stdout_buf, 0, sizeof(stdout_buf)); - - stream_status = get_string_from_pipe(fd, - stdout_buf, sizeof(stdout_buf) - 1); - if (stream_status != IO_STREAM_OKAY) - goto done; - - if (!lines) lines = smartlist_new(); - smartlist_split_string(lines, stdout_buf, "\n", 0, 0); - } - - done: - *stream_status_out = stream_status; - return lines; -} - -#endif /* defined(_WIN32) */ - -/** Reads from <b>fd</b> and stores input in <b>buf_out</b> making - * sure it's below <b>count</b> bytes. - * If the string has a trailing newline, we strip it off. - * - * This function is specifically created to handle input from managed - * proxies, according to the pluggable transports spec. Make sure it - * fits your needs before using it. - * - * Returns: - * IO_STREAM_CLOSED: If the stream is closed. - * IO_STREAM_EAGAIN: If there is nothing to read and we should check back - * later. - * IO_STREAM_TERM: If something is wrong with the stream. - * IO_STREAM_OKAY: If everything went okay and we got a string - * in <b>buf_out</b>. */ -enum stream_status -get_string_from_pipe(int fd, char *buf_out, size_t count) -{ - ssize_t ret; - - tor_assert(count <= INT_MAX); - - ret = read(fd, buf_out, count); - - if (ret == 0) - return IO_STREAM_CLOSED; - else if (ret < 0 && errno == EAGAIN) - return IO_STREAM_EAGAIN; - else if (ret < 0) - return IO_STREAM_TERM; - - if (buf_out[ret - 1] == '\n') { - /* Remove the trailing newline */ - buf_out[ret - 1] = '\0'; - } else - buf_out[ret] = '\0'; - - return IO_STREAM_OKAY; -} diff --git a/src/lib/process/subprocess.h b/src/lib/process/subprocess.h deleted file mode 100644 index aa3127d62d..0000000000 --- a/src/lib/process/subprocess.h +++ /dev/null @@ -1,134 +0,0 @@ -/* Copyright (c) 2003-2004, Roger Dingledine - * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ -/* See LICENSE for licensing information */ - -/** - * \file subprocess.h - * \brief Header for subprocess.c - **/ - -#ifndef TOR_SUBPROCESS_H -#define TOR_SUBPROCESS_H - -#include "lib/cc/torint.h" -#include "lib/testsupport/testsupport.h" -#include <stddef.h> -#ifdef _WIN32 -#include <windows.h> -#endif - -struct smartlist_t; - -void tor_disable_spawning_background_processes(void); - -typedef struct process_handle_t process_handle_t; -struct process_environment_t; -int tor_spawn_background(const char *const filename, const char **argv, - struct process_environment_t *env, - process_handle_t **process_handle_out); - -#define SPAWN_ERROR_MESSAGE "ERR: Failed to spawn background process - code " - -/** Status of an I/O stream. */ -enum stream_status { - IO_STREAM_OKAY, - IO_STREAM_EAGAIN, - IO_STREAM_TERM, - IO_STREAM_CLOSED -}; - -const char *stream_status_to_string(enum stream_status stream_status); - -enum stream_status get_string_from_pipe(int fd, char *buf, size_t count); - -/* Values of process_handle_t.status. */ -#define PROCESS_STATUS_NOTRUNNING 0 -#define PROCESS_STATUS_RUNNING 1 -#define PROCESS_STATUS_ERROR -1 - -#ifdef SUBPROCESS_PRIVATE -struct waitpid_callback_t; - -/** Structure to represent the state of a process with which Tor is - * communicating. The contents of this structure are private to util.c */ -struct process_handle_t { - /** One of the PROCESS_STATUS_* values */ - int status; -#ifdef _WIN32 - HANDLE stdin_pipe; - HANDLE stdout_pipe; - HANDLE stderr_pipe; - PROCESS_INFORMATION pid; -#else /* !(defined(_WIN32)) */ - int stdin_pipe; - int stdout_pipe; - int stderr_pipe; - pid_t pid; - /** If the process has not given us a SIGCHLD yet, this has the - * waitpid_callback_t that gets invoked once it has. Otherwise this - * contains NULL. */ - struct waitpid_callback_t *waitpid_cb; - /** The exit status reported by waitpid. */ - int waitpid_exit_status; -#endif /* defined(_WIN32) */ -}; -#endif /* defined(SUBPROCESS_PRIVATE) */ - -/* Return values of tor_get_exit_code() */ -#define PROCESS_EXIT_RUNNING 1 -#define PROCESS_EXIT_EXITED 0 -#define PROCESS_EXIT_ERROR -1 -int tor_get_exit_code(process_handle_t *process_handle, - int block, int *exit_code); -int tor_split_lines(struct smartlist_t *sl, char *buf, int len); -#ifdef _WIN32 -ssize_t tor_read_all_handle(HANDLE h, char *buf, size_t count, - const process_handle_t *process); -#else -ssize_t tor_read_all_handle(int fd, char *buf, size_t count, - const process_handle_t *process, - int *eof); -#endif /* defined(_WIN32) */ -ssize_t tor_read_all_from_process_stdout( - const process_handle_t *process_handle, char *buf, size_t count); -ssize_t tor_read_all_from_process_stderr( - const process_handle_t *process_handle, char *buf, size_t count); -char *tor_join_win_cmdline(const char *argv[]); - -int tor_process_get_pid(process_handle_t *process_handle); -#ifdef _WIN32 -HANDLE tor_process_get_stdout_pipe(process_handle_t *process_handle); -#else -int tor_process_get_stdout_pipe(process_handle_t *process_handle); -#endif - -#ifdef _WIN32 -MOCK_DECL(struct smartlist_t *, tor_get_lines_from_handle,(HANDLE *handle, - enum stream_status *stream_status)); -#else -MOCK_DECL(struct smartlist_t *, tor_get_lines_from_handle,(int fd, - enum stream_status *stream_status)); -#endif /* defined(_WIN32) */ - -int tor_terminate_process(process_handle_t *process_handle); - -MOCK_DECL(void, tor_process_handle_destroy,(process_handle_t *process_handle, - int also_terminate_process)); - -#ifdef SUBPROCESS_PRIVATE -/* Prototypes for private functions only used by util.c (and unit tests) */ - -#ifndef _WIN32 -STATIC int format_helper_exit_status(unsigned char child_state, - int saved_errno, char *hex_errno); - -/* Space for hex values of child state, a slash, saved_errno (with - leading minus) and newline (no null) */ -#define HEX_ERRNO_SIZE (sizeof(char) * 2 + 1 + \ - 1 + sizeof(int) * 2 + 1) -#endif /* !defined(_WIN32) */ - -#endif /* defined(SUBPROCESS_PRIVATE) */ - -#endif diff --git a/src/lib/process/waitpid.c b/src/lib/process/waitpid.c index 9b626394d2..2b38481aeb 100644 --- a/src/lib/process/waitpid.c +++ b/src/lib/process/waitpid.c @@ -16,7 +16,7 @@ #include "lib/log/log.h" #include "lib/log/util_bug.h" #include "lib/malloc/malloc.h" -#include "ht.h" +#include "ext/ht.h" #ifdef HAVE_SYS_WAIT_H #include <sys/wait.h> diff --git a/src/lib/process/winprocess_sys.c b/src/lib/process/winprocess_sys.c new file mode 100644 index 0000000000..1266babca8 --- /dev/null +++ b/src/lib/process/winprocess_sys.c @@ -0,0 +1,64 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file winprocess_sys.c + * \brief Subsystem object for windows process setup. + **/ + +#include "orconfig.h" +#include "lib/subsys/subsys.h" +#include "lib/process/winprocess_sys.h" + +#include <stdbool.h> +#include <stddef.h> + +#ifdef _WIN32 +#include <windows.h> + +#define WINPROCESS_SYS_ENABLED true + +static int +subsys_winprocess_initialize(void) +{ +#ifndef HeapEnableTerminationOnCorruption +#define HeapEnableTerminationOnCorruption 1 +#endif + + /* On heap corruption, just give up; don't try to play along. */ + HeapSetInformation(NULL, HeapEnableTerminationOnCorruption, NULL, 0); + + /* SetProcessDEPPolicy is only supported on 32-bit Windows. + * (On 64-bit Windows it always fails, and some compilers don't like the + * PSETDEP cast.) + * 32-bit Windows defines _WIN32. + * 64-bit Windows defines _WIN32 and _WIN64. */ +#ifndef _WIN64 + /* Call SetProcessDEPPolicy to permanently enable DEP. + The function will not resolve on earlier versions of Windows, + and failure is not dangerous. */ + HMODULE hMod = GetModuleHandleA("Kernel32.dll"); + if (hMod) { + typedef BOOL (WINAPI *PSETDEP)(DWORD); + PSETDEP setdeppolicy = (PSETDEP)GetProcAddress(hMod, + "SetProcessDEPPolicy"); + if (setdeppolicy) { + /* PROCESS_DEP_ENABLE | PROCESS_DEP_DISABLE_ATL_THUNK_EMULATION */ + setdeppolicy(3); + } + } +#endif /* !defined(_WIN64) */ + + return 0; +} +#else /* !defined(_WIN32) */ +#define WINPROCESS_SYS_ENABLED false +#define subsys_winprocess_initialize NULL +#endif /* defined(_WIN32) */ + +const subsys_fns_t sys_winprocess = { + .name = "winprocess", + .level = -100, + .supported = WINPROCESS_SYS_ENABLED, + .initialize = subsys_winprocess_initialize, +}; diff --git a/src/lib/process/winprocess_sys.h b/src/lib/process/winprocess_sys.h new file mode 100644 index 0000000000..7ab2aa04a6 --- /dev/null +++ b/src/lib/process/winprocess_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file winprocess_sys.h + * \brief Declare subsystem object for winprocess.c + **/ + +#ifndef TOR_WINPROCESS_SYS_H +#define TOR_WINPROCESS_SYS_H + +extern const struct subsys_fns_t sys_winprocess; + +#endif /* !defined(TOR_WINPROCESS_SYS_H) */ diff --git a/src/lib/sandbox/.may_include b/src/lib/sandbox/.may_include index 84906dfb3d..853dae7880 100644 --- a/src/lib/sandbox/.may_include +++ b/src/lib/sandbox/.may_include @@ -5,11 +5,10 @@ lib/container/*.h lib/err/*.h lib/log/*.h lib/malloc/*.h -lib/net/*.h lib/sandbox/*.h lib/sandbox/*.inc lib/string/*.h -ht.h -siphash.h -tor_queue.h +ext/ht.h +ext/siphash.h +ext/tor_queue.h diff --git a/src/lib/sandbox/sandbox.c b/src/lib/sandbox/sandbox.c index 8f577b0660..dfb41a0965 100644 --- a/src/lib/sandbox/sandbox.c +++ b/src/lib/sandbox/sandbox.c @@ -38,13 +38,12 @@ #include "lib/err/torerr.h" #include "lib/log/log.h" #include "lib/cc/torint.h" -#include "lib/net/resolve.h" #include "lib/malloc/malloc.h" #include "lib/string/scanf.h" -#include "tor_queue.h" -#include "ht.h" -#include "siphash.h" +#include "ext/tor_queue.h" +#include "ext/ht.h" +#include "ext/siphash.h" #define DEBUGGING_CLOSE @@ -798,6 +797,12 @@ sb_getsockopt(scmp_filter_ctx ctx, sandbox_cfg_t *filter) if (rc) return rc; + rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(getsockopt), + SCMP_CMP(1, SCMP_CMP_EQ, SOL_SOCKET), + SCMP_CMP(2, SCMP_CMP_EQ, SO_ACCEPTCONN)); + if (rc) + return rc; + #ifdef HAVE_SYSTEMD rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(getsockopt), SCMP_CMP(1, SCMP_CMP_EQ, SOL_SOCKET), @@ -1517,7 +1522,6 @@ install_syscall_filter(sandbox_cfg_t* cfg) // marking the sandbox as active sandbox_active = 1; - tor_make_getaddrinfo_cache_active(); end: seccomp_release(ctx); @@ -1764,9 +1768,4 @@ sandbox_is_active(void) return 0; } -void -sandbox_disable_getaddrinfo_cache(void) -{ -} - #endif /* !defined(USE_LIBSECCOMP) */ diff --git a/src/lib/smartlist_core/.may_include b/src/lib/smartlist_core/.may_include index a8507761a4..2f0c8d341e 100644 --- a/src/lib/smartlist_core/.may_include +++ b/src/lib/smartlist_core/.may_include @@ -4,4 +4,4 @@ lib/malloc/*.h lib/err/*.h lib/string/*.h lib/smartlist_core/*.h -lib/testsupport/testsupport.h +lib/testsupport/*.h diff --git a/src/lib/smartlist_core/smartlist_foreach.h b/src/lib/smartlist_core/smartlist_foreach.h index 0f6fe30074..a1fbcd444c 100644 --- a/src/lib/smartlist_core/smartlist_foreach.h +++ b/src/lib/smartlist_core/smartlist_foreach.h @@ -83,6 +83,19 @@ ++var ## _sl_idx) { \ var = (sl)->list[var ## _sl_idx]; +/** Iterates over the items in smartlist <b>sl</b> in reverse order, similar to + * SMARTLIST_FOREACH_BEGIN + * + * NOTE: This macro is incompatible with SMARTLIST_DEL_CURRENT. + */ +#define SMARTLIST_FOREACH_REVERSE_BEGIN(sl, type, var) \ + STMT_BEGIN \ + int var ## _sl_idx, var ## _sl_len=(sl)->num_used; \ + type var; \ + for (var ## _sl_idx = var ## _sl_len-1; var ## _sl_idx >= 0; \ + --var ## _sl_idx) { \ + var = (sl)->list[var ## _sl_idx]; + #define SMARTLIST_FOREACH_END(var) \ var = NULL; \ (void) var ## _sl_idx; \ diff --git a/src/lib/string/.may_include b/src/lib/string/.may_include index ec5c769831..1fb9127f19 100644 --- a/src/lib/string/.may_include +++ b/src/lib/string/.may_include @@ -6,5 +6,5 @@ lib/malloc/*.h lib/ctime/*.h lib/string/*.h -strlcat.c -strlcpy.c +ext/strlcat.c +ext/strlcpy.c diff --git a/src/lib/string/compat_string.c b/src/lib/string/compat_string.c index e125c921a4..187f784be5 100644 --- a/src/lib/string/compat_string.c +++ b/src/lib/string/compat_string.c @@ -14,10 +14,10 @@ /* Inline the strl functions if the platform doesn't have them. */ #ifndef HAVE_STRLCPY -#include "strlcpy.c" +#include "ext/strlcpy.c" #endif #ifndef HAVE_STRLCAT -#include "strlcat.c" +#include "ext/strlcat.c" #endif #include <stdlib.h> diff --git a/src/lib/string/util_string.c b/src/lib/string/util_string.c index f934f66f02..0c4e399008 100644 --- a/src/lib/string/util_string.c +++ b/src/lib/string/util_string.c @@ -212,21 +212,6 @@ strcmpstart(const char *s1, const char *s2) return strncmp(s1, s2, n); } -/** Compare the s1_len-byte string <b>s1</b> with <b>s2</b>, - * without depending on a terminating nul in s1. Sorting order is first by - * length, then lexically; return values are as for strcmp. - */ -int -strcmp_len(const char *s1, const char *s2, size_t s1_len) -{ - size_t s2_len = strlen(s2); - if (s1_len < s2_len) - return -1; - if (s1_len > s2_len) - return 1; - return fast_memcmp(s1, s2, s2_len); -} - /** Compares the first strlen(s2) characters of s1 with s2. Returns as for * strcasecmp. */ @@ -541,3 +526,16 @@ string_is_utf8(const char *str, size_t len) } return true; } + +/** As string_is_utf8(), but returns false if the string begins with a UTF-8 + * byte order mark (BOM). + */ +int +string_is_utf8_no_bom(const char *str, size_t len) +{ + if (len >= 3 && (!strcmpstart(str, "\uFEFF") || + !strcmpstart(str, "\uFFFE"))) { + return false; + } + return string_is_utf8(str, len); +} diff --git a/src/lib/string/util_string.h b/src/lib/string/util_string.h index d9fbf8c61e..da4fab159c 100644 --- a/src/lib/string/util_string.h +++ b/src/lib/string/util_string.h @@ -33,7 +33,6 @@ int tor_strisnonupper(const char *s); int tor_strisspace(const char *s); int strcmp_opt(const char *s1, const char *s2); int strcmpstart(const char *s1, const char *s2); -int strcmp_len(const char *s1, const char *s2, size_t len); int strcasecmpstart(const char *s1, const char *s2); int strcmpend(const char *s1, const char *s2); int strcasecmpend(const char *s1, const char *s2); @@ -53,5 +52,6 @@ const char *find_str_at_start_of_line(const char *haystack, int string_is_C_identifier(const char *string); int string_is_utf8(const char *str, size_t len); +int string_is_utf8_no_bom(const char *str, size_t len); #endif /* !defined(TOR_UTIL_STRING_H) */ diff --git a/src/lib/subsys/.may_include b/src/lib/subsys/.may_include new file mode 100644 index 0000000000..2b06e8519c --- /dev/null +++ b/src/lib/subsys/.may_include @@ -0,0 +1 @@ +orconfig.h diff --git a/src/lib/subsys/include.am b/src/lib/subsys/include.am new file mode 100644 index 0000000000..4741126b14 --- /dev/null +++ b/src/lib/subsys/include.am @@ -0,0 +1,3 @@ + +noinst_HEADERS += \ + src/lib/subsys/subsys.h diff --git a/src/lib/subsys/subsys.h b/src/lib/subsys/subsys.h new file mode 100644 index 0000000000..241ad7829c --- /dev/null +++ b/src/lib/subsys/subsys.h @@ -0,0 +1,95 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_SUBSYS_T +#define TOR_SUBSYS_T + +#include <stdbool.h> + +struct dispatch_connector_t; + +/** + * A subsystem is a part of Tor that is initialized, shut down, configured, + * and connected to other parts of Tor. + * + * All callbacks are optional -- if a callback is set to NULL, the subsystem + * manager will treat it as a no-op. + * + * You should use c99 named-field initializers with this structure: we + * will be adding more fields, often in the middle of the structure. + **/ +typedef struct subsys_fns_t { + /** + * The name of this subsystem. It should be a programmer-readable + * identifier. + **/ + const char *name; + + /** + * Whether this subsystem is supported -- that is, whether it is compiled + * into Tor. For most subsystems, this should be true. + **/ + bool supported; + + /** + * The 'initialization level' for the subsystem. It should run from -100 + * through +100. The subsystems are initialized from lowest level to + * highest, and shut down from highest level to lowest. + **/ + int level; + + /** + * Initialize any global components of this subsystem. + * + * This function MAY rely on any lower-level subsystem being initialized. + * + * This function MUST NOT rely on any runtime configuration information; + * it is only for global state or pre-configuration state. + * + * (If you need to do any setup that depends on configuration, you'll need + * to declare a configuration callback. (Not yet designed)) + * + * This function MUST NOT have any parts that can fail. + **/ + int (*initialize)(void); + + /** + * Connect a subsystem to the message dispatch system. + **/ + int (*add_pubsub)(struct dispatch_connector_t *); + + /** + * Perform any necessary pre-fork cleanup. This function may not fail. + */ + void (*prefork)(void); + + /** + * Perform any necessary post-fork setup. This function may not fail. + */ + void (*postfork)(void); + + /** + * Free any thread-local resources held by this subsystem. Called before + * the thread exits. + */ + void (*thread_cleanup)(void); + + /** + * Free all resources held by this subsystem. + * + * This function is not allowed to fail. + **/ + void (*shutdown)(void); + +} subsys_fns_t; + +#define MIN_SUBSYS_LEVEL -100 +#define MAX_SUBSYS_LEVEL 100 + +/* All tor "libraries" (in src/libs) should have a subsystem level equal to or + * less than this value. */ +#define SUBSYS_LEVEL_LIBS -10 + +#endif diff --git a/src/lib/term/.may_include b/src/lib/term/.may_include index c93a06e59e..306fa57b7a 100644 --- a/src/lib/term/.may_include +++ b/src/lib/term/.may_include @@ -5,5 +5,4 @@ lib/log/*.h lib/term/*.h lib/malloc/*.h -# From src/ext -tor_readpassphrase.h +ext/tor_readpassphrase.h diff --git a/src/lib/term/getpass.c b/src/lib/term/getpass.c index c6459f250f..8741344acf 100644 --- a/src/lib/term/getpass.c +++ b/src/lib/term/getpass.c @@ -36,7 +36,7 @@ SecureZeroMemory(PVOID ptr, SIZE_T cnt) #elif defined(HAVE_READPASSPHRASE_H) #include <readpassphrase.h> #else -#include "tor_readpassphrase.h" +#include "ext/tor_readpassphrase.h" #endif /* defined(_WIN32) || ... */ #include <stdlib.h> diff --git a/src/lib/thread/.may_include b/src/lib/thread/.may_include index fc56f46836..02711348c5 100644 --- a/src/lib/thread/.may_include +++ b/src/lib/thread/.may_include @@ -2,6 +2,7 @@ orconfig.h lib/cc/*.h lib/lock/*.h lib/log/*.h +lib/subsys/*.h lib/testsupport/*.h lib/thread/*.h lib/wallclock/*.h diff --git a/src/lib/thread/compat_threads.c b/src/lib/thread/compat_threads.c index 94ab021c52..35cfeba64c 100644 --- a/src/lib/thread/compat_threads.c +++ b/src/lib/thread/compat_threads.c @@ -14,9 +14,11 @@ #include "orconfig.h" #include <stdlib.h> #include "lib/thread/threads.h" +#include "lib/thread/thread_sys.h" #include "lib/log/log.h" #include "lib/log/util_bug.h" +#include "lib/subsys/subsys.h" #include <string.h> @@ -109,3 +111,17 @@ atomic_counter_exchange(atomic_counter_t *counter, size_t newval) return oldval; } #endif /* !defined(HAVE_WORKING_STDATOMIC) */ + +static int +subsys_threads_initialize(void) +{ + tor_threads_init(); + return 0; +} + +const subsys_fns_t sys_threads = { + .name = "threads", + .supported = true, + .level = -95, + .initialize = subsys_threads_initialize, +}; diff --git a/src/lib/thread/include.am b/src/lib/thread/include.am index 9ec23d166e..695795a2c8 100644 --- a/src/lib/thread/include.am +++ b/src/lib/thread/include.am @@ -23,5 +23,6 @@ src_lib_libtor_thread_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_thread_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) noinst_HEADERS += \ - src/lib/thread/threads.h \ - src/lib/thread/numcpus.h + src/lib/thread/numcpus.h \ + src/lib/thread/thread_sys.h \ + src/lib/thread/threads.h diff --git a/src/lib/thread/thread_sys.h b/src/lib/thread/thread_sys.h new file mode 100644 index 0000000000..c0daf2b5e9 --- /dev/null +++ b/src/lib/thread/thread_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file threads_sys.h + * \brief Declare subsystem object for threads library + **/ + +#ifndef TOR_THREADS_SYS_H +#define TOR_THREADS_SYS_H + +extern const struct subsys_fns_t sys_threads; + +#endif /* !defined(TOR_THREADS_SYS_H) */ diff --git a/src/lib/time/.may_include b/src/lib/time/.may_include index 2c7e37a836..ae01431b60 100644 --- a/src/lib/time/.may_include +++ b/src/lib/time/.may_include @@ -4,8 +4,10 @@ lib/cc/*.h lib/err/*.h lib/intmath/*.h lib/log/*.h +lib/subsys/*.h lib/time/*.h lib/wallclock/*.h +lib/defs/time.h # For load_windows_system_lib. lib/fs/winlib.h
\ No newline at end of file diff --git a/src/lib/time/compat_time.c b/src/lib/time/compat_time.c index 98854bad2c..a047d6c2cd 100644 --- a/src/lib/time/compat_time.c +++ b/src/lib/time/compat_time.c @@ -522,7 +522,9 @@ monotime_init_internal(void) GetTickCount64_fn = (GetTickCount64_fn_t) (void(*)(void)) GetProcAddress(h, "GetTickCount64"); } - // FreeLibrary(h) ? + // We can't call FreeLibrary(h) here, because freeing the handle may + // unload the library, and cause future calls to GetTickCount64_fn() + // to fail. See 29642 for details. } void @@ -787,8 +789,8 @@ monotime_absolute_nsec(void) return monotime_diff_nsec(&initialized_at, &now); } -uint64_t -monotime_absolute_usec(void) +MOCK_IMPL(uint64_t, +monotime_absolute_usec,(void)) { return monotime_absolute_nsec() / 1000; } diff --git a/src/lib/time/compat_time.h b/src/lib/time/compat_time.h index 480d426ac7..2cd4b3bee3 100644 --- a/src/lib/time/compat_time.h +++ b/src/lib/time/compat_time.h @@ -15,6 +15,102 @@ * of tens of milliseconds. */ +/* Q: Should you use monotime or monotime_coarse as your source? + * + * A: Generally, you get better precision with monotime, but better + * performance with monotime_coarse. + * + * Q: Should you use monotime_t or monotime_coarse_t directly? Should you use + * usec? msec? "stamp units?" + * + * A: Using monotime_t and monotime_coarse_t directly is most time-efficient, + * since no conversion needs to happen. But they can potentially use more + * memory than you would need for a usec/msec/"stamp unit" count. + * + * Converting to usec or msec on some platforms, and working with them in + * general, creates a risk of doing a 64-bit division. 64-bit division is + * expensive on 32-bit platforms, which still do exist. + * + * The "stamp unit" type is designed to give a type that is cheap to convert + * from monotime_coarse, has resolution of about 1-2ms, and fits nicely in a + * 32-bit integer. Its downside is that it does not correspond directly + * to a natural unit of time. + * + * There is not much point in using "coarse usec" or "coarse nsec", since the + * current coarse monotime implementations give you on the order of + * milliseconds of precision. + * + * Q: So, what backends is monotime_coarse using? + * + * A: Generally speaking, it uses "whatever monotonic-ish time implemenation + * does not require a context switch." The various implementations provide + * this by having a view of the current time in a read-only memory page that + * is updated with a frequency corresponding to the kernel's tick count. + * + * On Windows, monotime_coarse uses GetCount64() [or GetTickCount() on + * obsolete systems]. MSDN claims that the resolution is "typically in the + * range of 10-16 msec", but it has said that for years. Storing + * monotime_coarse_t uses 8 bytes. + * + * On OSX/iOS, monotime_coarse uses uses mach_approximate_time() where + * available, and falls back to regular monotime. The precision is not + * documented, but the implementation is open-source: it reads from a page + * that the kernel updates. Storing monotime_coarse_t uses 8 bytes. + * + * On unixy systems, monotime_coarse uses clock_gettime() with + * CLOCK_MONOTONIC_COARSE where available, and falls back to CLOCK_MONOTONIC. + * It typically uses vdso tricks to read from a page that the kernel updates. + * Its precision fixed, but you can get it with clock_getres(): on my Linux + * desktop, it claims to be 1 msec, but it will depend on the system HZ + * setting. Storing monotime_coarse_t uses 16 bytes. + * + * [TODO: Try CLOCK_MONOTONIC_FAST on foobsd.] + * + * Q: What backends is regular monotonic time using? + * + * A: In general, regular monotime uses something that requires a system call. + * On platforms where system calls are cheap, you win! Otherwise, you lose. + * + * On Windows, monotonic time uses QuereyPerformanceCounter. Storing + * monotime_t costs 8 bytes. + * + * On OSX/Apple, monotonic time uses mach_absolute_time. Storing + * monotime_t costs 8 bytes. + * + * On unixy systems, monotonic time uses CLOCK_MONOTONIC. Storing + * monotime_t costs 16 bytes. + * + * Q: Tell me about the costs of converting to a 64-bit nsec, usec, or msec + * count. + * + * A: Windows, coarse: Cheap, since it's all multiplication. + * + * Windows, precise: Expensive on 32-bit: it needs 64-bit division. + * + * Apple, all: Expensive on 32-bit: it needs 64-bit division. + * + * Unixy, all: Fairly cheap, since the only division required is dividing + * tv_nsec 1000, and nanoseconds-per-second fits in a 32-bit value. + * + * All, "timestamp units": Cheap everywhere: it never divides. + * + * Q: This is only somewhat related, but how much precision could I hope for + * from a libevent time.? + * + * A: Actually, it's _very_ related if you're timing in order to have a + * timeout happen. + * + * On Windows, it uses select: you could in theory have a microsecond + * resolution, but it usually isn't that accurate. + * + * On OSX, iOS, and BSD, you have kqueue: You could in theory have a nanosecond + * resolution, but it usually isn't that accurate. + * + * On Linux, you have epoll: It has a millisecond resolution. Some recent + * Libevents can also use timerfd for higher resolution if + * EVENT_BASE_FLAG_PRECISE_TIMER is set: Tor doesn't set that flag. + */ + #ifndef TOR_COMPAT_TIME_H #define TOR_COMPAT_TIME_H @@ -103,7 +199,7 @@ uint64_t monotime_absolute_nsec(void); /** * Return the number of microseconds since the timer system was initialized. */ -uint64_t monotime_absolute_usec(void); +MOCK_DECL(uint64_t, monotime_absolute_usec,(void)); /** * Return the number of milliseconds since the timer system was initialized. */ diff --git a/src/lib/time/include.am b/src/lib/time/include.am index a3f93a3744..dae16f49ac 100644 --- a/src/lib/time/include.am +++ b/src/lib/time/include.am @@ -7,6 +7,7 @@ endif src_lib_libtor_time_a_SOURCES = \ src/lib/time/compat_time.c \ + src/lib/time/time_sys.c \ src/lib/time/tvdiff.c src_lib_libtor_time_testing_a_SOURCES = \ @@ -16,4 +17,5 @@ src_lib_libtor_time_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) noinst_HEADERS += \ src/lib/time/compat_time.h \ + src/lib/time/time_sys.h \ src/lib/time/tvdiff.h diff --git a/src/lib/time/time_sys.c b/src/lib/time/time_sys.c new file mode 100644 index 0000000000..b3feb7b46a --- /dev/null +++ b/src/lib/time/time_sys.c @@ -0,0 +1,26 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file time_sys.c + * \brief Subsystem object for monotime setup. + **/ + +#include "orconfig.h" +#include "lib/subsys/subsys.h" +#include "lib/time/time_sys.h" +#include "lib/time/compat_time.h" + +static int +subsys_time_initialize(void) +{ + monotime_init(); + return 0; +} + +const subsys_fns_t sys_time = { + .name = "time", + .level = -90, + .supported = true, + .initialize = subsys_time_initialize, +}; diff --git a/src/lib/time/time_sys.h b/src/lib/time/time_sys.h new file mode 100644 index 0000000000..6a860ffd08 --- /dev/null +++ b/src/lib/time/time_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file time_sys.h + * \brief Declare subsystem object for the time module. + **/ + +#ifndef TOR_TIME_SYS_H +#define TOR_TIME_SYS_H + +extern const struct subsys_fns_t sys_time; + +#endif /* !defined(TOR_TIME_SYS_H) */ diff --git a/src/lib/time/tvdiff.c b/src/lib/time/tvdiff.c index a87d0d96dc..d7c245f57a 100644 --- a/src/lib/time/tvdiff.c +++ b/src/lib/time/tvdiff.c @@ -11,6 +11,7 @@ #include "lib/time/tvdiff.h" #include "lib/cc/compat_compiler.h" +#include "lib/defs/time.h" #include "lib/log/log.h" #ifdef _WIN32 @@ -20,8 +21,6 @@ #include <sys/time.h> #endif -#define TOR_USEC_PER_SEC 1000000 - /** Return the difference between start->tv_sec and end->tv_sec. * Returns INT64_MAX on overflow and underflow. */ diff --git a/src/lib/tls/.may_include b/src/lib/tls/.may_include index 2840e590b8..c550bde024 100644 --- a/src/lib/tls/.may_include +++ b/src/lib/tls/.may_include @@ -1,6 +1,7 @@ orconfig.h lib/arch/*.h +lib/buf/*.h lib/cc/*.h lib/container/*.h lib/crypt_ops/*.h @@ -11,7 +12,7 @@ lib/log/*.h lib/malloc/*.h lib/net/*.h lib/string/*.h -lib/testsupport/testsupport.h +lib/subsys/*.h +lib/testsupport/*.h lib/tls/*.h - -ciphers.inc +lib/tls/*.inc diff --git a/src/lib/tls/buffers_tls.c b/src/lib/tls/buffers_tls.c index e92cb9163f..bf03b61459 100644 --- a/src/lib/tls/buffers_tls.c +++ b/src/lib/tls/buffers_tls.c @@ -12,7 +12,7 @@ #define BUFFERS_PRIVATE #include "orconfig.h" #include <stddef.h> -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/tls/buffers_tls.h" #include "lib/cc/torint.h" #include "lib/log/log.h" diff --git a/src/lib/tls/include.am b/src/lib/tls/include.am index a664b29fb2..1817739eef 100644 --- a/src/lib/tls/include.am +++ b/src/lib/tls/include.am @@ -36,5 +36,6 @@ noinst_HEADERS += \ src/lib/tls/tortls.h \ src/lib/tls/tortls_internal.h \ src/lib/tls/tortls_st.h \ + src/lib/tls/tortls_sys.h \ src/lib/tls/x509.h \ src/lib/tls/x509_internal.h diff --git a/src/lib/tls/tortls.c b/src/lib/tls/tortls.c index 4ca7c7d9d3..1aff40c437 100644 --- a/src/lib/tls/tortls.c +++ b/src/lib/tls/tortls.c @@ -7,6 +7,7 @@ #define TOR_X509_PRIVATE #include "lib/tls/x509.h" #include "lib/tls/x509_internal.h" +#include "lib/tls/tortls_sys.h" #include "lib/tls/tortls.h" #include "lib/tls/tortls_st.h" #include "lib/tls/tortls_internal.h" @@ -15,6 +16,7 @@ #include "lib/crypt_ops/crypto_rsa.h" #include "lib/crypt_ops/crypto_rand.h" #include "lib/net/socket.h" +#include "lib/subsys/subsys.h" #ifdef _WIN32 #include <winsock2.h> @@ -440,3 +442,15 @@ tor_tls_verify(int severity, tor_tls_t *tls, crypto_pk_t **identity) return rv; } + +static void +subsys_tortls_shutdown(void) +{ + tor_tls_free_all(); +} + +const subsys_fns_t sys_tortls = { + .name = "tortls", + .level = -50, + .shutdown = subsys_tortls_shutdown +}; diff --git a/src/lib/tls/tortls_openssl.c b/src/lib/tls/tortls_openssl.c index 80b0df301f..b40f948a3b 100644 --- a/src/lib/tls/tortls_openssl.c +++ b/src/lib/tls/tortls_openssl.c @@ -464,7 +464,7 @@ static const char UNRESTRICTED_SERVER_CIPHER_LIST[] = /** List of ciphers that clients should advertise, omitting items that * our OpenSSL doesn't know about. */ static const char CLIENT_CIPHER_LIST[] = -#include "ciphers.inc" +#include "lib/tls/ciphers.inc" /* Tell it not to use SSLv2 ciphers, so that it can select an SSLv3 version * of any cipher we say. */ "!SSLv2" diff --git a/src/lib/tls/tortls_sys.h b/src/lib/tls/tortls_sys.h new file mode 100644 index 0000000000..4b04f85f0c --- /dev/null +++ b/src/lib/tls/tortls_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file tortls_sys.h + * \brief Declare subsystem object for the tortls module + **/ + +#ifndef TOR_TORTLS_SYS_H +#define TOR_TORTLS_SYS_H + +extern const struct subsys_fns_t sys_tortls; + +#endif /* !defined(TOR_TORTLS_SYS_H) */ diff --git a/src/lib/version/.may_include b/src/lib/version/.may_include new file mode 100644 index 0000000000..d159ceb41f --- /dev/null +++ b/src/lib/version/.may_include @@ -0,0 +1,3 @@ +orconfig.h +micro-revision.i +lib/version/*.h
\ No newline at end of file diff --git a/src/lib/log/git_revision.c b/src/lib/version/git_revision.c index 7d27549cad..900a1e12a0 100644 --- a/src/lib/log/git_revision.c +++ b/src/lib/version/git_revision.c @@ -4,7 +4,7 @@ /* See LICENSE for licensing information */ #include "orconfig.h" -#include "lib/log/git_revision.h" +#include "lib/version/git_revision.h" /** String describing which Tor Git repository version the source was * built from. This string is generated by a bit of shell kludging in diff --git a/src/lib/log/git_revision.h b/src/lib/version/git_revision.h index 79e3c6684b..79e3c6684b 100644 --- a/src/lib/log/git_revision.h +++ b/src/lib/version/git_revision.h diff --git a/src/lib/version/include.am b/src/lib/version/include.am new file mode 100644 index 0000000000..6944eb05e3 --- /dev/null +++ b/src/lib/version/include.am @@ -0,0 +1,25 @@ + +noinst_LIBRARIES += src/lib/libtor-version.a + +if UNITTESTS_ENABLED +noinst_LIBRARIES += src/lib/libtor-version-testing.a +endif + +src_lib_libtor_version_a_SOURCES = \ + src/lib/version/git_revision.c \ + src/lib/version/version.c + +src_lib_libtor_version_testing_a_SOURCES = \ + $(src_lib_libtor_version_a_SOURCES) +src_lib_libtor_version_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) +src_lib_libtor_version_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) + +# Declare that these object files depend on micro-revision.i. Without this +# rule, we could try to build them before micro-revision.i was created. +src/lib/version/git_revision.$(OBJEXT) \ + src/lib/version/src_lib_libtor_version_testing_a-git_revision.$(OBJEXT): \ + micro-revision.i + +noinst_HEADERS += \ + src/lib/version/git_revision.h \ + src/lib/version/torversion.h diff --git a/src/lib/version/torversion.h b/src/lib/version/torversion.h new file mode 100644 index 0000000000..7b0fb66ec0 --- /dev/null +++ b/src/lib/version/torversion.h @@ -0,0 +1,12 @@ +/* Copyright 2001-2004 Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_VERSION_H +#define TOR_VERSION_H + +const char *get_version(void); +const char *get_short_version(void); + +#endif /* !defined(TOR_VERSION_H) */ diff --git a/src/lib/version/version.c b/src/lib/version/version.c new file mode 100644 index 0000000000..434e6fb424 --- /dev/null +++ b/src/lib/version/version.c @@ -0,0 +1,50 @@ +/* Copyright 2001-2004 Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#include "lib/version/torversion.h" +#include "lib/version/git_revision.h" + +#include <stdio.h> +#include <string.h> + +/** A shorter version of this Tor process's version, for export in our router + * descriptor. (Does not include the git version, if any.) */ +static const char the_short_tor_version[] = + VERSION +#ifdef TOR_BUILD_TAG + " ("TOR_BUILD_TAG")" +#endif + ""; + +#define MAX_VERSION_LEN 128 + +/** The version of this Tor process, possibly including git version */ +static char the_tor_version[MAX_VERSION_LEN] = ""; + +/** Return the current Tor version. */ +const char * +get_version(void) +{ + if (the_tor_version[0] == 0) { + if (strlen(tor_git_revision)) { + snprintf(the_tor_version, sizeof(the_tor_version), + "%s (git-%s)", the_short_tor_version, tor_git_revision); + } else { + snprintf(the_tor_version, sizeof(the_tor_version), + "%s", the_short_tor_version); + } + the_tor_version[sizeof(the_tor_version)-1] = 0; + } + + return the_tor_version; +} + +/** Return the current Tor version, without any git tag. */ +const char * +get_short_version(void) +{ + return the_short_tor_version; +} diff --git a/src/lib/wallclock/.may_include b/src/lib/wallclock/.may_include index dc010da063..ce7a26472b 100644 --- a/src/lib/wallclock/.may_include +++ b/src/lib/wallclock/.may_include @@ -3,4 +3,5 @@ lib/cc/*.h lib/err/*.h lib/wallclock/*.h lib/string/*.h +lib/subsys/*.h lib/testsupport/*.h diff --git a/src/lib/wallclock/approx_time.c b/src/lib/wallclock/approx_time.c index ee498702d5..7b32804026 100644 --- a/src/lib/wallclock/approx_time.c +++ b/src/lib/wallclock/approx_time.c @@ -9,7 +9,9 @@ **/ #include "orconfig.h" +#include "lib/subsys/subsys.h" #include "lib/wallclock/approx_time.h" +#include "lib/wallclock/wallclock_sys.h" /* ===== * Cached time @@ -41,3 +43,17 @@ update_approx_time(time_t now) cached_approx_time = now; } #endif /* !defined(TIME_IS_FAST) */ + +static int +subsys_wallclock_initialize(void) +{ + update_approx_time(time(NULL)); + return 0; +} + +const subsys_fns_t sys_wallclock = { + .name = "wallclock", + .supported = true, + .level = -99, + .initialize = subsys_wallclock_initialize, +}; diff --git a/src/lib/wallclock/include.am b/src/lib/wallclock/include.am index 1961639bd7..2351252e0c 100644 --- a/src/lib/wallclock/include.am +++ b/src/lib/wallclock/include.am @@ -19,4 +19,5 @@ noinst_HEADERS += \ src/lib/wallclock/approx_time.h \ src/lib/wallclock/timeval.h \ src/lib/wallclock/time_to_tm.h \ - src/lib/wallclock/tor_gettimeofday.h + src/lib/wallclock/tor_gettimeofday.h \ + src/lib/wallclock/wallclock_sys.h diff --git a/src/lib/wallclock/wallclock_sys.h b/src/lib/wallclock/wallclock_sys.h new file mode 100644 index 0000000000..a30912b8fb --- /dev/null +++ b/src/lib/wallclock/wallclock_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file wallclock_sys.h + * \brief Declare subsystem object for the wallclock module. + **/ + +#ifndef TOR_WALLCLOCK_SYS_H +#define TOR_WALLCLOCK_SYS_H + +extern const struct subsys_fns_t sys_wallclock; + +#endif /* !defined(TOR_WALLCLOCK_SYS_H) */ diff --git a/src/rust/build.rs b/src/rust/build.rs index 123d5c0682..5626b35f75 100644 --- a/src/rust/build.rs +++ b/src/rust/build.rs @@ -149,8 +149,9 @@ pub fn main() { cfg.component("tor-sandbox-testing"); cfg.component("tor-encoding-testing"); cfg.component("tor-fs-testing"); - cfg.component("tor-time-testing"); cfg.component("tor-net-testing"); + cfg.component("tor-buf-testing"); + cfg.component("tor-time-testing"); cfg.component("tor-thread-testing"); cfg.component("tor-memarea-testing"); cfg.component("tor-log-testing"); @@ -162,6 +163,7 @@ pub fn main() { cfg.component("tor-malloc"); cfg.component("tor-wallclock"); cfg.component("tor-err-testing"); + cfg.component("tor-version-testing"); cfg.component("tor-intmath-testing"); cfg.component("tor-ctime-testing"); cfg.component("curve25519_donna"); diff --git a/src/rust/protover/ffi.rs b/src/rust/protover/ffi.rs index 6ee63adb10..066b08eddb 100644 --- a/src/rust/protover/ffi.rs +++ b/src/rust/protover/ffi.rs @@ -30,6 +30,7 @@ fn translate_to_rust(c_proto: uint32_t) -> Result<Protocol, ProtoverError> { 7 => Ok(Protocol::Desc), 8 => Ok(Protocol::Microdesc), 9 => Ok(Protocol::Cons), + 10 => Ok(Protocol::Padding), _ => Err(ProtoverError::UnknownProtocol), } } diff --git a/src/rust/protover/protover.rs b/src/rust/protover/protover.rs index 06fdf56c69..a08e835393 100644 --- a/src/rust/protover/protover.rs +++ b/src/rust/protover/protover.rs @@ -46,6 +46,7 @@ pub enum Protocol { LinkAuth, Microdesc, Relay, + Padding, } impl fmt::Display for Protocol { @@ -73,6 +74,7 @@ impl FromStr for Protocol { "LinkAuth" => Ok(Protocol::LinkAuth), "Microdesc" => Ok(Protocol::Microdesc), "Relay" => Ok(Protocol::Relay), + "Padding" => Ok(Protocol::Padding), _ => Err(ProtoverError::UnknownProtocol), } } @@ -163,7 +165,8 @@ pub(crate) fn get_supported_protocols_cstr() -> &'static CStr { Link=1-5 \ LinkAuth=3 \ Microdesc=1-2 \ - Relay=1-2" + Relay=1-2 \ + Padding=1" ) } else { cstr!( @@ -176,7 +179,8 @@ pub(crate) fn get_supported_protocols_cstr() -> &'static CStr { Link=1-5 \ LinkAuth=1,3 \ Microdesc=1-2 \ - Relay=1-2" + Relay=1-2 \ + Padding=1" ) } } diff --git a/src/rust/tor_util/strings.rs b/src/rust/tor_util/strings.rs index 2a19458f46..ede42c6ea8 100644 --- a/src/rust/tor_util/strings.rs +++ b/src/rust/tor_util/strings.rs @@ -105,11 +105,7 @@ macro_rules! cstr { ($($bytes:expr),*) => ( ::std::ffi::CStr::from_bytes_with_nul( concat!($($bytes),*, "\0").as_bytes() - ).unwrap_or( - unsafe{ - ::std::ffi::CStr::from_bytes_with_nul_unchecked(b"\0") - } - ) + ).unwrap_or_default() ) } diff --git a/src/test/Makefile.nmake b/src/test/Makefile.nmake index cfbe281b94..ca6a84cf8a 100644 --- a/src/test/Makefile.nmake +++ b/src/test/Makefile.nmake @@ -1,4 +1,4 @@ -all: test.exe test-child.exe bench.exe +all: test.exe bench.exe CFLAGS = /I ..\win32 /I ..\..\..\build-alpha\include /I ..\common /I ..\or \ /I ..\ext @@ -19,6 +19,7 @@ TEST_OBJECTS = test.obj test_addr.obj test_channel.obj test_channeltls.obj \ test_cell_formats.obj test_relay.obj test_replay.obj \ test_channelpadding.obj \ test_circuitstats.obj \ + test_circuitpadding.obj \ test_scheduler.obj test_introduce.obj test_hs.obj tinytest.obj tinytest.obj: ..\ext\tinytest.c @@ -30,8 +31,5 @@ test.exe: $(TEST_OBJECTS) bench.exe: bench.obj $(CC) $(CFLAGS) bench.obj $(LIBS) ..\common\*.lib /Fe$@ -test-child.exe: test-child.obj - $(CC) $(CFLAGS) test-child.obj /Fe$@ - clean: - del *.obj *.lib test.exe bench.exe test-child.exe + del *.obj *.lib test.exe bench.exe diff --git a/src/test/bench.c b/src/test/bench.c index 06c616c3b0..65fa617cbd 100644 --- a/src/test/bench.c +++ b/src/test/bench.c @@ -14,6 +14,8 @@ #include "core/crypto/onion_tap.h" #include "core/crypto/relay_crypto.h" +#include "lib/intmath/weakrng.h" + #ifdef ENABLE_OPENSSL #include <openssl/opensslv.h> #include <openssl/evp.h> @@ -24,6 +26,7 @@ #include "core/or/circuitlist.h" #include "app/config/config.h" +#include "app/main/subsysmgr.h" #include "lib/crypt_ops/crypto_curve25519.h" #include "lib/crypt_ops/crypto_dh.h" #include "core/crypto/onion_ntor.h" @@ -38,6 +41,9 @@ #include "lib/crypt_ops/digestset.h" #include "lib/crypt_ops/crypto_init.h" +#include "feature/dirparse/microdesc_parse.h" +#include "feature/nodelist/microdesc.h" + #if defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_PROCESS_CPUTIME_ID) static uint64_t nanostart; static inline uint64_t @@ -332,6 +338,65 @@ bench_ed25519(void) } static void +bench_rand_len(int len) +{ + const int N = 100000; + int i; + char *buf = tor_malloc(len); + uint64_t start,end; + + start = perftime(); + for (i = 0; i < N; ++i) { + crypto_rand(buf, len); + } + end = perftime(); + printf("crypto_rand(%d): %f nsec.\n", len, NANOCOUNT(start,end,N)); + + crypto_fast_rng_t *fr = crypto_fast_rng_new(); + start = perftime(); + for (i = 0; i < N; ++i) { + crypto_fast_rng_getbytes(fr,(uint8_t*)buf,len); + } + end = perftime(); + printf("crypto_fast_rng_getbytes(%d): %f nsec.\n", len, + NANOCOUNT(start,end,N)); + crypto_fast_rng_free(fr); + + if (len <= 32) { + start = perftime(); + for (i = 0; i < N; ++i) { + crypto_strongest_rand((uint8_t*)buf, len); + } + end = perftime(); + printf("crypto_strongest_rand(%d): %f nsec.\n", len, + NANOCOUNT(start,end,N)); + } + + if (len == 4) { + tor_weak_rng_t weak; + tor_init_weak_random(&weak, 1337); + + start = perftime(); + uint32_t t=0; + for (i = 0; i < N; ++i) { + t += tor_weak_random(&weak); + } + end = perftime(); + printf("weak_rand(4): %f nsec.\n", NANOCOUNT(start,end,N)); + } + + tor_free(buf); +} + +static void +bench_rand(void) +{ + bench_rand_len(4); + bench_rand_len(16); + bench_rand_len(128); +} + +static void bench_cell_aes(void) { uint64_t start, end; @@ -638,6 +703,41 @@ bench_ecdh_p224(void) } #endif +static void +bench_md_parse(void) +{ + uint64_t start, end; + const int N = 100000; + // selected arbitrarily + const char md_text[] = + "@last-listed 2018-12-14 18:14:14\n" + "onion-key\n" + "-----BEGIN RSA PUBLIC KEY-----\n" + "MIGJAoGBAMHkZeXNDX/49JqM2BVLmh1Fnb5iMVnatvZZTLJyedqDLkbXZ1WKP5oh\n" + "7ec14dj/k3ntpwHD4s2o3Lb6nfagWbug4+F/rNJ7JuFru/PSyOvDyHGNAuegOXph\n" + "3gTGjdDpv/yPoiadGebbVe8E7n6hO+XxM2W/4dqheKimF0/s9B7HAgMBAAE=\n" + "-----END RSA PUBLIC KEY-----\n" + "ntor-onion-key QgF/EjqlNG1wRHLIop/nCekEH+ETGZSgYOhu26eiTF4=\n" + "family $00E9A86E7733240E60D8435A7BBD634A23894098 " + "$329BD7545DEEEBBDC8C4285F243916F248972102 " + "$69E06EBB2573A4F89330BDF8BC869794A3E10E4D " + "$DCA2A3FAE50B3729DAA15BC95FB21AF03389818B\n" + "p accept 53,80,443,5222-5223,25565\n" + "id ed25519 BzffzY99z6Q8KltcFlUTLWjNTBU7yKK+uQhyi1Ivb3A\n"; + + reset_perftime(); + start = perftime(); + for (int i = 0; i < N; ++i) { + smartlist_t *s = microdescs_parse_from_string(md_text, NULL, 1, + SAVED_IN_CACHE, NULL); + SMARTLIST_FOREACH(s, microdesc_t *, md, microdesc_free(md)); + smartlist_free(s); + } + + end = perftime(); + printf("Microdesc parse: %f nsec\n", NANOCOUNT(start, end, N)); +} + typedef void (*bench_fn)(void); typedef struct benchmark_t { @@ -656,6 +756,7 @@ static struct benchmark_t benchmarks[] = { ENT(onion_TAP), ENT(onion_ntor), ENT(ed25519), + ENT(rand), ENT(cell_aes), ENT(cell_ops), @@ -665,6 +766,8 @@ static struct benchmark_t benchmarks[] = { ENT(ecdh_p256), ENT(ecdh_p224), #endif + + ENT(md_parse), {NULL,NULL,0} }; @@ -690,9 +793,10 @@ main(int argc, const char **argv) char *errmsg; or_options_t *options; - tor_threads_init(); + subsystems_init_upto(SUBSYS_LEVEL_LIBS); + flush_log_messages_from_startup(); + tor_compress_init(); - init_logging(1); if (argc == 4 && !strcmp(argv[1], "diff")) { const int N = 200; @@ -702,11 +806,13 @@ main(int argc, const char **argv) perror("X"); return 1; } + size_t f1len = strlen(f1); + size_t f2len = strlen(f2); for (i = 0; i < N; ++i) { - char *diff = consensus_diff_generate(f1, f2); + char *diff = consensus_diff_generate(f1, f1len, f2, f2len); tor_free(diff); } - char *diff = consensus_diff_generate(f1, f2); + char *diff = consensus_diff_generate(f1, f1len, f2, f2len); printf("%s", diff); tor_free(f1); tor_free(f2); @@ -737,7 +843,6 @@ main(int argc, const char **argv) init_protocol_warning_severity_level(); options = options_new(); - init_logging(1); options->command = CMD_RUN_UNITTESTS; options->DataDirectory = tor_strdup(""); options->KeyDirectory = tor_strdup(""); diff --git a/src/test/fuzz/fuzz_consensus.c b/src/test/fuzz/fuzz_consensus.c index 5947a3f48c..656ef0bdb2 100644 --- a/src/test/fuzz/fuzz_consensus.c +++ b/src/test/fuzz/fuzz_consensus.c @@ -61,13 +61,13 @@ int fuzz_main(const uint8_t *data, size_t sz) { networkstatus_t *ns; - char *str = tor_memdup_nulterm(data, sz); const char *eos = NULL; networkstatus_type_t tp = NS_TYPE_CONSENSUS; if (tor_memstr(data, MIN(sz, 1024), "tus vote")) tp = NS_TYPE_VOTE; const char *what = (tp == NS_TYPE_CONSENSUS) ? "consensus" : "vote"; - ns = networkstatus_parse_vote_from_string(str, + ns = networkstatus_parse_vote_from_string((const char *)data, + sz, &eos, tp); if (ns) { @@ -76,6 +76,6 @@ fuzz_main(const uint8_t *data, size_t sz) } else { log_debug(LD_GENERAL, "Parsing as %s failed", what); } - tor_free(str); + return 0; } diff --git a/src/test/fuzz/fuzz_diff.c b/src/test/fuzz/fuzz_diff.c index 1bc60e50ee..a31445666c 100644 --- a/src/test/fuzz/fuzz_diff.c +++ b/src/test/fuzz/fuzz_diff.c @@ -10,9 +10,11 @@ #include "test/fuzz/fuzzing.h" static int -mock_consensus_compute_digest_(const char *c, consensus_digest_t *d) +mock_consensus_compute_digest_(const char *c, size_t len, + consensus_digest_t *d) { (void)c; + (void)len; memset(d->sha3_256, 3, sizeof(d->sha3_256)); return 0; } @@ -42,28 +44,34 @@ fuzz_main(const uint8_t *stdin_buf, size_t data_size) if (! separator) return 0; size_t c1_len = separator - stdin_buf; - char *c1 = tor_memdup_nulterm(stdin_buf, c1_len); + const char *c1 = (const char *)stdin_buf; size_t c2_len = data_size - c1_len - SEPLEN; - char *c2 = tor_memdup_nulterm(separator + SEPLEN, c2_len); + const char *c2 = (const char *)separator + SEPLEN; - char *c3 = consensus_diff_generate(c1, c2); + const char *cp = memchr(c1, 0, c1_len); + if (cp) + c1_len = cp - c1; + + cp = memchr(c2, 0, c2_len); + if (cp) + c2_len = cp - c2; + + char *c3 = consensus_diff_generate(c1, c1_len, c2, c2_len); if (c3) { - char *c4 = consensus_diff_apply(c1, c3); + char *c4 = consensus_diff_apply(c1, c1_len, c3, strlen(c3)); tor_assert(c4); - if (strcmp(c2, c4)) { - printf("%s\n", escaped(c1)); - printf("%s\n", escaped(c2)); + int equal = (c2_len == strlen(c4)) && fast_memeq(c2, c4, c2_len); + if (! equal) { + //printf("%s\n", escaped(c1)); + //printf("%s\n", escaped(c2)); printf("%s\n", escaped(c3)); printf("%s\n", escaped(c4)); } - tor_assert(! strcmp(c2, c4)); + tor_assert(equal); tor_free(c3); tor_free(c4); } - tor_free(c1); - tor_free(c2); return 0; } - diff --git a/src/test/fuzz/fuzz_diff_apply.c b/src/test/fuzz/fuzz_diff_apply.c index 9bd3cb0bf8..d8a0f9e590 100644 --- a/src/test/fuzz/fuzz_diff_apply.c +++ b/src/test/fuzz/fuzz_diff_apply.c @@ -10,9 +10,11 @@ #include "test/fuzz/fuzzing.h" static int -mock_consensus_compute_digest_(const char *c, consensus_digest_t *d) +mock_consensus_compute_digest_(const char *c, size_t len, + consensus_digest_t *d) { (void)c; + (void)len; memset(d->sha3_256, 3, sizeof(d->sha3_256)); return 0; } @@ -50,16 +52,13 @@ fuzz_main(const uint8_t *stdin_buf, size_t data_size) if (! separator) return 0; size_t c1_len = separator - stdin_buf; - char *c1 = tor_memdup_nulterm(stdin_buf, c1_len); + const char *c1 = (const char *)stdin_buf; size_t c2_len = data_size - c1_len - SEPLEN; - char *c2 = tor_memdup_nulterm(separator + SEPLEN, c2_len); + const char *c2 = (const char *)separator + SEPLEN; - char *c3 = consensus_diff_apply(c1, c2); + char *c3 = consensus_diff_apply(c1, c1_len, c2, c2_len); - tor_free(c1); - tor_free(c2); tor_free(c3); return 0; } - diff --git a/src/test/fuzz/fuzz_http.c b/src/test/fuzz/fuzz_http.c index 2798c47d23..44393b3a10 100644 --- a/src/test/fuzz/fuzz_http.c +++ b/src/test/fuzz/fuzz_http.c @@ -8,7 +8,7 @@ #include "core/or/or.h" #include "lib/err/backtrace.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "app/config/config.h" #include "core/mainloop/connection.h" #include "feature/dircache/dircache.h" diff --git a/src/test/fuzz/fuzz_http_connect.c b/src/test/fuzz/fuzz_http_connect.c index a60fc36804..2a597cae74 100644 --- a/src/test/fuzz/fuzz_http_connect.c +++ b/src/test/fuzz/fuzz_http_connect.c @@ -8,7 +8,7 @@ #include "core/or/or.h" #include "lib/err/backtrace.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "app/config/config.h" #include "core/mainloop/connection.h" #include "core/or/connection_edge.h" diff --git a/src/test/fuzz/fuzz_socks.c b/src/test/fuzz/fuzz_socks.c index 06cb08391e..d6c416a0f9 100644 --- a/src/test/fuzz/fuzz_socks.c +++ b/src/test/fuzz/fuzz_socks.c @@ -6,7 +6,7 @@ #define BUFFERS_PRIVATE #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/err/backtrace.h" #include "lib/log/log.h" #include "core/proto/proto_socks.h" diff --git a/src/test/fuzz/fuzz_strops.c b/src/test/fuzz/fuzz_strops.c new file mode 100644 index 0000000000..64a6453050 --- /dev/null +++ b/src/test/fuzz/fuzz_strops.c @@ -0,0 +1,247 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file fuzz_strops.c + * \brief Fuzzers for various string encoding/decoding operations + **/ + +#include "orconfig.h" + +#include "lib/cc/torint.h" +#include "lib/ctime/di_ops.h" +#include "lib/encoding/binascii.h" +#include "lib/encoding/cstring.h" +#include "lib/encoding/kvline.h" +#include "lib/encoding/confline.h" +#include "lib/malloc/malloc.h" +#include "lib/log/escape.h" +#include "lib/log/util_bug.h" +#include "lib/intmath/muldiv.h" + +#include "test/fuzz/fuzzing.h" + +#include <stdio.h> +#include <string.h> + +int +fuzz_init(void) +{ + return 0; +} + +int +fuzz_cleanup(void) +{ + return 0; +} + +typedef struct chunk_t { + uint8_t *buf; + size_t len; +} chunk_t; + +#define chunk_free(ch) \ + FREE_AND_NULL(chunk_t, chunk_free_, (ch)) + +static chunk_t * +chunk_new(size_t len) +{ + chunk_t *ch = tor_malloc(sizeof(chunk_t)); + ch->buf = tor_malloc(len); + ch->len = len; + return ch; +} +static void +chunk_free_(chunk_t *ch) +{ + if (!ch) + return; + tor_free(ch->buf); + tor_free(ch); +} +static bool +chunk_eq(const chunk_t *a, const chunk_t *b) +{ + return a->len == b->len && fast_memeq(a->buf, b->buf, a->len); +} + +static chunk_t * +b16_dec(const chunk_t *inp) +{ + chunk_t *ch = chunk_new(CEIL_DIV(inp->len, 2)); + int r = base16_decode((char *)ch->buf, ch->len, (char *)inp->buf, inp->len); + if (r >= 0) { + ch->len = r; + } else { + chunk_free(ch); + } + return ch; +} +static chunk_t * +b16_enc(const chunk_t *inp) +{ + chunk_t *ch = chunk_new(inp->len * 2 + 1); + base16_encode((char *)ch->buf, ch->len, (char*)inp->buf, inp->len); + return ch; +} + +#if 0 +static chunk_t * +b32_dec(const chunk_t *inp) +{ + chunk_t *ch = chunk_new(inp->len);//XXXX + int r = base32_decode((char *)ch->buf, ch->len, (char *)inp->buf, inp->len); + if (r >= 0) { + ch->len = r; // XXXX we need some way to get the actual length of + // XXXX the output here. + } else { + chunk_free(ch); + } + return ch; +} +static chunk_t * +b32_enc(const chunk_t *inp) +{ + chunk_t *ch = chunk_new(base32_encoded_size(inp->len)); + base32_encode((char *)ch->buf, ch->len, (char*)inp->buf, inp->len); + ch->len = strlen((char *) ch->buf); + return ch; +} +#endif + +static chunk_t * +b64_dec(const chunk_t *inp) +{ + chunk_t *ch = chunk_new(inp->len);//XXXX This could be shorter. + int r = base64_decode((char *)ch->buf, ch->len, (char *)inp->buf, inp->len); + if (r >= 0) { + ch->len = r; + } else { + chunk_free(ch); + } + return ch; +} +static chunk_t * +b64_enc(const chunk_t *inp) +{ + chunk_t *ch = chunk_new(BASE64_BUFSIZE(inp->len)); + base64_encode((char *)ch->buf, ch->len, (char *)inp->buf, inp->len, 0); + ch->len = strlen((char *) ch->buf); + return ch; +} + +static chunk_t * +c_dec(const chunk_t *inp) +{ + char *s = tor_memdup_nulterm(inp->buf, inp->len); + chunk_t *ch = tor_malloc(sizeof(chunk_t)); + char *r = NULL; + (void) unescape_string(s, &r, &ch->len); + tor_free(s); + ch->buf = (uint8_t*) r; + if (!ch->buf) { + tor_free(ch); + } + return ch; +} +static chunk_t * +c_enc(const chunk_t *inp) +{ + char *s = tor_memdup_nulterm(inp->buf, inp->len); + chunk_t *ch = tor_malloc(sizeof(chunk_t)); + ch->buf = (uint8_t*)esc_for_log(s); + tor_free(s); + ch->len = strlen((char*)ch->buf); + return ch; +} + +static int kv_flags = 0; +static config_line_t * +kv_dec(const chunk_t *inp) +{ + char *s = tor_memdup_nulterm(inp->buf, inp->len); + config_line_t *res = kvline_parse(s, kv_flags); + tor_free(s); + return res; +} +static chunk_t * +kv_enc(const config_line_t *inp) +{ + char *s = kvline_encode(inp, kv_flags); + if (!s) + return NULL; + chunk_t *res = tor_malloc(sizeof(chunk_t)); + res->buf = (uint8_t*)s; + res->len = strlen(s); + return res; +} + +/* Given an encoder function, a decoder function, and a function to free + * the decoded object, check whether any string that successfully decoded + * will then survive an encode-decode-encode round-trip unchanged. + */ +#define ENCODE_ROUNDTRIP(E,D,FREE) \ + STMT_BEGIN { \ + bool err = false; \ + a = D(&inp); \ + if (!a) \ + return 0; \ + b = E(a); \ + tor_assert(b); \ + c = D(b); \ + tor_assert(c); \ + d = E(c); \ + tor_assert(d); \ + if (!chunk_eq(b,d)) { \ + printf("Unequal chunks: %s\n", \ + hex_str((char*)b->buf, b->len)); \ + printf(" vs %s\n", \ + hex_str((char*)d->buf, d->len)); \ + err = true; \ + } \ + FREE(a); \ + chunk_free(b); \ + FREE(c); \ + chunk_free(d); \ + tor_assert(!err); \ + } STMT_END + +int +fuzz_main(const uint8_t *stdin_buf, size_t data_size) +{ + if (!data_size) + return 0; + + chunk_t inp = { (uint8_t*)stdin_buf, data_size }; + chunk_t *b=NULL,*d=NULL; + void *a=NULL,*c=NULL; + + switch (stdin_buf[0]) { + case 0: + ENCODE_ROUNDTRIP(b16_enc, b16_dec, chunk_free_); + break; + case 1: + /* + XXXX see notes above about our base-32 functions. + ENCODE_ROUNDTRIP(b32_enc, b32_dec, chunk_free_); + */ + break; + case 2: + ENCODE_ROUNDTRIP(b64_enc, b64_dec, chunk_free_); + break; + case 3: + ENCODE_ROUNDTRIP(c_enc, c_dec, chunk_free_); + break; + case 5: + kv_flags = KV_QUOTED|KV_OMIT_KEYS; + ENCODE_ROUNDTRIP(kv_enc, kv_dec, config_free_lines_); + break; + case 6: + kv_flags = 0; + ENCODE_ROUNDTRIP(kv_enc, kv_dec, config_free_lines_); + break; + } + + return 0; +} diff --git a/src/test/fuzz/fuzz_vrs.c b/src/test/fuzz/fuzz_vrs.c index 967397d1af..7b61b8df2d 100644 --- a/src/test/fuzz/fuzz_vrs.c +++ b/src/test/fuzz/fuzz_vrs.c @@ -3,6 +3,7 @@ #define NS_PARSE_PRIVATE #define NETWORKSTATUS_PRIVATE #include "core/or/or.h" +#include "feature/dirauth/dirvote.h" #include "feature/dirparse/ns_parse.h" #include "feature/dirparse/unparseable.h" #include "lib/memarea/memarea.h" @@ -35,9 +36,12 @@ fuzz_init(void) dummy_vote = tor_malloc_zero(sizeof(*dummy_vote)); dummy_vote->known_flags = smartlist_new(); smartlist_split_string(dummy_vote->known_flags, - "Authority BadExit Exit Fast Guard HSDir " - "NoEdConsensus Running Stable V2Dir Valid", + DIRVOTE_UNIVERSAL_FLAGS, " ", 0, 0); + smartlist_split_string(dummy_vote->known_flags, + DIRVOTE_OPTIONAL_FLAGS, + " ", 0, 0); + smartlist_sort_strings(dummy_vote->known_flags); return 0; } @@ -53,24 +57,24 @@ fuzz_cleanup(void) int fuzz_main(const uint8_t *data, size_t sz) { - char *str = tor_memdup_nulterm(data, sz); const char *s; routerstatus_t *rs_ns = NULL, *rs_md = NULL, *rs_vote = NULL; vote_routerstatus_t *vrs = tor_malloc_zero(sizeof(*vrs)); smartlist_t *tokens = smartlist_new(); + const char *eos = (const char *)data + sz; - s = str; - rs_ns = routerstatus_parse_entry_from_string(area, &s, tokens, + s = (const char *)data; + rs_ns = routerstatus_parse_entry_from_string(area, &s, eos, tokens, NULL, NULL, 26, FLAV_NS); tor_assert(smartlist_len(tokens) == 0); - s = str; - rs_md = routerstatus_parse_entry_from_string(area, &s, tokens, + s = (const char *)data; + rs_md = routerstatus_parse_entry_from_string(area, &s, eos, tokens, NULL, NULL, 26, FLAV_MICRODESC); tor_assert(smartlist_len(tokens) == 0); - s = str; - rs_vote = routerstatus_parse_entry_from_string(area, &s, tokens, + s = (const char *)data; + rs_vote = routerstatus_parse_entry_from_string(area, &s, eos, tokens, dummy_vote, vrs, 26, FLAV_NS); tor_assert(smartlist_len(tokens) == 0); @@ -82,6 +86,6 @@ fuzz_main(const uint8_t *data, size_t sz) vote_routerstatus_free(vrs); memarea_clear(area); smartlist_free(tokens); - tor_free(str); + return 0; } diff --git a/src/test/fuzz/fuzzing_common.c b/src/test/fuzz/fuzzing_common.c index 8ea4898522..387c865a9b 100644 --- a/src/test/fuzz/fuzzing_common.c +++ b/src/test/fuzz/fuzzing_common.c @@ -3,12 +3,14 @@ #define CRYPTO_ED25519_PRIVATE #include "orconfig.h" #include "core/or/or.h" +#include "app/main/subsysmgr.h" #include "lib/err/backtrace.h" #include "app/config/config.h" #include "test/fuzz/fuzzing.h" #include "lib/compress/compress.h" #include "lib/crypt_ops/crypto_ed25519.h" #include "lib/crypt_ops/crypto_init.h" +#include "lib/version/torversion.h" static or_options_t *mock_options = NULL; static const or_options_t * @@ -94,12 +96,10 @@ disable_signature_checking(void) static void global_init(void) { - tor_threads_init(); - tor_compress_init(); + subsystems_init_upto(SUBSYS_LEVEL_LIBS); + flush_log_messages_from_startup(); - /* Initialise logging first */ - init_logging(1); - configure_backtrace_handler(get_version()); + tor_compress_init(); if (crypto_global_init(0, NULL, NULL) < 0) abort(); diff --git a/src/test/fuzz/include.am b/src/test/fuzz/include.am index 27eeced8c5..d0711f05d6 100644 --- a/src/test/fuzz/include.am +++ b/src/test/fuzz/include.am @@ -153,6 +153,16 @@ src_test_fuzz_fuzz_socks_LDADD = $(FUZZING_LIBS) endif if UNITTESTS_ENABLED +src_test_fuzz_fuzz_strops_SOURCES = \ + src/test/fuzz/fuzzing_common.c \ + src/test/fuzz/fuzz_strops.c +src_test_fuzz_fuzz_strops_CPPFLAGS = $(FUZZING_CPPFLAGS) +src_test_fuzz_fuzz_strops_CFLAGS = $(FUZZING_CFLAGS) +src_test_fuzz_fuzz_strops_LDFLAGS = $(FUZZING_LDFLAG) +src_test_fuzz_fuzz_strops_LDADD = $(FUZZING_LIBS) +endif + +if UNITTESTS_ENABLED src_test_fuzz_fuzz_vrs_SOURCES = \ src/test/fuzz/fuzzing_common.c \ src/test/fuzz/fuzz_vrs.c @@ -176,6 +186,7 @@ FUZZERS = \ src/test/fuzz/fuzz-iptsv2 \ src/test/fuzz/fuzz-microdesc \ src/test/fuzz/fuzz-socks \ + src/test/fuzz/fuzz-strops \ src/test/fuzz/fuzz-vrs endif @@ -291,6 +302,15 @@ src_test_fuzz_lf_fuzz_socks_LDADD = $(LIBFUZZER_LIBS) endif if UNITTESTS_ENABLED +src_test_fuzz_lf_fuzz_strops_SOURCES = \ + $(src_test_fuzz_fuzz_strops_SOURCES) +src_test_fuzz_lf_fuzz_strops_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) +src_test_fuzz_lf_fuzz_strops_CFLAGS = $(LIBFUZZER_CFLAGS) +src_test_fuzz_lf_fuzz_strops_LDFLAGS = $(LIBFUZZER_LDFLAG) +src_test_fuzz_lf_fuzz_strops_LDADD = $(LIBFUZZER_LIBS) +endif + +if UNITTESTS_ENABLED src_test_fuzz_lf_fuzz_vrs_SOURCES = \ $(src_test_fuzz_fuzz_vrs_SOURCES) src_test_fuzz_lf_fuzz_vrs_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) @@ -312,6 +332,7 @@ LIBFUZZER_FUZZERS = \ src/test/fuzz/lf-fuzz-iptsv2 \ src/test/fuzz/lf-fuzz-microdesc \ src/test/fuzz/lf-fuzz-socks \ + src/test/fuzz/lf-fuzz-strops \ src/test/fuzz/lf-fuzz-vrs else @@ -406,6 +427,13 @@ src_test_fuzz_liboss_fuzz_socks_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) endif if UNITTESTS_ENABLED +src_test_fuzz_liboss_fuzz_strops_a_SOURCES = \ + $(src_test_fuzz_fuzz_strops_SOURCES) +src_test_fuzz_liboss_fuzz_strops_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) +src_test_fuzz_liboss_fuzz_strops_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) +endif + +if UNITTESTS_ENABLED src_test_fuzz_liboss_fuzz_vrs_a_SOURCES = \ $(src_test_fuzz_fuzz_vrs_SOURCES) src_test_fuzz_liboss_fuzz_vrs_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) @@ -425,6 +453,7 @@ OSS_FUZZ_FUZZERS = \ src/test/fuzz/liboss-fuzz-iptsv2.a \ src/test/fuzz/liboss-fuzz-microdesc.a \ src/test/fuzz/liboss-fuzz-socks.a \ + src/test/fuzz/liboss-fuzz-strops.a \ src/test/fuzz/liboss-fuzz-vrs.a else diff --git a/src/test/include.am b/src/test/include.am index ecb7689579..d585c2a38a 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -65,10 +65,11 @@ noinst_PROGRAMS+= \ src/test/test \ src/test/test-slow \ src/test/test-memwipe \ - src/test/test-child \ + src/test/test-process \ src/test/test_workqueue \ src/test/test-switch-id \ - src/test/test-timers + src/test/test-timers \ + src/test/test-rng endif src_test_AM_CPPFLAGS = -DSHARE_DATADIR="\"$(datadir)\"" \ @@ -94,12 +95,14 @@ src_test_test_SOURCES += \ src/test/test_address.c \ src/test/test_address_set.c \ src/test/test_bridges.c \ + src/test/test_btrack.c \ src/test/test_buffers.c \ src/test/test_bwmgt.c \ src/test/test_cell_formats.c \ src/test/test_cell_queue.c \ src/test/test_channel.c \ src/test/test_channelpadding.c \ + src/test/test_circuitpadding.c \ src/test/test_channeltls.c \ src/test/test_checkdir.c \ src/test/test_circuitlist.c \ @@ -118,6 +121,7 @@ src_test_test_SOURCES += \ src/test/test_controller_events.c \ src/test/test_crypto.c \ src/test/test_crypto_ope.c \ + src/test/test_crypto_rng.c \ src/test/test_data.c \ src/test/test_dir.c \ src/test/test_dir_common.c \ @@ -146,6 +150,7 @@ src_test_test_SOURCES += \ src/test/test_logging.c \ src/test/test_mainloop.c \ src/test/test_microdesc.c \ + src/test/test_netinfo.c \ src/test/test_nodelist.c \ src/test/test_oom.c \ src/test/test_oos.c \ @@ -153,6 +158,8 @@ src_test_test_SOURCES += \ src/test/test_pem.c \ src/test/test_periodic_event.c \ src/test/test_policy.c \ + src/test/test_process.c \ + src/test/test_prob_distr.c \ src/test/test_procmon.c \ src/test/test_proto_http.c \ src/test/test_proto_misc.c \ @@ -177,10 +184,12 @@ src_test_test_SOURCES += \ src/test/test_util.c \ src/test/test_util_format.c \ src/test/test_util_process.c \ + src/test/test_voting_flags.c \ src/test/test_voting_schedule.c \ src/test/test_x509.c \ src/test/test_helpers.c \ src/test/test_dns.c \ + src/test/test_parsecommon.c \ src/test/testing_common.c \ src/test/testing_rsakeys.c \ src/ext/tinytest.c @@ -200,7 +209,8 @@ if UNITTESTS_ENABLED src_test_test_slow_SOURCES += \ src/test/test_slow.c \ src/test/test_crypto_slow.c \ - src/test/test_util_slow.c \ + src/test/test_process_slow.c \ + src/test/test_prob_distr.c \ src/test/testing_common.c \ src/test/testing_rsakeys.c \ src/ext/tinytest.c @@ -251,6 +261,12 @@ src_test_test_slow_CFLAGS = $(src_test_test_CFLAGS) src_test_test_slow_LDADD = $(src_test_test_LDADD) src_test_test_slow_LDFLAGS = $(src_test_test_LDFLAGS) +src_test_test_rng_CPPFLAGS = $(src_test_test_CPPFLAGS) +src_test_test_rng_CFLAGS = $(src_test_test_CFLAGS) +src_test_test_rng_SOURCES = src/test/test_rng.c +src_test_test_rng_LDFLAGS = $(src_test_test_LDFLAGS) +src_test_test_rng_LDADD = $(src_test_test_LDADD) + src_test_test_memwipe_CPPFLAGS = $(src_test_test_CPPFLAGS) # Don't use bugtrap cflags here: memwipe tests require memory violations. src_test_test_memwipe_CFLAGS = $(TEST_CFLAGS) diff --git a/src/test/prob_distr_mpfr_ref.c b/src/test/prob_distr_mpfr_ref.c new file mode 100644 index 0000000000..425733dc1b --- /dev/null +++ b/src/test/prob_distr_mpfr_ref.c @@ -0,0 +1,64 @@ +/* Copyright 2012-2019, The Tor Project, Inc + * See LICENSE for licensing information */ + +/** prob_distr_mpfr_ref.c + * + * Example reference file for GNU MPFR vectors tested in test_prob_distr.c . + * Code by Riastradh. + */ + +#include <complex.h> +#include <float.h> +#include <math.h> +#include <stdio.h> + +/* Must come after <stdio.h> so we get mpfr_printf. */ +#include <mpfr.h> + +/* gcc -o mpfr prob_distr_mpfr_ref.c -lmpfr -lm */ + +/* Computes logit(p) for p = .49999 */ +int +main(void) +{ + mpfr_t p, q, r; + mpfr_init(p); + mpfr_set_prec(p, 200); + mpfr_init(q); + mpfr_set_prec(q, 200); + mpfr_init(r); + mpfr_set_prec(r, 200); + mpfr_set_d(p, .49999, MPFR_RNDN); + mpfr_set_d(q, 1, MPFR_RNDN); + /* r := q - p = 1 - p */ + mpfr_sub(r, q, p, MPFR_RNDN); + /* q := p/r = p/(1 - p) */ + mpfr_div(q, p, r, MPFR_RNDN); + /* r := log(q) = log(p/(1 - p)) */ + mpfr_log(r, q, MPFR_RNDN); + mpfr_printf("mpfr 200-bit\t%.128Rg\n", r); + + /* + * Print a double approximation to logit three different ways. All + * three agree bit for bit on the libms I tried, with the nextafter + * adjustment (which is well within the 10 eps relative error bound + * advertised). Apparently I must have used the Goldberg expression + * for what I wrote down in the test case. + */ + printf("mpfr 53-bit\t%.17g\n", nextafter(mpfr_get_d(r, MPFR_RNDN), 0), 0); + volatile double p0 = .49999; + printf("log1p\t\t%.17g\n", nextafter(-log1p((1 - 2*p0)/p0), 0)); + volatile double x = (1 - 2*p0)/p0; + volatile double xp1 = x + 1; + printf("Goldberg\t%.17g\n", -x*log(xp1)/(xp1 - 1)); + + /* + * Print a bad approximation, using the naive expression, to see a + * lot of wrong digits, far beyond the 10 eps relative error attained + * by -log1p((1 - 2*p)/p). + */ + printf("naive\t\t%.17g\n", log(p0/(1 - p0))); + + fflush(stdout); + return ferror(stdout); +} diff --git a/src/test/test-child.c b/src/test/test-child.c deleted file mode 100644 index 11a1695cad..0000000000 --- a/src/test/test-child.c +++ /dev/null @@ -1,61 +0,0 @@ -/* Copyright (c) 2011-2019, The Tor Project, Inc. */ -/* See LICENSE for licensing information */ - -#include "orconfig.h" -#include <stdio.h> -#ifdef _WIN32 -#define WINDOWS_LEAN_AND_MEAN -#include <windows.h> -#else -#include <unistd.h> -#endif /* defined(_WIN32) */ -#include <string.h> - -#ifdef _WIN32 -#define SLEEP(sec) Sleep((sec)*1000) -#else -#define SLEEP(sec) sleep(sec) -#endif - -/** Trivial test program which prints out its command line arguments so we can - * check if tor_spawn_background() works */ -int -main(int argc, char **argv) -{ - int i; - int delay = 1; - int fast = 0; - - if (argc > 1) { - if (!strcmp(argv[1], "--hang")) { - delay = 60; - } else if (!strcmp(argv[1], "--fast")) { - fast = 1; - delay = 0; - } - } - - fprintf(stdout, "OUT\n"); - fprintf(stderr, "ERR\n"); - for (i = 1; i < argc; i++) - fprintf(stdout, "%s\n", argv[i]); - if (!fast) - fprintf(stdout, "SLEEPING\n"); - /* We need to flush stdout so that test_util_spawn_background_partial_read() - succeed. Otherwise ReadFile() will get the entire output in one */ - // XXX: Can we make stdio flush on newline? - fflush(stdout); - if (!fast) - SLEEP(1); - fprintf(stdout, "DONE\n"); - fflush(stdout); - if (fast) - return 0; - - while (--delay) { - SLEEP(1); - } - - return 0; -} - diff --git a/src/test/test-process.c b/src/test/test-process.c new file mode 100644 index 0000000000..eb28ad90e9 --- /dev/null +++ b/src/test/test-process.c @@ -0,0 +1,85 @@ +/* Copyright (c) 2011-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#include <stdio.h> +#ifdef _WIN32 +#define WINDOWS_LEAN_AND_MEAN +#include <windows.h> +#else +#include <unistd.h> +#endif /* defined(_WIN32) */ +#include <string.h> +#include <stdlib.h> + +#ifdef _WIN32 +#define SLEEP(sec) Sleep((sec)*1000) +#else +#define SLEEP(sec) sleep(sec) +#endif + +/* Trivial test program to test process_t. */ +int +main(int argc, char **argv) +{ + /* Does our process get the right arguments? */ + for (int i = 0; i < argc; ++i) { + fprintf(stdout, "argv[%d] = '%s'\n", i, argv[i]); + fflush(stdout); + } + + /* Make sure our process got our environment variable. */ + fprintf(stdout, "Environment variable TOR_TEST_ENV = '%s'\n", + getenv("TOR_TEST_ENV")); + fflush(stdout); + + /* Test line handling on stdout and stderr. */ + fprintf(stdout, "Output on stdout\nThis is a new line\n"); + fflush(stdout); + + fprintf(stderr, "Output on stderr\nThis is a new line\n"); + fflush(stderr); + + fprintf(stdout, "Partial line on stdout ..."); + fflush(stdout); + + fprintf(stderr, "Partial line on stderr ..."); + fflush(stderr); + + SLEEP(2); + + fprintf(stdout, "end of partial line on stdout\n"); + fflush(stdout); + fprintf(stderr, "end of partial line on stderr\n"); + fflush(stderr); + + /* Echo input from stdin. */ + char buffer[1024]; + + int count = 0; + + while (fgets(buffer, sizeof(buffer), stdin)) { + /* Strip the newline. */ + size_t size = strlen(buffer); + + if (size >= 1 && buffer[size - 1] == '\n') { + buffer[size - 1] = '\0'; + --size; + } + + if (size >= 1 && buffer[size - 1] == '\r') { + buffer[size - 1] = '\0'; + --size; + } + + fprintf(stdout, "Read line from stdin: '%s'\n", buffer); + fflush(stdout); + + if (++count == 3) + break; + } + + fprintf(stdout, "We are done for here, thank you!\n"); + + return 0; +} diff --git a/src/test/test.c b/src/test/test.c index 58b468775c..25e9da5591 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -37,7 +37,7 @@ #include "core/or/or.h" #include "lib/err/backtrace.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/or/circuitlist.h" #include "core/or/circuitstats.h" #include "lib/compress/compress.h" @@ -845,10 +845,11 @@ struct testgroup_t testgroups[] = { { "channeltls/", channeltls_tests }, { "checkdir/", checkdir_tests }, { "circuitbuild/", circuitbuild_tests }, + { "circuitpadding/", circuitpadding_tests }, { "circuitlist/", circuitlist_tests }, { "circuitmux/", circuitmux_tests }, - { "circuituse/", circuituse_tests }, { "circuitstats/", circuitstats_tests }, + { "circuituse/", circuituse_tests }, { "compat/libevent/", compat_libevent_tests }, { "config/", config_tests }, { "connection/", connection_tests }, @@ -857,6 +858,7 @@ struct testgroup_t testgroups[] = { { "consdiffmgr/", consdiffmgr_tests }, { "container/", container_tests }, { "control/", controller_tests }, + { "control/btrack/", btrack_tests }, { "control/event/", controller_event_tests }, { "crypto/", crypto_tests }, { "crypto/ope/", crypto_ope_tests }, @@ -864,38 +866,45 @@ struct testgroup_t testgroups[] = { { "crypto/openssl/", crypto_openssl_tests }, #endif { "crypto/pem/", pem_tests }, + { "crypto/rng/", crypto_rng_tests }, { "dir/", dir_tests }, - { "dir_handle_get/", dir_handle_get_tests }, { "dir/md/", microdesc_tests }, - { "dir/voting-schedule/", voting_schedule_tests }, + { "dir/voting/flags/", voting_flags_tests }, + { "dir/voting/schedule/", voting_schedule_tests }, + { "dir_handle_get/", dir_handle_get_tests }, + { "dns/", dns_tests }, { "dos/", dos_tests }, { "entryconn/", entryconn_tests }, { "entrynodes/", entrynodes_tests }, - { "guardfraction/", guardfraction_tests }, { "extorport/", extorport_tests }, { "geoip/", geoip_tests }, - { "legacy_hs/", hs_tests }, + { "guardfraction/", guardfraction_tests }, { "hs_cache/", hs_cache }, { "hs_cell/", hs_cell_tests }, + { "hs_client/", hs_client_tests }, { "hs_common/", hs_common_tests }, { "hs_config/", hs_config_tests }, { "hs_control/", hs_control_tests }, { "hs_descriptor/", hs_descriptor }, + { "hs_intropoint/", hs_intropoint_tests }, { "hs_ntor/", hs_ntor_tests }, { "hs_service/", hs_service_tests }, - { "hs_client/", hs_client_tests }, - { "hs_intropoint/", hs_intropoint_tests }, { "introduce/", introduce_tests }, { "keypin/", keypin_tests }, + { "legacy_hs/", hs_tests }, { "link-handshake/", link_handshake_tests }, { "mainloop/", mainloop_tests }, + { "netinfo/", netinfo_tests }, { "nodelist/", nodelist_tests }, { "oom/", oom_tests }, { "oos/", oos_tests }, { "options/", options_tests }, + { "parsecommon/", parsecommon_tests }, { "periodic-event/" , periodic_event_tests }, { "policy/" , policy_tests }, + { "prob_distr/", prob_distr_tests }, { "procmon/", procmon_tests }, + { "process/", process_tests }, { "proto/http/", proto_http_tests }, { "proto/misc/", proto_misc_tests }, { "protover/", protover_tests }, @@ -910,8 +919,8 @@ struct testgroup_t testgroups[] = { { "routerlist/", routerlist_tests }, { "routerset/" , routerset_tests }, { "scheduler/", scheduler_tests }, - { "socks/", socks_tests }, { "shared-random/", sr_tests }, + { "socks/", socks_tests }, { "status/" , status_tests }, { "storagedir/", storagedir_tests }, { "tortls/", tortls_tests }, @@ -921,10 +930,9 @@ struct testgroup_t testgroups[] = { { "tortls/x509/", x509_tests }, { "util/", util_tests }, { "util/format/", util_format_tests }, + { "util/handle/", handle_tests }, { "util/logging/", logging_tests }, { "util/process/", util_process_tests }, { "util/thread/", thread_tests }, - { "util/handle/", handle_tests }, - { "dns/", dns_tests }, END_OF_GROUPS }; diff --git a/src/test/test.h b/src/test/test.h index aacc9dba87..2564432985 100644 --- a/src/test/test.h +++ b/src/test/test.h @@ -177,22 +177,24 @@ extern const struct testcase_setup_t ed25519_test_setup; extern struct testcase_t accounting_tests[]; extern struct testcase_t addr_tests[]; -extern struct testcase_t address_tests[]; extern struct testcase_t address_set_tests[]; +extern struct testcase_t address_tests[]; extern struct testcase_t bridges_tests[]; -extern struct testcase_t bwmgt_tests[]; +extern struct testcase_t btrack_tests[]; extern struct testcase_t buffer_tests[]; +extern struct testcase_t bwmgt_tests[]; extern struct testcase_t cell_format_tests[]; extern struct testcase_t cell_queue_tests[]; extern struct testcase_t channel_tests[]; extern struct testcase_t channelpadding_tests[]; +extern struct testcase_t circuitpadding_tests[]; extern struct testcase_t channeltls_tests[]; extern struct testcase_t checkdir_tests[]; extern struct testcase_t circuitbuild_tests[]; extern struct testcase_t circuitlist_tests[]; extern struct testcase_t circuitmux_tests[]; -extern struct testcase_t circuituse_tests[]; extern struct testcase_t circuitstats_tests[]; +extern struct testcase_t circuituse_tests[]; extern struct testcase_t compat_libevent_tests[]; extern struct testcase_t config_tests[]; extern struct testcase_t connection_tests[]; @@ -200,44 +202,52 @@ extern struct testcase_t conscache_tests[]; extern struct testcase_t consdiff_tests[]; extern struct testcase_t consdiffmgr_tests[]; extern struct testcase_t container_tests[]; -extern struct testcase_t controller_tests[]; extern struct testcase_t controller_event_tests[]; -extern struct testcase_t crypto_tests[]; +extern struct testcase_t controller_tests[]; extern struct testcase_t crypto_ope_tests[]; extern struct testcase_t crypto_openssl_tests[]; -extern struct testcase_t dir_tests[]; +extern struct testcase_t crypto_rng_tests[]; +extern struct testcase_t crypto_tests[]; extern struct testcase_t dir_handle_get_tests[]; +extern struct testcase_t dir_tests[]; +extern struct testcase_t dns_tests[]; extern struct testcase_t dos_tests[]; extern struct testcase_t entryconn_tests[]; extern struct testcase_t entrynodes_tests[]; -extern struct testcase_t guardfraction_tests[]; extern struct testcase_t extorport_tests[]; extern struct testcase_t geoip_tests[]; -extern struct testcase_t hs_tests[]; +extern struct testcase_t guardfraction_tests[]; +extern struct testcase_t handle_tests[]; extern struct testcase_t hs_cache[]; extern struct testcase_t hs_cell_tests[]; +extern struct testcase_t hs_client_tests[]; extern struct testcase_t hs_common_tests[]; extern struct testcase_t hs_config_tests[]; extern struct testcase_t hs_control_tests[]; extern struct testcase_t hs_descriptor[]; +extern struct testcase_t hs_intropoint_tests[]; extern struct testcase_t hs_ntor_tests[]; extern struct testcase_t hs_service_tests[]; -extern struct testcase_t hs_client_tests[]; -extern struct testcase_t hs_intropoint_tests[]; +extern struct testcase_t hs_tests[]; extern struct testcase_t introduce_tests[]; extern struct testcase_t keypin_tests[]; extern struct testcase_t link_handshake_tests[]; extern struct testcase_t logging_tests[]; extern struct testcase_t mainloop_tests[]; extern struct testcase_t microdesc_tests[]; +extern struct testcase_t netinfo_tests[]; extern struct testcase_t nodelist_tests[]; extern struct testcase_t oom_tests[]; extern struct testcase_t oos_tests[]; extern struct testcase_t options_tests[]; +extern struct testcase_t parsecommon_tests[]; extern struct testcase_t pem_tests[]; extern struct testcase_t periodic_event_tests[]; extern struct testcase_t policy_tests[]; +extern struct testcase_t prob_distr_tests[]; +extern struct testcase_t slow_stochastic_prob_distr_tests[]; extern struct testcase_t procmon_tests[]; +extern struct testcase_t process_tests[]; extern struct testcase_t proto_http_tests[]; extern struct testcase_t proto_misc_tests[]; extern struct testcase_t protover_tests[]; @@ -252,23 +262,22 @@ extern struct testcase_t routerkeys_tests[]; extern struct testcase_t routerlist_tests[]; extern struct testcase_t routerset_tests[]; extern struct testcase_t scheduler_tests[]; -extern struct testcase_t storagedir_tests[]; extern struct testcase_t socks_tests[]; +extern struct testcase_t sr_tests[]; extern struct testcase_t status_tests[]; +extern struct testcase_t storagedir_tests[]; extern struct testcase_t thread_tests[]; -extern struct testcase_t tortls_tests[]; extern struct testcase_t tortls_openssl_tests[]; -extern struct testcase_t util_tests[]; +extern struct testcase_t tortls_tests[]; extern struct testcase_t util_format_tests[]; extern struct testcase_t util_process_tests[]; +extern struct testcase_t util_tests[]; +extern struct testcase_t voting_flags_tests[]; extern struct testcase_t voting_schedule_tests[]; -extern struct testcase_t dns_tests[]; -extern struct testcase_t handle_tests[]; -extern struct testcase_t sr_tests[]; extern struct testcase_t x509_tests[]; extern struct testcase_t slow_crypto_tests[]; -extern struct testcase_t slow_util_tests[]; +extern struct testcase_t slow_process_tests[]; extern struct testgroup_t testgroups[]; diff --git a/src/test/test_addr.c b/src/test/test_addr.c index 8868edce25..fb8df5f0fb 100644 --- a/src/test/test_addr.c +++ b/src/test/test_addr.c @@ -723,7 +723,7 @@ test_addr_ip6_helpers(void *arg) ; } -/** Test tor_addr_port_parse(). */ +/** Test tor_addr_parse() and tor_addr_port_parse(). */ static void test_addr_parse(void *arg) { @@ -734,6 +734,60 @@ test_addr_parse(void *arg) /* Correct call. */ (void)arg; + r= tor_addr_parse(&addr, "192.0.2.1"); + tt_int_op(r,OP_EQ, AF_INET); + tor_addr_to_str(buf, &addr, sizeof(buf), 0); + tt_str_op(buf,OP_EQ, "192.0.2.1"); + + r= tor_addr_parse(&addr, "11:22::33:44"); + tt_int_op(r,OP_EQ, AF_INET6); + tor_addr_to_str(buf, &addr, sizeof(buf), 0); + tt_str_op(buf,OP_EQ, "11:22::33:44"); + + r= tor_addr_parse(&addr, "[11:22::33:44]"); + tt_int_op(r,OP_EQ, AF_INET6); + tor_addr_to_str(buf, &addr, sizeof(buf), 0); + tt_str_op(buf,OP_EQ, "11:22::33:44"); + + r= tor_addr_parse(&addr, "11:22:33:44:55:66:1.2.3.4"); + tt_int_op(r,OP_EQ, AF_INET6); + tor_addr_to_str(buf, &addr, sizeof(buf), 0); + tt_str_op(buf,OP_EQ, "11:22:33:44:55:66:102:304"); + + r= tor_addr_parse(&addr, "11:22::33:44:1.2.3.4"); + tt_int_op(r,OP_EQ, AF_INET6); + tor_addr_to_str(buf, &addr, sizeof(buf), 0); + tt_str_op(buf,OP_EQ, "11:22::33:44:102:304"); + + /* Empty string. */ + r= tor_addr_parse(&addr, ""); + tt_int_op(r,OP_EQ, -1); + + /* Square brackets around IPv4 address. */ + r= tor_addr_parse(&addr, "[192.0.2.1]"); + tt_int_op(r,OP_EQ, -1); + + /* Only left square bracket. */ + r= tor_addr_parse(&addr, "[11:22::33:44"); + tt_int_op(r,OP_EQ, -1); + + /* Only right square bracket. */ + r= tor_addr_parse(&addr, "11:22::33:44]"); + tt_int_op(r,OP_EQ, -1); + + /* Leading colon. */ + r= tor_addr_parse(&addr, ":11:22::33:44"); + tt_int_op(r,OP_EQ, -1); + + /* Trailing colon. */ + r= tor_addr_parse(&addr, "11:22::33:44:"); + tt_int_op(r,OP_EQ, -1); + + /* Too many hex words in IPv4-mapped IPv6 address. */ + r= tor_addr_parse(&addr, "11:22:33:44:55:66:77:88:1.2.3.4"); + tt_int_op(r,OP_EQ, -1); + + /* Correct call. */ r= tor_addr_port_parse(LOG_DEBUG, "192.0.2.1:1234", &addr, &port, -1); diff --git a/src/test/test_address.c b/src/test/test_address.c index c33c30aee5..bf9ca047dc 100644 --- a/src/test/test_address.c +++ b/src/test/test_address.c @@ -24,6 +24,8 @@ #endif /* defined(HAVE_IFCONF_TO_SMARTLIST) */ #include "core/or/or.h" +#include "feature/nodelist/routerinfo_st.h" +#include "feature/nodelist/node_st.h" #include "feature/nodelist/nodelist.h" #include "lib/net/address.h" #include "test/test.h" @@ -1170,6 +1172,78 @@ test_address_tor_addr_in_same_network_family(void *ignored) return; } +static node_t * +helper_create_mock_node(char id_char) +{ + node_t *node = tor_malloc_zero(sizeof(node_t)); + routerinfo_t *ri = tor_malloc_zero(sizeof(routerinfo_t)); + tor_addr_make_null(&ri->ipv6_addr, AF_INET6); + node->ri = ri; + memset(node->identity, id_char, sizeof(node->identity)); + return node; +} + +static void +helper_free_mock_node(node_t *node) +{ + if (!node) + return; + tor_free(node->ri); + tor_free(node); +} + +#define NODE_SET_IPV4(node, ipv4_addr, ipv4_port) { \ + tor_addr_t addr; \ + tor_addr_parse(&addr, ipv4_addr); \ + node->ri->addr = tor_addr_to_ipv4h(&addr); \ + node->ri->or_port = ipv4_port; \ + } + +#define NODE_CLEAR_IPV4(node) { \ + node->ri->addr = 0; \ + node->ri->or_port = 0; \ + } + +#define NODE_SET_IPV6(node, ipv6_addr_str, ipv6_port) { \ + tor_addr_parse(&node->ri->ipv6_addr, ipv6_addr_str); \ + node->ri->ipv6_orport = ipv6_port; \ + } + +static void +test_address_tor_node_in_same_network_family(void *ignored) +{ + (void)ignored; + node_t *node_a = helper_create_mock_node('a'); + node_t *node_b = helper_create_mock_node('b'); + + NODE_SET_IPV4(node_a, "8.8.8.8", 1); + NODE_SET_IPV4(node_b, "8.8.4.4", 1); + + tt_int_op(nodes_in_same_family(node_a, node_b), OP_EQ, 1); + + NODE_SET_IPV4(node_a, "8.8.8.8", 1); + NODE_SET_IPV4(node_b, "1.1.1.1", 1); + + tt_int_op(nodes_in_same_family(node_a, node_b), OP_EQ, 0); + + NODE_CLEAR_IPV4(node_a); + NODE_SET_IPV6(node_a, "2001:470:20::2", 1); + + tt_int_op(nodes_in_same_family(node_a, node_b), OP_EQ, 0); + + NODE_CLEAR_IPV4(node_b); + NODE_SET_IPV6(node_b, "2606:4700:4700::1111", 1); + + tt_int_op(nodes_in_same_family(node_a, node_b), OP_EQ, 0); + + NODE_SET_IPV6(node_a, "2606:4700:4700::1001", 1); + tt_int_op(nodes_in_same_family(node_a, node_b), OP_EQ, 1); + + done: + helper_free_mock_node(node_a); + helper_free_mock_node(node_b); +} + #define ADDRESS_TEST(name, flags) \ { #name, test_address_ ## name, flags, NULL, NULL } @@ -1202,5 +1276,6 @@ struct testcase_t address_tests[] = { ADDRESS_TEST(tor_addr_to_mapped_ipv4h, 0), ADDRESS_TEST(tor_addr_eq_ipv4h, 0), ADDRESS_TEST(tor_addr_in_same_network_family, 0), + ADDRESS_TEST(tor_node_in_same_network_family, 0), END_OF_TESTCASES }; diff --git a/src/test/test_btrack.c b/src/test/test_btrack.c new file mode 100644 index 0000000000..48486fb5a1 --- /dev/null +++ b/src/test/test_btrack.c @@ -0,0 +1,100 @@ +/* Copyright (c) 2013-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "core/or/or.h" + +#include "test/test.h" +#include "test/log_test_helpers.h" + +#define OCIRC_EVENT_PRIVATE +#define ORCONN_EVENT_PRIVATE +#include "core/or/ocirc_event.h" +#include "core/or/orconn_event.h" + +static void +test_btrack_launch(void *arg) +{ + orconn_event_msg_t conn; + ocirc_event_msg_t circ; + + (void)arg; + conn.type = ORCONN_MSGTYPE_STATE; + conn.u.state.gid = 1; + conn.u.state.chan = 1; + conn.u.state.proxy_type = PROXY_NONE; + conn.u.state.state = OR_CONN_STATE_CONNECTING; + + setup_full_capture_of_logs(LOG_DEBUG); + orconn_event_publish(&conn); + expect_log_msg_containing("ORCONN gid=1 chan=1 proxy_type=0 state=1"); + expect_no_log_msg_containing("ORCONN BEST_"); + teardown_capture_of_logs(); + + circ.type = OCIRC_MSGTYPE_CHAN; + circ.u.chan.chan = 1; + circ.u.chan.onehop = true; + + setup_full_capture_of_logs(LOG_DEBUG); + ocirc_event_publish(&circ); + expect_log_msg_containing("ORCONN LAUNCH chan=1 onehop=1"); + expect_log_msg_containing("ORCONN BEST_ANY state -1->1 gid=1"); + teardown_capture_of_logs(); + + conn.u.state.gid = 2; + conn.u.state.chan = 2; + + setup_full_capture_of_logs(LOG_DEBUG); + orconn_event_publish(&conn); + expect_log_msg_containing("ORCONN gid=2 chan=2 proxy_type=0 state=1"); + expect_no_log_msg_containing("ORCONN BEST_"); + teardown_capture_of_logs(); + + circ.u.chan.chan = 2; + circ.u.chan.onehop = false; + + setup_full_capture_of_logs(LOG_DEBUG); + ocirc_event_publish(&circ); + expect_log_msg_containing("ORCONN LAUNCH chan=2 onehop=0"); + expect_log_msg_containing("ORCONN BEST_AP state -1->1 gid=2"); + teardown_capture_of_logs(); + + done: + ; +} + +static void +test_btrack_delete(void *arg) +{ + orconn_event_msg_t conn; + + (void)arg; + conn.type = ORCONN_MSGTYPE_STATE; + conn.u.state.gid = 1; + conn.u.state.chan = 1; + conn.u.state.proxy_type = PROXY_NONE; + conn.u.state.state = OR_CONN_STATE_CONNECTING; + + setup_full_capture_of_logs(LOG_DEBUG); + orconn_event_publish(&conn); + expect_log_msg_containing("ORCONN gid=1 chan=1 proxy_type=0"); + teardown_capture_of_logs(); + + conn.type = ORCONN_MSGTYPE_STATUS; + conn.u.status.gid = 1; + conn.u.status.status = OR_CONN_EVENT_CLOSED; + conn.u.status.reason = 0; + + setup_full_capture_of_logs(LOG_DEBUG); + orconn_event_publish(&conn); + expect_log_msg_containing("ORCONN DELETE gid=1 status=3 reason=0"); + teardown_capture_of_logs(); + + done: + ; +} + +struct testcase_t btrack_tests[] = { + { "launch", test_btrack_launch, TT_FORK, 0, NULL }, + { "delete", test_btrack_delete, TT_FORK, 0, NULL }, + END_OF_TESTCASES +}; diff --git a/src/test/test_buffers.c b/src/test/test_buffers.c index 2f9ad1fbe3..97311c85cc 100644 --- a/src/test/test_buffers.c +++ b/src/test/test_buffers.c @@ -6,7 +6,7 @@ #define BUFFERS_PRIVATE #define PROTO_HTTP_PRIVATE #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/tls/buffers_tls.h" #include "lib/tls/tortls.h" #include "lib/compress/compress.h" diff --git a/src/test/test_channelpadding.c b/src/test/test_channelpadding.c index 5da2d81377..5d012e462b 100644 --- a/src/test/test_channelpadding.c +++ b/src/test/test_channelpadding.c @@ -21,7 +21,7 @@ #include "test/log_test_helpers.h" #include "lib/tls/tortls.h" #include "lib/evloop/timers.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/or/cell_st.h" #include "feature/nodelist/networkstatus_st.h" diff --git a/src/test/test_channeltls.c b/src/test/test_channeltls.c index 10513e451d..054d3910e4 100644 --- a/src/test/test_channeltls.c +++ b/src/test/test_channeltls.c @@ -8,7 +8,7 @@ #define TOR_CHANNEL_INTERNAL_ #include "core/or/or.h" #include "lib/net/address.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/or/channel.h" #include "core/or/channeltls.h" #include "core/mainloop/connection.h" diff --git a/src/test/test_circuitbuild.c b/src/test/test_circuitbuild.c index 538f20781f..ccbaa02552 100644 --- a/src/test/test_circuitbuild.c +++ b/src/test/test_circuitbuild.c @@ -27,11 +27,11 @@ static smartlist_t dummy_nodes; static extend_info_t dummy_ei; static int -mock_count_acceptable_nodes(smartlist_t *nodes) +mock_count_acceptable_nodes(smartlist_t *nodes, int direct) { (void)nodes; - return DEFAULT_ROUTE_LEN + 1; + return direct ? 1 : DEFAULT_ROUTE_LEN + 1; } /* Test route lengths when the caller of new_route_len() doesn't diff --git a/src/test/test_circuitpadding.c b/src/test/test_circuitpadding.c new file mode 100644 index 0000000000..09a4c9a0ca --- /dev/null +++ b/src/test/test_circuitpadding.c @@ -0,0 +1,2128 @@ +#define TOR_CHANNEL_INTERNAL_ +#define TOR_TIMERS_PRIVATE +#define CIRCUITPADDING_PRIVATE +#define NETWORKSTATUS_PRIVATE + +#include "core/or/or.h" +#include "test.h" +#include "lib/testsupport/testsupport.h" +#include "core/or/connection_or.h" +#include "core/or/channel.h" +#include "core/or/channeltls.h" +#include <event.h> +#include "lib/evloop/compat_libevent.h" +#include "lib/time/compat_time.h" +#include "lib/defs/time.h" +#include "core/or/relay.h" +#include "core/or/circuitlist.h" +#include "core/or/circuitbuild.h" +#include "core/or/circuitpadding.h" +#include "core/crypto/relay_crypto.h" +#include "core/or/protover.h" +#include "feature/nodelist/nodelist.h" +#include "lib/evloop/compat_libevent.h" +#include "app/config/config.h" + +#include "feature/nodelist/routerstatus_st.h" +#include "feature/nodelist/networkstatus_st.h" +#include "feature/nodelist/node_st.h" +#include "core/or/cell_st.h" +#include "core/or/crypt_path_st.h" +#include "core/or/or_circuit_st.h" +#include "core/or/origin_circuit_st.h" + +/* Start our monotime mocking at 1 second past whatever monotime_init() + * thought the actual wall clock time was, for platforms with bad resolution + * and weird timevalues during monotime_init() before mocking. */ +#define MONOTIME_MOCK_START (monotime_absolute_nsec()+\ + TOR_NSEC_PER_USEC*TOR_USEC_PER_SEC) + +extern smartlist_t *connection_array; + +circid_t get_unique_circ_id_by_chan(channel_t *chan); +void helper_create_basic_machine(void); +static void helper_create_conditional_machines(void); + +static or_circuit_t * new_fake_orcirc(channel_t *nchan, channel_t *pchan); +channel_t *new_fake_channel(void); +void test_circuitpadding_negotiation(void *arg); +void test_circuitpadding_wronghop(void *arg); +void test_circuitpadding_conditions(void *arg); + +void test_circuitpadding_serialize(void *arg); +void test_circuitpadding_rtt(void *arg); +void test_circuitpadding_tokens(void *arg); + +static void +simulate_single_hop_extend(circuit_t *client, circuit_t *mid_relay, + int padding); +void free_fake_orcirc(circuit_t *circ); +void free_fake_origin_circuit(origin_circuit_t *circ); + +static int deliver_negotiated = 1; +static int64_t curr_mocked_time; + +static node_t padding_node; +static node_t non_padding_node; + +static channel_t dummy_channel; +static circpad_machine_spec_t circ_client_machine; + +static void +timers_advance_and_run(int64_t msec_update) +{ + curr_mocked_time += msec_update*TOR_NSEC_PER_MSEC; + monotime_coarse_set_mock_time_nsec(curr_mocked_time); + monotime_set_mock_time_nsec(curr_mocked_time); + timers_run_pending(); +} + +static void +nodes_init(void) +{ + padding_node.rs = tor_malloc_zero(sizeof(routerstatus_t)); + padding_node.rs->pv.supports_padding = 1; + + non_padding_node.rs = tor_malloc_zero(sizeof(routerstatus_t)); + non_padding_node.rs->pv.supports_padding = 0; +} + +static void +nodes_free(void) +{ + tor_free(padding_node.rs); + + tor_free(non_padding_node.rs); +} + +static const node_t * +node_get_by_id_mock(const char *identity_digest) +{ + if (identity_digest[0] == 1) { + return &padding_node; + } else if (identity_digest[0] == 0) { + return &non_padding_node; + } + + return NULL; +} + +static or_circuit_t * +new_fake_orcirc(channel_t *nchan, channel_t *pchan) +{ + or_circuit_t *orcirc = NULL; + circuit_t *circ = NULL; + crypt_path_t tmp_cpath; + char whatevs_key[CPATH_KEY_MATERIAL_LEN]; + + orcirc = tor_malloc_zero(sizeof(*orcirc)); + circ = &(orcirc->base_); + circ->magic = OR_CIRCUIT_MAGIC; + + //circ->n_chan = nchan; + circ->n_circ_id = get_unique_circ_id_by_chan(nchan); + circ->n_mux = NULL; /* ?? */ + cell_queue_init(&(circ->n_chan_cells)); + circ->n_hop = NULL; + circ->streams_blocked_on_n_chan = 0; + circ->streams_blocked_on_p_chan = 0; + circ->n_delete_pending = 0; + circ->p_delete_pending = 0; + circ->received_destroy = 0; + circ->state = CIRCUIT_STATE_OPEN; + circ->purpose = CIRCUIT_PURPOSE_OR; + circ->package_window = CIRCWINDOW_START_MAX; + circ->deliver_window = CIRCWINDOW_START_MAX; + circ->n_chan_create_cell = NULL; + + //orcirc->p_chan = pchan; + orcirc->p_circ_id = get_unique_circ_id_by_chan(pchan); + cell_queue_init(&(orcirc->p_chan_cells)); + + circuit_set_p_circid_chan(orcirc, orcirc->p_circ_id, pchan); + circuit_set_n_circid_chan(circ, circ->n_circ_id, nchan); + + memset(&tmp_cpath, 0, sizeof(tmp_cpath)); + if (circuit_init_cpath_crypto(&tmp_cpath, whatevs_key, + sizeof(whatevs_key), 0, 0)<0) { + log_warn(LD_BUG,"Circuit initialization failed"); + return NULL; + } + orcirc->crypto = tmp_cpath.crypto; + + return orcirc; +} + +void +free_fake_orcirc(circuit_t *circ) +{ + or_circuit_t *orcirc = TO_OR_CIRCUIT(circ); + + relay_crypto_clear(&orcirc->crypto); + + circpad_circuit_free_all_machineinfos(circ); + tor_free(circ); +} + +void +free_fake_origin_circuit(origin_circuit_t *circ) +{ + circpad_circuit_free_all_machineinfos(TO_CIRCUIT(circ)); + circuit_clear_cpath(circ); + tor_free(circ); +} + +void dummy_nop_timer(void); + +//static int dont_stop_libevent = 0; + +static circuit_t *client_side; +static circuit_t *relay_side; + +static int n_client_cells = 0; +static int n_relay_cells = 0; + +static int +circuit_package_relay_cell_mock(cell_t *cell, circuit_t *circ, + cell_direction_t cell_direction, + crypt_path_t *layer_hint, streamid_t on_stream, + const char *filename, int lineno); + +static void +circuitmux_attach_circuit_mock(circuitmux_t *cmux, circuit_t *circ, + cell_direction_t direction); + +static void +circuitmux_attach_circuit_mock(circuitmux_t *cmux, circuit_t *circ, + cell_direction_t direction) +{ + (void)cmux; + (void)circ; + (void)direction; + + return; +} + +static int +circuit_package_relay_cell_mock(cell_t *cell, circuit_t *circ, + cell_direction_t cell_direction, + crypt_path_t *layer_hint, streamid_t on_stream, + const char *filename, int lineno) +{ + (void)cell; (void)on_stream; (void)filename; (void)lineno; + + if (circ == client_side) { + if (cell->payload[0] == RELAY_COMMAND_PADDING_NEGOTIATE) { + // Deliver to relay + circpad_handle_padding_negotiate(relay_side, cell); + } else { + + int is_target_hop = circpad_padding_is_from_expected_hop(circ, + layer_hint); + tt_int_op(cell_direction, OP_EQ, CELL_DIRECTION_OUT); + tt_int_op(is_target_hop, OP_EQ, 1); + + // No need to pretend a padding cell was sent: This event is + // now emitted internally when the circuitpadding code sends them. + //circpad_cell_event_padding_sent(client_side); + + // Receive padding cell at middle + circpad_deliver_recognized_relay_cell_events(relay_side, + cell->payload[0], NULL); + } + n_client_cells++; + } else if (circ == relay_side) { + tt_int_op(cell_direction, OP_EQ, CELL_DIRECTION_IN); + + if (cell->payload[0] == RELAY_COMMAND_PADDING_NEGOTIATED) { + // XXX: blah need right layer_hint.. + if (deliver_negotiated) + circpad_handle_padding_negotiated(client_side, cell, + TO_ORIGIN_CIRCUIT(client_side) + ->cpath->next); + } else if (cell->payload[0] == RELAY_COMMAND_PADDING_NEGOTIATE) { + circpad_handle_padding_negotiate(client_side, cell); + } else { + // No need to pretend a padding cell was sent: This event is + // now emitted internally when the circuitpadding code sends them. + //circpad_cell_event_padding_sent(relay_side); + + // Receive padding cell at client + circpad_deliver_recognized_relay_cell_events(client_side, + cell->payload[0], + TO_ORIGIN_CIRCUIT(client_side)->cpath->next); + } + + n_relay_cells++; + } + + done: + timers_advance_and_run(1); + return 0; +} + +// Test reading and writing padding to strings (or options_t + consensus) +void +test_circuitpadding_serialize(void *arg) +{ + (void)arg; +} + +static signed_error_t +circpad_send_command_to_hop_mock(origin_circuit_t *circ, uint8_t hopnum, + uint8_t relay_command, const uint8_t *payload, + ssize_t payload_len) +{ + (void) circ; + (void) hopnum; + (void) relay_command; + (void) payload; + (void) payload_len; + return 0; +} + +void +test_circuitpadding_rtt(void *arg) +{ + /* Test Plan: + * + * 1. Test RTT measurement server side + * a. test usage of measured RTT + * 2. Test termination of RTT measurement + * a. test non-update of RTT + * 3. Test client side circuit and non-application of RTT.. + */ + circpad_delay_t rtt_estimate; + int64_t actual_mocked_monotime_start; + (void)arg; + + MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock); + MOCK(circpad_send_command_to_hop, circpad_send_command_to_hop_mock); + + dummy_channel.cmux = circuitmux_alloc(); + relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel, &dummy_channel)); + client_side = TO_CIRCUIT(origin_circuit_new()); + relay_side->purpose = CIRCUIT_PURPOSE_OR; + client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; + + monotime_init(); + monotime_enable_test_mocking(); + actual_mocked_monotime_start = MONOTIME_MOCK_START; + monotime_set_mock_time_nsec(actual_mocked_monotime_start); + monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start); + curr_mocked_time = actual_mocked_monotime_start; + + timers_initialize(); + circpad_machines_init(); + helper_create_basic_machine(); + + MOCK(circuit_package_relay_cell, + circuit_package_relay_cell_mock); + + client_side->padding_machine[0] = &circ_client_machine; + client_side->padding_info[0] = circpad_circuit_machineinfo_new(client_side, + 0); + + relay_side->padding_machine[0] = &circ_client_machine; + relay_side->padding_info[0] = circpad_circuit_machineinfo_new(client_side,0); + + /* Test 1: Test measuring RTT */ + circpad_cell_event_nonpadding_received((circuit_t*)relay_side); + tt_u64_op(relay_side->padding_info[0]->last_received_time_usec, OP_NE, 0); + + timers_advance_and_run(20); + + circpad_cell_event_nonpadding_sent((circuit_t*)relay_side); + tt_u64_op(relay_side->padding_info[0]->last_received_time_usec, OP_EQ, 0); + + tt_int_op(relay_side->padding_info[0]->rtt_estimate_usec, OP_GE, 19000); + tt_int_op(relay_side->padding_info[0]->rtt_estimate_usec, OP_LE, 30000); + tt_int_op(circpad_histogram_bin_to_usec(relay_side->padding_info[0], 0), + OP_EQ, + relay_side->padding_info[0]->rtt_estimate_usec+ + circpad_machine_current_state( + relay_side->padding_info[0])->start_usec); + + circpad_cell_event_nonpadding_received((circuit_t*)relay_side); + circpad_cell_event_nonpadding_received((circuit_t*)relay_side); + tt_u64_op(relay_side->padding_info[0]->last_received_time_usec, OP_NE, 0); + timers_advance_and_run(20); + circpad_cell_event_nonpadding_sent((circuit_t*)relay_side); + circpad_cell_event_nonpadding_sent((circuit_t*)relay_side); + tt_u64_op(relay_side->padding_info[0]->last_received_time_usec, OP_EQ, 0); + + tt_int_op(relay_side->padding_info[0]->rtt_estimate_usec, OP_GE, 20000); + tt_int_op(relay_side->padding_info[0]->rtt_estimate_usec, OP_LE, 21000); + tt_int_op(circpad_histogram_bin_to_usec(relay_side->padding_info[0], 0), + OP_EQ, + relay_side->padding_info[0]->rtt_estimate_usec+ + circpad_machine_current_state( + relay_side->padding_info[0])->start_usec); + + /* Test 2: Termination of RTT measurement (from the previous test) */ + tt_int_op(relay_side->padding_info[0]->stop_rtt_update, OP_EQ, 1); + rtt_estimate = relay_side->padding_info[0]->rtt_estimate_usec; + + circpad_cell_event_nonpadding_received((circuit_t*)relay_side); + timers_advance_and_run(4); + circpad_cell_event_nonpadding_sent((circuit_t*)relay_side); + + tt_int_op(relay_side->padding_info[0]->rtt_estimate_usec, OP_EQ, + rtt_estimate); + tt_u64_op(relay_side->padding_info[0]->last_received_time_usec, OP_EQ, 0); + tt_int_op(relay_side->padding_info[0]->stop_rtt_update, OP_EQ, 1); + tt_int_op(circpad_histogram_bin_to_usec(relay_side->padding_info[0], 0), + OP_EQ, + relay_side->padding_info[0]->rtt_estimate_usec+ + circpad_machine_current_state( + relay_side->padding_info[0])->start_usec); + + /* Test 3: Make sure client side machine properly ignores RTT */ + circpad_cell_event_nonpadding_received((circuit_t*)client_side); + tt_u64_op(client_side->padding_info[0]->last_received_time_usec, OP_EQ, 0); + + timers_advance_and_run(20); + circpad_cell_event_nonpadding_sent((circuit_t*)client_side); + tt_u64_op(client_side->padding_info[0]->last_received_time_usec, OP_EQ, 0); + + tt_int_op(client_side->padding_info[0]->rtt_estimate_usec, OP_EQ, 0); + tt_int_op(circpad_histogram_bin_to_usec(client_side->padding_info[0], 0), + OP_NE, client_side->padding_info[0]->rtt_estimate_usec); + tt_int_op(circpad_histogram_bin_to_usec(client_side->padding_info[0], 0), + OP_EQ, + circpad_machine_current_state( + client_side->padding_info[0])->start_usec); + done: + free_fake_orcirc(relay_side); + circuitmux_detach_all_circuits(dummy_channel.cmux, NULL); + circuitmux_free(dummy_channel.cmux); + timers_shutdown(); + monotime_disable_test_mocking(); + UNMOCK(circuit_package_relay_cell); + UNMOCK(circuitmux_attach_circuit); + tor_free(circ_client_machine.states); + + return; +} + +void +helper_create_basic_machine(void) +{ + /* Start, burst */ + circpad_machine_states_init(&circ_client_machine, 2); + + circ_client_machine.states[CIRCPAD_STATE_START]. + next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_BURST; + + circ_client_machine.states[CIRCPAD_STATE_BURST]. + next_state[CIRCPAD_EVENT_PADDING_RECV] = CIRCPAD_STATE_BURST; + circ_client_machine.states[CIRCPAD_STATE_BURST]. + next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_BURST; + + circ_client_machine.states[CIRCPAD_STATE_BURST]. + next_state[CIRCPAD_EVENT_NONPADDING_SENT] = CIRCPAD_STATE_CANCEL; + + // FIXME: Is this what we want? + circ_client_machine.states[CIRCPAD_STATE_BURST].token_removal = + CIRCPAD_TOKEN_REMOVAL_HIGHER; + + // FIXME: Tune this histogram + circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_len = 5; + circ_client_machine.states[CIRCPAD_STATE_BURST].start_usec = 500; + circ_client_machine.states[CIRCPAD_STATE_BURST].range_usec = 1000000; + circ_client_machine.states[CIRCPAD_STATE_BURST].histogram[0] = 1; + circ_client_machine.states[CIRCPAD_STATE_BURST].histogram[1] = 0; + circ_client_machine.states[CIRCPAD_STATE_BURST].histogram[2] = 2; + circ_client_machine.states[CIRCPAD_STATE_BURST].histogram[3] = 2; + circ_client_machine.states[CIRCPAD_STATE_BURST].histogram[4] = 2; + circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_total_tokens = 7; + circ_client_machine.states[CIRCPAD_STATE_BURST].use_rtt_estimate = 1; + + return; +} + +#define BIG_HISTOGRAM_LEN 10 + +/** Setup a machine with a big histogram */ +static void +helper_create_machine_with_big_histogram(circpad_removal_t removal_strategy) +{ + const int tokens_per_bin = 2; + + /* Start, burst */ + circpad_machine_states_init(&circ_client_machine, 2); + + circpad_state_t *burst_state = + &circ_client_machine.states[CIRCPAD_STATE_BURST]; + + circ_client_machine.states[CIRCPAD_STATE_START]. + next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_BURST; + + burst_state->next_state[CIRCPAD_EVENT_PADDING_RECV] = CIRCPAD_STATE_BURST; + burst_state->next_state[CIRCPAD_EVENT_NONPADDING_RECV] =CIRCPAD_STATE_BURST; + + burst_state->next_state[CIRCPAD_EVENT_NONPADDING_SENT] =CIRCPAD_STATE_CANCEL; + + burst_state->token_removal = CIRCPAD_TOKEN_REMOVAL_HIGHER; + + burst_state->histogram_len = BIG_HISTOGRAM_LEN; + burst_state->start_usec = 0; + burst_state->range_usec = 1000; + + int n_tokens = 0; + for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) { + burst_state->histogram[i] = tokens_per_bin; + n_tokens += tokens_per_bin; + } + + burst_state->histogram_total_tokens = n_tokens; + burst_state->length_dist.type = CIRCPAD_DIST_UNIFORM; + burst_state->length_dist.param1 = n_tokens; + burst_state->length_dist.param2 = n_tokens; + burst_state->max_length = n_tokens; + burst_state->length_includes_nonpadding = 1; + burst_state->use_rtt_estimate = 0; + burst_state->token_removal = removal_strategy; +} + +static circpad_decision_t +circpad_machine_schedule_padding_mock(circpad_machine_state_t *mi) +{ + (void)mi; + return 0; +} + +static uint64_t +mock_monotime_absolute_usec(void) +{ + return 100; +} + +/** Test higher token removal strategy by bin */ +static void +test_circuitpadding_token_removal_higher(void *arg) +{ + circpad_machine_state_t *mi; + (void)arg; + + /* Mock it up */ + MOCK(monotime_absolute_usec, mock_monotime_absolute_usec); + MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock); + + /* Setup test environment (time etc.) */ + client_side = (circuit_t *)origin_circuit_new(); + client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; + monotime_enable_test_mocking(); + + /* Create test machine */ + helper_create_machine_with_big_histogram(CIRCPAD_TOKEN_REMOVAL_HIGHER); + client_side->padding_machine[0] = &circ_client_machine; + client_side->padding_info[0] = + circpad_circuit_machineinfo_new(client_side, 0); + + /* move the machine to the right state */ + circpad_cell_event_nonpadding_received((circuit_t*)client_side); + tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, + CIRCPAD_STATE_BURST); + + /* Get the machine and setup tokens */ + mi = client_side->padding_info[0]; + tt_assert(mi); + + /*************************************************************************/ + + uint64_t current_time = monotime_absolute_usec(); + + /* Test left boundaries of each histogram bin: */ + const circpad_delay_t bin_left_bounds[] = + {0, 1, 7, 15, 31, 62, 125, 250, 500, CIRCPAD_DELAY_INFINITE}; + for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) { + tt_uint_op(bin_left_bounds[i], OP_EQ, + circpad_histogram_bin_to_usec(mi, i)); + } + + /* Check that all bins have two tokens right now */ + for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) { + tt_int_op(mi->histogram[i], OP_EQ, 2); + } + + /* This is the right order to remove tokens from this histogram. That is, we + * first remove tokens from the 4th bin since 57 usec is nearest to the 4th + * bin midpoint (31 + (62-31)/2 == 46). Then we remove from the 3rd bin for + * the same reason, then from the 5th, etc. */ + const int bin_removal_order[] = {4, 5, 6, 7, 8}; + unsigned i; + + /* Remove all tokens from all bins apart from the infinity bin */ + for (i = 0; i < sizeof(bin_removal_order)/sizeof(int) ; i++) { + int bin_to_remove = bin_removal_order[i]; + log_debug(LD_GENERAL, "Testing that %d attempt removes %d bin", + i, bin_to_remove); + + tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 2); + + mi->padding_scheduled_at_usec = current_time - 57; + circpad_machine_remove_token(mi); + + tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 1); + + mi->padding_scheduled_at_usec = current_time - 57; + circpad_machine_remove_token(mi); + + /* Test that we cleaned out this bin. Don't do this in the case of the last + bin since the tokens will get refilled */ + if (i != BIG_HISTOGRAM_LEN - 2) { + tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 0); + } + } + + /* Check that all lower bins are not touched */ + for (i=0; i < 4 ; i++) { + tt_int_op(mi->histogram[i], OP_EQ, 2); + } + + /* Test below the lowest bin, for coverage */ + tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, + CIRCPAD_STATE_BURST); + circ_client_machine.states[CIRCPAD_STATE_BURST].start_usec = 100; + mi->padding_scheduled_at_usec = current_time - 1; + circpad_machine_remove_token(mi); + tt_int_op(mi->histogram[0], OP_EQ, 1); + + done: + free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side)); + monotime_disable_test_mocking(); + tor_free(circ_client_machine.states); +} + +/** Test lower token removal strategy by bin */ +static void +test_circuitpadding_token_removal_lower(void *arg) +{ + circpad_machine_state_t *mi; + (void)arg; + + /* Mock it up */ + MOCK(monotime_absolute_usec, mock_monotime_absolute_usec); + MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock); + + /* Setup test environment (time etc.) */ + client_side = (circuit_t *)origin_circuit_new(); + client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; + monotime_enable_test_mocking(); + + /* Create test machine */ + helper_create_machine_with_big_histogram(CIRCPAD_TOKEN_REMOVAL_LOWER); + client_side->padding_machine[0] = &circ_client_machine; + client_side->padding_info[0] = + circpad_circuit_machineinfo_new(client_side, 0); + + /* move the machine to the right state */ + circpad_cell_event_nonpadding_received((circuit_t*)client_side); + tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, + CIRCPAD_STATE_BURST); + + /* Get the machine and setup tokens */ + mi = client_side->padding_info[0]; + tt_assert(mi); + + /*************************************************************************/ + + uint64_t current_time = monotime_absolute_usec(); + + /* Test left boundaries of each histogram bin: */ + const circpad_delay_t bin_left_bounds[] = + {0, 1, 7, 15, 31, 62, 125, 250, 500, CIRCPAD_DELAY_INFINITE}; + for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) { + tt_uint_op(bin_left_bounds[i], OP_EQ, + circpad_histogram_bin_to_usec(mi, i)); + } + + /* Check that all bins have two tokens right now */ + for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) { + tt_int_op(mi->histogram[i], OP_EQ, 2); + } + + /* This is the right order to remove tokens from this histogram. That is, we + * first remove tokens from the 4th bin since 57 usec is nearest to the 4th + * bin midpoint (31 + (62-31)/2 == 46). Then we remove from the 3rd bin for + * the same reason, then from the 5th, etc. */ + const int bin_removal_order[] = {4, 3, 2, 1, 0}; + unsigned i; + + /* Remove all tokens from all bins apart from the infinity bin */ + for (i = 0; i < sizeof(bin_removal_order)/sizeof(int) ; i++) { + int bin_to_remove = bin_removal_order[i]; + log_debug(LD_GENERAL, "Testing that %d attempt removes %d bin", + i, bin_to_remove); + + tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 2); + + mi->padding_scheduled_at_usec = current_time - 57; + circpad_machine_remove_token(mi); + + tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 1); + + mi->padding_scheduled_at_usec = current_time - 57; + circpad_machine_remove_token(mi); + + /* Test that we cleaned out this bin. Don't do this in the case of the last + bin since the tokens will get refilled */ + if (i != BIG_HISTOGRAM_LEN - 2) { + tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 0); + } + } + + /* Check that all higher bins are untouched */ + for (i = 5; i < BIG_HISTOGRAM_LEN ; i++) { + tt_int_op(mi->histogram[i], OP_EQ, 2); + } + + /* Test above the highest bin, for coverage */ + tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, + CIRCPAD_STATE_BURST); + circ_client_machine.states[CIRCPAD_STATE_BURST].start_usec = 100; + mi->padding_scheduled_at_usec = current_time - 29202; + circpad_machine_remove_token(mi); + tt_int_op(mi->histogram[BIG_HISTOGRAM_LEN-2], OP_EQ, 1); + + done: + free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side)); + monotime_disable_test_mocking(); + tor_free(circ_client_machine.states); +} + +/** Test closest token removal strategy by bin */ +static void +test_circuitpadding_closest_token_removal(void *arg) +{ + circpad_machine_state_t *mi; + (void)arg; + + /* Mock it up */ + MOCK(monotime_absolute_usec, mock_monotime_absolute_usec); + MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock); + + /* Setup test environment (time etc.) */ + client_side = (circuit_t *)origin_circuit_new(); + client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; + monotime_enable_test_mocking(); + + /* Create test machine */ + helper_create_machine_with_big_histogram(CIRCPAD_TOKEN_REMOVAL_CLOSEST); + client_side->padding_machine[0] = &circ_client_machine; + client_side->padding_info[0] = + circpad_circuit_machineinfo_new(client_side, 0); + + /* move the machine to the right state */ + circpad_cell_event_nonpadding_received((circuit_t*)client_side); + tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, + CIRCPAD_STATE_BURST); + + /* Get the machine and setup tokens */ + mi = client_side->padding_info[0]; + tt_assert(mi); + + /*************************************************************************/ + + uint64_t current_time = monotime_absolute_usec(); + + /* Test left boundaries of each histogram bin: */ + const circpad_delay_t bin_left_bounds[] = + {0, 1, 7, 15, 31, 62, 125, 250, 500, CIRCPAD_DELAY_INFINITE}; + for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) { + tt_uint_op(bin_left_bounds[i], OP_EQ, + circpad_histogram_bin_to_usec(mi, i)); + } + + /* Check that all bins have two tokens right now */ + for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) { + tt_int_op(mi->histogram[i], OP_EQ, 2); + } + + /* This is the right order to remove tokens from this histogram. That is, we + * first remove tokens from the 4th bin since 57 usec is nearest to the 4th + * bin midpoint (31 + (62-31)/2 == 46). Then we remove from the 3rd bin for + * the same reason, then from the 5th, etc. */ + const int bin_removal_order[] = {4, 3, 5, 2, 6, 1, 7, 0, 8, 9}; + + /* Remove all tokens from all bins apart from the infinity bin */ + for (int i = 0; i < BIG_HISTOGRAM_LEN-1 ; i++) { + int bin_to_remove = bin_removal_order[i]; + log_debug(LD_GENERAL, "Testing that %d attempt removes %d bin", + i, bin_to_remove); + + tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 2); + + mi->padding_scheduled_at_usec = current_time - 57; + circpad_machine_remove_token(mi); + + tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 1); + + mi->padding_scheduled_at_usec = current_time - 57; + circpad_machine_remove_token(mi); + + /* Test that we cleaned out this bin. Don't do this in the case of the last + bin since the tokens will get refilled */ + if (i != BIG_HISTOGRAM_LEN - 2) { + tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 0); + } + } + + /* Check that all bins have been refilled */ + for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) { + tt_int_op(mi->histogram[i], OP_EQ, 2); + } + + /* Test below the lowest bin, for coverage */ + tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, + CIRCPAD_STATE_BURST); + circ_client_machine.states[CIRCPAD_STATE_BURST].start_usec = 100; + mi->padding_scheduled_at_usec = current_time - 102; + mi->histogram[0] = 0; + circpad_machine_remove_token(mi); + tt_int_op(mi->histogram[1], OP_EQ, 1); + + /* Test above the highest bin, for coverage */ + tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, + CIRCPAD_STATE_BURST); + circ_client_machine.states[CIRCPAD_STATE_BURST].start_usec = 100; + mi->padding_scheduled_at_usec = current_time - 29202; + circpad_machine_remove_token(mi); + tt_int_op(mi->histogram[BIG_HISTOGRAM_LEN-2], OP_EQ, 1); + + done: + free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side)); + monotime_disable_test_mocking(); + tor_free(circ_client_machine.states); +} + +/** Test closest token removal strategy with usec */ +static void +test_circuitpadding_closest_token_removal_usec(void *arg) +{ + circpad_machine_state_t *mi; + (void)arg; + + /* Mock it up */ + MOCK(monotime_absolute_usec, mock_monotime_absolute_usec); + MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock); + + /* Setup test environment (time etc.) */ + client_side = (circuit_t *)origin_circuit_new(); + client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; + monotime_enable_test_mocking(); + + /* Create test machine */ + helper_create_machine_with_big_histogram(CIRCPAD_TOKEN_REMOVAL_CLOSEST_USEC); + client_side->padding_machine[0] = &circ_client_machine; + client_side->padding_info[0] = + circpad_circuit_machineinfo_new(client_side, 0); + + /* move the machine to the right state */ + circpad_cell_event_nonpadding_received((circuit_t*)client_side); + tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, + CIRCPAD_STATE_BURST); + + /* Get the machine and setup tokens */ + mi = client_side->padding_info[0]; + tt_assert(mi); + + /*************************************************************************/ + + uint64_t current_time = monotime_absolute_usec(); + + /* Test left boundaries of each histogram bin: */ + const circpad_delay_t bin_left_bounds[] = + {0, 1, 7, 15, 31, 62, 125, 250, 500, CIRCPAD_DELAY_INFINITE}; + for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) { + tt_uint_op(bin_left_bounds[i], OP_EQ, + circpad_histogram_bin_to_usec(mi, i)); + } + + /* XXX we want to test remove_token_exact and + circpad_machine_remove_closest_token() with usec */ + + /* Check that all bins have two tokens right now */ + for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) { + tt_int_op(mi->histogram[i], OP_EQ, 2); + } + + /* This is the right order to remove tokens from this histogram. That is, we + * first remove tokens from the 4th bin since 57 usec is nearest to the 4th + * bin midpoint (31 + (62-31)/2 == 46). Then we remove from the 3rd bin for + * the same reason, then from the 5th, etc. */ + const int bin_removal_order[] = {4, 3, 5, 2, 1, 0, 6, 7, 8, 9}; + + /* Remove all tokens from all bins apart from the infinity bin */ + for (int i = 0; i < BIG_HISTOGRAM_LEN-1 ; i++) { + int bin_to_remove = bin_removal_order[i]; + log_debug(LD_GENERAL, "Testing that %d attempt removes %d bin", + i, bin_to_remove); + + tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 2); + + mi->padding_scheduled_at_usec = current_time - 57; + circpad_machine_remove_token(mi); + + tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 1); + + mi->padding_scheduled_at_usec = current_time - 57; + circpad_machine_remove_token(mi); + + /* Test that we cleaned out this bin. Don't do this in the case of the last + bin since the tokens will get refilled */ + if (i != BIG_HISTOGRAM_LEN - 2) { + tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 0); + } + } + + /* Check that all bins have been refilled */ + for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) { + tt_int_op(mi->histogram[i], OP_EQ, 2); + } + + /* Test below the lowest bin, for coverage */ + tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, + CIRCPAD_STATE_BURST); + circ_client_machine.states[CIRCPAD_STATE_BURST].start_usec = 100; + mi->padding_scheduled_at_usec = current_time - 102; + mi->histogram[0] = 0; + circpad_machine_remove_token(mi); + tt_int_op(mi->histogram[1], OP_EQ, 1); + + /* Test above the highest bin, for coverage */ + tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, + CIRCPAD_STATE_BURST); + circ_client_machine.states[CIRCPAD_STATE_BURST].start_usec = 100; + mi->padding_scheduled_at_usec = current_time - 29202; + circpad_machine_remove_token(mi); + tt_int_op(mi->histogram[BIG_HISTOGRAM_LEN-2], OP_EQ, 1); + + done: + free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side)); + monotime_disable_test_mocking(); + tor_free(circ_client_machine.states); +} + +/** Test closest token removal strategy with usec */ +static void +test_circuitpadding_token_removal_exact(void *arg) +{ + circpad_machine_state_t *mi; + (void)arg; + + /* Mock it up */ + MOCK(monotime_absolute_usec, mock_monotime_absolute_usec); + MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock); + + /* Setup test environment (time etc.) */ + client_side = (circuit_t *)origin_circuit_new(); + client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; + monotime_enable_test_mocking(); + + /* Create test machine */ + helper_create_machine_with_big_histogram(CIRCPAD_TOKEN_REMOVAL_EXACT); + client_side->padding_machine[0] = &circ_client_machine; + client_side->padding_info[0] = + circpad_circuit_machineinfo_new(client_side, 0); + + /* move the machine to the right state */ + circpad_cell_event_nonpadding_received((circuit_t*)client_side); + tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, + CIRCPAD_STATE_BURST); + + /* Get the machine and setup tokens */ + mi = client_side->padding_info[0]; + tt_assert(mi); + + /**********************************************************************/ + uint64_t current_time = monotime_absolute_usec(); + + /* Ensure that we will clear out bin #4 with this usec */ + mi->padding_scheduled_at_usec = current_time - 57; + tt_int_op(mi->histogram[4], OP_EQ, 2); + circpad_machine_remove_token(mi); + mi->padding_scheduled_at_usec = current_time - 57; + tt_int_op(mi->histogram[4], OP_EQ, 1); + circpad_machine_remove_token(mi); + tt_int_op(mi->histogram[4], OP_EQ, 0); + + /* Ensure that we will not remove any other tokens even tho we try to, since + * this is what the exact strategy dictates */ + mi->padding_scheduled_at_usec = current_time - 57; + circpad_machine_remove_token(mi); + for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) { + if (i != 4) { + tt_int_op(mi->histogram[i], OP_EQ, 2); + } + } + + done: + free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side)); + monotime_disable_test_mocking(); + tor_free(circ_client_machine.states); +} + +#undef BIG_HISTOGRAM_LEN + +void +test_circuitpadding_tokens(void *arg) +{ + const circpad_state_t *state; + circpad_machine_state_t *mi; + int64_t actual_mocked_monotime_start; + (void)arg; + + /** Test plan: + * + * 1. Test symmetry between bin_to_usec and usec_to_bin + * a. Test conversion + * b. Test edge transitions (lower, upper) + * 2. Test remove higher on an empty bin + * a. Normal bin + * b. Infinity bin + * c. Bin 0 + * d. No higher + * 3. Test remove lower + * a. Normal bin + * b. Bin 0 + * c. No lower + * 4. Test remove closest + * a. Closest lower + * b. Closest higher + * c. Closest 0 + * d. Closest Infinity + */ + client_side = TO_CIRCUIT(origin_circuit_new()); + client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; + + monotime_init(); + monotime_enable_test_mocking(); + actual_mocked_monotime_start = MONOTIME_MOCK_START; + monotime_set_mock_time_nsec(actual_mocked_monotime_start); + monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start); + curr_mocked_time = actual_mocked_monotime_start; + + timers_initialize(); + + helper_create_basic_machine(); + client_side->padding_machine[0] = &circ_client_machine; + client_side->padding_info[0] = circpad_circuit_machineinfo_new(client_side, + 0); + + mi = client_side->padding_info[0]; + + // Pretend a non-padding cell was sent + circpad_cell_event_nonpadding_received((circuit_t*)client_side); + circpad_cell_event_nonpadding_sent((circuit_t*)client_side); + /* We have to save the infinity bin because one inf delay + * could have been chosen when we transition to burst */ + circpad_hist_token_t inf_bin = mi->histogram[4]; + + tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, + CIRCPAD_STATE_BURST); + + state = circpad_machine_current_state(client_side->padding_info[0]); + + // Test 0: convert bin->usec->bin + // Bin 0+1 have different semantics + for (int bin = 0; bin < 2; bin++) { + circpad_delay_t usec = + circpad_histogram_bin_to_usec(client_side->padding_info[0], bin); + int bin2 = circpad_histogram_usec_to_bin(client_side->padding_info[0], + usec); + tt_int_op(bin, OP_EQ, bin2); + } + for (int bin = 2; bin < state->histogram_len-1; bin++) { + circpad_delay_t usec = + circpad_histogram_bin_to_usec(client_side->padding_info[0], bin); + int bin2 = circpad_histogram_usec_to_bin(client_side->padding_info[0], + usec); + tt_int_op(bin, OP_EQ, bin2); + /* Verify we round down */ + bin2 = circpad_histogram_usec_to_bin(client_side->padding_info[0], + usec+3); + tt_int_op(bin, OP_EQ, bin2); + + bin2 = circpad_histogram_usec_to_bin(client_side->padding_info[0], + usec-1); + tt_int_op(bin, OP_EQ, bin2+1); + } + + // Test 1: converting usec->bin->usec->bin + // Bin 0+1 have different semantics. + for (circpad_delay_t i = 0; i <= state->start_usec+1; i++) { + int bin = circpad_histogram_usec_to_bin(client_side->padding_info[0], + i); + circpad_delay_t usec = + circpad_histogram_bin_to_usec(client_side->padding_info[0], bin); + int bin2 = circpad_histogram_usec_to_bin(client_side->padding_info[0], + usec); + tt_int_op(bin, OP_EQ, bin2); + tt_int_op(i, OP_LE, usec); + } + for (circpad_delay_t i = state->start_usec+1; + i <= state->start_usec + state->range_usec; i++) { + int bin = circpad_histogram_usec_to_bin(client_side->padding_info[0], + i); + circpad_delay_t usec = + circpad_histogram_bin_to_usec(client_side->padding_info[0], bin); + int bin2 = circpad_histogram_usec_to_bin(client_side->padding_info[0], + usec); + tt_int_op(bin, OP_EQ, bin2); + tt_int_op(i, OP_GE, usec); + } + + /* 2.a. Normal higher bin */ + { + tt_int_op(mi->histogram[2], OP_EQ, 2); + tt_int_op(mi->histogram[3], OP_EQ, 2); + circpad_machine_remove_higher_token(mi, + circpad_histogram_bin_to_usec(mi, 2)+1); + tt_int_op(mi->histogram[3], OP_EQ, 2); + tt_int_op(mi->histogram[2], OP_EQ, 1); + + circpad_machine_remove_higher_token(mi, + circpad_histogram_bin_to_usec(mi, 2)+1); + tt_int_op(mi->histogram[2], OP_EQ, 0); + + tt_int_op(mi->histogram[3], OP_EQ, 2); + circpad_machine_remove_higher_token(mi, + circpad_histogram_bin_to_usec(mi, 2)+1); + circpad_machine_remove_higher_token(mi, + circpad_histogram_bin_to_usec(mi, 2)+1); + tt_int_op(mi->histogram[3], OP_EQ, 0); + circpad_machine_remove_higher_token(mi, + circpad_histogram_bin_to_usec(mi, 2)+1); + tt_int_op(mi->histogram[3], OP_EQ, 0); + } + + /* 2.b. Higher Infinity bin */ + { + tt_int_op(mi->histogram[4], OP_EQ, inf_bin); + circpad_machine_remove_higher_token(mi, + circpad_histogram_bin_to_usec(mi, 2)+1); + tt_int_op(mi->histogram[4], OP_EQ, inf_bin); + + /* Test past the infinity bin */ + circpad_machine_remove_higher_token(mi, + circpad_histogram_bin_to_usec(mi, 5)+1000000); + + tt_int_op(mi->histogram[4], OP_EQ, inf_bin); + } + + /* 2.c. Bin 0 */ + { + tt_int_op(mi->histogram[0], OP_EQ, 0); + mi->histogram[0] = 1; + circpad_machine_remove_higher_token(mi, + state->start_usec/2); + tt_int_op(mi->histogram[0], OP_EQ, 0); + } + + /* Drain the infinity bin and cause a refill */ + while (inf_bin != 0) { + tt_int_op(mi->histogram[4], OP_EQ, inf_bin); + circpad_cell_event_nonpadding_received((circuit_t*)client_side); + inf_bin--; + } + + circpad_cell_event_nonpadding_sent((circuit_t*)client_side); + + // We should have refilled here. + tt_int_op(mi->histogram[4], OP_EQ, 2); + + /* 3.a. Bin 0 */ + { + tt_int_op(mi->histogram[0], OP_EQ, 1); + circpad_machine_remove_higher_token(mi, + state->start_usec/2); + tt_int_op(mi->histogram[0], OP_EQ, 0); + } + + /* 3.b. Test remove lower normal bin */ + { + tt_int_op(mi->histogram[3], OP_EQ, 2); + circpad_machine_remove_lower_token(mi, + circpad_histogram_bin_to_usec(mi, 3)+1); + circpad_machine_remove_lower_token(mi, + circpad_histogram_bin_to_usec(mi, 3)+1); + tt_int_op(mi->histogram[3], OP_EQ, 0); + tt_int_op(mi->histogram[2], OP_EQ, 2); + circpad_machine_remove_lower_token(mi, + circpad_histogram_bin_to_usec(mi, 3)+1); + circpad_machine_remove_lower_token(mi, + circpad_histogram_bin_to_usec(mi, 3)+1); + /* 3.c. No lower */ + circpad_machine_remove_lower_token(mi, + circpad_histogram_bin_to_usec(mi, 3)+1); + tt_int_op(mi->histogram[2], OP_EQ, 0); + } + + /* 4. Test remove closest + * a. Closest lower + * b. Closest higher + * c. Closest 0 + * d. Closest Infinity + */ + circpad_machine_setup_tokens(mi); + tt_int_op(mi->histogram[2], OP_EQ, 2); + circpad_machine_remove_closest_token(mi, + circpad_histogram_bin_to_usec(mi, 2)+1, 0); + circpad_machine_remove_closest_token(mi, + circpad_histogram_bin_to_usec(mi, 2)+1, 0); + tt_int_op(mi->histogram[2], OP_EQ, 0); + tt_int_op(mi->histogram[3], OP_EQ, 2); + circpad_machine_remove_closest_token(mi, + circpad_histogram_bin_to_usec(mi, 2)+1, 0); + circpad_machine_remove_closest_token(mi, + circpad_histogram_bin_to_usec(mi, 2)+1, 0); + tt_int_op(mi->histogram[3], OP_EQ, 0); + tt_int_op(mi->histogram[0], OP_EQ, 1); + circpad_machine_remove_closest_token(mi, + circpad_histogram_bin_to_usec(mi, 2)+1, 0); + tt_int_op(mi->histogram[0], OP_EQ, 0); + tt_int_op(mi->histogram[4], OP_EQ, 2); + circpad_machine_remove_closest_token(mi, + circpad_histogram_bin_to_usec(mi, 2)+1, 0); + tt_int_op(mi->histogram[4], OP_EQ, 2); + + /* 5. Test remove closest usec + * a. Closest 0 + * b. Closest lower (below midpoint) + * c. Closest higher (above midpoint) + * d. Closest Infinity + */ + circpad_machine_setup_tokens(mi); + + tt_int_op(mi->histogram[0], OP_EQ, 1); + circpad_machine_remove_closest_token(mi, + circpad_histogram_bin_to_usec(mi, 0)/3, 1); + tt_int_op(mi->histogram[0], OP_EQ, 0); + tt_int_op(mi->histogram[2], OP_EQ, 2); + circpad_machine_remove_closest_token(mi, + circpad_histogram_bin_to_usec(mi, 0)/3, 1); + circpad_machine_remove_closest_token(mi, + circpad_histogram_bin_to_usec(mi, 0)/3, 1); + tt_int_op(mi->histogram[2], OP_EQ, 0); + tt_int_op(mi->histogram[3], OP_EQ, 2); + circpad_machine_remove_closest_token(mi, + circpad_histogram_bin_to_usec(mi, 4), 1); + circpad_machine_remove_closest_token(mi, + circpad_histogram_bin_to_usec(mi, 4), 1); + tt_int_op(mi->histogram[3], OP_EQ, 0); + tt_int_op(mi->histogram[4], OP_EQ, 2); + circpad_machine_remove_closest_token(mi, + circpad_histogram_bin_to_usec(mi, 4), 1); + circpad_machine_remove_closest_token(mi, + circpad_histogram_bin_to_usec(mi, 4), 1); + tt_int_op(mi->histogram[4], OP_EQ, 2); + + // XXX: Need more coverage of the actual usec branches + + done: + free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side)); + monotime_disable_test_mocking(); + tor_free(circ_client_machine.states); +} + +void +test_circuitpadding_wronghop(void *arg) +{ + /** + * Test plan: + * 1. Padding sent from hop 1 and 3 to client + * 2. Send negotiated from hop 1 and 3 to client + * 3. Garbled negotiated cell + * 4. Padding negotiate sent to client + * 5. Send negotiate stop command for unknown machine + * 6. Send negotiated to relay + * 7. Garbled padding negotiate cell + */ + (void)arg; + uint32_t read_bw = 0, overhead_bw = 0; + cell_t cell; + signed_error_t ret; + origin_circuit_t *orig_client; + int64_t actual_mocked_monotime_start; + + MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock); + + /* Mock this function so that our cell counting tests don't get confused by + * padding that gets sent by scheduled timers. */ + MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock); + + client_side = (circuit_t *)origin_circuit_new(); + dummy_channel.cmux = circuitmux_alloc(); + relay_side = (circuit_t *)new_fake_orcirc(&dummy_channel, + &dummy_channel); + orig_client = TO_ORIGIN_CIRCUIT(client_side); + + relay_side->purpose = CIRCUIT_PURPOSE_OR; + client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; + nodes_init(); + + monotime_init(); + monotime_enable_test_mocking(); + actual_mocked_monotime_start = MONOTIME_MOCK_START; + monotime_set_mock_time_nsec(actual_mocked_monotime_start); + monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start); + curr_mocked_time = actual_mocked_monotime_start; + + timers_initialize(); + circpad_machines_init(); + + MOCK(node_get_by_id, + node_get_by_id_mock); + + MOCK(circuit_package_relay_cell, + circuit_package_relay_cell_mock); + + /* Build three hops */ + simulate_single_hop_extend(client_side, relay_side, 1); + simulate_single_hop_extend(client_side, relay_side, 1); + simulate_single_hop_extend(client_side, relay_side, 1); + + /* verify padding was negotiated */ + tt_ptr_op(relay_side->padding_machine[0], OP_NE, NULL); + tt_ptr_op(relay_side->padding_info[0], OP_NE, NULL); + + /* verify echo was sent */ + tt_int_op(n_relay_cells, OP_EQ, 1); + tt_int_op(n_client_cells, OP_EQ, 1); + + read_bw = orig_client->n_delivered_read_circ_bw; + overhead_bw = orig_client->n_overhead_read_circ_bw; + + /* 1. Test padding from first and third hop */ + circpad_deliver_recognized_relay_cell_events(client_side, + RELAY_COMMAND_DROP, + TO_ORIGIN_CIRCUIT(client_side)->cpath); + tt_int_op(read_bw, OP_EQ, + orig_client->n_delivered_read_circ_bw); + tt_int_op(overhead_bw, OP_EQ, + orig_client->n_overhead_read_circ_bw); + + circpad_deliver_recognized_relay_cell_events(client_side, + RELAY_COMMAND_DROP, + TO_ORIGIN_CIRCUIT(client_side)->cpath->next->next); + tt_int_op(read_bw, OP_EQ, + orig_client->n_delivered_read_circ_bw); + tt_int_op(overhead_bw, OP_EQ, + orig_client->n_overhead_read_circ_bw); + + circpad_deliver_recognized_relay_cell_events(client_side, + RELAY_COMMAND_DROP, + TO_ORIGIN_CIRCUIT(client_side)->cpath->next); + tt_int_op(read_bw, OP_EQ, + orig_client->n_delivered_read_circ_bw); + tt_int_op(overhead_bw, OP_LT, + orig_client->n_overhead_read_circ_bw); + + /* 2. Test padding negotiated not handled from hops 1,3 */ + ret = circpad_handle_padding_negotiated(client_side, &cell, + TO_ORIGIN_CIRCUIT(client_side)->cpath); + tt_int_op(ret, OP_EQ, -1); + + ret = circpad_handle_padding_negotiated(client_side, &cell, + TO_ORIGIN_CIRCUIT(client_side)->cpath->next->next); + tt_int_op(ret, OP_EQ, -1); + + /* 3. Garbled negotiated cell */ + memset(&cell, 255, sizeof(cell)); + ret = circpad_handle_padding_negotiated(client_side, &cell, + TO_ORIGIN_CIRCUIT(client_side)->cpath->next); + tt_int_op(ret, OP_EQ, -1); + + /* 4. Test that negotiate is dropped at origin */ + read_bw = orig_client->n_delivered_read_circ_bw; + overhead_bw = orig_client->n_overhead_read_circ_bw; + relay_send_command_from_edge(0, relay_side, + RELAY_COMMAND_PADDING_NEGOTIATE, + (void*)cell.payload, + (size_t)3, NULL); + tt_int_op(read_bw, OP_EQ, + orig_client->n_delivered_read_circ_bw); + tt_int_op(overhead_bw, OP_EQ, + orig_client->n_overhead_read_circ_bw); + + tt_int_op(n_relay_cells, OP_EQ, 2); + tt_int_op(n_client_cells, OP_EQ, 1); + + /* 5. Test that asking to stop the wrong machine does nothing */ + circpad_negotiate_padding(TO_ORIGIN_CIRCUIT(client_side), + 255, 2, CIRCPAD_COMMAND_STOP); + tt_ptr_op(client_side->padding_machine[0], OP_NE, NULL); + tt_ptr_op(client_side->padding_info[0], OP_NE, NULL); + tt_ptr_op(relay_side->padding_machine[0], OP_NE, NULL); + tt_ptr_op(relay_side->padding_info[0], OP_NE, NULL); + tt_int_op(n_relay_cells, OP_EQ, 3); + tt_int_op(n_client_cells, OP_EQ, 2); + + /* 6. Sending negotiated command to relay does nothing */ + ret = circpad_handle_padding_negotiated(relay_side, &cell, NULL); + tt_int_op(ret, OP_EQ, -1); + + /* 7. Test garbled negotated cell (bad command 255) */ + memset(&cell, 0, sizeof(cell)); + ret = circpad_handle_padding_negotiate(relay_side, &cell); + tt_int_op(ret, OP_EQ, -1); + tt_int_op(n_client_cells, OP_EQ, 2); + + /* Test 2: Test no padding */ + free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side)); + free_fake_orcirc(relay_side); + + client_side = (circuit_t *)origin_circuit_new(); + relay_side = (circuit_t *)new_fake_orcirc(&dummy_channel, + &dummy_channel); + relay_side->purpose = CIRCUIT_PURPOSE_OR; + client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; + + simulate_single_hop_extend(client_side, relay_side, 1); + simulate_single_hop_extend(client_side, relay_side, 0); + + /* verify no padding was negotiated */ + tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL); + tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL); + tt_int_op(n_relay_cells, OP_EQ, 3); + tt_int_op(n_client_cells, OP_EQ, 2); + + /* verify no echo was sent */ + tt_int_op(n_relay_cells, OP_EQ, 3); + tt_int_op(n_client_cells, OP_EQ, 2); + + /* Finish circuit */ + simulate_single_hop_extend(client_side, relay_side, 1); + + /* Spoof padding negotiated on circuit with no padding */ + circpad_padding_negotiated(relay_side, + CIRCPAD_MACHINE_CIRC_SETUP, + CIRCPAD_COMMAND_START, + CIRCPAD_RESPONSE_OK); + + /* verify no padding was negotiated */ + tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL); + tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL); + + circpad_padding_negotiated(relay_side, + CIRCPAD_MACHINE_CIRC_SETUP, + CIRCPAD_COMMAND_START, + CIRCPAD_RESPONSE_ERR); + + /* verify no padding was negotiated */ + tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL); + tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL); + + done: + free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side)); + free_fake_orcirc(relay_side); + circuitmux_detach_all_circuits(dummy_channel.cmux, NULL); + circuitmux_free(dummy_channel.cmux); + monotime_disable_test_mocking(); + UNMOCK(node_get_by_id); + UNMOCK(circuit_package_relay_cell); + UNMOCK(circuitmux_attach_circuit); + nodes_free(); +} + +void +test_circuitpadding_negotiation(void *arg) +{ + /** + * Test plan: + * 1. Test circuit where padding is supported by middle + * a. Make sure padding negotiation is sent + * b. Test padding negotiation delivery and parsing + * 2. Test circuit where padding is unsupported by middle + * a. Make sure padding negotiation is not sent + * 3. Test failure to negotiate a machine due to desync. + */ + int64_t actual_mocked_monotime_start; + (void)arg; + + MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock); + + client_side = TO_CIRCUIT(origin_circuit_new()); + dummy_channel.cmux = circuitmux_alloc(); + relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel, &dummy_channel)); + + relay_side->purpose = CIRCUIT_PURPOSE_OR; + client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; + nodes_init(); + + monotime_init(); + monotime_enable_test_mocking(); + actual_mocked_monotime_start = MONOTIME_MOCK_START; + monotime_set_mock_time_nsec(actual_mocked_monotime_start); + monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start); + curr_mocked_time = actual_mocked_monotime_start; + + timers_initialize(); + circpad_machines_init(); + + MOCK(node_get_by_id, + node_get_by_id_mock); + + MOCK(circuit_package_relay_cell, + circuit_package_relay_cell_mock); + + /* Build two hops */ + simulate_single_hop_extend(client_side, relay_side, 1); + simulate_single_hop_extend(client_side, relay_side, 1); + + /* verify padding was negotiated */ + tt_ptr_op(relay_side->padding_machine[0], OP_NE, NULL); + tt_ptr_op(relay_side->padding_info[0], OP_NE, NULL); + + /* verify echo was sent */ + tt_int_op(n_relay_cells, OP_EQ, 1); + tt_int_op(n_client_cells, OP_EQ, 1); + + /* Finish circuit */ + simulate_single_hop_extend(client_side, relay_side, 1); + + /* Test 2: Test no padding */ + free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side)); + free_fake_orcirc(relay_side); + + client_side = TO_CIRCUIT(origin_circuit_new()); + relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel, &dummy_channel)); + relay_side->purpose = CIRCUIT_PURPOSE_OR; + client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; + + simulate_single_hop_extend(client_side, relay_side, 1); + simulate_single_hop_extend(client_side, relay_side, 0); + + /* verify no padding was negotiated */ + tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL); + tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL); + tt_int_op(n_relay_cells, OP_EQ, 1); + tt_int_op(n_client_cells, OP_EQ, 1); + + /* verify no echo was sent */ + tt_int_op(n_relay_cells, OP_EQ, 1); + tt_int_op(n_client_cells, OP_EQ, 1); + + /* Finish circuit */ + simulate_single_hop_extend(client_side, relay_side, 1); + + /* Force negotiate padding. */ + circpad_negotiate_padding(TO_ORIGIN_CIRCUIT(client_side), + CIRCPAD_MACHINE_CIRC_SETUP, + 2, CIRCPAD_COMMAND_START); + + /* verify no padding was negotiated */ + tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL); + tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL); + + /* verify no echo was sent */ + tt_int_op(n_relay_cells, OP_EQ, 1); + tt_int_op(n_client_cells, OP_EQ, 1); + + /* 3. Test failure to negotiate a machine due to desync */ + free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side)); + free_fake_orcirc(relay_side); + + client_side = TO_CIRCUIT(origin_circuit_new()); + relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel, &dummy_channel)); + relay_side->purpose = CIRCUIT_PURPOSE_OR; + client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; + + SMARTLIST_FOREACH(relay_padding_machines, + circpad_machine_spec_t *, + m, tor_free(m->states); tor_free(m)); + smartlist_free(relay_padding_machines); + relay_padding_machines = smartlist_new(); + + simulate_single_hop_extend(client_side, relay_side, 1); + simulate_single_hop_extend(client_side, relay_side, 1); + + /* verify echo was sent */ + tt_int_op(n_client_cells, OP_EQ, 2); + tt_int_op(n_relay_cells, OP_EQ, 2); + + /* verify no padding was negotiated */ + tt_ptr_op(client_side->padding_info[0], OP_EQ, NULL); + tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL); + tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL); + tt_ptr_op(relay_side->padding_info[0], OP_EQ, NULL); + + done: + free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side)); + free_fake_orcirc(relay_side); + circuitmux_detach_all_circuits(dummy_channel.cmux, NULL); + circuitmux_free(dummy_channel.cmux); + monotime_disable_test_mocking(); + UNMOCK(node_get_by_id); + UNMOCK(circuit_package_relay_cell); + UNMOCK(circuitmux_attach_circuit); + nodes_free(); +} + +static void +simulate_single_hop_extend(circuit_t *client, circuit_t *mid_relay, + int padding) +{ + char whatevs_key[CPATH_KEY_MATERIAL_LEN]; + char digest[DIGEST_LEN]; + tor_addr_t addr; + + // Pretend a non-padding cell was sent + circpad_cell_event_nonpadding_sent((circuit_t*)client); + + // Receive extend cell at middle + circpad_cell_event_nonpadding_received((circuit_t*)mid_relay); + + // Advance time a tiny bit so we can calculate an RTT + curr_mocked_time += 10 * TOR_NSEC_PER_MSEC; + monotime_coarse_set_mock_time_nsec(curr_mocked_time); + monotime_set_mock_time_nsec(curr_mocked_time); + + // Receive extended cell at middle + circpad_cell_event_nonpadding_sent((circuit_t*)mid_relay); + + // Receive extended cell at first hop + circpad_cell_event_nonpadding_received((circuit_t*)client); + + // Add a hop to cpath + crypt_path_t *hop = tor_malloc_zero(sizeof(crypt_path_t)); + onion_append_to_cpath(&TO_ORIGIN_CIRCUIT(client)->cpath, hop); + + hop->magic = CRYPT_PATH_MAGIC; + hop->state = CPATH_STATE_OPEN; + + // add an extend info to indicate if this node supports padding or not. + // (set the first byte of the digest for our mocked node_get_by_id) + digest[0] = padding; + + hop->extend_info = extend_info_new( + padding ? "padding" : "non-padding", + digest, NULL, NULL, NULL, + &addr, padding); + + circuit_init_cpath_crypto(hop, whatevs_key, sizeof(whatevs_key), 0, 0); + + hop->package_window = circuit_initial_package_window(); + hop->deliver_window = CIRCWINDOW_START; + + // Signal that the hop was added + circpad_machine_event_circ_added_hop(TO_ORIGIN_CIRCUIT(client)); +} + +static circpad_machine_spec_t * +helper_create_conditional_machine(void) +{ + circpad_machine_spec_t *ret = + tor_malloc_zero(sizeof(circpad_machine_spec_t)); + + /* Start, burst */ + circpad_machine_states_init(ret, 2); + + ret->states[CIRCPAD_STATE_START]. + next_state[CIRCPAD_EVENT_PADDING_SENT] = CIRCPAD_STATE_BURST; + + ret->states[CIRCPAD_STATE_BURST]. + next_state[CIRCPAD_EVENT_PADDING_SENT] = CIRCPAD_STATE_BURST; + + ret->states[CIRCPAD_STATE_BURST]. + next_state[CIRCPAD_EVENT_LENGTH_COUNT] = CIRCPAD_STATE_END; + + ret->states[CIRCPAD_STATE_BURST].token_removal = + CIRCPAD_TOKEN_REMOVAL_NONE; + + ret->states[CIRCPAD_STATE_BURST].histogram_len = 3; + ret->states[CIRCPAD_STATE_BURST].start_usec = 0; + ret->states[CIRCPAD_STATE_BURST].range_usec = 1000000; + ret->states[CIRCPAD_STATE_BURST].histogram[0] = 6; + ret->states[CIRCPAD_STATE_BURST].histogram[1] = 0; + ret->states[CIRCPAD_STATE_BURST].histogram[1] = 0; + ret->states[CIRCPAD_STATE_BURST].histogram_total_tokens = 6; + ret->states[CIRCPAD_STATE_BURST].use_rtt_estimate = 0; + ret->states[CIRCPAD_STATE_BURST].length_includes_nonpadding = 1; + + return ret; +} + +static void +helper_create_conditional_machines(void) +{ + circpad_machine_spec_t *add = helper_create_conditional_machine(); + origin_padding_machines = smartlist_new(); + relay_padding_machines = smartlist_new(); + + add->machine_num = 2; + add->is_origin_side = 1; + add->should_negotiate_end = 1; + add->target_hopnum = 2; + + /* Let's have this one end after 4 packets */ + add->states[CIRCPAD_STATE_BURST].length_dist.type = CIRCPAD_DIST_UNIFORM; + add->states[CIRCPAD_STATE_BURST].length_dist.param1 = 4; + add->states[CIRCPAD_STATE_BURST].length_dist.param2 = 4; + add->states[CIRCPAD_STATE_BURST].max_length = 4; + + add->conditions.requires_vanguards = 0; + add->conditions.min_hops = 2; + add->conditions.state_mask = CIRCPAD_CIRC_BUILDING| + CIRCPAD_CIRC_NO_STREAMS|CIRCPAD_CIRC_HAS_RELAY_EARLY; + add->conditions.purpose_mask = CIRCPAD_PURPOSE_ALL; + + smartlist_add(origin_padding_machines, add); + + add = helper_create_conditional_machine(); + add->machine_num = 3; + add->is_origin_side = 1; + add->should_negotiate_end = 1; + add->target_hopnum = 2; + + /* Let's have this one end after 4 packets */ + add->states[CIRCPAD_STATE_BURST].length_dist.type = CIRCPAD_DIST_UNIFORM; + add->states[CIRCPAD_STATE_BURST].length_dist.param1 = 4; + add->states[CIRCPAD_STATE_BURST].length_dist.param2 = 4; + add->states[CIRCPAD_STATE_BURST].max_length = 4; + + add->conditions.requires_vanguards = 1; + add->conditions.min_hops = 3; + add->conditions.state_mask = CIRCPAD_CIRC_OPENED| + CIRCPAD_CIRC_STREAMS|CIRCPAD_CIRC_HAS_NO_RELAY_EARLY; + add->conditions.purpose_mask = CIRCPAD_PURPOSE_ALL; + smartlist_add(origin_padding_machines, add); + + add = helper_create_conditional_machine(); + add->machine_num = 2; + smartlist_add(relay_padding_machines, add); + + add = helper_create_conditional_machine(); + add->machine_num = 3; + smartlist_add(relay_padding_machines, add); +} + +void +test_circuitpadding_conditions(void *arg) +{ + /** + * Test plan: + * 0. Make a few origin and client machines with diff conditions + * * vanguards, purposes, has_opened circs, no relay early + * * Client side should_negotiate_end + * * Length limits + * 1. Test STATE_END transitions + * 2. Test new machine after end with same conditions + * 3. Test new machine due to changed conditions + * * Esp: built event, no relay early, no streams + * XXX: Diff test: + * 1. Test STATE_END with pending timers + * 2. Test marking a circuit before padding callback fires + * 3. Test freeing a circuit before padding callback fires + */ + int64_t actual_mocked_monotime_start; + (void)arg; + MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock); + + nodes_init(); + dummy_channel.cmux = circuitmux_alloc(); + relay_side = (circuit_t *)new_fake_orcirc(&dummy_channel, + &dummy_channel); + client_side = (circuit_t *)origin_circuit_new(); + relay_side->purpose = CIRCUIT_PURPOSE_OR; + client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; + + monotime_init(); + monotime_enable_test_mocking(); + actual_mocked_monotime_start = MONOTIME_MOCK_START; + monotime_set_mock_time_nsec(actual_mocked_monotime_start); + monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start); + curr_mocked_time = actual_mocked_monotime_start; + + timers_initialize(); + helper_create_conditional_machines(); + + MOCK(circuit_package_relay_cell, + circuit_package_relay_cell_mock); + MOCK(node_get_by_id, + node_get_by_id_mock); + + /* Simulate extend. This should result in the original machine getting + * added, since the circuit is not built */ + simulate_single_hop_extend(client_side, relay_side, 1); + simulate_single_hop_extend(client_side, relay_side, 1); + + /* Verify that machine #2 is added */ + tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 2); + tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 2); + + /* Deliver a padding cell to the client, to trigger burst state */ + circpad_cell_event_padding_sent(client_side); + + /* This should have trigger length shutdown condition on client.. */ + tt_ptr_op(client_side->padding_info[0], OP_EQ, NULL); + tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL); + + /* Verify machine is gone from both sides */ + tt_ptr_op(relay_side->padding_info[0], OP_EQ, NULL); + tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL); + + /* Send another event.. verify machine gets re-added properly + * (test race with shutdown) */ + simulate_single_hop_extend(client_side, relay_side, 1); + tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 2); + tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 2); + + TO_ORIGIN_CIRCUIT(client_side)->p_streams = 0; + circpad_machine_event_circ_has_no_streams(TO_ORIGIN_CIRCUIT(client_side)); + tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 2); + tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 2); + + /* Now make the circuit opened and send built event */ + TO_ORIGIN_CIRCUIT(client_side)->has_opened = 1; + circpad_machine_event_circ_built(TO_ORIGIN_CIRCUIT(client_side)); + tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 2); + tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 2); + + TO_ORIGIN_CIRCUIT(client_side)->remaining_relay_early_cells = 0; + circpad_machine_event_circ_has_no_relay_early( + TO_ORIGIN_CIRCUIT(client_side)); + tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 2); + tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 2); + + get_options_mutable()->HSLayer2Nodes = (void*)1; + TO_ORIGIN_CIRCUIT(client_side)->p_streams = (void*)1; + circpad_machine_event_circ_has_streams(TO_ORIGIN_CIRCUIT(client_side)); + + /* Verify different machine is added */ + tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 3); + tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 3); + + /* Hold off on negotiated */ + deliver_negotiated = 0; + + /* Deliver a padding cell to the client, to trigger burst state */ + circpad_cell_event_padding_sent(client_side); + + /* This should have trigger length shutdown condition on client + * but not the response for the padding machine */ + tt_ptr_op(client_side->padding_info[0], OP_EQ, NULL); + tt_ptr_op(client_side->padding_machine[0], OP_NE, NULL); + + /* Verify machine is gone from the relay (but negotiated not back yet */ + tt_ptr_op(relay_side->padding_info[0], OP_EQ, NULL); + tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL); + + /* Add another hop and verify it's back */ + simulate_single_hop_extend(client_side, relay_side, 1); + + tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 3); + tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 3); + + tt_ptr_op(client_side->padding_info[0], OP_NE, NULL); + tt_ptr_op(relay_side->padding_info[0], OP_NE, NULL); + + done: + /* XXX: Free everything */ + return; +} + +/** Helper function: Initializes a padding machine where every state uses the + * uniform probability distribution. */ +static void +helper_circpad_circ_distribution_machine_setup(int min, int max) +{ + circpad_machine_states_init(&circ_client_machine, 7); + + circpad_state_t *zero_st = &circ_client_machine.states[0]; + zero_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 1; + zero_st->iat_dist.type = CIRCPAD_DIST_UNIFORM; + zero_st->iat_dist.param1 = min; + zero_st->iat_dist.param2 = max; + zero_st->start_usec = min; + zero_st->range_usec = max; + + circpad_state_t *first_st = &circ_client_machine.states[1]; + first_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 2; + first_st->iat_dist.type = CIRCPAD_DIST_LOGISTIC; + first_st->iat_dist.param1 = min; + first_st->iat_dist.param2 = max; + first_st->start_usec = min; + first_st->range_usec = max; + + circpad_state_t *second_st = &circ_client_machine.states[2]; + second_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 3; + second_st->iat_dist.type = CIRCPAD_DIST_LOG_LOGISTIC; + second_st->iat_dist.param1 = min; + second_st->iat_dist.param2 = max; + second_st->start_usec = min; + second_st->range_usec = max; + + circpad_state_t *third_st = &circ_client_machine.states[3]; + third_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 4; + third_st->iat_dist.type = CIRCPAD_DIST_GEOMETRIC; + third_st->iat_dist.param1 = min; + third_st->iat_dist.param2 = max; + third_st->start_usec = min; + third_st->range_usec = max; + + circpad_state_t *fourth_st = &circ_client_machine.states[4]; + fourth_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 5; + fourth_st->iat_dist.type = CIRCPAD_DIST_WEIBULL; + fourth_st->iat_dist.param1 = min; + fourth_st->iat_dist.param2 = max; + fourth_st->start_usec = min; + fourth_st->range_usec = max; + + circpad_state_t *fifth_st = &circ_client_machine.states[5]; + fifth_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 6; + fifth_st->iat_dist.type = CIRCPAD_DIST_PARETO; + fifth_st->iat_dist.param1 = min; + fifth_st->iat_dist.param2 = max; + fifth_st->start_usec = min; + fifth_st->range_usec = max; +} + +/** Simple test that the padding delays sampled from a uniform distribution + * actually faill within the uniform distribution range. */ +/* TODO: Upgrade this test so that each state tests a different prob + * distribution */ +static void +test_circuitpadding_sample_distribution(void *arg) +{ + circpad_machine_state_t *mi; + int n_samples; + int n_states; + + (void) arg; + + /* mock this function so that we dont actually schedule any padding */ + MOCK(circpad_machine_schedule_padding, + circpad_machine_schedule_padding_mock); + + /* Initialize a machine with multiple probability distributions that should + * return values between 0 and 5 */ + circpad_machines_init(); + helper_circpad_circ_distribution_machine_setup(0, 10); + + /* Initialize machine and circuits */ + client_side = TO_CIRCUIT(origin_circuit_new()); + client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; + client_side->padding_machine[0] = &circ_client_machine; + client_side->padding_info[0] = + circpad_circuit_machineinfo_new(client_side, 0); + mi = client_side->padding_info[0]; + + /* For every state, sample a bunch of values from the distribution and ensure + * they fall within range. */ + for (n_states = 0 ; n_states < 6; n_states++) { + /* Make sure we in the right state */ + tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, n_states); + + for (n_samples = 0; n_samples < 100; n_samples++) { + circpad_delay_t delay = circpad_machine_sample_delay(mi); + tt_int_op(delay, OP_GE, 0); + tt_int_op(delay, OP_LE, 10); + } + + /* send a non-padding cell to move to the next machine state */ + circpad_cell_event_nonpadding_received((circuit_t*)client_side); + } + + done: + free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side)); + UNMOCK(circpad_machine_schedule_padding); +} + +static circpad_decision_t +circpad_machine_spec_transition_mock(circpad_machine_state_t *mi, + circpad_event_t event) +{ + (void) mi; + (void) event; + + return CIRCPAD_STATE_UNCHANGED; +} + +/* Test per-machine padding rate limits */ +static void +test_circuitpadding_machine_rate_limiting(void *arg) +{ + (void) arg; + bool retval; + circpad_machine_state_t *mi; + int i; + + /* Ignore machine transitions for the purposes of this function, we only + * really care about padding counts */ + MOCK(circpad_machine_spec_transition, circpad_machine_spec_transition_mock); + MOCK(circpad_send_command_to_hop, circpad_send_command_to_hop_mock); + + /* Setup machine and circuits */ + client_side = TO_CIRCUIT(origin_circuit_new()); + client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; + helper_create_basic_machine(); + client_side->padding_machine[0] = &circ_client_machine; + client_side->padding_info[0] = + circpad_circuit_machineinfo_new(client_side, 0); + mi = client_side->padding_info[0]; + /* Set up the machine info so that we can get through the basic functions */ + mi->state_length = CIRCPAD_STATE_LENGTH_INFINITE; + + /* First we are going to test the per-machine rate limits */ + circ_client_machine.max_padding_percent = 50; + circ_client_machine.allowed_padding_count = 100; + + /* Check padding limit, should be fine since we haven't sent anything yet. */ + retval = circpad_machine_reached_padding_limit(mi); + tt_int_op(retval, OP_EQ, 0); + + /* Send 99 padding cells which is below circpad_global_allowed_cells=100, so + * the rate limit will not trigger */ + for (i=0;i<99;i++) { + circpad_send_padding_cell_for_callback(mi); + } + retval = circpad_machine_reached_padding_limit(mi); + tt_int_op(retval, OP_EQ, 0); + + /* Now send another padding cell to pass circpad_global_allowed_cells=100, + and see that the limit will trigger */ + circpad_send_padding_cell_for_callback(mi); + retval = circpad_machine_reached_padding_limit(mi); + tt_int_op(retval, OP_EQ, 1); + + retval = circpad_machine_schedule_padding(mi); + tt_int_op(retval, OP_EQ, CIRCPAD_STATE_UNCHANGED); + + /* Cover wrap */ + for (;i<UINT16_MAX;i++) { + circpad_send_padding_cell_for_callback(mi); + } + tt_int_op(mi->padding_sent, OP_EQ, UINT16_MAX/2+1); + + tt_ptr_op(client_side->padding_info[0], OP_EQ, mi); + for (i=0;i<UINT16_MAX;i++) { + circpad_cell_event_nonpadding_sent(client_side); + } + + tt_int_op(mi->nonpadding_sent, OP_EQ, UINT16_MAX/2); + tt_int_op(mi->padding_sent, OP_EQ, UINT16_MAX/4+1); + + done: + free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side)); +} + +/* Test global padding rate limits */ +static void +test_circuitpadding_global_rate_limiting(void *arg) +{ + (void) arg; + bool retval; + circpad_machine_state_t *mi; + int i; + int64_t actual_mocked_monotime_start; + + /* Ignore machine transitions for the purposes of this function, we only + * really care about padding counts */ + MOCK(circpad_machine_spec_transition, circpad_machine_spec_transition_mock); + MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock); + MOCK(circuit_package_relay_cell, + circuit_package_relay_cell_mock); + MOCK(monotime_absolute_usec, mock_monotime_absolute_usec); + + monotime_init(); + monotime_enable_test_mocking(); + actual_mocked_monotime_start = MONOTIME_MOCK_START; + monotime_set_mock_time_nsec(actual_mocked_monotime_start); + monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start); + curr_mocked_time = actual_mocked_monotime_start; + timers_initialize(); + + client_side = (circuit_t *)origin_circuit_new(); + client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; + dummy_channel.cmux = circuitmux_alloc(); + + /* Setup machine and circuits */ + relay_side = (circuit_t *)new_fake_orcirc(&dummy_channel, &dummy_channel); + relay_side->purpose = CIRCUIT_PURPOSE_OR; + helper_create_basic_machine(); + relay_side->padding_machine[0] = &circ_client_machine; + relay_side->padding_info[0] = + circpad_circuit_machineinfo_new(relay_side, 0); + mi = relay_side->padding_info[0]; + /* Set up the machine info so that we can get through the basic functions */ + mi->state_length = CIRCPAD_STATE_LENGTH_INFINITE; + + simulate_single_hop_extend(client_side, relay_side, 1); + simulate_single_hop_extend(client_side, relay_side, 1); + + /* Now test the global limits by setting up the consensus */ + networkstatus_t vote1; + vote1.net_params = smartlist_new(); + smartlist_split_string(vote1.net_params, + "circpad_global_allowed_cells=100 circpad_global_max_padding_pct=50", + NULL, 0, 0); + /* Register global limits with the padding subsystem */ + circpad_new_consensus_params(&vote1); + + /* Check padding limit, should be fine since we haven't sent anything yet. */ + retval = circpad_machine_reached_padding_limit(mi); + tt_int_op(retval, OP_EQ, 0); + + /* Send 99 padding cells which is below circpad_global_allowed_cells=100, so + * the rate limit will not trigger */ + for (i=0;i<99;i++) { + circpad_send_padding_cell_for_callback(mi); + } + retval = circpad_machine_reached_padding_limit(mi); + tt_int_op(retval, OP_EQ, 0); + + /* Now send another padding cell to pass circpad_global_allowed_cells=100, + and see that the limit will trigger */ + circpad_send_padding_cell_for_callback(mi); + retval = circpad_machine_reached_padding_limit(mi); + tt_int_op(retval, OP_EQ, 1); + + retval = circpad_machine_schedule_padding(mi); + tt_int_op(retval, OP_EQ, CIRCPAD_STATE_UNCHANGED); + + /* Now send 92 non-padding cells to get near the + * circpad_global_max_padding_pct=50 limit; in particular with 96 non-padding + * cells, the padding traffic is still 51% of total traffic so limit should + * trigger */ + for (i=0;i<92;i++) { + circpad_cell_event_nonpadding_sent(relay_side); + } + retval = circpad_machine_reached_padding_limit(mi); + tt_int_op(retval, OP_EQ, 1); + + /* Send another non-padding cell to bring the padding traffic to 50% of total + * traffic and get past the limit */ + circpad_cell_event_nonpadding_sent(relay_side); + retval = circpad_machine_reached_padding_limit(mi); + tt_int_op(retval, OP_EQ, 0); + + done: + free_fake_orcirc(relay_side); + circuitmux_detach_all_circuits(dummy_channel.cmux, NULL); + circuitmux_free(dummy_channel.cmux); + SMARTLIST_FOREACH(vote1.net_params, char *, cp, tor_free(cp)); + smartlist_free(vote1.net_params); +} + +#define TEST_CIRCUITPADDING(name, flags) \ + { #name, test_##name, (flags), NULL, NULL } + +struct testcase_t circuitpadding_tests[] = { + TEST_CIRCUITPADDING(circuitpadding_tokens, TT_FORK), + TEST_CIRCUITPADDING(circuitpadding_negotiation, TT_FORK), + TEST_CIRCUITPADDING(circuitpadding_wronghop, TT_FORK), + TEST_CIRCUITPADDING(circuitpadding_conditions, TT_FORK), + TEST_CIRCUITPADDING(circuitpadding_rtt, TT_FORK), + TEST_CIRCUITPADDING(circuitpadding_sample_distribution, TT_FORK), + TEST_CIRCUITPADDING(circuitpadding_machine_rate_limiting, TT_FORK), + TEST_CIRCUITPADDING(circuitpadding_global_rate_limiting, TT_FORK), + TEST_CIRCUITPADDING(circuitpadding_token_removal_lower, TT_FORK), + TEST_CIRCUITPADDING(circuitpadding_token_removal_higher, TT_FORK), + TEST_CIRCUITPADDING(circuitpadding_closest_token_removal, TT_FORK), + TEST_CIRCUITPADDING(circuitpadding_closest_token_removal_usec, TT_FORK), + TEST_CIRCUITPADDING(circuitpadding_token_removal_exact, TT_FORK), + END_OF_TESTCASES +}; diff --git a/src/test/test_compat_libevent.c b/src/test/test_compat_libevent.c index 2f8646e897..5d625483da 100644 --- a/src/test/test_compat_libevent.c +++ b/src/test/test_compat_libevent.c @@ -187,4 +187,3 @@ struct testcase_t compat_libevent_tests[] = { TT_FORK, NULL, NULL }, END_OF_TESTCASES }; - diff --git a/src/test/test_config.c b/src/test/test_config.c index 8f011ce1f1..994016a710 100644 --- a/src/test/test_config.c +++ b/src/test/test_config.c @@ -54,6 +54,7 @@ #include "lib/meminfo/meminfo.h" #include "lib/net/gethostname.h" #include "lib/encoding/confline.h" +#include "lib/encoding/kvline.h" #ifdef HAVE_UNISTD_H #include <unistd.h> @@ -5847,6 +5848,7 @@ test_config_extended_fmt(void *arg) tt_str_op(lp->value, OP_EQ, "is back here"); tt_int_op(lp->command, OP_EQ, CONFIG_LINE_NORMAL); lp = lp->next; + tt_assert(!lp); config_free_lines(lines); /* Try with the "extended" flag enabled. */ @@ -5873,11 +5875,88 @@ test_config_extended_fmt(void *arg) tt_str_op(lp->value, OP_EQ, ""); tt_int_op(lp->command, OP_EQ, CONFIG_LINE_CLEAR); lp = lp->next; + tt_assert(!lp); done: config_free_lines(lines); } +static void +test_config_kvline_parse(void *arg) +{ + (void)arg; + + config_line_t *lines = NULL; + char *enc = NULL; + + lines = kvline_parse("A=B CD=EF", 0); + tt_assert(lines); + tt_str_op(lines->key, OP_EQ, "A"); + tt_str_op(lines->value, OP_EQ, "B"); + tt_str_op(lines->next->key, OP_EQ, "CD"); + tt_str_op(lines->next->value, OP_EQ, "EF"); + enc = kvline_encode(lines, 0); + tt_str_op(enc, OP_EQ, "A=B CD=EF"); + tor_free(enc); + enc = kvline_encode(lines, KV_QUOTED|KV_OMIT_KEYS); + tt_str_op(enc, OP_EQ, "A=B CD=EF"); + tor_free(enc); + config_free_lines(lines); + + lines = kvline_parse("AB CDE=F", 0); + tt_assert(! lines); + + lines = kvline_parse("AB CDE=F", KV_OMIT_KEYS); + tt_assert(lines); + tt_str_op(lines->key, OP_EQ, ""); + tt_str_op(lines->value, OP_EQ, "AB"); + tt_str_op(lines->next->key, OP_EQ, "CDE"); + tt_str_op(lines->next->value, OP_EQ, "F"); + tt_assert(lines); + enc = kvline_encode(lines, 0); + tt_assert(!enc); + enc = kvline_encode(lines, KV_QUOTED|KV_OMIT_KEYS); + tt_str_op(enc, OP_EQ, "AB CDE=F"); + tor_free(enc); + config_free_lines(lines); + + lines = kvline_parse("AB=C CDE=\"F G\"", 0); + tt_assert(!lines); + + lines = kvline_parse("AB=C CDE=\"F G\" \"GHI\" ", KV_QUOTED|KV_OMIT_KEYS); + tt_assert(lines); + tt_str_op(lines->key, OP_EQ, "AB"); + tt_str_op(lines->value, OP_EQ, "C"); + tt_str_op(lines->next->key, OP_EQ, "CDE"); + tt_str_op(lines->next->value, OP_EQ, "F G"); + tt_str_op(lines->next->next->key, OP_EQ, ""); + tt_str_op(lines->next->next->value, OP_EQ, "GHI"); + enc = kvline_encode(lines, 0); + tt_assert(!enc); + enc = kvline_encode(lines, KV_QUOTED|KV_OMIT_KEYS); + tt_str_op(enc, OP_EQ, "AB=C CDE=\"F G\" GHI"); + tor_free(enc); + config_free_lines(lines); + + lines = kvline_parse("A\"B=C CDE=\"F\" \"GHI\" ", KV_QUOTED|KV_OMIT_KEYS); + tt_assert(! lines); + + lines = kvline_parse("AB=", KV_QUOTED); + tt_assert(lines); + tt_str_op(lines->key, OP_EQ, "AB"); + tt_str_op(lines->value, OP_EQ, ""); + config_free_lines(lines); + + lines = kvline_parse("AB=", 0); + tt_assert(lines); + tt_str_op(lines->key, OP_EQ, "AB"); + tt_str_op(lines->value, OP_EQ, ""); + + done: + config_free_lines(lines); + tor_free(enc); +} + #define CONFIG_TEST(name, flags) \ { #name, test_config_ ## name, flags, NULL, NULL } @@ -5930,5 +6009,6 @@ struct testcase_t config_tests[] = { CONFIG_TEST(include_opened_file_list, 0), CONFIG_TEST(compute_max_mem_in_queues, 0), CONFIG_TEST(extended_fmt, 0), + CONFIG_TEST(kvline_parse, 0), END_OF_TESTCASES }; diff --git a/src/test/test_consdiff.c b/src/test/test_consdiff.c index 682ba5b970..7c4c92ea42 100644 --- a/src/test/test_consdiff.c +++ b/src/test/test_consdiff.c @@ -14,6 +14,39 @@ #define tt_str_eq_line(a,b) \ tt_assert(line_str_eq((b),(a))) +static int +consensus_split_lines_(smartlist_t *out, const char *s, memarea_t *area) +{ + size_t len = strlen(s); + return consensus_split_lines(out, s, len, area); +} + +static int +consensus_compute_digest_(const char *cons, + consensus_digest_t *digest_out) +{ + size_t len = strlen(cons); + char *tmp = tor_memdup(cons, len); + // We use memdup here to ensure that the input is NOT nul-terminated. + // This makes it likelier for us to spot bugs. + int r = consensus_compute_digest(tmp, len, digest_out); + tor_free(tmp); + return r; +} + +static int +consensus_compute_digest_as_signed_(const char *cons, + consensus_digest_t *digest_out) +{ + size_t len = strlen(cons); + char *tmp = tor_memdup(cons, len); + // We use memdup here to ensure that the input is NOT nul-terminated. + // This makes it likelier for us to spot bugs. + int r = consensus_compute_digest_as_signed(tmp, len, digest_out); + tor_free(tmp); + return r; +} + static void test_consdiff_smartlist_slice(void *arg) { @@ -58,7 +91,7 @@ test_consdiff_smartlist_slice_string_pos(void *arg) /* Create a regular smartlist. */ (void)arg; - consensus_split_lines(sl, "a\nd\nc\na\nb\n", area); + consensus_split_lines_(sl, "a\nd\nc\na\nb\n", area); /* See that smartlist_slice_string_pos respects the bounds of the slice. */ sls = smartlist_slice(sl, 2, 5); @@ -87,8 +120,8 @@ test_consdiff_lcs_lengths(void *arg) int e_lengths2[] = { 0, 1, 1, 2, 3, 4 }; (void)arg; - consensus_split_lines(sl1, "a\nb\nc\nd\ne\n", area); - consensus_split_lines(sl2, "a\nc\nd\ni\ne\n", area); + consensus_split_lines_(sl1, "a\nb\nc\nd\ne\n", area); + consensus_split_lines_(sl2, "a\nc\nd\ni\ne\n", area); sls1 = smartlist_slice(sl1, 0, -1); sls2 = smartlist_slice(sl2, 0, -1); @@ -119,10 +152,10 @@ test_consdiff_trim_slices(void *arg) memarea_t *area = memarea_new(); (void)arg; - consensus_split_lines(sl1, "a\nb\nb\nb\nd\n", area); - consensus_split_lines(sl2, "a\nc\nc\nc\nd\n", area); - consensus_split_lines(sl3, "a\nb\nb\nb\na\n", area); - consensus_split_lines(sl4, "c\nb\nb\nb\nc\n", area); + consensus_split_lines_(sl1, "a\nb\nb\nb\nd\n", area); + consensus_split_lines_(sl2, "a\nc\nc\nc\nd\n", area); + consensus_split_lines_(sl3, "a\nb\nb\nb\na\n", area); + consensus_split_lines_(sl4, "c\nb\nb\nb\nc\n", area); sls1 = smartlist_slice(sl1, 0, -1); sls2 = smartlist_slice(sl2, 0, -1); sls3 = smartlist_slice(sl3, 0, -1); @@ -165,8 +198,8 @@ test_consdiff_set_changed(void *arg) memarea_t *area = memarea_new(); (void)arg; - consensus_split_lines(sl1, "a\nb\na\na\n", area); - consensus_split_lines(sl2, "a\na\na\na\n", area); + consensus_split_lines_(sl1, "a\nb\na\na\n", area); + consensus_split_lines_(sl2, "a\na\na\na\n", area); /* Length of sls1 is 0. */ sls1 = smartlist_slice(sl1, 0, 0); @@ -240,8 +273,8 @@ test_consdiff_calc_changes(void *arg) memarea_t *area = memarea_new(); (void)arg; - consensus_split_lines(sl1, "a\na\na\na\n", area); - consensus_split_lines(sl2, "a\na\na\na\n", area); + consensus_split_lines_(sl1, "a\na\na\na\n", area); + consensus_split_lines_(sl2, "a\na\na\na\n", area); sls1 = smartlist_slice(sl1, 0, -1); sls2 = smartlist_slice(sl2, 0, -1); @@ -259,7 +292,7 @@ test_consdiff_calc_changes(void *arg) tt_assert(!bitarray_is_set(changed2, 3)); smartlist_clear(sl2); - consensus_split_lines(sl2, "a\nb\na\nb\n", area); + consensus_split_lines_(sl2, "a\nb\na\nb\n", area); tor_free(sls1); tor_free(sls2); sls1 = smartlist_slice(sl1, 0, -1); @@ -282,7 +315,7 @@ test_consdiff_calc_changes(void *arg) bitarray_clear(changed1, 3); smartlist_clear(sl2); - consensus_split_lines(sl2, "b\nb\nb\nb\n", area); + consensus_split_lines_(sl2, "b\nb\nb\nb\n", area); tor_free(sls1); tor_free(sls2); sls1 = smartlist_slice(sl1, 0, -1); @@ -610,8 +643,8 @@ test_consdiff_gen_ed_diff(void *arg) /* Test 'a', 'c' and 'd' together. See that it is done in reverse order. */ smartlist_clear(cons1); smartlist_clear(cons2); - consensus_split_lines(cons1, "A\nB\nC\nD\nE\n", area); - consensus_split_lines(cons2, "A\nC\nO\nE\nU\n", area); + consensus_split_lines_(cons1, "A\nB\nC\nD\nE\n", area); + consensus_split_lines_(cons2, "A\nC\nO\nE\nU\n", area); diff = gen_ed_diff(cons1, cons2, area); tt_ptr_op(NULL, OP_NE, diff); tt_int_op(7, OP_EQ, smartlist_len(diff)); @@ -627,8 +660,8 @@ test_consdiff_gen_ed_diff(void *arg) smartlist_clear(cons1); smartlist_clear(cons2); - consensus_split_lines(cons1, "B\n", area); - consensus_split_lines(cons2, "A\nB\n", area); + consensus_split_lines_(cons1, "B\n", area); + consensus_split_lines_(cons2, "A\nB\n", area); diff = gen_ed_diff(cons1, cons2, area); tt_ptr_op(NULL, OP_NE, diff); tt_int_op(3, OP_EQ, smartlist_len(diff)); @@ -656,7 +689,7 @@ test_consdiff_apply_ed_diff(void *arg) diff = smartlist_new(); setup_capture_of_logs(LOG_WARN); - consensus_split_lines(cons1, "A\nB\nC\nD\nE\n", area); + consensus_split_lines_(cons1, "A\nB\nC\nD\nE\n", area); /* Command without range. */ smartlist_add_linecpy(diff, area, "a"); @@ -829,7 +862,7 @@ test_consdiff_apply_ed_diff(void *arg) smartlist_clear(diff); /* Test appending text, 'a'. */ - consensus_split_lines(diff, "3a\nU\nO\n.\n0a\nV\n.\n", area); + consensus_split_lines_(diff, "3a\nU\nO\n.\n0a\nV\n.\n", area); cons2 = apply_ed_diff(cons1, diff, 0); tt_ptr_op(NULL, OP_NE, cons2); tt_int_op(8, OP_EQ, smartlist_len(cons2)); @@ -846,7 +879,7 @@ test_consdiff_apply_ed_diff(void *arg) smartlist_free(cons2); /* Test deleting text, 'd'. */ - consensus_split_lines(diff, "4d\n1,2d\n", area); + consensus_split_lines_(diff, "4d\n1,2d\n", area); cons2 = apply_ed_diff(cons1, diff, 0); tt_ptr_op(NULL, OP_NE, cons2); tt_int_op(2, OP_EQ, smartlist_len(cons2)); @@ -857,7 +890,7 @@ test_consdiff_apply_ed_diff(void *arg) smartlist_free(cons2); /* Test changing text, 'c'. */ - consensus_split_lines(diff, "4c\nT\nX\n.\n1,2c\nM\n.\n", area); + consensus_split_lines_(diff, "4c\nT\nX\n.\n1,2c\nM\n.\n", area); cons2 = apply_ed_diff(cons1, diff, 0); tt_ptr_op(NULL, OP_NE, cons2); tt_int_op(5, OP_EQ, smartlist_len(cons2)); @@ -871,7 +904,7 @@ test_consdiff_apply_ed_diff(void *arg) smartlist_free(cons2); /* Test 'a', 'd' and 'c' together. */ - consensus_split_lines(diff, "4c\nT\nX\n.\n2d\n0a\nM\n.\n", area); + consensus_split_lines_(diff, "4c\nT\nX\n.\n2d\n0a\nM\n.\n", area); cons2 = apply_ed_diff(cons1, diff, 0); tt_ptr_op(NULL, OP_NE, cons2); tt_int_op(6, OP_EQ, smartlist_len(cons2)); @@ -918,12 +951,12 @@ test_consdiff_gen_diff(void *arg) ); tt_int_op(0, OP_EQ, - consensus_compute_digest_as_signed(cons1_str, &digests1)); + consensus_compute_digest_as_signed_(cons1_str, &digests1)); tt_int_op(0, OP_EQ, - consensus_compute_digest(cons2_str, &digests2)); + consensus_compute_digest_(cons2_str, &digests2)); - consensus_split_lines(cons1, cons1_str, area); - consensus_split_lines(cons2, cons2_str, area); + consensus_split_lines_(cons1, cons1_str, area); + consensus_split_lines_(cons2, cons2_str, area); diff = consdiff_gen_diff(cons1, cons2, &digests1, &digests2, area); tt_ptr_op(NULL, OP_EQ, diff); @@ -937,9 +970,9 @@ test_consdiff_gen_diff(void *arg) "directory-signature foo bar\nbar\n" ); tt_int_op(0, OP_EQ, - consensus_compute_digest_as_signed(cons1_str, &digests1)); + consensus_compute_digest_as_signed_(cons1_str, &digests1)); smartlist_clear(cons1); - consensus_split_lines(cons1, cons1_str, area); + consensus_split_lines_(cons1, cons1_str, area); diff = consdiff_gen_diff(cons1, cons2, &digests1, &digests2, area); tt_ptr_op(NULL, OP_NE, diff); tt_int_op(11, OP_EQ, smartlist_len(diff)); @@ -991,8 +1024,8 @@ test_consdiff_apply_diff(void *arg) "directory-signature foo bar\nbar\n" ); tt_int_op(0, OP_EQ, - consensus_compute_digest(cons1_str, &digests1)); - consensus_split_lines(cons1, cons1_str, area); + consensus_compute_digest_(cons1_str, &digests1)); + consensus_split_lines_(cons1, cons1_str, area); /* diff doesn't have enough lines. */ cons2 = consdiff_apply_diff(cons1, diff, &digests1); @@ -1182,4 +1215,3 @@ struct testcase_t consdiff_tests[] = { CONSDIFF_LEGACY(apply_diff), END_OF_TESTCASES }; - diff --git a/src/test/test_consdiffmgr.c b/src/test/test_consdiffmgr.c index 254a5ba5d0..74226b8c52 100644 --- a/src/test/test_consdiffmgr.c +++ b/src/test/test_consdiffmgr.c @@ -21,6 +21,23 @@ #include "test/test.h" #include "test/log_test_helpers.h" +#define consdiffmgr_add_consensus consdiffmgr_add_consensus_nulterm + +static char * +consensus_diff_apply_(const char *c, const char *d) +{ + size_t c_len = strlen(c); + size_t d_len = strlen(d); + // We use memdup here to ensure that the input is NOT nul-terminated. + // This makes it likelier for us to spot bugs. + char *c_tmp = tor_memdup(c, c_len); + char *d_tmp = tor_memdup(d, d_len); + char *result = consensus_diff_apply(c_tmp, c_len, d_tmp, d_len); + tor_free(c_tmp); + tor_free(d_tmp); + return result; +} + // ============================== Setup/teardown the consdiffmgr // These functions get run before/after each test in this module @@ -153,7 +170,8 @@ lookup_diff_from(consensus_cache_entry_t **out, const char *str1) { uint8_t digest[DIGEST256_LEN]; - if (router_get_networkstatus_v3_sha3_as_signed(digest, str1)<0) { + if (router_get_networkstatus_v3_sha3_as_signed(digest, + str1, strlen(str1))<0) { TT_FAIL(("Unable to compute sha3-as-signed")); return CONSDIFF_NOT_FOUND; } @@ -175,14 +193,15 @@ lookup_apply_and_verify_diff(consensus_flavor_t flav, consensus_cache_entry_incref(ent); size_t size; - char *diff_string = NULL; - int r = uncompress_or_copy(&diff_string, &size, ent); + const char *diff_string = NULL; + char *diff_owned = NULL; + int r = uncompress_or_set_ptr(&diff_string, &size, &diff_owned, ent); consensus_cache_entry_decref(ent); if (diff_string == NULL || r < 0) return -1; - char *applied = consensus_diff_apply(str1, diff_string); - tor_free(diff_string); + char *applied = consensus_diff_apply(str1, strlen(str1), diff_string, size); + tor_free(diff_owned); if (applied == NULL) return -1; @@ -282,7 +301,8 @@ test_consdiffmgr_add(void *arg) (void) arg; time_t now = approx_time(); - char *body = NULL; + const char *body = NULL; + char *body_owned = NULL; consensus_cache_entry_t *ent = NULL; networkstatus_t *ns_tmp = fake_ns_new(FLAV_NS, now); @@ -324,7 +344,7 @@ test_consdiffmgr_add(void *arg) tt_assert(ent); consensus_cache_entry_incref(ent); size_t s; - r = uncompress_or_copy(&body, &s, ent); + r = uncompress_or_set_ptr(&body, &s, &body_owned, ent); tt_int_op(r, OP_EQ, 0); tt_int_op(s, OP_EQ, 4); tt_mem_op(body, OP_EQ, "quux", 4); @@ -337,7 +357,7 @@ test_consdiffmgr_add(void *arg) networkstatus_vote_free(ns_tmp); teardown_capture_of_logs(); consensus_cache_entry_decref(ent); - tor_free(body); + tor_free(body_owned); } static void @@ -370,7 +390,8 @@ test_consdiffmgr_make_diffs(void *arg) ns = fake_ns_new(FLAV_MICRODESC, now-3600); md_ns_body = fake_ns_body_new(FLAV_MICRODESC, now-3600); r = consdiffmgr_add_consensus(md_ns_body, ns); - router_get_networkstatus_v3_sha3_as_signed(md_ns_sha3, md_ns_body); + router_get_networkstatus_v3_sha3_as_signed(md_ns_sha3, md_ns_body, + strlen(md_ns_body)); networkstatus_vote_free(ns); tt_int_op(r, OP_EQ, 0); @@ -414,7 +435,7 @@ test_consdiffmgr_make_diffs(void *arg) r = consensus_cache_entry_get_body(diff, &diff_body, &diff_size); tt_int_op(r, OP_EQ, 0); diff_text = tor_memdup_nulterm(diff_body, diff_size); - applied = consensus_diff_apply(md_ns_body, diff_text); + applied = consensus_diff_apply_(md_ns_body, diff_text); tt_assert(applied); tt_str_op(applied, OP_EQ, md_ns_body_2); diff --git a/src/test/test_containers.c b/src/test/test_containers.c index aedd2f7a89..a0832f868e 100644 --- a/src/test/test_containers.c +++ b/src/test/test_containers.c @@ -96,6 +96,30 @@ test_container_smartlist_basic(void *arg) tor_free(v555); } +/** Test SMARTLIST_FOREACH_REVERSE_BEGIN loop macro */ +static void +test_container_smartlist_foreach_reverse(void *arg) +{ + smartlist_t *sl = smartlist_new(); + int i; + + (void) arg; + + /* Add integers to smartlist in increasing order */ + for (i=0;i<100;i++) { + smartlist_add(sl, (void*)(uintptr_t)i); + } + + /* Pop them out in reverse and test their value */ + SMARTLIST_FOREACH_REVERSE_BEGIN(sl, void*, k) { + i--; + tt_ptr_op(k, OP_EQ, (void*)(uintptr_t)i); + } SMARTLIST_FOREACH_END(k); + + done: + smartlist_free(sl); +} + /** Run unit tests for smartlist-of-strings functionality. */ static void test_container_smartlist_strings(void *arg) @@ -1281,6 +1305,7 @@ test_container_smartlist_strings_eq(void *arg) struct testcase_t container_tests[] = { CONTAINER_LEGACY(smartlist_basic), CONTAINER_LEGACY(smartlist_strings), + CONTAINER_LEGACY(smartlist_foreach_reverse), CONTAINER_LEGACY(smartlist_overlap), CONTAINER_LEGACY(smartlist_digests), CONTAINER_LEGACY(smartlist_join), diff --git a/src/test/test_controller_events.c b/src/test/test_controller_events.c index 70d36e53d4..647eac43c7 100644 --- a/src/test/test_controller_events.c +++ b/src/test/test_controller_events.c @@ -4,10 +4,14 @@ #define CONNECTION_PRIVATE #define TOR_CHANNEL_INTERNAL_ #define CONTROL_PRIVATE +#define OCIRC_EVENT_PRIVATE +#define ORCONN_EVENT_PRIVATE #include "core/or/or.h" #include "core/or/channel.h" #include "core/or/channeltls.h" #include "core/or/circuitlist.h" +#include "core/or/ocirc_event.h" +#include "core/or/orconn_event.h" #include "core/mainloop/connection.h" #include "feature/control/control.h" #include "test/test.h" @@ -351,10 +355,10 @@ test_cntev_dirboot_defer_desc(void *arg) /* This event should get deferred */ control_event_boot_dir(BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS, 0); assert_bootmsg("0 TAG=starting"); - control_event_bootstrap(BOOTSTRAP_STATUS_CONN_DIR, 0); - assert_bootmsg("5 TAG=conn_dir"); + control_event_bootstrap(BOOTSTRAP_STATUS_CONN, 0); + assert_bootmsg("5 TAG=conn"); control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE, 0); - assert_bootmsg("10 TAG=handshake_dir"); + assert_bootmsg("14 TAG=handshake"); /* The deferred event should appear */ control_event_boot_first_orconn(); assert_bootmsg("45 TAG=requesting_descriptors"); @@ -374,21 +378,159 @@ test_cntev_dirboot_defer_orconn(void *arg) control_event_bootstrap(BOOTSTRAP_STATUS_STARTING, 0); assert_bootmsg("0 TAG=starting"); /* This event should get deferred */ - control_event_boot_dir(BOOTSTRAP_STATUS_CONN_OR, 0); + control_event_boot_dir(BOOTSTRAP_STATUS_ENOUGH_DIRINFO, 0); assert_bootmsg("0 TAG=starting"); - control_event_bootstrap(BOOTSTRAP_STATUS_CONN_DIR, 0); - assert_bootmsg("5 TAG=conn_dir"); + control_event_bootstrap(BOOTSTRAP_STATUS_CONN, 0); + assert_bootmsg("5 TAG=conn"); control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE, 0); - assert_bootmsg("10 TAG=handshake_dir"); + assert_bootmsg("14 TAG=handshake"); /* The deferred event should appear */ control_event_boot_first_orconn(); - assert_bootmsg("80 TAG=conn_or"); + assert_bootmsg("75 TAG=enough_dirinfo"); done: tor_free(saved_event_str); UNMOCK(queue_control_event_string); } -#define TEST(name, flags) \ +static void +setup_orconn_state(orconn_event_msg_t *msg, uint64_t gid, uint64_t chan, + int proxy_type) +{ + msg->type = ORCONN_MSGTYPE_STATE; + msg->u.state.gid = gid; + msg->u.state.chan = chan; + msg->u.state.proxy_type = proxy_type; +} + +static void +send_orconn_state(orconn_event_msg_t *msg, uint8_t state) +{ + msg->u.state.state = state; + orconn_event_publish(msg); +} + +static void +send_ocirc_chan(uint32_t gid, uint64_t chan, bool onehop) +{ + ocirc_event_msg_t msg; + + msg.type = OCIRC_MSGTYPE_CHAN; + msg.u.chan.gid = gid; + msg.u.chan.chan = chan; + msg.u.chan.onehop = onehop; + ocirc_event_publish(&msg); +} + +static void +test_cntev_orconn_state(void *arg) +{ + orconn_event_msg_t conn; + + (void)arg; + MOCK(queue_control_event_string, mock_queue_control_event_string); + control_testing_set_global_event_mask(EVENT_MASK_(EVENT_STATUS_CLIENT)); + setup_orconn_state(&conn, 1, 1, PROXY_NONE); + + send_orconn_state(&conn, OR_CONN_STATE_CONNECTING); + send_ocirc_chan(1, 1, true); + assert_bootmsg("5 TAG=conn"); + send_orconn_state(&conn, OR_CONN_STATE_TLS_HANDSHAKING); + assert_bootmsg("10 TAG=conn_done"); + send_orconn_state(&conn, OR_CONN_STATE_OR_HANDSHAKING_V3); + assert_bootmsg("14 TAG=handshake"); + send_orconn_state(&conn, OR_CONN_STATE_OPEN); + assert_bootmsg("15 TAG=handshake_done"); + + conn.u.state.gid = 2; + conn.u.state.chan = 2; + send_orconn_state(&conn, OR_CONN_STATE_CONNECTING); + /* It doesn't know it's an origin circuit yet */ + assert_bootmsg("15 TAG=handshake_done"); + send_ocirc_chan(2, 2, false); + assert_bootmsg("80 TAG=ap_conn"); + send_orconn_state(&conn, OR_CONN_STATE_TLS_HANDSHAKING); + assert_bootmsg("85 TAG=ap_conn_done"); + send_orconn_state(&conn, OR_CONN_STATE_OR_HANDSHAKING_V3); + assert_bootmsg("89 TAG=ap_handshake"); + send_orconn_state(&conn, OR_CONN_STATE_OPEN); + assert_bootmsg("90 TAG=ap_handshake_done"); + + done: + tor_free(saved_event_str); + UNMOCK(queue_control_event_string); +} + +static void +test_cntev_orconn_state_pt(void *arg) +{ + orconn_event_msg_t conn; + + (void)arg; + MOCK(queue_control_event_string, mock_queue_control_event_string); + control_testing_set_global_event_mask(EVENT_MASK_(EVENT_STATUS_CLIENT)); + setup_orconn_state(&conn, 1, 1, PROXY_PLUGGABLE); + send_ocirc_chan(1, 1, true); + + send_orconn_state(&conn, OR_CONN_STATE_CONNECTING); + assert_bootmsg("1 TAG=conn_pt"); + send_orconn_state(&conn, OR_CONN_STATE_PROXY_HANDSHAKING); + assert_bootmsg("2 TAG=conn_done_pt"); + send_orconn_state(&conn, OR_CONN_STATE_TLS_HANDSHAKING); + assert_bootmsg("10 TAG=conn_done"); + send_orconn_state(&conn, OR_CONN_STATE_OR_HANDSHAKING_V3); + assert_bootmsg("14 TAG=handshake"); + send_orconn_state(&conn, OR_CONN_STATE_OPEN); + assert_bootmsg("15 TAG=handshake_done"); + + send_ocirc_chan(2, 2, false); + conn.u.state.gid = 2; + conn.u.state.chan = 2; + send_orconn_state(&conn, OR_CONN_STATE_CONNECTING); + assert_bootmsg("76 TAG=ap_conn_pt"); + send_orconn_state(&conn, OR_CONN_STATE_PROXY_HANDSHAKING); + assert_bootmsg("77 TAG=ap_conn_done_pt"); + + done: + tor_free(saved_event_str); + UNMOCK(queue_control_event_string); +} + +static void +test_cntev_orconn_state_proxy(void *arg) +{ + orconn_event_msg_t conn; + + (void)arg; + MOCK(queue_control_event_string, mock_queue_control_event_string); + control_testing_set_global_event_mask(EVENT_MASK_(EVENT_STATUS_CLIENT)); + setup_orconn_state(&conn, 1, 1, PROXY_CONNECT); + send_ocirc_chan(1, 1, true); + + send_orconn_state(&conn, OR_CONN_STATE_CONNECTING); + assert_bootmsg("3 TAG=conn_proxy"); + send_orconn_state(&conn, OR_CONN_STATE_PROXY_HANDSHAKING); + assert_bootmsg("4 TAG=conn_done_proxy"); + send_orconn_state(&conn, OR_CONN_STATE_TLS_HANDSHAKING); + assert_bootmsg("10 TAG=conn_done"); + send_orconn_state(&conn, OR_CONN_STATE_OR_HANDSHAKING_V3); + assert_bootmsg("14 TAG=handshake"); + send_orconn_state(&conn, OR_CONN_STATE_OPEN); + assert_bootmsg("15 TAG=handshake_done"); + + send_ocirc_chan(2, 2, false); + conn.u.state.gid = 2; + conn.u.state.chan = 2; + send_orconn_state(&conn, OR_CONN_STATE_CONNECTING); + assert_bootmsg("78 TAG=ap_conn_proxy"); + send_orconn_state(&conn, OR_CONN_STATE_PROXY_HANDSHAKING); + assert_bootmsg("79 TAG=ap_conn_done_proxy"); + + done: + tor_free(saved_event_str); + UNMOCK(queue_control_event_string); +} + +#define TEST(name, flags) \ { #name, test_cntev_ ## name, flags, 0, NULL } struct testcase_t controller_event_tests[] = { @@ -398,5 +540,8 @@ struct testcase_t controller_event_tests[] = { TEST(event_mask, TT_FORK), TEST(dirboot_defer_desc, TT_FORK), TEST(dirboot_defer_orconn, TT_FORK), + TEST(orconn_state, TT_FORK), + TEST(orconn_state_pt, TT_FORK), + TEST(orconn_state_proxy, TT_FORK), END_OF_TESTCASES }; diff --git a/src/test/test_crypto.c b/src/test/test_crypto.c index fa79f4cc47..0b57448bcf 100644 --- a/src/test/test_crypto.c +++ b/src/test/test_crypto.c @@ -254,168 +254,6 @@ test_crypto_openssl_version(void *arg) ; } -/** Run unit tests for our random number generation function and its wrappers. - */ -static void -test_crypto_rng(void *arg) -{ - int i, j, allok; - char data1[100], data2[100]; - double d; - char *h=NULL; - - /* Try out RNG. */ - (void)arg; - tt_assert(! crypto_seed_rng()); - crypto_rand(data1, 100); - crypto_rand(data2, 100); - tt_mem_op(data1,OP_NE, data2,100); - allok = 1; - for (i = 0; i < 100; ++i) { - uint64_t big; - char *host; - j = crypto_rand_int(100); - if (j < 0 || j >= 100) - allok = 0; - big = crypto_rand_uint64(UINT64_C(1)<<40); - if (big >= (UINT64_C(1)<<40)) - allok = 0; - big = crypto_rand_uint64(UINT64_C(5)); - if (big >= 5) - allok = 0; - d = crypto_rand_double(); - tt_assert(d >= 0); - tt_assert(d < 1.0); - host = crypto_random_hostname(3,8,"www.",".onion"); - if (strcmpstart(host,"www.") || - strcmpend(host,".onion") || - strlen(host) < 13 || - strlen(host) > 18) - allok = 0; - tor_free(host); - } - - /* Make sure crypto_random_hostname clips its inputs properly. */ - h = crypto_random_hostname(20000, 9000, "www.", ".onion"); - tt_assert(! strcmpstart(h,"www.")); - tt_assert(! strcmpend(h,".onion")); - tt_int_op(63+4+6, OP_EQ, strlen(h)); - - tt_assert(allok); - done: - tor_free(h); -} - -static void -test_crypto_rng_range(void *arg) -{ - int got_smallest = 0, got_largest = 0; - int i; - - (void)arg; - for (i = 0; i < 1000; ++i) { - int x = crypto_rand_int_range(5,9); - tt_int_op(x, OP_GE, 5); - tt_int_op(x, OP_LT, 9); - if (x == 5) - got_smallest = 1; - if (x == 8) - got_largest = 1; - } - /* These fail with probability 1/10^603. */ - tt_assert(got_smallest); - tt_assert(got_largest); - - got_smallest = got_largest = 0; - const uint64_t ten_billion = 10 * ((uint64_t)1000000000000); - for (i = 0; i < 1000; ++i) { - uint64_t x = crypto_rand_uint64_range(ten_billion, ten_billion+10); - tt_u64_op(x, OP_GE, ten_billion); - tt_u64_op(x, OP_LT, ten_billion+10); - if (x == ten_billion) - got_smallest = 1; - if (x == ten_billion+9) - got_largest = 1; - } - - tt_assert(got_smallest); - tt_assert(got_largest); - - const time_t now = time(NULL); - for (i = 0; i < 2000; ++i) { - time_t x = crypto_rand_time_range(now, now+60); - tt_i64_op(x, OP_GE, now); - tt_i64_op(x, OP_LT, now+60); - if (x == now) - got_smallest = 1; - if (x == now+59) - got_largest = 1; - } - - tt_assert(got_smallest); - tt_assert(got_largest); - done: - ; -} - -static void -test_crypto_rng_strongest(void *arg) -{ - const char *how = arg; - int broken = 0; - - if (how == NULL) { - ; - } else if (!strcmp(how, "nosyscall")) { - break_strongest_rng_syscall = 1; - } else if (!strcmp(how, "nofallback")) { - break_strongest_rng_fallback = 1; - } else if (!strcmp(how, "broken")) { - broken = break_strongest_rng_syscall = break_strongest_rng_fallback = 1; - } - -#define N 128 - uint8_t combine_and[N]; - uint8_t combine_or[N]; - int i, j; - - memset(combine_and, 0xff, N); - memset(combine_or, 0, N); - - for (i = 0; i < 100; ++i) { /* 2^-100 chances just don't happen. */ - uint8_t output[N]; - memset(output, 0, N); - if (how == NULL) { - /* this one can't fail. */ - crypto_strongest_rand(output, sizeof(output)); - } else { - int r = crypto_strongest_rand_raw(output, sizeof(output)); - if (r == -1) { - if (broken) { - goto done; /* we're fine. */ - } - /* This function is allowed to break, but only if it always breaks. */ - tt_int_op(i, OP_EQ, 0); - tt_skip(); - } else { - tt_assert(! broken); - } - } - for (j = 0; j < N; ++j) { - combine_and[j] &= output[j]; - combine_or[j] |= output[j]; - } - } - - for (j = 0; j < N; ++j) { - tt_int_op(combine_and[j], OP_EQ, 0); - tt_int_op(combine_or[j], OP_EQ, 0xff); - } - done: - ; -#undef N -} - /** Run unit tests for our AES128 functionality */ static void test_crypto_aes128(void *arg) @@ -3140,15 +2978,6 @@ test_crypto_failure_modes(void *arg) struct testcase_t crypto_tests[] = { CRYPTO_LEGACY(formats), - CRYPTO_LEGACY(rng), - { "rng_range", test_crypto_rng_range, 0, NULL, NULL }, - { "rng_strongest", test_crypto_rng_strongest, TT_FORK, NULL, NULL }, - { "rng_strongest_nosyscall", test_crypto_rng_strongest, TT_FORK, - &passthrough_setup, (void*)"nosyscall" }, - { "rng_strongest_nofallback", test_crypto_rng_strongest, TT_FORK, - &passthrough_setup, (void*)"nofallback" }, - { "rng_strongest_broken", test_crypto_rng_strongest, TT_FORK, - &passthrough_setup, (void*)"broken" }, { "openssl_version", test_crypto_openssl_version, TT_FORK, NULL, NULL }, { "aes_AES", test_crypto_aes128, TT_FORK, &passthrough_setup, (void*)"aes" }, { "aes_EVP", test_crypto_aes128, TT_FORK, &passthrough_setup, (void*)"evp" }, diff --git a/src/test/test_crypto_rng.c b/src/test/test_crypto_rng.c new file mode 100644 index 0000000000..23b0c66514 --- /dev/null +++ b/src/test/test_crypto_rng.c @@ -0,0 +1,324 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#define CRYPTO_RAND_PRIVATE +#include "core/or/or.h" +#include "test/test.h" +#include "lib/crypt_ops/aes.h" +#include "lib/crypt_ops/crypto_format.h" +#include "lib/crypt_ops/crypto_rand.h" + +/** Run unit tests for our random number generation function and its wrappers. + */ +static void +test_crypto_rng(void *arg) +{ + int i, j, allok; + char data1[100], data2[100]; + double d; + char *h=NULL; + + /* Try out RNG. */ + (void)arg; + tt_assert(! crypto_seed_rng()); + crypto_rand(data1, 100); + crypto_rand(data2, 100); + tt_mem_op(data1,OP_NE, data2,100); + allok = 1; + for (i = 0; i < 100; ++i) { + uint64_t big; + char *host; + j = crypto_rand_int(100); + if (j < 0 || j >= 100) + allok = 0; + big = crypto_rand_uint64(UINT64_C(1)<<40); + if (big >= (UINT64_C(1)<<40)) + allok = 0; + big = crypto_rand_uint64(UINT64_C(5)); + if (big >= 5) + allok = 0; + d = crypto_rand_double(); + tt_assert(d >= 0); + tt_assert(d < 1.0); + host = crypto_random_hostname(3,8,"www.",".onion"); + if (strcmpstart(host,"www.") || + strcmpend(host,".onion") || + strlen(host) < 13 || + strlen(host) > 18) + allok = 0; + tor_free(host); + } + + /* Make sure crypto_random_hostname clips its inputs properly. */ + h = crypto_random_hostname(20000, 9000, "www.", ".onion"); + tt_assert(! strcmpstart(h,"www.")); + tt_assert(! strcmpend(h,".onion")); + tt_int_op(63+4+6, OP_EQ, strlen(h)); + + tt_assert(allok); + done: + tor_free(h); +} + +static void +test_crypto_rng_range(void *arg) +{ + int got_smallest = 0, got_largest = 0; + int i; + + (void)arg; + for (i = 0; i < 1000; ++i) { + int x = crypto_rand_int_range(5,9); + tt_int_op(x, OP_GE, 5); + tt_int_op(x, OP_LT, 9); + if (x == 5) + got_smallest = 1; + if (x == 8) + got_largest = 1; + } + /* These fail with probability 1/10^603. */ + tt_assert(got_smallest); + tt_assert(got_largest); + + got_smallest = got_largest = 0; + const uint64_t ten_billion = 10 * ((uint64_t)1000000000000); + for (i = 0; i < 1000; ++i) { + uint64_t x = crypto_rand_uint64_range(ten_billion, ten_billion+10); + tt_u64_op(x, OP_GE, ten_billion); + tt_u64_op(x, OP_LT, ten_billion+10); + if (x == ten_billion) + got_smallest = 1; + if (x == ten_billion+9) + got_largest = 1; + } + + tt_assert(got_smallest); + tt_assert(got_largest); + + const time_t now = time(NULL); + for (i = 0; i < 2000; ++i) { + time_t x = crypto_rand_time_range(now, now+60); + tt_i64_op(x, OP_GE, now); + tt_i64_op(x, OP_LT, now+60); + if (x == now) + got_smallest = 1; + if (x == now+59) + got_largest = 1; + } + + tt_assert(got_smallest); + tt_assert(got_largest); + done: + ; +} + +static void +test_crypto_rng_strongest(void *arg) +{ + const char *how = arg; + int broken = 0; + + if (how == NULL) { + ; + } else if (!strcmp(how, "nosyscall")) { + break_strongest_rng_syscall = 1; + } else if (!strcmp(how, "nofallback")) { + break_strongest_rng_fallback = 1; + } else if (!strcmp(how, "broken")) { + broken = break_strongest_rng_syscall = break_strongest_rng_fallback = 1; + } + +#define N 128 + uint8_t combine_and[N]; + uint8_t combine_or[N]; + int i, j; + + memset(combine_and, 0xff, N); + memset(combine_or, 0, N); + + for (i = 0; i < 100; ++i) { /* 2^-100 chances just don't happen. */ + uint8_t output[N]; + memset(output, 0, N); + if (how == NULL) { + /* this one can't fail. */ + crypto_strongest_rand(output, sizeof(output)); + } else { + int r = crypto_strongest_rand_raw(output, sizeof(output)); + if (r == -1) { + if (broken) { + goto done; /* we're fine. */ + } + /* This function is allowed to break, but only if it always breaks. */ + tt_int_op(i, OP_EQ, 0); + tt_skip(); + } else { + tt_assert(! broken); + } + } + for (j = 0; j < N; ++j) { + combine_and[j] &= output[j]; + combine_or[j] |= output[j]; + } + } + + for (j = 0; j < N; ++j) { + tt_int_op(combine_and[j], OP_EQ, 0); + tt_int_op(combine_or[j], OP_EQ, 0xff); + } + done: + ; +#undef N +} + +static void +test_crypto_rng_fast(void *arg) +{ + (void)arg; + crypto_fast_rng_t *rng = crypto_fast_rng_new(); + tt_assert(rng); + + /* Rudimentary black-block test to make sure that our prng outputs + * have all bits sometimes on and all bits sometimes off. */ + uint64_t m1 = 0, m2 = ~(uint64_t)0; + const int N = 128; + + for (int i=0; i < N; ++i) { + uint64_t v; + crypto_fast_rng_getbytes(rng, (void*)&v, sizeof(v)); + m1 |= v; + m2 &= v; + } + + tt_u64_op(m1, OP_EQ, ~(uint64_t)0); + tt_u64_op(m2, OP_EQ, 0); + + /* Check range functions. */ + int counts[5]; + memset(counts, 0, sizeof(counts)); + for (int i=0; i < N; ++i) { + unsigned u = crypto_fast_rng_get_uint(rng, 5); + tt_int_op(u, OP_GE, 0); + tt_int_op(u, OP_LT, 5); + counts[u]++; + + uint64_t u64 = crypto_fast_rng_get_uint64(rng, UINT64_C(1)<<40); + tt_u64_op(u64, OP_GE, 0); + tt_u64_op(u64, OP_LT, UINT64_C(1)<<40); + + double d = crypto_fast_rng_get_double(rng); + tt_assert(d >= 0.0); + tt_assert(d < 1.0); + } + + /* All values should have come up once. */ + for (int i=0; i<5; ++i) { + tt_int_op(counts[i], OP_GT, 0); + } + + done: + crypto_fast_rng_free(rng); +} + +static void +test_crypto_rng_fast_whitebox(void *arg) +{ + (void)arg; + const size_t buflen = crypto_fast_rng_get_bytes_used_per_stream(); + char *buf = tor_malloc_zero(buflen); + char *buf2 = tor_malloc_zero(buflen); + char *buf3 = NULL, *buf4 = NULL; + + crypto_cipher_t *cipher = NULL, *cipher2 = NULL; + uint8_t seed[CRYPTO_FAST_RNG_SEED_LEN]; + memset(seed, 0, sizeof(seed)); + + /* Start with a prng with zero key and zero IV. */ + crypto_fast_rng_t *rng = crypto_fast_rng_new_from_seed(seed); + tt_assert(rng); + + /* We'll use a stream cipher to keep in sync */ + cipher = crypto_cipher_new_with_iv_and_bits(seed, seed+32, 256); + + /* The first 48 bytes are used for the next seed -- let's make sure we have + * them. + */ + memset(seed, 0, sizeof(seed)); + crypto_cipher_crypt_inplace(cipher, (char*)seed, sizeof(seed)); + + /* if we get 128 bytes, they should match the bytes from the aes256-counter + * stream, starting at position 48. + */ + crypto_fast_rng_getbytes(rng, (uint8_t*)buf, 128); + memset(buf2, 0, 128); + crypto_cipher_crypt_inplace(cipher, buf2, 128); + tt_mem_op(buf, OP_EQ, buf2, 128); + + /* Try that again, with an odd number of bytes. */ + crypto_fast_rng_getbytes(rng, (uint8_t*)buf, 199); + memset(buf2, 0, 199); + crypto_cipher_crypt_inplace(cipher, buf2, 199); + tt_mem_op(buf, OP_EQ, buf2, 199); + + /* Make sure that refilling works as expected: skip all but the last 5 bytes + * of this steam. */ + size_t skip = buflen - (199+128) - 5; + crypto_fast_rng_getbytes(rng, (uint8_t*)buf, skip); + crypto_cipher_crypt_inplace(cipher, buf2, skip); + + /* Now get the next 128 bytes. The first 5 will come from this stream, and + * the next 5 will come from the stream keyed by the new value of 'seed'. */ + crypto_fast_rng_getbytes(rng, (uint8_t*)buf, 128); + memset(buf2, 0, 128); + crypto_cipher_crypt_inplace(cipher, buf2, 5); + crypto_cipher_free(cipher); + cipher = crypto_cipher_new_with_iv_and_bits(seed, seed+32, 256); + memset(seed, 0, sizeof(seed)); + crypto_cipher_crypt_inplace(cipher, (char*)seed, sizeof(seed)); + crypto_cipher_crypt_inplace(cipher, buf2+5, 128-5); + tt_mem_op(buf, OP_EQ, buf2, 128); + + /* And check the next 7 bytes to make sure we didn't discard anything. */ + crypto_fast_rng_getbytes(rng, (uint8_t*)buf, 7); + memset(buf2, 0, 7); + crypto_cipher_crypt_inplace(cipher, buf2, 7); + tt_mem_op(buf, OP_EQ, buf2, 7); + + /* Now try the optimization for long outputs. */ + buf3 = tor_malloc(65536); + crypto_fast_rng_getbytes(rng, (uint8_t*)buf3, 65536); + + buf4 = tor_malloc_zero(65536); + uint8_t seed2[CRYPTO_FAST_RNG_SEED_LEN]; + memset(seed2, 0, sizeof(seed2)); + crypto_cipher_crypt_inplace(cipher, (char*)seed2, sizeof(seed2)); + cipher2 = crypto_cipher_new_with_iv_and_bits(seed2, seed2+32, 256); + crypto_cipher_crypt_inplace(cipher2, buf4, 65536); + tt_mem_op(buf3, OP_EQ, buf4, 65536); + + done: + crypto_fast_rng_free(rng); + crypto_cipher_free(cipher); + crypto_cipher_free(cipher2); + tor_free(buf); + tor_free(buf2); + tor_free(buf3); + tor_free(buf4); +} + +struct testcase_t crypto_rng_tests[] = { + { "rng", test_crypto_rng, 0, NULL, NULL }, + { "rng_range", test_crypto_rng_range, 0, NULL, NULL }, + { "rng_strongest", test_crypto_rng_strongest, TT_FORK, NULL, NULL }, + { "rng_strongest_nosyscall", test_crypto_rng_strongest, TT_FORK, + &passthrough_setup, (void*)"nosyscall" }, + { "rng_strongest_nofallback", test_crypto_rng_strongest, TT_FORK, + &passthrough_setup, (void*)"nofallback" }, + { "rng_strongest_broken", test_crypto_rng_strongest, TT_FORK, + &passthrough_setup, (void*)"broken" }, + { "fast", test_crypto_rng_fast, 0, NULL, NULL }, + { "fast_whitebox", test_crypto_rng_fast_whitebox, 0, NULL, NULL }, + END_OF_TESTCASES +}; diff --git a/src/test/test_dir.c b/src/test/test_dir.c index 0e44c47f3f..07a2641c9f 100644 --- a/src/test/test_dir.c +++ b/src/test/test_dir.c @@ -91,9 +91,29 @@ #ifdef HAVE_SYS_STAT_H #include <sys/stat.h> #endif +#ifdef HAVE_UNISTD_H +#include <unistd.h> +#endif #define NS_MODULE dir +static networkstatus_t * +networkstatus_parse_vote_from_string_(const char *s, + const char **eos_out, + enum networkstatus_type_t ns_type) +{ + size_t len = strlen(s); + // memdup so that it won't be nul-terminated. + char *tmp = tor_memdup(s, len); + networkstatus_t *result = + networkstatus_parse_vote_from_string(tmp, len, eos_out, ns_type); + if (eos_out && *eos_out) { + *eos_out = s + (*eos_out - tmp); + } + tor_free(tmp); + return result; +} + static void test_dir_nicknames(void *arg) { @@ -1767,7 +1787,8 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) write_str_to_file(fname, "", 0); setup_capture_of_logs(LOG_WARN); tt_int_op(-1, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, - bw_file_headers)); + bw_file_headers, + NULL)); expect_log_msg("Empty bandwidth file\n"); teardown_capture_of_logs(); bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL); @@ -1783,7 +1804,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) write_str_to_file(fname, content, 0); tor_free(content); tt_int_op(-1, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, - bw_file_headers)); + bw_file_headers, + NULL)); + bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL); tt_str_op("", OP_EQ, bw_file_headers_str); SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c)); @@ -1794,7 +1817,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) write_str_to_file(fname, header_lines_v100, 0); bw_file_headers = smartlist_new(); tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, - bw_file_headers)); + bw_file_headers, + NULL)); + bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL); tt_str_op(bw_file_headers_str_v100, OP_EQ, bw_file_headers_str); SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c)); @@ -1807,7 +1832,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) write_str_to_file(fname, content, 0); tor_free(content); tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, - bw_file_headers)); + bw_file_headers, + NULL)); + bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL); tt_str_op(bw_file_headers_str_v100, OP_EQ, bw_file_headers_str); SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c)); @@ -1818,7 +1845,8 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) tor_asprintf(&content, "%s%s", header_lines_v100, relay_lines_v100); write_str_to_file(fname, content, 0); tor_free(content); - tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, NULL)); + tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, NULL, + NULL)); /* Test bandwidth file including v1.1.0 bandwidth headers and * v1.0.0 relay lines. bw_file_headers will contain the v1.1.0 headers. */ @@ -1828,7 +1856,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) write_str_to_file(fname, content, 0); tor_free(content); tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, - bw_file_headers)); + bw_file_headers, + NULL)); + bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL); tt_str_op(bw_file_headers_str_v110, OP_EQ, bw_file_headers_str); SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c)); @@ -1844,7 +1874,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) write_str_to_file(fname, content, 0); tor_free(content); tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, - bw_file_headers)); + bw_file_headers, + NULL)); + bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL); tt_str_op(bw_file_headers_str_v100, OP_EQ, bw_file_headers_str); SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c)); @@ -1861,7 +1893,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) write_str_to_file(fname, content, 0); tor_free(content); tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, - bw_file_headers)); + bw_file_headers, + NULL)); + bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL); tt_str_op(bw_file_headers_str_v100, OP_EQ, bw_file_headers_str); SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c)); @@ -1872,7 +1906,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) bw_file_headers = smartlist_new(); write_str_to_file(fname, header_lines_v110_no_terminator, 0); tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, - bw_file_headers)); + bw_file_headers, + NULL)); + bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL); tt_str_op(bw_file_headers_str_v110, OP_EQ, bw_file_headers_str); SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c)); @@ -1883,7 +1919,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) bw_file_headers = smartlist_new(); write_str_to_file(fname, header_lines_v110, 0); tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, - bw_file_headers)); + bw_file_headers, + NULL)); + bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL); tt_str_op(bw_file_headers_str_v110, OP_EQ, bw_file_headers_str); SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c)); @@ -1898,7 +1936,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) write_str_to_file(fname, content, 0); tor_free(content); tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, - bw_file_headers)); + bw_file_headers, + NULL)); + bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL); tt_str_op(bw_file_headers_str_v110, OP_EQ, bw_file_headers_str); SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c)); @@ -1913,7 +1953,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) write_str_to_file(fname, content, 0); tor_free(content); tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, - bw_file_headers)); + bw_file_headers, + NULL)); + bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL); tt_str_op(bw_file_headers_str_v110, OP_EQ, bw_file_headers_str); SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c)); @@ -1929,7 +1971,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) write_str_to_file(fname, content, 0); tor_free(content); tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, - bw_file_headers)); + bw_file_headers, + NULL)); + bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL); tt_str_op(bw_file_headers_str_v110, OP_EQ, bw_file_headers_str); SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c)); @@ -1946,7 +1990,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) write_str_to_file(fname, content, 0); tor_free(content); tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, - bw_file_headers)); + bw_file_headers, + NULL)); + bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL); tt_str_op(bw_file_headers_str_bad, OP_EQ, bw_file_headers_str); SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c)); @@ -1964,7 +2010,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) write_str_to_file(fname, content, 0); tor_free(content); tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, - bw_file_headers)); + bw_file_headers, + NULL)); + tt_int_op(MAX_BW_FILE_HEADER_COUNT_IN_VOTE, OP_EQ, smartlist_len(bw_file_headers)); bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL); @@ -1985,7 +2033,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) write_str_to_file(fname, content, 0); tor_free(content); tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, - bw_file_headers)); + bw_file_headers, + NULL)); + tt_int_op(MAX_BW_FILE_HEADER_COUNT_IN_VOTE, OP_EQ, smartlist_len(bw_file_headers)); /* force bw_file_headers to be bigger than @@ -2014,7 +2064,8 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) /* Read the bandwidth file */ setup_full_capture_of_logs(LOG_DEBUG); - tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, NULL)); + tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, NULL, + NULL)); expect_log_msg_containing("Ignoring bandwidth file line"); teardown_capture_of_logs(); @@ -2032,11 +2083,13 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) /* Read the bandwidth file */ setup_full_capture_of_logs(LOG_DEBUG); - tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, NULL)); + tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, NULL, + NULL)); expect_log_msg_not_containing("Ignoring bandwidth file line"); teardown_capture_of_logs(); done: + unlink(fname); tor_free(fname); tor_free(header_lines_v100); tor_free(header_lines_v110_no_terminator); @@ -2863,11 +2916,17 @@ test_a_networkstatus( MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m); /* Parse certificates and keys. */ - cert1 = mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); + cert1 = mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, + strlen(AUTHORITY_CERT_1), + NULL); tt_assert(cert1); - cert2 = authority_cert_parse_from_string(AUTHORITY_CERT_2, NULL); + cert2 = authority_cert_parse_from_string(AUTHORITY_CERT_2, + strlen(AUTHORITY_CERT_2), + NULL); tt_assert(cert2); - cert3 = authority_cert_parse_from_string(AUTHORITY_CERT_3, NULL); + cert3 = authority_cert_parse_from_string(AUTHORITY_CERT_3, + strlen(AUTHORITY_CERT_3), + NULL); tt_assert(cert3); sign_skey_1 = crypto_pk_new(); sign_skey_2 = crypto_pk_new(); @@ -2969,7 +3028,7 @@ test_a_networkstatus( sign_skey_leg1, FLAV_NS); tt_assert(consensus_text); - con = networkstatus_parse_vote_from_string(consensus_text, NULL, + con = networkstatus_parse_vote_from_string_(consensus_text, NULL, NS_TYPE_CONSENSUS); tt_assert(con); //log_notice(LD_GENERAL, "<<%s>>\n<<%s>>\n<<%s>>\n", @@ -2981,7 +3040,7 @@ test_a_networkstatus( sign_skey_leg1, FLAV_MICRODESC); tt_assert(consensus_text_md); - con_md = networkstatus_parse_vote_from_string(consensus_text_md, NULL, + con_md = networkstatus_parse_vote_from_string_(consensus_text_md, NULL, NS_TYPE_CONSENSUS); tt_assert(con_md); tt_int_op(con_md->flavor,OP_EQ, FLAV_MICRODESC); @@ -3080,13 +3139,13 @@ test_a_networkstatus( tt_assert(consensus_text3); tt_assert(consensus_text_md2); tt_assert(consensus_text_md3); - con2 = networkstatus_parse_vote_from_string(consensus_text2, NULL, + con2 = networkstatus_parse_vote_from_string_(consensus_text2, NULL, NS_TYPE_CONSENSUS); - con3 = networkstatus_parse_vote_from_string(consensus_text3, NULL, + con3 = networkstatus_parse_vote_from_string_(consensus_text3, NULL, NS_TYPE_CONSENSUS); - con_md2 = networkstatus_parse_vote_from_string(consensus_text_md2, NULL, + con_md2 = networkstatus_parse_vote_from_string_(consensus_text_md2, NULL, NS_TYPE_CONSENSUS); - con_md3 = networkstatus_parse_vote_from_string(consensus_text_md3, NULL, + con_md3 = networkstatus_parse_vote_from_string_(consensus_text_md3, NULL, NS_TYPE_CONSENSUS); tt_assert(con2); tt_assert(con3); @@ -3864,6 +3923,62 @@ mock_get_options(void) return mock_options; } +/** + * Test dirauth_get_b64_digest_bw_file. + * This function should be near the other bwauth functions, but it needs + * mock_get_options, that is only defined here. + */ + +static void +test_dir_bwauth_bw_file_digest256(void *arg) +{ + (void)arg; + const char *content = + "1541171221\n" + "node_id=$68A483E05A2ABDCA6DA5A3EF8DB5177638A27F80 " + "master_key_ed25519=YaqV4vbvPYKucElk297eVdNArDz9HtIwUoIeo0+cVIpQ " + "bw=760 nick=Test time=2018-05-08T16:13:26\n"; + + char *fname = tor_strdup(get_fname("V3BandwidthsFile")); + /* Initialize to a wrong digest. */ + uint8_t digest[DIGEST256_LEN] = "01234567890123456789abcdefghijkl"; + + /* Digest of an empty string. Initialize to a wrong digest. */ + char digest_empty_str[DIGEST256_LEN] = "01234567890123456789abcdefghijkl"; + crypto_digest256(digest_empty_str, "", 0, DIGEST_SHA256); + + /* Digest of the content. Initialize to a wrong digest. */ + char digest_expected[DIGEST256_LEN] = "01234567890123456789abcdefghijkl"; + crypto_digest256(digest_expected, content, strlen(content), DIGEST_SHA256); + + /* When the bandwidth file can not be found. */ + tt_int_op(-1, OP_EQ, + dirserv_read_measured_bandwidths(fname, + NULL, NULL, digest)); + tt_mem_op(digest, OP_EQ, digest_empty_str, DIGEST256_LEN); + + /* When there is a timestamp but it is too old. */ + write_str_to_file(fname, content, 0); + tt_int_op(-1, OP_EQ, + dirserv_read_measured_bandwidths(fname, + NULL, NULL, digest)); + /* The digest will be correct. */ + tt_mem_op(digest, OP_EQ, digest_expected, DIGEST256_LEN); + + update_approx_time(1541171221); + + /* When there is a bandwidth file and it can be read. */ + tt_int_op(0, OP_EQ, + dirserv_read_measured_bandwidths(fname, + NULL, NULL, digest)); + tt_mem_op(digest, OP_EQ, digest_expected, DIGEST256_LEN); + + done: + unlink(fname); + tor_free(fname); + update_approx_time(time(NULL)); +} + static void reset_routerstatus(routerstatus_t *rs, const char *hex_identity_digest, @@ -6087,6 +6202,80 @@ test_dir_find_dl_min_delay(void* data) } static void +test_dir_matching_flags(void *arg) +{ + (void) arg; + routerstatus_t *rs_noflags = NULL; + routerstatus_t *rs = NULL; + char *s = NULL; + + smartlist_t *tokens = smartlist_new(); + memarea_t *area = memarea_new(); + + int expected_val_when_unused = 0; + + const char *ex_noflags = + "r example hereiswhereyouridentitygoes 2015-08-30 12:00:00 " + "192.168.0.1 9001 0\n" + "m thisoneislongerbecauseitisa256bitmddigest33\n" + "s\n"; + const char *cp = ex_noflags; + rs_noflags = routerstatus_parse_entry_from_string( + area, &cp, + cp + strlen(cp), + tokens, NULL, NULL, + MAX_SUPPORTED_CONSENSUS_METHOD, FLAV_MICRODESC); + tt_assert(rs_noflags); + +#define FLAG(string, field) STMT_BEGIN { \ + tor_asprintf(&s,\ + "r example hereiswhereyouridentitygoes 2015-08-30 12:00:00 " \ + "192.168.0.1 9001 0\n" \ + "m thisoneislongerbecauseitisa256bitmddigest33\n" \ + "s %s\n", string); \ + cp = s; \ + rs = routerstatus_parse_entry_from_string( \ + area, &cp, \ + cp + strlen(cp), \ + tokens, NULL, NULL, \ + MAX_SUPPORTED_CONSENSUS_METHOD, FLAV_MICRODESC); \ + /* the field should usually be 0 when no flags are listed */ \ + tt_int_op(rs_noflags->field, OP_EQ, expected_val_when_unused); \ + /* the field should be 1 when this flags islisted */ \ + tt_int_op(rs->field, OP_EQ, 1); \ + tor_free(s); \ + routerstatus_free(rs); \ +} STMT_END + + FLAG("Authority", is_authority); + FLAG("BadExit", is_bad_exit); + FLAG("Exit", is_exit); + FLAG("Fast", is_fast); + FLAG("Guard", is_possible_guard); + FLAG("HSDir", is_hs_dir); + FLAG("Stable", is_stable); + FLAG("StaleDesc", is_staledesc); + FLAG("V2Dir", is_v2_dir); + + // These flags are assumed to be set whether they're declared or not. + expected_val_when_unused = 1; + FLAG("Running", is_flagged_running); + FLAG("Valid", is_valid); + expected_val_when_unused = 0; + + // These flags are no longer used, but still parsed. + FLAG("Named", is_named); + FLAG("Unnamed", is_unnamed); + + done: + tor_free(s); + routerstatus_free(rs); + routerstatus_free(rs_noflags); + memarea_drop_all(area); + smartlist_free(tokens); +} + +static void test_dir_assumed_flags(void *arg) { (void)arg; @@ -6101,9 +6290,10 @@ test_dir_assumed_flags(void *arg) "192.168.0.1 9001 0\n" "m thisoneislongerbecauseitisa256bitmddigest33\n" "s Fast Guard Stable\n"; + const char *eos = str1 + strlen(str1); const char *cp = str1; - rs = routerstatus_parse_entry_from_string(area, &cp, tokens, NULL, NULL, + rs = routerstatus_parse_entry_from_string(area, &cp, eos, tokens, NULL, NULL, 24, FLAV_MICRODESC); tt_assert(rs); tt_assert(rs->is_flagged_running); @@ -6370,6 +6560,7 @@ struct testcase_t dir_tests[] = { DIR_LEGACY(measured_bw_kb_line_is_after_headers), DIR_LEGACY(measured_bw_kb_cache), DIR_LEGACY(dirserv_read_measured_bandwidths), + DIR(bwauth_bw_file_digest256, 0), DIR_LEGACY(param_voting), DIR(param_voting_lookup, 0), DIR_LEGACY(v3_networkstatus), @@ -6410,6 +6601,7 @@ struct testcase_t dir_tests[] = { DIR_ARG(find_dl_min_delay, TT_FORK, "cfr"), DIR_ARG(find_dl_min_delay, TT_FORK, "car"), DIR(assumed_flags, 0), + DIR(matching_flags, 0), DIR(networkstatus_compute_bw_weights_v10, 0), DIR(platform_str, 0), DIR(networkstatus_consensus_has_ipv6, TT_FORK), diff --git a/src/test/test_dir_common.c b/src/test/test_dir_common.c index 3723d6c31b..0b87e29873 100644 --- a/src/test/test_dir_common.c +++ b/src/test/test_dir_common.c @@ -42,14 +42,20 @@ dir_common_authority_pk_init(authority_cert_t **cert1, { /* Parse certificates and keys. */ authority_cert_t *cert; - cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); + cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, + strlen(AUTHORITY_CERT_1), + NULL); tt_assert(cert); tt_assert(cert->identity_key); *cert1 = cert; tt_assert(*cert1); - *cert2 = authority_cert_parse_from_string(AUTHORITY_CERT_2, NULL); + *cert2 = authority_cert_parse_from_string(AUTHORITY_CERT_2, + strlen(AUTHORITY_CERT_2), + NULL); tt_assert(*cert2); - *cert3 = authority_cert_parse_from_string(AUTHORITY_CERT_3, NULL); + *cert3 = authority_cert_parse_from_string(AUTHORITY_CERT_3, + strlen(AUTHORITY_CERT_3), + NULL); tt_assert(*cert3); *sign_skey_1 = crypto_pk_new(); *sign_skey_2 = crypto_pk_new(); @@ -266,7 +272,9 @@ dir_common_add_rs_and_parse(networkstatus_t *vote, networkstatus_t **vote_out, /* dump the vote and try to parse it. */ v_text = format_networkstatus_vote(sign_skey, vote); tt_assert(v_text); - *vote_out = networkstatus_parse_vote_from_string(v_text, NULL, NS_TYPE_VOTE); + *vote_out = networkstatus_parse_vote_from_string(v_text, + strlen(v_text), + NULL, NS_TYPE_VOTE); done: if (v_text) @@ -424,4 +432,3 @@ dir_common_construct_vote_3(networkstatus_t **vote, authority_cert_t *cert, return 0; } - diff --git a/src/test/test_dir_handle_get.c b/src/test/test_dir_handle_get.c index 90691fff94..e57bd02584 100644 --- a/src/test/test_dir_handle_get.c +++ b/src/test/test_dir_handle_get.c @@ -72,6 +72,8 @@ ENABLE_GCC_WARNING(overlength-strings) #define NOT_ENOUGH_CONSENSUS_SIGNATURES "HTTP/1.0 404 " \ "Consensus not signed by sufficient number of requested authorities\r\n\r\n" +#define consdiffmgr_add_consensus consdiffmgr_add_consensus_nulterm + static dir_connection_t * new_dir_conn(void) { @@ -1275,7 +1277,9 @@ test_dir_handle_get_server_keys_authority(void* data) size_t body_used = 0; (void) data; - mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE, NULL); + mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE, + strlen(TEST_CERTIFICATE), + NULL); MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m); MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock); @@ -1425,7 +1429,9 @@ test_dir_handle_get_server_keys_sk(void* data) size_t body_used = 0; (void) data; - mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE, NULL); + mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE, + strlen(TEST_CERTIFICATE), + NULL); MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m); MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock); @@ -2217,6 +2223,31 @@ test_dir_handle_get_status_vote_next_authority_not_found(void* data) tor_free(header); } +static void +test_dir_handle_get_status_vote_next_bandwidth_not_found(void* data) +{ + dir_connection_t *conn = NULL; + char *header = NULL; + (void) data; + + MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock); + + conn = new_dir_conn(); + + tt_int_op(0, OP_EQ, directory_handle_command_get(conn, + GET("/tor/status-vote/next/bandwdith"), NULL, 0)); + + fetch_from_buf_http(TO_CONN(conn)->outbuf, &header, MAX_HEADERS_SIZE, + NULL, NULL, 1, 0); + tt_assert(header); + tt_str_op(NOT_FOUND, OP_EQ, header); + + done: + UNMOCK(connection_write_to_buf_impl_); + connection_free_minimal(TO_CONN(conn)); + tor_free(header); +} + NS_DECL(const char*, dirvote_get_pending_consensus, (consensus_flavor_t flav)); @@ -2393,7 +2424,9 @@ test_dir_handle_get_status_vote_next_authority(void* data) routerlist_free_all(); dirvote_free_all(); - mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE, NULL); + mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE, + strlen(TEST_CERTIFICATE), + NULL); /* create a trusted ds */ ds = trusted_dir_server_new("ds", "127.0.0.1", 9059, 9060, NULL, digest, @@ -2455,6 +2488,85 @@ test_dir_handle_get_status_vote_next_authority(void* data) } static void +test_dir_handle_get_status_vote_next_bandwidth(void* data) +{ + dir_connection_t *conn = NULL; + char *header = NULL, *body = NULL; + size_t body_used = 0; + (void) data; + + const char *content = + "1541171221\n" + "node_id=$68A483E05A2ABDCA6DA5A3EF8DB5177638A27F80 " + "master_key_ed25519=YaqV4vbvPYKucElk297eVdNArDz9HtIwUoIeo0+cVIpQ " + "bw=760 nick=Test time=2018-05-08T16:13:26\n"; + + init_mock_options(); + MOCK(get_options, mock_get_options); + mock_options->V3BandwidthsFile = tor_strdup( + get_fname_rnd("V3BandwidthsFile") + ); + + write_str_to_file(mock_options->V3BandwidthsFile, content, 0); + + MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock); + + conn = new_dir_conn(); + tt_int_op(0, OP_EQ, directory_handle_command_get(conn, + GET("/tor/status-vote/next/bandwidth"), NULL, 0)); + + fetch_from_buf_http(TO_CONN(conn)->outbuf, &header, MAX_HEADERS_SIZE, + &body, &body_used, strlen(content)+1, 0); + + tt_assert(header); + tt_ptr_op(strstr(header, "HTTP/1.0 200 OK\r\n"), OP_EQ, header); + tt_assert(strstr(header, "Content-Type: text/plain\r\n")); + tt_assert(strstr(header, "Content-Encoding: identity\r\n")); + tt_assert(strstr(header, "Content-Length: 167\r\n")); + + /* Check cache lifetime */ + char expbuf[RFC1123_TIME_LEN+1]; + time_t now = approx_time(); + /* BANDWIDTH_CACHE_LIFETIME is defined in dircache.c. */ + format_rfc1123_time(expbuf, (time_t)(now + 30*60)); + char *expires = NULL; + /* Change to 'Cache-control: max-age=%d' if using http/1.1. */ + tor_asprintf(&expires, "Expires: %s\r\n", expbuf); + tt_assert(strstr(header, expires)); + + tt_int_op(body_used, OP_EQ, strlen(body)); + tt_str_op(content, OP_EQ, body); + + tor_free(header); + tor_free(body); + + /* Request the file using compression, the result should be the same. */ + tt_int_op(0, OP_EQ, directory_handle_command_get(conn, + GET("/tor/status-vote/next/bandwidth.z"), NULL, 0)); + + fetch_from_buf_http(TO_CONN(conn)->outbuf, &header, MAX_HEADERS_SIZE, + &body, &body_used, strlen(content)+1, 0); + + tt_assert(header); + tt_ptr_op(strstr(header, "HTTP/1.0 200 OK\r\n"), OP_EQ, header); + tt_assert(strstr(header, "Content-Encoding: deflate\r\n")); + + /* Since using connection_write_to_buf_mock instead of mocking + * connection_buf_add_compress, the content is not actually compressed. + * If it would, the size and content would be different than the original. + */ + + done: + UNMOCK(get_options); + UNMOCK(connection_write_to_buf_impl_); + connection_free_minimal(TO_CONN(conn)); + tor_free(header); + tor_free(body); + tor_free(expires); + or_options_free(mock_options); +} + +static void test_dir_handle_get_status_vote_current_authority(void* data) { dir_connection_t *conn = NULL; @@ -2471,7 +2583,9 @@ test_dir_handle_get_status_vote_current_authority(void* data) routerlist_free_all(); dirvote_free_all(); - mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE, NULL); + mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE, + strlen(TEST_CERTIFICATE), + NULL); /* create a trusted ds */ ds = trusted_dir_server_new("ds", "127.0.0.1", 9059, 9060, NULL, digest, @@ -2627,6 +2741,8 @@ struct testcase_t dir_handle_get_tests[] = { DIR_HANDLE_CMD(status_vote_current_authority, 0), DIR_HANDLE_CMD(status_vote_next_authority_not_found, 0), DIR_HANDLE_CMD(status_vote_next_authority, 0), + DIR_HANDLE_CMD(status_vote_next_bandwidth_not_found, 0), + DIR_HANDLE_CMD(status_vote_next_bandwidth, 0), DIR_HANDLE_CMD(status_vote_current_consensus_ns_not_enough_sigs, TT_FORK), DIR_HANDLE_CMD(status_vote_current_consensus_ns_not_found, TT_FORK), DIR_HANDLE_CMD(status_vote_current_consensus_too_old, TT_FORK), diff --git a/src/test/test_dns.c b/src/test/test_dns.c index 41a56f65d8..231e6965f7 100644 --- a/src/test/test_dns.c +++ b/src/test/test_dns.c @@ -1,6 +1,7 @@ /* Copyright (c) 2015-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ +#include "orconfig.h" #include "core/or/or.h" #include "test/test.h" @@ -13,9 +14,71 @@ #include "core/or/edge_connection_st.h" #include "core/or/or_circuit_st.h" +#include "app/config/or_options_st.h" +#include "app/config/config.h" + +#include <event2/event.h> +#include <event2/dns.h> #define NS_MODULE dns +#ifdef HAVE_EVDNS_BASE_GET_NAMESERVER_ADDR +#define NS_SUBMODULE configure_nameservers_fallback + +static or_options_t options = { + .ORPort_set = 1, +}; + +static const or_options_t * +mock_get_options(void) +{ + return &options; +} + +static void +NS(test_main)(void *arg) +{ + (void)arg; + tor_addr_t *nameserver_addr = NULL; + + MOCK(get_options, mock_get_options); + + options.ServerDNSResolvConfFile = (char *)"no_such_file!!!"; + + dns_init(); // calls configure_nameservers() + + tt_int_op(number_of_configured_nameservers(), OP_EQ, 1); + + nameserver_addr = configured_nameserver_address(0); + + tt_assert(tor_addr_family(nameserver_addr) == AF_INET); + tt_assert(tor_addr_eq_ipv4h(nameserver_addr, 0x7f000001)); + +#ifndef _WIN32 + tor_free(nameserver_addr); + + options.ServerDNSResolvConfFile = (char *)"/dev/null"; + + dns_init(); + + tt_int_op(number_of_configured_nameservers(), OP_EQ, 1); + + nameserver_addr = configured_nameserver_address(0); + + tt_assert(tor_addr_family(nameserver_addr) == AF_INET); + tt_assert(tor_addr_eq_ipv4h(nameserver_addr, 0x7f000001)); +#endif + + UNMOCK(get_options); + + done: + tor_free(nameserver_addr); + return; +} + +#undef NS_SUBMODULE +#endif + #define NS_SUBMODULE clip_ttl static void @@ -736,6 +799,9 @@ NS(test_main)(void *arg) #undef NS_SUBMODULE struct testcase_t dns_tests[] = { +#ifdef HAVE_EVDNS_BASE_GET_NAMESERVER_ADDR + TEST_CASE(configure_nameservers_fallback), +#endif TEST_CASE(clip_ttl), TEST_CASE(resolve), TEST_CASE_ASPECT(resolve_impl, addr_is_ip_no_need_to_resolve), diff --git a/src/test/test_entrynodes.c b/src/test/test_entrynodes.c index a486b13ae1..9e213393b0 100644 --- a/src/test/test_entrynodes.c +++ b/src/test/test_entrynodes.c @@ -127,6 +127,9 @@ big_fake_network_cleanup(const struct testcase_t *testcase, void *ptr) return 1; /* NOP */ } +#define REASONABLY_FUTURE " reasonably-future" +#define REASONABLY_PAST " reasonably-past" + /* Unittest setup function: Setup a fake network. */ static void * big_fake_network_setup(const struct testcase_t *testcase) @@ -138,9 +141,10 @@ big_fake_network_setup(const struct testcase_t *testcase) const int N_NODES = 271; const char *argument = testcase->setup_data; - int reasonably_live_consensus = 0; + int reasonably_future_consensus = 0, reasonably_past_consensus = 0; if (argument) { - reasonably_live_consensus = strstr(argument, "reasonably-live") != NULL; + reasonably_future_consensus = strstr(argument, REASONABLY_FUTURE) != NULL; + reasonably_past_consensus = strstr(argument, REASONABLY_PAST) != NULL; } big_fake_net_nodes = smartlist_new(); @@ -198,11 +202,15 @@ big_fake_network_setup(const struct testcase_t *testcase) dummy_state = tor_malloc_zero(sizeof(or_state_t)); dummy_consensus = tor_malloc_zero(sizeof(networkstatus_t)); - if (reasonably_live_consensus) { - /* Make the dummy consensus valid from 4 hours ago, but expired an hour + if (reasonably_future_consensus) { + /* Make the dummy consensus valid in 6 hours, and expiring in 7 hours. */ + dummy_consensus->valid_after = approx_time() + 6*3600; + dummy_consensus->valid_until = approx_time() + 7*3600; + } else if (reasonably_past_consensus) { + /* Make the dummy consensus valid from 16 hours ago, but expired 12 hours * ago. */ - dummy_consensus->valid_after = approx_time() - 4*3600; - dummy_consensus->valid_until = approx_time() - 3600; + dummy_consensus->valid_after = approx_time() - 16*3600; + dummy_consensus->valid_until = approx_time() - 12*3600; } else { /* Make the dummy consensus valid for an hour either side of now. */ dummy_consensus->valid_after = approx_time() - 3600; @@ -3039,13 +3047,17 @@ static const struct testcase_setup_t upgrade_circuits = { #define BFN_TEST(name) \ EN_TEST_BASE(name, TT_FORK, &big_fake_network, NULL), \ - { #name "_reasonably_live", test_entry_guard_ ## name, TT_FORK, \ - &big_fake_network, (void*)("reasonably-live") } + { #name "_reasonably_future", test_entry_guard_ ## name, TT_FORK, \ + &big_fake_network, (void*)(REASONABLY_FUTURE) }, \ + { #name "_reasonably_past", test_entry_guard_ ## name, TT_FORK, \ + &big_fake_network, (void*)(REASONABLY_PAST) } #define UPGRADE_TEST(name, arg) \ EN_TEST_BASE(name, TT_FORK, &upgrade_circuits, arg), \ - { #name "_reasonably_live", test_entry_guard_ ## name, TT_FORK, \ - &upgrade_circuits, (void*)(arg " reasonably-live") } + { #name "_reasonably_future", test_entry_guard_ ## name, TT_FORK, \ + &upgrade_circuits, (void*)(arg REASONABLY_FUTURE) }, \ + { #name "_reasonably_past", test_entry_guard_ ## name, TT_FORK, \ + &upgrade_circuits, (void*)(arg REASONABLY_PAST) } struct testcase_t entrynodes_tests[] = { NO_PREFIX_TEST(node_preferred_orport), diff --git a/src/test/test_extorport.c b/src/test/test_extorport.c index 0c34d37a71..aeb71ec583 100644 --- a/src/test/test_extorport.c +++ b/src/test/test_extorport.c @@ -5,7 +5,7 @@ #define EXT_ORPORT_PRIVATE #define MAINLOOP_PRIVATE #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/mainloop/connection.h" #include "core/or/connection_or.h" #include "app/config/config.h" diff --git a/src/test/test_helpers.c b/src/test/test_helpers.c index 802d0a9ebe..13de1e154b 100644 --- a/src/test/test_helpers.c +++ b/src/test/test_helpers.c @@ -14,7 +14,7 @@ #include "orconfig.h" #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "app/config/config.h" #include "app/config/confparse.h" #include "core/mainloop/connection.h" diff --git a/src/test/test_hs_common.c b/src/test/test_hs_common.c index 2aff179687..eb7f3bfbb0 100644 --- a/src/test/test_hs_common.c +++ b/src/test/test_hs_common.c @@ -630,7 +630,7 @@ test_disaster_srv(void *arg) get_disaster_srv(1, srv_one); get_disaster_srv(2, srv_two); - /* Check that the cached ones where updated */ + /* Check that the cached ones were updated */ tt_mem_op(cached_disaster_srv_one, OP_EQ, srv_one, DIGEST256_LEN); tt_mem_op(cached_disaster_srv_two, OP_EQ, srv_two, DIGEST256_LEN); diff --git a/src/test/test_hs_service.c b/src/test/test_hs_service.c index 32b08ecf37..43bf894383 100644 --- a/src/test/test_hs_service.c +++ b/src/test/test_hs_service.c @@ -267,7 +267,7 @@ helper_clone_authorized_client(const hs_service_authorized_client_t *client) /* Helper: Return a newly allocated service object with the identity keypair * sets and the current descriptor. Then register it to the global map. - * Caller should us hs_free_all() to free this service or remove it from the + * Caller should use hs_free_all() to free this service or remove it from the * global map before freeing. */ static hs_service_t * helper_create_service(void) @@ -289,6 +289,20 @@ helper_create_service(void) return service; } +/* Helper: Deallocate a given service object, its child objects and + * remove it from onion service map. + * */ +static void +helper_destroy_service(hs_service_t *service) +{ + if (!service) + return; + + remove_service(get_hs_service_map(), service); + + hs_service_free(service); +} + /* Helper: Return a newly allocated service object with clients. */ static hs_service_t * helper_create_service_with_clients(int num_clients) @@ -1396,7 +1410,6 @@ static void test_build_update_descriptors(void *arg) { int ret; - time_t now = time(NULL); node_t *node; hs_service_t *service; hs_service_intro_point_t *ip_cur, *ip_next; @@ -1422,7 +1435,8 @@ test_build_update_descriptors(void *arg) voting_schedule_recalculate_timing(get_options(), mock_ns.valid_after); update_approx_time(mock_ns.valid_after+1); - now = mock_ns.valid_after+1; + + time_t now = mock_ns.valid_after+1; /* Create a service without a current descriptor to trigger a build. */ service = helper_create_service(); @@ -1626,6 +1640,7 @@ test_build_descriptors(void *arg) { int ret; time_t now = time(NULL); + hs_service_t *last_service = NULL; (void) arg; @@ -1650,19 +1665,27 @@ test_build_descriptors(void *arg) * is disabled. */ { hs_service_t *service = helper_create_service(); + last_service = service; service_descriptor_free(service->desc_current); service->desc_current = NULL; build_all_descriptors(now); + tt_assert(service->desc_current); + tt_assert(service->desc_current->desc); + hs_desc_superencrypted_data_t *superencrypted; superencrypted = &service->desc_current->desc->superencrypted_data; tt_int_op(smartlist_len(superencrypted->clients), OP_EQ, 16); + + helper_destroy_service(service); + last_service = NULL; } /* Generate a valid number of fake auth clients when the number of * clients is zero. */ { hs_service_t *service = helper_create_service_with_clients(0); + last_service = service; service_descriptor_free(service->desc_current); service->desc_current = NULL; @@ -1670,12 +1693,16 @@ test_build_descriptors(void *arg) hs_desc_superencrypted_data_t *superencrypted; superencrypted = &service->desc_current->desc->superencrypted_data; tt_int_op(smartlist_len(superencrypted->clients), OP_EQ, 16); + + helper_destroy_service(service); + last_service = NULL; } /* Generate a valid number of fake auth clients when the number of * clients is not a multiple of 16. */ { hs_service_t *service = helper_create_service_with_clients(20); + last_service = service; service_descriptor_free(service->desc_current); service->desc_current = NULL; @@ -1683,12 +1710,16 @@ test_build_descriptors(void *arg) hs_desc_superencrypted_data_t *superencrypted; superencrypted = &service->desc_current->desc->superencrypted_data; tt_int_op(smartlist_len(superencrypted->clients), OP_EQ, 32); + + helper_destroy_service(service); + last_service = NULL; } /* Do not generate any fake desc client when the number of clients is * a multiple of 16 but not zero. */ { hs_service_t *service = helper_create_service_with_clients(32); + last_service = service; service_descriptor_free(service->desc_current); service->desc_current = NULL; @@ -1696,9 +1727,13 @@ test_build_descriptors(void *arg) hs_desc_superencrypted_data_t *superencrypted; superencrypted = &service->desc_current->desc->superencrypted_data; tt_int_op(smartlist_len(superencrypted->clients), OP_EQ, 32); + + helper_destroy_service(service); + last_service = NULL; } done: + helper_destroy_service(last_service); hs_free_all(); } diff --git a/src/test/test_logging.c b/src/test/test_logging.c index 95a2fce757..6416e98a4e 100644 --- a/src/test/test_logging.c +++ b/src/test/test_logging.c @@ -9,7 +9,6 @@ #include "lib/err/torerr.h" #include "lib/log/log.h" #include "test/test.h" -#include "lib/process/subprocess.h" #ifdef HAVE_UNISTD_H #include <unistd.h> @@ -117,22 +116,27 @@ test_sigsafe_err(void *arg) content = read_file_to_str(fn, 0, NULL); tt_ptr_op(content, OP_NE, NULL); - tor_split_lines(lines, content, (int)strlen(content)); + smartlist_split_string(lines, content, "\n", 0, 0); tt_int_op(smartlist_len(lines), OP_GE, 5); - if (strstr(smartlist_get(lines, 0), "opening new log file")) + if (strstr(smartlist_get(lines, 0), "opening new log file")) { + void *item = smartlist_get(lines, 0); smartlist_del_keeporder(lines, 0); + tor_free(item); + } + tt_assert(strstr(smartlist_get(lines, 0), "Say, this isn't too cool")); - /* Next line is blank. */ - tt_assert(!strcmpstart(smartlist_get(lines, 1), "==============")); - tt_assert(!strcmpstart(smartlist_get(lines, 2), "Minimal.")); - /* Next line is blank. */ - tt_assert(!strcmpstart(smartlist_get(lines, 3), "==============")); - tt_str_op(smartlist_get(lines, 4), OP_EQ, + tt_str_op(smartlist_get(lines, 1), OP_EQ, ""); + tt_assert(!strcmpstart(smartlist_get(lines, 2), "==============")); + tt_assert(!strcmpstart(smartlist_get(lines, 3), "Minimal.")); + tt_str_op(smartlist_get(lines, 4), OP_EQ, ""); + tt_assert(!strcmpstart(smartlist_get(lines, 5), "==============")); + tt_str_op(smartlist_get(lines, 6), OP_EQ, "Testing any attempt to manually log from a signal."); done: tor_free(content); + SMARTLIST_FOREACH(lines, char *, x, tor_free(x)); smartlist_free(lines); } diff --git a/src/test/test_mainloop.c b/src/test/test_mainloop.c index 089ea812cf..ed6b8a9b66 100644 --- a/src/test/test_mainloop.c +++ b/src/test/test_mainloop.c @@ -6,11 +6,23 @@ * \brief Tests for functions closely related to the Tor main loop */ +#define CONFIG_PRIVATE +#define MAINLOOP_PRIVATE +#define STATEFILE_PRIVATE + #include "test/test.h" #include "test/log_test_helpers.h" #include "core/or/or.h" +#include "core/mainloop/connection.h" #include "core/mainloop/mainloop.h" +#include "core/mainloop/netstatus.h" + +#include "feature/hs/hs_service.h" + +#include "app/config/config.h" +#include "app/config/statefile.h" +#include "app/config/or_state_st.h" static const uint64_t BILLION = 1000000000; @@ -131,12 +143,227 @@ test_mainloop_update_time_jumps(void *arg) monotime_disable_test_mocking(); } +static int schedule_rescan_called = 0; +static void +mock_schedule_rescan_periodic_events(void) +{ + ++schedule_rescan_called; +} + +static void +test_mainloop_user_activity(void *arg) +{ + (void)arg; + const time_t start = 1542658829; + update_approx_time(start); + + MOCK(schedule_rescan_periodic_events, mock_schedule_rescan_periodic_events); + + reset_user_activity(start); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start); + + set_network_participation(false); + + // reset can move backwards and forwards, but does not change network + // participation. + reset_user_activity(start-10); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start-10); + reset_user_activity(start+10); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start+10); + + tt_int_op(schedule_rescan_called, OP_EQ, 0); + tt_int_op(false, OP_EQ, is_participating_on_network()); + + // "note" can only move forward. Calling it from a non-participating + // state makes us rescan the periodic callbacks and set participation. + note_user_activity(start+20); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start+20); + tt_int_op(true, OP_EQ, is_participating_on_network()); + tt_int_op(schedule_rescan_called, OP_EQ, 1); + + // Calling it again will move us forward, but not call rescan again. + note_user_activity(start+25); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start+25); + tt_int_op(true, OP_EQ, is_participating_on_network()); + tt_int_op(schedule_rescan_called, OP_EQ, 1); + + // We won't move backwards. + note_user_activity(start+20); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start+25); + tt_int_op(true, OP_EQ, is_participating_on_network()); + tt_int_op(schedule_rescan_called, OP_EQ, 1); + + // We _will_ adjust if the clock jumps though. + netstatus_note_clock_jumped(500); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start+525); + + netstatus_note_clock_jumped(-400); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start+125); + + done: + UNMOCK(schedule_rescan_periodic_events); +} + +static unsigned int +mock_get_num_services(void) +{ + return 1; +} + +static connection_t * +mock_connection_gbtu(int type) +{ + (void) type; + return (void *)"hello fellow connections"; +} + +static void +test_mainloop_check_participation(void *arg) +{ + (void)arg; + or_options_t *options = options_new(); + const time_t start = 1542658829; + const time_t ONE_DAY = 24*60*60; + + // Suppose we've been idle for a day or two + reset_user_activity(start - 2*ONE_DAY); + set_network_participation(true); + check_network_participation_callback(start, options); + tt_int_op(is_participating_on_network(), OP_EQ, false); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start-2*ONE_DAY); + + // suppose we've been idle for 2 days... but we are a server. + reset_user_activity(start - 2*ONE_DAY); + options->ORPort_set = 1; + set_network_participation(true); + check_network_participation_callback(start+2, options); + tt_int_op(is_participating_on_network(), OP_EQ, true); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start+2); + options->ORPort_set = 0; + + // idle for 2 days, but we have a hidden service. + reset_user_activity(start - 2*ONE_DAY); + set_network_participation(true); + MOCK(hs_service_get_num_services, mock_get_num_services); + check_network_participation_callback(start+3, options); + tt_int_op(is_participating_on_network(), OP_EQ, true); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start+3); + UNMOCK(hs_service_get_num_services); + + // idle for 2 days but we have at least one user connection + MOCK(connection_get_by_type_nonlinked, mock_connection_gbtu); + reset_user_activity(start - 2*ONE_DAY); + set_network_participation(true); + options->DormantTimeoutDisabledByIdleStreams = 1; + check_network_participation_callback(start+10, options); + tt_int_op(is_participating_on_network(), OP_EQ, true); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start+10); + + // as above, but DormantTimeoutDisabledByIdleStreams is not set + reset_user_activity(start - 2*ONE_DAY); + set_network_participation(true); + options->DormantTimeoutDisabledByIdleStreams = 0; + check_network_participation_callback(start+13, options); + tt_int_op(is_participating_on_network(), OP_EQ, false); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start-2*ONE_DAY); + UNMOCK(connection_get_by_type_nonlinked); + options->DormantTimeoutDisabledByIdleStreams = 1; + + // idle for 2 days but DormantClientTimeout is 3 days + reset_user_activity(start - 2*ONE_DAY); + set_network_participation(true); + options->DormantClientTimeout = ONE_DAY * 3; + check_network_participation_callback(start+30, options); + tt_int_op(is_participating_on_network(), OP_EQ, true); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start-2*ONE_DAY); + + done: + or_options_free(options); + UNMOCK(hs_service_get_num_services); + UNMOCK(connection_get_by_type_nonlinked); +} + +static void +test_mainloop_dormant_load_state(void *arg) +{ + (void)arg; + or_state_t *state = or_state_new(); + const time_t start = 1543956575; + + reset_user_activity(0); + set_network_participation(false); + + // When we construct a new state, it starts out in "auto" mode. + tt_int_op(state->Dormant, OP_EQ, -1); + + // Initializing from "auto" makes us start out (by default) non-Dormant, + // with activity right now. + netstatus_load_from_state(state, start); + tt_assert(is_participating_on_network()); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start); + + // Initializing from dormant clears the last user activity time, and + // makes us dormant. + state->Dormant = 1; + netstatus_load_from_state(state, start); + tt_assert(! is_participating_on_network()); + tt_i64_op(get_last_user_activity_time(), OP_EQ, 0); + + // Initializing from non-dormant sets the last user activity time, and + // makes us non-dormant. + state->Dormant = 0; + state->MinutesSinceUserActivity = 123; + netstatus_load_from_state(state, start); + tt_assert(is_participating_on_network()); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start - 123*60); + + // If we would start dormant, but DormantCanceledByStartup is set, then + // we start up non-dormant. + state->Dormant = 1; + get_options_mutable()->DormantCanceledByStartup = 1; + netstatus_load_from_state(state, start); + tt_assert(is_participating_on_network()); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start); + + done: + or_state_free(state); +} + +static void +test_mainloop_dormant_save_state(void *arg) +{ + (void)arg; + or_state_t *state = or_state_new(); + const time_t start = 1543956575; + + // Can we save a non-dormant state correctly? + reset_user_activity(start - 1000); + set_network_participation(true); + netstatus_flush_to_state(state, start); + + tt_int_op(state->Dormant, OP_EQ, 0); + tt_int_op(state->MinutesSinceUserActivity, OP_EQ, 1000 / 60); + + // Can we save a dormant state correctly? + set_network_participation(false); + netstatus_flush_to_state(state, start); + + tt_int_op(state->Dormant, OP_EQ, 1); + tt_int_op(state->MinutesSinceUserActivity, OP_EQ, 0); + + done: + or_state_free(state); +} + #define MAINLOOP_TEST(name) \ { #name, test_mainloop_## name , TT_FORK, NULL, NULL } struct testcase_t mainloop_tests[] = { MAINLOOP_TEST(update_time_normal), MAINLOOP_TEST(update_time_jumps), + MAINLOOP_TEST(user_activity), + MAINLOOP_TEST(check_participation), + MAINLOOP_TEST(dormant_load_state), + MAINLOOP_TEST(dormant_save_state), END_OF_TESTCASES }; - diff --git a/src/test/test_microdesc.c b/src/test/test_microdesc.c index 4c4317d81a..53ee799185 100644 --- a/src/test/test_microdesc.c +++ b/src/test/test_microdesc.c @@ -11,6 +11,7 @@ #include "feature/dirparse/routerparse.h" #include "feature/nodelist/microdesc.h" #include "feature/nodelist/networkstatus.h" +#include "feature/nodelist/nodefamily.h" #include "feature/nodelist/routerlist.h" #include "feature/nodelist/torcert.h" @@ -70,6 +71,7 @@ test_md_cache(void *data) const char *test_md3_noannotation = strchr(test_md3, '\n')+1; time_t time1, time2, time3; char *fn = NULL, *s = NULL; + char *encoded_family = NULL; (void)data; options = get_options_mutable(); @@ -172,8 +174,9 @@ test_md_cache(void *data) tt_ptr_op(md1->family, OP_EQ, NULL); tt_ptr_op(md3->family, OP_NE, NULL); - tt_int_op(smartlist_len(md3->family), OP_EQ, 3); - tt_str_op(smartlist_get(md3->family, 0), OP_EQ, "nodeX"); + + encoded_family = nodefamily_format(md3->family); + tt_str_op(encoded_family, OP_EQ, "nodex nodey nodez"); /* Now rebuild the cache! */ tt_int_op(microdesc_cache_rebuild(mc, 1), OP_EQ, 0); @@ -254,6 +257,7 @@ test_md_cache(void *data) smartlist_free(wanted); tor_free(s); tor_free(fn); + tor_free(encoded_family); } static const char truncated_md[] = @@ -417,6 +421,28 @@ static const char test_md2_21[] = "ntor-onion-key hbxdRnfVUJJY7+KcT4E3Rs7/zuClbN3hJrjSBiEGMgI=\n" "id ed25519 wqfLzgfCtRfYNg88LsL1QpzxS0itapJ1aj6TbnByx/Q\n"; +static const char test_md2_withfamily_28[] = + "onion-key\n" + "-----BEGIN RSA PUBLIC KEY-----\n" + "MIGJAoGBAL2R8EfubUcahxha4u02P4VAR0llQIMwFAmrHPjzcK7apcQgDOf2ovOA\n" + "+YQnJFxlpBmCoCZC6ssCi+9G0mqo650lFuTMP5I90BdtjotfzESfTykHLiChyvhd\n" + "l0dlqclb2SU/GKem/fLRXH16aNi72CdSUu/1slKs/70ILi34QixRAgMBAAE=\n" + "-----END RSA PUBLIC KEY-----\n" + "ntor-onion-key hbxdRnfVUJJY7+KcT4E3Rs7/zuClbN3hJrjSBiEGMgI=\n" + "family OtherNode !Strange\n" + "id ed25519 wqfLzgfCtRfYNg88LsL1QpzxS0itapJ1aj6TbnByx/Q\n"; + +static const char test_md2_withfamily_29[] = + "onion-key\n" + "-----BEGIN RSA PUBLIC KEY-----\n" + "MIGJAoGBAL2R8EfubUcahxha4u02P4VAR0llQIMwFAmrHPjzcK7apcQgDOf2ovOA\n" + "+YQnJFxlpBmCoCZC6ssCi+9G0mqo650lFuTMP5I90BdtjotfzESfTykHLiChyvhd\n" + "l0dlqclb2SU/GKem/fLRXH16aNi72CdSUu/1slKs/70ILi34QixRAgMBAAE=\n" + "-----END RSA PUBLIC KEY-----\n" + "ntor-onion-key hbxdRnfVUJJY7+KcT4E3Rs7/zuClbN3hJrjSBiEGMgI=\n" + "family !Strange $B7E27F104213C36F13E7E9829182845E495997A0 othernode\n" + "id ed25519 wqfLzgfCtRfYNg88LsL1QpzxS0itapJ1aj6TbnByx/Q\n"; + static void test_md_generate(void *arg) { @@ -447,6 +473,17 @@ test_md_generate(void *arg) tt_assert(ed25519_pubkey_eq(md->ed25519_identity_pkey, &ri->cache_info.signing_key_cert->signing_key)); + // Try family encoding. + microdesc_free(md); + ri->declared_family = smartlist_new(); + smartlist_add_strdup(ri->declared_family, "OtherNode !Strange"); + md = dirvote_create_microdescriptor(ri, 28); + tt_str_op(md->body, OP_EQ, test_md2_withfamily_28); + + microdesc_free(md); + md = dirvote_create_microdescriptor(ri, 29); + tt_str_op(md->body, OP_EQ, test_md2_withfamily_29); + done: microdesc_free(md); routerinfo_free(ri); diff --git a/src/test/test_netinfo.c b/src/test/test_netinfo.c new file mode 100644 index 0000000000..27d276d42f --- /dev/null +++ b/src/test/test_netinfo.c @@ -0,0 +1,48 @@ +/* Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#include "core/or/or.h" +#include "trunnel/netinfo.h" +#include "test/test.h" + +static void +test_netinfo_unsupported_addr(void *arg) +{ + const uint8_t wire_data[] = + { // TIME + 0x00, 0x00, 0x00, 0x01, + // OTHERADDR + 0x04, // ATYPE + 0x04, // ALEN + 0x08, 0x08, 0x08, 0x08, // AVAL + 0x01, // NMYADDR + 0x03, // ATYPE (unsupported) + 0x05, // ALEN + 'a', 'd', 'r', 'r', '!' // AVAL (unsupported) + }; + + (void)arg; + + netinfo_cell_t *parsed_cell = NULL; + + ssize_t parsed = netinfo_cell_parse(&parsed_cell, wire_data, + sizeof(wire_data)); + + tt_assert(parsed == sizeof(wire_data)); + + netinfo_addr_t *addr = netinfo_cell_get_my_addrs(parsed_cell, 0); + tt_assert(addr); + + tt_int_op(3, OP_EQ, netinfo_addr_get_addr_type(addr)); + tt_int_op(5, OP_EQ, netinfo_addr_get_len(addr)); + + done: + netinfo_cell_free(parsed_cell); +} + +struct testcase_t netinfo_tests[] = { + { "unsupported_addr", test_netinfo_unsupported_addr, 0, NULL, NULL }, + END_OF_TESTCASES +}; + diff --git a/src/test/test_nodelist.c b/src/test/test_nodelist.c index 53eb0413e5..8d6d3cb974 100644 --- a/src/test/test_nodelist.c +++ b/src/test/test_nodelist.c @@ -6,15 +6,19 @@ * \brief Unit tests for nodelist related functions. **/ +#define NODELIST_PRIVATE + #include "core/or/or.h" #include "lib/crypt_ops/crypto_rand.h" #include "feature/nodelist/networkstatus.h" +#include "feature/nodelist/nodefamily.h" #include "feature/nodelist/nodelist.h" #include "feature/nodelist/torcert.h" #include "feature/nodelist/microdesc_st.h" #include "feature/nodelist/networkstatus_st.h" #include "feature/nodelist/node_st.h" +#include "feature/nodelist/nodefamily_st.h" #include "feature/nodelist/routerinfo_st.h" #include "feature/nodelist/routerstatus_st.h" @@ -231,6 +235,411 @@ test_nodelist_ed_id(void *arg) #undef N_NODES } +static void +test_nodelist_nodefamily(void *arg) +{ + (void)arg; + /* hex ID digests */ + const char h1[] = "5B435D6869206861206C65207363617270652070"; + const char h2[] = "75C3B220616E6461726520696E206769726F2061"; + const char h3[] = "2074726F766172206461206D616E67696172652C"; + const char h4[] = "206D656E747265206E6F6E2076616C65206C2769"; + const char h5[] = "6E766572736F2E202D2D5072696D6F204C657669"; + + /* binary ID digests */ + uint8_t d1[DIGEST_LEN], d2[DIGEST_LEN], d3[DIGEST_LEN], d4[DIGEST_LEN], + d5[DIGEST_LEN]; + base16_decode((char*)d1, sizeof(d1), h1, strlen(h1)); + base16_decode((char*)d2, sizeof(d2), h2, strlen(h2)); + base16_decode((char*)d3, sizeof(d3), h3, strlen(h3)); + base16_decode((char*)d4, sizeof(d4), h4, strlen(h4)); + base16_decode((char*)d5, sizeof(d5), h5, strlen(h5)); + + char *enc=NULL, *enc2=NULL; + + nodefamily_t *nf1 = NULL; + nodefamily_t *nf2 = NULL; + nodefamily_t *nf3 = NULL; + + enc = nodefamily_format(NULL); + tt_str_op(enc, OP_EQ, ""); + tor_free(enc); + + /* Make sure that sorting and de-duplication work. */ + tor_asprintf(&enc, "$%s hello", h1); + nf1 = nodefamily_parse(enc, NULL, 0); + tt_assert(nf1); + tor_free(enc); + + tor_asprintf(&enc, "hello hello $%s hello", h1); + nf2 = nodefamily_parse(enc, NULL, 0); + tt_assert(nf2); + tt_ptr_op(nf1, OP_EQ, nf2); + tor_free(enc); + + tor_asprintf(&enc, "%s $%s hello", h1, h1); + nf3 = nodefamily_parse(enc, NULL, 0); + tt_assert(nf3); + tt_ptr_op(nf1, OP_EQ, nf3); + tor_free(enc); + + tt_assert(nodefamily_contains_rsa_id(nf1, d1)); + tt_assert(! nodefamily_contains_rsa_id(nf1, d2)); + tt_assert(nodefamily_contains_nickname(nf1, "hello")); + tt_assert(nodefamily_contains_nickname(nf1, "HELLO")); + tt_assert(! nodefamily_contains_nickname(nf1, "goodbye")); + + tt_int_op(nf1->refcnt, OP_EQ, 3); + nodefamily_free(nf3); + tt_int_op(nf1->refcnt, OP_EQ, 2); + + /* Try parsing with a provided self RSA digest. */ + nf3 = nodefamily_parse("hello ", d1, 0); + tt_assert(nf3); + tt_ptr_op(nf1, OP_EQ, nf3); + + /* Do we get the expected result when we re-encode? */ + tor_asprintf(&enc, "$%s hello", h1); + enc2 = nodefamily_format(nf1); + tt_str_op(enc2, OP_EQ, enc); + tor_free(enc2); + tor_free(enc); + + /* Make sure that we get a different result if we give a different digest. */ + nodefamily_free(nf3); + tor_asprintf(&enc, "hello $%s hello", h3); + nf3 = nodefamily_parse(enc, NULL, 0); + tt_assert(nf3); + tt_ptr_op(nf1, OP_NE, nf3); + tor_free(enc); + + tt_assert(nodefamily_contains_rsa_id(nf3, d3)); + tt_assert(! nodefamily_contains_rsa_id(nf3, d2)); + tt_assert(! nodefamily_contains_rsa_id(nf3, d1)); + tt_assert(nodefamily_contains_nickname(nf3, "hello")); + tt_assert(! nodefamily_contains_nickname(nf3, "goodbye")); + + nodefamily_free(nf1); + nodefamily_free(nf2); + nodefamily_free(nf3); + + /* Try one with several digests, all with nicknames appended, in different + formats. */ + tor_asprintf(&enc, "%s $%s $%s=res $%s~ist", h1, h2, h3, h4); + nf1 = nodefamily_parse(enc, d5, 0); + tt_assert(nf1); + tt_assert(nodefamily_contains_rsa_id(nf1, d1)); + tt_assert(nodefamily_contains_rsa_id(nf1, d2)); + tt_assert(nodefamily_contains_rsa_id(nf1, d3)); + tt_assert(nodefamily_contains_rsa_id(nf1, d4)); + tt_assert(nodefamily_contains_rsa_id(nf1, d5)); + /* Nicknames aren't preserved when ids are present, since node naming is + * deprecated */ + tt_assert(! nodefamily_contains_nickname(nf3, "res")); + tor_free(enc); + tor_asprintf(&enc, "$%s $%s $%s $%s $%s", h4, h3, h1, h5, h2); + enc2 = nodefamily_format(nf1); + tt_str_op(enc, OP_EQ, enc2); + tor_free(enc); + tor_free(enc2); + + /* Try ones where we parse the empty string. */ + nf2 = nodefamily_parse("", NULL, 0); + nf3 = nodefamily_parse("", d4, 0); + tt_assert(nf2); + tt_assert(nf3); + tt_ptr_op(nf2, OP_NE, nf3); + + tt_assert(! nodefamily_contains_rsa_id(nf2, d4)); + tt_assert(nodefamily_contains_rsa_id(nf3, d4)); + tt_assert(! nodefamily_contains_rsa_id(nf2, d5)); + tt_assert(! nodefamily_contains_rsa_id(nf3, d5)); + tt_assert(! nodefamily_contains_nickname(nf2, "fred")); + tt_assert(! nodefamily_contains_nickname(nf3, "bosco")); + + /* The NULL family should contain nothing. */ + tt_assert(! nodefamily_contains_rsa_id(NULL, d4)); + tt_assert(! nodefamily_contains_rsa_id(NULL, d5)); + + done: + tor_free(enc); + tor_free(enc2); + nodefamily_free(nf1); + nodefamily_free(nf2); + nodefamily_free(nf3); + nodefamily_free_all(); +} + +static void +test_nodelist_nodefamily_parse_err(void *arg) +{ + (void)arg; + nodefamily_t *nf1 = NULL; + char *enc = NULL; + const char *semibogus = + "sdakljfdslkfjdsaklfjdkl9sdf " // too long for nickname + "$jkASDFLkjsadfjhkl " // not hex + "$7468696e67732d696e2d7468656d73656c766573 " // ok + "reticulatogranulate "// ok + "$73656d69616e7468726f706f6c6f676963616c6c79 " // too long for hex + "$616273656e746d696e6465646e6573736573" // too short for hex + ; + + setup_capture_of_logs(LOG_WARN); + + // We only get two items when we parse this. + for (int reject = 0; reject <= 1; ++reject) { + for (int log_at_warn = 0; log_at_warn <= 1; ++log_at_warn) { + unsigned flags = log_at_warn ? NF_WARN_MALFORMED : 0; + flags |= reject ? NF_REJECT_MALFORMED : 0; + nf1 = nodefamily_parse(semibogus, NULL, flags); + if (reject) { + tt_assert(nf1 == NULL); + } else { + tt_assert(nf1); + enc = nodefamily_format(nf1); + tt_str_op(enc, OP_EQ, + "$7468696E67732D696E2D7468656D73656C766573 " + "reticulatogranulate"); + tor_free(enc); + } + + if (log_at_warn) { + expect_log_msg_containing("$616273656e746d696e6465646e6573736573"); + expect_log_msg_containing("sdakljfdslkfjdsaklfjdkl9sdf"); + } else { + tt_int_op(mock_saved_log_n_entries(), OP_EQ, 0); + } + mock_clean_saved_logs(); + } + } + + done: + tor_free(enc); + nodefamily_free(nf1); + teardown_capture_of_logs(); +} + +static const node_t * +mock_node_get_by_id(const char *id) +{ + if (fast_memeq(id, "!!!!!!!!!!!!!!!!!!!!", DIGEST_LEN)) + return NULL; + + // use tor_free, not node_free. + node_t *fake_node = tor_malloc_zero(sizeof(node_t)); + memcpy(fake_node->identity, id, DIGEST_LEN); + return fake_node; +} + +static const node_t * +mock_node_get_by_nickname(const char *nn, unsigned flags) +{ + (void)flags; + if (!strcmp(nn, "nonesuch")) + return NULL; + + // use tor_free, not node_free. + node_t *fake_node = tor_malloc_zero(sizeof(node_t)); + strlcpy(fake_node->identity, nn, DIGEST_LEN); + return fake_node; +} + +static void +test_nodelist_nodefamily_lookup(void *arg) +{ + (void)arg; + MOCK(node_get_by_nickname, mock_node_get_by_nickname); + MOCK(node_get_by_id, mock_node_get_by_id); + smartlist_t *sl = smartlist_new(); + nodefamily_t *nf1 = NULL; + char *mem_op_hex_tmp = NULL; + + // 'null' is allowed. + nodefamily_add_nodes_to_smartlist(NULL, sl); + tt_int_op(smartlist_len(sl), OP_EQ, 0); + + // Try a real family + nf1 = nodefamily_parse("$EEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE " + "$2121212121212121212121212121212121212121 " + "$3333333333333333333333333333333333333333 " + "erewhon nonesuch", NULL, 0); + tt_assert(nf1); + nodefamily_add_nodes_to_smartlist(nf1, sl); + // There were 5 elements; 2 were dropped because the mocked lookup failed. + tt_int_op(smartlist_len(sl), OP_EQ, 3); + + const node_t *n = smartlist_get(sl, 0); + test_memeq_hex(n->identity, "3333333333333333333333333333333333333333"); + n = smartlist_get(sl, 1); + test_memeq_hex(n->identity, "EEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE"); + n = smartlist_get(sl, 2); + tt_str_op(n->identity, OP_EQ, "erewhon"); + + done: + UNMOCK(node_get_by_nickname); + UNMOCK(node_get_by_id); + SMARTLIST_FOREACH(sl, node_t *, fake_node, tor_free(fake_node)); + smartlist_free(sl); + nodefamily_free(nf1); + tor_free(mem_op_hex_tmp); +} + +static void +test_nodelist_nickname_matches(void *arg) +{ + (void)arg; + node_t mock_node; + routerstatus_t mock_rs; + memset(&mock_node, 0, sizeof(mock_node)); + memset(&mock_rs, 0, sizeof(mock_rs)); + + strlcpy(mock_rs.nickname, "evilgeniuses", sizeof(mock_rs.nickname)); + mock_node.rs = &mock_rs; + memcpy(mock_node.identity, ".forabettertomorrow.", DIGEST_LEN); + +#define match(x) tt_assert(node_nickname_matches(&mock_node, (x))) +#define no_match(x) tt_assert(! node_nickname_matches(&mock_node, (x))) + + match("evilgeniuses"); + match("EvilGeniuses"); + match("EvilGeniuses"); + match("2e666f7261626574746572746f6d6f72726f772e"); + match("2E666F7261626574746572746F6D6F72726F772E"); + match("$2e666f7261626574746572746f6d6f72726f772e"); + match("$2E666F7261626574746572746F6D6F72726F772E"); + match("$2E666F7261626574746572746F6D6F72726F772E~evilgeniuses"); + match("$2E666F7261626574746572746F6D6F72726F772E~EVILGENIUSES"); + + no_match("evilgenius"); + no_match("evilgeniuseses"); + no_match("evil.genius"); + no_match("$2E666F7261626574746572746F6D6F72726FFFFF"); + no_match("2E666F7261626574746572746F6D6F72726FFFFF"); + no_match("$2E666F7261626574746572746F6D6F72726F772E~fred"); + no_match("$2E666F7261626574746572746F6D6F72726F772E=EVILGENIUSES"); + done: + ; +} + +static void +test_nodelist_node_nodefamily(void *arg) +{ + (void)arg; + node_t mock_node1; + routerstatus_t mock_rs; + microdesc_t mock_md; + + node_t mock_node2; + routerinfo_t mock_ri; + + smartlist_t *nodes=smartlist_new(); + + memset(&mock_node1, 0, sizeof(mock_node1)); + memset(&mock_node2, 0, sizeof(mock_node2)); + memset(&mock_rs, 0, sizeof(mock_rs)); + memset(&mock_md, 0, sizeof(mock_md)); + memset(&mock_ri, 0, sizeof(mock_ri)); + + mock_node1.rs = &mock_rs; + mock_node1.md = &mock_md; + + mock_node2.ri = &mock_ri; + + strlcpy(mock_rs.nickname, "nodeone", sizeof(mock_rs.nickname)); + mock_ri.nickname = tor_strdup("nodetwo"); + + memcpy(mock_node1.identity, "NodeOneNode1NodeOne1", DIGEST_LEN); + memcpy(mock_node2.identity, "SecondNodeWe'reTestn", DIGEST_LEN); + + // empty families. + tt_assert(! node_family_contains(&mock_node1, &mock_node2)); + tt_assert(! node_family_contains(&mock_node2, &mock_node1)); + + // Families contain nodes, but not these nodes + mock_ri.declared_family = smartlist_new(); + smartlist_add(mock_ri.declared_family, (char*)"NodeThree"); + mock_md.family = nodefamily_parse("NodeFour", NULL, 0); + tt_assert(! node_family_contains(&mock_node1, &mock_node2)); + tt_assert(! node_family_contains(&mock_node2, &mock_node1)); + + // Families contain one another. + smartlist_add(mock_ri.declared_family, (char*) + "4e6f64654f6e654e6f6465314e6f64654f6e6531"); + tt_assert(! node_family_contains(&mock_node1, &mock_node2)); + tt_assert(node_family_contains(&mock_node2, &mock_node1)); + + nodefamily_free(mock_md.family); + mock_md.family = nodefamily_parse( + "NodeFour " + "5365636f6e644e6f64655765277265546573746e", NULL, 0); + tt_assert(node_family_contains(&mock_node1, &mock_node2)); + tt_assert(node_family_contains(&mock_node2, &mock_node1)); + + // Try looking up families now. + MOCK(node_get_by_nickname, mock_node_get_by_nickname); + MOCK(node_get_by_id, mock_node_get_by_id); + + node_lookup_declared_family(nodes, &mock_node1); + tt_int_op(smartlist_len(nodes), OP_EQ, 2); + const node_t *n = smartlist_get(nodes, 0); + tt_mem_op(n->identity, OP_EQ, "SecondNodeWe'reTestn", DIGEST_LEN); + n = smartlist_get(nodes, 1); + tt_str_op(n->identity, OP_EQ, "nodefour"); + + // free, try the other one. + SMARTLIST_FOREACH(nodes, node_t *, x, tor_free(x)); + smartlist_clear(nodes); + + node_lookup_declared_family(nodes, &mock_node2); + tt_int_op(smartlist_len(nodes), OP_EQ, 2); + n = smartlist_get(nodes, 0); + // This gets a truncated hex hex ID since it was looked up by name + tt_str_op(n->identity, OP_EQ, "NodeThree"); + n = smartlist_get(nodes, 1); + tt_str_op(n->identity, OP_EQ, "4e6f64654f6e654e6f6"); + + done: + UNMOCK(node_get_by_nickname); + UNMOCK(node_get_by_id); + smartlist_free(mock_ri.declared_family); + nodefamily_free(mock_md.family); + tor_free(mock_ri.nickname); + // use tor_free, these aren't real nodes + SMARTLIST_FOREACH(nodes, node_t *, x, tor_free(x)); + smartlist_free(nodes); +} + +static void +test_nodelist_nodefamily_canonicalize(void *arg) +{ + (void)arg; + char *c = NULL; + + c = nodefamily_canonicalize("", NULL, 0); + tt_str_op(c, OP_EQ, ""); + tor_free(c); + + uint8_t own_id[20]; + memset(own_id, 0, sizeof(own_id)); + c = nodefamily_canonicalize( + "alice BOB caroL %potrzebie !!!@#@# " + "$bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb=fred " + "ffffffffffffffffffffffffffffffffffffffff " + "$cccccccccccccccccccccccccccccccccccccccc ", own_id, 0); + tt_str_op(c, OP_EQ, + "!!!@#@# " + "$0000000000000000000000000000000000000000 " + "$BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB " + "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC " + "$FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF " + "%potrzebie " + "alice bob carol"); + + done: + tor_free(c); +} + #define NODE(name, flags) \ { #name, test_nodelist_##name, (flags), NULL, NULL } @@ -239,6 +648,11 @@ struct testcase_t nodelist_tests[] = { NODE(node_get_verbose_nickname_not_named, TT_FORK), NODE(node_is_dir, TT_FORK), NODE(ed_id, TT_FORK), + NODE(nodefamily, TT_FORK), + NODE(nodefamily_parse_err, TT_FORK), + NODE(nodefamily_lookup, TT_FORK), + NODE(nickname_matches, 0), + NODE(node_nodefamily, TT_FORK), + NODE(nodefamily_canonicalize, 0), END_OF_TESTCASES }; - diff --git a/src/test/test_oom.c b/src/test/test_oom.c index b813fa43a3..da6b2ee14d 100644 --- a/src/test/test_oom.c +++ b/src/test/test_oom.c @@ -8,7 +8,7 @@ #define CIRCUITLIST_PRIVATE #define CONNECTION_PRIVATE #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/or/circuitlist.h" #include "lib/evloop/compat_libevent.h" #include "core/mainloop/connection.h" diff --git a/src/test/test_options.c b/src/test/test_options.c index 66b0e7ef11..f12e6b6763 100644 --- a/src/test/test_options.c +++ b/src/test/test_options.c @@ -429,6 +429,7 @@ get_options_test_data(const char *conf) // with options_init(), but about a dozen tests break when I do that. // Being kinda lame and just fixing the immedate breakage for now.. result->opt->ConnectionPadding = -1; // default must be "auto" + result->opt->DormantClientTimeout = 1800; // must be over 600. rv = config_get_lines(conf, &cl, 1); tt_int_op(rv, OP_EQ, 0); diff --git a/src/test/test_parsecommon.c b/src/test/test_parsecommon.c new file mode 100644 index 0000000000..0c8f467a45 --- /dev/null +++ b/src/test/test_parsecommon.c @@ -0,0 +1,594 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "core/or/or.h" +#include "test/test.h" +#include "lib/memarea/memarea.h" +#include "lib/encoding/binascii.h" +#include "feature/dirparse/parsecommon.h" +#include "test/log_test_helpers.h" + +static void +test_parsecommon_tokenize_string_null(void *arg) +{ + + memarea_t *area = memarea_new(); + smartlist_t *tokens = smartlist_new(); + + (void)arg; + + const char *str_with_null = "a\0bccccccccc"; + + int retval = + tokenize_string(area, str_with_null, + str_with_null + 3, + tokens, NULL, 0); + + tt_int_op(retval, OP_EQ, -1); + + done: + memarea_drop_all(area); + smartlist_free(tokens); + return; +} + +static void +test_parsecommon_tokenize_string_multiple_lines(void *arg) +{ + memarea_t *area = memarea_new(); + smartlist_t *tokens = smartlist_new(); + + (void)arg; + + token_rule_t table[] = { + T01("uptime", K_UPTIME, GE(1), NO_OBJ), + T01("hibernating", K_HIBERNATING, GE(1), NO_OBJ), + T1( "published", K_PUBLISHED, CONCAT_ARGS, NO_OBJ), + END_OF_TABLE, + }; + + char *str = tor_strdup( + "hibernating 0\nuptime 1024\n" + "published 2018-10-15 10:00:00\n"); + + int retval = + tokenize_string(area, str, NULL, + tokens, table, 0); + + tt_int_op(smartlist_len(tokens), OP_EQ, 3); + directory_token_t *token = smartlist_get(tokens, 0); + + tt_int_op(token->tp, OP_EQ, K_HIBERNATING); + + token = smartlist_get(tokens, 1); + + tt_int_op(token->tp, OP_EQ, K_UPTIME); + + token = smartlist_get(tokens, 2); + + tt_int_op(token->tp, OP_EQ, K_PUBLISHED); + + tt_int_op(retval, OP_EQ, 0); + + done: + tor_free(str); + memarea_drop_all(area); + smartlist_free(tokens); + return; +} + +static void +test_parsecommon_tokenize_string_min_cnt(void *arg) +{ + memarea_t *area = memarea_new(); + smartlist_t *tokens = smartlist_new(); + + (void)arg; + + token_rule_t table[] = { + T01("uptime", K_UPTIME, EQ(2), NO_OBJ), + T01("hibernating", K_HIBERNATING, GE(1), NO_OBJ), + END_OF_TABLE, + }; + + // Missing "uptime" + char *str = tor_strdup("uptime 1024\nhibernating 0\n"); + + int retval = + tokenize_string(area, str, NULL, + tokens, table, 0); + + tt_int_op(retval, OP_EQ, -1); + + done: + tor_free(str); + memarea_drop_all(area); + smartlist_free(tokens); + return; +} + +static void +test_parsecommon_tokenize_string_max_cnt(void *arg) +{ + memarea_t *area = memarea_new(); + smartlist_t *tokens = smartlist_new(); + + (void)arg; + + token_rule_t table[] = { + T01("uptime", K_UPTIME, EQ(1), NO_OBJ), + T01("hibernating", K_HIBERNATING, GE(1), NO_OBJ), + END_OF_TABLE, + }; + + // "uptime" expected once, but occurs twice in input. + char *str = tor_strdup( + "uptime 1024\nuptime 2048\nhibernating 0\n"); + + int retval = + tokenize_string(area, str, NULL, + tokens, table, 0); + + tt_int_op(retval, OP_EQ, -1); + + done: + tor_free(str); + memarea_drop_all(area); + smartlist_free(tokens); + return; +} + +static void +test_parsecommon_tokenize_string_at_start(void *arg) +{ + memarea_t *area = memarea_new(); + smartlist_t *tokens = smartlist_new(); + + (void)arg; + + token_rule_t table[] = { + T1_START("client-name", C_CLIENT_NAME, CONCAT_ARGS, NO_OBJ), + T01("uptime", K_UPTIME, EQ(1), NO_OBJ), + END_OF_TABLE, + }; + + // "client-name" is not the first line. + char *str = tor_strdup( + "uptime 1024\nclient-name Alice\n"); + + int retval = + tokenize_string(area, str, NULL, tokens, table, 0); + + tt_int_op(retval, OP_EQ, -1); + + done: + tor_free(str); + memarea_drop_all(area); + smartlist_free(tokens); + return; +} + +static void +test_parsecommon_tokenize_string_at_end(void *arg) +{ + memarea_t *area = memarea_new(); + smartlist_t *tokens = smartlist_new(); + + (void)arg; + + token_rule_t table[] = { + T1_END("client-name", C_CLIENT_NAME, CONCAT_ARGS, NO_OBJ), + T01("uptime", K_UPTIME, EQ(1), NO_OBJ), + END_OF_TABLE, + }; + + // "client-name" is not the last line. + char *str = tor_strdup( + "client-name Alice\nuptime 1024\n"); + + int retval = + tokenize_string(area, str, NULL, tokens, table, 0); + + tt_int_op(retval, OP_EQ, -1); + + done: + tor_free(str); + memarea_drop_all(area); + smartlist_free(tokens); + return; +} + +static void +test_parsecommon_tokenize_string_no_annotations(void *arg) +{ + memarea_t *area = memarea_new(); + smartlist_t *tokens = smartlist_new(); + + (void)arg; + + token_rule_t table[] = { + A01("@last-listed", A_LAST_LISTED, CONCAT_ARGS, NO_OBJ), + END_OF_TABLE, + }; + + char *str = tor_strdup("@last-listed 2018-09-21 15:30:03\n"); + + int retval = + tokenize_string(area, str, NULL, tokens, table, 0); + + tt_int_op(retval, OP_EQ, -1); + + done: + tor_free(str); + memarea_drop_all(area); + smartlist_free(tokens); + return; +} + +static void +test_parsecommon_get_next_token_success(void *arg) +{ + memarea_t *area = memarea_new(); + const char *str = "uptime 1024"; + const char *end = str + strlen(str); + const char **s = &str; + token_rule_t table = T01("uptime", K_UPTIME, GE(1), NO_OBJ); + (void)arg; + + directory_token_t *token = get_next_token(area, s, end, &table); + + tt_int_op(token->tp, OP_EQ, K_UPTIME); + tt_int_op(token->n_args, OP_EQ, 1); + tt_str_op(*(token->args), OP_EQ, "1024"); + tt_assert(!token->object_type); + tt_int_op(token->object_size, OP_EQ, 0); + tt_assert(!token->object_body); + + tt_ptr_op(*s, OP_EQ, end); + + done: + memarea_drop_all(area); + return; +} + +static void +test_parsecommon_get_next_token_concat_args(void *arg) +{ + memarea_t *area = memarea_new(); + const char *str = "proto A=1 B=2"; + const char *end = str + strlen(str); + const char **s = &str; + token_rule_t rule = T01("proto", K_PROTO, CONCAT_ARGS, NO_OBJ); + (void)arg; + + directory_token_t *token = get_next_token(area, s, end, &rule); + + tt_int_op(token->tp, OP_EQ, K_PROTO); + tt_int_op(token->n_args, OP_EQ, 1); + tt_str_op(*(token->args), OP_EQ, "A=1 B=2"); + + done: + memarea_drop_all(area); +} + +static void +test_parsecommon_get_next_token_parse_keys(void *arg) +{ + (void)arg; + + memarea_t *area = memarea_new(); + const char *str = + "onion-key\n" + "-----BEGIN RSA PUBLIC KEY-----\n" + "MIGJAoGBAMDdIya33BfNlHOkzoTKSTT8EjD64waMfUr372syVHiFjHhObwKwGA5u\n" + "sHaMIe9r+Ij/4C1dKyuXkcz3DOl6gWNhTD7dZ89I+Okoh1jWe30jxCiAcywC22p5\n" + "XLhrDkX1A63Z7XCH9ltwU2WMqWsVM98N2GR6MTujP7wtqdLExYN1AgMBAAE=\n" + "-----END RSA PUBLIC KEY-----\n"; + + const char *end = str + strlen(str); + const char **s = (const char **)&str; + directory_token_t *token = NULL; + directory_token_t *token2 = NULL; + + token_rule_t rule = T1("onion-key", R_IPO_ONION_KEY, NO_ARGS, NEED_KEY_1024); + + token = get_next_token(area, s, end, &rule); + tt_assert(token); + + tt_int_op(token->tp, OP_EQ, R_IPO_ONION_KEY); + tt_int_op(token->n_args, OP_EQ, 0); + tt_str_op(token->object_type, OP_EQ, "RSA PUBLIC KEY"); + tt_int_op(token->object_size, OP_EQ, 140); + tt_assert(token->object_body); + tt_assert(token->key); + tt_assert(!token->error); + + const char *str2 = + "client-key\n" + "-----BEGIN RSA PRIVATE KEY-----\n" + "MIICXAIBAAKBgQCwS810a2auH2PQchOBz9smNgjlDu31aq0IYlUohSYbhcv5AJ+d\n" + "DY0nfZWzS+mZPwzL3UiEnTt6PVv7AgoZ5V9ZJWJTKIURjJpkK0mstfJKHKIZhf84\n" + "pmFfRej9GQViB6NLtp1obOXJgJixSlMfw9doDI4NoAnEISCyH/tD77Qs2wIDAQAB\n" + "AoGAbDg8CKkdQOnX9c7xFpCnsE8fKqz9eddgHHNwXw1NFTwOt+2gDWKSMZmv2X5S\n" + "CVZg3owZxf5W0nT0D6Ny2+6nliak7foYAvkD0BsCiBhgftwC0zAo6k5rIbUKB3PJ\n" + "QLFXgpJhqWuXkODyt/hS/GTernR437WVSEGp1bnALqiFabECQQDaqHOxzoWY/nvH\n" + "KrfUi8EhqCnqERlRHwrW0MQZ1RPvF16OPPma+xa+ht/amfh3vYN5tZY82Zm43gGl\n" + "XWL5cZhNAkEAzmdSootYVnqLLLRMfHKXnO1XbaEcA/08MDNKGlSclBJixFenE8jX\n" + "iQsUbHwMJuGONvzWpRGPBP2f8xBd28ZtxwJARY+LZshtpfNniz/ixYJESaHG28je\n" + "xfjbKOW3TQSFV+2WTifFvHEeljQwKMoMyoMGvYRwLCGJjs9JtMLVxsdFjQJBAKwD\n" + "3BBvBQ39TuPQ1zWX4tb7zjMlY83HTFP3Sriq71tP/1QWoL2SUl56B2lp8E6vB/C3\n" + "wsMK4SCNprHRYAd7VZ0CQDKn6Zhd11P94PLs0msybFEh1VXr6CEW/BrxBgbL4ls6\n" + "dbX5XO0z4Ra8gYXgObgimhyMDYO98Idt5+Z3HIdyrSc=\n" + "-----END RSA PRIVATE KEY-----\n"; + + const char *end2 = str2 + strlen(str2); + const char **s2 = (const char **)&str2; + + token_rule_t rule2 = T01("client-key", C_CLIENT_KEY, NO_ARGS, + NEED_SKEY_1024); + + token2 = get_next_token(area, s2, end2, &rule2); + tt_assert(token2); + + tt_int_op(token2->tp, OP_EQ, C_CLIENT_KEY); + tt_int_op(token2->n_args, OP_EQ, 0); + tt_str_op(token2->object_type, OP_EQ, "RSA PRIVATE KEY"); + tt_int_op(token2->object_size, OP_EQ, 608); + tt_assert(token2->object_body); + tt_assert(token2->key); + tt_assert(!token->error); + + done: + if (token) token_clear(token); + if (token2) token_clear(token2); + memarea_drop_all(area); +} + +static void +test_parsecommon_get_next_token_object(void *arg) +{ + memarea_t *area = memarea_new(); + + const char *str = + "directory-signature 0232AF901C31A04EE9848595AF9BB7620D4C5B2E " + "CD1FD971855430880D3C31E0331C5C55800C2F79\n" + "-----BEGIN SIGNATURE-----\n" + "dLTbc1Lad/OWKBJhA/dERzDHumswTAzBFAWAz2vnQhLsebs1SOm0W/vceEsiEkiF\n" + "A+JJSzIyfywJc6Mnk7aKMEIFjOO/MaxuAp4zv+q+JonJkF0ExjMqvKR0D6pSFmfN\n" + "cnemnxGHxNuPDnKl0imbWKmWDsHtwgi4zWeTq3MekfMOXKi6gIh+bDFzCs9/Vquh\n" + "uNKJI1jW/A2DEKeaSAODEv9VoCsYSvbVVEuHCBWjeNAurd5aL26BrAolW6m7pkD6\n" + "I+cQ8dQG6Wa/Zt6gLXtBbOP2o/iDI7ahDP9diNkBI/rm4nfp9j4piTwsqpi7xz9J\n" + "Ua9DEZB9KbJHVX1rGShrLA==\n" + "-----END SIGNATURE-----\n"; + + const char *end = str + strlen(str); + const char **s = &str; + token_rule_t rule = T("directory-signature", K_DIRECTORY_SIGNATURE, + GE(2), NEED_OBJ); + (void)arg; + + directory_token_t *token = get_next_token(area, s, end, &rule); + + tt_int_op(token->tp, OP_EQ, K_DIRECTORY_SIGNATURE); + tt_int_op(token->n_args, OP_EQ, 2); + tt_str_op(token->args[0], OP_EQ, + "0232AF901C31A04EE9848595AF9BB7620D4C5B2E"); + tt_str_op(token->args[1], OP_EQ, + "CD1FD971855430880D3C31E0331C5C55800C2F79"); + + tt_assert(!token->error); + + char decoded[256]; + const char *signature = + "dLTbc1Lad/OWKBJhA/dERzDHumswTAzBFAWAz2vnQhLsebs1SOm0W/vceEsiEkiF\n" + "A+JJSzIyfywJc6Mnk7aKMEIFjOO/MaxuAp4zv+q+JonJkF0ExjMqvKR0D6pSFmfN\n" + "cnemnxGHxNuPDnKl0imbWKmWDsHtwgi4zWeTq3MekfMOXKi6gIh+bDFzCs9/Vquh\n" + "uNKJI1jW/A2DEKeaSAODEv9VoCsYSvbVVEuHCBWjeNAurd5aL26BrAolW6m7pkD6\n" + "I+cQ8dQG6Wa/Zt6gLXtBbOP2o/iDI7ahDP9diNkBI/rm4nfp9j4piTwsqpi7xz9J\n" + "Ua9DEZB9KbJHVX1rGShrLA==\n"; + tt_assert(signature); + size_t signature_len = strlen(signature); + base64_decode(decoded, sizeof(decoded), signature, signature_len); + + tt_str_op(token->object_type, OP_EQ, "SIGNATURE"); + tt_int_op(token->object_size, OP_EQ, 256); + tt_mem_op(token->object_body, OP_EQ, decoded, 256); + + tt_assert(!token->key); + + done: + memarea_drop_all(area); +} + +static void +test_parsecommon_get_next_token_err_too_many_args(void *arg) +{ + memarea_t *area = memarea_new(); + const char *str = "uptime 1024 1024 1024"; + const char *end = str + strlen(str); + const char **s = &str; + token_rule_t table = T01("uptime", K_UPTIME, EQ(1), NO_OBJ); + (void)arg; + + directory_token_t *token = get_next_token(area, s, end, &table); + + tt_int_op(token->tp, OP_EQ, ERR_); + tt_str_op(token->error, OP_EQ, "Too many arguments to uptime"); + + done: + memarea_drop_all(area); + return; +} + +static void +test_parsecommon_get_next_token_err_too_few_args(void *arg) +{ + memarea_t *area = memarea_new(); + const char *str = "uptime"; + const char *end = str + strlen(str); + const char **s = &str; + token_rule_t table = T01("uptime", K_UPTIME, EQ(1), NO_OBJ); + (void)arg; + + directory_token_t *token = get_next_token(area, s, end, &table); + + tt_int_op(token->tp, OP_EQ, ERR_); + tt_str_op(token->error, OP_EQ, "Too few arguments to uptime"); + + done: + memarea_drop_all(area); + return; +} + +static void +test_parsecommon_get_next_token_err_obj_missing_endline(void *arg) +{ + memarea_t *area = memarea_new(); + + const char *str = + "directory-signature 0232AF901C31A04EE9848595AF9BB7620D4C5B2E " + "CD1FD971855430880D3C31E0331C5C55800C2F79\n" + "-----BEGIN SIGNATURE-----\n" + "dLTbc1Lad/OWKBJhA/dERzDHumswTAzBFAWAz2vnQhLsebs1SOm0W/vceEsiEkiF\n" + "A+JJSzIyfywJc6Mnk7aKMEIFjOO/MaxuAp4zv+q+JonJkF0ExjMqvKR0D6pSFmfN\n" + "cnemnxGHxNuPDnKl0imbWKmWDsHtwgi4zWeTq3MekfMOXKi6gIh+bDFzCs9/Vquh\n" + "uNKJI1jW/A2DEKeaSAODEv9VoCsYSvbVVEuHCBWjeNAurd5aL26BrAolW6m7pkD6\n" + "I+cQ8dQG6Wa/Zt6gLXtBbOP2o/iDI7ahDP9diNkBI/rm4nfp9j4piTwsqpi7xz9J\n" + "Ua9DEZB9KbJHVX1rGShrLA==\n"; + + const char *end = str + strlen(str); + const char **s = &str; + token_rule_t rule = T("directory-signature", K_DIRECTORY_SIGNATURE, + GE(2), NEED_OBJ); + (void)arg; + + directory_token_t *token = get_next_token(area, s, end, &rule); + + tt_int_op(token->tp, OP_EQ, ERR_); + tt_str_op(token->error, OP_EQ, "Malformed object: missing object end line"); + + done: + memarea_drop_all(area); + return; +} + +static void +test_parsecommon_get_next_token_err_bad_beginline(void *arg) +{ + memarea_t *area = memarea_new(); + + const char *str = + "directory-signature 0232AF901C31A04EE9848595AF9BB7620D4C5B2E " + "CD1FD971855430880D3C31E0331C5C55800C2F79\n" + "-----BEGIN SIGNATURE-Z---\n" + "dLTbc1Lad/OWKBJhA/dERzDHumswTAzBFAWAz2vnQhLsebs1SOm0W/vceEsiEkiF\n" + "A+JJSzIyfywJc6Mnk7aKMEIFjOO/MaxuAp4zv+q+JonJkF0ExjMqvKR0D6pSFmfN\n" + "cnemnxGHxNuPDnKl0imbWKmWDsHtwgi4zWeTq3MekfMOXKi6gIh+bDFzCs9/Vquh\n" + "uNKJI1jW/A2DEKeaSAODEv9VoCsYSvbVVEuHCBWjeNAurd5aL26BrAolW6m7pkD6\n" + "I+cQ8dQG6Wa/Zt6gLXtBbOP2o/iDI7ahDP9diNkBI/rm4nfp9j4piTwsqpi7xz9J\n" + "Ua9DEZB9KbJHVX1rGShrLA==\n" + "-----END SIGNATURE-----\n"; + + const char *end = str + strlen(str); + const char **s = &str; + token_rule_t rule = T("directory-signature", K_DIRECTORY_SIGNATURE, + GE(2), NEED_OBJ); + (void)arg; + + directory_token_t *token = get_next_token(area, s, end, &rule); + + tt_int_op(token->tp, OP_EQ, ERR_); + tt_str_op(token->error, OP_EQ, "Malformed object: bad begin line"); + + done: + memarea_drop_all(area); + return; +} + +static void +test_parsecommon_get_next_token_err_tag_mismatch(void *arg) +{ + memarea_t *area = memarea_new(); + + const char *str = + "directory-signature 0232AF901C31A04EE9848595AF9BB7620D4C5B2E " + "CD1FD971855430880D3C31E0331C5C55800C2F79\n" + "-----BEGIN SIGNATURE-----\n" + "dLTbc1Lad/OWKBJhA/dERzDHumswTAzBFAWAz2vnQhLsebs1SOm0W/vceEsiEkiF\n" + "A+JJSzIyfywJc6Mnk7aKMEIFjOO/MaxuAp4zv+q+JonJkF0ExjMqvKR0D6pSFmfN\n" + "cnemnxGHxNuPDnKl0imbWKmWDsHtwgi4zWeTq3MekfMOXKi6gIh+bDFzCs9/Vquh\n" + "uNKJI1jW/A2DEKeaSAODEv9VoCsYSvbVVEuHCBWjeNAurd5aL26BrAolW6m7pkD6\n" + "I+cQ8dQG6Wa/Zt6gLXtBbOP2o/iDI7ahDP9diNkBI/rm4nfp9j4piTwsqpi7xz9J\n" + "Ua9DEZB9KbJHVX1rGShrLA==\n" + "-----END SOMETHINGELSE-----\n"; + + const char *end = str + strlen(str); + const char **s = &str; + token_rule_t rule = T("directory-signature", K_DIRECTORY_SIGNATURE, + GE(2), NEED_OBJ); + (void)arg; + + directory_token_t *token = get_next_token(area, s, end, &rule); + + tt_int_op(token->tp, OP_EQ, ERR_); + tt_str_op(token->error, OP_EQ, + "Malformed object: mismatched end tag SIGNATURE"); + + done: + memarea_drop_all(area); + return; +} + +static void +test_parsecommon_get_next_token_err_bad_base64(void *arg) +{ + memarea_t *area = memarea_new(); + + const char *str = + "directory-signature 0232AF901C31A04EE9848595AF9BB7620D4C5B2E " + "CD1FD971855430880D3C31E0331C5C55800C2F79\n" + "-----BEGIN SIGNATURE-----\n" + "%%@%%%%%%%!!!'\n" + "-----END SIGNATURE-----\n"; + + const char *end = str + strlen(str); + const char **s = &str; + token_rule_t rule = T("directory-signature", K_DIRECTORY_SIGNATURE, + GE(2), NEED_OBJ); + (void)arg; + + directory_token_t *token = get_next_token(area, s, end, &rule); + + tt_int_op(token->tp, OP_EQ, ERR_); + tt_str_op(token->error, OP_EQ, "Malformed object: bad base64-encoded data"); + + done: + memarea_drop_all(area); + return; +} + +#define PARSECOMMON_TEST(name) \ + { #name, test_parsecommon_ ## name, 0, NULL, NULL } + +struct testcase_t parsecommon_tests[] = { + PARSECOMMON_TEST(tokenize_string_null), + PARSECOMMON_TEST(tokenize_string_multiple_lines), + PARSECOMMON_TEST(tokenize_string_min_cnt), + PARSECOMMON_TEST(tokenize_string_max_cnt), + PARSECOMMON_TEST(tokenize_string_at_start), + PARSECOMMON_TEST(tokenize_string_at_end), + PARSECOMMON_TEST(tokenize_string_no_annotations), + PARSECOMMON_TEST(get_next_token_success), + PARSECOMMON_TEST(get_next_token_concat_args), + PARSECOMMON_TEST(get_next_token_parse_keys), + PARSECOMMON_TEST(get_next_token_object), + PARSECOMMON_TEST(get_next_token_err_too_many_args), + PARSECOMMON_TEST(get_next_token_err_too_few_args), + PARSECOMMON_TEST(get_next_token_err_obj_missing_endline), + PARSECOMMON_TEST(get_next_token_err_bad_beginline), + PARSECOMMON_TEST(get_next_token_err_tag_mismatch), + PARSECOMMON_TEST(get_next_token_err_bad_base64), + END_OF_TESTCASES +}; diff --git a/src/test/test_periodic_event.c b/src/test/test_periodic_event.c index 4a53639dad..ebac20838f 100644 --- a/src/test/test_periodic_event.c +++ b/src/test/test_periodic_event.c @@ -19,6 +19,7 @@ #include "feature/hibernate/hibernate.h" #include "feature/hs/hs_service.h" #include "core/mainloop/mainloop.h" +#include "core/mainloop/netstatus.h" #include "core/mainloop/periodic.h" /** Helper function: This is replaced in some tests for the event callbacks so @@ -50,6 +51,8 @@ test_pe_initialize(void *arg) * need to run the main loop and then wait for a second delaying the unit * tests. Instead, we'll test the callback work indepedently elsewhere. */ initialize_periodic_events(); + set_network_participation(false); + rescan_periodic_events(get_options()); /* Validate that all events have been set up. */ for (int i = 0; periodic_events[i].name; ++i) { @@ -59,7 +62,9 @@ test_pe_initialize(void *arg) tt_u64_op(item->last_action_time, OP_EQ, 0); /* Every event must have role(s) assign to it. This is done statically. */ tt_u64_op(item->roles, OP_NE, 0); - tt_uint_op(periodic_event_is_enabled(item), OP_EQ, 0); + int should_be_enabled = (item->roles & PERIODIC_EVENT_ROLE_ALL) && + !(item->flags & PERIODIC_EVENT_FLAG_NEED_NET); + tt_uint_op(periodic_event_is_enabled(item), OP_EQ, should_be_enabled); } done: @@ -79,6 +84,8 @@ test_pe_launch(void *arg) * network gets enabled. */ consider_hibernation(time(NULL)); + set_network_participation(true); + /* Hack: We'll set a dumb fn() of each events so they don't get called when * dispatching them. We just want to test the state of the callbacks, not * the whole code path. */ @@ -90,6 +97,7 @@ test_pe_launch(void *arg) options = get_options_mutable(); options->SocksPort_set = 1; periodic_events_on_new_options(options); + #if 0 /* Lets make sure that before intialization, we can't scan the periodic * events list and launch them. Lets try by being a Client. */ @@ -106,13 +114,12 @@ test_pe_launch(void *arg) /* Now that we've initialized, rescan the list to launch. */ periodic_events_on_new_options(options); + int mask = PERIODIC_EVENT_ROLE_CLIENT|PERIODIC_EVENT_ROLE_ALL| + PERIODIC_EVENT_ROLE_NET_PARTICIPANT; for (int i = 0; periodic_events[i].name; ++i) { periodic_event_item_t *item = &periodic_events[i]; - if (item->roles & PERIODIC_EVENT_ROLE_CLIENT) { - tt_int_op(periodic_event_is_enabled(item), OP_EQ, 1); - } else { - tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0); - } + int should_be_enabled = !!(item->roles & mask); + tt_int_op(periodic_event_is_enabled(item), OP_EQ, should_be_enabled); // enabled or not, the event has not yet been run. tt_u64_op(item->last_action_time, OP_EQ, 0); } @@ -124,7 +131,8 @@ test_pe_launch(void *arg) unsigned roles = get_my_roles(options); tt_uint_op(roles, OP_EQ, - PERIODIC_EVENT_ROLE_RELAY|PERIODIC_EVENT_ROLE_DIRSERVER); + PERIODIC_EVENT_ROLE_RELAY|PERIODIC_EVENT_ROLE_DIRSERVER| + PERIODIC_EVENT_ROLE_ALL|PERIODIC_EVENT_ROLE_NET_PARTICIPANT); for (int i = 0; periodic_events[i].name; ++i) { periodic_event_item_t *item = &periodic_events[i]; @@ -144,17 +152,23 @@ test_pe_launch(void *arg) /* Disable everything and we'll enable them ALL. */ options->SocksPort_set = 0; options->ORPort_set = 0; + options->DisableNetwork = 1; + set_network_participation(false); periodic_events_on_new_options(options); for (int i = 0; periodic_events[i].name; ++i) { periodic_event_item_t *item = &periodic_events[i]; - tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0); + int should_be_enabled = (item->roles & PERIODIC_EVENT_ROLE_ALL) && + !(item->flags & PERIODIC_EVENT_FLAG_NEED_NET); + tt_int_op(periodic_event_is_enabled(item), OP_EQ, should_be_enabled); } /* Enable everything. */ options->SocksPort_set = 1; options->ORPort_set = 1; options->BridgeRelay = 1; options->AuthoritativeDir = 1; options->V3AuthoritativeDir = 1; options->BridgeAuthoritativeDir = 1; + options->DisableNetwork = 0; + set_network_participation(true); register_dummy_hidden_service(&service); periodic_events_on_new_options(options); /* Note down the reference because we need to remove this service from the @@ -165,7 +179,8 @@ test_pe_launch(void *arg) for (int i = 0; periodic_events[i].name; ++i) { periodic_event_item_t *item = &periodic_events[i]; - tt_int_op(periodic_event_is_enabled(item), OP_EQ, 1); + tt_int_op(periodic_event_is_enabled(item), OP_EQ, + (item->roles != PERIODIC_EVENT_ROLE_CONTROLEV)); } done: @@ -187,42 +202,49 @@ test_pe_get_roles(void *arg) or_options_t *options = get_options_mutable(); tt_assert(options); + set_network_participation(true); + + const int ALL = PERIODIC_EVENT_ROLE_ALL | + PERIODIC_EVENT_ROLE_NET_PARTICIPANT; /* Nothing configured, should be no roles. */ + tt_assert(net_is_disabled()); roles = get_my_roles(options); - tt_int_op(roles, OP_EQ, 0); + tt_int_op(roles, OP_EQ, ALL); /* Indicate we have a SocksPort, roles should be come Client. */ options->SocksPort_set = 1; roles = get_my_roles(options); - tt_int_op(roles, OP_EQ, PERIODIC_EVENT_ROLE_CLIENT); + tt_int_op(roles, OP_EQ, PERIODIC_EVENT_ROLE_CLIENT|ALL); /* Now, we'll add a ORPort so should now be a Relay + Client. */ options->ORPort_set = 1; roles = get_my_roles(options); tt_int_op(roles, OP_EQ, (PERIODIC_EVENT_ROLE_CLIENT | PERIODIC_EVENT_ROLE_RELAY | - PERIODIC_EVENT_ROLE_DIRSERVER)); + PERIODIC_EVENT_ROLE_DIRSERVER | ALL)); /* Now add a Bridge. */ options->BridgeRelay = 1; roles = get_my_roles(options); tt_int_op(roles, OP_EQ, (PERIODIC_EVENT_ROLE_CLIENT | PERIODIC_EVENT_ROLE_RELAY | - PERIODIC_EVENT_ROLE_BRIDGE | PERIODIC_EVENT_ROLE_DIRSERVER)); + PERIODIC_EVENT_ROLE_BRIDGE | PERIODIC_EVENT_ROLE_DIRSERVER | + ALL)); tt_assert(roles & PERIODIC_EVENT_ROLE_ROUTER); /* Unset client so we can solely test Router role. */ options->SocksPort_set = 0; roles = get_my_roles(options); tt_int_op(roles, OP_EQ, - PERIODIC_EVENT_ROLE_ROUTER | PERIODIC_EVENT_ROLE_DIRSERVER); + PERIODIC_EVENT_ROLE_ROUTER | PERIODIC_EVENT_ROLE_DIRSERVER | + ALL); /* Reset options so we can test authorities. */ options->SocksPort_set = 0; options->ORPort_set = 0; options->BridgeRelay = 0; roles = get_my_roles(options); - tt_int_op(roles, OP_EQ, 0); + tt_int_op(roles, OP_EQ, ALL); /* Now upgrade to Dirauth. */ options->DirPort_set = 1; @@ -230,7 +252,7 @@ test_pe_get_roles(void *arg) options->V3AuthoritativeDir = 1; roles = get_my_roles(options); tt_int_op(roles, OP_EQ, - PERIODIC_EVENT_ROLE_DIRAUTH|PERIODIC_EVENT_ROLE_DIRSERVER); + PERIODIC_EVENT_ROLE_DIRAUTH|PERIODIC_EVENT_ROLE_DIRSERVER|ALL); tt_assert(roles & PERIODIC_EVENT_ROLE_AUTHORITIES); /* Now Bridge Authority. */ @@ -238,7 +260,7 @@ test_pe_get_roles(void *arg) options->BridgeAuthoritativeDir = 1; roles = get_my_roles(options); tt_int_op(roles, OP_EQ, - PERIODIC_EVENT_ROLE_BRIDGEAUTH|PERIODIC_EVENT_ROLE_DIRSERVER); + PERIODIC_EVENT_ROLE_BRIDGEAUTH|PERIODIC_EVENT_ROLE_DIRSERVER|ALL); tt_assert(roles & PERIODIC_EVENT_ROLE_AUTHORITIES); /* Move that bridge auth to become a relay. */ @@ -246,7 +268,7 @@ test_pe_get_roles(void *arg) roles = get_my_roles(options); tt_int_op(roles, OP_EQ, (PERIODIC_EVENT_ROLE_BRIDGEAUTH | PERIODIC_EVENT_ROLE_RELAY - | PERIODIC_EVENT_ROLE_DIRSERVER)); + | PERIODIC_EVENT_ROLE_DIRSERVER|ALL)); tt_assert(roles & PERIODIC_EVENT_ROLE_AUTHORITIES); /* And now an Hidden service. */ @@ -257,7 +279,8 @@ test_pe_get_roles(void *arg) remove_service(get_hs_service_map(), &service); tt_int_op(roles, OP_EQ, (PERIODIC_EVENT_ROLE_BRIDGEAUTH | PERIODIC_EVENT_ROLE_RELAY | - PERIODIC_EVENT_ROLE_HS_SERVICE | PERIODIC_EVENT_ROLE_DIRSERVER)); + PERIODIC_EVENT_ROLE_HS_SERVICE | PERIODIC_EVENT_ROLE_DIRSERVER | + ALL)); tt_assert(roles & PERIODIC_EVENT_ROLE_AUTHORITIES); done: diff --git a/src/test/test_policy.c b/src/test/test_policy.c index 9c001c294a..46d4a1b94a 100644 --- a/src/test/test_policy.c +++ b/src/test/test_policy.c @@ -2024,6 +2024,20 @@ test_policies_fascist_firewall_allows_address(void *arg) expect_ap); \ STMT_END +/** Mock the preferred address function to return zero (prefer IPv4). */ +static int +mock_fascist_firewall_rand_prefer_ipv6_addr_use_ipv4(void) +{ + return 0; +} + +/** Mock the preferred address function to return one (prefer IPv6). */ +static int +mock_fascist_firewall_rand_prefer_ipv6_addr_use_ipv6(void) +{ + return 1; +} + /** Run unit tests for fascist_firewall_choose_address */ static void test_policies_fascist_firewall_choose_address(void *arg) @@ -2422,6 +2436,42 @@ test_policies_fascist_firewall_choose_address(void *arg) CHECK_CHOSEN_ADDR_RN(fake_rs, fake_node, FIREWALL_DIR_CONNECTION, 1, 1, ipv4_dir_ap); + /* Test ClientAutoIPv6ORPort and pretend we prefer IPv4. */ + memset(&mock_options, 0, sizeof(or_options_t)); + mock_options.ClientAutoIPv6ORPort = 1; + mock_options.ClientUseIPv4 = 1; + mock_options.ClientUseIPv6 = 1; + MOCK(fascist_firewall_rand_prefer_ipv6_addr, + mock_fascist_firewall_rand_prefer_ipv6_addr_use_ipv4); + /* Simulate the initialisation of fake_node.ipv6_preferred */ + fake_node.ipv6_preferred = fascist_firewall_prefer_ipv6_orport( + &mock_options); + + CHECK_CHOSEN_ADDR_RN(fake_rs, fake_node, FIREWALL_OR_CONNECTION, 0, 1, + ipv4_or_ap); + CHECK_CHOSEN_ADDR_RN(fake_rs, fake_node, FIREWALL_OR_CONNECTION, 1, 1, + ipv4_or_ap); + + UNMOCK(fascist_firewall_rand_prefer_ipv6_addr); + + /* Test ClientAutoIPv6ORPort and pretend we prefer IPv6. */ + memset(&mock_options, 0, sizeof(or_options_t)); + mock_options.ClientAutoIPv6ORPort = 1; + mock_options.ClientUseIPv4 = 1; + mock_options.ClientUseIPv6 = 1; + MOCK(fascist_firewall_rand_prefer_ipv6_addr, + mock_fascist_firewall_rand_prefer_ipv6_addr_use_ipv6); + /* Simulate the initialisation of fake_node.ipv6_preferred */ + fake_node.ipv6_preferred = fascist_firewall_prefer_ipv6_orport( + &mock_options); + + CHECK_CHOSEN_ADDR_RN(fake_rs, fake_node, FIREWALL_OR_CONNECTION, 0, 1, + ipv6_or_ap); + CHECK_CHOSEN_ADDR_RN(fake_rs, fake_node, FIREWALL_OR_CONNECTION, 1, 1, + ipv6_or_ap); + + UNMOCK(fascist_firewall_rand_prefer_ipv6_addr); + done: UNMOCK(get_options); } diff --git a/src/test/test_prob_distr.c b/src/test/test_prob_distr.c new file mode 100644 index 0000000000..37cfdae7d9 --- /dev/null +++ b/src/test/test_prob_distr.c @@ -0,0 +1,1456 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file test_prob_distr.c + * \brief Test probability distributions. + * \detail + * + * For each probability distribution we do two kinds of tests: + * + * a) We do numerical deterministic testing of their cdf/icdf/sf/isf functions + * and the various relationships between them for each distribution. We also + * do deterministic tests on their sampling functions. Test vectors for + * these tests were computed from alternative implementations and were + * eyeballed to make sure they make sense + * (e.g. src/test/prob_distr_mpfr_ref.c computes logit(p) using GNU mpfr + * with 200-bit precision and is then tested in test_logit_logistic()). + * + * b) We do stochastic hypothesis testing (G-test) to ensure that sampling from + * the given distributions is distributed properly. The stochastic tests are + * slow and their false positive rate is not well suited for CI, so they are + * currently disabled-by-default and put into 'tests-slow'. + */ + +#define PROB_DISTR_PRIVATE + +#include "orconfig.h" + +#include "test/test.h" + +#include "core/or/or.h" + +#include "lib/math/prob_distr.h" +#include "lib/math/fp.h" +#include "lib/crypt_ops/crypto_rand.h" + +#include <float.h> +#include <math.h> +#include <stdbool.h> +#include <stddef.h> +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> + +/** + * Return floor(d) converted to size_t, as a workaround for complaints + * under -Wbad-function-cast for (size_t)floor(d). + */ +static size_t +floor_to_size_t(double d) +{ + double integral_d = floor(d); + return (size_t)integral_d; +} + +/** + * Return ceil(d) converted to size_t, as a workaround for complaints + * under -Wbad-function-cast for (size_t)ceil(d). + */ +static size_t +ceil_to_size_t(double d) +{ + double integral_d = ceil(d); + return (size_t)integral_d; +} + +/* + * Geometric(p) distribution, supported on {1, 2, 3, ...}. + * + * Compute the probability mass function Geom(n; p) of the number of + * trials before the first success when success has probability p. + */ +static double +logpmf_geometric(unsigned n, double p) +{ + /* This is actually a check against 1, but we do >= so that the compiler + does not raise a -Wfloat-equal */ + if (p >= 1) { + if (n == 1) + return 0; + else + return -HUGE_VAL; + } + return (n - 1)*log1p(-p) + log(p); +} + +/** + * Compute the logistic function, translated in output by 1/2: + * logistichalf(x) = logistic(x) - 1/2. Well-conditioned on the entire + * real plane, with maximum condition number 1 at 0. + * + * This implementation gives relative error bounded by 5 eps. + */ +static double +logistichalf(double x) +{ + /* + * Rewrite this with the identity + * + * 1/(1 + e^{-x}) - 1/2 + * = (1 - 1/2 - e^{-x}/2)/(1 + e^{-x}) + * = (1/2 - e^{-x}/2)/(1 + e^{-x}) + * = (1 - e^{-x})/[2 (1 + e^{-x})] + * = -(e^{-x} - 1)/[2 (1 + e^{-x})], + * + * which we can evaluate by -expm1(-x)/[2 (1 + exp(-x))]. + * + * Suppose exp has error d0, + has error d1, expm1 has error + * d2, and / has error d3, so we evaluate + * + * -(1 + d2) (1 + d3) (e^{-x} - 1) + * / [2 (1 + d1) (1 + (1 + d0) e^{-x})]. + * + * In the denominator, + * + * 1 + (1 + d0) e^{-x} + * = 1 + e^{-x} + d0 e^{-x} + * = (1 + e^{-x}) (1 + d0 e^{-x}/(1 + e^{-x})), + * + * so the relative error of the numerator is + * + * d' = d2 + d3 + d2 d3, + * and of the denominator, + * d'' = d1 + d0 e^{-x}/(1 + e^{-x}) + d0 d1 e^{-x}/(1 + e^{-x}) + * = d1 + d0 L(-x) + d0 d1 L(-x), + * + * where L(-x) is logistic(-x). By Lemma 1 the relative error + * of the quotient is bounded by + * + * 2|d2 + d3 + d2 d3 - d1 - d0 L(x) + d0 d1 L(x)|, + * + * Since 0 < L(x) < 1, this is bounded by + * + * 2|d2| + 2|d3| + 2|d2 d3| + 2|d1| + 2|d0| + 2|d0 d1| + * <= 4 eps + 2 eps^2. + */ + if (x < log(DBL_EPSILON/8)) { + /* + * Avoid overflow in e^{-x}. When x < log(eps/4), we + * we further have x < logit(eps/4), so that + * logistic(x) < eps/4. Hence the relative error of + * logistic(x) - 1/2 from -1/2 is bounded by eps/2, and + * so the relative error of -1/2 from logistic(x) - 1/2 + * is bounded by eps. + */ + return -0.5; + } else { + return -expm1(-x)/(2*(1 + exp(-x))); + } +} + +/** + * Compute the log of the sum of the exps. Caller should arrange the + * array in descending order to minimize error because I don't want to + * deal with using temporary space and the one caller in this file + * arranges that anyway. + * + * Warning: This implementation does not handle infinite or NaN inputs + * sensibly, because I don't need that here at the moment. (NaN, or + * -inf and +inf together, should yield NaN; +inf and finite should + * yield +inf; otherwise all -inf should be ignored because exp(-inf) = + * 0.) + */ +static double +logsumexp(double *A, size_t n) +{ + double maximum, sum; + size_t i; + + if (n == 0) + return log(0); + + maximum = A[0]; + for (i = 1; i < n; i++) { + if (A[i] > maximum) + maximum = A[i]; + } + + sum = 0; + for (i = n; i --> 0;) + sum += exp(A[i] - maximum); + + return log(sum) + maximum; +} + +/** + * Compute log(1 - e^x). Defined only for negative x so that e^x < 1. + * This is the complement of a probability in log space. + */ +static double +log1mexp(double x) +{ + + /* + * We want to compute log on [0, 1/2) but log1p on [1/2, +inf), + * so partition x at -log(2) = log(1/2). + */ + if (-log(2) < x) + return log(-expm1(x)); + else + return log1p(-exp(x)); +} + +/* + * Tests of numerical errors in computing logit, logistic, and the + * various cdfs, sfs, icdfs, and isfs. + */ + +#define arraycount(A) (sizeof(A)/sizeof(A[0])) + +/** Return relative error between <b>actual</b> and <b>expected</b>. + * Special cases: If <b>expected</b> is zero or infinite, return 1 if + * <b>actual</b> is equal to <b>expected</b> and 0 if not, since the + * usual notion of relative error is undefined but we only use this + * for testing relerr(e, a) <= bound. If either is NaN, return NaN, + * which has the property that NaN <= bound is false no matter what + * bound is. + * + * Beware: if you test !(relerr(e, a) > bound), then then the result + * is true when a is NaN because NaN > bound is false too. See + * CHECK_RELERR for correct use to decide when to report failure. + */ +static double +relerr(double expected, double actual) +{ + /* + * To silence -Wfloat-equal, we have to test for equality using + * inequalities: we have (fabs(expected) <= 0) iff (expected == 0), + * and (actual <= expected && actual >= expected) iff actual == + * expected whether expected is zero or infinite. + */ + if (fabs(expected) <= 0 || tor_isinf(expected)) { + if (actual <= expected && actual >= expected) + return 0; + else + return 1; + } else { + return fabs((expected - actual)/expected); + } +} + +/** Check that relative error of <b>expected</b> and <b>actual</b> is within + * <b>relerr_bound</b>. Caller must arrange to have i and relerr_bound in + * scope. */ +#define CHECK_RELERR(expected, actual) do { \ + double check_expected = (expected); \ + double check_actual = (actual); \ + const char *str_expected = #expected; \ + const char *str_actual = #actual; \ + double check_relerr = relerr(expected, actual); \ + if (!(relerr(check_expected, check_actual) <= relerr_bound)) { \ + log_warn(LD_GENERAL, "%s:%d: case %u: relerr(%s=%.17e, %s=%.17e)" \ + " = %.17e > %.17e\n", \ + __func__, __LINE__, (unsigned) i, \ + str_expected, check_expected, \ + str_actual, check_actual, \ + check_relerr, relerr_bound); \ + ok = false; \ + } \ +} while (0) + +/* Check that a <= b. + * Caller must arrange to have i in scope. */ +#define CHECK_LE(a, b) do { \ + double check_a = (a); \ + double check_b = (b); \ + const char *str_a = #a; \ + const char *str_b = #b; \ + if (!(check_a <= check_b)) { \ + log_warn(LD_GENERAL, "%s:%d: case %u: %s=%.17e > %s=%.17e\n", \ + __func__, __LINE__, (unsigned) i, \ + str_a, check_a, str_b, check_b); \ + ok = false; \ + } \ +} while (0) + +/** + * Test the logit and logistic functions. Confirm that they agree with + * the cdf, sf, icdf, and isf of the standard Logistic distribution. + * Confirm that the sampler for the standard logistic distribution maps + * [0, 1] into the right subinterval for the inverse transform, for + * this implementation. + */ +static void +test_logit_logistic(void *arg) +{ + (void) arg; + + static const struct { + double x; /* x = logit(p) */ + double p; /* p = logistic(x) */ + double phalf; /* p - 1/2 = logistic(x) - 1/2 */ + } cases[] = { + { -HUGE_VAL, 0, -0.5 }, + { -1000, 0, -0.5 }, + { -710, 4.47628622567513e-309, -0.5 }, + { -708, 3.307553003638408e-308, -0.5 }, + { -2, .11920292202211755, -.3807970779778824 }, + { -1.0000001, .2689414017088022, -.23105859829119776 }, + { -1, .2689414213699951, -.23105857863000487 }, + { -0.9999999, .26894144103118883, -.2310585589688111 }, + /* see src/test/prob_distr_mpfr_ref.c for computation */ + { -4.000000000537333e-5, .49999, -1.0000000000010001e-5 }, + { -4.000000000533334e-5, .49999, -.00001 }, + { -4.000000108916878e-9, .499999999, -1.0000000272292198e-9 }, + { -4e-9, .499999999, -1e-9 }, + { -4e-16, .5, -1e-16 }, + { -4e-300, .5, -1e-300 }, + { 0, .5, 0 }, + { 4e-300, .5, 1e-300 }, + { 4e-16, .5, 1e-16 }, + { 3.999999886872274e-9, .500000001, 9.999999717180685e-10 }, + { 4e-9, .500000001, 1e-9 }, + { 4.0000000005333336e-5, .50001, .00001 }, + { 8.000042667076272e-3, .502, .002 }, + { 0.9999999, .7310585589688111, .2310585589688111 }, + { 1, .7310585786300049, .23105857863000487 }, + { 1.0000001, .7310585982911977, .23105859829119774 }, + { 2, .8807970779778823, .3807970779778824 }, + { 708, 1, .5 }, + { 710, 1, .5 }, + { 1000, 1, .5 }, + { HUGE_VAL, 1, .5 }, + }; + double relerr_bound = 3e-15; /* >10eps */ + size_t i; + bool ok = true; + + for (i = 0; i < arraycount(cases); i++) { + double x = cases[i].x; + double p = cases[i].p; + double phalf = cases[i].phalf; + + /* + * cdf is logistic, icdf is logit, and symmetry for + * sf/isf. + */ + CHECK_RELERR(logistic(x), cdf_logistic(x, 0, 1)); + CHECK_RELERR(logistic(-x), sf_logistic(x, 0, 1)); + CHECK_RELERR(logit(p), icdf_logistic(p, 0, 1)); + CHECK_RELERR(-logit(p), isf_logistic(p, 0, 1)); + + CHECK_RELERR(cdf_logistic(x, 0, 1), cdf_logistic(x*2, 0, 2)); + CHECK_RELERR(sf_logistic(x, 0, 1), sf_logistic(x*2, 0, 2)); + CHECK_RELERR(icdf_logistic(p, 0, 1), icdf_logistic(p, 0, 2)/2); + CHECK_RELERR(isf_logistic(p, 0, 1), isf_logistic(p, 0, 2)/2); + + CHECK_RELERR(cdf_logistic(x, 0, 1), cdf_logistic(x/2, 0, .5)); + CHECK_RELERR(sf_logistic(x, 0, 1), sf_logistic(x/2, 0, .5)); + CHECK_RELERR(icdf_logistic(p, 0, 1), icdf_logistic(p, 0,.5)*2); + CHECK_RELERR(isf_logistic(p, 0, 1), isf_logistic(p, 0, .5)*2); + + CHECK_RELERR(cdf_logistic(x, 0, 1), cdf_logistic(x*2 + 1, 1, 2)); + CHECK_RELERR(sf_logistic(x, 0, 1), sf_logistic(x*2 + 1, 1, 2)); + + /* + * For p near 0 and p near 1/2, the arithmetic of + * translating by 1 loses precision. + */ + if (fabs(p) > DBL_EPSILON && fabs(p) < 0.4) { + CHECK_RELERR(icdf_logistic(p, 0, 1), + (icdf_logistic(p, 1, 2) - 1)/2); + CHECK_RELERR(isf_logistic(p, 0, 1), + (isf_logistic(p, 1, 2) - 1)/2); + } + + CHECK_RELERR(p, logistic(x)); + CHECK_RELERR(phalf, logistichalf(x)); + + /* + * On the interior floating-point numbers, either logit or + * logithalf had better give the correct answer. + * + * For probabilities near 0, we can get much finer resolution with + * logit, and for probabilities near 1/2, we can get much finer + * resolution with logithalf by representing them using p - 1/2. + * + * E.g., we can write -.00001 for phalf, and .49999 for p, but the + * difference 1/2 - .00001 gives 1.0000000000010001e-5 in binary64 + * arithmetic. So test logit(.49999) which should give the same + * answer as logithalf(-1.0000000000010001e-5), namely + * -4.000000000537333e-5, and also test logithalf(-.00001) which + * gives -4.000000000533334e-5 instead -- but don't expect + * logit(.49999) to give -4.000000000533334e-5 even though it looks + * like 1/2 - .00001. + * + * A naive implementation of logit will just use log(p/(1 - p)) and + * give the answer -4.000000000551673e-05 for .49999, which is + * wrong in a lot of digits, which happens because log is + * ill-conditioned near 1 and thus amplifies whatever relative + * error we made in computing p/(1 - p). + */ + if ((0 < p && p < 1) || tor_isinf(x)) { + if (phalf >= p - 0.5 && phalf <= p - 0.5) + CHECK_RELERR(x, logit(p)); + if (p >= 0.5 + phalf && p <= 0.5 + phalf) + CHECK_RELERR(x, logithalf(phalf)); + } + + CHECK_RELERR(-phalf, logistichalf(-x)); + if (fabs(phalf) < 0.5 || tor_isinf(x)) + CHECK_RELERR(-x, logithalf(-phalf)); + if (p < 1 || tor_isinf(x)) { + CHECK_RELERR(1 - p, logistic(-x)); + if (p > .75 || tor_isinf(x)) + CHECK_RELERR(-x, logit(1 - p)); + } else { + CHECK_LE(logistic(-x), 1e-300); + } + } + + for (i = 0; i <= 100; i++) { + double p0 = (double)i/100; + + CHECK_RELERR(logit(p0/(1 + M_E)), sample_logistic(0, 0, p0)); + CHECK_RELERR(-logit(p0/(1 + M_E)), sample_logistic(1, 0, p0)); + CHECK_RELERR(logithalf(p0*(0.5 - 1/(1 + M_E))), + sample_logistic(0, 1, p0)); + CHECK_RELERR(-logithalf(p0*(0.5 - 1/(1 + M_E))), + sample_logistic(1, 1, p0)); + } + + if (!ok) + printf("fail logit/logistic / logistic cdf/sf\n"); + + tt_assert(ok); + + done: + ; +} + +/** + * Test the cdf, sf, icdf, and isf of the LogLogistic distribution. + */ +static void +test_log_logistic(void *arg) +{ + (void) arg; + + static const struct { + /* x is a point in the support of the LogLogistic distribution */ + double x; + /* 'p' is the probability that a random variable X for a given LogLogistic + * probability ditribution will take value less-or-equal to x */ + double p; + /* 'np' is the probability that a random variable X for a given LogLogistic + * probability distribution will take value greater-or-equal to x. */ + double np; + } cases[] = { + { 0, 0, 1 }, + { 1e-300, 1e-300, 1 }, + { 1e-17, 1e-17, 1 }, + { 1e-15, 1e-15, .999999999999999 }, + { .1, .09090909090909091, .90909090909090909 }, + { .25, .2, .8 }, + { .5, .33333333333333333, .66666666666666667 }, + { .75, .42857142857142855, .5714285714285714 }, + { .9999, .49997499874993756, .5000250012500626 }, + { .99999999, .49999999749999996, .5000000025 }, + { .999999999999999, .49999999999999994, .5000000000000002 }, + { 1, .5, .5 }, + }; + double relerr_bound = 3e-15; + size_t i; + bool ok = true; + + for (i = 0; i < arraycount(cases); i++) { + double x = cases[i].x; + double p = cases[i].p; + double np = cases[i].np; + + CHECK_RELERR(p, cdf_log_logistic(x, 1, 1)); + CHECK_RELERR(p, cdf_log_logistic(x/2, .5, 1)); + CHECK_RELERR(p, cdf_log_logistic(x*2, 2, 1)); + CHECK_RELERR(p, cdf_log_logistic(sqrt(x), 1, 2)); + CHECK_RELERR(p, cdf_log_logistic(sqrt(x)/2, .5, 2)); + CHECK_RELERR(p, cdf_log_logistic(sqrt(x)*2, 2, 2)); + if (2*sqrt(DBL_MIN) < x) { + CHECK_RELERR(p, cdf_log_logistic(x*x, 1, .5)); + CHECK_RELERR(p, cdf_log_logistic(x*x/2, .5, .5)); + CHECK_RELERR(p, cdf_log_logistic(x*x*2, 2, .5)); + } + + CHECK_RELERR(np, sf_log_logistic(x, 1, 1)); + CHECK_RELERR(np, sf_log_logistic(x/2, .5, 1)); + CHECK_RELERR(np, sf_log_logistic(x*2, 2, 1)); + CHECK_RELERR(np, sf_log_logistic(sqrt(x), 1, 2)); + CHECK_RELERR(np, sf_log_logistic(sqrt(x)/2, .5, 2)); + CHECK_RELERR(np, sf_log_logistic(sqrt(x)*2, 2, 2)); + if (2*sqrt(DBL_MIN) < x) { + CHECK_RELERR(np, sf_log_logistic(x*x, 1, .5)); + CHECK_RELERR(np, sf_log_logistic(x*x/2, .5, .5)); + CHECK_RELERR(np, sf_log_logistic(x*x*2, 2, .5)); + } + + CHECK_RELERR(np, cdf_log_logistic(1/x, 1, 1)); + CHECK_RELERR(np, cdf_log_logistic(1/(2*x), .5, 1)); + CHECK_RELERR(np, cdf_log_logistic(2/x, 2, 1)); + CHECK_RELERR(np, cdf_log_logistic(1/sqrt(x), 1, 2)); + CHECK_RELERR(np, cdf_log_logistic(1/(2*sqrt(x)), .5, 2)); + CHECK_RELERR(np, cdf_log_logistic(2/sqrt(x), 2, 2)); + if (2*sqrt(DBL_MIN) < x && x < 1/(2*sqrt(DBL_MIN))) { + CHECK_RELERR(np, cdf_log_logistic(1/(x*x), 1, .5)); + CHECK_RELERR(np, cdf_log_logistic(1/(2*x*x), .5, .5)); + CHECK_RELERR(np, cdf_log_logistic(2/(x*x), 2, .5)); + } + + CHECK_RELERR(p, sf_log_logistic(1/x, 1, 1)); + CHECK_RELERR(p, sf_log_logistic(1/(2*x), .5, 1)); + CHECK_RELERR(p, sf_log_logistic(2/x, 2, 1)); + CHECK_RELERR(p, sf_log_logistic(1/sqrt(x), 1, 2)); + CHECK_RELERR(p, sf_log_logistic(1/(2*sqrt(x)), .5, 2)); + CHECK_RELERR(p, sf_log_logistic(2/sqrt(x), 2, 2)); + if (2*sqrt(DBL_MIN) < x && x < 1/(2*sqrt(DBL_MIN))) { + CHECK_RELERR(p, sf_log_logistic(1/(x*x), 1, .5)); + CHECK_RELERR(p, sf_log_logistic(1/(2*x*x), .5, .5)); + CHECK_RELERR(p, sf_log_logistic(2/(x*x), 2, .5)); + } + + CHECK_RELERR(x, icdf_log_logistic(p, 1, 1)); + CHECK_RELERR(x/2, icdf_log_logistic(p, .5, 1)); + CHECK_RELERR(x*2, icdf_log_logistic(p, 2, 1)); + CHECK_RELERR(x, icdf_log_logistic(p, 1, 1)); + CHECK_RELERR(sqrt(x)/2, icdf_log_logistic(p, .5, 2)); + CHECK_RELERR(sqrt(x)*2, icdf_log_logistic(p, 2, 2)); + CHECK_RELERR(sqrt(x), icdf_log_logistic(p, 1, 2)); + CHECK_RELERR(x*x/2, icdf_log_logistic(p, .5, .5)); + CHECK_RELERR(x*x*2, icdf_log_logistic(p, 2, .5)); + + if (np < .9) { + CHECK_RELERR(x, isf_log_logistic(np, 1, 1)); + CHECK_RELERR(x/2, isf_log_logistic(np, .5, 1)); + CHECK_RELERR(x*2, isf_log_logistic(np, 2, 1)); + CHECK_RELERR(sqrt(x), isf_log_logistic(np, 1, 2)); + CHECK_RELERR(sqrt(x)/2, isf_log_logistic(np, .5, 2)); + CHECK_RELERR(sqrt(x)*2, isf_log_logistic(np, 2, 2)); + CHECK_RELERR(x*x, isf_log_logistic(np, 1, .5)); + CHECK_RELERR(x*x/2, isf_log_logistic(np, .5, .5)); + CHECK_RELERR(x*x*2, isf_log_logistic(np, 2, .5)); + + CHECK_RELERR(1/x, icdf_log_logistic(np, 1, 1)); + CHECK_RELERR(1/(2*x), icdf_log_logistic(np, .5, 1)); + CHECK_RELERR(2/x, icdf_log_logistic(np, 2, 1)); + CHECK_RELERR(1/sqrt(x), icdf_log_logistic(np, 1, 2)); + CHECK_RELERR(1/(2*sqrt(x)), + icdf_log_logistic(np, .5, 2)); + CHECK_RELERR(2/sqrt(x), icdf_log_logistic(np, 2, 2)); + CHECK_RELERR(1/(x*x), icdf_log_logistic(np, 1, .5)); + CHECK_RELERR(1/(2*x*x), icdf_log_logistic(np, .5, .5)); + CHECK_RELERR(2/(x*x), icdf_log_logistic(np, 2, .5)); + } + + CHECK_RELERR(1/x, isf_log_logistic(p, 1, 1)); + CHECK_RELERR(1/(2*x), isf_log_logistic(p, .5, 1)); + CHECK_RELERR(2/x, isf_log_logistic(p, 2, 1)); + CHECK_RELERR(1/sqrt(x), isf_log_logistic(p, 1, 2)); + CHECK_RELERR(1/(2*sqrt(x)), isf_log_logistic(p, .5, 2)); + CHECK_RELERR(2/sqrt(x), isf_log_logistic(p, 2, 2)); + CHECK_RELERR(1/(x*x), isf_log_logistic(p, 1, .5)); + CHECK_RELERR(1/(2*x*x), isf_log_logistic(p, .5, .5)); + CHECK_RELERR(2/(x*x), isf_log_logistic(p, 2, .5)); + } + + for (i = 0; i <= 100; i++) { + double p0 = (double)i/100; + + CHECK_RELERR(0.5*p0/(1 - 0.5*p0), sample_log_logistic(0, p0)); + CHECK_RELERR((1 - 0.5*p0)/(0.5*p0), + sample_log_logistic(1, p0)); + } + + if (!ok) + printf("fail log logistic cdf/sf\n"); + + tt_assert(ok); + + done: + ; +} + +/** + * Test the cdf, sf, icdf, isf of the Weibull distribution. + */ +static void +test_weibull(void *arg) +{ + (void) arg; + + static const struct { + /* x is a point in the support of the Weibull distribution */ + double x; + /* 'p' is the probability that a random variable X for a given Weibull + * probability ditribution will take value less-or-equal to x */ + double p; + /* 'np' is the probability that a random variable X for a given Weibull + * probability distribution will take value greater-or-equal to x. */ + double np; + } cases[] = { + { 0, 0, 1 }, + { 1e-300, 1e-300, 1 }, + { 1e-17, 1e-17, 1 }, + { .1, .09516258196404043, .9048374180359595 }, + { .5, .3934693402873666, .6065306597126334 }, + { .6931471805599453, .5, .5 }, + { 1, .6321205588285577, .36787944117144233 }, + { 10, .9999546000702375, 4.5399929762484854e-5 }, + { 36, .9999999999999998, 2.319522830243569e-16 }, + { 37, .9999999999999999, 8.533047625744066e-17 }, + { 38, 1, 3.1391327920480296e-17 }, + { 100, 1, 3.720075976020836e-44 }, + { 708, 1, 3.307553003638408e-308 }, + { 710, 1, 4.47628622567513e-309 }, + { 1000, 1, 0 }, + { HUGE_VAL, 1, 0 }, + }; + double relerr_bound = 3e-15; + size_t i; + bool ok = true; + + for (i = 0; i < arraycount(cases); i++) { + double x = cases[i].x; + double p = cases[i].p; + double np = cases[i].np; + + CHECK_RELERR(p, cdf_weibull(x, 1, 1)); + CHECK_RELERR(p, cdf_weibull(x/2, .5, 1)); + CHECK_RELERR(p, cdf_weibull(x*2, 2, 1)); + /* For 0 < x < sqrt(DBL_MIN), x^2 loses lots of bits. */ + if (x <= 0 || + sqrt(DBL_MIN) <= x) { + CHECK_RELERR(p, cdf_weibull(x*x, 1, .5)); + CHECK_RELERR(p, cdf_weibull(x*x/2, .5, .5)); + CHECK_RELERR(p, cdf_weibull(x*x*2, 2, .5)); + } + CHECK_RELERR(p, cdf_weibull(sqrt(x), 1, 2)); + CHECK_RELERR(p, cdf_weibull(sqrt(x)/2, .5, 2)); + CHECK_RELERR(p, cdf_weibull(sqrt(x)*2, 2, 2)); + CHECK_RELERR(np, sf_weibull(x, 1, 1)); + CHECK_RELERR(np, sf_weibull(x/2, .5, 1)); + CHECK_RELERR(np, sf_weibull(x*2, 2, 1)); + CHECK_RELERR(np, sf_weibull(x*x, 1, .5)); + CHECK_RELERR(np, sf_weibull(x*x/2, .5, .5)); + CHECK_RELERR(np, sf_weibull(x*x*2, 2, .5)); + if (x >= 10) { + /* + * exp amplifies the error of sqrt(x)^2 + * proportionally to exp(x); for large inputs + * this is significant. + */ + double t = -expm1(-x*(2*DBL_EPSILON + DBL_EPSILON)); + relerr_bound = t + DBL_EPSILON + t*DBL_EPSILON; + if (relerr_bound < 3e-15) + /* + * The tests are written only to 16 + * decimal places anyway even if your + * `double' is, say, i387 binary80, for + * whatever reason. + */ + relerr_bound = 3e-15; + CHECK_RELERR(np, sf_weibull(sqrt(x), 1, 2)); + CHECK_RELERR(np, sf_weibull(sqrt(x)/2, .5, 2)); + CHECK_RELERR(np, sf_weibull(sqrt(x)*2, 2, 2)); + } + + if (p <= 0.75) { + /* + * For p near 1, not enough precision near 1 to + * recover x. + */ + CHECK_RELERR(x, icdf_weibull(p, 1, 1)); + CHECK_RELERR(x/2, icdf_weibull(p, .5, 1)); + CHECK_RELERR(x*2, icdf_weibull(p, 2, 1)); + } + if (p >= 0.25 && !tor_isinf(x) && np > 0) { + /* + * For p near 0, not enough precision in np + * near 1 to recover x. For 0, isf gives inf, + * even if p is precise enough for the icdf to + * work. + */ + CHECK_RELERR(x, isf_weibull(np, 1, 1)); + CHECK_RELERR(x/2, isf_weibull(np, .5, 1)); + CHECK_RELERR(x*2, isf_weibull(np, 2, 1)); + } + } + + for (i = 0; i <= 100; i++) { + double p0 = (double)i/100; + + CHECK_RELERR(3*sqrt(-log(p0/2)), sample_weibull(0, p0, 3, 2)); + CHECK_RELERR(3*sqrt(-log1p(-p0/2)), + sample_weibull(1, p0, 3, 2)); + } + + if (!ok) + printf("fail Weibull cdf/sf\n"); + + tt_assert(ok); + + done: + ; +} + +/** + * Test the cdf, sf, icdf, and isf of the generalized Pareto + * distribution. + */ +static void +test_genpareto(void *arg) +{ + (void) arg; + + struct { + /* xi is the 'xi' parameter of the generalized Pareto distribution, and the + * rest are the same as in the above tests */ + double xi, x, p, np; + } cases[] = { + { 0, 0, 0, 1 }, + { 1e-300, .004, 3.992010656008528e-3, .9960079893439915 }, + { 1e-300, .1, .09516258196404043, .9048374180359595 }, + { 1e-300, 1, .6321205588285577, .36787944117144233 }, + { 1e-300, 10, .9999546000702375, 4.5399929762484854e-5 }, + { 1e-200, 1e-16, 9.999999999999999e-17, .9999999999999999 }, + { 1e-16, 1e-200, 9.999999999999998e-201, 1 }, + { 1e-16, 1e-16, 1e-16, 1 }, + { 1e-16, .004, 3.992010656008528e-3, .9960079893439915 }, + { 1e-16, .1, .09516258196404043, .9048374180359595 }, + { 1e-16, 1, .6321205588285577, .36787944117144233 }, + { 1e-16, 10, .9999546000702375, 4.539992976248509e-5 }, + { 1e-10, 1e-6, 9.999995000001667e-7, .9999990000005 }, + { 1e-8, 1e-8, 9.999999950000001e-9, .9999999900000001 }, + { 1, 1e-300, 1e-300, 1 }, + { 1, 1e-16, 1e-16, .9999999999999999 }, + { 1, .1, .09090909090909091, .9090909090909091 }, + { 1, 1, .5, .5 }, + { 1, 10, .9090909090909091, .0909090909090909 }, + { 1, 100, .9900990099009901, .0099009900990099 }, + { 1, 1000, .999000999000999, 9.990009990009992e-4 }, + { 10, 1e-300, 1e-300, 1 }, + { 10, 1e-16, 9.999999999999995e-17, .9999999999999999 }, + { 10, .1, .06696700846319258, .9330329915368074 }, + { 10, 1, .21320655780322778, .7867934421967723 }, + { 10, 10, .3696701667040189, .6303298332959811 }, + { 10, 100, .49886285755007337, .5011371424499267 }, + { 10, 1000, .6018968102992647, .3981031897007353 }, + }; + double xi_array[] = { -1.5, -1, -1e-30, 0, 1e-30, 1, 1.5 }; + size_t i, j; + double relerr_bound = 3e-15; + bool ok = true; + + for (i = 0; i < arraycount(cases); i++) { + double xi = cases[i].xi; + double x = cases[i].x; + double p = cases[i].p; + double np = cases[i].np; + + CHECK_RELERR(p, cdf_genpareto(x, 0, 1, xi)); + CHECK_RELERR(p, cdf_genpareto(x*2, 0, 2, xi)); + CHECK_RELERR(p, cdf_genpareto(x/2, 0, .5, xi)); + CHECK_RELERR(np, sf_genpareto(x, 0, 1, xi)); + CHECK_RELERR(np, sf_genpareto(x*2, 0, 2, xi)); + CHECK_RELERR(np, sf_genpareto(x/2, 0, .5, xi)); + + if (p < .5) { + CHECK_RELERR(x, icdf_genpareto(p, 0, 1, xi)); + CHECK_RELERR(x*2, icdf_genpareto(p, 0, 2, xi)); + CHECK_RELERR(x/2, icdf_genpareto(p, 0, .5, xi)); + } + if (np < .5) { + CHECK_RELERR(x, isf_genpareto(np, 0, 1, xi)); + CHECK_RELERR(x*2, isf_genpareto(np, 0, 2, xi)); + CHECK_RELERR(x/2, isf_genpareto(np, 0, .5, xi)); + } + } + + for (i = 0; i < arraycount(xi_array); i++) { + for (j = 0; j <= 100; j++) { + double p0 = (j == 0 ? 2*DBL_MIN : (double)j/100); + + /* This is actually a check against 0, but we do <= so that the compiler + does not raise a -Wfloat-equal */ + if (fabs(xi_array[i]) <= 0) { + /* + * When xi == 0, the generalized Pareto + * distribution reduces to an + * exponential distribution. + */ + CHECK_RELERR(-log(p0/2), + sample_genpareto(0, p0, 0)); + CHECK_RELERR(-log1p(-p0/2), + sample_genpareto(1, p0, 0)); + } else { + CHECK_RELERR(expm1(-xi_array[i]*log(p0/2))/xi_array[i], + sample_genpareto(0, p0, xi_array[i])); + CHECK_RELERR((j == 0 ? DBL_MIN : + expm1(-xi_array[i]*log1p(-p0/2))/xi_array[i]), + sample_genpareto(1, p0, xi_array[i])); + } + + CHECK_RELERR(isf_genpareto(p0/2, 0, 1, xi_array[i]), + sample_genpareto(0, p0, xi_array[i])); + CHECK_RELERR(icdf_genpareto(p0/2, 0, 1, xi_array[i]), + sample_genpareto(1, p0, xi_array[i])); + } + } + + tt_assert(ok); + + done: + ; +} + +/** + * Test the deterministic sampler for uniform distribution on [a, b]. + * + * This currently only tests whether the outcome lies within [a, b]. + */ +static void +test_uniform_interval(void *arg) +{ + (void) arg; + struct { + /* Sample from a uniform distribution with parameters 'a' and 'b', using + * 't' as the sampling index. */ + double t, a, b; + } cases[] = { + { 0, 0, 0 }, + { 0, 0, 1 }, + { 0, 1.0000000000000007, 3.999999999999995 }, + { 0, 4000, 4000 }, + { 0.42475836677491291, 4000, 4000 }, + { 0, -DBL_MAX, DBL_MAX }, + { 0.25, -DBL_MAX, DBL_MAX }, + { 0.5, -DBL_MAX, DBL_MAX }, + }; + size_t i = 0; + bool ok = true; + + for (i = 0; i < arraycount(cases); i++) { + double t = cases[i].t; + double a = cases[i].a; + double b = cases[i].b; + + CHECK_LE(a, sample_uniform_interval(t, a, b)); + CHECK_LE(sample_uniform_interval(t, a, b), b); + + CHECK_LE(a, sample_uniform_interval(1 - t, a, b)); + CHECK_LE(sample_uniform_interval(1 - t, a, b), b); + + CHECK_LE(sample_uniform_interval(t, -b, -a), -a); + CHECK_LE(-b, sample_uniform_interval(t, -b, -a)); + + CHECK_LE(sample_uniform_interval(1 - t, -b, -a), -a); + CHECK_LE(-b, sample_uniform_interval(1 - t, -b, -a)); + } + + tt_assert(ok); + + done: + ; +} + +/********************** Stochastic tests ****************************/ + +/* + * Psi test, sometimes also called G-test. The psi test statistic, + * suitably scaled, has chi^2 distribution, but the psi test tends to + * have better statistical power in practice to detect deviations than + * the chi^2 test does. (The chi^2 test statistic is the first term of + * the Taylor expansion of the psi test statistic.) The psi test is + * generic, for any CDF; particular distributions might have higher- + * power tests to distinguish them from predictable deviations or bugs. + * + * We choose the psi critical value so that a single psi test has + * probability below alpha = 1% of spuriously failing even if all the + * code is correct. But the false positive rate for a suite of n tests + * is higher: 1 - Binom(0; n, alpha) = 1 - (1 - alpha)^n. For n = 10, + * this is about 10%, and for n = 100 it is well over 50%. + * + * Given that these tests will run with every CI job, we want to drive down the + * false positive rate. We can drive it down by running each test four times, + * and accepting it if it passes at least once; in that case, it is as if we + * used Binom(4; 2, alpha) = alpha^4 as the false positive rate for each test, + * and for n = 10 tests, it would be 9.99999959506e-08. If each CI build has 14 + * jobs, then the chance of a CI build failing is 1.39999903326e-06, which + * means that a CI build will break with probability 50% after about 495106 + * builds. + * + * The critical value for a chi^2 distribution with 100 degrees of + * freedom and false positive rate alpha = 1% was taken from: + * + * NIST/SEMATECH e-Handbook of Statistical Methods, Section + * 1.3.6.7.4 `Critical Values of the Chi-Square Distribution', + * <http://www.itl.nist.gov/div898/handbook/eda/section3/eda3674.htm>, + * retrieved 2018-10-28. + */ + +static const size_t NSAMPLES = 100000; +/* Number of chances we give to the test to succeed. */ +static const unsigned NTRIALS = 4; +/* Number of times we want the test to pass per NTRIALS. */ +static const unsigned NPASSES_MIN = 1; + +#define PSI_DF 100 /* degrees of freedom */ +static const double PSI_CRITICAL = 135.807; /* critical value, alpha = .01 */ + +/** + * Perform a psi test on an array of sample counts, C, adding up to N + * samples, and an array of log expected probabilities, logP, + * representing the null hypothesis for the distribution of samples + * counted. Return false if the psi test rejects the null hypothesis, + * true if otherwise. + */ +static bool +psi_test(const size_t C[PSI_DF], const double logP[PSI_DF], size_t N) +{ + double psi = 0; + double c = 0; /* Kahan compensation */ + double t, u; + size_t i; + + for (i = 0; i < PSI_DF; i++) { + /* + * c*log(c/(n*p)) = (1/n) * f*log(f/p) where f = c/n is + * the frequency, and f*log(f/p) ---> 0 as f ---> 0, so + * this is a reasonable choice. Further, any mass that + * _fails_ to turn up in this bin will inflate another + * bin instead, so we don't really lose anything by + * ignoring empty bins even if they have high + * probability. + */ + if (C[i] == 0) + continue; + t = C[i]*(log((double)C[i]/N) - logP[i]) - c; + u = psi + t; + c = (u - psi) - t; + psi = u; + } + psi *= 2; + + return psi <= PSI_CRITICAL; +} + +static bool +test_stochastic_geometric_impl(double p) +{ + const struct geometric geometric = { + .base = GEOMETRIC(geometric), + .p = p, + }; + double logP[PSI_DF] = {0}; + unsigned ntry = NTRIALS, npass = 0; + unsigned i; + size_t j; + + /* Compute logP[i] = Geom(i + 1; p). */ + for (i = 0; i < PSI_DF - 1; i++) + logP[i] = logpmf_geometric(i + 1, p); + + /* Compute logP[n-1] = log (1 - (P[0] + P[1] + ... + P[n-2])). */ + logP[PSI_DF - 1] = log1mexp(logsumexp(logP, PSI_DF - 1)); + + while (ntry --> 0) { + size_t C[PSI_DF] = {0}; + + for (j = 0; j < NSAMPLES; j++) { + double n_tmp = dist_sample(&geometric.base); + + /* Must be an integer. (XXX -Wfloat-equal) */ + tor_assert(ceil(n_tmp) <= n_tmp && ceil(n_tmp) >= n_tmp); + + /* Must be a positive integer. */ + tor_assert(n_tmp >= 1); + + /* Probability of getting a value in the billions is negligible. */ + tor_assert(n_tmp <= (double)UINT_MAX); + + unsigned n = (unsigned) n_tmp; + + if (n > PSI_DF) + n = PSI_DF; + C[n - 1]++; + } + + if (psi_test(C, logP, NSAMPLES)) { + if (++npass >= NPASSES_MIN) + break; + } + } + + if (npass >= NPASSES_MIN) { + /* printf("pass %s sampler\n", "geometric"); */ + return true; + } else { + printf("fail %s sampler\n", "geometric"); + return false; + } +} + +/** + * Divide the support of <b>dist</b> into histogram bins in <b>logP</b>. Start + * at the 1st percentile and ending at the 99th percentile. Pick the bin + * boundaries using linear interpolation so that they are uniformly spaced. + * + * In each bin logP[i] we insert the expected log-probability that a sampled + * value will fall into that bin. We will use this as the null hypothesis of + * the psi test. + * + * Set logP[i] = log(CDF(x_i) - CDF(x_{i-1})), where x_-1 = -inf, x_n = + * +inf, and x_i = i*(hi - lo)/(n - 2). + */ +static void +bin_cdfs(const struct dist *dist, double lo, double hi, double *logP, size_t n) +{ +#define CDF(x) dist_cdf(dist, x) +#define SF(x) dist_sf(dist, x) + const double w = (hi - lo)/(n - 2); + double halfway = dist_icdf(dist, 0.5); + double x_0, x_1; + size_t i; + size_t n2 = ceil_to_size_t((halfway - lo)/w); + + tor_assert(lo <= halfway); + tor_assert(halfway <= hi); + tor_assert(n2 <= n); + + x_1 = lo; + logP[0] = log(CDF(x_1) - 0); /* 0 = CDF(-inf) */ + for (i = 1; i < n2; i++) { + x_0 = x_1; + /* do the linear interpolation */ + x_1 = (i <= n/2 ? lo + i*w : hi - (n - 2 - i)*w); + /* set the expected log-probability */ + logP[i] = log(CDF(x_1) - CDF(x_0)); + } + x_0 = hi; + logP[n - 1] = log(SF(x_0) - 0); /* 0 = SF(+inf) = 1 - CDF(+inf) */ + + /* In this loop we are filling out the high part of the array. We are using + * SF because in these cases the CDF is near 1 where precision is lower. So + * instead we are using SF near 0 where the precision is higher. We have + * SF(t) = 1 - CDF(t). */ + for (i = 1; i < n - n2; i++) { + x_1 = x_0; + /* do the linear interpolation */ + x_0 = (i <= n/2 ? hi - i*w : lo + (n - 2 - i)*w); + /* set the expected log-probability */ + logP[n - i - 1] = log(SF(x_0) - SF(x_1)); + } +#undef SF +#undef CDF +} + +/** + * Draw NSAMPLES samples from dist, counting the number of samples x in + * the ith bin C[i] if x_{i-1} <= x < x_i, where x_-1 = -inf, x_n = + * +inf, and x_i = i*(hi - lo)/(n - 2). + */ +static void +bin_samples(const struct dist *dist, double lo, double hi, size_t *C, size_t n) +{ + const double w = (hi - lo)/(n - 2); + size_t i; + + for (i = 0; i < NSAMPLES; i++) { + double x = dist_sample(dist); + size_t bin; + + if (x < lo) + bin = 0; + else if (x < hi) + bin = 1 + floor_to_size_t((x - lo)/w); + else + bin = n - 1; + tor_assert(bin < n); + C[bin]++; + } +} + +/** + * Carry out a Psi test on <b>dist</b>. + * + * Sample NSAMPLES from dist, putting them in bins from -inf to lo to + * hi to +inf, and apply up to two psi tests. True if at least one psi + * test passes; false if not. False positive rate should be bounded by + * 0.01^2 = 0.0001. + */ +static bool +test_psi_dist_sample(const struct dist *dist) +{ + double logP[PSI_DF] = {0}; + unsigned ntry = NTRIALS, npass = 0; + double lo = dist_icdf(dist, 1/(double)(PSI_DF + 2)); + double hi = dist_isf(dist, 1/(double)(PSI_DF + 2)); + + /* Create the null hypothesis in logP */ + bin_cdfs(dist, lo, hi, logP, PSI_DF); + + /* Now run the test */ + while (ntry --> 0) { + size_t C[PSI_DF] = {0}; + bin_samples(dist, lo, hi, C, PSI_DF); + if (psi_test(C, logP, NSAMPLES)) { + if (++npass >= NPASSES_MIN) + break; + } + } + + /* Did we fail or succeed? */ + if (npass >= NPASSES_MIN) { + /* printf("pass %s sampler\n", dist_name(dist));*/ + return true; + } else { + printf("fail %s sampler\n", dist_name(dist)); + return false; + } +} + +/* This is the seed of the deterministic randomness */ +static uint8_t rng_seed[16]; +static crypto_xof_t *rng_xof = NULL; + +/** Initialize the seed of the deterministic randomness. */ +static void +init_deterministic_rand(void) +{ + crypto_rand((char*)rng_seed, sizeof(rng_seed)); + crypto_xof_free(rng_xof); + rng_xof = crypto_xof_new(); + crypto_xof_add_bytes(rng_xof, rng_seed, sizeof(rng_seed)); +} + +static void +teardown_deterministic_rand(void) +{ + crypto_xof_free(rng_xof); +} + +static void +dump_seed(void) +{ + printf("\n" + "NOTE: This is a stochastic test, and we expect it to fail from\n" + "time to time, with some low probability. If you see it fail more\n" + "than one trial in 100, though, please tell us.\n\n" + "Seed: %s\n", + hex_str((const char*)rng_seed, sizeof(rng_seed))); +} + +/** Produce deterministic randomness for the stochastic tests using the global + * deterministic_rand_counter seed + * + * This function produces deterministic data over multiple calls iff it's + * called in the same call order with the same 'n' parameter (which is the + * case for the psi test). If not, outputs will deviate. */ +static void +crypto_rand_deterministic(char *out, size_t n) +{ + /* Use a XOF to squeeze bytes out of that silly counter */ + tor_assert(rng_xof); + crypto_xof_squeeze_bytes(rng_xof, (uint8_t*)out, n); +} + +static void +test_stochastic_uniform(void *arg) +{ + (void) arg; + + const struct uniform uniform01 = { + .base = UNIFORM(uniform01), + .a = 0, + .b = 1, + }; + const struct uniform uniform_pos = { + .base = UNIFORM(uniform_pos), + .a = 1.23, + .b = 4.56, + }; + const struct uniform uniform_neg = { + .base = UNIFORM(uniform_neg), + .a = -10, + .b = -1, + }; + const struct uniform uniform_cross = { + .base = UNIFORM(uniform_cross), + .a = -1.23, + .b = 4.56, + }; + const struct uniform uniform_subnormal = { + .base = UNIFORM(uniform_subnormal), + .a = 4e-324, + .b = 4e-310, + }; + const struct uniform uniform_subnormal_cross = { + .base = UNIFORM(uniform_subnormal_cross), + .a = -4e-324, + .b = 4e-310, + }; + bool ok = true, tests_failed = true; + + init_deterministic_rand(); + MOCK(crypto_rand, crypto_rand_deterministic); + + ok &= test_psi_dist_sample(&uniform01.base); + ok &= test_psi_dist_sample(&uniform_pos.base); + ok &= test_psi_dist_sample(&uniform_neg.base); + ok &= test_psi_dist_sample(&uniform_cross.base); + ok &= test_psi_dist_sample(&uniform_subnormal.base); + ok &= test_psi_dist_sample(&uniform_subnormal_cross.base); + + tt_assert(ok); + + tests_failed = false; + + done: + if (tests_failed) { + dump_seed(); + } + teardown_deterministic_rand(); + UNMOCK(crypto_rand); +} + +static bool +test_stochastic_logistic_impl(double mu, double sigma) +{ + const struct logistic dist = { + .base = LOGISTIC(dist), + .mu = mu, + .sigma = sigma, + }; + + /* XXX Consider some fancier logistic test. */ + return test_psi_dist_sample(&dist.base); +} + +static bool +test_stochastic_log_logistic_impl(double alpha, double beta) +{ + const struct log_logistic dist = { + .base = LOG_LOGISTIC(dist), + .alpha = alpha, + .beta = beta, + }; + + /* XXX Consider some fancier log logistic test. */ + return test_psi_dist_sample(&dist.base); +} + +static bool +test_stochastic_weibull_impl(double lambda, double k) +{ + const struct weibull dist = { + .base = WEIBULL(dist), + .lambda = lambda, + .k = k, + }; + +/* + * XXX Consider applying a Tiku-Singh test: + * + * M.L. Tiku and M. Singh, `Testing the two-parameter + * Weibull distribution', Communications in Statistics -- + * Theory and Methods A10(9), 1981, 907--918. + *https://www.tandfonline.com/doi/pdf/10.1080/03610928108828082?needAccess=true + */ + return test_psi_dist_sample(&dist.base); +} + +static bool +test_stochastic_genpareto_impl(double mu, double sigma, double xi) +{ + const struct genpareto dist = { + .base = GENPARETO(dist), + .mu = mu, + .sigma = sigma, + .xi = xi, + }; + + /* XXX Consider some fancier GPD test. */ + return test_psi_dist_sample(&dist.base); +} + +static void +test_stochastic_genpareto(void *arg) +{ + bool ok = 0; + bool tests_failed = true; + (void) arg; + + init_deterministic_rand(); + MOCK(crypto_rand, crypto_rand_deterministic); + + ok = test_stochastic_genpareto_impl(0, 1, -0.25); + tt_assert(ok); + ok = test_stochastic_genpareto_impl(0, 1, -1e-30); + tt_assert(ok); + ok = test_stochastic_genpareto_impl(0, 1, 0); + tt_assert(ok); + ok = test_stochastic_genpareto_impl(0, 1, 1e-30); + tt_assert(ok); + ok = test_stochastic_genpareto_impl(0, 1, 0.25); + tt_assert(ok); + ok = test_stochastic_genpareto_impl(-1, 1, -0.25); + tt_assert(ok); + ok = test_stochastic_genpareto_impl(1, 2, 0.25); + tt_assert(ok); + + tests_failed = false; + + done: + if (tests_failed) { + dump_seed(); + } + teardown_deterministic_rand(); + UNMOCK(crypto_rand); +} + +static void +test_stochastic_geometric(void *arg) +{ + bool ok = 0; + bool tests_failed = true; + + (void) arg; + + init_deterministic_rand(); + MOCK(crypto_rand, crypto_rand_deterministic); + + ok = test_stochastic_geometric_impl(0.1); + tt_assert(ok); + ok = test_stochastic_geometric_impl(0.5); + tt_assert(ok); + ok = test_stochastic_geometric_impl(0.9); + tt_assert(ok); + ok = test_stochastic_geometric_impl(1); + tt_assert(ok); + + tests_failed = false; + + done: + if (tests_failed) { + dump_seed(); + } + teardown_deterministic_rand(); + UNMOCK(crypto_rand); +} + +static void +test_stochastic_logistic(void *arg) +{ + bool ok = 0; + bool tests_failed = true; + (void) arg; + + init_deterministic_rand(); + MOCK(crypto_rand, crypto_rand_deterministic); + + ok = test_stochastic_logistic_impl(0, 1); + tt_assert(ok); + ok = test_stochastic_logistic_impl(0, 1e-16); + tt_assert(ok); + ok = test_stochastic_logistic_impl(1, 10); + tt_assert(ok); + ok = test_stochastic_logistic_impl(-10, 100); + tt_assert(ok); + + tests_failed = false; + + done: + if (tests_failed) { + dump_seed(); + } + teardown_deterministic_rand(); + UNMOCK(crypto_rand); +} + +static void +test_stochastic_log_logistic(void *arg) +{ + bool ok = 0; + bool tests_failed = true; + (void) arg; + + init_deterministic_rand(); + MOCK(crypto_rand, crypto_rand_deterministic); + + ok = test_stochastic_log_logistic_impl(1, 1); + tt_assert(ok); + ok = test_stochastic_log_logistic_impl(1, 10); + tt_assert(ok); + ok = test_stochastic_log_logistic_impl(M_E, 1e-1); + tt_assert(ok); + ok = test_stochastic_log_logistic_impl(exp(-10), 1e-2); + tt_assert(ok); + + tests_failed = false; + + done: + if (tests_failed) { + dump_seed(); + } + teardown_deterministic_rand(); + UNMOCK(crypto_rand); +} + +static void +test_stochastic_weibull(void *arg) +{ + bool ok = 0; + bool tests_failed = true; + (void) arg; + + init_deterministic_rand(); + MOCK(crypto_rand, crypto_rand_deterministic); + + ok = test_stochastic_weibull_impl(1, 0.5); + tt_assert(ok); + ok = test_stochastic_weibull_impl(1, 1); + tt_assert(ok); + ok = test_stochastic_weibull_impl(1, 1.5); + tt_assert(ok); + ok = test_stochastic_weibull_impl(1, 2); + tt_assert(ok); + ok = test_stochastic_weibull_impl(10, 1); + tt_assert(ok); + + tests_failed = false; + + done: + if (tests_failed) { + dump_seed(); + } + teardown_deterministic_rand(); + UNMOCK(crypto_rand); +} + +struct testcase_t prob_distr_tests[] = { + { "logit_logistics", test_logit_logistic, TT_FORK, NULL, NULL }, + { "log_logistic", test_log_logistic, TT_FORK, NULL, NULL }, + { "weibull", test_weibull, TT_FORK, NULL, NULL }, + { "genpareto", test_genpareto, TT_FORK, NULL, NULL }, + { "uniform_interval", test_uniform_interval, TT_FORK, NULL, NULL }, + END_OF_TESTCASES +}; + +struct testcase_t slow_stochastic_prob_distr_tests[] = { + { "stochastic_genpareto", test_stochastic_genpareto, TT_FORK, NULL, NULL }, + { "stochastic_geometric", test_stochastic_geometric, TT_FORK, NULL, NULL }, + { "stochastic_uniform", test_stochastic_uniform, TT_FORK, NULL, NULL }, + { "stochastic_logistic", test_stochastic_logistic, TT_FORK, NULL, NULL }, + { "stochastic_log_logistic", test_stochastic_log_logistic, TT_FORK, NULL, + NULL }, + { "stochastic_weibull", test_stochastic_weibull, TT_FORK, NULL, NULL }, + END_OF_TESTCASES +}; diff --git a/src/test/test_process.c b/src/test/test_process.c new file mode 100644 index 0000000000..7cc01d2442 --- /dev/null +++ b/src/test/test_process.c @@ -0,0 +1,669 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file test_process.c + * \brief Test cases for the Process API. + */ + +#include "orconfig.h" +#include "core/or/or.h" +#include "test/test.h" +#include "lib/process/env.h" + +#define PROCESS_PRIVATE +#include "lib/process/process.h" +#define PROCESS_UNIX_PRIVATE +#include "lib/process/process_unix.h" +#define PROCESS_WIN32_PRIVATE +#include "lib/process/process_win32.h" + +static const char *stdout_read_buffer; +static const char *stderr_read_buffer; + +struct process_data_t { + smartlist_t *stdout_data; + smartlist_t *stderr_data; + smartlist_t *stdin_data; + process_exit_code_t exit_code; +}; + +typedef struct process_data_t process_data_t; + +static process_data_t * +process_data_new(void) +{ + process_data_t *process_data = tor_malloc_zero(sizeof(process_data_t)); + process_data->stdout_data = smartlist_new(); + process_data->stderr_data = smartlist_new(); + process_data->stdin_data = smartlist_new(); + return process_data; +} + +static void +process_data_free(process_data_t *process_data) +{ + if (process_data == NULL) + return; + + SMARTLIST_FOREACH(process_data->stdout_data, char *, x, tor_free(x)); + SMARTLIST_FOREACH(process_data->stderr_data, char *, x, tor_free(x)); + SMARTLIST_FOREACH(process_data->stdin_data, char *, x, tor_free(x)); + + smartlist_free(process_data->stdout_data); + smartlist_free(process_data->stderr_data); + smartlist_free(process_data->stdin_data); + tor_free(process_data); +} + +static int +process_mocked_read_stdout(process_t *process, buf_t *buffer) +{ + (void)process; + + if (stdout_read_buffer != NULL) { + buf_add_string(buffer, stdout_read_buffer); + stdout_read_buffer = NULL; + } + + return (int)buf_datalen(buffer); +} + +static int +process_mocked_read_stderr(process_t *process, buf_t *buffer) +{ + (void)process; + + if (stderr_read_buffer != NULL) { + buf_add_string(buffer, stderr_read_buffer); + stderr_read_buffer = NULL; + } + + return (int)buf_datalen(buffer); +} + +static void +process_mocked_write_stdin(process_t *process, buf_t *buffer) +{ + const size_t size = buf_datalen(buffer); + + if (size == 0) + return; + + char *data = tor_malloc_zero(size + 1); + process_data_t *process_data = process_get_data(process); + + buf_get_bytes(buffer, data, size); + smartlist_add(process_data->stdin_data, data); +} + +static void +process_stdout_callback(process_t *process, const char *data, size_t size) +{ + tt_ptr_op(process, OP_NE, NULL); + tt_ptr_op(data, OP_NE, NULL); + tt_int_op(strlen(data), OP_EQ, size); + + process_data_t *process_data = process_get_data(process); + smartlist_add(process_data->stdout_data, tor_strdup(data)); + + done: + return; +} + +static void +process_stderr_callback(process_t *process, const char *data, size_t size) +{ + tt_ptr_op(process, OP_NE, NULL); + tt_ptr_op(data, OP_NE, NULL); + tt_int_op(strlen(data), OP_EQ, size); + + process_data_t *process_data = process_get_data(process); + smartlist_add(process_data->stderr_data, tor_strdup(data)); + + done: + return; +} + +static bool +process_exit_callback(process_t *process, process_exit_code_t exit_code) +{ + tt_ptr_op(process, OP_NE, NULL); + + process_data_t *process_data = process_get_data(process); + process_data->exit_code = exit_code; + + done: + /* Do not free up our process_t. */ + return false; +} + +static void +test_default_values(void *arg) +{ + (void)arg; + process_t *process = process_new("/path/to/nothing"); + + /* We are not running by default. */ + tt_int_op(PROCESS_STATUS_NOT_RUNNING, OP_EQ, process_get_status(process)); + + /* We use the line protocol by default. */ + tt_int_op(PROCESS_PROTOCOL_LINE, OP_EQ, process_get_protocol(process)); + + /* We don't set any custom data by default. */ + tt_ptr_op(NULL, OP_EQ, process_get_data(process)); + + /* Our command was given to the process_t's constructor in process_new(). */ + tt_str_op("/path/to/nothing", OP_EQ, process_get_command(process)); + + /* Make sure we are listed in the list of proccesses. */ + tt_assert(smartlist_contains(process_get_all_processes(), + process)); + + /* Default PID is 0. */ + tt_u64_op(0, OP_EQ, process_get_pid(process)); + + /* Our arguments should be empty. */ + tt_int_op(0, OP_EQ, + smartlist_len(process_get_arguments(process))); + + done: + process_free(process); +} + +static void +test_environment(void *arg) +{ + (void)arg; + + process_t *process = process_new(""); + process_environment_t *env = NULL; + + process_set_environment(process, "E", "F"); + process_set_environment(process, "C", "D"); + process_set_environment(process, "A", "B"); + + env = process_get_environment(process); + tt_mem_op(env->windows_environment_block, OP_EQ, + "A=B\0C=D\0E=F\0", 12); + tt_str_op(env->unixoid_environment_block[0], OP_EQ, + "A=B"); + tt_str_op(env->unixoid_environment_block[1], OP_EQ, + "C=D"); + tt_str_op(env->unixoid_environment_block[2], OP_EQ, + "E=F"); + tt_ptr_op(env->unixoid_environment_block[3], OP_EQ, + NULL); + process_environment_free(env); + + /* Reset our environment. */ + smartlist_t *new_env = smartlist_new(); + smartlist_add(new_env, (char *)"FOO=bar"); + smartlist_add(new_env, (char *)"HELLO=world"); + + process_reset_environment(process, new_env); + smartlist_free(new_env); + + env = process_get_environment(process); + tt_mem_op(env->windows_environment_block, OP_EQ, + "FOO=bar\0HELLO=world\0", 20); + tt_str_op(env->unixoid_environment_block[0], OP_EQ, + "FOO=bar"); + tt_str_op(env->unixoid_environment_block[1], OP_EQ, + "HELLO=world"); + tt_ptr_op(env->unixoid_environment_block[2], OP_EQ, + NULL); + + done: + process_environment_free(env); + process_free(process); +} + +static void +test_stringified_types(void *arg) +{ + (void)arg; + + /* process_protocol_t values. */ + tt_str_op("Raw", OP_EQ, process_protocol_to_string(PROCESS_PROTOCOL_RAW)); + tt_str_op("Line", OP_EQ, process_protocol_to_string(PROCESS_PROTOCOL_LINE)); + + /* process_status_t values. */ + tt_str_op("not running", OP_EQ, + process_status_to_string(PROCESS_STATUS_NOT_RUNNING)); + tt_str_op("running", OP_EQ, + process_status_to_string(PROCESS_STATUS_RUNNING)); + tt_str_op("error", OP_EQ, + process_status_to_string(PROCESS_STATUS_ERROR)); + + done: + return; +} + +static void +test_line_protocol_simple(void *arg) +{ + (void)arg; + + process_data_t *process_data = process_data_new(); + + process_t *process = process_new(""); + process_set_data(process, process_data); + + process_set_stdout_read_callback(process, process_stdout_callback); + process_set_stderr_read_callback(process, process_stderr_callback); + + MOCK(process_read_stdout, process_mocked_read_stdout); + MOCK(process_read_stderr, process_mocked_read_stderr); + + /* Make sure we are running with the line protocol. */ + tt_int_op(PROCESS_PROTOCOL_LINE, OP_EQ, process_get_protocol(process)); + + tt_int_op(0, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(0, OP_EQ, smartlist_len(process_data->stderr_data)); + + stdout_read_buffer = "Hello stdout\n"; + process_notify_event_stdout(process); + tt_ptr_op(NULL, OP_EQ, stdout_read_buffer); + + stderr_read_buffer = "Hello stderr\r\n"; + process_notify_event_stderr(process); + tt_ptr_op(NULL, OP_EQ, stderr_read_buffer); + + /* Data should be ready. */ + tt_int_op(1, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(1, OP_EQ, smartlist_len(process_data->stderr_data)); + + /* Check if the data is correct. */ + tt_str_op(smartlist_get(process_data->stdout_data, 0), OP_EQ, + "Hello stdout"); + tt_str_op(smartlist_get(process_data->stderr_data, 0), OP_EQ, + "Hello stderr"); + + done: + process_data_free(process_data); + process_free(process); + + UNMOCK(process_read_stdout); + UNMOCK(process_read_stderr); +} + +static void +test_line_protocol_multi(void *arg) +{ + (void)arg; + + process_data_t *process_data = process_data_new(); + + process_t *process = process_new(""); + process_set_data(process, process_data); + process_set_stdout_read_callback(process, process_stdout_callback); + process_set_stderr_read_callback(process, process_stderr_callback); + + MOCK(process_read_stdout, process_mocked_read_stdout); + MOCK(process_read_stderr, process_mocked_read_stderr); + + /* Make sure we are running with the line protocol. */ + tt_int_op(PROCESS_PROTOCOL_LINE, OP_EQ, process_get_protocol(process)); + + tt_int_op(0, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(0, OP_EQ, smartlist_len(process_data->stderr_data)); + + stdout_read_buffer = "Hello stdout\r\nOnion Onion Onion\nA B C D\r\n\r\n"; + process_notify_event_stdout(process); + tt_ptr_op(NULL, OP_EQ, stdout_read_buffer); + + stderr_read_buffer = "Hello stderr\nFoo bar baz\nOnion Onion Onion\n"; + process_notify_event_stderr(process); + tt_ptr_op(NULL, OP_EQ, stderr_read_buffer); + + /* Data should be ready. */ + tt_int_op(4, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(3, OP_EQ, smartlist_len(process_data->stderr_data)); + + /* Check if the data is correct. */ + tt_str_op(smartlist_get(process_data->stdout_data, 0), OP_EQ, + "Hello stdout"); + tt_str_op(smartlist_get(process_data->stdout_data, 1), OP_EQ, + "Onion Onion Onion"); + tt_str_op(smartlist_get(process_data->stdout_data, 2), OP_EQ, + "A B C D"); + tt_str_op(smartlist_get(process_data->stdout_data, 3), OP_EQ, + ""); + + tt_str_op(smartlist_get(process_data->stderr_data, 0), OP_EQ, + "Hello stderr"); + tt_str_op(smartlist_get(process_data->stderr_data, 1), OP_EQ, + "Foo bar baz"); + tt_str_op(smartlist_get(process_data->stderr_data, 2), OP_EQ, + "Onion Onion Onion"); + + done: + process_data_free(process_data); + process_free(process); + + UNMOCK(process_read_stdout); + UNMOCK(process_read_stderr); +} + +static void +test_line_protocol_partial(void *arg) +{ + (void)arg; + + process_data_t *process_data = process_data_new(); + + process_t *process = process_new(""); + process_set_data(process, process_data); + process_set_stdout_read_callback(process, process_stdout_callback); + process_set_stderr_read_callback(process, process_stderr_callback); + + MOCK(process_read_stdout, process_mocked_read_stdout); + MOCK(process_read_stderr, process_mocked_read_stderr); + + /* Make sure we are running with the line protocol. */ + tt_int_op(PROCESS_PROTOCOL_LINE, OP_EQ, process_get_protocol(process)); + + tt_int_op(0, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(0, OP_EQ, smartlist_len(process_data->stderr_data)); + + stdout_read_buffer = "Hello stdout this is a partial line ..."; + process_notify_event_stdout(process); + tt_ptr_op(NULL, OP_EQ, stdout_read_buffer); + + stderr_read_buffer = "Hello stderr this is a partial line ..."; + process_notify_event_stderr(process); + tt_ptr_op(NULL, OP_EQ, stderr_read_buffer); + + /* Data should NOT be ready. */ + tt_int_op(0, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(0, OP_EQ, smartlist_len(process_data->stderr_data)); + + stdout_read_buffer = " the end\nAnother partial string goes here ..."; + process_notify_event_stdout(process); + tt_ptr_op(NULL, OP_EQ, stdout_read_buffer); + + stderr_read_buffer = " the end\nAnother partial string goes here ..."; + process_notify_event_stderr(process); + tt_ptr_op(NULL, OP_EQ, stderr_read_buffer); + + /* Some data should be ready. */ + tt_int_op(1, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(1, OP_EQ, smartlist_len(process_data->stderr_data)); + + stdout_read_buffer = " the end\nFoo bar baz\n"; + process_notify_event_stdout(process); + tt_ptr_op(NULL, OP_EQ, stdout_read_buffer); + + stderr_read_buffer = " the end\nFoo bar baz\n"; + process_notify_event_stderr(process); + tt_ptr_op(NULL, OP_EQ, stderr_read_buffer); + + /* Some data should be ready. */ + tt_int_op(3, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(3, OP_EQ, smartlist_len(process_data->stderr_data)); + + /* Check if the data is correct. */ + tt_str_op(smartlist_get(process_data->stdout_data, 0), OP_EQ, + "Hello stdout this is a partial line ... the end"); + tt_str_op(smartlist_get(process_data->stdout_data, 1), OP_EQ, + "Another partial string goes here ... the end"); + tt_str_op(smartlist_get(process_data->stdout_data, 2), OP_EQ, + "Foo bar baz"); + + tt_str_op(smartlist_get(process_data->stderr_data, 0), OP_EQ, + "Hello stderr this is a partial line ... the end"); + tt_str_op(smartlist_get(process_data->stderr_data, 1), OP_EQ, + "Another partial string goes here ... the end"); + tt_str_op(smartlist_get(process_data->stderr_data, 2), OP_EQ, + "Foo bar baz"); + + done: + process_data_free(process_data); + process_free(process); + + UNMOCK(process_read_stdout); + UNMOCK(process_read_stderr); +} + +static void +test_raw_protocol_simple(void *arg) +{ + (void)arg; + + process_data_t *process_data = process_data_new(); + + process_t *process = process_new(""); + process_set_data(process, process_data); + process_set_protocol(process, PROCESS_PROTOCOL_RAW); + + process_set_stdout_read_callback(process, process_stdout_callback); + process_set_stderr_read_callback(process, process_stderr_callback); + + MOCK(process_read_stdout, process_mocked_read_stdout); + MOCK(process_read_stderr, process_mocked_read_stderr); + + /* Make sure we are running with the raw protocol. */ + tt_int_op(PROCESS_PROTOCOL_RAW, OP_EQ, process_get_protocol(process)); + + tt_int_op(0, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(0, OP_EQ, smartlist_len(process_data->stderr_data)); + + stdout_read_buffer = "Hello stdout\n"; + process_notify_event_stdout(process); + tt_ptr_op(NULL, OP_EQ, stdout_read_buffer); + + stderr_read_buffer = "Hello stderr\n"; + process_notify_event_stderr(process); + tt_ptr_op(NULL, OP_EQ, stderr_read_buffer); + + /* Data should be ready. */ + tt_int_op(1, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(1, OP_EQ, smartlist_len(process_data->stderr_data)); + + stdout_read_buffer = "Hello, again, stdout\nThis contains multiple lines"; + process_notify_event_stdout(process); + tt_ptr_op(NULL, OP_EQ, stdout_read_buffer); + + stderr_read_buffer = "Hello, again, stderr\nThis contains multiple lines"; + process_notify_event_stderr(process); + tt_ptr_op(NULL, OP_EQ, stderr_read_buffer); + + /* Data should be ready. */ + tt_int_op(2, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(2, OP_EQ, smartlist_len(process_data->stderr_data)); + + /* Check if the data is correct. */ + tt_str_op(smartlist_get(process_data->stdout_data, 0), OP_EQ, + "Hello stdout\n"); + tt_str_op(smartlist_get(process_data->stdout_data, 1), OP_EQ, + "Hello, again, stdout\nThis contains multiple lines"); + + tt_str_op(smartlist_get(process_data->stderr_data, 0), OP_EQ, + "Hello stderr\n"); + tt_str_op(smartlist_get(process_data->stderr_data, 1), OP_EQ, + "Hello, again, stderr\nThis contains multiple lines"); + + done: + process_data_free(process_data); + process_free(process); + + UNMOCK(process_read_stdout); + UNMOCK(process_read_stderr); +} + +static void +test_write_simple(void *arg) +{ + (void)arg; + + process_data_t *process_data = process_data_new(); + + process_t *process = process_new(""); + process_set_data(process, process_data); + + MOCK(process_write_stdin, process_mocked_write_stdin); + + process_write(process, (uint8_t *)"Hello world\n", 12); + process_notify_event_stdin(process); + tt_int_op(1, OP_EQ, smartlist_len(process_data->stdin_data)); + + process_printf(process, "Hello %s !\n", "moon"); + process_notify_event_stdin(process); + tt_int_op(2, OP_EQ, smartlist_len(process_data->stdin_data)); + + done: + process_data_free(process_data); + process_free(process); + + UNMOCK(process_write_stdin); +} + +static void +test_exit_simple(void *arg) +{ + (void)arg; + + process_data_t *process_data = process_data_new(); + + process_t *process = process_new(""); + process_set_data(process, process_data); + process_set_exit_callback(process, process_exit_callback); + + /* Our default is 0. */ + tt_u64_op(0, OP_EQ, process_data->exit_code); + + /* Fake that we are a running process. */ + process_set_status(process, PROCESS_STATUS_RUNNING); + tt_int_op(process_get_status(process), OP_EQ, PROCESS_STATUS_RUNNING); + + /* Fake an exit. */ + process_notify_event_exit(process, 1337); + + /* Check if our state changed and if our callback fired. */ + tt_int_op(process_get_status(process), OP_EQ, PROCESS_STATUS_NOT_RUNNING); + tt_u64_op(1337, OP_EQ, process_data->exit_code); + + done: + process_set_data(process, process_data); + process_data_free(process_data); + process_free(process); +} + +static void +test_argv_simple(void *arg) +{ + (void)arg; + + process_t *process = process_new("/bin/cat"); + char **argv = NULL; + + /* Setup some arguments. */ + process_append_argument(process, "foo"); + process_append_argument(process, "bar"); + process_append_argument(process, "baz"); + + /* Check the number of elements. */ + tt_int_op(3, OP_EQ, + smartlist_len(process_get_arguments(process))); + + /* Let's try to convert it into a Unix style char **argv. */ + argv = process_get_argv(process); + + /* Check our values. */ + tt_str_op(argv[0], OP_EQ, "/bin/cat"); + tt_str_op(argv[1], OP_EQ, "foo"); + tt_str_op(argv[2], OP_EQ, "bar"); + tt_str_op(argv[3], OP_EQ, "baz"); + tt_ptr_op(argv[4], OP_EQ, NULL); + + done: + tor_free(argv); + process_free(process); +} + +static void +test_unix(void *arg) +{ + (void)arg; +#ifndef _WIN32 + process_t *process = process_new(""); + + /* On Unix all processes should have a Unix process handle. */ + tt_ptr_op(NULL, OP_NE, process_get_unix_process(process)); + + done: + process_free(process); +#endif +} + +static void +test_win32(void *arg) +{ + (void)arg; +#ifdef _WIN32 + process_t *process = process_new(""); + char *joined_argv = NULL; + + /* On Win32 all processes should have a Win32 process handle. */ + tt_ptr_op(NULL, OP_NE, process_get_win32_process(process)); + + /* Based on some test cases from "Parsing C++ Command-Line Arguments" in + * MSDN but we don't exercise all quoting rules because tor_join_win_cmdline + * will try to only generate simple cases for the child process to parse; + * i.e. we never embed quoted strings in arguments. */ + + const char *argvs[][4] = { + {"a", "bb", "CCC", NULL}, // Normal + {NULL, NULL, NULL, NULL}, // Empty argument list + {"", NULL, NULL, NULL}, // Empty argument + {"\"a", "b\"b", "CCC\"", NULL}, // Quotes + {"a\tbc", "dd dd", "E", NULL}, // Whitespace + {"a\\\\\\b", "de fg", "H", NULL}, // Backslashes + {"a\\\"b", "\\c", "D\\", NULL}, // Backslashes before quote + {"a\\\\b c", "d", "E", NULL}, // Backslashes not before quote + { NULL } // Terminator + }; + + const char *cmdlines[] = { + "a bb CCC", + "", + "\"\"", + "\\\"a b\\\"b CCC\\\"", + "\"a\tbc\" \"dd dd\" E", + "a\\\\\\b \"de fg\" H", + "a\\\\\\\"b \\c D\\", + "\"a\\\\b c\" d E", + NULL // Terminator + }; + + int i; + + for (i=0; cmdlines[i]!=NULL; i++) { + log_info(LD_GENERAL, "Joining argvs[%d], expecting <%s>", i, cmdlines[i]); + joined_argv = tor_join_win_cmdline(argvs[i]); + tt_str_op(cmdlines[i],OP_EQ, joined_argv); + tor_free(joined_argv); + } + + done: + tor_free(joined_argv); + process_free(process); +#endif +} + +struct testcase_t process_tests[] = { + { "default_values", test_default_values, TT_FORK, NULL, NULL }, + { "environment", test_environment, TT_FORK, NULL, NULL }, + { "stringified_types", test_stringified_types, TT_FORK, NULL, NULL }, + { "line_protocol_simple", test_line_protocol_simple, TT_FORK, NULL, NULL }, + { "line_protocol_multi", test_line_protocol_multi, TT_FORK, NULL, NULL }, + { "line_protocol_partial", test_line_protocol_partial, TT_FORK, NULL, NULL }, + { "raw_protocol_simple", test_raw_protocol_simple, TT_FORK, NULL, NULL }, + { "write_simple", test_write_simple, TT_FORK, NULL, NULL }, + { "exit_simple", test_exit_simple, TT_FORK, NULL, NULL }, + { "argv_simple", test_argv_simple, TT_FORK, NULL, NULL }, + { "unix", test_unix, TT_FORK, NULL, NULL }, + { "win32", test_win32, TT_FORK, NULL, NULL }, + END_OF_TESTCASES +}; diff --git a/src/test/test_process_slow.c b/src/test/test_process_slow.c new file mode 100644 index 0000000000..d269a068ec --- /dev/null +++ b/src/test/test_process_slow.c @@ -0,0 +1,365 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file test_process_slow.c + * \brief Slow test cases for the Process API. + */ + +#define MAINLOOP_PRIVATE +#include "orconfig.h" +#include "core/or/or.h" +#include "core/mainloop/mainloop.h" +#include "lib/evloop/compat_libevent.h" +#include "lib/process/process.h" +#include "lib/process/waitpid.h" +#include "test/test.h" + +#ifndef BUILDDIR +#define BUILDDIR "." +#endif + +#ifdef _WIN32 +#define TEST_PROCESS "test-process.exe" +#else +#define TEST_PROCESS BUILDDIR "/src/test/test-process" +#endif /* defined(_WIN32) */ + +/** Timer that ticks once a second and stop the event loop after 5 ticks. */ +static periodic_timer_t *main_loop_timeout_timer; + +/** How many times have our timer ticked? */ +static int timer_tick_count; + +struct process_data_t { + smartlist_t *stdout_data; + smartlist_t *stderr_data; + smartlist_t *stdin_data; + process_exit_code_t exit_code; + bool did_exit; +}; + +typedef struct process_data_t process_data_t; + +static process_data_t * +process_data_new(void) +{ + process_data_t *process_data = tor_malloc_zero(sizeof(process_data_t)); + process_data->stdout_data = smartlist_new(); + process_data->stderr_data = smartlist_new(); + process_data->stdin_data = smartlist_new(); + return process_data; +} + +static void +process_data_free(process_data_t *process_data) +{ + if (process_data == NULL) + return; + + SMARTLIST_FOREACH(process_data->stdout_data, char *, x, tor_free(x)); + SMARTLIST_FOREACH(process_data->stderr_data, char *, x, tor_free(x)); + SMARTLIST_FOREACH(process_data->stdin_data, char *, x, tor_free(x)); + + smartlist_free(process_data->stdout_data); + smartlist_free(process_data->stderr_data); + smartlist_free(process_data->stdin_data); + tor_free(process_data); +} + +static void +process_stdout_callback(process_t *process, const char *data, size_t size) +{ + tt_ptr_op(process, OP_NE, NULL); + tt_ptr_op(data, OP_NE, NULL); + tt_int_op(strlen(data), OP_EQ, size); + + process_data_t *process_data = process_get_data(process); + smartlist_add(process_data->stdout_data, tor_strdup(data)); + + done: + return; +} + +static void +process_stderr_callback(process_t *process, const char *data, size_t size) +{ + tt_ptr_op(process, OP_NE, NULL); + tt_ptr_op(data, OP_NE, NULL); + tt_int_op(strlen(data), OP_EQ, size); + + process_data_t *process_data = process_get_data(process); + smartlist_add(process_data->stderr_data, tor_strdup(data)); + + done: + return; +} + +static bool +process_exit_callback(process_t *process, process_exit_code_t exit_code) +{ + process_status_t status; + + tt_ptr_op(process, OP_NE, NULL); + + process_data_t *process_data = process_get_data(process); + process_data->exit_code = exit_code; + process_data->did_exit = true; + + /* Check if our process is still running? */ + status = process_get_status(process); + tt_int_op(status, OP_EQ, PROCESS_STATUS_NOT_RUNNING); + + done: + /* Do not free up our process_t. */ + return false; +} + +#ifdef _WIN32 +static const char * +get_win32_test_binary_path(void) +{ + static char buffer[MAX_PATH]; + + /* Get the absolute path of our binary: \path\to\test-slow.exe. */ + GetModuleFileNameA(GetModuleHandle(0), buffer, sizeof(buffer)); + + /* Find our process name. */ + char *offset = strstr(buffer, "test-slow.exe"); + tt_ptr_op(offset, OP_NE, NULL); + + /* Change test-slow.exe to test-process.exe. */ + memcpy(offset, TEST_PROCESS, strlen(TEST_PROCESS)); + + return buffer; + done: + return NULL; +} +#endif + +static void +main_loop_timeout_cb(periodic_timer_t *timer, void *data) +{ + /* Sanity check. */ + tt_ptr_op(timer, OP_EQ, main_loop_timeout_timer); + tt_ptr_op(data, OP_NE, NULL); + + /* Our process data. */ + process_data_t *process_data = data; + + /* Our process did exit. */ + if (process_data->did_exit) + tor_shutdown_event_loop_and_exit(0); + + /* Have we been called 10 times we exit the main loop. */ + timer_tick_count++; + + tt_int_op(timer_tick_count, OP_LT, 10); + +#ifndef _WIN32 + /* Call waitpid callbacks. */ + notify_pending_waitpid_callbacks(); +#endif + + return; + done: + /* Exit with an error. */ + tor_shutdown_event_loop_and_exit(-1); +} + +static void +run_main_loop(process_data_t *process_data) +{ + int ret; + + /* Wake up after 1 seconds. */ + static const struct timeval interval = {1, 0}; + + timer_tick_count = 0; + main_loop_timeout_timer = periodic_timer_new(tor_libevent_get_base(), + &interval, + main_loop_timeout_cb, + process_data); + + /* Run our main loop. */ + ret = run_main_loop_until_done(); + + /* Clean up our main loop timeout timer. */ + tt_int_op(ret, OP_EQ, 0); + + done: + periodic_timer_free(main_loop_timeout_timer); +} + +static void +test_callbacks(void *arg) +{ + (void)arg; + const char *filename = NULL; + +#ifdef _WIN32 + filename = get_win32_test_binary_path(); +#else + filename = TEST_PROCESS; +#endif + + /* Process callback data. */ + process_data_t *process_data = process_data_new(); + + /* Setup our process. */ + process_t *process = process_new(filename); + process_set_data(process, process_data); + process_set_stdout_read_callback(process, process_stdout_callback); + process_set_stderr_read_callback(process, process_stderr_callback); + process_set_exit_callback(process, process_exit_callback); + + /* Set environment variable. */ + process_set_environment(process, "TOR_TEST_ENV", "Hello, from Tor!"); + + /* Add some arguments. */ + process_append_argument(process, "This is the first one"); + process_append_argument(process, "Second one"); + process_append_argument(process, "Third: Foo bar baz"); + + /* Run our process. */ + process_status_t status; + + status = process_exec(process); + tt_int_op(status, OP_EQ, PROCESS_STATUS_RUNNING); + + /* Write some lines to stdin. */ + process_printf(process, "Hi process!\r\n"); + process_printf(process, "Can you read more than one line?\n"); + process_printf(process, "Can you read partial ..."); + process_printf(process, " lines?\r\n"); + + /* Start our main loop. */ + run_main_loop(process_data); + + /* We returned. Let's see what our event loop said. */ + tt_int_op(smartlist_len(process_data->stdout_data), OP_EQ, 12); + tt_int_op(smartlist_len(process_data->stderr_data), OP_EQ, 3); + tt_assert(process_data->did_exit); + tt_u64_op(process_data->exit_code, OP_EQ, 0); + + /* Check stdout output. */ + char argv0_expected[256]; + tor_snprintf(argv0_expected, sizeof(argv0_expected), + "argv[0] = '%s'", filename); + + tt_str_op(smartlist_get(process_data->stdout_data, 0), OP_EQ, + argv0_expected); + tt_str_op(smartlist_get(process_data->stdout_data, 1), OP_EQ, + "argv[1] = 'This is the first one'"); + tt_str_op(smartlist_get(process_data->stdout_data, 2), OP_EQ, + "argv[2] = 'Second one'"); + tt_str_op(smartlist_get(process_data->stdout_data, 3), OP_EQ, + "argv[3] = 'Third: Foo bar baz'"); + tt_str_op(smartlist_get(process_data->stdout_data, 4), OP_EQ, + "Environment variable TOR_TEST_ENV = 'Hello, from Tor!'"); + tt_str_op(smartlist_get(process_data->stdout_data, 5), OP_EQ, + "Output on stdout"); + tt_str_op(smartlist_get(process_data->stdout_data, 6), OP_EQ, + "This is a new line"); + tt_str_op(smartlist_get(process_data->stdout_data, 7), OP_EQ, + "Partial line on stdout ...end of partial line on stdout"); + tt_str_op(smartlist_get(process_data->stdout_data, 8), OP_EQ, + "Read line from stdin: 'Hi process!'"); + tt_str_op(smartlist_get(process_data->stdout_data, 9), OP_EQ, + "Read line from stdin: 'Can you read more than one line?'"); + tt_str_op(smartlist_get(process_data->stdout_data, 10), OP_EQ, + "Read line from stdin: 'Can you read partial ... lines?'"); + tt_str_op(smartlist_get(process_data->stdout_data, 11), OP_EQ, + "We are done for here, thank you!"); + + /* Check stderr output. */ + tt_str_op(smartlist_get(process_data->stderr_data, 0), OP_EQ, + "Output on stderr"); + tt_str_op(smartlist_get(process_data->stderr_data, 1), OP_EQ, + "This is a new line"); + tt_str_op(smartlist_get(process_data->stderr_data, 2), OP_EQ, + "Partial line on stderr ...end of partial line on stderr"); + + done: + process_data_free(process_data); + process_free(process); +} + +static void +test_callbacks_terminate(void *arg) +{ + (void)arg; + const char *filename = NULL; + +#ifdef _WIN32 + filename = get_win32_test_binary_path(); +#else + filename = TEST_PROCESS; +#endif + + /* Process callback data. */ + process_data_t *process_data = process_data_new(); + + /* Setup our process. */ + process_t *process = process_new(filename); + process_set_data(process, process_data); + process_set_exit_callback(process, process_exit_callback); + + /* Run our process. */ + process_status_t status; + + status = process_exec(process); + tt_int_op(status, OP_EQ, PROCESS_STATUS_RUNNING); + + /* Zap our process. */ + bool success; + + success = process_terminate(process); + tt_assert(success); + + /* Start our main loop. */ + run_main_loop(process_data); + + /* Check if we did exit. */ + tt_assert(process_data->did_exit); + + done: + process_data_free(process_data); + process_free(process); +} + +static void +test_nonexistent_executable(void *arg) +{ + (void)arg; + + /* Process callback data. */ + process_data_t *process_data = process_data_new(); + + /* Setup our process. */ + process_t *process = process_new("binary-does-not-exist"); + process_set_data(process, process_data); + process_set_exit_callback(process, process_exit_callback); + + /* Run our process. */ + process_exec(process); + + /* Start our main loop. */ + run_main_loop(process_data); + + /* Ensure that the exit callback was actually called even though the binary + * did not exist. + */ + tt_assert(process_data->did_exit); + + done: + process_data_free(process_data); + process_free(process); +} + +struct testcase_t slow_process_tests[] = { + { "callbacks", test_callbacks, 0, NULL, NULL }, + { "callbacks_terminate", test_callbacks_terminate, 0, NULL, NULL }, + { "nonexistent_executable", test_nonexistent_executable, 0, NULL, NULL }, + END_OF_TESTCASES +}; diff --git a/src/test/test_proto_http.c b/src/test/test_proto_http.c index 08990c0b6a..f9339e8dd3 100644 --- a/src/test/test_proto_http.c +++ b/src/test/test_proto_http.c @@ -8,7 +8,7 @@ #include "core/or/or.h" #include "test/test.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/proto/proto_http.h" #include "test/log_test_helpers.h" diff --git a/src/test/test_proto_misc.c b/src/test/test_proto_misc.c index af9cf7eee2..18669a7772 100644 --- a/src/test/test_proto_misc.c +++ b/src/test/test_proto_misc.c @@ -8,7 +8,7 @@ #include "core/or/or.h" #include "test/test.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/or/connection_or.h" #include "feature/relay/ext_orport.h" #include "core/proto/proto_cell.h" diff --git a/src/test/test_pt.c b/src/test/test_pt.c index 1f9786648a..d2996f4cc3 100644 --- a/src/test/test_pt.c +++ b/src/test/test_pt.c @@ -8,7 +8,7 @@ #define UTIL_PRIVATE #define STATEFILE_PRIVATE #define CONTROL_PRIVATE -#define SUBPROCESS_PRIVATE +#define PROCESS_PRIVATE #include "core/or/or.h" #include "app/config/config.h" #include "app/config/confparse.h" @@ -17,12 +17,14 @@ #include "core/or/circuitbuild.h" #include "app/config/statefile.h" #include "test/test.h" -#include "lib/process/subprocess.h" #include "lib/encoding/confline.h" #include "lib/net/resolve.h" +#include "lib/process/process.h" #include "app/config/or_state_st.h" +#include "test/log_test_helpers.h" + static void reset_mp(managed_proxy_t *mp) { @@ -288,41 +290,35 @@ test_pt_get_extrainfo_string(void *arg) tor_free(s); } -#ifdef _WIN32 -#define STDIN_HANDLE HANDLE* -#else -#define STDIN_HANDLE int -#endif - -static smartlist_t * -tor_get_lines_from_handle_replacement(STDIN_HANDLE handle, - enum stream_status *stream_status_out) +static int +process_read_stdout_replacement(process_t *process, buf_t *buffer) { + (void)process; static int times_called = 0; - smartlist_t *retval_sl = smartlist_new(); - - (void) handle; - (void) stream_status_out; /* Generate some dummy CMETHOD lines the first 5 times. The 6th time, send 'CMETHODS DONE' to finish configuring the proxy. */ - if (times_called++ != 5) { - smartlist_add_asprintf(retval_sl, "SMETHOD mock%d 127.0.0.1:555%d", + times_called++; + + if (times_called <= 5) { + buf_add_printf(buffer, "SMETHOD mock%d 127.0.0.1:555%d\n", times_called, times_called); - } else { - smartlist_add_strdup(retval_sl, "SMETHODS DONE"); + } else if (times_called <= 6) { + buf_add_string(buffer, "SMETHODS DONE\n"); + } else if (times_called <= 7) { + buf_add_string(buffer, "LOG SEVERITY=error MESSAGE=\"Oh noes, something " + "bad happened. What do we do!?\"\n"); + buf_add_string(buffer, "LOG SEVERITY=warning MESSAGE=\"warning msg\"\n"); + buf_add_string(buffer, "LOG SEVERITY=notice MESSAGE=\"notice msg\"\n"); + buf_add_string(buffer, "LOG SEVERITY=info MESSAGE=\"info msg\"\n"); + buf_add_string(buffer, "LOG SEVERITY=debug MESSAGE=\"debug msg\"\n"); + } else if (times_called <= 8) { + buf_add_string(buffer, "STATUS TRANSPORT=a K_1=a K_2=b K_3=\"foo bar\"\n"); + buf_add_string(buffer, "STATUS TRANSPORT=b K_1=a K_2=b K_3=\"foo bar\"\n"); + buf_add_string(buffer, "STATUS TRANSPORT=c K_1=a K_2=b K_3=\"foo bar\"\n"); } - return retval_sl; -} - -/* NOP mock */ -static void -tor_process_handle_destroy_replacement(process_handle_t *process_handle, - int also_terminate_process) -{ - (void) process_handle; - (void) also_terminate_process; + return (int)buf_datalen(buffer); } static or_state_t *dummy_state = NULL; @@ -357,10 +353,7 @@ test_pt_configure_proxy(void *arg) dummy_state = tor_malloc_zero(sizeof(or_state_t)); - MOCK(tor_get_lines_from_handle, - tor_get_lines_from_handle_replacement); - MOCK(tor_process_handle_destroy, - tor_process_handle_destroy_replacement); + MOCK(process_read_stdout, process_read_stdout_replacement); MOCK(get_or_state, get_or_state_replacement); MOCK(queue_control_event_string, @@ -372,24 +365,34 @@ test_pt_configure_proxy(void *arg) mp->conf_state = PT_PROTO_ACCEPTING_METHODS; mp->transports = smartlist_new(); mp->transports_to_launch = smartlist_new(); - mp->process_handle = tor_malloc_zero(sizeof(process_handle_t)); mp->argv = tor_malloc_zero(sizeof(char*)*2); mp->argv[0] = tor_strdup("<testcase>"); mp->is_server = 1; + /* Configure the process. */ + mp->process = process_new(""); + process_set_stdout_read_callback(mp->process, managed_proxy_stdout_callback); + process_set_data(mp->process, mp); + /* Test the return value of configure_proxy() by calling it some times while it is uninitialized and then finally finalizing its configuration. */ for (i = 0 ; i < 5 ; i++) { + /* force a read from our mocked stdout reader. */ + process_notify_event_stdout(mp->process); + /* try to configure our proxy. */ retval = configure_proxy(mp); /* retval should be zero because proxy hasn't finished configuring yet */ tt_int_op(retval, OP_EQ, 0); /* check the number of registered transports */ - tt_assert(smartlist_len(mp->transports) == i+1); + tt_int_op(smartlist_len(mp->transports), OP_EQ, i+1); /* check that the mp is still waiting for transports */ tt_assert(mp->conf_state == PT_PROTO_ACCEPTING_METHODS); } + /* Get the SMETHOD DONE written to the process. */ + process_notify_event_stdout(mp->process); + /* this last configure_proxy() should finalize the proxy configuration. */ retval = configure_proxy(mp); /* retval should be 1 since the proxy finished configuring */ @@ -412,6 +415,49 @@ test_pt_configure_proxy(void *arg) tt_str_op(smartlist_get(controlevent_msgs, 4), OP_EQ, "650 TRANSPORT_LAUNCHED server mock5 127.0.0.1 5555\r\n"); + /* Get the log message out. */ + setup_full_capture_of_logs(LOG_ERR); + process_notify_event_stdout(mp->process); + expect_single_log_msg_containing("Oh noes, something bad happened"); + teardown_capture_of_logs(); + + tt_int_op(controlevent_n, OP_EQ, 10); + tt_int_op(controlevent_event, OP_EQ, EVENT_PT_LOG); + tt_int_op(smartlist_len(controlevent_msgs), OP_EQ, 10); + tt_str_op(smartlist_get(controlevent_msgs, 5), OP_EQ, + "650 PT_LOG PT=<testcase> SEVERITY=error " + "MESSAGE=\"Oh noes, " + "something bad happened. What do we do!?\"\r\n"); + tt_str_op(smartlist_get(controlevent_msgs, 6), OP_EQ, + "650 PT_LOG PT=<testcase> SEVERITY=warning " + "MESSAGE=\"warning msg\"\r\n"); + tt_str_op(smartlist_get(controlevent_msgs, 7), OP_EQ, + "650 PT_LOG PT=<testcase> SEVERITY=notice " + "MESSAGE=\"notice msg\"\r\n"); + tt_str_op(smartlist_get(controlevent_msgs, 8), OP_EQ, + "650 PT_LOG PT=<testcase> SEVERITY=info " + "MESSAGE=\"info msg\"\r\n"); + tt_str_op(smartlist_get(controlevent_msgs, 9), OP_EQ, + "650 PT_LOG PT=<testcase> SEVERITY=debug " + "MESSAGE=\"debug msg\"\r\n"); + + /* Get the STATUS messages out. */ + process_notify_event_stdout(mp->process); + + tt_int_op(controlevent_n, OP_EQ, 13); + tt_int_op(controlevent_event, OP_EQ, EVENT_PT_STATUS); + tt_int_op(smartlist_len(controlevent_msgs), OP_EQ, 13); + + tt_str_op(smartlist_get(controlevent_msgs, 10), OP_EQ, + "650 PT_STATUS " + "PT=<testcase> TRANSPORT=a K_1=a K_2=b K_3=\"foo bar\"\r\n"); + tt_str_op(smartlist_get(controlevent_msgs, 11), OP_EQ, + "650 PT_STATUS " + "PT=<testcase> TRANSPORT=b K_1=a K_2=b K_3=\"foo bar\"\r\n"); + tt_str_op(smartlist_get(controlevent_msgs, 12), OP_EQ, + "650 PT_STATUS " + "PT=<testcase> TRANSPORT=c K_1=a K_2=b K_3=\"foo bar\"\r\n"); + { /* check that the transport info were saved properly in the tor state */ config_line_t *transport_in_state = NULL; smartlist_t *transport_info_sl = smartlist_new(); @@ -434,9 +480,9 @@ test_pt_configure_proxy(void *arg) } done: + teardown_capture_of_logs(); or_state_free(dummy_state); - UNMOCK(tor_get_lines_from_handle); - UNMOCK(tor_process_handle_destroy); + UNMOCK(process_read_stdout); UNMOCK(get_or_state); UNMOCK(queue_control_event_string); if (controlevent_msgs) { @@ -449,7 +495,7 @@ test_pt_configure_proxy(void *arg) smartlist_free(mp->transports); } smartlist_free(mp->transports_to_launch); - tor_free(mp->process_handle); + process_free(mp->process); tor_free(mp->argv[0]); tor_free(mp->argv); tor_free(mp); diff --git a/src/test/test_rebind.py b/src/test/test_rebind.py index 30a587858f..c9b9200b2d 100644 --- a/src/test/test_rebind.py +++ b/src/test/test_rebind.py @@ -22,9 +22,10 @@ def skip(msg): def try_connecting_to_socksport(): socks_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - if socks_socket.connect_ex(('127.0.0.1', socks_port)): + e = socks_socket.connect_ex(('127.0.0.1', socks_port)) + if e: tor_process.terminate() - fail('Cannot connect to SOCKSPort') + fail('Cannot connect to SOCKSPort: error ' + os.strerror(e)) socks_socket.close() def wait_for_log(s): diff --git a/src/test/test_rng.c b/src/test/test_rng.c new file mode 100644 index 0000000000..c749de112a --- /dev/null +++ b/src/test/test_rng.c @@ -0,0 +1,59 @@ +/* Copyright (c) 2016-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/* + * Example usage: + * + * ./src/test/test-rng --emit | dieharder -g 200 -a + * + * Remember, dieharder can tell you that your RNG is completely broken, but if + * your RNG is not _completely_ broken, dieharder cannot tell you whether your + * RNG is actually secure. + */ + +#include "orconfig.h" + +#ifdef HAVE_UNISTD_H +#include <unistd.h> +#endif +#include <stdio.h> +#include <string.h> +#include <errno.h> + +#include "lib/crypt_ops/crypto_rand.h" + +int +main(int argc, char **argv) +{ + uint8_t buf[0x123]; + + if (argc != 2 || strcmp(argv[1], "--emit")) { + fprintf(stderr, "If you want me to fill stdout with a bunch of random " + "bytes, you need to say --emit.\n"); + return 1; + } + + if (crypto_seed_rng() < 0) { + fprintf(stderr, "Can't seed RNG.\n"); + return 1; + } + +#if 0 + while (1) { + crypto_rand(buf, sizeof(buf)); + if (write(1 /*stdout*/, buf, sizeof(buf)) != sizeof(buf)) { + fprintf(stderr, "write() failed: %s\n", strerror(errno)); + return 1; + } + } +#endif + + crypto_fast_rng_t *rng = crypto_fast_rng_new(); + while (1) { + crypto_fast_rng_getbytes(rng, buf, sizeof(buf)); + if (write(1 /*stdout*/, buf, sizeof(buf)) != sizeof(buf)) { + fprintf(stderr, "write() failed: %s\n", strerror(errno)); + return 1; + } + } +} diff --git a/src/test/test_router.c b/src/test/test_router.c index 601881a124..ea0ee3e84c 100644 --- a/src/test/test_router.c +++ b/src/test/test_router.c @@ -7,16 +7,25 @@ * \brief Unittests for code in router.c **/ +#define CONFIG_PRIVATE +#define ROUTER_PRIVATE + #include "core/or/or.h" #include "app/config/config.h" #include "core/mainloop/mainloop.h" #include "feature/hibernate/hibernate.h" +#include "feature/nodelist/networkstatus.h" +#include "feature/nodelist/networkstatus_st.h" +#include "feature/nodelist/node_st.h" +#include "feature/nodelist/nodelist.h" #include "feature/nodelist/routerinfo_st.h" #include "feature/nodelist/routerlist.h" +#include "feature/nodelist/routerstatus_st.h" #include "feature/relay/router.h" #include "feature/stats/rephist.h" #include "lib/crypt_ops/crypto_curve25519.h" #include "lib/crypt_ops/crypto_ed25519.h" +#include "lib/encoding/confline.h" /* Test suite stuff */ #include "test/test.h" @@ -231,11 +240,254 @@ test_router_check_descriptor_bandwidth_changed(void *arg) UNMOCK(we_are_hibernating); } +static networkstatus_t *mock_ns = NULL; +static networkstatus_t * +mock_networkstatus_get_live_consensus(time_t now) +{ + (void)now; + return mock_ns; +} + +static routerstatus_t *mock_rs = NULL; +static const routerstatus_t * +mock_networkstatus_vote_find_entry(networkstatus_t *ns, const char *digest) +{ + (void)ns; + (void)digest; + return mock_rs; +} + +static void +test_router_mark_if_too_old(void *arg) +{ + (void)arg; + time_t now = approx_time(); + MOCK(networkstatus_get_live_consensus, + mock_networkstatus_get_live_consensus); + MOCK(networkstatus_vote_find_entry, mock_networkstatus_vote_find_entry); + + routerstatus_t rs; + networkstatus_t ns; + memset(&rs, 0, sizeof(rs)); + memset(&ns, 0, sizeof(ns)); + mock_ns = &ns; + mock_ns->valid_after = now-3600; + mock_rs = &rs; + mock_rs->published_on = now - 10; + + // no reason to mark this time. + desc_clean_since = now-10; + desc_dirty_reason = NULL; + mark_my_descriptor_dirty_if_too_old(now); + tt_i64_op(desc_clean_since, OP_EQ, now-10); + + // Doesn't appear in consensus? Still don't mark it. + mock_ns = NULL; + mark_my_descriptor_dirty_if_too_old(now); + tt_i64_op(desc_clean_since, OP_EQ, now-10); + mock_ns = &ns; + + // No new descriptor in a long time? Mark it. + desc_clean_since = now - 3600 * 96; + mark_my_descriptor_dirty_if_too_old(now); + tt_i64_op(desc_clean_since, OP_EQ, 0); + tt_str_op(desc_dirty_reason, OP_EQ, "time for new descriptor"); + + // Version in consensus published a long time ago? We won't mark it + // if it's been clean for only a short time. + desc_clean_since = now - 10; + desc_dirty_reason = NULL; + mock_rs->published_on = now - 3600 * 96; + mark_my_descriptor_dirty_if_too_old(now); + tt_i64_op(desc_clean_since, OP_EQ, now - 10); + + // ... but if it's been clean a while, we mark. + desc_clean_since = now - 2 * 3600; + mark_my_descriptor_dirty_if_too_old(now); + tt_i64_op(desc_clean_since, OP_EQ, 0); + tt_str_op(desc_dirty_reason, OP_EQ, + "version listed in consensus is quite old"); + + // same deal if we're marked stale. + desc_clean_since = now - 2 * 3600; + desc_dirty_reason = NULL; + mock_rs->published_on = now - 10; + mock_rs->is_staledesc = 1; + mark_my_descriptor_dirty_if_too_old(now); + tt_i64_op(desc_clean_since, OP_EQ, 0); + tt_str_op(desc_dirty_reason, OP_EQ, + "listed as stale in consensus"); + + // same deal if we're absent from the consensus. + desc_clean_since = now - 2 * 3600; + desc_dirty_reason = NULL; + mock_rs = NULL; + mark_my_descriptor_dirty_if_too_old(now); + tt_i64_op(desc_clean_since, OP_EQ, 0); + tt_str_op(desc_dirty_reason, OP_EQ, + "not listed in consensus"); + + done: + UNMOCK(networkstatus_get_live_consensus); + UNMOCK(networkstatus_vote_find_entry); +} + +static node_t fake_node; +static const node_t * +mock_node_get_by_nickname(const char *name, unsigned flags) +{ + (void)flags; + if (!strcasecmp(name, "crumpet")) + return &fake_node; + else + return NULL; +} + +static void +test_router_get_my_family(void *arg) +{ + (void)arg; + or_options_t *options = options_new(); + smartlist_t *sl = NULL; + char *join = NULL; + // Overwrite the result of router_get_my_identity_digest(). This + // happens to be okay, but only for testing. + set_server_identity_key_digest_testing( + (const uint8_t*)"holeinthebottomofthe"); + + setup_capture_of_logs(LOG_WARN); + + // No family listed -- so there's no list. + sl = get_my_declared_family(options); + tt_ptr_op(sl, OP_EQ, NULL); + expect_no_log_entry(); + +#define CLEAR() do { \ + if (sl) { \ + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); \ + smartlist_free(sl); \ + } \ + tor_free(join); \ + mock_clean_saved_logs(); \ + } while (0) + + // Add a single nice friendly hex member. This should be enough + // to have our own ID added. + tt_ptr_op(options->MyFamily, OP_EQ, NULL); + config_line_append(&options->MyFamily, "MyFamily", + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"); + + sl = get_my_declared_family(options); + tt_ptr_op(sl, OP_NE, NULL); + tt_int_op(smartlist_len(sl), OP_EQ, 2); + join = smartlist_join_strings(sl, " ", 0, NULL); + tt_str_op(join, OP_EQ, + "$686F6C65696E746865626F74746F6D6F66746865 " + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"); + expect_no_log_entry(); + CLEAR(); + + // Add a hex member with a ~. The ~ part should get removed. + config_line_append(&options->MyFamily, "MyFamily", + "$0123456789abcdef0123456789abcdef01234567~Muffin"); + sl = get_my_declared_family(options); + tt_ptr_op(sl, OP_NE, NULL); + tt_int_op(smartlist_len(sl), OP_EQ, 3); + join = smartlist_join_strings(sl, " ", 0, NULL); + tt_str_op(join, OP_EQ, + "$0123456789ABCDEF0123456789ABCDEF01234567 " + "$686F6C65696E746865626F74746F6D6F66746865 " + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"); + expect_no_log_entry(); + CLEAR(); + + // Nickname lookup will fail, so a nickname will appear verbatim. + config_line_append(&options->MyFamily, "MyFamily", + "BAGEL"); + sl = get_my_declared_family(options); + tt_ptr_op(sl, OP_NE, NULL); + tt_int_op(smartlist_len(sl), OP_EQ, 4); + join = smartlist_join_strings(sl, " ", 0, NULL); + tt_str_op(join, OP_EQ, + "$0123456789ABCDEF0123456789ABCDEF01234567 " + "$686F6C65696E746865626F74746F6D6F66746865 " + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA " + "bagel"); + expect_single_log_msg_containing( + "There is a router named \"BAGEL\" in my declared family, but " + "I have no descriptor for it."); + CLEAR(); + + // A bogus digest should fail entirely. + config_line_append(&options->MyFamily, "MyFamily", + "$painauchocolat"); + sl = get_my_declared_family(options); + tt_ptr_op(sl, OP_NE, NULL); + tt_int_op(smartlist_len(sl), OP_EQ, 4); + join = smartlist_join_strings(sl, " ", 0, NULL); + tt_str_op(join, OP_EQ, + "$0123456789ABCDEF0123456789ABCDEF01234567 " + "$686F6C65696E746865626F74746F6D6F66746865 " + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA " + "bagel"); + // "BAGEL" is still there, but it won't make a warning, because we already + // warned about it. + expect_single_log_msg_containing( + "There is a router named \"$painauchocolat\" in my declared " + "family, but that isn't a legal digest or nickname. Skipping it."); + CLEAR(); + + // Let's introduce a node we can look up by nickname + memset(&fake_node, 0, sizeof(fake_node)); + memcpy(fake_node.identity, "whydoyouasknonononon", DIGEST_LEN); + MOCK(node_get_by_nickname, mock_node_get_by_nickname); + + config_line_append(&options->MyFamily, "MyFamily", + "CRUmpeT"); + sl = get_my_declared_family(options); + tt_ptr_op(sl, OP_NE, NULL); + tt_int_op(smartlist_len(sl), OP_EQ, 5); + join = smartlist_join_strings(sl, " ", 0, NULL); + tt_str_op(join, OP_EQ, + "$0123456789ABCDEF0123456789ABCDEF01234567 " + "$686F6C65696E746865626F74746F6D6F66746865 " + "$776879646F796F7561736B6E6F6E6F6E6F6E6F6E " + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA " + "bagel"); + // "BAGEL" is still there, but it won't make a warning, because we already + // warned about it. Some with "$painauchocolat". + expect_single_log_msg_containing( + "There is a router named \"CRUmpeT\" in my declared " + "family, but it wasn't listed by digest. Please consider saying " + "$776879646F796F7561736B6E6F6E6F6E6F6E6F6E instead, if that's " + "what you meant."); + CLEAR(); + UNMOCK(node_get_by_nickname); + + // Try a singleton list containing only us: It should give us NULL. + config_free_lines(options->MyFamily); + config_line_append(&options->MyFamily, "MyFamily", + "$686F6C65696E746865626F74746F6D6F66746865"); + sl = get_my_declared_family(options); + tt_ptr_op(sl, OP_EQ, NULL); + expect_no_log_entry(); + + done: + or_options_free(options); + teardown_capture_of_logs(); + CLEAR(); + UNMOCK(node_get_by_nickname); + +#undef CLEAR +} + #define ROUTER_TEST(name, flags) \ { #name, test_router_ ## name, flags, NULL, NULL } struct testcase_t router_tests[] = { ROUTER_TEST(check_descriptor_bandwidth_changed, TT_FORK), ROUTER_TEST(dump_router_to_string_no_bridge_distribution_method, TT_FORK), + ROUTER_TEST(mark_if_too_old, TT_FORK), + ROUTER_TEST(get_my_family, TT_FORK), END_OF_TESTCASES }; diff --git a/src/test/test_routerlist.c b/src/test/test_routerlist.c index 95c9176faa..84ec8cc462 100644 --- a/src/test/test_routerlist.c +++ b/src/test/test_routerlist.c @@ -46,7 +46,7 @@ #include "feature/nodelist/routerstatus_st.h" #include "lib/encoding/confline.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "test/test.h" #include "test/test_dir_common.h" @@ -265,7 +265,9 @@ test_router_pick_directory_server_impl(void *arg) /* Init SR subsystem. */ MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m); - mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); + mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, + strlen(AUTHORITY_CERT_1), + NULL); sr_init(0); UNMOCK(get_my_v3_authority_cert); @@ -275,7 +277,9 @@ test_router_pick_directory_server_impl(void *arg) construct_consensus(&consensus_text_md, now); tt_assert(consensus_text_md); - con_md = networkstatus_parse_vote_from_string(consensus_text_md, NULL, + con_md = networkstatus_parse_vote_from_string(consensus_text_md, + strlen(consensus_text_md), + NULL, NS_TYPE_CONSENSUS); tt_assert(con_md); tt_int_op(con_md->flavor,OP_EQ, FLAV_MICRODESC); @@ -301,7 +305,6 @@ test_router_pick_directory_server_impl(void *arg) tt_assert(!networkstatus_consensus_is_bootstrapping(con_md->valid_until + 24*60*60)); /* These times are outside the test validity period */ - tt_assert(networkstatus_consensus_is_bootstrapping(now)); tt_assert(networkstatus_consensus_is_bootstrapping(now + 2*24*60*60)); tt_assert(networkstatus_consensus_is_bootstrapping(now - 2*24*60*60)); @@ -475,7 +478,9 @@ test_directory_guard_fetch_with_no_dirinfo(void *arg) /* Initialize the SRV subsystem */ MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m); - mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); + mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, + strlen(AUTHORITY_CERT_1), + NULL); sr_init(0); UNMOCK(get_my_v3_authority_cert); @@ -648,7 +653,9 @@ test_skew_common(void *arg, time_t now, unsigned long *offset) /* Initialize the SRV subsystem */ MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m); - mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); + mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, + strlen(AUTHORITY_CERT_1), + NULL); sr_init(0); UNMOCK(get_my_v3_authority_cert); @@ -662,7 +669,8 @@ test_skew_common(void *arg, time_t now, unsigned long *offset) MOCK(clock_skew_warning, mock_clock_skew_warning); /* Caller will call teardown_capture_of_logs() */ setup_capture_of_logs(LOG_WARN); - retval = networkstatus_set_current_consensus(consensus, "microdesc", 0, + retval = networkstatus_set_current_consensus(consensus, strlen(consensus), + "microdesc", 0, NULL); done: diff --git a/src/test/test_shared_random.c b/src/test/test_shared_random.c index b4fe6eef64..5fa7e80d07 100644 --- a/src/test/test_shared_random.c +++ b/src/test/test_shared_random.c @@ -58,14 +58,17 @@ trusteddirserver_get_by_v3_auth_digest_m(const char *digest) } /* Setup a minimal dirauth environment by initializing the SR state and - * making sure the options are set to be an authority directory. */ + * making sure the options are set to be an authority directory. + * You must only call this function once per process. */ static void init_authority_state(void) { MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m); or_options_t *options = get_options_mutable(); - mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); + mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, + strlen(AUTHORITY_CERT_1), + NULL); tt_assert(mock_cert); options->AuthoritativeDir = 1; tt_int_op(load_ed_keys(options, time(NULL)), OP_GE, 0); @@ -310,6 +313,7 @@ test_get_start_time_of_current_run(void *arg) retval = parse_rfc1123_time("Mon, 19 Apr 2015 23:00:00 UTC", &mock_consensus.valid_after); + tt_int_op(retval, OP_EQ, 0); retval = parse_rfc1123_time("Mon, 20 Apr 2015 00:08:00 UTC", ¤t_time); @@ -424,7 +428,9 @@ test_sr_commit(void *arg) { /* Setup a minimal dirauth environment for this test */ or_options_t *options = get_options_mutable(); - auth_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); + auth_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, + strlen(AUTHORITY_CERT_1), + NULL); tt_assert(auth_cert); options->AuthoritativeDir = 1; @@ -835,7 +841,9 @@ test_sr_setup_commits(void) { /* Setup a minimal dirauth environment for this test */ or_options_t *options = get_options_mutable(); - auth_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); + auth_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, + strlen(AUTHORITY_CERT_1), + NULL); tt_assert(auth_cert); options->AuthoritativeDir = 1; @@ -1073,12 +1081,13 @@ test_sr_get_majority_srv_from_votes(void *arg) smartlist_free(votes); } +/* Test utils that don't depend on authority state */ static void -test_utils(void *arg) +test_utils_general(void *arg) { (void) arg; - /* Testing srv_dup(). */ + /* Testing sr_srv_dup(). */ { sr_srv_t *srv = NULL, *dup_srv = NULL; const char *srv_value = @@ -1086,7 +1095,7 @@ test_utils(void *arg) srv = tor_malloc_zero(sizeof(*srv)); srv->num_reveals = 42; memcpy(srv->value, srv_value, sizeof(srv->value)); - dup_srv = srv_dup(srv); + dup_srv = sr_srv_dup(srv); tt_assert(dup_srv); tt_u64_op(dup_srv->num_reveals, OP_EQ, srv->num_reveals); tt_mem_op(dup_srv->value, OP_EQ, srv->value, sizeof(srv->value)); @@ -1137,9 +1146,19 @@ test_utils(void *arg) tt_str_op(get_phase_str(SR_PHASE_COMMIT), OP_EQ, "commit"); } + done: + return; +} + +/* Test utils that depend on authority state */ +static void +test_utils_auth(void *arg) +{ + (void)arg; + init_authority_state(); + /* Testing phase transition */ { - init_authority_state(); set_sr_phase(SR_PHASE_COMMIT); tt_int_op(is_phase_transition(SR_PHASE_REVEAL), OP_EQ, 1); tt_int_op(is_phase_transition(SR_PHASE_COMMIT), OP_EQ, 0); @@ -1150,8 +1169,193 @@ test_utils(void *arg) tt_int_op(is_phase_transition(42), OP_EQ, 1); } + /* Testing get, set, delete, clean SRVs */ + + { + /* Just set the previous SRV */ + test_sr_setup_srv(0); + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL); + state_del_previous_srv(); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL); + } + + { + /* Delete the SRVs one at a time */ + test_sr_setup_srv(1); + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + state_del_current_srv(); + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL); + state_del_previous_srv(); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL); + + /* And in the opposite order */ + test_sr_setup_srv(1); + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + state_del_previous_srv(); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + state_del_current_srv(); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL); + + /* And both at once */ + test_sr_setup_srv(1); + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + sr_state_clean_srvs(); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL); + + /* And do the gets and sets multiple times */ + test_sr_setup_srv(1); + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + state_del_previous_srv(); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + state_del_previous_srv(); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + sr_state_clean_srvs(); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL); + state_del_current_srv(); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL); + sr_state_clean_srvs(); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL); + state_del_current_srv(); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL); + } + + { + /* Now set the SRVs to NULL instead */ + test_sr_setup_srv(1); + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + sr_state_set_current_srv(NULL); + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL); + sr_state_set_previous_srv(NULL); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL); + + /* And in the opposite order */ + test_sr_setup_srv(1); + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + sr_state_set_previous_srv(NULL); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + sr_state_set_current_srv(NULL); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL); + + /* And both at once */ + test_sr_setup_srv(1); + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + sr_state_clean_srvs(); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL); + + /* And do the gets and sets multiple times */ + test_sr_setup_srv(1); + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + sr_state_set_previous_srv(NULL); + sr_state_set_previous_srv(NULL); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + sr_state_set_current_srv(NULL); + sr_state_set_previous_srv(NULL); + sr_state_set_current_srv(NULL); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL); + } + + { + /* Now copy the values across */ + test_sr_setup_srv(1); + /* Check that the pointers are non-NULL, and different from each other */ + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, + sr_state_get_current_srv()); + /* Check that the content is different */ + tt_mem_op(sr_state_get_previous_srv(), OP_NE, + sr_state_get_current_srv(), sizeof(sr_srv_t)); + /* Set the current to the previous: the protocol goes the other way */ + sr_state_set_current_srv(sr_srv_dup(sr_state_get_previous_srv())); + /* Check that the pointers are non-NULL, and different from each other */ + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, + sr_state_get_current_srv()); + /* Check that the content is the same */ + tt_mem_op(sr_state_get_previous_srv(), OP_EQ, + sr_state_get_current_srv(), sizeof(sr_srv_t)); + } + + { + /* Now copy a value onto itself */ + test_sr_setup_srv(1); + /* Check that the pointers are non-NULL, and different from each other */ + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, + sr_state_get_current_srv()); + /* Take a copy of the old value */ + sr_srv_t old_current_srv; + memcpy(&old_current_srv, sr_state_get_current_srv(), sizeof(sr_srv_t)); + /* Check that the content is different */ + tt_mem_op(sr_state_get_previous_srv(), OP_NE, + sr_state_get_current_srv(), sizeof(sr_srv_t)); + /* Set the current to the current: the protocol never replaces an SRV with + * the same value */ + sr_state_set_current_srv(sr_srv_dup(sr_state_get_current_srv())); + /* Check that the pointers are non-NULL, and different from each other */ + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, + sr_state_get_current_srv()); + /* Check that the content is different between current and previous */ + tt_mem_op(sr_state_get_previous_srv(), OP_NE, + sr_state_get_current_srv(), sizeof(sr_srv_t)); + /* Check that the content is the same as the old content */ + tt_mem_op(&old_current_srv, OP_EQ, + sr_state_get_current_srv(), sizeof(sr_srv_t)); + } + + /* I don't think we can say "expect a BUG()" in our tests. */ +#if 0 + { + /* Now copy a value onto itself without sr_srv_dup(). + * This should fail with a BUG() warning. */ + test_sr_setup_srv(1); + sr_state_set_current_srv(sr_state_get_current_srv()); + sr_state_set_previous_srv(sr_state_get_previous_srv()); + } +#endif + done: - return; + sr_state_free_all(); } static void @@ -1159,6 +1363,7 @@ test_state_transition(void *arg) { sr_state_t *state = NULL; time_t now = time(NULL); + sr_srv_t *cur = NULL; (void) arg; @@ -1197,44 +1402,47 @@ test_state_transition(void *arg) /* Test SRV rotation in our state. */ { - const sr_srv_t *cur, *prev; test_sr_setup_srv(1); - cur = sr_state_get_current_srv(); + tt_assert(sr_state_get_current_srv()); + /* Take a copy of the data, because the state owns the pointer */ + cur = sr_srv_dup(sr_state_get_current_srv()); tt_assert(cur); - /* After, current srv should be the previous and then set to NULL. */ + /* After, the previous SRV should be the same as the old current SRV, and + * the current SRV should be set to NULL */ state_rotate_srv(); - prev = sr_state_get_previous_srv(); - tt_assert(prev == cur); + tt_mem_op(sr_state_get_previous_srv(), OP_EQ, cur, sizeof(sr_srv_t)); tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL); sr_state_clean_srvs(); + tor_free(cur); } /* New protocol run. */ { - const sr_srv_t *cur; /* Setup some new SRVs so we can confirm that a new protocol run * actually makes them rotate and compute new ones. */ test_sr_setup_srv(1); - cur = sr_state_get_current_srv(); - tt_assert(cur); + tt_assert(sr_state_get_current_srv()); + /* Take a copy of the data, because the state owns the pointer */ + cur = sr_srv_dup(sr_state_get_current_srv()); set_sr_phase(SR_PHASE_REVEAL); MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m); new_protocol_run(now); UNMOCK(get_my_v3_authority_cert); /* Rotation happened. */ - tt_assert(sr_state_get_previous_srv() == cur); + tt_mem_op(sr_state_get_previous_srv(), OP_EQ, cur, sizeof(sr_srv_t)); /* We are going into COMMIT phase so we had to rotate our SRVs. Usually * our current SRV would be NULL but a new protocol run should make us * compute a new SRV. */ tt_assert(sr_state_get_current_srv()); /* Also, make sure we did change the current. */ - tt_assert(sr_state_get_current_srv() != cur); + tt_mem_op(sr_state_get_current_srv(), OP_NE, cur, sizeof(sr_srv_t)); /* We should have our commitment alone. */ tt_int_op(digestmap_size(state->commits), OP_EQ, 1); tt_int_op(state->n_reveal_rounds, OP_EQ, 0); tt_int_op(state->n_commit_rounds, OP_EQ, 0); /* 46 here since we were at 45 just before. */ tt_u64_op(state->n_protocol_runs, OP_EQ, 46); + tor_free(cur); } /* Cleanup of SRVs. */ @@ -1245,6 +1453,7 @@ test_state_transition(void *arg) } done: + tor_free(cur); sr_state_free_all(); } @@ -1440,7 +1649,8 @@ struct testcase_t sr_tests[] = { { "sr_compute_srv", test_sr_compute_srv, TT_FORK, NULL, NULL }, { "sr_get_majority_srv_from_votes", test_sr_get_majority_srv_from_votes, TT_FORK, NULL, NULL }, - { "utils", test_utils, TT_FORK, NULL, NULL }, + { "utils_general", test_utils_general, TT_FORK, NULL, NULL }, + { "utils_auth", test_utils_auth, TT_FORK, NULL, NULL }, { "state_transition", test_state_transition, TT_FORK, NULL, NULL }, { "state_update", test_state_update, TT_FORK, NULL, NULL }, diff --git a/src/test/test_slow.c b/src/test/test_slow.c index bda67b2d92..c3e7edd408 100644 --- a/src/test/test_slow.c +++ b/src/test/test_slow.c @@ -20,7 +20,8 @@ struct testgroup_t testgroups[] = { { "slow/crypto/", slow_crypto_tests }, - { "slow/util/", slow_util_tests }, + { "slow/process/", slow_process_tests }, + { "slow/prob_distr/", slow_stochastic_prob_distr_tests }, END_OF_GROUPS }; diff --git a/src/test/test_socks.c b/src/test/test_socks.c index 783f4726ee..a4a768ce84 100644 --- a/src/test/test_socks.c +++ b/src/test/test_socks.c @@ -4,7 +4,7 @@ /* See LICENSE for licensing information */ #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "app/config/config.h" #include "core/mainloop/connection.h" #include "core/proto/proto_socks.h" diff --git a/src/test/test_util.c b/src/test/test_util.c index 3a0eb15157..f1ffae7af8 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -10,18 +10,20 @@ #define UTIL_PRIVATE #define UTIL_MALLOC_PRIVATE #define SOCKET_PRIVATE -#define SUBPROCESS_PRIVATE +#define PROCESS_WIN32_PRIVATE #include "lib/testsupport/testsupport.h" #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "app/config/config.h" #include "feature/control/control.h" #include "feature/client/transports.h" #include "lib/crypt_ops/crypto_format.h" #include "lib/crypt_ops/crypto_rand.h" +#include "lib/defs/time.h" #include "test/test.h" #include "lib/memarea/memarea.h" #include "lib/process/waitpid.h" +#include "lib/process/process_win32.h" #include "test/log_test_helpers.h" #include "lib/compress/compress.h" #include "lib/compress/compress_zstd.h" @@ -30,7 +32,6 @@ #include "lib/fs/winlib.h" #include "lib/process/env.h" #include "lib/process/pidfile.h" -#include "lib/process/subprocess.h" #include "lib/intmath/weakrng.h" #include "lib/thread/numcpus.h" #include "lib/math/fp.h" @@ -39,6 +40,7 @@ #include "lib/time/tvdiff.h" #include "lib/encoding/confline.h" #include "lib/net/socketpair.h" +#include "lib/malloc/map_anon.h" #ifdef HAVE_PWD_H #include <pwd.h> @@ -58,6 +60,12 @@ #ifdef HAVE_UNISTD_H #include <unistd.h> #endif +#ifdef HAVE_SYS_MMAN_H +#include <sys/mman.h> +#endif +#ifdef HAVE_SYS_WAIT_H +#include <sys/wait.h> +#endif #ifdef _WIN32 #include <tchar.h> @@ -69,6 +77,28 @@ #define INFINITY_DBL ((double)INFINITY) #define NAN_DBL ((double)NAN) +/** Test the tor_isinf() wrapper */ +static void +test_tor_isinf(void *arg) +{ + (void) arg; + + tt_assert(tor_isinf(INFINITY_DBL)); + + tt_assert(!tor_isinf(NAN_DBL)); + tt_assert(!tor_isinf(DBL_EPSILON)); + tt_assert(!tor_isinf(DBL_MAX)); + tt_assert(!tor_isinf(DBL_MIN)); + + tt_assert(!tor_isinf(0.0)); + tt_assert(!tor_isinf(0.1)); + tt_assert(!tor_isinf(3)); + tt_assert(!tor_isinf(3.14)); + + done: + ; +} + /* XXXX this is a minimal wrapper to make the unit tests compile with the * changed tor_timegm interface. */ static time_t @@ -404,7 +434,6 @@ test_util_time(void *arg) /* Assume tv_usec is an unsigned integer until proven otherwise */ #define TV_USEC_MAX UINT_MAX -#define TOR_USEC_PER_SEC 1000000 /* Overflows in the result type */ @@ -2179,15 +2208,6 @@ test_util_strmisc(void *arg) tt_int_op(strcmp_opt(NULL, "foo"), OP_LT, 0); tt_int_op(strcmp_opt("foo", NULL), OP_GT, 0); - /* Test strcmp_len */ - tt_int_op(strcmp_len("foo", "bar", 3), OP_GT, 0); - tt_int_op(strcmp_len("foo", "bar", 2), OP_LT, 0); - tt_int_op(strcmp_len("foo2", "foo1", 4), OP_GT, 0); - tt_int_op(strcmp_len("foo2", "foo1", 3), OP_LT, 0); /* Really stop at len */ - tt_int_op(strcmp_len("foo2", "foo", 3), OP_EQ, 0); /* Really stop at len */ - tt_int_op(strcmp_len("blah", "", 4), OP_GT, 0); - tt_int_op(strcmp_len("blah", "", 0), OP_EQ, 0); - done: tor_free(cp_tmp); } @@ -4050,6 +4070,13 @@ test_util_string_is_utf8(void *ptr) tt_int_op(1, OP_EQ, string_is_utf8("ascii\x7f\n", 7)); tt_int_op(1, OP_EQ, string_is_utf8("Risqu\u00e9=1", 9)); + /* Test the utf8_no_bom function */ + tt_int_op(0, OP_EQ, string_is_utf8_no_bom("\uFEFF", 3)); + tt_int_op(0, OP_EQ, string_is_utf8_no_bom("\uFFFE", 3)); + tt_int_op(0, OP_EQ, string_is_utf8_no_bom("\uFEFFlove", 7)); + tt_int_op(1, OP_EQ, string_is_utf8_no_bom("loveandrespect", + strlen("loveandrespect"))); + // Validate exactly 'len' bytes. tt_int_op(0, OP_EQ, string_is_utf8("\0\x80", 2)); tt_int_op(0, OP_EQ, string_is_utf8("Risqu\u00e9=1", 6)); @@ -4320,204 +4347,6 @@ test_util_load_win_lib(void *ptr) } #endif /* defined(_WIN32) */ -#ifndef _WIN32 -static void -clear_hex_errno(char *hex_errno) -{ - memset(hex_errno, '\0', HEX_ERRNO_SIZE + 1); -} - -static void -test_util_exit_status(void *ptr) -{ - /* Leave an extra byte for a \0 so we can do string comparison */ - char hex_errno[HEX_ERRNO_SIZE + 1]; - int n; - - (void)ptr; - - clear_hex_errno(hex_errno); - tt_str_op("",OP_EQ, hex_errno); - - clear_hex_errno(hex_errno); - n = format_helper_exit_status(0, 0, hex_errno); - tt_str_op("0/0\n",OP_EQ, hex_errno); - tt_int_op(n,OP_EQ, strlen(hex_errno)); - -#if SIZEOF_INT == 4 - - clear_hex_errno(hex_errno); - n = format_helper_exit_status(0, 0x7FFFFFFF, hex_errno); - tt_str_op("0/7FFFFFFF\n",OP_EQ, hex_errno); - tt_int_op(n,OP_EQ, strlen(hex_errno)); - - clear_hex_errno(hex_errno); - n = format_helper_exit_status(0xFF, -0x80000000, hex_errno); - tt_str_op("FF/-80000000\n",OP_EQ, hex_errno); - tt_int_op(n,OP_EQ, strlen(hex_errno)); - tt_int_op(n,OP_EQ, HEX_ERRNO_SIZE); - -#elif SIZEOF_INT == 8 - - clear_hex_errno(hex_errno); - n = format_helper_exit_status(0, 0x7FFFFFFFFFFFFFFF, hex_errno); - tt_str_op("0/7FFFFFFFFFFFFFFF\n",OP_EQ, hex_errno); - tt_int_op(n,OP_EQ, strlen(hex_errno)); - - clear_hex_errno(hex_errno); - n = format_helper_exit_status(0xFF, -0x8000000000000000, hex_errno); - tt_str_op("FF/-8000000000000000\n",OP_EQ, hex_errno); - tt_int_op(n,OP_EQ, strlen(hex_errno)); - tt_int_op(n,OP_EQ, HEX_ERRNO_SIZE); - -#endif /* SIZEOF_INT == 4 || ... */ - - clear_hex_errno(hex_errno); - n = format_helper_exit_status(0x7F, 0, hex_errno); - tt_str_op("7F/0\n",OP_EQ, hex_errno); - tt_int_op(n,OP_EQ, strlen(hex_errno)); - - clear_hex_errno(hex_errno); - n = format_helper_exit_status(0x08, -0x242, hex_errno); - tt_str_op("8/-242\n",OP_EQ, hex_errno); - tt_int_op(n,OP_EQ, strlen(hex_errno)); - - clear_hex_errno(hex_errno); - tt_str_op("",OP_EQ, hex_errno); - - done: - ; -} -#endif /* !defined(_WIN32) */ - -#ifndef _WIN32 -static void -test_util_string_from_pipe(void *ptr) -{ - int test_pipe[2] = {-1, -1}; - int retval = 0; - enum stream_status status = IO_STREAM_TERM; - ssize_t retlen; - char buf[4] = { 0 }; - - (void)ptr; - - errno = 0; - - /* Set up a pipe to test on */ - retval = pipe(test_pipe); - tt_int_op(retval, OP_EQ, 0); - - /* Send in a string. */ - retlen = write(test_pipe[1], "ABC", 3); - tt_int_op(retlen, OP_EQ, 3); - - status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1); - tt_int_op(errno, OP_EQ, 0); - tt_int_op(status, OP_EQ, IO_STREAM_OKAY); - tt_str_op(buf, OP_EQ, "ABC"); - errno = 0; - - /* Send in a string that contains a nul. */ - retlen = write(test_pipe[1], "AB\0", 3); - tt_int_op(retlen, OP_EQ, 3); - - status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1); - tt_int_op(errno, OP_EQ, 0); - tt_int_op(status, OP_EQ, IO_STREAM_OKAY); - tt_str_op(buf, OP_EQ, "AB"); - errno = 0; - - /* Send in a string that contains a nul only. */ - retlen = write(test_pipe[1], "\0", 1); - tt_int_op(retlen, OP_EQ, 1); - - status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1); - tt_int_op(errno, OP_EQ, 0); - tt_int_op(status, OP_EQ, IO_STREAM_OKAY); - tt_str_op(buf, OP_EQ, ""); - errno = 0; - - /* Send in a string that contains a trailing newline. */ - retlen = write(test_pipe[1], "AB\n", 3); - tt_int_op(retlen, OP_EQ, 3); - - status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1); - tt_int_op(errno, OP_EQ, 0); - tt_int_op(status, OP_EQ, IO_STREAM_OKAY); - tt_str_op(buf, OP_EQ, "AB"); - errno = 0; - - /* Send in a string that contains a newline only. */ - retlen = write(test_pipe[1], "\n", 1); - tt_int_op(retlen, OP_EQ, 1); - - status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1); - tt_int_op(errno, OP_EQ, 0); - tt_int_op(status, OP_EQ, IO_STREAM_OKAY); - tt_str_op(buf, OP_EQ, ""); - errno = 0; - - /* Send in a string and check that we nul terminate return values. */ - retlen = write(test_pipe[1], "AAA", 3); - tt_int_op(retlen, OP_EQ, 3); - - status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1); - tt_int_op(errno, OP_EQ, 0); - tt_int_op(status, OP_EQ, IO_STREAM_OKAY); - tt_str_op(buf, OP_EQ, "AAA"); - tt_mem_op(buf, OP_EQ, "AAA\0", sizeof(buf)); - errno = 0; - - retlen = write(test_pipe[1], "B", 1); - tt_int_op(retlen, OP_EQ, 1); - - memset(buf, '\xff', sizeof(buf)); - status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1); - tt_int_op(errno, OP_EQ, 0); - tt_int_op(status, OP_EQ, IO_STREAM_OKAY); - tt_str_op(buf, OP_EQ, "B"); - tt_mem_op(buf, OP_EQ, "B\0\xff\xff", sizeof(buf)); - errno = 0; - - /* Send in multiple lines. */ - retlen = write(test_pipe[1], "A\nB", 3); - tt_int_op(retlen, OP_EQ, 3); - - status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1); - tt_int_op(errno, OP_EQ, 0); - tt_int_op(status, OP_EQ, IO_STREAM_OKAY); - tt_str_op(buf, OP_EQ, "A\nB"); - errno = 0; - - /* Send in a line and close */ - retlen = write(test_pipe[1], "AB", 2); - tt_int_op(retlen, OP_EQ, 2); - retval = close(test_pipe[1]); - tt_int_op(retval, OP_EQ, 0); - test_pipe[1] = -1; - - status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1); - tt_int_op(errno, OP_EQ, 0); - tt_int_op(status, OP_EQ, IO_STREAM_OKAY); - tt_str_op(buf, OP_EQ, "AB"); - errno = 0; - - /* Check for EOF */ - status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1); - tt_int_op(errno, OP_EQ, 0); - tt_int_op(status, OP_EQ, IO_STREAM_CLOSED); - errno = 0; - - done: - if (test_pipe[0] != -1) - close(test_pipe[0]); - if (test_pipe[1] != -1) - close(test_pipe[1]); -} - -#endif /* !defined(_WIN32) */ - /** * Test for format_hex_number_sigsafe() */ @@ -4612,57 +4441,6 @@ test_util_format_dec_number(void *ptr) return; } -/** - * Test that we can properly format a Windows command line - */ -static void -test_util_join_win_cmdline(void *ptr) -{ - /* Based on some test cases from "Parsing C++ Command-Line Arguments" in - * MSDN but we don't exercise all quoting rules because tor_join_win_cmdline - * will try to only generate simple cases for the child process to parse; - * i.e. we never embed quoted strings in arguments. */ - - const char *argvs[][4] = { - {"a", "bb", "CCC", NULL}, // Normal - {NULL, NULL, NULL, NULL}, // Empty argument list - {"", NULL, NULL, NULL}, // Empty argument - {"\"a", "b\"b", "CCC\"", NULL}, // Quotes - {"a\tbc", "dd dd", "E", NULL}, // Whitespace - {"a\\\\\\b", "de fg", "H", NULL}, // Backslashes - {"a\\\"b", "\\c", "D\\", NULL}, // Backslashes before quote - {"a\\\\b c", "d", "E", NULL}, // Backslashes not before quote - { NULL } // Terminator - }; - - const char *cmdlines[] = { - "a bb CCC", - "", - "\"\"", - "\\\"a b\\\"b CCC\\\"", - "\"a\tbc\" \"dd dd\" E", - "a\\\\\\b \"de fg\" H", - "a\\\\\\\"b \\c D\\", - "\"a\\\\b c\" d E", - NULL // Terminator - }; - - int i; - char *joined_argv = NULL; - - (void)ptr; - - for (i=0; cmdlines[i]!=NULL; i++) { - log_info(LD_GENERAL, "Joining argvs[%d], expecting <%s>", i, cmdlines[i]); - joined_argv = tor_join_win_cmdline(argvs[i]); - tt_str_op(cmdlines[i],OP_EQ, joined_argv); - tor_free(joined_argv); - } - - done: - tor_free(joined_argv); -} - #define MAX_SPLIT_LINE_COUNT 4 struct split_lines_test_t { const char *orig_line; // Line to be split (may contain \0's) @@ -4670,67 +4448,6 @@ struct split_lines_test_t { const char *split_line[MAX_SPLIT_LINE_COUNT]; // Split lines }; -/** - * Test that we properly split a buffer into lines - */ -static void -test_util_split_lines(void *ptr) -{ - /* Test cases. orig_line of last test case must be NULL. - * The last element of split_line[i] must be NULL. */ - struct split_lines_test_t tests[] = { - {"", 0, {NULL}}, - {"foo", 3, {"foo", NULL}}, - {"\n\rfoo\n\rbar\r\n", 12, {"foo", "bar", NULL}}, - {"fo o\r\nb\tar", 10, {"fo o", "b.ar", NULL}}, - {"\x0f""f\0o\0\n\x01""b\0r\0\r", 12, {".f.o.", ".b.r.", NULL}}, - {"line 1\r\nline 2", 14, {"line 1", "line 2", NULL}}, - {"line 1\r\n\r\nline 2", 16, {"line 1", "line 2", NULL}}, - {"line 1\r\n\r\r\r\nline 2", 18, {"line 1", "line 2", NULL}}, - {"line 1\r\n\n\n\n\rline 2", 18, {"line 1", "line 2", NULL}}, - {"line 1\r\n\r\t\r\nline 3", 18, {"line 1", ".", "line 3", NULL}}, - {"\n\t\r\t\nline 3", 11, {".", ".", "line 3", NULL}}, - {NULL, 0, { NULL }} - }; - - int i, j; - char *orig_line=NULL; - smartlist_t *sl=NULL; - - (void)ptr; - - for (i=0; tests[i].orig_line; i++) { - sl = smartlist_new(); - /* Allocate space for string and trailing NULL */ - orig_line = tor_memdup(tests[i].orig_line, tests[i].orig_length + 1); - tor_split_lines(sl, orig_line, tests[i].orig_length); - - j = 0; - log_info(LD_GENERAL, "Splitting test %d of length %d", - i, tests[i].orig_length); - SMARTLIST_FOREACH_BEGIN(sl, const char *, line) { - /* Check we have not got too many lines */ - tt_int_op(MAX_SPLIT_LINE_COUNT, OP_GT, j); - /* Check that there actually should be a line here */ - tt_ptr_op(tests[i].split_line[j], OP_NE, NULL); - log_info(LD_GENERAL, "Line %d of test %d, should be <%s>", - j, i, tests[i].split_line[j]); - /* Check that the line is as expected */ - tt_str_op(line,OP_EQ, tests[i].split_line[j]); - j++; - } SMARTLIST_FOREACH_END(line); - /* Check that we didn't miss some lines */ - tt_ptr_op(NULL,OP_EQ, tests[i].split_line[j]); - tor_free(orig_line); - smartlist_free(sl); - sl = NULL; - } - - done: - tor_free(orig_line); - smartlist_free(sl); -} - static void test_util_di_ops(void *arg) { @@ -5834,6 +5551,18 @@ test_util_ipv4_validation(void *arg) } static void +test_util_ipv6_validation(void *arg) +{ + (void)arg; + + tt_assert(string_is_valid_ipv6_address("2a00:1450:401b:800::200e")); + tt_assert(!string_is_valid_ipv6_address("11:22::33:44:")); + + done: + return; +} + +static void test_util_writepid(void *arg) { (void) arg; @@ -6397,6 +6126,104 @@ test_util_log_mallinfo(void *arg) tor_free(mem); } +static void +test_util_map_anon(void *arg) +{ + (void)arg; + char *ptr = NULL; + size_t sz = 16384; + + /* Basic checks. */ + ptr = tor_mmap_anonymous(sz, 0); + tt_ptr_op(ptr, OP_NE, 0); + ptr[sz-1] = 3; + tt_int_op(ptr[0], OP_EQ, 0); + tt_int_op(ptr[sz-2], OP_EQ, 0); + tt_int_op(ptr[sz-1], OP_EQ, 3); + + /* Try again, with a private (non-swappable) mapping. */ + tor_munmap_anonymous(ptr, sz); + ptr = tor_mmap_anonymous(sz, ANONMAP_PRIVATE); + tt_ptr_op(ptr, OP_NE, 0); + ptr[sz-1] = 10; + tt_int_op(ptr[0], OP_EQ, 0); + tt_int_op(ptr[sz/2], OP_EQ, 0); + tt_int_op(ptr[sz-1], OP_EQ, 10); + + /* Now let's test a drop-on-fork mapping. */ + tor_munmap_anonymous(ptr, sz); + ptr = tor_mmap_anonymous(sz, ANONMAP_NOINHERIT); + tt_ptr_op(ptr, OP_NE, 0); + ptr[sz-1] = 10; + tt_int_op(ptr[0], OP_EQ, 0); + tt_int_op(ptr[sz/2], OP_EQ, 0); + tt_int_op(ptr[sz-1], OP_EQ, 10); + + done: + tor_munmap_anonymous(ptr, sz); +} + +static void +test_util_map_anon_nofork(void *arg) +{ + (void)arg; +#if !defined(HAVE_MADVISE) && !defined(HAVE_MINHERIT) + /* The operating system doesn't support this. */ + tt_skip(); + done: + ; +#else + /* We have the right OS support. We're going to try marking the buffer as + * either zero-on-fork or as drop-on-fork, whichever is supported. Then we + * will fork and send a byte back to the parent process. This will either + * crash, or send zero. */ + + char *ptr = NULL; + size_t sz = 16384; + int pipefd[2] = {-1, -1}; + + tor_munmap_anonymous(ptr, sz); + ptr = tor_mmap_anonymous(sz, ANONMAP_NOINHERIT); + tt_ptr_op(ptr, OP_NE, 0); + memset(ptr, 0xd0, sz); + + tt_int_op(0, OP_EQ, pipe(pipefd)); + pid_t child = fork(); + if (child == 0) { + /* We're in the child. */ + close(pipefd[0]); + ssize_t r = write(pipefd[1], &ptr[sz-1], 1); /* This may crash. */ + close(pipefd[1]); + if (r < 0) + exit(1); + exit(0); + } + tt_int_op(child, OP_GT, 0); + /* In the parent. */ + close(pipefd[1]); + pipefd[1] = -1; + char buf[1]; + ssize_t r = read(pipefd[0], buf, 1); +#if defined(INHERIT_ZERO) || defined(MADV_WIPEONFORK) + tt_int_op((int)r, OP_EQ, 1); // child should send us a byte. + tt_int_op(buf[0], OP_EQ, 0); +#else + tt_int_op(r, OP_LE, 0); // child said nothing; it should have crashed. +#endif + int ws; + waitpid(child, &ws, 0); + + done: + tor_munmap_anonymous(ptr, sz); + if (pipefd[0] >= 0) { + close(pipefd[0]); + } + if (pipefd[1] >= 0) { + close(pipefd[1]); + } +#endif +} + #define UTIL_LEGACY(name) \ { #name, test_util_ ## name , 0, NULL, NULL } @@ -6490,12 +6317,8 @@ struct testcase_t util_tests[] = { UTIL_TEST(nowrap_math, 0), UTIL_TEST(num_cpus, 0), UTIL_TEST_WIN_ONLY(load_win_lib, 0), - UTIL_TEST_NO_WIN(exit_status, 0), - UTIL_TEST_NO_WIN(string_from_pipe, 0), UTIL_TEST(format_hex_number, 0), UTIL_TEST(format_dec_number, 0), - UTIL_TEST(join_win_cmdline, 0), - UTIL_TEST(split_lines, 0), UTIL_TEST(n_bits_set, 0), UTIL_TEST(eat_whitespace, 0), UTIL_TEST(sl_new_from_text_lines, 0), @@ -6512,6 +6335,7 @@ struct testcase_t util_tests[] = { UTIL_TEST(mathlog, 0), UTIL_TEST(fraction, 0), UTIL_TEST(weak_random, 0), + { "tor_isinf", test_tor_isinf, TT_FORK, NULL, NULL }, { "socket_ipv4", test_util_socket, TT_FORK, &passthrough_setup, (void*)"4" }, { "socket_ipv6", test_util_socket, TT_FORK, @@ -6524,6 +6348,7 @@ struct testcase_t util_tests[] = { UTIL_TEST(hostname_validation, 0), UTIL_TEST(dest_validation_edgecase, 0), UTIL_TEST(ipv4_validation, 0), + UTIL_TEST(ipv6_validation, 0), UTIL_TEST(writepid, 0), UTIL_TEST(get_avail_disk_space, 0), UTIL_TEST(touch_file, 0), @@ -6536,5 +6361,7 @@ struct testcase_t util_tests[] = { UTIL_TEST(htonll, 0), UTIL_TEST(get_unquoted_path, 0), UTIL_TEST(log_mallinfo, 0), + UTIL_TEST(map_anon, 0), + UTIL_TEST(map_anon_nofork, TT_SKIP /* See bug #29535 */), END_OF_TESTCASES }; diff --git a/src/test/test_util_format.c b/src/test/test_util_format.c index d344d0e95c..3a0b41faa5 100644 --- a/src/test/test_util_format.c +++ b/src/test/test_util_format.c @@ -392,10 +392,13 @@ test_util_format_encoded_size(void *arg) base64_encode(outbuf, sizeof(outbuf), (char *)inbuf, i, 0); tt_int_op(strlen(outbuf), OP_EQ, base64_encode_size(i, 0)); + tt_int_op(i, OP_LE, base64_decode_maxsize(strlen(outbuf))); + base64_encode(outbuf, sizeof(outbuf), (char *)inbuf, i, BASE64_ENCODE_MULTILINE); tt_int_op(strlen(outbuf), OP_EQ, base64_encode_size(i, BASE64_ENCODE_MULTILINE)); + tt_int_op(i, OP_LE, base64_decode_maxsize(strlen(outbuf))); } done: @@ -417,4 +420,3 @@ struct testcase_t util_format_tests[] = { { "encoded_size", test_util_format_encoded_size, 0, NULL, NULL }, END_OF_TESTCASES }; - diff --git a/src/test/test_util_slow.c b/src/test/test_util_slow.c deleted file mode 100644 index 29e30eaa11..0000000000 --- a/src/test/test_util_slow.c +++ /dev/null @@ -1,396 +0,0 @@ -/* Copyright (c) 2001-2004, Roger Dingledine. - * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ -/* See LICENSE for licensing information */ - -#include "orconfig.h" -#define UTIL_PRIVATE -#define SUBPROCESS_PRIVATE -#include "lib/crypt_ops/crypto_cipher.h" -#include "lib/log/log.h" -#include "lib/process/subprocess.h" -#include "lib/process/waitpid.h" -#include "lib/string/printf.h" -#include "lib/time/compat_time.h" -#include "test/test.h" - -#include <errno.h> -#include <string.h> - -#ifndef BUILDDIR -#define BUILDDIR "." -#endif - -#ifdef _WIN32 -#define notify_pending_waitpid_callbacks() STMT_NIL -#define TEST_CHILD "test-child.exe" -#define EOL "\r\n" -#else -#define TEST_CHILD (BUILDDIR "/src/test/test-child") -#define EOL "\n" -#endif /* defined(_WIN32) */ - -#ifdef _WIN32 -/* I've assumed Windows doesn't have the gap between fork and exec - * that causes the race condition on unix-like platforms */ -#define MATCH_PROCESS_STATUS(s1,s2) ((s1) == (s2)) - -#else /* !(defined(_WIN32)) */ -/* work around a race condition of the timing of SIGCHLD handler updates - * to the process_handle's fields, and checks of those fields - * - * TODO: Once we can signal failure to exec, change PROCESS_STATUS_RUNNING to - * PROCESS_STATUS_ERROR (and similarly with *_OR_NOTRUNNING) */ -#define PROCESS_STATUS_RUNNING_OR_NOTRUNNING (PROCESS_STATUS_RUNNING+1) -#define IS_RUNNING_OR_NOTRUNNING(s) \ - ((s) == PROCESS_STATUS_RUNNING || (s) == PROCESS_STATUS_NOTRUNNING) -/* well, this is ugly */ -#define MATCH_PROCESS_STATUS(s1,s2) \ - ( (s1) == (s2) \ - ||((s1) == PROCESS_STATUS_RUNNING_OR_NOTRUNNING \ - && IS_RUNNING_OR_NOTRUNNING(s2)) \ - ||((s2) == PROCESS_STATUS_RUNNING_OR_NOTRUNNING \ - && IS_RUNNING_OR_NOTRUNNING(s1))) - -#endif /* defined(_WIN32) */ - -/** Helper function for testing tor_spawn_background */ -static void -run_util_spawn_background(const char *argv[], const char *expected_out, - const char *expected_err, int expected_exit, - int expected_status) -{ - int retval, exit_code; - ssize_t pos; - process_handle_t *process_handle=NULL; - char stdout_buf[100], stderr_buf[100]; - int status; - - /* Start the program */ -#ifdef _WIN32 - status = tor_spawn_background(NULL, argv, NULL, &process_handle); -#else - status = tor_spawn_background(argv[0], argv, NULL, &process_handle); -#endif - - notify_pending_waitpid_callbacks(); - - /* the race condition doesn't affect status, - * because status isn't updated by the SIGCHLD handler, - * but we still need to handle PROCESS_STATUS_RUNNING_OR_NOTRUNNING */ - tt_assert(MATCH_PROCESS_STATUS(expected_status, status)); - if (status == PROCESS_STATUS_ERROR) { - tt_ptr_op(process_handle, OP_EQ, NULL); - return; - } - - tt_ptr_op(process_handle, OP_NE, NULL); - - /* When a spawned process forks, fails, then exits very quickly, - * (this typically occurs when exec fails) - * there is a race condition between the SIGCHLD handler - * updating the process_handle's fields, and this test - * checking the process status in those fields. - * The SIGCHLD update can occur before or after the code below executes. - * This causes intermittent failures in spawn_background_fail(), - * typically when the machine is under load. - * We use PROCESS_STATUS_RUNNING_OR_NOTRUNNING to avoid this issue. */ - - /* the race condition affects the change in - * process_handle->status from RUNNING to NOTRUNNING */ - tt_assert(MATCH_PROCESS_STATUS(expected_status, process_handle->status)); - -#ifndef _WIN32 - notify_pending_waitpid_callbacks(); - /* the race condition affects the change in - * process_handle->waitpid_cb to NULL, - * so we skip the check if expected_status is ambiguous, - * that is, PROCESS_STATUS_RUNNING_OR_NOTRUNNING */ - tt_assert(process_handle->waitpid_cb != NULL - || expected_status == PROCESS_STATUS_RUNNING_OR_NOTRUNNING); -#endif /* !defined(_WIN32) */ - -#ifdef _WIN32 - tt_assert(process_handle->stdout_pipe != INVALID_HANDLE_VALUE); - tt_assert(process_handle->stderr_pipe != INVALID_HANDLE_VALUE); - tt_assert(process_handle->stdin_pipe != INVALID_HANDLE_VALUE); -#else - tt_assert(process_handle->stdout_pipe >= 0); - tt_assert(process_handle->stderr_pipe >= 0); - tt_assert(process_handle->stdin_pipe >= 0); -#endif /* defined(_WIN32) */ - - /* Check stdout */ - pos = tor_read_all_from_process_stdout(process_handle, stdout_buf, - sizeof(stdout_buf) - 1); - tt_assert(pos >= 0); - stdout_buf[pos] = '\0'; - tt_int_op(strlen(expected_out),OP_EQ, pos); - tt_str_op(expected_out,OP_EQ, stdout_buf); - - notify_pending_waitpid_callbacks(); - - /* Check it terminated correctly */ - retval = tor_get_exit_code(process_handle, 1, &exit_code); - tt_int_op(PROCESS_EXIT_EXITED,OP_EQ, retval); - tt_int_op(expected_exit,OP_EQ, exit_code); - // TODO: Make test-child exit with something other than 0 - -#ifndef _WIN32 - notify_pending_waitpid_callbacks(); - tt_ptr_op(process_handle->waitpid_cb, OP_EQ, NULL); -#endif - - /* Check stderr */ - pos = tor_read_all_from_process_stderr(process_handle, stderr_buf, - sizeof(stderr_buf) - 1); - tt_assert(pos >= 0); - stderr_buf[pos] = '\0'; - tt_str_op(expected_err,OP_EQ, stderr_buf); - tt_int_op(strlen(expected_err),OP_EQ, pos); - - notify_pending_waitpid_callbacks(); - - done: - if (process_handle) - tor_process_handle_destroy(process_handle, 1); -} - -/** Check that we can launch a process and read the output */ -static void -test_util_spawn_background_ok(void *ptr) -{ - const char *argv[] = {TEST_CHILD, "--test", NULL}; - const char *expected_out = "OUT"EOL "--test"EOL "SLEEPING"EOL "DONE" EOL; - const char *expected_err = "ERR"EOL; - - (void)ptr; - - run_util_spawn_background(argv, expected_out, expected_err, 0, - PROCESS_STATUS_RUNNING); -} - -/** Check that failing to find the executable works as expected */ -static void -test_util_spawn_background_fail(void *ptr) -{ - const char *argv[] = {BUILDDIR "/src/test/no-such-file", "--test", NULL}; - const char *expected_err = ""; - char expected_out[1024]; - char code[32]; -#ifdef _WIN32 - const int expected_status = PROCESS_STATUS_ERROR; -#else - /* TODO: Once we can signal failure to exec, set this to be - * PROCESS_STATUS_RUNNING_OR_ERROR */ - const int expected_status = PROCESS_STATUS_RUNNING_OR_NOTRUNNING; -#endif /* defined(_WIN32) */ - - memset(expected_out, 0xf0, sizeof(expected_out)); - memset(code, 0xf0, sizeof(code)); - - (void)ptr; - - tor_snprintf(code, sizeof(code), "%x/%x", - 9 /* CHILD_STATE_FAILEXEC */ , ENOENT); - tor_snprintf(expected_out, sizeof(expected_out), - "ERR: Failed to spawn background process - code %s\n", code); - - run_util_spawn_background(argv, expected_out, expected_err, 255, - expected_status); -} - -/** Test that reading from a handle returns a partial read rather than - * blocking */ -static void -test_util_spawn_background_partial_read_impl(int exit_early) -{ - const int expected_exit = 0; - const int expected_status = PROCESS_STATUS_RUNNING; - - int retval, exit_code; - ssize_t pos = -1; - process_handle_t *process_handle=NULL; - int status; - char stdout_buf[100], stderr_buf[100]; - - const char *argv[] = {TEST_CHILD, "--test", NULL}; - const char *expected_out[] = { "OUT" EOL "--test" EOL "SLEEPING" EOL, - "DONE" EOL, - NULL }; - const char *expected_err = "ERR" EOL; - -#ifndef _WIN32 - int eof = 0; -#endif - int expected_out_ctr; - - if (exit_early) { - argv[1] = "--hang"; - expected_out[0] = "OUT"EOL "--hang"EOL "SLEEPING" EOL; - } - - /* Start the program */ -#ifdef _WIN32 - status = tor_spawn_background(NULL, argv, NULL, &process_handle); -#else - status = tor_spawn_background(argv[0], argv, NULL, &process_handle); -#endif - tt_int_op(expected_status,OP_EQ, status); - tt_assert(process_handle); - tt_int_op(expected_status,OP_EQ, process_handle->status); - - /* Check stdout */ - for (expected_out_ctr = 0; expected_out[expected_out_ctr] != NULL;) { -#ifdef _WIN32 - pos = tor_read_all_handle(process_handle->stdout_pipe, stdout_buf, - sizeof(stdout_buf) - 1, NULL); -#else - /* Check that we didn't read the end of file last time */ - tt_assert(!eof); - pos = tor_read_all_handle(process_handle->stdout_pipe, stdout_buf, - sizeof(stdout_buf) - 1, NULL, &eof); -#endif /* defined(_WIN32) */ - log_info(LD_GENERAL, "tor_read_all_handle() returned %d", (int)pos); - - /* We would have blocked, keep on trying */ - if (0 == pos) - continue; - - tt_assert(pos > 0); - stdout_buf[pos] = '\0'; - tt_str_op(expected_out[expected_out_ctr],OP_EQ, stdout_buf); - tt_int_op(strlen(expected_out[expected_out_ctr]),OP_EQ, pos); - expected_out_ctr++; - } - - if (exit_early) { - tor_process_handle_destroy(process_handle, 1); - process_handle = NULL; - goto done; - } - - /* The process should have exited without writing more */ -#ifdef _WIN32 - pos = tor_read_all_handle(process_handle->stdout_pipe, stdout_buf, - sizeof(stdout_buf) - 1, - process_handle); - tt_int_op(0,OP_EQ, pos); -#else /* !(defined(_WIN32)) */ - if (!eof) { - /* We should have got all the data, but maybe not the EOF flag */ - pos = tor_read_all_handle(process_handle->stdout_pipe, stdout_buf, - sizeof(stdout_buf) - 1, - process_handle, &eof); - tt_int_op(0,OP_EQ, pos); - tt_assert(eof); - } - /* Otherwise, we got the EOF on the last read */ -#endif /* defined(_WIN32) */ - - /* Check it terminated correctly */ - retval = tor_get_exit_code(process_handle, 1, &exit_code); - tt_int_op(PROCESS_EXIT_EXITED,OP_EQ, retval); - tt_int_op(expected_exit,OP_EQ, exit_code); - - // TODO: Make test-child exit with something other than 0 - - /* Check stderr */ - pos = tor_read_all_from_process_stderr(process_handle, stderr_buf, - sizeof(stderr_buf) - 1); - tt_assert(pos >= 0); - stderr_buf[pos] = '\0'; - tt_str_op(expected_err,OP_EQ, stderr_buf); - tt_int_op(strlen(expected_err),OP_EQ, pos); - - done: - tor_process_handle_destroy(process_handle, 1); -} - -static void -test_util_spawn_background_partial_read(void *arg) -{ - (void)arg; - test_util_spawn_background_partial_read_impl(0); -} - -static void -test_util_spawn_background_exit_early(void *arg) -{ - (void)arg; - test_util_spawn_background_partial_read_impl(1); -} - -static void -test_util_spawn_background_waitpid_notify(void *arg) -{ - int retval, exit_code; - process_handle_t *process_handle=NULL; - int status; - int ms_timer; - - const char *argv[] = {TEST_CHILD, "--fast", NULL}; - - (void) arg; - -#ifdef _WIN32 - status = tor_spawn_background(NULL, argv, NULL, &process_handle); -#else - status = tor_spawn_background(argv[0], argv, NULL, &process_handle); -#endif - - tt_int_op(status, OP_EQ, PROCESS_STATUS_RUNNING); - tt_ptr_op(process_handle, OP_NE, NULL); - - /* We're not going to look at the stdout/stderr output this time. Instead, - * we're testing whether notify_pending_waitpid_calbacks() can report the - * process exit (on unix) and/or whether tor_get_exit_code() can notice it - * (on windows) */ - -#ifndef _WIN32 - ms_timer = 30*1000; - tt_ptr_op(process_handle->waitpid_cb, OP_NE, NULL); - while (process_handle->waitpid_cb && ms_timer > 0) { - tor_sleep_msec(100); - ms_timer -= 100; - notify_pending_waitpid_callbacks(); - } - tt_int_op(ms_timer, OP_GT, 0); - tt_ptr_op(process_handle->waitpid_cb, OP_EQ, NULL); -#endif /* !defined(_WIN32) */ - - ms_timer = 30*1000; - while (((retval = tor_get_exit_code(process_handle, 0, &exit_code)) - == PROCESS_EXIT_RUNNING) && ms_timer > 0) { - tor_sleep_msec(100); - ms_timer -= 100; - } - tt_int_op(ms_timer, OP_GT, 0); - - tt_int_op(retval, OP_EQ, PROCESS_EXIT_EXITED); - - done: - tor_process_handle_destroy(process_handle, 1); -} - -#undef TEST_CHILD -#undef EOL - -#undef MATCH_PROCESS_STATUS - -#ifndef _WIN32 -#undef PROCESS_STATUS_RUNNING_OR_NOTRUNNING -#undef IS_RUNNING_OR_NOTRUNNING -#endif - -#define UTIL_TEST(name, flags) \ - { #name, test_util_ ## name, flags, NULL, NULL } - -struct testcase_t slow_util_tests[] = { - UTIL_TEST(spawn_background_ok, 0), - UTIL_TEST(spawn_background_fail, 0), - UTIL_TEST(spawn_background_partial_read, 0), - UTIL_TEST(spawn_background_exit_early, 0), - UTIL_TEST(spawn_background_waitpid_notify, 0), - END_OF_TESTCASES -}; diff --git a/src/test/test_voting_flags.c b/src/test/test_voting_flags.c new file mode 100644 index 0000000000..5c9eebd00e --- /dev/null +++ b/src/test/test_voting_flags.c @@ -0,0 +1,191 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" + +#define VOTEFLAGS_PRIVATE + +#include "core/or/or.h" + +#include "feature/dirauth/voteflags.h" +#include "feature/nodelist/node_st.h" +#include "feature/nodelist/routerstatus_st.h" +#include "feature/nodelist/routerinfo_st.h" + +#include "app/config/config.h" + +#include "test/test.h" + +typedef struct { + time_t now; + routerinfo_t ri; + node_t node; + + routerstatus_t expected; +} flag_vote_test_cfg_t; + +static void +setup_cfg(flag_vote_test_cfg_t *c) +{ + memset(c, 0, sizeof(*c)); + + c->now = approx_time(); + + c->ri.nickname = (char *) "testing100"; + strlcpy(c->expected.nickname, "testing100", sizeof(c->expected.nickname)); + + memset(c->ri.cache_info.identity_digest, 0xff, DIGEST_LEN); + memset(c->ri.cache_info.signed_descriptor_digest, 0xee, DIGEST_LEN); + + c->ri.cache_info.published_on = c->now - 100; + c->expected.published_on = c->now - 100; + + c->ri.addr = 0x7f010105; + c->expected.addr = 0x7f010105; + c->ri.or_port = 9090; + c->expected.or_port = 9090; + + tor_addr_make_null(&c->ri.ipv6_addr, AF_INET6); + tor_addr_make_null(&c->expected.ipv6_addr, AF_INET6); + + // By default we have no loaded information about stability or speed, + // so we'll default to voting "yeah sure." on these two. + c->expected.is_fast = 1; + c->expected.is_stable = 1; +} + +static bool +check_result(flag_vote_test_cfg_t *c) +{ + bool result = false; + routerstatus_t rs; + memset(&rs, 0, sizeof(rs)); + set_routerstatus_from_routerinfo(&rs, &c->node, &c->ri, c->now, 0); + + tt_i64_op(rs.published_on, OP_EQ, c->expected.published_on); + tt_str_op(rs.nickname, OP_EQ, c->expected.nickname); + + // identity_digest and descriptor_digest are not set here. + + tt_uint_op(rs.addr, OP_EQ, c->expected.addr); + tt_uint_op(rs.or_port, OP_EQ, c->expected.or_port); + tt_uint_op(rs.dir_port, OP_EQ, c->expected.dir_port); + + tt_assert(tor_addr_eq(&rs.ipv6_addr, &c->expected.ipv6_addr)); + tt_uint_op(rs.ipv6_orport, OP_EQ, c->expected.ipv6_orport); + +#define FLAG(flagname) \ + tt_uint_op(rs.flagname, OP_EQ, c->expected.flagname) + + FLAG(is_authority); + FLAG(is_exit); + FLAG(is_stable); + FLAG(is_fast); + FLAG(is_flagged_running); + FLAG(is_named); + FLAG(is_unnamed); + FLAG(is_valid); + FLAG(is_possible_guard); + FLAG(is_bad_exit); + FLAG(is_hs_dir); + FLAG(is_v2_dir); + FLAG(is_staledesc); + FLAG(has_bandwidth); + FLAG(has_exitsummary); + FLAG(bw_is_unmeasured); + + result = true; + + done: + return result; +} + +static void +test_voting_flags_minimal(void *arg) +{ + flag_vote_test_cfg_t *cfg = arg; + (void) check_result(cfg); +} + +static void +test_voting_flags_ipv6(void *arg) +{ + flag_vote_test_cfg_t *cfg = arg; + + tt_assert(tor_addr_parse(&cfg->ri.ipv6_addr, "f00::b42") == AF_INET6); + cfg->ri.ipv6_orport = 9091; + // no change in expected results, since we aren't set up with ipv6 + // connectivity. + if (!check_result(cfg)) + goto done; + + get_options_mutable()->AuthDirHasIPv6Connectivity = 1; + // no change in expected results, since last_reachable6 won't be set. + if (!check_result(cfg)) + goto done; + + cfg->node.last_reachable6 = cfg->now - 10; + // now that lastreachable6 is set, we expect to see the result. + tt_assert(tor_addr_parse(&cfg->expected.ipv6_addr, "f00::b42") == AF_INET6); + cfg->expected.ipv6_orport = 9091; + if (!check_result(cfg)) + goto done; + done: + ; +} + +static void +test_voting_flags_staledesc(void *arg) +{ + flag_vote_test_cfg_t *cfg = arg; + time_t now = cfg->now; + + cfg->ri.cache_info.published_on = now - DESC_IS_STALE_INTERVAL + 10; + cfg->expected.published_on = now - DESC_IS_STALE_INTERVAL + 10; + // no change in expectations for is_staledesc + if (!check_result(cfg)) + goto done; + + cfg->ri.cache_info.published_on = now - DESC_IS_STALE_INTERVAL - 10; + cfg->expected.published_on = now - DESC_IS_STALE_INTERVAL - 10; + cfg->expected.is_staledesc = 1; + if (!check_result(cfg)) + goto done; + + done: + ; +} + +static void * +setup_voting_flags_test(const struct testcase_t *testcase) +{ + (void)testcase; + flag_vote_test_cfg_t *cfg = tor_malloc_zero(sizeof(*cfg)); + setup_cfg(cfg); + return cfg; +} + +static int +teardown_voting_flags_test(const struct testcase_t *testcase, void *arg) +{ + (void)testcase; + flag_vote_test_cfg_t *cfg = arg; + tor_free(cfg); + return 1; +} + +static const struct testcase_setup_t voting_flags_setup = { + .setup_fn = setup_voting_flags_test, + .cleanup_fn = teardown_voting_flags_test, +}; + +#define T(name,flags) \ + { #name, test_voting_flags_##name, (flags), &voting_flags_setup, NULL } + +struct testcase_t voting_flags_tests[] = { + T(minimal, 0), + T(ipv6, TT_FORK), + // TODO: Add more of these tests. + T(staledesc, TT_FORK), + END_OF_TESTCASES +}; diff --git a/src/test/testing_common.c b/src/test/testing_common.c index 62d40a42fa..8fc8ef7830 100644 --- a/src/test/testing_common.c +++ b/src/test/testing_common.c @@ -25,6 +25,8 @@ #include "lib/compress/compress.h" #include "lib/evloop/compat_libevent.h" #include "lib/crypt_ops/crypto_init.h" +#include "lib/version/torversion.h" +#include "app/main/subsysmgr.h" #include <stdio.h> #ifdef HAVE_FCNTL_H @@ -230,12 +232,12 @@ void tinytest_prefork(void) { free_pregenerated_keys(); - crypto_prefork(); + subsystems_prefork(); } void tinytest_postfork(void) { - crypto_postfork(); + subsystems_postfork(); init_pregenerated_keys(); } @@ -259,24 +261,15 @@ main(int c, const char **v) int loglevel = LOG_ERR; int accel_crypto = 0; - /* We must initialise logs before we call tor_assert() */ - init_logging(1); + subsystems_init_upto(SUBSYS_LEVEL_LIBS); - update_approx_time(time(NULL)); options = options_new(); - tor_threads_init(); - tor_compress_init(); - - network_init(); - - monotime_init(); struct tor_libevent_cfg cfg; memset(&cfg, 0, sizeof(cfg)); tor_libevent_initialize(&cfg); control_initialize_event_queue(); - configure_backtrace_handler(get_version()); for (i_out = i = 1; i < c; ++i) { if (!strcmp(v[i], "--warn")) { @@ -312,6 +305,7 @@ main(int c, const char **v) s.masks[LOG_WARN-LOG_ERR] |= LD_BUG; add_callback_log(&s, log_callback_failure); } + flush_log_messages_from_startup(); init_protocol_warning_severity_level(); options->command = CMD_RUN_UNITTESTS; @@ -352,8 +346,6 @@ main(int c, const char **v) free_pregenerated_keys(); - crypto_global_cleanup(); - if (have_failed) return 1; else diff --git a/src/tools/include.am b/src/tools/include.am index f7aa7e0d1e..72dfe6017c 100644 --- a/src/tools/include.am +++ b/src/tools/include.am @@ -5,9 +5,11 @@ noinst_PROGRAMS+= src/tools/tor-cov-resolve endif src_tools_tor_resolve_SOURCES = src/tools/tor-resolve.c -src_tools_tor_resolve_LDFLAGS = +src_tools_tor_resolve_LDFLAGS = @TOR_LDFLAGS_openssl@ src_tools_tor_resolve_LDADD = \ + src/trunnel/libor-trunnel.a \ $(TOR_UTIL_LIBS) \ + $(TOR_CRYPTO_LIBS) $(TOR_LIBS_CRYPTLIB)\ $(rust_ldadd) \ @TOR_LIB_MATH@ @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_USERENV@ @@ -15,8 +17,11 @@ if COVERAGE_ENABLED src_tools_tor_cov_resolve_SOURCES = src/tools/tor-resolve.c src_tools_tor_cov_resolve_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_tools_tor_cov_resolve_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +src_tools_tor_cov_resolve_LDFLAGS = @TOR_LDFLAGS_openssl@ src_tools_tor_cov_resolve_LDADD = \ + src/trunnel/libor-trunnel.a \ $(TOR_UTIL_TESTING_LIBS) \ + $(TOR_CRYPTO_TESTING_LIBS) $(TOR_LIBS_CRYPTLIB) \ @TOR_LIB_MATH@ @TOR_LIB_WS32@ endif diff --git a/src/tools/tor-resolve.c b/src/tools/tor-resolve.c index 6a84abe557..98b3a4a74c 100644 --- a/src/tools/tor-resolve.c +++ b/src/tools/tor-resolve.c @@ -15,6 +15,7 @@ #include "lib/string/util_string.h" #include "lib/net/socks5_status.h" +#include "trunnel/socks5.h" #include <stdio.h> #include <stdlib.h> @@ -49,68 +50,204 @@ static void usage(void) ATTR_NORETURN; +/** + * Set <b>out</b> to a pointer to newly allocated buffer containing + * SOCKS4a RESOLVE request with <b>username</b> and <b>hostname</b>. + * Return number of bytes in the buffer if succeeded or -1 if failed. + */ +static ssize_t +build_socks4a_resolve_request(uint8_t **out, + const char *username, + const char *hostname) +{ + tor_assert(out); + tor_assert(username); + tor_assert(hostname); + + const char *errmsg = NULL; + uint8_t *output = NULL; + socks4_client_request_t *rq = socks4_client_request_new(); + + socks4_client_request_set_version(rq, 4); + socks4_client_request_set_command(rq, CMD_RESOLVE); + socks4_client_request_set_port(rq, 0); + socks4_client_request_set_addr(rq, 0x00000001u); + socks4_client_request_set_username(rq, username); + socks4_client_request_set_socks4a_addr_hostname(rq, hostname); + + errmsg = socks4_client_request_check(rq); + if (errmsg) { + goto cleanup; + } + + ssize_t encoded_len = socks4_client_request_encoded_len(rq); + if (encoded_len <= 0) { + errmsg = "socks4_client_request_encoded_len failed"; + goto cleanup; + } + + output = tor_malloc(encoded_len); + memset(output, 0, encoded_len); + + encoded_len = socks4_client_request_encode(output, encoded_len, rq); + if (encoded_len <= 0) { + errmsg = "socks4_client_request_encode failed"; + goto cleanup; + } + + *out = output; + + cleanup: + socks4_client_request_free(rq); + if (errmsg) { + log_err(LD_NET, "build_socks4a_resolve_request failed: %s", errmsg); + *out = NULL; + tor_free(output); + } + return errmsg ? -1 : encoded_len; +} + +#define SOCKS5_ATYPE_HOSTNAME 0x03 +#define SOCKS5_ATYPE_IPV4 0x01 +#define SOCKS5_ATYPE_IPV6 0x04 + +/** + * Set <b>out</b> to pointer to newly allocated buffer containing + * SOCKS5 RESOLVE/RESOLVE_PTR request with given <b>hostname<b>. + * Generate a reverse request if <b>reverse</b> is true. + * Return the number of bytes in the buffer if succeeded or -1 if failed. + */ +static ssize_t +build_socks5_resolve_request(uint8_t **out, + const char *hostname, + int reverse) +{ + const char *errmsg = NULL; + uint8_t *outbuf = NULL; + int is_ip_address; + tor_addr_t addr; + size_t addrlen; + int ipv6; + is_ip_address = tor_addr_parse(&addr, hostname) != -1; + if (!is_ip_address && reverse) { + log_err(LD_GENERAL, "Tried to do a reverse lookup on a non-IP!"); + return -1; + } + ipv6 = reverse && tor_addr_family(&addr) == AF_INET6; + addrlen = reverse ? (ipv6 ? 16 : 4) : 1 + strlen(hostname); + if (addrlen > UINT8_MAX) { + log_err(LD_GENERAL, "Hostname is too long!"); + return -1; + } + + socks5_client_request_t *rq = socks5_client_request_new(); + + socks5_client_request_set_version(rq, 5); + /* RESOLVE_PTR or RESOLVE */ + socks5_client_request_set_command(rq, reverse ? CMD_RESOLVE_PTR : + CMD_RESOLVE); + socks5_client_request_set_reserved(rq, 0); + + uint8_t atype = SOCKS5_ATYPE_HOSTNAME; + if (reverse) + atype = ipv6 ? SOCKS5_ATYPE_IPV6 : SOCKS5_ATYPE_IPV4; + + socks5_client_request_set_atype(rq, atype); + + switch (atype) { + case SOCKS5_ATYPE_IPV4: { + socks5_client_request_set_dest_addr_ipv4(rq, + tor_addr_to_ipv4h(&addr)); + } break; + case SOCKS5_ATYPE_IPV6: { + uint8_t *ipv6_array = + socks5_client_request_getarray_dest_addr_ipv6(rq); + + tor_assert(ipv6_array); + + memcpy(ipv6_array, tor_addr_to_in6_addr8(&addr), 16); + } break; + + case SOCKS5_ATYPE_HOSTNAME: { + domainname_t *dn = domainname_new(); + domainname_set_len(dn, addrlen - 1); + domainname_setlen_name(dn, addrlen - 1); + char *dn_buf = domainname_getarray_name(dn); + memcpy(dn_buf, hostname, addrlen - 1); + + errmsg = domainname_check(dn); + + if (errmsg) { + domainname_free(dn); + goto cleanup; + } else { + socks5_client_request_set_dest_addr_domainname(rq, dn); + } + } break; + default: + tor_assert_unreached(); + break; + } + + socks5_client_request_set_dest_port(rq, 0); + + errmsg = socks5_client_request_check(rq); + if (errmsg) { + goto cleanup; + } + + ssize_t encoded_len = socks5_client_request_encoded_len(rq); + if (encoded_len < 0) { + errmsg = "Cannot predict encoded length"; + goto cleanup; + } + + outbuf = tor_malloc(encoded_len); + memset(outbuf, 0, encoded_len); + + encoded_len = socks5_client_request_encode(outbuf, encoded_len, rq); + if (encoded_len < 0) { + errmsg = "encoding failed"; + goto cleanup; + } + + *out = outbuf; + + cleanup: + socks5_client_request_free(rq); + if (errmsg) { + tor_free(outbuf); + log_err(LD_NET, "build_socks5_resolve_request failed with error: %s", + errmsg); + } + + return errmsg ? -1 : encoded_len; +} + /** Set *<b>out</b> to a newly allocated SOCKS4a resolve request with * <b>username</b> and <b>hostname</b> as provided. Return the number * of bytes in the request. */ static ssize_t -build_socks_resolve_request(char **out, +build_socks_resolve_request(uint8_t **out, const char *username, const char *hostname, int reverse, int version) { - size_t len = 0; tor_assert(out); tor_assert(username); tor_assert(hostname); + tor_assert(version == 4 || version == 5); + if (version == 4) { - len = 8 + strlen(username) + 1 + strlen(hostname) + 1; - *out = tor_malloc(len); - (*out)[0] = 4; /* SOCKS version 4 */ - (*out)[1] = '\xF0'; /* Command: resolve. */ - set_uint16((*out)+2, htons(0)); /* port: 0. */ - set_uint32((*out)+4, htonl(0x00000001u)); /* addr: 0.0.0.1 */ - memcpy((*out)+8, username, strlen(username)+1); - memcpy((*out)+8+strlen(username)+1, hostname, strlen(hostname)+1); + return build_socks4a_resolve_request(out, username, hostname); } else if (version == 5) { - int is_ip_address; - tor_addr_t addr; - size_t addrlen; - int ipv6; - is_ip_address = tor_addr_parse(&addr, hostname) != -1; - if (!is_ip_address && reverse) { - log_err(LD_GENERAL, "Tried to do a reverse lookup on a non-IP!"); - return -1; - } - ipv6 = reverse && tor_addr_family(&addr) == AF_INET6; - addrlen = reverse ? (ipv6 ? 16 : 4) : 1 + strlen(hostname); - if (addrlen > UINT8_MAX) { - log_err(LD_GENERAL, "Hostname is too long!"); - return -1; - } - len = 6 + addrlen; - *out = tor_malloc(len); - (*out)[0] = 5; /* SOCKS version 5 */ - (*out)[1] = reverse ? '\xF1' : '\xF0'; /* RESOLVE_PTR or RESOLVE */ - (*out)[2] = 0; /* reserved. */ - if (reverse) { - (*out)[3] = ipv6 ? 4 : 1; - if (ipv6) - memcpy((*out)+4, tor_addr_to_in6_addr8(&addr), 16); - else - set_uint32((*out)+4, tor_addr_to_ipv4n(&addr)); - } else { - (*out)[3] = 3; - (*out)[4] = (char)(uint8_t)(addrlen - 1); - memcpy((*out)+5, hostname, addrlen - 1); - } - set_uint16((*out)+4+addrlen, 0); /* port */ - } else { - tor_assert(0); + return build_socks5_resolve_request(out, hostname, reverse); } - return len; + tor_assert_unreached(); + return -1; } static void @@ -134,34 +271,50 @@ parse_socks4a_resolve_response(const char *hostname, const char *response, size_t len, tor_addr_t *addr_out) { + int result = 0; uint8_t status; tor_assert(response); tor_assert(addr_out); - if (len < RESPONSE_LEN_4) { + socks4_server_reply_t *reply; + + ssize_t parsed = socks4_server_reply_parse(&reply, + (const uint8_t *)response, + len); + + if (parsed == -1) { + log_warn(LD_PROTOCOL, "Failed parsing SOCKS4a response"); + result = -1; goto cleanup; + } + + if (parsed == -2) { log_warn(LD_PROTOCOL,"Truncated socks response."); - return -1; + result = -1; goto cleanup; } - if (((uint8_t)response[0])!=0) { /* version: 0 */ + + if (socks4_server_reply_get_version(reply) != 0) { /* version: 0 */ log_warn(LD_PROTOCOL,"Nonzero version in socks response: bad format."); - return -1; + result = -1; goto cleanup; } - status = (uint8_t)response[1]; - if (get_uint16(response+2)!=0) { /* port: 0 */ + if (socks4_server_reply_get_port(reply) != 0) { /* port: 0 */ log_warn(LD_PROTOCOL,"Nonzero port in socks response: bad format."); - return -1; + result = -1; goto cleanup; } + status = socks4_server_reply_get_status(reply); if (status != 90) { log_warn(LD_NET,"Got status response '%d': socks request failed.", status); if (!strcasecmpend(hostname, ".onion")) { onion_warning(hostname); - return -1; + result = -1; goto cleanup; } - return -1; + result = -1; goto cleanup; } - tor_addr_from_ipv4n(addr_out, get_uint32(response+4)); - return 0; + tor_addr_from_ipv4h(addr_out, socks4_server_reply_get_addr(reply)); + + cleanup: + socks4_server_reply_free(reply); + return result; } /* It would be nice to let someone know what SOCKS5 issue a user may have */ @@ -205,7 +358,7 @@ do_resolve(const char *hostname, int s = -1; struct sockaddr_storage ss; socklen_t socklen; - char *req = NULL; + uint8_t *req = NULL; ssize_t len = 0; tor_assert(hostname); @@ -230,23 +383,57 @@ do_resolve(const char *hostname, } if (version == 5) { - char method_buf[2]; - if (write_all_to_socket(s, "\x05\x01\x00", 3) != 3) { + socks5_client_version_t *v = socks5_client_version_new(); + + socks5_client_version_set_version(v, 5); + socks5_client_version_set_n_methods(v, 1); + socks5_client_version_setlen_methods(v, 1); + socks5_client_version_set_methods(v, 0, 0x00); + + tor_assert(!socks5_client_version_check(v)); + ssize_t encoded_len = socks5_client_version_encoded_len(v); + tor_assert(encoded_len > 0); + + uint8_t *buf = tor_malloc(encoded_len); + encoded_len = socks5_client_version_encode(buf, encoded_len, v); + tor_assert(encoded_len > 0); + + socks5_client_version_free(v); + + if (write_all_to_socket(s, (const char *)buf, + encoded_len) != encoded_len) { log_err(LD_NET, "Error sending SOCKS5 method list."); + tor_free(buf); + goto err; } - if (read_all_from_socket(s, method_buf, 2) != 2) { + + tor_free(buf); + + uint8_t method_buf[2]; + + if (read_all_from_socket(s, (char *)method_buf, 2) != 2) { log_err(LD_NET, "Error reading SOCKS5 methods."); goto err; } - if (method_buf[0] != '\x05') { - log_err(LD_NET, "Unrecognized socks version: %u", - (unsigned)method_buf[0]); + + socks5_server_method_t *m; + ssize_t parsed = socks5_server_method_parse(&m, method_buf, + sizeof(method_buf)); + + if (parsed < 2) { + log_err(LD_NET, "Failed to parse SOCKS5 method selection " + "message"); goto err; } - if (method_buf[1] != '\x00') { + + uint8_t method = socks5_server_method_get_method(m); + + socks5_server_method_free(m); + + if (method != 0x00) { log_err(LD_NET, "Unrecognized socks authentication method: %u", - (unsigned)method_buf[1]); + (unsigned)method); goto err; } } @@ -257,7 +444,7 @@ do_resolve(const char *hostname, tor_assert(!req); goto err; } - if (write_all_to_socket(s, req, len) != len) { + if (write_all_to_socket(s, (const char *)req, len) != len) { log_sock_error("sending SOCKS request", s); tor_free(req); goto err; @@ -276,55 +463,59 @@ do_resolve(const char *hostname, goto err; } } else { - char reply_buf[16]; - if (read_all_from_socket(s, reply_buf, 4) != 4) { - log_err(LD_NET, "Error reading SOCKS5 response."); + uint8_t reply_buf[512]; + + len = read_all_from_socket(s, (char *)reply_buf, + sizeof(reply_buf)); + + socks5_server_reply_t *reply; + + ssize_t parsed = socks5_server_reply_parse(&reply, + reply_buf, + len); + if (parsed == -1) { + log_err(LD_NET, "Failed parsing SOCKS5 response"); goto err; } - if (reply_buf[0] != 5) { - log_err(LD_NET, "Bad SOCKS5 reply version."); + + if (parsed == -2) { + log_err(LD_NET, "Truncated SOCKS5 response"); goto err; } + /* Give a user some useful feedback about SOCKS5 errors */ - if (reply_buf[1] != 0) { + uint8_t reply_field = socks5_server_reply_get_reply(reply); + if (reply_field != 0) { log_warn(LD_NET,"Got SOCKS5 status response '%u': %s", - (unsigned)reply_buf[1], - socks5_reason_to_string(reply_buf[1])); - if (reply_buf[1] == 4 && !strcasecmpend(hostname, ".onion")) { + (unsigned)reply_field, + socks5_reason_to_string(reply_field)); + if (reply_field == 4 && !strcasecmpend(hostname, ".onion")) { onion_warning(hostname); } + + socks5_server_reply_free(reply); goto err; } - if (reply_buf[3] == 1) { + + uint8_t atype = socks5_server_reply_get_atype(reply); + + if (atype == SOCKS5_ATYPE_IPV4) { /* IPv4 address */ - if (read_all_from_socket(s, reply_buf, 4) != 4) { - log_err(LD_NET, "Error reading address in socks5 response."); - goto err; - } - tor_addr_from_ipv4n(result_addr, get_uint32(reply_buf)); - } else if (reply_buf[3] == 4) { + tor_addr_from_ipv4h(result_addr, + socks5_server_reply_get_bind_addr_ipv4(reply)); + } else if (atype == SOCKS5_ATYPE_IPV6) { /* IPv6 address */ - if (read_all_from_socket(s, reply_buf, 16) != 16) { - log_err(LD_NET, "Error reading address in socks5 response."); - goto err; - } - tor_addr_from_ipv6_bytes(result_addr, reply_buf); - } else if (reply_buf[3] == 3) { + tor_addr_from_ipv6_bytes(result_addr, + (const char *)socks5_server_reply_getarray_bind_addr_ipv6(reply)); + } else if (atype == SOCKS5_ATYPE_HOSTNAME) { /* Domain name */ - size_t result_len; - if (read_all_from_socket(s, reply_buf, 1) != 1) { - log_err(LD_NET, "Error reading address_length in socks5 response."); - goto err; - } - result_len = *(uint8_t*)(reply_buf); - *result_hostname = tor_malloc(result_len+1); - if (read_all_from_socket(s, *result_hostname, result_len) - != (int) result_len) { - log_err(LD_NET, "Error reading hostname in socks5 response."); - goto err; - } - (*result_hostname)[result_len] = '\0'; + domainname_t *dn = + socks5_server_reply_get_bind_addr_domainname(reply); + + *result_hostname = tor_strdup(domainname_getstr_name(dn)); } + + socks5_server_reply_free(reply); } tor_close_socket(s); diff --git a/src/trunnel/circpad_negotiation.c b/src/trunnel/circpad_negotiation.c new file mode 100644 index 0000000000..236be06ada --- /dev/null +++ b/src/trunnel/circpad_negotiation.c @@ -0,0 +1,549 @@ +/* circpad_negotiation.c -- generated by Trunnel v1.5.2. + * https://gitweb.torproject.org/trunnel.git + * You probably shouldn't edit this file. + */ +#include <stdlib.h> +#include "trunnel-impl.h" + +#include "circpad_negotiation.h" + +#define TRUNNEL_SET_ERROR_CODE(obj) \ + do { \ + (obj)->trunnel_error_code_ = 1; \ + } while (0) + +#if defined(__COVERITY__) || defined(__clang_analyzer__) +/* If we're running a static analysis tool, we don't want it to complain + * that some of our remaining-bytes checks are dead-code. */ +int circpadnegotiation_deadcode_dummy__ = 0; +#define OR_DEADCODE_DUMMY || circpadnegotiation_deadcode_dummy__ +#else +#define OR_DEADCODE_DUMMY +#endif + +#define CHECK_REMAINING(nbytes, label) \ + do { \ + if (remaining < (nbytes) OR_DEADCODE_DUMMY) { \ + goto label; \ + } \ + } while (0) + +circpad_negotiate_t * +circpad_negotiate_new(void) +{ + circpad_negotiate_t *val = trunnel_calloc(1, sizeof(circpad_negotiate_t)); + if (NULL == val) + return NULL; + val->command = CIRCPAD_COMMAND_START; + return val; +} + +/** Release all storage held inside 'obj', but do not free 'obj'. + */ +static void +circpad_negotiate_clear(circpad_negotiate_t *obj) +{ + (void) obj; +} + +void +circpad_negotiate_free(circpad_negotiate_t *obj) +{ + if (obj == NULL) + return; + circpad_negotiate_clear(obj); + trunnel_memwipe(obj, sizeof(circpad_negotiate_t)); + trunnel_free_(obj); +} + +uint8_t +circpad_negotiate_get_version(const circpad_negotiate_t *inp) +{ + return inp->version; +} +int +circpad_negotiate_set_version(circpad_negotiate_t *inp, uint8_t val) +{ + if (! ((val == 0))) { + TRUNNEL_SET_ERROR_CODE(inp); + return -1; + } + inp->version = val; + return 0; +} +uint8_t +circpad_negotiate_get_command(const circpad_negotiate_t *inp) +{ + return inp->command; +} +int +circpad_negotiate_set_command(circpad_negotiate_t *inp, uint8_t val) +{ + if (! ((val == CIRCPAD_COMMAND_START || val == CIRCPAD_COMMAND_STOP))) { + TRUNNEL_SET_ERROR_CODE(inp); + return -1; + } + inp->command = val; + return 0; +} +uint8_t +circpad_negotiate_get_machine_type(const circpad_negotiate_t *inp) +{ + return inp->machine_type; +} +int +circpad_negotiate_set_machine_type(circpad_negotiate_t *inp, uint8_t val) +{ + inp->machine_type = val; + return 0; +} +uint8_t +circpad_negotiate_get_echo_request(const circpad_negotiate_t *inp) +{ + return inp->echo_request; +} +int +circpad_negotiate_set_echo_request(circpad_negotiate_t *inp, uint8_t val) +{ + if (! ((val == 0 || val == 1))) { + TRUNNEL_SET_ERROR_CODE(inp); + return -1; + } + inp->echo_request = val; + return 0; +} +const char * +circpad_negotiate_check(const circpad_negotiate_t *obj) +{ + if (obj == NULL) + return "Object was NULL"; + if (obj->trunnel_error_code_) + return "A set function failed on this object"; + if (! (obj->version == 0)) + return "Integer out of bounds"; + if (! (obj->command == CIRCPAD_COMMAND_START || obj->command == CIRCPAD_COMMAND_STOP)) + return "Integer out of bounds"; + if (! (obj->echo_request == 0 || obj->echo_request == 1)) + return "Integer out of bounds"; + return NULL; +} + +ssize_t +circpad_negotiate_encoded_len(const circpad_negotiate_t *obj) +{ + ssize_t result = 0; + + if (NULL != circpad_negotiate_check(obj)) + return -1; + + + /* Length of u8 version IN [0] */ + result += 1; + + /* Length of u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP] */ + result += 1; + + /* Length of u8 machine_type */ + result += 1; + + /* Length of u8 echo_request IN [0, 1] */ + result += 1; + return result; +} +int +circpad_negotiate_clear_errors(circpad_negotiate_t *obj) +{ + int r = obj->trunnel_error_code_; + obj->trunnel_error_code_ = 0; + return r; +} +ssize_t +circpad_negotiate_encode(uint8_t *output, const size_t avail, const circpad_negotiate_t *obj) +{ + ssize_t result = 0; + size_t written = 0; + uint8_t *ptr = output; + const char *msg; +#ifdef TRUNNEL_CHECK_ENCODED_LEN + const ssize_t encoded_len = circpad_negotiate_encoded_len(obj); +#endif + + if (NULL != (msg = circpad_negotiate_check(obj))) + goto check_failed; + +#ifdef TRUNNEL_CHECK_ENCODED_LEN + trunnel_assert(encoded_len >= 0); +#endif + + /* Encode u8 version IN [0] */ + trunnel_assert(written <= avail); + if (avail - written < 1) + goto truncated; + trunnel_set_uint8(ptr, (obj->version)); + written += 1; ptr += 1; + + /* Encode u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP] */ + trunnel_assert(written <= avail); + if (avail - written < 1) + goto truncated; + trunnel_set_uint8(ptr, (obj->command)); + written += 1; ptr += 1; + + /* Encode u8 machine_type */ + trunnel_assert(written <= avail); + if (avail - written < 1) + goto truncated; + trunnel_set_uint8(ptr, (obj->machine_type)); + written += 1; ptr += 1; + + /* Encode u8 echo_request IN [0, 1] */ + trunnel_assert(written <= avail); + if (avail - written < 1) + goto truncated; + trunnel_set_uint8(ptr, (obj->echo_request)); + written += 1; ptr += 1; + + + trunnel_assert(ptr == output + written); +#ifdef TRUNNEL_CHECK_ENCODED_LEN + { + trunnel_assert(encoded_len >= 0); + trunnel_assert((size_t)encoded_len == written); + } + +#endif + + return written; + + truncated: + result = -2; + goto fail; + check_failed: + (void)msg; + result = -1; + goto fail; + fail: + trunnel_assert(result < 0); + return result; +} + +/** As circpad_negotiate_parse(), but do not allocate the output + * object. + */ +static ssize_t +circpad_negotiate_parse_into(circpad_negotiate_t *obj, const uint8_t *input, const size_t len_in) +{ + const uint8_t *ptr = input; + size_t remaining = len_in; + ssize_t result = 0; + (void)result; + + /* Parse u8 version IN [0] */ + CHECK_REMAINING(1, truncated); + obj->version = (trunnel_get_uint8(ptr)); + remaining -= 1; ptr += 1; + if (! (obj->version == 0)) + goto fail; + + /* Parse u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP] */ + CHECK_REMAINING(1, truncated); + obj->command = (trunnel_get_uint8(ptr)); + remaining -= 1; ptr += 1; + if (! (obj->command == CIRCPAD_COMMAND_START || obj->command == CIRCPAD_COMMAND_STOP)) + goto fail; + + /* Parse u8 machine_type */ + CHECK_REMAINING(1, truncated); + obj->machine_type = (trunnel_get_uint8(ptr)); + remaining -= 1; ptr += 1; + + /* Parse u8 echo_request IN [0, 1] */ + CHECK_REMAINING(1, truncated); + obj->echo_request = (trunnel_get_uint8(ptr)); + remaining -= 1; ptr += 1; + if (! (obj->echo_request == 0 || obj->echo_request == 1)) + goto fail; + trunnel_assert(ptr + remaining == input + len_in); + return len_in - remaining; + + truncated: + return -2; + fail: + result = -1; + return result; +} + +ssize_t +circpad_negotiate_parse(circpad_negotiate_t **output, const uint8_t *input, const size_t len_in) +{ + ssize_t result; + *output = circpad_negotiate_new(); + if (NULL == *output) + return -1; + result = circpad_negotiate_parse_into(*output, input, len_in); + if (result < 0) { + circpad_negotiate_free(*output); + *output = NULL; + } + return result; +} +circpad_negotiated_t * +circpad_negotiated_new(void) +{ + circpad_negotiated_t *val = trunnel_calloc(1, sizeof(circpad_negotiated_t)); + if (NULL == val) + return NULL; + val->command = CIRCPAD_COMMAND_START; + val->response = CIRCPAD_RESPONSE_ERR; + return val; +} + +/** Release all storage held inside 'obj', but do not free 'obj'. + */ +static void +circpad_negotiated_clear(circpad_negotiated_t *obj) +{ + (void) obj; +} + +void +circpad_negotiated_free(circpad_negotiated_t *obj) +{ + if (obj == NULL) + return; + circpad_negotiated_clear(obj); + trunnel_memwipe(obj, sizeof(circpad_negotiated_t)); + trunnel_free_(obj); +} + +uint8_t +circpad_negotiated_get_version(const circpad_negotiated_t *inp) +{ + return inp->version; +} +int +circpad_negotiated_set_version(circpad_negotiated_t *inp, uint8_t val) +{ + if (! ((val == 0))) { + TRUNNEL_SET_ERROR_CODE(inp); + return -1; + } + inp->version = val; + return 0; +} +uint8_t +circpad_negotiated_get_command(const circpad_negotiated_t *inp) +{ + return inp->command; +} +int +circpad_negotiated_set_command(circpad_negotiated_t *inp, uint8_t val) +{ + if (! ((val == CIRCPAD_COMMAND_START || val == CIRCPAD_COMMAND_STOP))) { + TRUNNEL_SET_ERROR_CODE(inp); + return -1; + } + inp->command = val; + return 0; +} +uint8_t +circpad_negotiated_get_response(const circpad_negotiated_t *inp) +{ + return inp->response; +} +int +circpad_negotiated_set_response(circpad_negotiated_t *inp, uint8_t val) +{ + if (! ((val == CIRCPAD_RESPONSE_ERR || val == CIRCPAD_RESPONSE_OK))) { + TRUNNEL_SET_ERROR_CODE(inp); + return -1; + } + inp->response = val; + return 0; +} +uint8_t +circpad_negotiated_get_machine_type(const circpad_negotiated_t *inp) +{ + return inp->machine_type; +} +int +circpad_negotiated_set_machine_type(circpad_negotiated_t *inp, uint8_t val) +{ + inp->machine_type = val; + return 0; +} +const char * +circpad_negotiated_check(const circpad_negotiated_t *obj) +{ + if (obj == NULL) + return "Object was NULL"; + if (obj->trunnel_error_code_) + return "A set function failed on this object"; + if (! (obj->version == 0)) + return "Integer out of bounds"; + if (! (obj->command == CIRCPAD_COMMAND_START || obj->command == CIRCPAD_COMMAND_STOP)) + return "Integer out of bounds"; + if (! (obj->response == CIRCPAD_RESPONSE_ERR || obj->response == CIRCPAD_RESPONSE_OK)) + return "Integer out of bounds"; + return NULL; +} + +ssize_t +circpad_negotiated_encoded_len(const circpad_negotiated_t *obj) +{ + ssize_t result = 0; + + if (NULL != circpad_negotiated_check(obj)) + return -1; + + + /* Length of u8 version IN [0] */ + result += 1; + + /* Length of u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP] */ + result += 1; + + /* Length of u8 response IN [CIRCPAD_RESPONSE_ERR, CIRCPAD_RESPONSE_OK] */ + result += 1; + + /* Length of u8 machine_type */ + result += 1; + return result; +} +int +circpad_negotiated_clear_errors(circpad_negotiated_t *obj) +{ + int r = obj->trunnel_error_code_; + obj->trunnel_error_code_ = 0; + return r; +} +ssize_t +circpad_negotiated_encode(uint8_t *output, const size_t avail, const circpad_negotiated_t *obj) +{ + ssize_t result = 0; + size_t written = 0; + uint8_t *ptr = output; + const char *msg; +#ifdef TRUNNEL_CHECK_ENCODED_LEN + const ssize_t encoded_len = circpad_negotiated_encoded_len(obj); +#endif + + if (NULL != (msg = circpad_negotiated_check(obj))) + goto check_failed; + +#ifdef TRUNNEL_CHECK_ENCODED_LEN + trunnel_assert(encoded_len >= 0); +#endif + + /* Encode u8 version IN [0] */ + trunnel_assert(written <= avail); + if (avail - written < 1) + goto truncated; + trunnel_set_uint8(ptr, (obj->version)); + written += 1; ptr += 1; + + /* Encode u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP] */ + trunnel_assert(written <= avail); + if (avail - written < 1) + goto truncated; + trunnel_set_uint8(ptr, (obj->command)); + written += 1; ptr += 1; + + /* Encode u8 response IN [CIRCPAD_RESPONSE_ERR, CIRCPAD_RESPONSE_OK] */ + trunnel_assert(written <= avail); + if (avail - written < 1) + goto truncated; + trunnel_set_uint8(ptr, (obj->response)); + written += 1; ptr += 1; + + /* Encode u8 machine_type */ + trunnel_assert(written <= avail); + if (avail - written < 1) + goto truncated; + trunnel_set_uint8(ptr, (obj->machine_type)); + written += 1; ptr += 1; + + + trunnel_assert(ptr == output + written); +#ifdef TRUNNEL_CHECK_ENCODED_LEN + { + trunnel_assert(encoded_len >= 0); + trunnel_assert((size_t)encoded_len == written); + } + +#endif + + return written; + + truncated: + result = -2; + goto fail; + check_failed: + (void)msg; + result = -1; + goto fail; + fail: + trunnel_assert(result < 0); + return result; +} + +/** As circpad_negotiated_parse(), but do not allocate the output + * object. + */ +static ssize_t +circpad_negotiated_parse_into(circpad_negotiated_t *obj, const uint8_t *input, const size_t len_in) +{ + const uint8_t *ptr = input; + size_t remaining = len_in; + ssize_t result = 0; + (void)result; + + /* Parse u8 version IN [0] */ + CHECK_REMAINING(1, truncated); + obj->version = (trunnel_get_uint8(ptr)); + remaining -= 1; ptr += 1; + if (! (obj->version == 0)) + goto fail; + + /* Parse u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP] */ + CHECK_REMAINING(1, truncated); + obj->command = (trunnel_get_uint8(ptr)); + remaining -= 1; ptr += 1; + if (! (obj->command == CIRCPAD_COMMAND_START || obj->command == CIRCPAD_COMMAND_STOP)) + goto fail; + + /* Parse u8 response IN [CIRCPAD_RESPONSE_ERR, CIRCPAD_RESPONSE_OK] */ + CHECK_REMAINING(1, truncated); + obj->response = (trunnel_get_uint8(ptr)); + remaining -= 1; ptr += 1; + if (! (obj->response == CIRCPAD_RESPONSE_ERR || obj->response == CIRCPAD_RESPONSE_OK)) + goto fail; + + /* Parse u8 machine_type */ + CHECK_REMAINING(1, truncated); + obj->machine_type = (trunnel_get_uint8(ptr)); + remaining -= 1; ptr += 1; + trunnel_assert(ptr + remaining == input + len_in); + return len_in - remaining; + + truncated: + return -2; + fail: + result = -1; + return result; +} + +ssize_t +circpad_negotiated_parse(circpad_negotiated_t **output, const uint8_t *input, const size_t len_in) +{ + ssize_t result; + *output = circpad_negotiated_new(); + if (NULL == *output) + return -1; + result = circpad_negotiated_parse_into(*output, input, len_in); + if (result < 0) { + circpad_negotiated_free(*output); + *output = NULL; + } + return result; +} diff --git a/src/trunnel/circpad_negotiation.h b/src/trunnel/circpad_negotiation.h new file mode 100644 index 0000000000..d09080dc16 --- /dev/null +++ b/src/trunnel/circpad_negotiation.h @@ -0,0 +1,195 @@ +/* circpad_negotiation.h -- generated by Trunnel v1.5.2. + * https://gitweb.torproject.org/trunnel.git + * You probably shouldn't edit this file. + */ +#ifndef TRUNNEL_CIRCPAD_NEGOTIATION_H +#define TRUNNEL_CIRCPAD_NEGOTIATION_H + +#include <stdint.h> +#include "trunnel.h" + +#define CIRCPAD_COMMAND_STOP 1 +#define CIRCPAD_COMMAND_START 2 +#define CIRCPAD_RESPONSE_OK 1 +#define CIRCPAD_RESPONSE_ERR 2 +#define CIRCPAD_MACHINE_CIRC_SETUP 1 +/** + * This command tells the relay to alter its min and max netflow + * timeout range values, and send padding at that rate (resuming + * if stopped). */ +#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_CIRCPAD_NEGOTIATE) +struct circpad_negotiate_st { + uint8_t version; + uint8_t command; + /** Machine type is left unbounded because we can specify + * new machines in the consensus */ + uint8_t machine_type; + /** If true, send a relay_drop reply.. */ + uint8_t echo_request; + uint8_t trunnel_error_code_; +}; +#endif +typedef struct circpad_negotiate_st circpad_negotiate_t; +/** + * This command tells the relay to alter its min and max netflow + * timeout range values, and send padding at that rate (resuming + * if stopped). */ +#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_CIRCPAD_NEGOTIATED) +struct circpad_negotiated_st { + uint8_t version; + uint8_t command; + uint8_t response; + /** Machine type is left unbounded because we can specify + * new machines in the consensus */ + uint8_t machine_type; + uint8_t trunnel_error_code_; +}; +#endif +typedef struct circpad_negotiated_st circpad_negotiated_t; +/** Return a newly allocated circpad_negotiate with all elements set + * to zero. + */ +circpad_negotiate_t *circpad_negotiate_new(void); +/** Release all storage held by the circpad_negotiate in 'victim'. (Do + * nothing if 'victim' is NULL.) + */ +void circpad_negotiate_free(circpad_negotiate_t *victim); +/** Try to parse a circpad_negotiate from the buffer in 'input', using + * up to 'len_in' bytes from the input buffer. On success, return the + * number of bytes consumed and set *output to the newly allocated + * circpad_negotiate_t. On failure, return -2 if the input appears + * truncated, and -1 if the input is otherwise invalid. + */ +ssize_t circpad_negotiate_parse(circpad_negotiate_t **output, const uint8_t *input, const size_t len_in); +/** Return the number of bytes we expect to need to encode the + * circpad_negotiate in 'obj'. On failure, return a negative value. + * Note that this value may be an overestimate, and can even be an + * underestimate for certain unencodeable objects. + */ +ssize_t circpad_negotiate_encoded_len(const circpad_negotiate_t *obj); +/** Try to encode the circpad_negotiate from 'input' into the buffer + * at 'output', using up to 'avail' bytes of the output buffer. On + * success, return the number of bytes used. On failure, return -2 if + * the buffer was not long enough, and -1 if the input was invalid. + */ +ssize_t circpad_negotiate_encode(uint8_t *output, size_t avail, const circpad_negotiate_t *input); +/** Check whether the internal state of the circpad_negotiate in 'obj' + * is consistent. Return NULL if it is, and a short message if it is + * not. + */ +const char *circpad_negotiate_check(const circpad_negotiate_t *obj); +/** Clear any errors that were set on the object 'obj' by its setter + * functions. Return true iff errors were cleared. + */ +int circpad_negotiate_clear_errors(circpad_negotiate_t *obj); +/** Return the value of the version field of the circpad_negotiate_t + * in 'inp' + */ +uint8_t circpad_negotiate_get_version(const circpad_negotiate_t *inp); +/** Set the value of the version field of the circpad_negotiate_t in + * 'inp' to 'val'. Return 0 on success; return -1 and set the error + * code on 'inp' on failure. + */ +int circpad_negotiate_set_version(circpad_negotiate_t *inp, uint8_t val); +/** Return the value of the command field of the circpad_negotiate_t + * in 'inp' + */ +uint8_t circpad_negotiate_get_command(const circpad_negotiate_t *inp); +/** Set the value of the command field of the circpad_negotiate_t in + * 'inp' to 'val'. Return 0 on success; return -1 and set the error + * code on 'inp' on failure. + */ +int circpad_negotiate_set_command(circpad_negotiate_t *inp, uint8_t val); +/** Return the value of the machine_type field of the + * circpad_negotiate_t in 'inp' + */ +uint8_t circpad_negotiate_get_machine_type(const circpad_negotiate_t *inp); +/** Set the value of the machine_type field of the circpad_negotiate_t + * in 'inp' to 'val'. Return 0 on success; return -1 and set the error + * code on 'inp' on failure. + */ +int circpad_negotiate_set_machine_type(circpad_negotiate_t *inp, uint8_t val); +/** Return the value of the echo_request field of the + * circpad_negotiate_t in 'inp' + */ +uint8_t circpad_negotiate_get_echo_request(const circpad_negotiate_t *inp); +/** Set the value of the echo_request field of the circpad_negotiate_t + * in 'inp' to 'val'. Return 0 on success; return -1 and set the error + * code on 'inp' on failure. + */ +int circpad_negotiate_set_echo_request(circpad_negotiate_t *inp, uint8_t val); +/** Return a newly allocated circpad_negotiated with all elements set + * to zero. + */ +circpad_negotiated_t *circpad_negotiated_new(void); +/** Release all storage held by the circpad_negotiated in 'victim'. + * (Do nothing if 'victim' is NULL.) + */ +void circpad_negotiated_free(circpad_negotiated_t *victim); +/** Try to parse a circpad_negotiated from the buffer in 'input', + * using up to 'len_in' bytes from the input buffer. On success, + * return the number of bytes consumed and set *output to the newly + * allocated circpad_negotiated_t. On failure, return -2 if the input + * appears truncated, and -1 if the input is otherwise invalid. + */ +ssize_t circpad_negotiated_parse(circpad_negotiated_t **output, const uint8_t *input, const size_t len_in); +/** Return the number of bytes we expect to need to encode the + * circpad_negotiated in 'obj'. On failure, return a negative value. + * Note that this value may be an overestimate, and can even be an + * underestimate for certain unencodeable objects. + */ +ssize_t circpad_negotiated_encoded_len(const circpad_negotiated_t *obj); +/** Try to encode the circpad_negotiated from 'input' into the buffer + * at 'output', using up to 'avail' bytes of the output buffer. On + * success, return the number of bytes used. On failure, return -2 if + * the buffer was not long enough, and -1 if the input was invalid. + */ +ssize_t circpad_negotiated_encode(uint8_t *output, size_t avail, const circpad_negotiated_t *input); +/** Check whether the internal state of the circpad_negotiated in + * 'obj' is consistent. Return NULL if it is, and a short message if + * it is not. + */ +const char *circpad_negotiated_check(const circpad_negotiated_t *obj); +/** Clear any errors that were set on the object 'obj' by its setter + * functions. Return true iff errors were cleared. + */ +int circpad_negotiated_clear_errors(circpad_negotiated_t *obj); +/** Return the value of the version field of the circpad_negotiated_t + * in 'inp' + */ +uint8_t circpad_negotiated_get_version(const circpad_negotiated_t *inp); +/** Set the value of the version field of the circpad_negotiated_t in + * 'inp' to 'val'. Return 0 on success; return -1 and set the error + * code on 'inp' on failure. + */ +int circpad_negotiated_set_version(circpad_negotiated_t *inp, uint8_t val); +/** Return the value of the command field of the circpad_negotiated_t + * in 'inp' + */ +uint8_t circpad_negotiated_get_command(const circpad_negotiated_t *inp); +/** Set the value of the command field of the circpad_negotiated_t in + * 'inp' to 'val'. Return 0 on success; return -1 and set the error + * code on 'inp' on failure. + */ +int circpad_negotiated_set_command(circpad_negotiated_t *inp, uint8_t val); +/** Return the value of the response field of the circpad_negotiated_t + * in 'inp' + */ +uint8_t circpad_negotiated_get_response(const circpad_negotiated_t *inp); +/** Set the value of the response field of the circpad_negotiated_t in + * 'inp' to 'val'. Return 0 on success; return -1 and set the error + * code on 'inp' on failure. + */ +int circpad_negotiated_set_response(circpad_negotiated_t *inp, uint8_t val); +/** Return the value of the machine_type field of the + * circpad_negotiated_t in 'inp' + */ +uint8_t circpad_negotiated_get_machine_type(const circpad_negotiated_t *inp); +/** Set the value of the machine_type field of the + * circpad_negotiated_t in 'inp' to 'val'. Return 0 on success; return + * -1 and set the error code on 'inp' on failure. + */ +int circpad_negotiated_set_machine_type(circpad_negotiated_t *inp, uint8_t val); + + +#endif diff --git a/src/trunnel/circpad_negotiation.trunnel b/src/trunnel/circpad_negotiation.trunnel new file mode 100644 index 0000000000..abbc929cc5 --- /dev/null +++ b/src/trunnel/circpad_negotiation.trunnel @@ -0,0 +1,44 @@ +/* These are the padding negotiation commands */ +const CIRCPAD_COMMAND_STOP = 1; +const CIRCPAD_COMMAND_START = 2; + +/* Responses to commands */ +const CIRCPAD_RESPONSE_OK = 1; +const CIRCPAD_RESPONSE_ERR = 2; + +/* Built-in machine types */ + +/* 1) Machine that obscures circuit setup */ +const CIRCPAD_MACHINE_CIRC_SETUP = 1; + +/** + * This command tells the relay to alter its min and max netflow + * timeout range values, and send padding at that rate (resuming + * if stopped). */ +struct circpad_negotiate { + u8 version IN [0]; + u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP]; + + /** Machine type is left unbounded because we can specify + * new machines in the consensus */ + u8 machine_type; + + /** If true, send a relay_drop reply.. */ + // FIXME-MP-AP: Maybe we just say to transition to the first state + // here instead.. Also what about delay before responding? + u8 echo_request IN [0,1]; +}; + +/** + * This command tells the relay to alter its min and max netflow + * timeout range values, and send padding at that rate (resuming + * if stopped). */ +struct circpad_negotiated { + u8 version IN [0]; + u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP]; + u8 response IN [CIRCPAD_RESPONSE_OK, CIRCPAD_RESPONSE_ERR]; + + /** Machine type is left unbounded because we can specify + * new machines in the consensus */ + u8 machine_type; +}; diff --git a/src/trunnel/include.am b/src/trunnel/include.am index 03c1753e96..4f4f1d3624 100644 --- a/src/trunnel/include.am +++ b/src/trunnel/include.am @@ -11,7 +11,8 @@ TRUNNELINPUTS = \ src/trunnel/link_handshake.trunnel \ src/trunnel/pwbox.trunnel \ src/trunnel/channelpadding_negotiation.trunnel \ - src/trunner/socks5.trunnel + src/trunnel/socks5.trunnel \ + src/trunnel/circpad_negotiation.trunnel TRUNNELSOURCES = \ src/ext/trunnel/trunnel.c \ @@ -23,7 +24,9 @@ TRUNNELSOURCES = \ src/trunnel/hs/cell_introduce1.c \ src/trunnel/hs/cell_rendezvous.c \ src/trunnel/channelpadding_negotiation.c \ - src/trunnel/socks5.c + src/trunnel/socks5.c \ + src/trunnel/netinfo.c \ + src/trunnel/circpad_negotiation.c TRUNNELHEADERS = \ src/ext/trunnel/trunnel.h \ @@ -37,7 +40,9 @@ TRUNNELHEADERS = \ src/trunnel/hs/cell_introduce1.h \ src/trunnel/hs/cell_rendezvous.h \ src/trunnel/channelpadding_negotiation.h \ - src/trunnel/socks5.h + src/trunnel/socks5.h \ + src/trunnel/netinfo.h \ + src/trunnel/circpad_negotiation.h src_trunnel_libor_trunnel_a_SOURCES = $(TRUNNELSOURCES) src_trunnel_libor_trunnel_a_CPPFLAGS = \ diff --git a/src/trunnel/netinfo.c b/src/trunnel/netinfo.c new file mode 100644 index 0000000000..5d815b9b12 --- /dev/null +++ b/src/trunnel/netinfo.c @@ -0,0 +1,723 @@ +/* netinfo.c -- generated by Trunnel v1.5.2. + * https://gitweb.torproject.org/trunnel.git + * You probably shouldn't edit this file. + */ +#include <stdlib.h> +#include "trunnel-impl.h" + +#include "netinfo.h" + +#define TRUNNEL_SET_ERROR_CODE(obj) \ + do { \ + (obj)->trunnel_error_code_ = 1; \ + } while (0) + +#if defined(__COVERITY__) || defined(__clang_analyzer__) +/* If we're running a static analysis tool, we don't want it to complain + * that some of our remaining-bytes checks are dead-code. */ +int netinfo_deadcode_dummy__ = 0; +#define OR_DEADCODE_DUMMY || netinfo_deadcode_dummy__ +#else +#define OR_DEADCODE_DUMMY +#endif + +#define CHECK_REMAINING(nbytes, label) \ + do { \ + if (remaining < (nbytes) OR_DEADCODE_DUMMY) { \ + goto label; \ + } \ + } while (0) + +netinfo_addr_t * +netinfo_addr_new(void) +{ + netinfo_addr_t *val = trunnel_calloc(1, sizeof(netinfo_addr_t)); + if (NULL == val) + return NULL; + return val; +} + +/** Release all storage held inside 'obj', but do not free 'obj'. + */ +static void +netinfo_addr_clear(netinfo_addr_t *obj) +{ + (void) obj; +} + +void +netinfo_addr_free(netinfo_addr_t *obj) +{ + if (obj == NULL) + return; + netinfo_addr_clear(obj); + trunnel_memwipe(obj, sizeof(netinfo_addr_t)); + trunnel_free_(obj); +} + +uint8_t +netinfo_addr_get_addr_type(const netinfo_addr_t *inp) +{ + return inp->addr_type; +} +int +netinfo_addr_set_addr_type(netinfo_addr_t *inp, uint8_t val) +{ + inp->addr_type = val; + return 0; +} +uint8_t +netinfo_addr_get_len(const netinfo_addr_t *inp) +{ + return inp->len; +} +int +netinfo_addr_set_len(netinfo_addr_t *inp, uint8_t val) +{ + inp->len = val; + return 0; +} +uint32_t +netinfo_addr_get_addr_ipv4(const netinfo_addr_t *inp) +{ + return inp->addr_ipv4; +} +int +netinfo_addr_set_addr_ipv4(netinfo_addr_t *inp, uint32_t val) +{ + inp->addr_ipv4 = val; + return 0; +} +size_t +netinfo_addr_getlen_addr_ipv6(const netinfo_addr_t *inp) +{ + (void)inp; return 16; +} + +uint8_t +netinfo_addr_get_addr_ipv6(netinfo_addr_t *inp, size_t idx) +{ + trunnel_assert(idx < 16); + return inp->addr_ipv6[idx]; +} + +uint8_t +netinfo_addr_getconst_addr_ipv6(const netinfo_addr_t *inp, size_t idx) +{ + return netinfo_addr_get_addr_ipv6((netinfo_addr_t*)inp, idx); +} +int +netinfo_addr_set_addr_ipv6(netinfo_addr_t *inp, size_t idx, uint8_t elt) +{ + trunnel_assert(idx < 16); + inp->addr_ipv6[idx] = elt; + return 0; +} + +uint8_t * +netinfo_addr_getarray_addr_ipv6(netinfo_addr_t *inp) +{ + return inp->addr_ipv6; +} +const uint8_t * +netinfo_addr_getconstarray_addr_ipv6(const netinfo_addr_t *inp) +{ + return (const uint8_t *)netinfo_addr_getarray_addr_ipv6((netinfo_addr_t*)inp); +} +const char * +netinfo_addr_check(const netinfo_addr_t *obj) +{ + if (obj == NULL) + return "Object was NULL"; + if (obj->trunnel_error_code_) + return "A set function failed on this object"; + switch (obj->addr_type) { + + case NETINFO_ADDR_TYPE_IPV4: + break; + + case NETINFO_ADDR_TYPE_IPV6: + break; + + default: + break; + } + return NULL; +} + +ssize_t +netinfo_addr_encoded_len(const netinfo_addr_t *obj) +{ + ssize_t result = 0; + + if (NULL != netinfo_addr_check(obj)) + return -1; + + + /* Length of u8 addr_type */ + result += 1; + + /* Length of u8 len */ + result += 1; + switch (obj->addr_type) { + + case NETINFO_ADDR_TYPE_IPV4: + + /* Length of u32 addr_ipv4 */ + result += 4; + break; + + case NETINFO_ADDR_TYPE_IPV6: + + /* Length of u8 addr_ipv6[16] */ + result += 16; + break; + + default: + break; + } + return result; +} +int +netinfo_addr_clear_errors(netinfo_addr_t *obj) +{ + int r = obj->trunnel_error_code_; + obj->trunnel_error_code_ = 0; + return r; +} +ssize_t +netinfo_addr_encode(uint8_t *output, const size_t avail, const netinfo_addr_t *obj) +{ + ssize_t result = 0; + size_t written = 0; + uint8_t *ptr = output; + const char *msg; +#ifdef TRUNNEL_CHECK_ENCODED_LEN + const ssize_t encoded_len = netinfo_addr_encoded_len(obj); +#endif + + uint8_t *backptr_len = NULL; + + if (NULL != (msg = netinfo_addr_check(obj))) + goto check_failed; + +#ifdef TRUNNEL_CHECK_ENCODED_LEN + trunnel_assert(encoded_len >= 0); +#endif + + /* Encode u8 addr_type */ + trunnel_assert(written <= avail); + if (avail - written < 1) + goto truncated; + trunnel_set_uint8(ptr, (obj->addr_type)); + written += 1; ptr += 1; + + /* Encode u8 len */ + backptr_len = ptr; + trunnel_assert(written <= avail); + if (avail - written < 1) + goto truncated; + trunnel_set_uint8(ptr, (obj->len)); + written += 1; ptr += 1; + { + size_t written_before_union = written; + + /* Encode union addr[addr_type] */ + trunnel_assert(written <= avail); + switch (obj->addr_type) { + + case NETINFO_ADDR_TYPE_IPV4: + + /* Encode u32 addr_ipv4 */ + trunnel_assert(written <= avail); + if (avail - written < 4) + goto truncated; + trunnel_set_uint32(ptr, trunnel_htonl(obj->addr_ipv4)); + written += 4; ptr += 4; + break; + + case NETINFO_ADDR_TYPE_IPV6: + + /* Encode u8 addr_ipv6[16] */ + trunnel_assert(written <= avail); + if (avail - written < 16) + goto truncated; + memcpy(ptr, obj->addr_ipv6, 16); + written += 16; ptr += 16; + break; + + default: + break; + } + /* Write the length field back to len */ + trunnel_assert(written >= written_before_union); +#if UINT8_MAX < SIZE_MAX + if (written - written_before_union > UINT8_MAX) + goto check_failed; +#endif + trunnel_set_uint8(backptr_len, (written - written_before_union)); + } + + + trunnel_assert(ptr == output + written); +#ifdef TRUNNEL_CHECK_ENCODED_LEN + { + trunnel_assert(encoded_len >= 0); + trunnel_assert((size_t)encoded_len == written); + } + +#endif + + return written; + + truncated: + result = -2; + goto fail; + check_failed: + (void)msg; + result = -1; + goto fail; + fail: + trunnel_assert(result < 0); + return result; +} + +/** As netinfo_addr_parse(), but do not allocate the output object. + */ +static ssize_t +netinfo_addr_parse_into(netinfo_addr_t *obj, const uint8_t *input, const size_t len_in) +{ + const uint8_t *ptr = input; + size_t remaining = len_in; + ssize_t result = 0; + (void)result; + + /* Parse u8 addr_type */ + CHECK_REMAINING(1, truncated); + obj->addr_type = (trunnel_get_uint8(ptr)); + remaining -= 1; ptr += 1; + + /* Parse u8 len */ + CHECK_REMAINING(1, truncated); + obj->len = (trunnel_get_uint8(ptr)); + remaining -= 1; ptr += 1; + { + size_t remaining_after; + CHECK_REMAINING(obj->len, truncated); + remaining_after = remaining - obj->len; + remaining = obj->len; + + /* Parse union addr[addr_type] */ + switch (obj->addr_type) { + + case NETINFO_ADDR_TYPE_IPV4: + + /* Parse u32 addr_ipv4 */ + CHECK_REMAINING(4, fail); + obj->addr_ipv4 = trunnel_ntohl(trunnel_get_uint32(ptr)); + remaining -= 4; ptr += 4; + break; + + case NETINFO_ADDR_TYPE_IPV6: + + /* Parse u8 addr_ipv6[16] */ + CHECK_REMAINING(16, fail); + memcpy(obj->addr_ipv6, ptr, 16); + remaining -= 16; ptr += 16; + break; + + default: + /* Skip to end of union */ + ptr += remaining; remaining = 0; + break; + } + if (remaining != 0) + goto fail; + remaining = remaining_after; + } + trunnel_assert(ptr + remaining == input + len_in); + return len_in - remaining; + + truncated: + return -2; + fail: + result = -1; + return result; +} + +ssize_t +netinfo_addr_parse(netinfo_addr_t **output, const uint8_t *input, const size_t len_in) +{ + ssize_t result; + *output = netinfo_addr_new(); + if (NULL == *output) + return -1; + result = netinfo_addr_parse_into(*output, input, len_in); + if (result < 0) { + netinfo_addr_free(*output); + *output = NULL; + } + return result; +} +netinfo_cell_t * +netinfo_cell_new(void) +{ + netinfo_cell_t *val = trunnel_calloc(1, sizeof(netinfo_cell_t)); + if (NULL == val) + return NULL; + return val; +} + +/** Release all storage held inside 'obj', but do not free 'obj'. + */ +static void +netinfo_cell_clear(netinfo_cell_t *obj) +{ + (void) obj; + netinfo_addr_free(obj->other_addr); + obj->other_addr = NULL; + { + + unsigned idx; + for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->my_addrs); ++idx) { + netinfo_addr_free(TRUNNEL_DYNARRAY_GET(&obj->my_addrs, idx)); + } + } + TRUNNEL_DYNARRAY_WIPE(&obj->my_addrs); + TRUNNEL_DYNARRAY_CLEAR(&obj->my_addrs); +} + +void +netinfo_cell_free(netinfo_cell_t *obj) +{ + if (obj == NULL) + return; + netinfo_cell_clear(obj); + trunnel_memwipe(obj, sizeof(netinfo_cell_t)); + trunnel_free_(obj); +} + +uint32_t +netinfo_cell_get_timestamp(const netinfo_cell_t *inp) +{ + return inp->timestamp; +} +int +netinfo_cell_set_timestamp(netinfo_cell_t *inp, uint32_t val) +{ + inp->timestamp = val; + return 0; +} +struct netinfo_addr_st * +netinfo_cell_get_other_addr(netinfo_cell_t *inp) +{ + return inp->other_addr; +} +const struct netinfo_addr_st * +netinfo_cell_getconst_other_addr(const netinfo_cell_t *inp) +{ + return netinfo_cell_get_other_addr((netinfo_cell_t*) inp); +} +int +netinfo_cell_set_other_addr(netinfo_cell_t *inp, struct netinfo_addr_st *val) +{ + if (inp->other_addr && inp->other_addr != val) + netinfo_addr_free(inp->other_addr); + return netinfo_cell_set0_other_addr(inp, val); +} +int +netinfo_cell_set0_other_addr(netinfo_cell_t *inp, struct netinfo_addr_st *val) +{ + inp->other_addr = val; + return 0; +} +uint8_t +netinfo_cell_get_n_my_addrs(const netinfo_cell_t *inp) +{ + return inp->n_my_addrs; +} +int +netinfo_cell_set_n_my_addrs(netinfo_cell_t *inp, uint8_t val) +{ + inp->n_my_addrs = val; + return 0; +} +size_t +netinfo_cell_getlen_my_addrs(const netinfo_cell_t *inp) +{ + return TRUNNEL_DYNARRAY_LEN(&inp->my_addrs); +} + +struct netinfo_addr_st * +netinfo_cell_get_my_addrs(netinfo_cell_t *inp, size_t idx) +{ + return TRUNNEL_DYNARRAY_GET(&inp->my_addrs, idx); +} + + const struct netinfo_addr_st * +netinfo_cell_getconst_my_addrs(const netinfo_cell_t *inp, size_t idx) +{ + return netinfo_cell_get_my_addrs((netinfo_cell_t*)inp, idx); +} +int +netinfo_cell_set_my_addrs(netinfo_cell_t *inp, size_t idx, struct netinfo_addr_st * elt) +{ + netinfo_addr_t *oldval = TRUNNEL_DYNARRAY_GET(&inp->my_addrs, idx); + if (oldval && oldval != elt) + netinfo_addr_free(oldval); + return netinfo_cell_set0_my_addrs(inp, idx, elt); +} +int +netinfo_cell_set0_my_addrs(netinfo_cell_t *inp, size_t idx, struct netinfo_addr_st * elt) +{ + TRUNNEL_DYNARRAY_SET(&inp->my_addrs, idx, elt); + return 0; +} +int +netinfo_cell_add_my_addrs(netinfo_cell_t *inp, struct netinfo_addr_st * elt) +{ +#if SIZE_MAX >= UINT8_MAX + if (inp->my_addrs.n_ == UINT8_MAX) + goto trunnel_alloc_failed; +#endif + TRUNNEL_DYNARRAY_ADD(struct netinfo_addr_st *, &inp->my_addrs, elt, {}); + return 0; + trunnel_alloc_failed: + TRUNNEL_SET_ERROR_CODE(inp); + return -1; +} + +struct netinfo_addr_st * * +netinfo_cell_getarray_my_addrs(netinfo_cell_t *inp) +{ + return inp->my_addrs.elts_; +} +const struct netinfo_addr_st * const * +netinfo_cell_getconstarray_my_addrs(const netinfo_cell_t *inp) +{ + return (const struct netinfo_addr_st * const *)netinfo_cell_getarray_my_addrs((netinfo_cell_t*)inp); +} +int +netinfo_cell_setlen_my_addrs(netinfo_cell_t *inp, size_t newlen) +{ + struct netinfo_addr_st * *newptr; +#if UINT8_MAX < SIZE_MAX + if (newlen > UINT8_MAX) + goto trunnel_alloc_failed; +#endif + newptr = trunnel_dynarray_setlen(&inp->my_addrs.allocated_, + &inp->my_addrs.n_, inp->my_addrs.elts_, newlen, + sizeof(inp->my_addrs.elts_[0]), (trunnel_free_fn_t) netinfo_addr_free, + &inp->trunnel_error_code_); + if (newlen != 0 && newptr == NULL) + goto trunnel_alloc_failed; + inp->my_addrs.elts_ = newptr; + return 0; + trunnel_alloc_failed: + TRUNNEL_SET_ERROR_CODE(inp); + return -1; +} +const char * +netinfo_cell_check(const netinfo_cell_t *obj) +{ + if (obj == NULL) + return "Object was NULL"; + if (obj->trunnel_error_code_) + return "A set function failed on this object"; + { + const char *msg; + if (NULL != (msg = netinfo_addr_check(obj->other_addr))) + return msg; + } + { + const char *msg; + + unsigned idx; + for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->my_addrs); ++idx) { + if (NULL != (msg = netinfo_addr_check(TRUNNEL_DYNARRAY_GET(&obj->my_addrs, idx)))) + return msg; + } + } + if (TRUNNEL_DYNARRAY_LEN(&obj->my_addrs) != obj->n_my_addrs) + return "Length mismatch for my_addrs"; + return NULL; +} + +ssize_t +netinfo_cell_encoded_len(const netinfo_cell_t *obj) +{ + ssize_t result = 0; + + if (NULL != netinfo_cell_check(obj)) + return -1; + + + /* Length of u32 timestamp */ + result += 4; + + /* Length of struct netinfo_addr other_addr */ + result += netinfo_addr_encoded_len(obj->other_addr); + + /* Length of u8 n_my_addrs */ + result += 1; + + /* Length of struct netinfo_addr my_addrs[n_my_addrs] */ + { + + unsigned idx; + for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->my_addrs); ++idx) { + result += netinfo_addr_encoded_len(TRUNNEL_DYNARRAY_GET(&obj->my_addrs, idx)); + } + } + return result; +} +int +netinfo_cell_clear_errors(netinfo_cell_t *obj) +{ + int r = obj->trunnel_error_code_; + obj->trunnel_error_code_ = 0; + return r; +} +ssize_t +netinfo_cell_encode(uint8_t *output, const size_t avail, const netinfo_cell_t *obj) +{ + ssize_t result = 0; + size_t written = 0; + uint8_t *ptr = output; + const char *msg; +#ifdef TRUNNEL_CHECK_ENCODED_LEN + const ssize_t encoded_len = netinfo_cell_encoded_len(obj); +#endif + + if (NULL != (msg = netinfo_cell_check(obj))) + goto check_failed; + +#ifdef TRUNNEL_CHECK_ENCODED_LEN + trunnel_assert(encoded_len >= 0); +#endif + + /* Encode u32 timestamp */ + trunnel_assert(written <= avail); + if (avail - written < 4) + goto truncated; + trunnel_set_uint32(ptr, trunnel_htonl(obj->timestamp)); + written += 4; ptr += 4; + + /* Encode struct netinfo_addr other_addr */ + trunnel_assert(written <= avail); + result = netinfo_addr_encode(ptr, avail - written, obj->other_addr); + if (result < 0) + goto fail; /* XXXXXXX !*/ + written += result; ptr += result; + + /* Encode u8 n_my_addrs */ + trunnel_assert(written <= avail); + if (avail - written < 1) + goto truncated; + trunnel_set_uint8(ptr, (obj->n_my_addrs)); + written += 1; ptr += 1; + + /* Encode struct netinfo_addr my_addrs[n_my_addrs] */ + { + + unsigned idx; + for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->my_addrs); ++idx) { + trunnel_assert(written <= avail); + result = netinfo_addr_encode(ptr, avail - written, TRUNNEL_DYNARRAY_GET(&obj->my_addrs, idx)); + if (result < 0) + goto fail; /* XXXXXXX !*/ + written += result; ptr += result; + } + } + + + trunnel_assert(ptr == output + written); +#ifdef TRUNNEL_CHECK_ENCODED_LEN + { + trunnel_assert(encoded_len >= 0); + trunnel_assert((size_t)encoded_len == written); + } + +#endif + + return written; + + truncated: + result = -2; + goto fail; + check_failed: + (void)msg; + result = -1; + goto fail; + fail: + trunnel_assert(result < 0); + return result; +} + +/** As netinfo_cell_parse(), but do not allocate the output object. + */ +static ssize_t +netinfo_cell_parse_into(netinfo_cell_t *obj, const uint8_t *input, const size_t len_in) +{ + const uint8_t *ptr = input; + size_t remaining = len_in; + ssize_t result = 0; + (void)result; + + /* Parse u32 timestamp */ + CHECK_REMAINING(4, truncated); + obj->timestamp = trunnel_ntohl(trunnel_get_uint32(ptr)); + remaining -= 4; ptr += 4; + + /* Parse struct netinfo_addr other_addr */ + result = netinfo_addr_parse(&obj->other_addr, ptr, remaining); + if (result < 0) + goto relay_fail; + trunnel_assert((size_t)result <= remaining); + remaining -= result; ptr += result; + + /* Parse u8 n_my_addrs */ + CHECK_REMAINING(1, truncated); + obj->n_my_addrs = (trunnel_get_uint8(ptr)); + remaining -= 1; ptr += 1; + + /* Parse struct netinfo_addr my_addrs[n_my_addrs] */ + TRUNNEL_DYNARRAY_EXPAND(netinfo_addr_t *, &obj->my_addrs, obj->n_my_addrs, {}); + { + netinfo_addr_t * elt; + unsigned idx; + for (idx = 0; idx < obj->n_my_addrs; ++idx) { + result = netinfo_addr_parse(&elt, ptr, remaining); + if (result < 0) + goto relay_fail; + trunnel_assert((size_t)result <= remaining); + remaining -= result; ptr += result; + TRUNNEL_DYNARRAY_ADD(netinfo_addr_t *, &obj->my_addrs, elt, {netinfo_addr_free(elt);}); + } + } + trunnel_assert(ptr + remaining == input + len_in); + return len_in - remaining; + + truncated: + return -2; + relay_fail: + trunnel_assert(result < 0); + return result; + trunnel_alloc_failed: + return -1; +} + +ssize_t +netinfo_cell_parse(netinfo_cell_t **output, const uint8_t *input, const size_t len_in) +{ + ssize_t result; + *output = netinfo_cell_new(); + if (NULL == *output) + return -1; + result = netinfo_cell_parse_into(*output, input, len_in); + if (result < 0) { + netinfo_cell_free(*output); + *output = NULL; + } + return result; +} diff --git a/src/trunnel/netinfo.h b/src/trunnel/netinfo.h new file mode 100644 index 0000000000..ac46e603ba --- /dev/null +++ b/src/trunnel/netinfo.h @@ -0,0 +1,226 @@ +/* netinfo.h -- generated by Trunnel v1.5.2. + * https://gitweb.torproject.org/trunnel.git + * You probably shouldn't edit this file. + */ +#ifndef TRUNNEL_NETINFO_H +#define TRUNNEL_NETINFO_H + +#include <stdint.h> +#include "trunnel.h" + +#define NETINFO_ADDR_TYPE_IPV4 4 +#define NETINFO_ADDR_TYPE_IPV6 6 +#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_NETINFO_ADDR) +struct netinfo_addr_st { + uint8_t addr_type; + uint8_t len; + uint32_t addr_ipv4; + uint8_t addr_ipv6[16]; + uint8_t trunnel_error_code_; +}; +#endif +typedef struct netinfo_addr_st netinfo_addr_t; +#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_NETINFO_CELL) +struct netinfo_cell_st { + uint32_t timestamp; + struct netinfo_addr_st *other_addr; + uint8_t n_my_addrs; + TRUNNEL_DYNARRAY_HEAD(, struct netinfo_addr_st *) my_addrs; + uint8_t trunnel_error_code_; +}; +#endif +typedef struct netinfo_cell_st netinfo_cell_t; +/** Return a newly allocated netinfo_addr with all elements set to + * zero. + */ +netinfo_addr_t *netinfo_addr_new(void); +/** Release all storage held by the netinfo_addr in 'victim'. (Do + * nothing if 'victim' is NULL.) + */ +void netinfo_addr_free(netinfo_addr_t *victim); +/** Try to parse a netinfo_addr from the buffer in 'input', using up + * to 'len_in' bytes from the input buffer. On success, return the + * number of bytes consumed and set *output to the newly allocated + * netinfo_addr_t. On failure, return -2 if the input appears + * truncated, and -1 if the input is otherwise invalid. + */ +ssize_t netinfo_addr_parse(netinfo_addr_t **output, const uint8_t *input, const size_t len_in); +/** Return the number of bytes we expect to need to encode the + * netinfo_addr in 'obj'. On failure, return a negative value. Note + * that this value may be an overestimate, and can even be an + * underestimate for certain unencodeable objects. + */ +ssize_t netinfo_addr_encoded_len(const netinfo_addr_t *obj); +/** Try to encode the netinfo_addr from 'input' into the buffer at + * 'output', using up to 'avail' bytes of the output buffer. On + * success, return the number of bytes used. On failure, return -2 if + * the buffer was not long enough, and -1 if the input was invalid. + */ +ssize_t netinfo_addr_encode(uint8_t *output, size_t avail, const netinfo_addr_t *input); +/** Check whether the internal state of the netinfo_addr in 'obj' is + * consistent. Return NULL if it is, and a short message if it is not. + */ +const char *netinfo_addr_check(const netinfo_addr_t *obj); +/** Clear any errors that were set on the object 'obj' by its setter + * functions. Return true iff errors were cleared. + */ +int netinfo_addr_clear_errors(netinfo_addr_t *obj); +/** Return the value of the addr_type field of the netinfo_addr_t in + * 'inp' + */ +uint8_t netinfo_addr_get_addr_type(const netinfo_addr_t *inp); +/** Set the value of the addr_type field of the netinfo_addr_t in + * 'inp' to 'val'. Return 0 on success; return -1 and set the error + * code on 'inp' on failure. + */ +int netinfo_addr_set_addr_type(netinfo_addr_t *inp, uint8_t val); +/** Return the value of the len field of the netinfo_addr_t in 'inp' + */ +uint8_t netinfo_addr_get_len(const netinfo_addr_t *inp); +/** Set the value of the len field of the netinfo_addr_t in 'inp' to + * 'val'. Return 0 on success; return -1 and set the error code on + * 'inp' on failure. + */ +int netinfo_addr_set_len(netinfo_addr_t *inp, uint8_t val); +/** Return the value of the addr_ipv4 field of the netinfo_addr_t in + * 'inp' + */ +uint32_t netinfo_addr_get_addr_ipv4(const netinfo_addr_t *inp); +/** Set the value of the addr_ipv4 field of the netinfo_addr_t in + * 'inp' to 'val'. Return 0 on success; return -1 and set the error + * code on 'inp' on failure. + */ +int netinfo_addr_set_addr_ipv4(netinfo_addr_t *inp, uint32_t val); +/** Return the (constant) length of the array holding the addr_ipv6 + * field of the netinfo_addr_t in 'inp'. + */ +size_t netinfo_addr_getlen_addr_ipv6(const netinfo_addr_t *inp); +/** Return the element at position 'idx' of the fixed array field + * addr_ipv6 of the netinfo_addr_t in 'inp'. + */ +uint8_t netinfo_addr_get_addr_ipv6(netinfo_addr_t *inp, size_t idx); +/** As netinfo_addr_get_addr_ipv6, but take and return a const pointer + */ +uint8_t netinfo_addr_getconst_addr_ipv6(const netinfo_addr_t *inp, size_t idx); +/** Change the element at position 'idx' of the fixed array field + * addr_ipv6 of the netinfo_addr_t in 'inp', so that it will hold the + * value 'elt'. + */ +int netinfo_addr_set_addr_ipv6(netinfo_addr_t *inp, size_t idx, uint8_t elt); +/** Return a pointer to the 16-element array field addr_ipv6 of 'inp'. + */ +uint8_t * netinfo_addr_getarray_addr_ipv6(netinfo_addr_t *inp); +/** As netinfo_addr_get_addr_ipv6, but take and return a const pointer + */ +const uint8_t * netinfo_addr_getconstarray_addr_ipv6(const netinfo_addr_t *inp); +/** Return a newly allocated netinfo_cell with all elements set to + * zero. + */ +netinfo_cell_t *netinfo_cell_new(void); +/** Release all storage held by the netinfo_cell in 'victim'. (Do + * nothing if 'victim' is NULL.) + */ +void netinfo_cell_free(netinfo_cell_t *victim); +/** Try to parse a netinfo_cell from the buffer in 'input', using up + * to 'len_in' bytes from the input buffer. On success, return the + * number of bytes consumed and set *output to the newly allocated + * netinfo_cell_t. On failure, return -2 if the input appears + * truncated, and -1 if the input is otherwise invalid. + */ +ssize_t netinfo_cell_parse(netinfo_cell_t **output, const uint8_t *input, const size_t len_in); +/** Return the number of bytes we expect to need to encode the + * netinfo_cell in 'obj'. On failure, return a negative value. Note + * that this value may be an overestimate, and can even be an + * underestimate for certain unencodeable objects. + */ +ssize_t netinfo_cell_encoded_len(const netinfo_cell_t *obj); +/** Try to encode the netinfo_cell from 'input' into the buffer at + * 'output', using up to 'avail' bytes of the output buffer. On + * success, return the number of bytes used. On failure, return -2 if + * the buffer was not long enough, and -1 if the input was invalid. + */ +ssize_t netinfo_cell_encode(uint8_t *output, size_t avail, const netinfo_cell_t *input); +/** Check whether the internal state of the netinfo_cell in 'obj' is + * consistent. Return NULL if it is, and a short message if it is not. + */ +const char *netinfo_cell_check(const netinfo_cell_t *obj); +/** Clear any errors that were set on the object 'obj' by its setter + * functions. Return true iff errors were cleared. + */ +int netinfo_cell_clear_errors(netinfo_cell_t *obj); +/** Return the value of the timestamp field of the netinfo_cell_t in + * 'inp' + */ +uint32_t netinfo_cell_get_timestamp(const netinfo_cell_t *inp); +/** Set the value of the timestamp field of the netinfo_cell_t in + * 'inp' to 'val'. Return 0 on success; return -1 and set the error + * code on 'inp' on failure. + */ +int netinfo_cell_set_timestamp(netinfo_cell_t *inp, uint32_t val); +/** Return the value of the other_addr field of the netinfo_cell_t in + * 'inp' + */ +struct netinfo_addr_st * netinfo_cell_get_other_addr(netinfo_cell_t *inp); +/** As netinfo_cell_get_other_addr, but take and return a const + * pointer + */ +const struct netinfo_addr_st * netinfo_cell_getconst_other_addr(const netinfo_cell_t *inp); +/** Set the value of the other_addr field of the netinfo_cell_t in + * 'inp' to 'val'. Free the old value if any. Steals the referenceto + * 'val'.Return 0 on success; return -1 and set the error code on + * 'inp' on failure. + */ +int netinfo_cell_set_other_addr(netinfo_cell_t *inp, struct netinfo_addr_st *val); +/** As netinfo_cell_set_other_addr, but does not free the previous + * value. + */ +int netinfo_cell_set0_other_addr(netinfo_cell_t *inp, struct netinfo_addr_st *val); +/** Return the value of the n_my_addrs field of the netinfo_cell_t in + * 'inp' + */ +uint8_t netinfo_cell_get_n_my_addrs(const netinfo_cell_t *inp); +/** Set the value of the n_my_addrs field of the netinfo_cell_t in + * 'inp' to 'val'. Return 0 on success; return -1 and set the error + * code on 'inp' on failure. + */ +int netinfo_cell_set_n_my_addrs(netinfo_cell_t *inp, uint8_t val); +/** Return the length of the dynamic array holding the my_addrs field + * of the netinfo_cell_t in 'inp'. + */ +size_t netinfo_cell_getlen_my_addrs(const netinfo_cell_t *inp); +/** Return the element at position 'idx' of the dynamic array field + * my_addrs of the netinfo_cell_t in 'inp'. + */ +struct netinfo_addr_st * netinfo_cell_get_my_addrs(netinfo_cell_t *inp, size_t idx); +/** As netinfo_cell_get_my_addrs, but take and return a const pointer + */ + const struct netinfo_addr_st * netinfo_cell_getconst_my_addrs(const netinfo_cell_t *inp, size_t idx); +/** Change the element at position 'idx' of the dynamic array field + * my_addrs of the netinfo_cell_t in 'inp', so that it will hold the + * value 'elt'. Free the previous value, if any. + */ +int netinfo_cell_set_my_addrs(netinfo_cell_t *inp, size_t idx, struct netinfo_addr_st * elt); +/** As netinfo_cell_set_my_addrs, but does not free the previous + * value. + */ +int netinfo_cell_set0_my_addrs(netinfo_cell_t *inp, size_t idx, struct netinfo_addr_st * elt); +/** Append a new element 'elt' to the dynamic array field my_addrs of + * the netinfo_cell_t in 'inp'. + */ +int netinfo_cell_add_my_addrs(netinfo_cell_t *inp, struct netinfo_addr_st * elt); +/** Return a pointer to the variable-length array field my_addrs of + * 'inp'. + */ +struct netinfo_addr_st * * netinfo_cell_getarray_my_addrs(netinfo_cell_t *inp); +/** As netinfo_cell_get_my_addrs, but take and return a const pointer + */ +const struct netinfo_addr_st * const * netinfo_cell_getconstarray_my_addrs(const netinfo_cell_t *inp); +/** Change the length of the variable-length array field my_addrs of + * 'inp' to 'newlen'.Fill extra elements with NULL; free removed + * elements. Return 0 on success; return -1 and set the error code on + * 'inp' on failure. + */ +int netinfo_cell_setlen_my_addrs(netinfo_cell_t *inp, size_t newlen); + + +#endif diff --git a/src/trunnel/netinfo.trunnel b/src/trunnel/netinfo.trunnel new file mode 100644 index 0000000000..2c4b7a7591 --- /dev/null +++ b/src/trunnel/netinfo.trunnel @@ -0,0 +1,24 @@ +// Warning: make sure these values are consistent with RESOLVED_TYPE_* +// constants in Tor code and numbers in Section 6.4 of tor-spec.txt. + +const NETINFO_ADDR_TYPE_IPV4 = 4; +const NETINFO_ADDR_TYPE_IPV6 = 6; + +struct netinfo_addr { + u8 addr_type; + u8 len; + union addr[addr_type] with length len { + NETINFO_ADDR_TYPE_IPV4: u32 ipv4; + NETINFO_ADDR_TYPE_IPV6: u8 ipv6[16]; + default: ignore; + }; + +} + +struct netinfo_cell { + u32 timestamp; + struct netinfo_addr other_addr; + u8 n_my_addrs; + struct netinfo_addr my_addrs[n_my_addrs]; +} + diff --git a/src/trunnel/socks5.c b/src/trunnel/socks5.c index 9e5f6fcfed..057a52b042 100644 --- a/src/trunnel/socks5.c +++ b/src/trunnel/socks5.c @@ -1694,7 +1694,6 @@ socks4_server_reply_new(void) socks4_server_reply_t *val = trunnel_calloc(1, sizeof(socks4_server_reply_t)); if (NULL == val) return NULL; - val->version = 4; return val; } @@ -1724,7 +1723,7 @@ socks4_server_reply_get_version(const socks4_server_reply_t *inp) int socks4_server_reply_set_version(socks4_server_reply_t *inp, uint8_t val) { - if (! ((val == 4))) { + if (! ((val == 0 || val == 4))) { TRUNNEL_SET_ERROR_CODE(inp); return -1; } @@ -1771,7 +1770,7 @@ socks4_server_reply_check(const socks4_server_reply_t *obj) return "Object was NULL"; if (obj->trunnel_error_code_) return "A set function failed on this object"; - if (! (obj->version == 4)) + if (! (obj->version == 0 || obj->version == 4)) return "Integer out of bounds"; return NULL; } @@ -1785,7 +1784,7 @@ socks4_server_reply_encoded_len(const socks4_server_reply_t *obj) return -1; - /* Length of u8 version IN [4] */ + /* Length of u8 version IN [0, 4] */ result += 1; /* Length of u8 status */ @@ -1823,7 +1822,7 @@ socks4_server_reply_encode(uint8_t *output, const size_t avail, const socks4_ser trunnel_assert(encoded_len >= 0); #endif - /* Encode u8 version IN [4] */ + /* Encode u8 version IN [0, 4] */ trunnel_assert(written <= avail); if (avail - written < 1) goto truncated; @@ -1886,11 +1885,11 @@ socks4_server_reply_parse_into(socks4_server_reply_t *obj, const uint8_t *input, ssize_t result = 0; (void)result; - /* Parse u8 version IN [4] */ + /* Parse u8 version IN [0, 4] */ CHECK_REMAINING(1, truncated); obj->version = (trunnel_get_uint8(ptr)); remaining -= 1; ptr += 1; - if (! (obj->version == 4)) + if (! (obj->version == 0 || obj->version == 4)) goto fail; /* Parse u8 status */ diff --git a/src/trunnel/socks5.trunnel b/src/trunnel/socks5.trunnel index b86ec03b9d..17fa2ed996 100644 --- a/src/trunnel/socks5.trunnel +++ b/src/trunnel/socks5.trunnel @@ -86,7 +86,7 @@ struct socks4_client_request { } struct socks4_server_reply { - u8 version IN [4]; + u8 version IN [0,4]; u8 status; u16 port; u32 addr; diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h index baad733d0f..bf746a19ac 100644 --- a/src/win32/orconfig.h +++ b/src/win32/orconfig.h @@ -218,7 +218,7 @@ #define USING_TWOS_COMPLEMENT /* Version number of package */ -#define VERSION "0.3.5.9-dev" +#define VERSION "0.4.0.6-dev" |