diff options
434 files changed, 15927 insertions, 4589 deletions
diff --git a/.appveyor.yml b/.appveyor.yml index 69cbd8e2d6..b838055f46 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -76,6 +76,8 @@ test_script: $buildpath = @("C:\msys64\${env:compiler_path}\bin") + $oldpath $env:Path = $buildpath -join ';' Set-Location "${env:build}" + Copy-Item "C:/msys64/${env:compiler_path}/bin/libssp-0.dll" -Destination "${env:build}/src/test" + Copy-Item "C:/msys64/${env:compiler_path}/bin/zlib1.dll" -Destination "${env:build}/src/test" Execute-Bash "VERBOSE=1 make -j2 check" } diff --git a/.gitignore b/.gitignore index f1ce903a11..6a49285b8a 100644 --- a/.gitignore +++ b/.gitignore @@ -158,6 +158,8 @@ uptime-*.json # /src/lib /src/lib/libcurve25519_donna.a +/src/lib/libtor-buf.a +/src/lib/libtor-buf-testing.a /src/lib/libtor-compress.a /src/lib/libtor-compress-testing.a /src/lib/libtor-container.a @@ -213,6 +215,8 @@ uptime-*.json /src/lib/libtor-tls.a /src/lib/libtor-tls-testing.a /src/lib/libtor-trace.a +/src/lib/libtor-version.a +/src/lib/libtor-version-testing.a /src/lib/libtor-wallclock.a /src/lib/libtor-wallclock-testing.a @@ -240,7 +244,7 @@ uptime-*.json /src/test/test /src/test/test-slow /src/test/test-bt-cl -/src/test/test-child +/src/test/test-process /src/test/test-memwipe /src/test/test-ntor-cl /src/test/test-hs-ntor-cl @@ -250,7 +254,7 @@ uptime-*.json /src/test/test.exe /src/test/test-slow.exe /src/test/test-bt-cl.exe -/src/test/test-child.exe +/src/test/test-process.exe /src/test/test-ntor-cl.exe /src/test/test-hs-ntor-cl.exe /src/test/test-memwipe.exe diff --git a/.travis.yml b/.travis.yml index b5713d6933..2ea529e252 100644 --- a/.travis.yml +++ b/.travis.yml @@ -193,6 +193,9 @@ install: - if [[ "$RUST_OPTIONS" != "" ]]; then rustup --version; fi - if [[ "$RUST_OPTIONS" != "" ]]; then rustc --version; fi - if [[ "$RUST_OPTIONS" != "" ]]; then cargo --version; fi + ## Get python version + - python --version + ## run stem tests if they are enabled. - if [[ "$TEST_STEM" != "" ]]; then pushd stem; python -c "from stem import stem; print(stem.__version__);"; git log -1; popd; fi script: @@ -1,3 +1,154 @@ +Changes in version 0.3.5.6-rc - 2018-12-18 + Tor 0.3.5.6-rc fixes numerous small bugs in earlier versions of Tor. + It is the first release candidate in the 0.3.5.x series; if no further + huge bugs are found, our next release may be the stable 0.3.5.x. + + o Minor features (continuous integration, Windows): + - Always show the configure and test logs, and upload them as build + artifacts, when building for Windows using Appveyor CI. + Implements 28459. + + o Minor features (fallback directory list): + - Replace the 150 fallbacks originally introduced in Tor + 0.3.3.1-alpha in January 2018 (of which ~115 were still + functional), with a list of 157 fallbacks (92 new, 65 existing, 85 + removed) generated in December 2018. Closes ticket 24803. + + o Minor features (geoip): + - Update geoip and geoip6 to the December 5 2018 Maxmind GeoLite2 + Country database. Closes ticket 28744. + + o Minor bugfixes (compilation): + - Add missing dependency on libgdi32.dll for tor-print-ed-signing- + cert.exe on Windows. Fixes bug 28485; bugfix on 0.3.5.1-alpha. + + o Minor bugfixes (continuous integration, Windows): + - Explicitly specify the path to the OpenSSL library and do not + download OpenSSL from Pacman, but instead use the library that is + already provided by AppVeyor. Fixes bug 28574; bugfix on master. + + o Minor bugfixes (onion service v3): + - When deleting an ephemeral onion service (DEL_ONION), do not close + any rendezvous circuits in order to let the existing client + connections finish by themselves or closed by the application. The + HS v2 is doing that already so now we have the same behavior for + all versions. Fixes bug 28619; bugfix on 0.3.3.1-alpha. + + o Minor bugfixes (restart-in-process, boostrap): + - Add missing resets of bootstrap tracking state when shutting down + (regression caused by ticket 27169). Fixes bug 28524; bugfix + on 0.3.5.1-alpha. + + o Minor bugfixes (testing): + - Use a separate DataDirectory for the test_rebind script. + Previously, this script would run using the default DataDirectory, + and sometimes fail. Fixes bug 28562; bugfix on 0.3.5.1-alpha. + Patch from Taylor R Campbell. + - Stop leaking memory in an entry guard unit test. Fixes bug 28554; + bugfix on 0.3.0.1-alpha. + + o Minor bugfixes (Windows): + - Correctly identify Windows 8.1, Windows 10, and Windows Server + 2008 and later from their NT versions. Fixes bug 28096; bugfix on + 0.2.2.34; reported by Keifer Bly. + - On recent Windows versions, the GetVersionEx() function may report + an earlier Windows version than the running OS. To avoid user + confusion, add "[or later]" to Tor's version string on affected + versions of Windows. Fixes bug 28096; bugfix on 0.2.2.34; reported + by Keifer Bly. + - Remove Windows versions that were never supported by the + GetVersionEx() function. Stop duplicating the latest Windows + version in get_uname(). Fixes bug 28096; bugfix on 0.2.2.34; + reported by Keifer Bly. + + o Testing: + - Increase logging and tag all log entries with timestamps in + test_rebind.py. Provides diagnostics for issue 28229. + + o Code simplification and refactoring (shared random, dirauth): + - Change many tor_assert() to use BUG() instead. The idea is to not + crash a dirauth but rather scream loudly with a stacktrace and let + it continue run. The shared random subsystem is very resilient and + if anything wrong happens with it, at worst a non coherent value + will be put in the vote and discarded by the other authorities. + Closes ticket 19566. + + o Documentation (onion services): + - Document in the man page that changing ClientOnionAuthDir value or + adding a new file in the directory will not work at runtime upon + sending a HUP if Sandbox 1. Closes ticket 28128. + - Note in the man page that the only real way to fully revoke an + onion service v3 client authorization is by restarting the tor + process. Closes ticket 28275. + + +Changes in version 0.3.5.5-alpha - 2018-11-16 + Tor 0.3.5.5-alpha includes numerous bugfixes on earlier releases, + including several that we hope to backport to older release series in + the future. + + o Major bugfixes (OpenSSL, portability): + - Fix our usage of named groups when running as a TLS 1.3 client in + OpenSSL 1.1.1. Previously, we only initialized EC groups when + running as a relay, which caused clients to fail to negotiate TLS + 1.3 with relays. Fixes bug 28245; bugfix on 0.2.9.15 (when TLS 1.3 + support was added). + + o Minor features (geoip): + - Update geoip and geoip6 to the November 6 2018 Maxmind GeoLite2 + Country database. Closes ticket 28395. + + o Minor bugfixes (compilation): + - Initialize a variable unconditionally in aes_new_cipher(), since + some compilers cannot tell that we always initialize it before + use. Fixes bug 28413; bugfix on 0.2.9.3-alpha. + + o Minor bugfixes (connection, relay): + - Avoid a logging a BUG() stacktrace when closing connection held + open because the write side is rate limited but not the read side. + Now, the connection read side is simply shut down until Tor is + able to flush the connection and close it. Fixes bug 27750; bugfix + on 0.3.4.1-alpha. + + o Minor bugfixes (continuous integration, Windows): + - Manually configure the zstd compiler options, when building using + mingw on Appveyor Windows CI. The MSYS2 mingw zstd package does + not come with a pkg-config file. Fixes bug 28454; bugfix + on 0.3.4.1-alpha. + - Stop using an external OpenSSL install, and stop installing MSYS2 + packages, when building using mingw on Appveyor Windows CI. Fixes + bug 28399; bugfix on 0.3.4.1-alpha. + + o Minor bugfixes (documentation): + - Make Doxygen work again after the code movement in the 0.3.5 + source tree. Fixes bug 28435; bugfix on 0.3.5.1-alpha. + + o Minor bugfixes (Linux seccomp2 sandbox): + - Permit the "shutdown()" system call, which is apparently used by + OpenSSL under some circumstances. Fixes bug 28183; bugfix + on 0.2.5.1-alpha. + + o Minor bugfixes (logging): + - Stop talking about the Named flag in log messages. Clients have + ignored the Named flag since 0.3.2. Fixes bug 28441; bugfix + on 0.3.2.1-alpha. + + o Minor bugfixes (memory leaks): + - Fix a harmless memory leak in libtorrunner.a. Fixes bug 28419; + bugfix on 0.3.3.1-alpha. Patch from Martin Kepplinger. + + o Minor bugfixes (onion services): + - On an intro point for a version 3 onion service, stop closing + introduction circuits on an NACK. This lets the client decide + whether to reuse the circuit or discard it. Previously, we closed + intro circuits when sending NACKs. Fixes bug 27841; bugfix on + 0.3.2.1-alpha. Patch by Neel Chaunan. + - When replacing a descriptor in the client cache, make sure to + close all client introduction circuits for the old descriptor, so + we don't end up with unusable leftover circuits. Fixes bug 27471; + bugfix on 0.3.2.1-alpha. + + Changes in version 0.3.5.4-alpha - 2018-11-08 Tor 0.3.5.4-alpha includes numerous bugfixes on earlier versions and improves our continuous integration support. It continues our attempts diff --git a/Makefile.am b/Makefile.am index c759fffea3..84050f8b34 100644 --- a/Makefile.am +++ b/Makefile.am @@ -31,9 +31,7 @@ TESTING_TOR_BINARY=$(top_builddir)/src/app/tor$(EXEEXT) endif if USE_RUST -## this MUST be $(), otherwise am__DEPENDENCIES will not track it -rust_ldadd=$(top_builddir)/$(TOR_RUST_LIB_PATH) \ - $(TOR_RUST_EXTRA_LIBS) +rust_ldadd=$(top_builddir)/$(TOR_RUST_LIB_PATH) else rust_ldadd= endif @@ -42,6 +40,7 @@ endif TOR_UTIL_LIBS = \ src/lib/libtor-geoip.a \ src/lib/libtor-process.a \ + src/lib/libtor-buf.a \ src/lib/libtor-time.a \ src/lib/libtor-fs.a \ src/lib/libtor-encoding.a \ @@ -62,6 +61,7 @@ TOR_UTIL_LIBS = \ src/lib/libtor-malloc.a \ src/lib/libtor-wallclock.a \ src/lib/libtor-err.a \ + src/lib/libtor-version.a \ src/lib/libtor-intmath.a \ src/lib/libtor-ctime.a @@ -71,6 +71,7 @@ if UNITTESTS_ENABLED TOR_UTIL_TESTING_LIBS = \ src/lib/libtor-geoip-testing.a \ src/lib/libtor-process-testing.a \ + src/lib/libtor-buf-testing.a \ src/lib/libtor-time-testing.a \ src/lib/libtor-fs-testing.a \ src/lib/libtor-encoding-testing.a \ @@ -91,6 +92,7 @@ TOR_UTIL_TESTING_LIBS = \ src/lib/libtor-malloc-testing.a \ src/lib/libtor-wallclock-testing.a \ src/lib/libtor-err-testing.a \ + src/lib/libtor-version-testing.a \ src/lib/libtor-intmath.a \ src/lib/libtor-ctime-testing.a endif @@ -416,7 +418,7 @@ endif .PHONY: update-versions update-versions: - $(PERL) $(top_builddir)/scripts/maint/updateVersions.pl + abs_top_srcdir="$(abs_top_srcdir)" $(PYTHON) $(top_srcdir)/scripts/maint/update_versions.py .PHONY: callgraph callgraph: diff --git a/changes/bug21900 b/changes/bug21900 new file mode 100644 index 0000000000..686cb6c584 --- /dev/null +++ b/changes/bug21900 @@ -0,0 +1,4 @@ + o Minor bugfixes (DNS): + - Gracefully handle empty or absent resolve.conf file by falling + back to using localhost DNS service and hoping it works. Fixes + bug 21900; bugfix on 0.2.1.10-alpha. diff --git a/changes/bug23082 b/changes/bug23082 new file mode 100644 index 0000000000..fc4b52c364 --- /dev/null +++ b/changes/bug23082 @@ -0,0 +1,4 @@ + o Minor bugfixes (networking): + - Introduce additional checks into tor_addr_parse() to + reject certain incorrect inputs that previously were + not detected. Fixes bug 23082; bugfix on 0.2.0.10-alpha. diff --git a/changes/bug24393 b/changes/bug24393 new file mode 100644 index 0000000000..e190192319 --- /dev/null +++ b/changes/bug24393 @@ -0,0 +1,6 @@ + o Minor features (ipv6): + - When using addrs_in_same_network_family(), check IPv6 subnets as well as + IPv4 ones where possible when a client chooses circuit paths. Previously, + we used this function only for IPv4 subnets. Closes ticket 24393. Patch + by Neel Chauhan. + diff --git a/changes/bug24661 b/changes/bug24661 new file mode 100644 index 0000000000..a915a93e0e --- /dev/null +++ b/changes/bug24661 @@ -0,0 +1,3 @@ + o Minor bugfixes (client, guard selection): + - When Tor's consensus has expired, but is still reasonably live, use it + to select guards. Fixes bug 24661; bugfix on 0.3.0.1-alpha. diff --git a/changes/bug25885 b/changes/bug25885 new file mode 100644 index 0000000000..1b89acfe06 --- /dev/null +++ b/changes/bug25885 @@ -0,0 +1,7 @@ + o Minor bugfixes (guards): + - In count_acceptable_nodes(), check if we have at least one bridge + or guard node, and two non-guard nodes for a circuit. Previously, + we have added up the sum of all nodes with a descriptor, but that + could cause us to build circuits that fail if we had either too + many bridges, or not enough guard nodes. Fixes bug 25885; bugfix + on 0.3.6.1-alpha. Patch by Neel Chauhan. diff --git a/changes/bug27197 b/changes/bug27197 deleted file mode 100644 index e389f85065..0000000000 --- a/changes/bug27197 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (protover, rust): - - Reject extra commas in version string. Fixes bug 27197; bugfix on - 0.3.3.3-alpha. diff --git a/changes/bug27707 b/changes/bug27707 new file mode 100644 index 0000000000..e114222741 --- /dev/null +++ b/changes/bug27707 @@ -0,0 +1,3 @@ + o Minor features (log messages): + - Improve log message in HSv3 service that could print out negative + revision counters. Closes ticket 27707. Patch by "ffmancera".
\ No newline at end of file diff --git a/changes/bug27740 b/changes/bug27740 deleted file mode 100644 index 76a17b7dda..0000000000 --- a/changes/bug27740 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (rust): - - Return a string that can be safely freed by C code, not one created by - the rust allocator, in protover_all_supported(). Fixes bug 27740; bugfix - on 0.3.3.1-alpha. diff --git a/changes/bug27741 b/changes/bug27741 deleted file mode 100644 index 531e264b63..0000000000 --- a/changes/bug27741 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (rust, directory authority): - - Fix an API mismatch in the rust implementation of - protover_compute_vote(). This bug could have caused crashes on any - directory authorities running Tor with Rust (which we do not yet - recommend). Fixes bug 27741; bugfix on 0.3.3.6. diff --git a/changes/bug27750 b/changes/bug27750 deleted file mode 100644 index c234788b1c..0000000000 --- a/changes/bug27750 +++ /dev/null @@ -1,6 +0,0 @@ - o Minor bugfixes (connection, relay): - - Avoid a wrong BUG() stacktrace in case a closing connection is being held - open because the write side is rate limited but not the read side. Now, - the connection read side is simply shutdown instead of kept open until tor - is able to flush the connection and then fully close it. Fixes bug 27750; - bugfix on 0.3.4.1-alpha. diff --git a/changes/bug27800 b/changes/bug27800 deleted file mode 100644 index 63d5dbc681..0000000000 --- a/changes/bug27800 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (directory authority): - - Log additional info when we get a relay that shares an ed25519 - ID with a different relay, instead making a BUG() warning. - Fixes bug 27800; bugfix on 0.3.2.1-alpha. diff --git a/changes/bug27804 b/changes/bug27804 deleted file mode 100644 index fa7fec0bc5..0000000000 --- a/changes/bug27804 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (rust): - - Fix a potential null dereference in protover_all_supported(). - Add a test for it. Fixes bug 27804; bugfix on 0.3.3.1-alpha. diff --git a/changes/bug27841 b/changes/bug27841 deleted file mode 100644 index 9cd1da7275..0000000000 --- a/changes/bug27841 +++ /dev/null @@ -1,7 +0,0 @@ - o Minor bugfixes (onion services): - - On an intro point for a version 3 onion service, we do not close - an introduction circuit on an NACK. This lets the client decide - whether to reuse the circuit or discard it. Previously, we closed - intro circuits on NACKs. Fixes bug 27841; bugfix on 0.3.2.1-alpha. - Patch by Neel Chaunan - diff --git a/changes/bug27929 b/changes/bug27929 new file mode 100644 index 0000000000..dab57a2eca --- /dev/null +++ b/changes/bug27929 @@ -0,0 +1,5 @@ + o Minor bugfixes (periodic events): + - Refrain from calling routerlist_remove_old_routers() from + check_descriptor_callback(). Instead, create a new periodic + event that will run once every hour even if Tor is not configured + as onion router. Fixes bug 27929; bugfix on 0.2.8.1-alpha. diff --git a/changes/bug27948 b/changes/bug27948 deleted file mode 100644 index fea16f3d0f..0000000000 --- a/changes/bug27948 +++ /dev/null @@ -1,6 +0,0 @@ - o Minor bugfixes (tests): - - Treat backtrace test failures as expected on BSD-derived systems - (NetBSD, OpenBSD, and macOS/Darwin) until we solve bug 17808. - (FreeBSD failures have been treated as expected since 18204 in 0.2.8.) - Fixes bug 27948; bugfix on 0.2.5.2-alpha. - diff --git a/changes/bug27963_timeradd b/changes/bug27963_timeradd deleted file mode 100644 index 34b361cf8d..0000000000 --- a/changes/bug27963_timeradd +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (compilation, opensolaris): - - Add a missing include to compat_pthreads.c, to fix compilation - on OpenSolaris and its descendants. Fixes bug 27963; bugfix - on 0.3.5.1-alpha. diff --git a/changes/bug27968 b/changes/bug27968 deleted file mode 100644 index 78c8eee33a..0000000000 --- a/changes/bug27968 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (testing): - - Avoid hangs and race conditions in test_rebind.py. - Fixes bug 27968; bugfix on 0.3.5.1-alpha. diff --git a/changes/bug28096 b/changes/bug28096 deleted file mode 100644 index 6847df9798..0000000000 --- a/changes/bug28096 +++ /dev/null @@ -1,13 +0,0 @@ - o Minor bugfixes (Windows): - - Correctly identify Windows 8.1, Windows 10, and Windows Server 2008 - and later from their NT versions. - Fixes bug 28096; bugfix on 0.2.2.34; reported by Keifer Bly. - - On recent Windows versions, the GetVersionEx() function may report - an earlier Windows version than the running OS. To avoid user - confusion, add "[or later]" to Tor's version string on affected - versions of Windows. - Fixes bug 28096; bugfix on 0.2.2.34; reported by Keifer Bly. - - Remove Windows versions that were never supported by the - GetVersionEx() function. Stop duplicating the latest Windows - version in get_uname(). - Fixes bug 28096; bugfix on 0.2.2.34; reported by Keifer Bly. diff --git a/changes/bug28115 b/changes/bug28115 deleted file mode 100644 index e3e29968eb..0000000000 --- a/changes/bug28115 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (portability): - - Make the OPE code (which is used for v3 onion services) run correctly - on big-endian platforms. Fixes bug 28115; bugfix on 0.3.5.1-alpha. diff --git a/changes/bug28127 b/changes/bug28127 deleted file mode 100644 index 541128c88e..0000000000 --- a/changes/bug28127 +++ /dev/null @@ -1,7 +0,0 @@ - o Minor bugfixes (onion services): - - Unless we have explicitly set HiddenServiceVersion, detect the onion - service version and then look for invalid options. Previously, we - did the reverse, but that broke existing configs which were pointed - to a v2 hidden service and had options like HiddenServiceAuthorizeClient - set Fixes bug 28127; bugfix on 0.3.5.1-alpha. Patch by Neel Chauhan. - diff --git a/changes/bug28183 b/changes/bug28183 deleted file mode 100644 index 8d35dcdc01..0000000000 --- a/changes/bug28183 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (Linux seccomp2 sandbox): - - Permit the "shutdown()" system call, which is apparently - used by OpenSSL under some circumstances. Fixes bug 28183; - bugfix on 0.2.5.1-alpha. diff --git a/changes/bug28202 b/changes/bug28202 deleted file mode 100644 index 182daac4f1..0000000000 --- a/changes/bug28202 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (C correctness): - - Avoid undefined behavior in an end-of-string check when parsing the - BEGIN line in a directory object. Fixes bug 28202; bugfix on - 0.2.0.3-alpha. diff --git a/changes/bug28245 b/changes/bug28245 deleted file mode 100644 index d7e6deb810..0000000000 --- a/changes/bug28245 +++ /dev/null @@ -1,6 +0,0 @@ - o Major bugfixes (OpenSSL, portability): - - Fix our usage of named groups when running as a TLS 1.3 client in - OpenSSL 1.1.1. Previously, we only initialized EC groups when running - as a server, which caused clients to fail to negotiate TLS 1.3 with - relays. Fixes bug 28245; bugfix on 0.2.9.15 when TLS 1.3 support was - added. diff --git a/changes/bug28298 b/changes/bug28298 deleted file mode 100644 index 8db340f3df..0000000000 --- a/changes/bug28298 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (configuration): - - Resume refusing to start with relative file paths and RunAsDaemon - set (regression from the fix for bug 22731). Fixes bug 28298; - bugfix on 0.3.3.1-alpha. diff --git a/changes/bug28303 b/changes/bug28303 deleted file mode 100644 index 80f1302e5e..0000000000 --- a/changes/bug28303 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (compilation): - - Fix a pair of missing headers on OpenBSD. Fixes bug 28303; - bugfix on 0.3.5.1-alpha. Patch from Kris Katterjohn. diff --git a/changes/bug28348_034 b/changes/bug28348_034 deleted file mode 100644 index 3913c03a4c..0000000000 --- a/changes/bug28348_034 +++ /dev/null @@ -1,5 +0,0 @@ - o Major bugfixes (embedding, main loop): - - When DisableNetwork becomes set, actually disable periodic events that - are already enabled. (Previously, we would refrain from enabling new - ones, but we would leave the old ones turned on.) - Fixes bug 28348; bugfix on 0.3.4.1-alpha. diff --git a/changes/bug28399 b/changes/bug28399 deleted file mode 100644 index 9096db70b0..0000000000 --- a/changes/bug28399 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (continuous integration, Windows): - - Stop using an external OpenSSL install, and stop installing MSYS2 - packages, when building using mingw on Appveyor Windows CI. - Fixes bug 28399; bugfix on 0.3.4.1-alpha. diff --git a/changes/bug28413 b/changes/bug28413 deleted file mode 100644 index 4c88bea7e7..0000000000 --- a/changes/bug28413 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (compilation): - - Initialize a variable in aes_new_cipher(), since some compilers - cannot tell that we always initialize it before use. Fixes bug 28413; - bugfix on 0.2.9.3-alpha. diff --git a/changes/bug28419 b/changes/bug28419 deleted file mode 100644 index 52ceb0a2a7..0000000000 --- a/changes/bug28419 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (memory leaks): - - Fix a harmless memory leak in libtorrunner.a. Fixes bug 28419; - bugfix on 0.3.3.1-alpha. Patch from Martin Kepplinger.
\ No newline at end of file diff --git a/changes/bug28435 b/changes/bug28435 deleted file mode 100644 index 2a886cb8b7..0000000000 --- a/changes/bug28435 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (documentation): - - Make Doxygen work again after the 0.3.5 source tree moves. - Fixes bug 28435; bugfix on 0.3.5.1-alpha. diff --git a/changes/bug28441 b/changes/bug28441 deleted file mode 100644 index d259b9f742..0000000000 --- a/changes/bug28441 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (logging): - - Stop talking about the Named flag in log messages. Clients have - ignored the Named flag since 0.3.2. Fixes bug 28441; - bugfix on 0.3.2.1-alpha. diff --git a/changes/bug28454 b/changes/bug28454 deleted file mode 100644 index ca46ae2777..0000000000 --- a/changes/bug28454 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (continuous integration, Windows): - - Manually configure the zstd compiler options, when building using - mingw on Appveyor Windows CI. The MSYS2 mingw zstd package does not - come with a pkg-config file. Fixes bug 28454; bugfix on 0.3.4.1-alpha. diff --git a/changes/bug28485 b/changes/bug28485 deleted file mode 100644 index a8309ae21f..0000000000 --- a/changes/bug28485 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (compilation): - - Add missing dependency on libgdi32.dll for tor-print-ed-signing-cert.exe - on Windows. Fixes bug 28485; bugfix on 0.3.5.1-alpha. diff --git a/changes/bug28518 b/changes/bug28518 new file mode 100644 index 0000000000..d7ebab29bb --- /dev/null +++ b/changes/bug28518 @@ -0,0 +1,4 @@ + o Minor features (FreeBSD): + - Warn relay operators if the "net.inet.ip.random_id" sysctl (IP ID + randomization) is disabled on their relay if it is running on FreeBSD + based operating systems. Closes ticket 28518. diff --git a/changes/bug28524 b/changes/bug28524 deleted file mode 100644 index 1cad700422..0000000000 --- a/changes/bug28524 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (restart-in-process, boostrap): - - Add missing resets of bootstrap tracking state when shutting - down (regression caused by ticket 27169). Fixes bug 28524; - bugfix on 0.3.5.1-alpha. diff --git a/changes/bug28554 b/changes/bug28554 deleted file mode 100644 index 9a0b281406..0000000000 --- a/changes/bug28554 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (unit tests, guard selection): - - Stop leaking memory in an entry guard unit test. Fixes bug 28554; - bugfix on 0.3.0.1-alpha. diff --git a/changes/bug28562 b/changes/bug28562 deleted file mode 100644 index e14362164d..0000000000 --- a/changes/bug28562 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (testing): - - Use a separate DataDirectory for the test_rebind script. - Previously, this script would run using the default DataDirectory, - and sometimes fail. Fixes bug 28562; bugfix on 0.3.5.1-alpha. - Patch from Taylor R Campbell. diff --git a/changes/bug28569 b/changes/bug28569 new file mode 100644 index 0000000000..45a57a80ae --- /dev/null +++ b/changes/bug28569 @@ -0,0 +1,3 @@ + o Minor bugfixes (unit tests, directory clients): + - Mark outdated dirservers when Tor only has a reasonably live consensus. + Fixes bug 28569; bugfix on 0.3.2.5-alpha. diff --git a/changes/bug28619 b/changes/bug28619 deleted file mode 100644 index 86be8cb2fb..0000000000 --- a/changes/bug28619 +++ /dev/null @@ -1,6 +0,0 @@ - o Minor bugfixes (hidden service v3): - - When deleting an ephemeral onion service (DEL_ONION), do not close any - rendezvous circuits in order to let the existing client connections - finish by themselves or closed by the application. The HS v2 is doing - that already so now we have the same behavior for all versions. Fixes - bug 28619; bugfix on 0.3.3.1-alpha. diff --git a/changes/doc28560 b/changes/doc28560 new file mode 100644 index 0000000000..c3356bda0a --- /dev/null +++ b/changes/doc28560 @@ -0,0 +1,3 @@ + o Documentation (hidden service manpage, sandbox): + - Mention that you cannot add new Onion Service if Tor is already + running with Sandbox enabled. Closes ticket 28560. diff --git a/changes/document_version b/changes/document_version new file mode 100644 index 0000000000..a45992b6b5 --- /dev/null +++ b/changes/document_version @@ -0,0 +1,2 @@ + o Documentation: + - Document the exact output of "tor --version". Closes ticket 28889. diff --git a/changes/feature27244 b/changes/feature27244 new file mode 100644 index 0000000000..a4debbbe53 --- /dev/null +++ b/changes/feature27244 @@ -0,0 +1,5 @@ + o Minor features (memory usage): + - Tor clients no longer need to keep the full text of a consensus in + memory in order to parse it, or apply a diff to it. Instead, they + use mmap() to read the consensus files from disk. Closes ticket + 27244. diff --git a/changes/feature27367 b/changes/feature27367 new file mode 100644 index 0000000000..99c0839621 --- /dev/null +++ b/changes/feature27367 @@ -0,0 +1,4 @@ + o Minor features (parsing): + - Directory authorities now validate that router descriptors and ExtraInfo + documents are in a valid subset of UTF-8, and reject them if not. + Closes ticket 27367. diff --git a/changes/geoip-2018-11-06 b/changes/geoip-2018-11-06 deleted file mode 100644 index 5c18ea4244..0000000000 --- a/changes/geoip-2018-11-06 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (geoip): - - Update geoip and geoip6 to the November 6 2018 Maxmind GeoLite2 - Country database. Closes ticket 28395. - diff --git a/changes/geoip-2018-12-05 b/changes/geoip-2018-12-05 deleted file mode 100644 index 20ccf2d8a5..0000000000 --- a/changes/geoip-2018-12-05 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (geoip): - - Update geoip and geoip6 to the December 5 2018 Maxmind GeoLite2 - Country database. Closes ticket 28744. - diff --git a/changes/prop297 b/changes/prop297 new file mode 100644 index 0000000000..4f93b232d2 --- /dev/null +++ b/changes/prop297 @@ -0,0 +1,7 @@ + o Minor features (required protocols): + - Tor no longer exits if it is missing a required protocol, if the + consensus that requires the protocol predates the release date of the + version of Tor. This change prevents Tor releases from exiting because + of an old cached consensus, on the theory that a newer cached + consensus might not require the protocol. Implements proposal 297; + closes ticket 27735. diff --git a/changes/rust_asan b/changes/rust_asan deleted file mode 100644 index 1ca7ae6888..0000000000 --- a/changes/rust_asan +++ /dev/null @@ -1,8 +0,0 @@ - o Major bugfixes (compilation, rust): - - Rust tests can now build and run successfully with the - --enable-fragile-hardening option enabled. - Doing this currently requires the rust beta channel; it will - be possible with stable rust as of rust version 1.31 is out. - Patch from Alex Crichton. - Fixes bugs 27272, 27273, and 27274. - Bugfix on 0.3.1.1-alpha. diff --git a/changes/subsystems b/changes/subsystems new file mode 100644 index 0000000000..a51fb8e2b1 --- /dev/null +++ b/changes/subsystems @@ -0,0 +1,6 @@ + o Major features (refactoring): + - Tor now uses an explicit list of its own subsystems when initializing + and shutting down. Previously, these systems were managed implicitly + though various places throughout the codebase. (There still some + subsystems using the old system.) + Closes ticket 28330. diff --git a/changes/ticket19566 b/changes/ticket19566 deleted file mode 100644 index bf7071e660..0000000000 --- a/changes/ticket19566 +++ /dev/null @@ -1,6 +0,0 @@ - o Code simplification and refactoring (shared random, dirauth): - - Change many tor_assert() to use BUG() instead. The idea is to not crash - a dirauth but rather scream loudly with a stacktrace and let it continue - run. The shared random subsystem is very resilient and if anything wrong - happens with it, at worst a non coherent value will be put in the vote - and discarded by the other authorities. Closes ticket 19566. diff --git a/changes/ticket24803 b/changes/ticket24803 deleted file mode 100644 index e76a9eeab9..0000000000 --- a/changes/ticket24803 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor features (fallback directory list): - - Replace the 150 fallbacks originally introduced in Tor 0.3.3.1-alpha in - January 2018 (of which ~115 were still functional), with a list of - 157 fallbacks (92 new, 65 existing, 85 removed) generated in - December 2018. Closes ticket 24803. diff --git a/changes/ticket24805 b/changes/ticket24805 new file mode 100644 index 0000000000..4ba6f6ecd4 --- /dev/null +++ b/changes/ticket24805 @@ -0,0 +1,3 @@ + o Minor features (fallback directory list): + - Update the fallback whitelist based on operator opt-ins and opt-outs. + Closes ticket 24805, patch by Phoul. diff --git a/changes/ticket24838 b/changes/ticket24838 new file mode 100644 index 0000000000..d068e31b91 --- /dev/null +++ b/changes/ticket24838 @@ -0,0 +1,6 @@ + o Minor features (fallback directory mirrors): + - Accept relays that are a fuzzy match to a fallback whitelist entry. + If a relay matches at least one fingerprint, IPv4 address, or IPv6 + address in the fallback whitelist, it can become a fallback. This + reduces the work required to keep the list up to date. + Closes ticket 24838. diff --git a/changes/ticket26360 b/changes/ticket26360 new file mode 100644 index 0000000000..80afbd1c17 --- /dev/null +++ b/changes/ticket26360 @@ -0,0 +1,4 @@ + o Minor bugfixes (pluggable transports): + - Make sure that data is continously read from standard out and error of the + PT child-process to avoid deadlocking when the pipes' buffer is full. + Fixes bug 26360; bugfix on 0.2.3.6-alpha. diff --git a/changes/ticket26770 b/changes/ticket26770 new file mode 100644 index 0000000000..7f3e92e9dd --- /dev/null +++ b/changes/ticket26770 @@ -0,0 +1,8 @@ + o Minor features (directory authority, relay): + - Authorities now vote on a "StaleDesc" flag to indicate that a relay's + descriptor is so old that the relay should upload again soon. Relays + understand this flag, and treat it as a signal to upload a new + descriptor. This flag will eventually let us remove the 'published' + date from routerstatus entries, and save a great deal of space in our + consensus diffs. Closes ticket 26770; implements proposal 293. + diff --git a/changes/ticket27167 b/changes/ticket27167 new file mode 100644 index 0000000000..81c66630c8 --- /dev/null +++ b/changes/ticket27167 @@ -0,0 +1,11 @@ + o Major features (bootstrap): + - Report the first connection to a relay as the earliest phases of + bootstrap progress, regardless of whether it's a connection for + building application circuits. This allows finer-grained + reporting of early progress than previously possible with the + improvements of ticket 27169. Closes tickets 27167 and 27103. + Addresses ticket 27308. + - Separately report the intermediate stage of having connected to + a proxy or pluggable transport, versus succesfully using that + proxy or pluggable transport to connect to a relay. Closes + tickets 27100 and 28884. diff --git a/changes/ticket27225 b/changes/ticket27225 new file mode 100644 index 0000000000..4c05a269d6 --- /dev/null +++ b/changes/ticket27225 @@ -0,0 +1,5 @@ + o Minor features (performance): + - Avoid parsing the same protocol-versions string over and over + in summarize_protover_flags(). This should save us a huge number + of malloc calls on startup, and may reduce memory fragmentation with + some allocators. Closes ticket 27225. diff --git a/changes/ticket27325 b/changes/ticket27325 new file mode 100644 index 0000000000..2cf2bb7d69 --- /dev/null +++ b/changes/ticket27325 @@ -0,0 +1,4 @@ + o Code simplification and refactoring: + - Reimplement NETINFO cell parsing and generation to rely on + trunnel-generated wire format handling code. Closes ticket + 27325. diff --git a/changes/ticket27359 b/changes/ticket27359 new file mode 100644 index 0000000000..bddc90634d --- /dev/null +++ b/changes/ticket27359 @@ -0,0 +1,3 @@ + o Minor features (memory usage): + - Store microdescriptor family lists with a more compact representation + to save memory. Closes ticket 27359. diff --git a/changes/ticket27402 b/changes/ticket27402 new file mode 100644 index 0000000000..b79fb56760 --- /dev/null +++ b/changes/ticket27402 @@ -0,0 +1,10 @@ + o Minor feature (bootstrap): + - When reporting bootstrap progress, stop distinguishing between + situations where it seems that only internal paths are available + and situations where it seems that external paths are available. + Previously, tor would often erroneously report that it had only + internal paths. Closes ticket 27402. + + o Code simplification and refactoring: + - Split out bootstrap progress reporting from control.c into a + separate file. Part of ticket 27402. diff --git a/changes/ticket27471 b/changes/ticket27471 deleted file mode 100644 index ffe77d268e..0000000000 --- a/changes/ticket27471 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (hidden service v3, client): - - When replacing a descriptor in the client cache with a newer descriptor, - make sure to close all client introduction circuits of the old - descriptor so we don't end up with unusable leftover circuits. Fixes bug - 27471; bugfix on 0.3.2.1-alpha. diff --git a/changes/ticket27490 b/changes/ticket27490 new file mode 100644 index 0000000000..523477dfea --- /dev/null +++ b/changes/ticket27490 @@ -0,0 +1,6 @@ + o Minor features (ipv6): + - We add an option ClientAutoIPv6ORPort which makes clients randomly + prefer a node's IPv4 or IPv6 ORPort. The random preference is set + every time a node is loaded from a new consensus or bridge config. + Closes ticket 27490. Patch by Neel Chauhan. + diff --git a/changes/ticket27549 b/changes/ticket27549 new file mode 100644 index 0000000000..51d0f24757 --- /dev/null +++ b/changes/ticket27549 @@ -0,0 +1,3 @@ + o Code simplification and refactoring (hidden service v3): + - Consolidate the authorized client descriptor cookie computation code + from client and service into one function. Closes ticket 27549. diff --git a/changes/ticket27620 b/changes/ticket27620 new file mode 100644 index 0000000000..6c491696d0 --- /dev/null +++ b/changes/ticket27620 @@ -0,0 +1,3 @@ + o Code simplification and refactoring: + - Rework SOCKS wire format handling to rely on trunnel-generated + parsing/generation code. Resolves ticket 27620. diff --git a/changes/ticket27625 b/changes/ticket27625 new file mode 100644 index 0000000000..33d40adf34 --- /dev/null +++ b/changes/ticket27625 @@ -0,0 +1,4 @@ + o Testing: + - Write some unit tests for tokenize_string() and + get_next_token() functions. Resolves ticket 27625. + diff --git a/changes/ticket27751 b/changes/ticket27751 deleted file mode 100644 index 593c473b61..0000000000 --- a/changes/ticket27751 +++ /dev/null @@ -1,2 +0,0 @@ - o Minor features (continuous integration): - - Add a Travis CI build for --enable-nss on Linux gcc. Closes ticket 27751. diff --git a/changes/ticket27838 b/changes/ticket27838 deleted file mode 100644 index 1699730d7a..0000000000 --- a/changes/ticket27838 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (hidden service v3): - - Build the service descriptor signing key certificate before uploading so - we always have a fresh one leaving no chances for it to expire service - side. Fixes bug 27838; bugfix on 0.3.2.1-alpha. diff --git a/changes/ticket27913 b/changes/ticket27913 deleted file mode 100644 index 81ce725932..0000000000 --- a/changes/ticket27913 +++ /dev/null @@ -1,3 +0,0 @@ - o Testing: - - Add new CI job to Travis configuration that runs stem-based - integration tests. Closes ticket 27913. diff --git a/changes/ticket27993 b/changes/ticket27993 new file mode 100644 index 0000000000..78ee7c2054 --- /dev/null +++ b/changes/ticket27993 @@ -0,0 +1,3 @@ + o Minor features (developer tooling): + - Provide git hook script to prevent "fixup!" and "squash!" commits from + ending up in master. Closes ticket 27993. diff --git a/changes/ticket27995 b/changes/ticket27995 deleted file mode 100644 index 8c75425749..0000000000 --- a/changes/ticket27995 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (hidden service v3, client authorization): - - Fix an assert() when adding a client authorization for the first time - and then sending a HUP signal to the service. Before that, tor would - stop abruptly. Fixes bug 27995; bugfix on 0.3.5.1-alpha. diff --git a/changes/ticket28006 b/changes/ticket28006 new file mode 100644 index 0000000000..95a4b2cae4 --- /dev/null +++ b/changes/ticket28006 @@ -0,0 +1,3 @@ + o Code simplification and refactoring: + - Fix issues that shellcheck found in chutney-git-bisect.sh. + Resolves ticket 28006. diff --git a/changes/ticket28007 b/changes/ticket28007 new file mode 100644 index 0000000000..1ac87862eb --- /dev/null +++ b/changes/ticket28007 @@ -0,0 +1,3 @@ + o Code simplification and refactoring: + - Cleanup scan-build.sh to silence shellcheck warnings. + Closes ticket 28007. diff --git a/changes/ticket28008 b/changes/ticket28008 new file mode 100644 index 0000000000..1f0de1a14d --- /dev/null +++ b/changes/ticket28008 @@ -0,0 +1,3 @@ + o Code simplification and refactoring: + - Fix shellcheck warnings in scripts/test/coverage. Resolves issue + 28008. diff --git a/changes/ticket28009 b/changes/ticket28009 new file mode 100644 index 0000000000..1d986d4211 --- /dev/null +++ b/changes/ticket28009 @@ -0,0 +1,3 @@ + o Code simplification and refactoring: + - Fix shellcheck warnings in cov-diff script. Resolves issue + 28009. diff --git a/changes/ticket28010 b/changes/ticket28010 new file mode 100644 index 0000000000..4fca17d022 --- /dev/null +++ b/changes/ticket28010 @@ -0,0 +1,3 @@ + o Code simplification and refactoring: + - Fix shellcheck warnings in run_trunnel.sh. Resolves issue + 28010. diff --git a/changes/ticket28011 b/changes/ticket28011 new file mode 100644 index 0000000000..5efc3c917b --- /dev/null +++ b/changes/ticket28011 @@ -0,0 +1,3 @@ + o Code simplification and refactoring: + - Fix shellcheck warnings in run_calltool.sh. Resolves + ticket 28011. diff --git a/changes/ticket28012 b/changes/ticket28012 new file mode 100644 index 0000000000..b2fe83e02a --- /dev/null +++ b/changes/ticket28012 @@ -0,0 +1,3 @@ + o Code simplification and refactoring: + - Fix issues that shellcheck found in updateRustDependencies.sh. + Resolves ticket 28012. diff --git a/changes/ticket28026 b/changes/ticket28026 deleted file mode 100644 index a6911c2cab..0000000000 --- a/changes/ticket28026 +++ /dev/null @@ -1,3 +0,0 @@ - o Documentation (hidden service manpage): - - Improve HSv3 client authorization by making some options more explicit - and detailed. Closes ticket 28026. Patch by "mtigas". diff --git a/changes/ticket28077 b/changes/ticket28077 new file mode 100644 index 0000000000..2b5afb1678 --- /dev/null +++ b/changes/ticket28077 @@ -0,0 +1,3 @@ + o Code simplification and refactoring: + - Remove unnecessarily unsafe code from the rust macro cstr!. Closes + ticket 28077. diff --git a/changes/ticket28100 b/changes/ticket28100 new file mode 100644 index 0000000000..b8e3271013 --- /dev/null +++ b/changes/ticket28100 @@ -0,0 +1,3 @@ + o Minor features (HTTP standards compliance): + - Don't send Content-Type: application/octet-stream for transparently + compressed documents, which confused browsers. Closes ticket 28100. diff --git a/changes/ticket28113 b/changes/ticket28113 deleted file mode 100644 index 30dd825a9b..0000000000 --- a/changes/ticket28113 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (relay shutdown, systemd): - - Notify systemd of ShutdownWaitLength so it can be set to longer than - systemd's TimeoutStopSec. In tor's systemd service file, set - TimeoutSec to 60 seconds, to allow tor some time to shut down. - Fixes bug 28113; bugfix on 0.2.6.2-alpha. diff --git a/changes/ticket28128 b/changes/ticket28128 deleted file mode 100644 index 6d08c74242..0000000000 --- a/changes/ticket28128 +++ /dev/null @@ -1,4 +0,0 @@ - o Documentation (hidden service manpage, sandbox): - - Document in the man page that changing ClientOnionAuthDir value or - adding a new file in the directory will not work at runtime upon sending - a HUP if Sandbox 1. Closes ticket 28128. diff --git a/changes/ticket28179 b/changes/ticket28179 new file mode 100644 index 0000000000..f548c4a79a --- /dev/null +++ b/changes/ticket28179 @@ -0,0 +1,5 @@ + o Minor features (process): + - Add new Process API for handling child processes. This + new API allows Tor to have bi-directional + communication with child processes on both Unix and Windows. + Closes ticket 28179. diff --git a/changes/ticket28180 b/changes/ticket28180 new file mode 100644 index 0000000000..59de1c6251 --- /dev/null +++ b/changes/ticket28180 @@ -0,0 +1,3 @@ + o Minor features (pluggable transports): + - Add support for logging to Tor's logging subsystem from a pluggable + transport process. Closes ticket 28180 diff --git a/changes/ticket28229_diag b/changes/ticket28229_diag deleted file mode 100644 index cd02b81faa..0000000000 --- a/changes/ticket28229_diag +++ /dev/null @@ -1,3 +0,0 @@ - o Testing: - - Increase logging and tag all log entries with timestamps - in test_rebind.py. Provides diagnostics for issue 28229. diff --git a/changes/ticket28266 b/changes/ticket28266 new file mode 100644 index 0000000000..e0bc171080 --- /dev/null +++ b/changes/ticket28266 @@ -0,0 +1,10 @@ + o Minor features (directory authority): + - Directory authorities support a new consensus algorithm, + under which microdescriptor entries are encoded in a canonical + form. This improves their compressibility in transit and on the client. + Closes ticket 28266; implements proposal 298. + + o Minor features (relay): + - When listing relay families, list them in canonical form including the + relay's own identity, and try to give a more useful set of warnings. + Part of ticket 28266 and proposal 298. diff --git a/changes/ticket28275 b/changes/ticket28275 deleted file mode 100644 index eadca86b7b..0000000000 --- a/changes/ticket28275 +++ /dev/null @@ -1,4 +0,0 @@ - o Documentation (hidden service v3, man page): - - Note in the man page that the only real way to fully revoke an onion - service v3 client authorization is by restarting the tor process. Closes - ticket 28275. diff --git a/changes/ticket28318 b/changes/ticket28318 deleted file mode 100644 index 24dc1e9580..0000000000 --- a/changes/ticket28318 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor features (Windows, continuous integration): - - Build tor on Windows Server 2012 R2 and Windows Server 2016 using - Appveyor's CI. Closes ticket 28318. diff --git a/changes/ticket28335 b/changes/ticket28335 new file mode 100644 index 0000000000..eecf7c7fd9 --- /dev/null +++ b/changes/ticket28335 @@ -0,0 +1,7 @@ + o Major features (client): + - When Tor is running as a client, and it is unused for a long time, it + can now enter a "dormant" state. When Tor is dormant, it avoids + network activity and CPU wakeups until it is reawoken either by a user + request or by a controller command. For more information, see + the configuration options starting with "Dormant". Implements tickets + 2149 and 28335. diff --git a/changes/ticket28362 b/changes/ticket28362 new file mode 100644 index 0000000000..4ac22d50f2 --- /dev/null +++ b/changes/ticket28362 @@ -0,0 +1,6 @@ + o Code simplification and refactoring: + - The .may_include files that we use to describe our + directory-by-directory dependency structure now describe a noncircular + dependency graph over the directories that they cover. + Our checkIncludes.py tool now enforces this. + Closes ticket 28362. diff --git a/changes/ticket28459 b/changes/ticket28459 deleted file mode 100644 index 6b5839b52b..0000000000 --- a/changes/ticket28459 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (continuous integration, Windows): - - Always show the configure and test logs, and upload them as build - artifacts, when building for Windows using Appveyor CI. - Implements 28459. diff --git a/changes/ticket28551 b/changes/ticket28551 new file mode 100644 index 0000000000..46ba9d713b --- /dev/null +++ b/changes/ticket28551 @@ -0,0 +1,3 @@ + o Minor features (Continuous Integration): + - Log Python version during each Travis CI job. Resolves issue + 28551. diff --git a/changes/ticket28574 b/changes/ticket28574 deleted file mode 100644 index 562810f511..0000000000 --- a/changes/ticket28574 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (continuous integration, Windows): - - Explicitly specify the path to the OpenSSL library and do not download - OpenSSL from Pacman, but instead use the library that is already provided - by AppVeyor. Fixes bug 28574; bugfix on master. diff --git a/changes/ticket28624 b/changes/ticket28624 new file mode 100644 index 0000000000..353f962be9 --- /dev/null +++ b/changes/ticket28624 @@ -0,0 +1,5 @@ + o Minor features (battery management, client, dormant mode): + - The client's memory of whether it is "dormant", and how long it has + spend idle, persists across invocations. Implements ticket 28624. + - There is a DormantOnFirstStartup option that integrators can use if + they expect that in many cases, Tor will be installed but not used. diff --git a/changes/ticket28768 b/changes/ticket28768 new file mode 100644 index 0000000000..27d90febc8 --- /dev/null +++ b/changes/ticket28768 @@ -0,0 +1,4 @@ + o Minor features (fallback directory mirrors): + - Accept fallbacks that deliver reasonably live consensuses. + (Consensuses that will become valid less than 24 hours in the future, + or that expired less than 24 hours ago.) Closes ticket 28768. diff --git a/changes/ticket28839 b/changes/ticket28839 new file mode 100644 index 0000000000..e9f81dc405 --- /dev/null +++ b/changes/ticket28839 @@ -0,0 +1,3 @@ + o Minor features (performance): + - Speed up microdesriptor parsing by about 30%, to help + improve startup time. Closes ticket 28839. diff --git a/changes/ticket28840 b/changes/ticket28840 new file mode 100644 index 0000000000..05d3fbb94c --- /dev/null +++ b/changes/ticket28840 @@ -0,0 +1,3 @@ + o Minor features (testing): + - Allow HeartbeatPeriod of less than 30 minutes in testing Tor networks. + Closes ticket 28840, patch by robgjansen diff --git a/changes/ticket28846 b/changes/ticket28846 new file mode 100644 index 0000000000..efb5b9938e --- /dev/null +++ b/changes/ticket28846 @@ -0,0 +1,3 @@ + o Minor features (pluggable transports): + - Add support for emitting STATUS updates to Tor's control port from a + pluggable transport process. Closes ticket 28846. diff --git a/changes/ticket28847 b/changes/ticket28847 new file mode 100644 index 0000000000..63100c5813 --- /dev/null +++ b/changes/ticket28847 @@ -0,0 +1,3 @@ + o Minor features (process): + - Use the subsystem module to initialize and shut down the process module. + Closes ticket 28847. diff --git a/changes/ticket28852 b/changes/ticket28852 new file mode 100644 index 0000000000..a58cc3ba0e --- /dev/null +++ b/changes/ticket28852 @@ -0,0 +1,4 @@ + o Minor features (performance): + - Remove a needless memset() call from get_token_arguments, + thereby speeding up the tokenization of directory objects by about + 20%. Closes ticket 28852. diff --git a/changes/ticket28853 b/changes/ticket28853 new file mode 100644 index 0000000000..e76f6bd8c9 --- /dev/null +++ b/changes/ticket28853 @@ -0,0 +1,3 @@ + o Minor features (performance): + - Replace parse_short_policy() with a faster implementation, to improve + microdescriptor parsing time. Closes ticket 28853. diff --git a/configure.ac b/configure.ac index d992b8d01b..76fc12be50 100644 --- a/configure.ac +++ b/configure.ac @@ -4,10 +4,19 @@ dnl Copyright (c) 2007-2018, The Tor Project, Inc. dnl See LICENSE for licensing information AC_PREREQ([2.63]) -AC_INIT([tor],[0.3.5.6-rc-dev]) +AC_INIT([tor],[0.4.0.0-alpha-dev]) AC_CONFIG_SRCDIR([src/app/main/tor_main.c]) AC_CONFIG_MACRO_DIR([m4]) +# DO NOT EDIT THIS DEFINITION BY HAND UNLESS YOU KNOW WHAT YOU'RE DOING. +# +# The update_versions.py script updates this definition when the +# version number changes. Tor uses it to make sure that it +# only shuts down for missing "required protocols" when those protocols +# are listed as required by a consensus after this date. +AC_DEFINE(APPROX_RELEASE_DATE, ["2019-01-15"], # for 0.4.0.0-alpha-dev + [Approximate date when this software was released. (Updated when the version changes.)]) + # "foreign" means we don't follow GNU package layout standards # "1.11" means we require automake version 1.11 or newer # "subdir-objects" means put .o files in the same directory as the .c files @@ -503,15 +512,6 @@ if test "x$enable_rust" = "xyes"; then fi fi - dnl This is a workaround for #46797 - dnl (a.k.a https://github.com/rust-lang/rust/issues/46797 ). Once the - dnl upstream bug is fixed, we can remove this workaround. - case "$host_os" in - darwin*) - TOR_RUST_EXTRA_LIBS="-lresolv" - ;; - esac - dnl For now both MSVC and MinGW rust libraries will output static libs with dnl the MSVC naming convention. if test "$bwin32" = "true"; then @@ -556,8 +556,6 @@ if test "x$enable_rust" = "xyes"; then AC_MSG_RESULT([$RUSTC_VERSION]) fi -AC_SUBST(TOR_RUST_EXTRA_LIBS) - AC_SEARCH_LIBS(socket, [socket network]) AC_SEARCH_LIBS(gethostbyname, [nsl]) AC_SEARCH_LIBS(dlopen, [dl]) @@ -812,6 +810,8 @@ fi dnl Now check for particular libevent functions. AC_CHECK_FUNCS([evutil_secure_rng_set_urandom_device_file \ evutil_secure_rng_add_bytes \ + evdns_base_get_nameserver_addr \ + ]) LIBS="$save_LIBS" @@ -2445,7 +2445,6 @@ AC_CONFIG_FILES([ src/config/torrc.minimal src/rust/.cargo/config scripts/maint/checkOptionDocs.pl - scripts/maint/updateVersions.pl warning_flags ]) diff --git a/contrib/win32build/tor-mingw.nsi.in b/contrib/win32build/tor-mingw.nsi.in index d64808e191..af01a2b499 100644 --- a/contrib/win32build/tor-mingw.nsi.in +++ b/contrib/win32build/tor-mingw.nsi.in @@ -8,7 +8,7 @@ !include "LogicLib.nsh" !include "FileFunc.nsh" !insertmacro GetParameters -!define VERSION "0.3.5.6-rc-dev" +!define VERSION "0.4.0.0-alpha-dev" !define INSTALLER "tor-${VERSION}-win32.exe" !define WEBSITE "https://www.torproject.org/" !define LICENSE "LICENSE" diff --git a/doc/HACKING/Maintaining.md b/doc/HACKING/Maintaining.md new file mode 100644 index 0000000000..22d62b5471 --- /dev/null +++ b/doc/HACKING/Maintaining.md @@ -0,0 +1,113 @@ +# Maintaining Tor + +This document details the duties and processes on maintaining the Tor code +base. + +The first section describes who is the current Tor maintainer and what are the +responsabilities. Tor has one main single maintainer but does have many +committers and subsystem maintainers. + +The second third section describes how the **alpha and master** branches are +maintained and by whom. + +Finally, the last section describes how the **stable** branches are maintained +and by whom. + +This document does not cover how Tor is released, please see +[ReleasingTor.md](ReleasingTor.md) for that information. + +## Tor Maintainer + +The current maintainer is Nick Mathewson <nickm@torproject.org>. + +The maintainer takes final decisions in terms of engineering, architecture and +protocol design. Releasing Tor falls under their responsability. + +## Alpha and Master Branches + +The Tor repository always has at all time a **master** branch which contains +the upstream ongoing development. + +It may also contains a branch for a released feature freezed version which is +called the **alpha** branch. The git tag and version number is always +postfixed with `-alpha[-dev]`. For example: `tor-0.3.5.0-alpha-dev` or +`tor-0.3.5.3-alpha`. + +Tor is separated into subsystems and some of those are maintained by other +developers than the main maintainer. Those people have commit access to the +code base but only commit (in most cases) into the subsystem they maintain. + +Upstream merges are restricted to the alpha and master branches. Subsystem +maintainers should never push a patch into a stable branch which is the +responsability of the [stable branch maintainer](#stable-branches). + +### Who + +In alphabetical order, the following people have upstream commit access and +maintain the following subsystems: + +- David Goulet <dgoulet@torproject.org> + * Onion Service (including Shared Random). + ***keywords:*** *[tor-hs]* + * Channels, Circuitmux, Connection, Scheduler. + ***keywords:*** *[tor-chan, tor-cmux, tor-sched, tor-conn]* + * Cell Logic (Handling/Parsing). + ***keywords:*** *[tor-cell]* + * Threading backend. + ***keywords:*** *[tor-thread]* + +- George Kadianakis <asn@torproject.org> + * Onion Service (including Shared Random). + ***keywords:*** *[tor-hs]* + * Guard. + ***keywords:*** *[tor-guard]* + * Pluggable Transport (excluding Bridge networking). + ***keywords:*** *[tor-pt]* + +### Tasks + +These are the tasks of a subsystem maintainer: + +1. Regurlarly go over `merge_ready` tickets relevant to the related subsystem + and for the current alpha or development (master branch) Milestone. + +2. A subsystem maintainer is expected to contribute to any design changes + (including proposals) or large patch set about the subsystem. + +3. Leave their ego at the door. Mistakes will be made but they have to be + taking care of seriously. Learn and move on quickly. + +### Merging Policy + +These are few important items to follow when merging code upstream: + +1. To merge code upstream, the patch must have passed our CI (currently + github.com/torproject), have a corresponding ticket and reviewed by + **at least** one person that is not the original coder. + + Example A: If Alice writes a patch then Bob, a Tor network team member, + reviews it and flags it `merge_ready`. Then, the maintainter is required + to look at the patch and makes a decision. + + Example B: If the maintainer writes a patch then Bob, a Tor network + team member, reviews it and flags it `merge_ready`, then the maintainer + can merge the code upstream. + +2. Maintainer makes sure the commit message should describe what was fixed + and, if it applies, how was it fixed. It should also always refer to + the ticket number. + +3. Trivial patches such as comment change, documentation, syntax issues or + typos can be merged without a ticket or reviewers. + +4. Tor uses the "merge forward" method that is if a patch applies to the + alpha branch, it has to be merged there first and then merged forward + into master. + +5. Maintainer should always consult with the network team about any doubts, + mis-understandings or unknowns of a patch. Final word will always go to the + main Tor maintainer. + +## Stable Branches + +(Currently being drafted and reviewed by the network team.) diff --git a/doc/HACKING/ReleasingTor.md b/doc/HACKING/ReleasingTor.md index 55a40fc89b..7334b1b34a 100644 --- a/doc/HACKING/ReleasingTor.md +++ b/doc/HACKING/ReleasingTor.md @@ -20,30 +20,29 @@ new Tor release: === I. Make sure it works -1. Use it for a while, as a client, as a relay, as a hidden service, - and as a directory authority. See if it has any obvious bugs, and - resolve those. +1. Make sure that CI passes: have a look at Travis, Appveyor, and + Jenkins. Make sure you're looking at the right branches. - As applicable, merge the `maint-X` branch into the `release-X` branch. - But you've been doing that all along, right? + If there are any unexplained failures, try to fix them or figure them + out. -2. Are all of the jenkins builders happy? See jenkins.torproject.org. +2. Verify that there are no big outstanding issues. You might find such + issues -- - What about the bsd buildbots? - See http://buildbot.pixelminers.net/builders/ + * On Trac - What about Coverity Scan? + * On coverity scan - What about clang scan-build? + * On OSS-Fuzz - Does 'make distcheck' complain? +3. Run checks that aren't covered above, including: - How about 'make test-stem' and 'make test-network' and - `make test-network-full`? + * clang scan-build. (See the script in ./scripts/test/scan_build.sh) - - Are all those tests still happy with --enable-expensive-hardening ? + * make test-network and make test-network-all (with + --enable-expensive-hardening) - Any memory leaks? + * Running Tor yourself and making sure that it actually works for you. === II. Write a changelog @@ -55,11 +54,14 @@ new Tor release: of them and reordering to focus on what users and funders would find interesting and understandable. - To do this, first run `./scripts/maint/lintChanges.py changes/*` and - fix as many warnings as you can. Then run `./scripts/maint/sortChanges.py - changes/* > changelog.in` to combine headings and sort the entries. - After that, it's time to hand-edit and fix the issues that lintChanges - can't find: + To do this, run + `./scripts/maint/sortChanges.py changes/* > changelog.in` + to combine headings and sort the entries. Copy the changelog.in file + into the ChangeLog. Run 'format_changelog.py' (see below) to clean + up the line breaks. + + After that, it's time to hand-edit and fix the issues that + lintChanges can't find: 1. Within each section, sort by "version it's a bugfix on", else by numerical ticket order. @@ -68,8 +70,6 @@ new Tor release: Make stuff very terse - Make sure each section name ends with a colon - Describe the user-visible problem right away Mention relevant config options by name. If they're rare or unusual, @@ -79,7 +79,9 @@ new Tor release: Present and imperative tense: not past. - 'Relays', not 'servers' or 'nodes' or 'Tor relays'. + "Relays", not "servers" or "nodes" or "Tor relays". + + "Onion services", not "hidden services". "Stop FOOing", not "Fix a bug where we would FOO". @@ -100,12 +102,14 @@ new Tor release: For stable releases that backport things from later, we try to compose their releases, we try to make sure that we keep the changelog entries - identical to their original versions, with a 'backport from 0.x.y.z' + identical to their original versions, with a "backport from 0.x.y.z" note added to each section. So in this case, once you have the items from the changes files copied together, don't use them to build a new changelog: instead, look up the corrected versions that were merged into ChangeLog in the master branch, and use those. + Add "backport from X.Y.Z" in the section header for these entries. + 2. Compose a short release blurb to highlight the user-facing changes. Insert said release blurb into the ChangeLog stanza. If it's a stable release, add it to the ReleaseNotes file too. If we're adding @@ -131,44 +135,57 @@ new Tor release: === III. Making the source release. 1. In `maint-0.?.x`, bump the version number in `configure.ac` and run - `perl scripts/maint/updateVersions.pl` to update version numbers in other + `make update-versions` to update version numbers in other places, and commit. Then merge `maint-0.?.x` into `release-0.?.x`. - (NOTE: To bump the version number, edit `configure.ac`, and then run - either `make`, or `perl scripts/maint/updateVersions.pl`, depending on - your version.) - When you merge the maint branch forward to the next maint branch, or into master, merge it with "-s ours" to avoid a needless version bump. 2. Make distcheck, put the tarball up in somewhere (how about your - homedir on your homedir on people.torproject.org?) , and tell `#tor` - about it. Wait a while to see if anybody has problems building it. - (Though jenkins is usually pretty good about catching these things.) + homedir on your homedir on people.torproject.org?) , and tell `#tor-dev` + about it. + + If you want, wait until at least one person has built it + successfully. (We used to say "wait for others to test it", but our + CI has successfully caught these kinds of errors for the last several + years.) + + +3. Make sure that the new version is recommended in the latest consensus. + (Otherwise, users will get confused when it complains to them + about its status.) + + If it is not, you'll need to poke Roger, Weasel, and Sebastian again: see + item 0.1 at the start of this document. === IV. Commit, upload, announce 1. Sign the tarball, then sign and push the git tag: gpg -ba <the_tarball> - git tag -u <keyid> tor-0.3.x.y-status - git push origin tag tor-0.3.x.y-status + git tag -s tor-0.4.x.y-<status> + git push origin tag tor-0.4.x.y-<status> + + (You must do this before you update the website: the website scripts + rely on finding the version by tag.) - (You must do this before you update the website: it relies on finding - the version by tag.) + (If your default PGP key is not the one you want to sign with, then say + "-u <keyid>" instead of "-s".) 2. scp the tarball and its sig to the dist website, i.e. - `/srv/dist-master.torproject.org/htdocs/` on dist-master. When you want - it to go live, you run "static-update-component dist.torproject.org" - on dist-master. + `/srv/dist-master.torproject.org/htdocs/` on dist-master. Run + "static-update-component dist.torproject.org" on dist-master. In the webwml.git repository, `include/versions.wmi` and `Makefile` - to note the new version. + to note the new version. Push these changes to master. (NOTE: Due to #17805, there can only be one stable version listed at once. Nonetheless, do not call your version "alpha" if it is stable, or people will get confused.) + (NOTE: It will take a while for the website update scripts to update + the website.) + 3. Email the packagers (cc'ing tor-team) that a new tarball is up. The current list of packagers is: @@ -186,29 +203,35 @@ new Tor release: Also, email tor-packagers@lists.torproject.org. + Mention where to download the tarball (https://dist.torproject.org). + + Include a link to the changelog. + + 4. Add the version number to Trac. To do this, go to Trac, log in, select "Admin" near the top of the screen, then select "Versions" from the menu on the left. At the right, there will be an "Add version" box. By convention, we enter the version in the form "Tor: - 0.2.2.23-alpha" (or whatever the version is), and we select the date as + 0.4.0.1-alpha" (or whatever the version is), and we select the date as the date in the ChangeLog. -5. Double-check: did the version get recommended in the consensus yet? Is - the website updated? If not, don't announce until they have the - up-to-date versions, or people will get confused. +5. Wait for the download page to be updated. (If you don't do this before you + announce, people will be confused.) 6. Mail the release blurb and ChangeLog to tor-talk (development release) or tor-announce (stable). Post the changelog on the blog as well. You can generate a - blog-formatted version of the changelog with the -B option to - format-changelog. + blog-formatted version of the changelog with + `./scripts/maint/format_changelog.py --B` When you post, include an estimate of when the next TorBrowser releases will come out that include this Tor release. This will usually track https://wiki.mozilla.org/RapidRelease/Calendar , but it can vary. + For templates to use when announcing, see: + https://trac.torproject.org/projects/tor/wiki/org/teams/NetworkTeam/AnnouncementTemplates === V. Aftermath and cleanup @@ -216,7 +239,7 @@ new Tor release: `maint-x.y.z` branch to "newversion-dev", and do a `merge -s ours` merge to avoid taking that change into master. -2. Forward-port the ChangeLog (and ReleaseNotes if appropriate). +2. Forward-port the ChangeLog (and ReleaseNotes if appropriate) to the + master branch. 3. Keep an eye on the blog post, to moderate comments and answer questions. - diff --git a/doc/tor.1.txt b/doc/tor.1.txt index 581783dd65..4ff789a931 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -89,7 +89,9 @@ COMMAND-LINE OPTIONS future version. (This is a warning, not a promise.) [[opt-version]] **--version**:: - Display Tor version and exit. + Display Tor version and exit. The output is a single line of the format + "Tor version [version number]." (The version number format + is as specified in version-spec.txt.) [[opt-quiet]] **--quiet**|**--hush**:: Override the default console log. By default, Tor starts out logging @@ -598,20 +600,26 @@ GENERAL OPTIONS Otherwise the sandbox will be disabled. The option is currently an experimental feature. It only works on Linux-based operating systems, and only when Tor has been built with the libseccomp library. This option - can not be changed while tor is running. + can not be changed while tor is running. + + - When the Sandbox is 1, the following options can not be changed when tor + When the **Sandbox** is 1, the following options can not be changed when tor is running: - Address - ConnLimit - CookieAuthFile - DirPortFrontPage - ExtORPortCookieAuthFile - Logs - ServerDNSResolvConfFile - Tor must remain in client or server mode (some changes to ClientOnly and - ORPort are not allowed). - ClientOnionAuthDir and any files in it won't reload on HUP signal. + **Address**, + **ConnLimit**, + **CookieAuthFile**, + **DirPortFrontPage**, + **ExtORPortCookieAuthFile**, + **Logs**, + **ServerDNSResolvConfFile**, + **ClientOnionAuthDir** (and any files in it won't reload on HUP signal). + + + Launching new Onion Services through the control port is not supported + with current syscall sandboxing implementation. + + + Tor must remain in client or server mode (some changes to **ClientOnly** + and **ORPort** are not allowed). Currently, if **Sandbox** is 1, + **ControlPort** command "GETINFO address" will not work. + + (Default: 0) [[Socks4Proxy]] **Socks4Proxy** __host__[:__port__]:: @@ -669,7 +677,8 @@ GENERAL OPTIONS + The currently recognized domains are: general, crypto, net, config, fs, protocol, mm, http, app, control, circ, rend, bug, dir, dirserv, or, edge, - acct, hist, handshake, heartbeat, channel, sched, guard, consdiff, and dos. + acct, hist, handshake, heartbeat, channel, sched, guard, consdiff, dos, + process, pt, and btrack. Domain names are case-insensitive. + + For example, "`Log [handshake]debug [~net,~mm]info notice stdout`" sends @@ -1749,6 +1758,12 @@ The following options are useful only for clients (that is, if other clients prefer IPv4. Other things may influence the choice. This option breaks a tie to the favor of IPv6. (Default: auto) +[[ClientAutoIPv6ORPort]] **ClientAutoIPv6ORPort** **0**|**1**:: + If this option is set to 1, Tor clients randomly prefer a node's IPv4 or + IPv6 ORPort. The random preference is set every time a node is loaded + from a new consensus or bridge config. When this option is set to 1, + **ClientPreferIPv6ORPort** is ignored. (Default: 0) + [[PathsNeededToBuildCircuits]] **PathsNeededToBuildCircuits** __NUM__:: Tor clients don't build circuits for user traffic until they know about enough of the network so that they could potentially construct @@ -1790,6 +1805,28 @@ The following options are useful only for clients (that is, if Try this many simultaneous connections to download a consensus before waiting for one to complete, timeout, or error out. (Default: 3) +[[DormantClientTimeout]] **DormantClientTimeout** __N__ **minutes**|**hours**|**days**|**weeks**:: + If Tor spends this much time without any client activity, + enter a dormant state where automatic circuits are not built, and + directory information is not fetched. + Does not affect servers or onion services. Must be at least 10 minutes. + (Default: 24 hours) + +[[DormantTimeoutDisabledByIdleStreams]] **DormantTimeoutDisabledByIdleStreams** **0**|**1**:: + If true, then any open client stream (even one not reading or writing) + counts as client activity for the purpose of DormantClientTimeout. + If false, then only network activity counts. (Default: 1) + +[[DormantOnFirstStartup]] **DormantOnFirstStartup** **0**|**1**:: + If true, then the first time Tor starts up with a fresh DataDirectory, + it starts in dormant mode, and takes no actions until the user has made + a request. (This mode is recommended if installing a Tor client for a + user who might not actually use it.) If false, Tor bootstraps the first + time it is started, whether it sees a user request or not. + + + After the first time Tor starts, it begins in dormant mode if it was + dormant before, and not otherwise. (Default: 0) + SERVER OPTIONS -------------- @@ -2212,7 +2249,8 @@ is non-zero): __filename__. The file format is the same as the standard Unix "**resolv.conf**" file (7). This option, like all other ServerDNS options, only affects name lookups that your server does on behalf of clients. - (Defaults to use the system DNS configuration.) + (Defaults to use the system DNS configuration or a localhost DNS service + in case no nameservers are found in a given configuration.) [[ServerDNSAllowBrokenConfig]] **ServerDNSAllowBrokenConfig** **0**|**1**:: If this option is false, Tor exits immediately if there are problems @@ -2801,6 +2839,8 @@ The following options are used to configure a hidden service. Store data files for a hidden service in DIRECTORY. Every hidden service must have a separate directory. You may use this option multiple times to specify multiple services. If DIRECTORY does not exist, Tor will create it. + Please note that you cannot add new Onion Service to already running Tor + instance if **Sandbox** is enabled. (Note: in current versions of Tor, if DIRECTORY is a relative path, it will be relative to the current working directory of Tor instance, not to its DataDirectory. Do not @@ -3255,10 +3295,6 @@ __CacheDirectory__**/cached-microdescs** and **cached-microdescs.new**:: router. The ".new" file is an append-only journal; when it gets too large, all entries are merged into a new cached-microdescs file. -__CacheDirectory__**/cached-routers** and **cached-routers.new**:: - Obsolete versions of cached-descriptors and cached-descriptors.new. When - Tor can't find the newer files, it looks here instead. - __DataDirectory__**/state**:: A set of persistent key-value mappings. These are documented in the file. These include: diff --git a/scripts/codegen/fuzzing_include_am.py b/scripts/codegen/fuzzing_include_am.py index 3c948d87cf..a944584453 100755 --- a/scripts/codegen/fuzzing_include_am.py +++ b/scripts/codegen/fuzzing_include_am.py @@ -13,6 +13,7 @@ FUZZERS = """ iptsv2 microdesc socks + strops vrs """ @@ -23,12 +24,12 @@ FUZZING_CPPFLAGS = \ FUZZING_CFLAGS = \ $(AM_CFLAGS) $(TEST_CFLAGS) FUZZING_LDFLAG = \ - @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ @TOR_LDFLAGS_libevent@ + @TOR_LDFLAGS_zlib@ $(TOR_LDFLAGS_CRYPTLIB) @TOR_LDFLAGS_libevent@ FUZZING_LIBS = \ $(TOR_INTERNAL_TESTING_LIBS) \ $(rust_ldadd) \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \ - @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ \ + @TOR_LIBEVENT_LIBS@ $(TOR_LIBS_CRYPTLIB) \ @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ @CURVE25519_LIBS@ \ @TOR_SYSTEMD_LIBS@ \ @TOR_LZMA_LIBS@ \ diff --git a/scripts/codegen/run_trunnel.sh b/scripts/codegen/run_trunnel.sh index 428804342a..645b3c2158 100755 --- a/scripts/codegen/run_trunnel.sh +++ b/scripts/codegen/run_trunnel.sh @@ -9,9 +9,7 @@ OPTIONS="--require-version=1.5.1" # Get all .trunnel files recursively from that directory so we can support # multiple sub-directories. -for file in `find ./src/trunnel/ -name '*.trunnel'`; do - python -m trunnel ${OPTIONS} $file -done +find ./src/trunnel/ -name '*.trunnel' -exec python -m trunnel ${OPTIONS} {} \; python -m trunnel ${OPTIONS} --write-c-files --target-dir=./src/ext/trunnel/ diff --git a/scripts/maint/checkIncludes.py b/scripts/maint/checkIncludes.py index 46a3f39638..3afd9bbebe 100755 --- a/scripts/maint/checkIncludes.py +++ b/scripts/maint/checkIncludes.py @@ -33,6 +33,9 @@ else: def open_file(fname): return open(fname, 'r', encoding='utf-8') +def warn(msg): + print(msg, file=sys.stderr) + def err(msg): """ Declare that an error has happened, and remember that there has been an error. """ @@ -48,14 +51,34 @@ def fname_is_c(fname): INCLUDE_PATTERN = re.compile(r'\s*#\s*include\s+"([^"]*)"') RULES_FNAME = ".may_include" +ALLOWED_PATTERNS = [ + re.compile(r'^.*\*\.(h|inc)$'), + re.compile(r'^.*/.*\.h$'), + re.compile(r'^ext/.*\.c$'), + re.compile(r'^orconfig.h$'), + re.compile(r'^micro-revision.i$'), +] + +def pattern_is_normal(s): + for p in ALLOWED_PATTERNS: + if p.match(s): + return True + return False + class Rules(object): """ A 'Rules' object is the parsed version of a .may_include file. """ def __init__(self, dirpath): self.dirpath = dirpath + if dirpath.startswith("src/"): + self.incpath = dirpath[4:] + else: + self.incpath = dirpath self.patterns = [] self.usedPatterns = set() def addPattern(self, pattern): + if not pattern_is_normal(pattern): + warn("Unusual pattern {} in {}".format(pattern, self.dirpath)) self.patterns.append(pattern) def includeOk(self, path): @@ -86,6 +109,20 @@ class Rules(object): if p not in self.usedPatterns: print("Pattern {} in {} was never used.".format(p, self.dirpath)) + def getAllowedDirectories(self): + allowed = [] + for p in self.patterns: + m = re.match(r'^(.*)/\*\.(h|inc)$', p) + if m: + allowed.append(m.group(1)) + continue + m = re.match(r'^(.*)/[^/]*$', p) + if m: + allowed.append(m.group(1)) + continue + + return allowed + def load_include_rules(fname): """ Read a rules file from 'fname', and return it as a Rules object. """ result = Rules(os.path.split(fname)[0]) @@ -98,6 +135,9 @@ def load_include_rules(fname): return result list_unused = False +log_sorted_levels = False + +uses_dirs = { } for dirpath, dirnames, fnames in os.walk("src"): if ".may_include" in fnames: @@ -108,8 +148,34 @@ for dirpath, dirnames, fnames in os.walk("src"): if list_unused: rules.noteUnusedRules() + uses_dirs[rules.incpath] = rules.getAllowedDirectories() + if trouble: err( """To change which includes are allowed in a C file, edit the {} files in its enclosing directory.""".format(RULES_FNAME)) sys.exit(1) + +all_levels = [] + +n = 0 +while uses_dirs: + n += 0 + cur_level = [] + for k in list(uses_dirs): + uses_dirs[k] = [ d for d in uses_dirs[k] + if (d in uses_dirs and d != k)] + if uses_dirs[k] == []: + cur_level.append(k) + for k in cur_level: + del uses_dirs[k] + n += 1 + if cur_level and log_sorted_levels: + print(n, cur_level) + if n > 100: + break + +if uses_dirs: + print("There are circular .may_include dependencies in here somewhere:", + uses_dirs) + sys.exit(1) diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist index 79551948c6..60d3e7bb85 100644 --- a/scripts/maint/fallback.whitelist +++ b/scripts/maint/fallback.whitelist @@ -1,34 +1,23 @@ # updateFallbackDirs.py directory mirror whitelist # -# Format: -# IPv4:DirPort orport=<ORPort> id=<ID> [ ipv6=<IPv6>:<IPv6 ORPort> ] -# or use: -# scripts/maint/generateFallbackDirLine.py fingerprint ... +# At least one of these keys must match for a directory mirror to be included +# in the fallback list: +# id +# ipv4 +# ipv6 +# The ports and nickname are ignored. Missing or extra ipv6 addresses +# are ignored. # -# All attributes must match for the directory mirror to be included. -# If the fallback has an ipv6 key, the whitelist line must also have -# it, and vice versa, otherwise they don't match. -# (The blacklist overrides the whitelist.) - -# To replace this list with the hard-coded fallback list (for testing), use -# a command similar to: -# cat src/app/config/fallback_dirs.inc | grep \" | grep -v weight | \ -# tr -d '\n' | \ -# sed 's/"" / /g' | sed 's/""/"/g' | tr \" '\n' | grep -v '^$' \ -# > scripts/maint/fallback.whitelist +# The latest relay details from Onionoo are included in the generated list. # -# When testing before a release, exclusions due to changed details will result -# in a warning, unless the IPv4 address or port change happened recently. -# Then it is only logged at info level, as part of the eligibility check. -# Exclusions due to stability also are only shown at info level. +# To check the hard-coded fallback list (for testing), use: +# $ updateFallbackDirs.py check_existing # -# Add the number of selected, slow, and excluded relays, and compare that to -# the number of hard-coded relays. If it's less, use info-level logs to find -# out why each of the missing relays was excluded. - # If a relay operator wants their relay to be a FallbackDir, # enter the following information here: -# <IPv4>:<DirPort> orport=<ORPort> id=<ID> [ ipv6=<IPv6>:<IPv6 ORPort> ] +# <IPv4>:<DirPort> orport=<ORPort> id=<ID> ( ipv6=[<IPv6>]:<IPv6 ORPort> )? +# or use: +# scripts/maint/generateFallbackDirLine.py fingerprint ... # https://lists.torproject.org/pipermail/tor-relays/2015-December/008362.html # https://trac.torproject.org/projects/tor/ticket/22321#comment:22 @@ -57,7 +46,6 @@ # https://lists.torproject.org/pipermail/tor-relays/2015-December/008379.html # Email sent directly to teor, verified using relay contact info 91.121.84.137:4951 orport=4051 id=6DE61A6F72C1E5418A66BFED80DFB63E4C77668F -91.121.84.137:4952 orport=4052 id=9FBEB75E8BC142565F12CBBE078D63310236A334 # https://lists.torproject.org/pipermail/tor-relays/2015-December/008381.html # Sent additional emails to teor with updated relays @@ -173,8 +161,6 @@ # Email sent directly to teor, verified using relay contact info 94.242.246.24:23 orport=8080 id=EC116BCB80565A408CE67F8EC3FE3B0B02C3A065 ipv6=[2a01:608:ffff:ff07::1:24]:9004 -176.126.252.11:443 orport=9001 id=B0279A521375F3CB2AE210BDBFC645FDD2E1973A ipv6=[2a02:59e0:0:7::11]:9003 -176.126.252.12:21 orport=8080 id=379FB450010D17078B3766C2273303C358C3A442 ipv6=[2a02:59e0:0:7::12]:81 94.242.246.23:443 orport=9001 id=F65E0196C94DFFF48AFBF2F5F9E3E19AAE583FD0 ipv6=[2a01:608:ffff:ff07::1:23]:9003 85.248.227.164:444 orport=9002 id=B84F248233FEA90CAD439F292556A3139F6E1B82 ipv6=[2a00:1298:8011:212::164]:9004 85.248.227.163:443 orport=9001 id=C793AB88565DDD3C9E4C6F15CCB9D8C7EF964CE9 ipv6=[2a00:1298:8011:212::163]:9003 @@ -297,9 +283,6 @@ 151.80.42.103:9030 orport=9001 id=9007C1D8E4F03D506A4A011B907A9E8D04E3C605 ipv6=[2001:41d0:e:f67::114]:9001 # Email sent directly to teor, verified using relay contact info -5.39.92.199:80 orport=443 id=0BEA4A88D069753218EAAAD6D22EA87B9A1319D6 ipv6=[2001:41d0:8:b1c7::1]:443 - -# Email sent directly to teor, verified using relay contact info 176.31.159.231:80 orport=443 id=D5DBCC0B4F029F80C7B8D33F20CF7D97F0423BB1 176.31.159.230:80 orport=443 id=631748AFB41104D77ADBB7E5CD4F8E8AE876E683 195.154.79.128:80 orport=443 id=C697612CA5AED06B8D829FCC6065B9287212CB2F @@ -324,10 +307,14 @@ 185.66.250.141:9030 orport=9001 id=B1726B94885CE3AC3910CA8B60622B97B98E2529 # Email sent directly to teor, verified using relay contact info +# Email sent directly to Phoul 185.104.120.7:9030 orport=443 id=445F1C853966624FB3CF1E12442570DC553CC2EC ipv6=[2a06:3000::120:7]:443 -185.104.120.2:9030 orport=21 id=518FF8708698E1DA09C823C36D35DF89A2CAD956 -185.104.120.4:9030 orport=9001 id=F92B3CB9BBE0CB22409843FB1AE4DBCD5EFAC835 -185.104.120.3:9030 orport=21 id=707C1B61AC72227B34487B56D04BAA3BA1179CE8 ipv6=[2a06:3000::120:3]:21 +185.104.120.2:9030 orport=21 id=518FF8708698E1DA09C823C36D35DF89A2CAD956 ipv6=[2a06:3000::120:2]:443 +185.104.120.4:9030 orport=9001 id=F92B3CB9BBE0CB22409843FB1AE4DBCD5EFAC835 ipv6=[2a06:3000::120:4]:443 +185.104.120.3:9030 orport=21 id=707C1B61AC72227B34487B56D04BAA3BA1179CE8 ipv6=[2a06:3000::120:3]:443 +185.104.120.5:80 orport=443 id=3EBDF84DE3B16F0EBF7D51450F07913A02EFDA6C ipv6=[2a06:3000::120:5]:443 +185.104.120.60:80 orport=443 id=D05C9C7068EB5A45F670D5E38A14907EE6223141 ipv6=[2a06:3000::120:60]:443 + # Email sent directly to teor, verified using relay contact info 37.187.102.186:9030 orport=9001 id=489D94333DF66D57FFE34D9D59CC2D97E2CB0053 ipv6=[2001:41d0:a:26ba::1]:9001 @@ -379,10 +366,6 @@ 91.219.237.229:80 orport=443 id=1ECD73B936CB6E6B3CD647CC204F108D9DF2C9F7 # Email sent directly to teor, verified using relay contact info -46.101.151.222:80 orport=443 id=1DBAED235E3957DE1ABD25B4206BE71406FB61F8 -178.62.60.37:80 orport=443 id=175921396C7C426309AB03775A9930B6F611F794 - -# Email sent directly to teor, verified using relay contact info 178.62.197.82:80 orport=443 id=0D3EBA17E1C78F1E9900BABDB23861D46FCAF163 # Email sent directly to teor, verified using relay contact info @@ -406,9 +389,6 @@ # Email sent directly to teor, verified using relay contact info 5.199.142.236:9030 orport=9001 id=F4C0EDAA0BF0F7EC138746F8FEF1CE26C7860265 -# Email sent directly to teor -188.166.133.133:9030 orport=9001 id=774555642FDC1E1D4FDF2E0C31B7CA9501C5C9C7 ipv6=[2a03:b0c0:2:d0::26c0:1]:9001 # dropsy - # Email sent directly to teor, verified using relay contact info 46.8.249.10:80 orport=443 id=31670150090A7C3513CB7914B9610E786391A95D @@ -564,7 +544,6 @@ 185.100.84.82:80 orport=443 id=7D05A38E39FC5D29AFE6BE487B9B4DC9E635D09E # Email sent directly to teor, verified using relay contact info -164.132.77.175:9030 orport=9001 id=3B33F6FCA645AD4E91428A3AF7DC736AD9FB727B 78.24.75.53:9030 orport=9001 id=DEB73705B2929AE9BE87091607388939332EF123 # Email sent directly to teor, verified using relay contact info @@ -685,6 +664,7 @@ # Email sent directly to teor, verified using relay contact info # Assume details update is permanent 188.40.128.246:9030 orport=9001 id=AD19490C7DBB26D3A68EFC824F67E69B0A96E601 ipv6=[2a01:4f8:221:1ac1:dead:beef:7005:9001]:9001 # sputnik +129.13.131.140:80 orport=443 id=F2DFE5FA1E4CF54F8E761A6D304B9B4EC69BDAE8 ipv6=[2a00:1398:5:f604:cafe:cafe:cafe:9001]:443 # AlleKochenKaffee # Email sent directly to teor, verified using relay contact info 88.198.253.13:9030 orport=9001 id=DF924196D69AAE3C00C115A9CCDF7BB62A175310 ipv6=[2a01:4f8:11a:b1f::2]:9001 @@ -696,8 +676,8 @@ # Email sent directly to teor, verified using relay contact info 176.10.104.240:80 orport=443 id=0111BA9B604669E636FFD5B503F382A4B7AD6E80 176.10.104.240:8080 orport=8443 id=AD86CD1A49573D52A7B6F4A35750F161AAD89C88 -176.10.104.243:80 orport=443 id=88487BDD980BF6E72092EE690E8C51C0AA4A538C 176.10.104.243:8080 orport=8443 id=95DA61AEF23A6C851028C1AA88AD8593F659E60F +94.230.208.147:80 orport=443 id=9AA3FF35E7A549D2337E962333D366E102FE4D50 ipv6=[2a02:418:6017::147]:443 # Email sent directly to teor, verified using relay contact info 107.170.101.39:9030 orport=443 id=30973217E70AF00EBE51797FF6D9AA720A902EAA @@ -738,13 +718,14 @@ 185.220.101.24:10024 orport=20024 id=FDA70EC93DB01E3CB418CB6943B0C68464B18B4C # niftyrat # Email sent directly to teor, verified using relay contact info -64.113.32.29:9030 orport=9001 id=30C19B81981F450C402306E2E7CFB6C3F79CB6B2 +198.232.165.2:9030 orport=9001 id=30C19B81981F450C402306E2E7CFB6C3F79CB6B2 # Emails sent directly to teor, verified using relay contact info 51.254.101.242:9002 orport=9001 id=4CC9CC9195EC38645B699A33307058624F660CCF # Emails sent directly to teor, verified using relay contact info -85.214.62.48:80 orport=443 id=6A7551EEE18F78A9813096E82BF84F740D32B911 +# Updated IP https://trac.torproject.org/projects/tor/ticket/24805#comment:16 +94.130.186.5:80 orport=443 id=6A7551EEE18F78A9813096E82BF84F740D32B911 # Email sent directly to teor, verified using relay contact info 173.255.245.116:9030 orport=9001 id=91E4015E1F82DAF0121D62267E54A1F661AB6DC7 @@ -756,13 +737,11 @@ 51.254.136.195:80 orport=443 id=7BB70F8585DFC27E75D692970C0EEB0F22983A63 # Email sent directly to teor, verified using relay contact info -163.172.13.165:9030 orport=9001 id=33DA0CAB7C27812EFF2E22C9705630A54D101FEB ipv6=[2001:bc8:38cb:201::8]:9001 - -# Email sent directly to teor, verified using relay contact info 5.196.88.122:9030 orport=9001 id=0C2C599AFCB26F5CFC2C7592435924C1D63D9484 ipv6=[2001:41d0:a:fb7a::1]:9001 # Email sent directly to teor, verified using relay contact info 5.9.158.75:80 orport=443 id=1AF72E8906E6C49481A791A6F8F84F8DFEBBB2BA ipv6=[2a01:4f8:190:514a::2]:443 +5.9.158.75:9030 orport=9001 id=D11D11877769B9E617537B4B46BFB92B443DE33D ipv6=[2a01:4f8:190:514a::2]:9001 # Email sent directly to teor, verified using relay contact info 46.101.169.151:9030 orport=9001 id=D760C5B436E42F93D77EF2D969157EEA14F9B39C ipv6=[2a03:b0c0:3:d0::74f:a001]:9001 @@ -834,9 +813,9 @@ 195.154.122.54:80 orport=443 id=64E99CB34C595A02A3165484BD1215E7389322C6 # Email sent directly to teor, verified using relay contact info +# Email sent directly to Phoul 185.100.86.128:9030 orport=9001 id=9B31F1F1C1554F9FFB3455911F82E818EF7C7883 185.100.85.101:9030 orport=9001 id=4061C553CA88021B8302F0814365070AAE617270 -31.171.155.108:9030 orport=9001 id=D3E5EDDBE5159388704D6785BE51930AAFACEC6F # Email sent directly to teor, verified using relay contact info 89.163.247.43:9030 orport=9001 id=BC7ACFAC04854C77167C7D66B7E471314ED8C410 ipv6=[2001:4ba0:fff7:25::5]:9001 @@ -943,10 +922,13 @@ 199.249.223.66:80 orport=443 id=C5A53BCC174EF8FD0DCB223E4AA929FA557DEDB2 # Quintex17 # https://lists.torproject.org/pipermail/tor-relays/2017-December/013914.html +# https://lists.torproject.org/pipermail/tor-relays/2018-January/014063.html 5.196.23.64:9030 orport=9001 id=775B0FAFDE71AADC23FFC8782B7BEB1D5A92733E # Aerodynamik01 217.182.75.181:9030 orport=9001 id=EFEACD781604EB80FBC025EDEDEA2D523AEAAA2F # Aerodynamik02 193.70.43.76:9030 orport=9001 id=484A10BA2B8D48A5F0216674C8DD50EF27BC32F3 # Aerodynamik03 149.56.141.138:9030 orport=9001 id=1938EBACBB1A7BFA888D9623C90061130E63BB3F # Aerodynamik04 +54.37.73.111:9030 orport=9001 id=92412EA1B9AA887D462B51D816777002F4D58907 # Aerodynamik05 +54.37.17.235:9030 orport=9001 id=360CBA08D1E24F513162047BDB54A1015E531534 # Aerodynamik06 # https://lists.torproject.org/pipermail/tor-relays/2017-December/013917.html 104.200.20.46:80 orport=9001 id=78E2BE744A53631B4AAB781468E94C52AB73968B # bynumlawtor @@ -960,16 +942,10 @@ # Email sent directly to teor 62.210.254.132:80 orport=443 id=8456DFA94161CDD99E480C2A2992C366C6564410 # turingmachine -# Email sent directly to teor -80.127.117.180:80 orport=443 id=328E54981C6DDD7D89B89E418724A4A7881E3192 ipv6=[2001:985:e77:10::4]:443 # sjc01 - # https://lists.torproject.org/pipermail/tor-relays/2017-December/013960.html 51.15.205.214:9030 orport=9001 id=8B6556601612F1E2AFCE2A12FFFAF8482A76DD1F ipv6=[2001:bc8:4400:2500::5:b07]:9001 # titania1 51.15.205.214:9031 orport=9002 id=5E363D72488276160D062DDD2DFA25CFEBAF5EA9 ipv6=[2001:bc8:4400:2500::5:b07]:9002 # titania2 -# Email sent directly to teor -185.129.249.124:9030 orport=9001 id=1FA8F638298645BE58AC905276680889CB795A94 # treadstone - # https://lists.torproject.org/pipermail/tor-relays/2017-December/014000.html 24.117.231.229:34175 orport=45117 id=CE24412AD69444954B4015E293AE53DDDAFEA3D6 # Anosognosia @@ -995,3 +971,94 @@ # https://lists.torproject.org/pipermail/tor-relays/2018-January/014024.html 82.161.212.209:9030 orport=9001 id=4E8CE6F5651E7342C1E7E5ED031E82078134FB0D ipv6=[2001:980:d7ed:1:ff:b0ff:fe00:d0b]:9001 # ymkeo + +# https://lists.torproject.org/pipermail/tor-relays/2018-January/014055.html +37.157.255.35:9030 orport=9090 id=361D33C96D0F161275EE67E2C91EE10B276E778B # cxx4freedom + +# https://lists.torproject.org/pipermail/tor-relays/2018-January/014064.html +87.118.122.120:80 orport=443 id=A2A6616723B511D8E068BB71705191763191F6B2 # otheontelth + +# https://lists.torproject.org/pipermail/tor-relays/2018-January/014069.html +185.100.86.182:9030 orport=8080 id=E51620B90DCB310138ED89EDEDD0A5C361AAE24E # NormalCitizen + +# https://lists.torproject.org/pipermail/tor-relays/2018-January/014267.html +51.15.72.211:80 orport=9001 id=D122094E396DF8BA560843E7B983B0EA649B7DF9 ipv6=[2001:bc8:4700:2300::1b:f09]:9001 # gjtorrelay + +# Email sent directly to Phoul +185.34.33.2:9265 orport=31415 id=D71B1CA1C9DC7E8CA64158E106AD770A21160FEE # lqdn + +# Email sent directly to Phoul +78.156.110.135:9091 orport=9090 id=F48FD1AED068496D51D1384BC7497C04E4985DA6 # SkynetC2 + +# Email sent directly to Phoul +5.200.21.144:80 orport=443 id=0C039F35C2E40DCB71CD8A07E97C7FD7787D42D6 # libel +64.79.152.132:80 orport=443 id=375DCBB2DBD94E5263BC0C015F0C9E756669617E # ebola + +# https://lists.torproject.org/pipermail/tor-relays/2018-June/015524.html +132.248.241.5:9030 orport=9001 id=4661DE96D3F8E923994B05218F23760C8D7935A4 + +# https://lists.torproject.org/pipermail/tor-relays/2018-June/015522.html +96.253.78.108:80 orport=442 id=924B24AFA7F075D059E8EEB284CC400B33D3D036 + +# Email sent directly to Phoul +163.172.218.10:9030 orport=9001 id=78809B6C50CB6491DB3A72C60EC39DC85BF72D1F ipv6=[2001:bc8:3f23:1100::1]:9001 +163.172.218.10:9130 orport=9101 id=B247BA9E0AEA93E6D7BF4080CFBB964034AF2B28 ipv6=[2001:bc8:3f23:1100::1]:9101 + +# Email sent directly to Phoul +158.255.212.178:8080 orport=8443 id=D941D380E5228E7B4D372AF4D484629A96DC48B9 ipv6=[2a03:f80:ed15:158:255:212:178:2]:8443 + +# Email sent directly to Phoul +45.79.108.130:9030 orport=9001 id=AEDAC7081AE14B8D241ECF0FF17A2858AB4383D0 ipv6=[2600:3c01:e000:131::8000:0]:9001 + +# Email sent directly to Phoul +51.254.147.57:80 orport=443 id=EB80A8D52F07238B576C42CEAB98ADD084EE075E +217.182.51.248:80 orport=443 id=D6BA940D3255AB40DC5EE5B0B285FA143E1F9865 + +# https://lists.torproject.org/pipermail/tor-relays/2018-June/015541.html +195.191.81.7:9030 orport=9001 id=41A3C16269C7B63DB6EB741DBDDB4E1F586B1592 ipv6=[2a00:1908:fffc:ffff:c0a6:ccff:fe62:e1a1]:9001 +51.254.96.208:9030 orport=9001 id=8101421BEFCCF4C271D5483C5AABCAAD245BBB9D ipv6=[2001:41d0:401:3100::30dc]:9001 +163.172.154.162:9030 orport=9001 id=F741E5124CB12700DA946B78C9B2DD175D6CD2A1 ipv6=[2001:bc8:4400:2100::17:419]:9001 +51.15.78.0:9030 orport=9001 id=15BE17C99FACE24470D40AF782D6A9C692AB36D6 ipv6=[2001:bc8:4700:2300::16:c0b]:9001 +54.37.139.118:9030 orport=9001 id=90A5D1355C4B5840E950EB61E673863A6AE3ACA1 ipv6=[2001:41d0:601:1100::1b8]:9001 +51.38.65.160:9030 orport=9001 id=3CB4193EF4E239FCEDC4DC43468E0B0D6B67ACC3 ipv6=[2001:41d0:801:2000::f6e]:9001 + +# Email sent directly to Phoul +54.37.138.138:8080 orport=993 id=1576BE143D8727745BB2BCDDF183291B3C3EFEFC + +# Email sent directly to Phoul +67.215.255.140:9030 orport=9001 id=23917BB3F3994BC61F0C9D7AD19B069F9E150D26 + +# Email sent directly to Phoul +195.154.105.170:9030 orport=9001 id=E947C029087FA1C3499BEF5D4372947C51223D8F + +# Email sent directly to Phoul +23.129.64.101:80 orport=443 id=2EB20285FE55927B7AECC47BB94F22534FBC3941 ipv6=[2620:18c:0:1001::101]:443 +23.129.64.102:80 orport=443 id=CA9739E2805A3CD73CF75BBCB6785C32394240E3 ipv6=[2620:18c:0:1001::102]:443 +23.129.64.103:80 orport=443 id=8ED84B53BD9556CCBB036073A1AD508EC27CBE52 ipv6=[2620:18c:0:1001::103]:443 + +# Email sent directly to Phoul +37.139.8.104:9030 orport=9001 id=7088D485934E8A403B81531F8C90BDC75FA43C98 ipv6=[2a03:b0c0:0:1010::24c:1001]:9001 + +# Email sent directly to Phoul +178.254.7.88:9030 orpport=9001 id=85A885433E50B1874F11CEC9BE98451E24660976 + +# https://lists.torproject.org/pipermail/tor-relays/2018-August/015869.html +5.45.111.149:80 orport=443 id=D405FCCF06ADEDF898DF2F29C9348DCB623031BA ipv6=[2a03:4000:6:2388:df98:15f9:b34d:443]:443 + +# https://trac.torproject.org/projects/tor/ticket/27297 +37.252.185.182:9030 orport=8080 id=113143469021882C3A4B82F084F8125B08EE471E ipv6=[2a00:63c1:a:182::2]:8080 + +# Email sent directly to Phoul +139.99.130.178:80 orport=443 id=867B95CACD64653FEEC4D2CEFC5C49B4620307A7 + +# Email sent directly to Phoul +104.131.11.214:9030 orport=8080 id=32828476F4F84E15C42B4C360A5CD8DE4C3C2BE7 + +# Email sent directly to Phoul / Teor +178.175.139.122:80 orport=443 id=490FB3FAAF8837407D94CA7E1DEF025DEF0F3516 ipv6=[2a00:1dc0:3002::3]:443 + +# Email sent directly to Phoul +192.42.116.16:80 orport=443 id=81B75D534F91BFB7C57AB67DA10BCEF622582AE8 + +# https://lists.torproject.org/pipermail/tor-relays/2018-November/016610.html +24.117.194.80:80 orport=443 id=B6C4C9A43658F686F8892CA5666717532F72979C diff --git a/scripts/maint/pre-push.git-hook b/scripts/maint/pre-push.git-hook new file mode 100755 index 0000000000..26296023fb --- /dev/null +++ b/scripts/maint/pre-push.git-hook @@ -0,0 +1,61 @@ +#!/bin/bash + +# To install this script, copy it into .git/hooks/pre-push path in your +# local copy of git repository. Make sure it has permission to execute. +# +# This is git pre-push hook script to prevent "fixup!" and "squash!" commits +# from ending up in upstream branches (master, release-* or maint-*). +# +# The following sample script was used as starting point: +# https://github.com/git/git/blob/master/templates/hooks--pre-push.sample + +z40=0000000000000000000000000000000000000000 + +CUR_BRANCH=$(git rev-parse --abbrev-ref HEAD) +if [ "$CUR_BRANCH" != "master" ] && [[ $CUR_BRANCH != release-* ]] && + [[ $CUR_BRANCH != maint-* ]] +then + exit 0 +fi + +echo "Running pre-push hook" + +# shellcheck disable=SC2034 +while read -r local_ref local_sha remote_ref remote_sha +do + if [ "$local_sha" = $z40 ] + then + # Handle delete + : + else + if [ "$remote_sha" = $z40 ] + then + # New branch, examine all commits + range="$local_sha" + else + # Update to existing branch, examine new commits + range="$remote_sha..$local_sha" + fi + + # Check for fixup! commit + commit=$(git rev-list -n 1 --grep '^fixup!' "$range") + if [ -n "$commit" ] + then + echo >&2 "Found fixup! commit in $local_ref, not pushing" + echo >&2 "If you really want to push this, use --no-verify." + exit 1 + fi + + # Check for squash! commit + commit=$(git rev-list -n 1 --grep '^squash!' "$range") + if [ -n "$commit" ] + then + echo >&2 "Found squash! commit in $local_ref, not pushing" + echo >&2 "If you really want to push this, use --no-verify." + exit 1 + fi + fi +done + +exit 0 + diff --git a/scripts/maint/run_calltool.sh b/scripts/maint/run_calltool.sh index efb8706fea..b0268322f4 100755 --- a/scripts/maint/run_calltool.sh +++ b/scripts/maint/run_calltool.sh @@ -15,10 +15,10 @@ SUBITEMS="fn_graph fn_invgraph fn_scc fn_scc_weaklinks module_graph module_invgr for calculation in $SUBITEMS; do echo "======== $calculation" - python -m calltool $calculation > callgraph/$calculation + python -m calltool "$calculation" > callgraph/"$calculation" done -echo <<EOF > callgraph/README +cat <<EOF > callgraph/README This directory holds output from calltool, as run on Tor. For more information about each of these files, see the NOTES and README files in the calltool distribution. diff --git a/scripts/maint/updateFallbackDirs.py b/scripts/maint/updateFallbackDirs.py index 0ea3992d8f..14372d0e83 100755 --- a/scripts/maint/updateFallbackDirs.py +++ b/scripts/maint/updateFallbackDirs.py @@ -18,8 +18,8 @@ # Optionally uses ipaddress (python 3 builtin) or py2-ipaddress (package) # for netblock analysis. # -# Then read the logs to make sure the fallbacks aren't dominated by a single -# netblock or port. +# After running this script, read the logs to make sure the fallbacks aren't +# dominated by a single netblock or port. # Script by weasel, April 2015 # Portions by gsathya & karsten, 2013 @@ -39,8 +39,6 @@ import urllib import urllib2 import hashlib import dateutil.parser -# bson_lazy provides bson -#from bson import json_util import copy import re @@ -100,19 +98,29 @@ MUST_BE_RUNNING_NOW = (PERFORM_IPV4_DIRPORT_CHECKS # Clients have been using microdesc consensuses by default for a while now DOWNLOAD_MICRODESC_CONSENSUS = True -# If a relay delivers an expired consensus, if it expired less than this many -# seconds ago, we still allow the relay. This should never be less than -90, -# as all directory mirrors should have downloaded a consensus 90 minutes -# before it expires. It should never be more than 24 hours, because clients -# reject consensuses that are older than REASONABLY_LIVE_TIME. -# For the consensus expiry check to be accurate, the machine running this -# script needs an accurate clock. +# If a relay delivers an invalid consensus, if it will become valid less than +# this many seconds in the future, or expired less than this many seconds ago, +# accept the relay as a fallback. For the consensus expiry check to be +# accurate, the machine running this script needs an accurate clock. # -# Relays on 0.3.0 and later return a 404 when they are about to serve an -# expired consensus. This makes them fail the download check. -# We use a tolerance of 0, so that 0.2.x series relays also fail the download -# check if they serve an expired consensus. -CONSENSUS_EXPIRY_TOLERANCE = 0 +# Relays on 0.3.0 and later return a 404 when they are about to serve a +# consensus that expired more than 24 hours ago. 0.2.9 and earlier relays +# will serve consensuses that are very old. +# +# Relays on 0.3.5.6-rc? and later return a 404 when they are about to serve a +# consensus that will become valid more than 24 hours in the future. Older +# relays don't serve future consensuses. +# +# A 404 makes relays fail the download check. We use a tolerance of 24 hours, +# so that 0.2.9 relays also fail the download check if they serve a consensus +# that is not reasonably live. +# +# REASONABLY_LIVE_TIME should never be more than Tor's REASONABLY_LIVE_TIME, +# (24 hours), because clients reject consensuses that are older than that. +# Clients on 0.3.5.5-alpha? and earlier also won't select guards from +# consensuses that have expired, but can bootstrap if they already have guards +# in their state file. +REASONABLY_LIVE_TIME = 24*60*60 # Output fallback name, flags, bandwidth, and ContactInfo in a C comment? OUTPUT_COMMENTS = True if OUTPUT_CANDIDATES else False @@ -912,61 +920,181 @@ class Candidate(object): return False return True - def is_in_whitelist(self, relaylist): - """ A fallback matches if each key in the whitelist line matches: + def id_matches(self, id, exact=False): + """ Does this fallback's id match id? + exact is ignored. """ + return self._fpr == id + + def ipv4_addr_matches(self, ipv4_addr, exact=False): + """ Does this fallback's IPv4 address match ipv4_addr? + exact is ignored. """ + return self.dirip == ipv4_addr + + def ipv4_dirport_matches(self, ipv4_dirport, exact=False): + """ Does this fallback's IPv4 dirport match ipv4_dirport? + If exact is False, always return True. """ + if exact: + return self.dirport == int(ipv4_dirport) + else: + return True + + def ipv4_and_dirport_matches(self, ipv4_addr, ipv4_dirport, exact=False): + """ Does this fallback's IPv4 address match ipv4_addr? + If exact is True, also check ipv4_dirport. """ + ipv4_match = self.ipv4_addr_matches(ipv4_addr, exact=exact) + if exact: + return ipv4_match and self.ipv4_dirport_matches(ipv4_dirport, + exact=exact) + else: + return ipv4_match + + def ipv4_orport_matches(self, ipv4_orport, exact=False): + """ Does this fallback's IPv4 orport match ipv4_orport? + If exact is False, always return True. """ + if exact: + return self.orport == int(ipv4_orport) + else: + return True + + def ipv4_and_orport_matches(self, ipv4_addr, ipv4_orport, exact=False): + """ Does this fallback's IPv4 address match ipv4_addr? + If exact is True, also check ipv4_orport. """ + ipv4_match = self.ipv4_addr_matches(ipv4_addr, exact=exact) + if exact: + return ipv4_match and self.ipv4_orport_matches(ipv4_orport, + exact=exact) + else: + return ipv4_match + + def ipv6_addr_matches(self, ipv6_addr, exact=False): + """ Does this fallback's IPv6 address match ipv6_addr? + Both addresses must be present to match. + exact is ignored. """ + if self.has_ipv6() and ipv6_addr is not None: + # Check that we have a bracketed IPv6 address without a port + assert(ipv6_addr.startswith('[') and ipv6_addr.endswith(']')) + return self.ipv6addr == ipv6_addr + else: + return False + + def ipv6_orport_matches(self, ipv6_orport, exact=False): + """ Does this fallback's IPv6 orport match ipv6_orport? + Both ports must be present to match. + If exact is False, always return True. """ + if exact: + return (self.has_ipv6() and ipv6_orport is not None and + self.ipv6orport == int(ipv6_orport)) + else: + return True + + def ipv6_and_orport_matches(self, ipv6_addr, ipv6_orport, exact=False): + """ Does this fallback's IPv6 address match ipv6_addr? + If exact is True, also check ipv6_orport. """ + ipv6_match = self.ipv6_addr_matches(ipv6_addr, exact=exact) + if exact: + return ipv6_match and self.ipv6_orport_matches(ipv6_orport, + exact=exact) + else: + return ipv6_match + + def entry_matches_exact(self, entry): + """ Is entry an exact match for this fallback? + A fallback is an exact match for entry if each key in entry matches: ipv4 dirport orport id - ipv6 address and port (if present) + ipv6 address and port (if present in the fallback or the whitelist) If the fallback has an ipv6 key, the whitelist line must also have - it, and vice versa, otherwise they don't match. """ - ipv6 = None - if self.has_ipv6(): - ipv6 = '%s:%d'%(self.ipv6addr, self.ipv6orport) - for entry in relaylist: - if entry['id'] != self._fpr: - # can't log here unless we match an IP and port, because every relay's - # fingerprint is compared to every entry's fingerprint - if entry['ipv4'] == self.dirip and int(entry['orport']) == self.orport: - logging.warning('%s excluded: has OR %s:%d changed fingerprint to ' + - '%s?', entry['id'], self.dirip, self.orport, - self._fpr) - if self.has_ipv6() and entry.has_key('ipv6') and entry['ipv6'] == ipv6: - logging.warning('%s excluded: has OR %s changed fingerprint to ' + - '%s?', entry['id'], ipv6, self._fpr) - continue - if entry['ipv4'] != self.dirip: - logging.warning('%s excluded: has it changed IPv4 from %s to %s?', - self._fpr, entry['ipv4'], self.dirip) - continue - if int(entry['dirport']) != self.dirport: - logging.warning('%s excluded: has it changed DirPort from %s:%d to ' + - '%s:%d?', self._fpr, self.dirip, int(entry['dirport']), - self.dirip, self.dirport) - continue - if int(entry['orport']) != self.orport: - logging.warning('%s excluded: has it changed ORPort from %s:%d to ' + - '%s:%d?', self._fpr, self.dirip, int(entry['orport']), - self.dirip, self.orport) - continue - if entry.has_key('ipv6') and self.has_ipv6(): - # if both entry and fallback have an ipv6 address, compare them - if entry['ipv6'] != ipv6: - logging.warning('%s excluded: has it changed IPv6 ORPort from %s ' + - 'to %s?', self._fpr, entry['ipv6'], ipv6) - continue - # if the fallback has an IPv6 address but the whitelist entry - # doesn't, or vice versa, the whitelist entry doesn't match - elif entry.has_key('ipv6') and not self.has_ipv6(): - logging.warning('%s excluded: has it lost its former IPv6 address %s?', - self._fpr, entry['ipv6']) - continue - elif not entry.has_key('ipv6') and self.has_ipv6(): - logging.warning('%s excluded: has it gained an IPv6 address %s?', - self._fpr, ipv6) - continue + it, otherwise they don't match. + + Logs a warning-level message if the fallback would be an exact match, + but one of the id, ipv4, ipv4 orport, ipv4 dirport, or ipv6 orport + have changed. """ + if not self.id_matches(entry['id'], exact=True): + # can't log here unless we match an IP and port, because every relay's + # fingerprint is compared to every entry's fingerprint + if self.ipv4_and_orport_matches(entry['ipv4'], + entry['orport'], + exact=True): + logging.warning('%s excluded: has OR %s:%d changed fingerprint to ' + + '%s?', entry['id'], self.dirip, self.orport, + self._fpr) + if self.ipv6_and_orport_matches(entry.get('ipv6_addr'), + entry.get('ipv6_orport'), + exact=True): + logging.warning('%s excluded: has OR %s changed fingerprint to ' + + '%s?', entry['id'], entry['ipv6'], self._fpr) + return False + if not self.ipv4_addr_matches(entry['ipv4'], exact=True): + logging.warning('%s excluded: has it changed IPv4 from %s to %s?', + self._fpr, entry['ipv4'], self.dirip) + return False + if not self.ipv4_dirport_matches(entry['dirport'], exact=True): + logging.warning('%s excluded: has it changed DirPort from %s:%d to ' + + '%s:%d?', self._fpr, self.dirip, int(entry['dirport']), + self.dirip, self.dirport) + return False + if not self.ipv4_orport_matches(entry['orport'], exact=True): + logging.warning('%s excluded: has it changed ORPort from %s:%d to ' + + '%s:%d?', self._fpr, self.dirip, int(entry['orport']), + self.dirip, self.orport) + return False + if entry.has_key('ipv6') and self.has_ipv6(): + # if both entry and fallback have an ipv6 address, compare them + if not self.ipv6_and_orport_matches(entry['ipv6_addr'], + entry['ipv6_orport'], + exact=True): + logging.warning('%s excluded: has it changed IPv6 ORPort from %s ' + + 'to %s:%d?', self._fpr, entry['ipv6'], + self.ipv6addr, self.ipv6orport) + return False + # if the fallback has an IPv6 address but the whitelist entry + # doesn't, or vice versa, the whitelist entry doesn't match + elif entry.has_key('ipv6') and not self.has_ipv6(): + logging.warning('%s excluded: has it lost its former IPv6 address %s?', + self._fpr, entry['ipv6']) + return False + elif not entry.has_key('ipv6') and self.has_ipv6(): + logging.warning('%s excluded: has it gained an IPv6 address %s:%d?', + self._fpr, self.ipv6addr, self.ipv6orport) + return False + return True + + def entry_matches_fuzzy(self, entry): + """ Is entry a fuzzy match for this fallback? + A fallback is a fuzzy match for entry if at least one of these keys + in entry matches: + id + ipv4 + ipv6 (if present in both the fallback and whitelist) + The ports and nickname are ignored. Missing or extra ipv6 addresses + are ignored. + + Doesn't log any warning messages. """ + if self.id_matches(entry['id'], exact=False): return True + if self.ipv4_addr_matches(entry['ipv4'], exact=False): + return True + if entry.has_key('ipv6') and self.has_ipv6(): + # if both entry and fallback have an ipv6 address, compare them + if self.ipv6_addr_matches(entry['ipv6_addr'], exact=False): + return True + return False + + def is_in_whitelist(self, relaylist, exact=False): + """ If exact is True (existing fallback list), check if this fallback is + an exact match for any whitelist entry, using entry_matches_exact(). + + If exact is False (new fallback whitelist), check if this fallback is + a fuzzy match for any whitelist entry, using entry_matches_fuzzy(). """ + for entry in relaylist: + if exact: + if self.entry_matches_exact(entry): + return True + else: + if self.entry_matches_fuzzy(entry): + return True return False def cw_to_bw_factor(self): @@ -1124,6 +1252,7 @@ class Candidate(object): ).run()[0] end = datetime.datetime.utcnow() time_since_expiry = (end - consensus.valid_until).total_seconds() + time_until_valid = (consensus.valid_after - end).total_seconds() except Exception, stem_error: end = datetime.datetime.utcnow() log_excluded('Unable to retrieve a consensus from %s: %s', nickname, @@ -1141,8 +1270,17 @@ class Candidate(object): download_failed = True elif (time_since_expiry > 0): status = 'outdated consensus, expired %ds ago'%(int(time_since_expiry)) - if time_since_expiry <= CONSENSUS_EXPIRY_TOLERANCE: - status += ', tolerating up to %ds'%(CONSENSUS_EXPIRY_TOLERANCE) + if time_since_expiry <= REASONABLY_LIVE_TIME: + status += ', tolerating up to %ds'%(REASONABLY_LIVE_TIME) + level = logging.INFO + else: + status += ', invalid' + level = logging.WARNING + download_failed = True + elif (time_until_valid > 0): + status = 'future consensus, valid in %ds'%(int(time_until_valid)) + if time_until_valid <= REASONABLY_LIVE_TIME: + status += ', tolerating up to %ds'%(REASONABLY_LIVE_TIME) level = logging.INFO else: status += ', invalid' @@ -1400,7 +1538,7 @@ class CandidateList(dict): each line's key/value pairs are placed in a dictonary, (of string -> string key/value pairs), and these dictionaries are placed in an array. - comments start with # and are ignored """ + comments start with # and are ignored. """ file_data = file_obj['data'] file_name = file_obj['name'] relaylist = [] @@ -1440,18 +1578,28 @@ class CandidateList(dict): relay_entry['dirport'] = ipv4_maybe_dirport_split[1] elif kvl == 2: relay_entry[key_value_split[0]] = key_value_split[1] + # split ipv6 addresses and orports + if key_value_split[0] == 'ipv6': + ipv6_orport_split = key_value_split[1].rsplit(':', 1) + ipv6l = len(ipv6_orport_split) + if ipv6l != 2: + print '#error Bad %s IPv6 item: %s, format is [ipv6]:orport.'%( + file_name, item) + relay_entry['ipv6_addr'] = ipv6_orport_split[0] + relay_entry['ipv6_orport'] = ipv6_orport_split[1] relaylist.append(relay_entry) return relaylist - # apply the fallback whitelist - def apply_filter_lists(self, whitelist_obj): + def apply_filter_lists(self, whitelist_obj, exact=False): + """ Apply the fallback whitelist_obj to this fallback list, + passing exact to is_in_whitelist(). """ excluded_count = 0 logging.debug('Applying whitelist') # parse the whitelist whitelist = self.load_relaylist(whitelist_obj) filtered_fallbacks = [] for f in self.fallbacks: - in_whitelist = f.is_in_whitelist(whitelist) + in_whitelist = f.is_in_whitelist(whitelist, exact=exact) if in_whitelist: # include filtered_fallbacks.append(f) @@ -2064,14 +2212,14 @@ def process_existing(): logging.getLogger('stem').setLevel(logging.INFO) whitelist = {'data': parse_fallback_file(FALLBACK_FILE_NAME), 'name': FALLBACK_FILE_NAME} - list_fallbacks(whitelist) + list_fallbacks(whitelist, exact=True) def process_default(): logging.basicConfig(level=logging.WARNING) logging.getLogger('stem').setLevel(logging.WARNING) whitelist = {'data': read_from_file(WHITELIST_FILE_NAME, MAX_LIST_FILE_SIZE), 'name': WHITELIST_FILE_NAME} - list_fallbacks(whitelist) + list_fallbacks(whitelist, exact=False) ## Main Function def main(): @@ -2092,10 +2240,10 @@ def log_excluded(msg, *args): else: logging.info(msg, *args) -def list_fallbacks(whitelist): +def list_fallbacks(whitelist, exact=False): """ Fetches required onionoo documents and evaluates the - fallback directory criteria for each of the relays """ - + fallback directory criteria for each of the relays, + passing exact to apply_filter_lists(). """ print "/* type=fallback */" print ("/* version={} */" .format(cleanse_c_multiline_comment(FALLBACK_FORMAT_VERSION))) @@ -2135,7 +2283,7 @@ def list_fallbacks(whitelist): # warning that the details have changed from those in the whitelist. # instead, there will be an info-level log during the eligibility check. initial_count = len(candidates.fallbacks) - excluded_count = candidates.apply_filter_lists(whitelist) + excluded_count = candidates.apply_filter_lists(whitelist, exact=exact) print candidates.summarise_filters(initial_count, excluded_count) eligible_count = len(candidates.fallbacks) diff --git a/scripts/maint/updateRustDependencies.sh b/scripts/maint/updateRustDependencies.sh index a5a92579d3..6d0587351f 100755 --- a/scripts/maint/updateRustDependencies.sh +++ b/scripts/maint/updateRustDependencies.sh @@ -20,26 +20,26 @@ set -e -HERE=`dirname $(realpath $0)` -TOPLEVEL=`dirname $(dirname $HERE)` +HERE=$(dirname "$(realpath "$0")") +TOPLEVEL=$(dirname "$(dirname "$HERE")") TOML="$TOPLEVEL/src/rust/Cargo.toml" VENDORED="$TOPLEVEL/src/ext/rust/crates" -CARGO=`which cargo` +CARGO=$(command -v cargo) if ! test -f "$TOML" ; then - printf "Error: Couldn't find workspace Cargo.toml in expected location: %s\n" "$TOML" + printf "Error: Couldn't find workspace Cargo.toml in expected location: %s\\n" "$TOML" fi if ! test -d "$VENDORED" ; then - printf "Error: Couldn't find directory for Rust dependencies! Expected location: %s\n" "$VENDORED" + printf "Error: Couldn't find directory for Rust dependencies! Expected location: %s\\n" "$VENDORED" fi if test -z "$CARGO" ; then - printf "Error: cargo must be installed and in your \$PATH\n" + printf "Error: cargo must be installed and in your \$PATH\\n" fi -if test -z `cargo --list | grep vendor` ; then - printf "Error: cargo-vendor not installed\n" +if test -z "$(cargo --list | grep vendor)" ; then + printf "Error: cargo-vendor not installed\\n" fi -$CARGO vendor -v --locked --explicit-version --no-delete --sync $TOML $VENDORED +$CARGO vendor -v --locked --explicit-version --no-delete --sync "$TOML" "$VENDORED" diff --git a/scripts/maint/updateVersions.pl.in b/scripts/maint/updateVersions.pl.in deleted file mode 100755 index 65c51a1f2d..0000000000 --- a/scripts/maint/updateVersions.pl.in +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/perl -w - -$CONFIGURE_IN = '@abs_top_srcdir@/configure.ac'; -$ORCONFIG_H = '@abs_top_srcdir@/src/win32/orconfig.h'; -$TOR_NSI = '@abs_top_srcdir@/contrib/win32build/tor-mingw.nsi.in'; - -$quiet = 1; - -sub demand { - my $fn = shift; - die "Missing file $fn" unless (-f $fn); -} - -demand($CONFIGURE_IN); -demand($ORCONFIG_H); -demand($TOR_NSI); - -# extract version from configure.ac - -open(F, $CONFIGURE_IN) or die "$!"; -$version = undef; -while (<F>) { - if (/AC_INIT\(\[tor\],\s*\[([^\]]*)\]\)/) { - $version = $1; - last; - } -} -die "No version found" unless $version; -print "Tor version is $version\n" unless $quiet; -close F; - -sub correctversion { - my ($fn, $defchar) = @_; - undef $/; - open(F, $fn) or die "$!"; - my $s = <F>; - close F; - if ($s =~ /^$defchar(?:)define\s+VERSION\s+\"([^\"]+)\"/m) { - $oldver = $1; - if ($oldver ne $version) { - print "Version mismatch in $fn: It thinks that the version is $oldver. I think it's $version. Fixing.\n"; - $line = $defchar . "define VERSION \"$version\""; - open(F, ">$fn.bak"); - print F $s; - close F; - $s =~ s/^$defchar(?:)define\s+VERSION.*?$/$line/m; - open(F, ">$fn"); - print F $s; - close F; - } else { - print "$fn has the correct version. Good.\n" unless $quiet; - } - } else { - print "Didn't find a version line in $fn -- uh oh.\n"; - } -} - -correctversion($TOR_NSI, "!"); -correctversion($ORCONFIG_H, "#"); diff --git a/scripts/maint/update_versions.py b/scripts/maint/update_versions.py new file mode 100755 index 0000000000..8067f2c6c8 --- /dev/null +++ b/scripts/maint/update_versions.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python + +from __future__ import print_function + +import io +import os +import re +import sys +import time + +def P(path): + """ + Give 'path' as a path relative to the abs_top_srcdir environment + variable. + """ + return os.path.join( + os.environ.get('abs_top_srcdir', "."), + path) + +def warn(msg): + """ + Print an warning message. + """ + print("WARNING: {}".format(msg), file=sys.stderr) + +def find_version(infile): + """ + Given an open file (or some other iterator of lines) holding a + configure.ac file, find the current version line. + """ + for line in infile: + m = re.search(r'AC_INIT\(\[tor\],\s*\[([^\]]*)\]\)', line) + if m: + return m.group(1) + + return None + +def update_version_in(infile, outfile, regex, versionline): + """ + Copy every line from infile to outfile. If any line matches 'regex', + replace it with 'versionline'. Return True if any line was changed; + false otherwise. + + 'versionline' is either a string -- in which case it is used literally, + or a function that receives the output of 'regex.match'. + """ + found = False + have_changed = False + for line in infile: + m = regex.match(line) + if m: + found = True + oldline = line + if type(versionline) == type(u""): + line = versionline + else: + line = versionline(m) + if not line.endswith("\n"): + line += "\n" + if oldline != line: + have_changed = True + outfile.write(line) + + if not found: + warn("didn't find any version line to replace in {}".format(infile.name)) + + return have_changed + +def replace_on_change(fname, change): + """ + If "change" is true, replace fname with fname.tmp. Otherwise, + delete fname.tmp. Log what we're doing to stderr. + """ + if not change: + print("No change in {}".format(fname)) + os.unlink(fname+".tmp") + else: + print("Updating {}".format(fname)) + os.rename(fname+".tmp", fname) + + +def update_file(fname, + regex, + versionline, + encoding="utf-8"): + """ + Replace any line matching 'regex' in 'fname' with 'versionline'. + Do not modify 'fname' if there are no changes made. Use the + provided encoding to read and write. + """ + with io.open(fname, "r", encoding=encoding) as f, \ + io.open(fname+".tmp", "w", encoding=encoding) as outf: + have_changed = update_version_in(f, outf, regex, versionline) + + replace_on_change(fname, have_changed) + +# Find out our version +with open("configure.ac") as f: + version = find_version(f) + +# If we have no version, we can't proceed. +if version == None: + print("No version found in configure.ac", file=sys.stderr()) + sys.exit(1) + +print("The version is {}".format(version)) + +today = time.strftime("%Y-%m-%d", time.gmtime()) + +# In configure.ac, we replace the definition of APPROX_RELEASE_DATE +# with "{today} for {version}", but only if the version does not match +# what is already there. +def replace_fn(m): + if m.group(1) != version: + # The version changed -- we change the date. + return u'AC_DEFINE(APPROX_RELEASE_DATE, ["{}"], # for {}'.format(today, version) + else: + # No changes. + return m.group(0) +update_file(P("configure.ac"), + re.compile(r'AC_DEFINE\(APPROX_RELEASE_DATE.* for (.*)'), + replace_fn) + +# In tor-mingw.nsi.in, we replace the definition of VERSION. +update_file(P("contrib/win32build/tor-mingw.nsi.in"), + re.compile(r'!define VERSION .*'), + u'!define VERSION "{}"'.format(version), + encoding="iso-8859-1") + +# In src/win32/orconfig.h, we replace the definition of VERSION. +update_file(P("src/win32/orconfig.h"), + re.compile(r'#define VERSION .*'), + u'#define VERSION "{}"'.format(version)) diff --git a/scripts/test/chutney-git-bisect.sh b/scripts/test/chutney-git-bisect.sh index 8a3f2c70c8..dc1319a27a 100755 --- a/scripts/test/chutney-git-bisect.sh +++ b/scripts/test/chutney-git-bisect.sh @@ -20,7 +20,7 @@ if [ ! -z "$1" ]; then fi if [ ! -z "$2" ]; then - cd "$2" + cd "$2" || exit fi CHUTNEY_TEST_CMD="make test-network-all" @@ -54,9 +54,9 @@ while [ "$i" -le "$CHUTNEY_TRIES" ]; do echo "test '$CHUTNEY_TEST_CMD' succeeded after $i/$CHUTNEY_TRIES attempts, good" exit 0 fi - i=$[$i+1] + i=$((i+1)) done -i=$[$i-1] +i=$((i-1)) echo "test '$CHUTNEY_TEST_CMD' failed $i/$CHUTNEY_TRIES attempts, bad" exit 1 diff --git a/scripts/test/cov-diff b/scripts/test/cov-diff index 6179dff63e..f3ca856888 100755 --- a/scripts/test/cov-diff +++ b/scripts/test/cov-diff @@ -7,9 +7,9 @@ DIRA="$1" DIRB="$2" -for B in $DIRB/*; do - A=$DIRA/`basename $B` - if [ -f $A ]; then +for B in "$DIRB"/*; do + A=$DIRA/$(basename "$B") + if [ -f "$A" ]; then perl -pe 's/^\s*\!*\d+(\*?):/ 1$1:/; s/^([^:]+:)[\d\s]+:/$1/; s/^ *-:(Runs|Programs):.*//;' "$A" > "$A.tmp" else cat /dev/null > "$A.tmp" diff --git a/scripts/test/coverage b/scripts/test/coverage index 59d468ee1e..7a417cdc89 100755 --- a/scripts/test/coverage +++ b/scripts/test/coverage @@ -8,30 +8,30 @@ dst=$1 for fn in src/core/*/*.c src/feature/*/*.c src/app/*/*.c src/lib/*/*.c; do - BN=`basename $fn` - DN=`dirname $fn` - F=`echo $BN | sed -e 's/\.c$//;'` + BN=$(basename "$fn") + DN=$(dirname "$fn") + F=$(echo "$BN" | sed -e 's/\.c$//;') GC="${BN}.gcov" # Figure out the object file names - ONS=`echo ${DN}/src_*-${F}.o` + ONS=$(echo "${DN}"/src_*-"${F}".o) ONS_WILDCARD_LITERAL="${DN}/src_*-${F}.o" # If the wildcard didn't expand, no files if [ "$ONS" != "${ONS_WILDCARD_LITERAL}" ] then for on in $ONS; do # We should have a gcno file - GCNO=`echo $on | sed -e 's/\.o$/\.gcno/;'` - if [ -e $GCNO ] + GCNO=$(echo "$on" | sed -e 's/\.o$/\.gcno/;') + if [ -e "$GCNO" ] then # No need to test for gcda, since gcov assumes no execution # if it's absent - rm -f $GC - gcov -o $on $fn - if [ -e $GC ] + rm -f "$GC" + gcov -o "$on" "$fn" + if [ -e "$GC" ] then if [ -d "$dst" ] then - mv $GC $dst/$GC + mv "$GC" "$dst"/"$GC" fi else echo "gcov -o $on $fn didn't make a .gcov file" diff --git a/scripts/test/scan-build.sh b/scripts/test/scan-build.sh index 8d126cbcee..26e05ff101 100755 --- a/scripts/test/scan-build.sh +++ b/scripts/test/scan-build.sh @@ -33,6 +33,7 @@ CHECKERS="\ -enable-checker security.insecureAPI.strcpy \ " +# shellcheck disable=SC2034 # These have high false-positive rates. EXTRA_CHECKERS="\ -enable-checker alpha.security.ArrayBoundV2 \ @@ -40,6 +41,7 @@ EXTRA_CHECKERS="\ -enable-checker alpha.core.CastSize \ " +# shellcheck disable=SC2034 # These don't seem to generate anything useful NOISY_CHECKERS="\ -enable-checker alpha.clone.CloneChecker \ @@ -52,6 +54,7 @@ else OUTPUTARG="" fi +# shellcheck disable=SC2086 scan-build \ $CHECKERS \ ./configure @@ -61,11 +64,13 @@ scan-build \ # Make this not get scanned for dead assignments, since it has lots of # dead assignments we don't care about. +# shellcheck disable=SC2086 scan-build \ $CHECKERS \ -disable-checker deadcode.DeadStores \ make -j5 -k ./src/ext/ed25519/ref10/libed25519_ref10.a +# shellcheck disable=SC2086 scan-build \ $CHECKERS $OUTPUTARG \ make -j5 -k diff --git a/src/app/config/config.c b/src/app/config/config.c index 4a8f94da0e..32b487dd24 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -64,6 +64,7 @@ #include "app/config/confparse.h" #include "app/config/statefile.h" #include "app/main/main.h" +#include "app/main/subsysmgr.h" #include "core/mainloop/connection.h" #include "core/mainloop/cpuworker.h" #include "core/mainloop/mainloop.h" @@ -112,9 +113,9 @@ #include "lib/crypt_ops/crypto_rand.h" #include "lib/crypt_ops/crypto_util.h" #include "lib/encoding/confline.h" -#include "lib/log/git_revision.h" #include "lib/net/resolve.h" #include "lib/sandbox/sandbox.h" +#include "lib/version/torversion.h" #ifdef ENABLE_NSS #include "lib/crypt_ops/crypto_nss_mgt.h" @@ -141,7 +142,7 @@ #include "lib/process/pidfile.h" #include "lib/process/restrict.h" #include "lib/process/setuid.h" -#include "lib/process/subprocess.h" +#include "lib/process/process.h" #include "lib/net/gethostname.h" #include "lib/thread/numcpus.h" @@ -339,6 +340,7 @@ static config_var_t option_vars_[] = { V(ClientOnly, BOOL, "0"), V(ClientPreferIPv6ORPort, AUTOBOOL, "auto"), V(ClientPreferIPv6DirPort, AUTOBOOL, "auto"), + V(ClientAutoIPv6ORPort, BOOL, "0"), V(ClientRejectInternalAddresses, BOOL, "1"), V(ClientTransportPlugin, LINELIST, NULL), V(ClientUseIPv6, BOOL, "0"), @@ -388,6 +390,9 @@ static config_var_t option_vars_[] = { OBSOLETE("DynamicDHGroups"), VPORT(DNSPort), OBSOLETE("DNSListenAddress"), + V(DormantClientTimeout, INTERVAL, "24 hours"), + V(DormantTimeoutDisabledByIdleStreams, BOOL, "1"), + V(DormantOnFirstStartup, BOOL, "0"), /* DoS circuit creation options. */ V(DoSCircuitCreationEnabled, AUTOBOOL, "auto"), V(DoSCircuitCreationMinConnections, UINT, "0"), @@ -972,42 +977,6 @@ set_options(or_options_t *new_val, char **msg) return 0; } -/** The version of this Tor process, as parsed. */ -static char *the_tor_version = NULL; -/** A shorter version of this Tor process's version, for export in our router - * descriptor. (Does not include the git version, if any.) */ -static char *the_short_tor_version = NULL; - -/** Return the current Tor version. */ -const char * -get_version(void) -{ - if (the_tor_version == NULL) { - if (strlen(tor_git_revision)) { - tor_asprintf(&the_tor_version, "%s (git-%s)", get_short_version(), - tor_git_revision); - } else { - the_tor_version = tor_strdup(get_short_version()); - } - } - return the_tor_version; -} - -/** Return the current Tor version, without any git tag. */ -const char * -get_short_version(void) -{ - - if (the_short_tor_version == NULL) { -#ifdef TOR_BUILD_TAG - tor_asprintf(&the_short_tor_version, "%s (%s)", VERSION, TOR_BUILD_TAG); -#else - the_short_tor_version = tor_strdup(VERSION); -#endif - } - return the_short_tor_version; -} - /** Release additional memory allocated in options */ STATIC void @@ -1067,9 +1036,6 @@ config_free_all(void) tor_free(torrc_defaults_fname); tor_free(global_dirfrontpagecontents); - tor_free(the_short_tor_version); - tor_free(the_tor_version); - cleanup_protocol_warning_severity_level(); have_parsed_cmdline = 0; @@ -1432,10 +1398,10 @@ options_act_reversible(const or_options_t *old_options, char **msg) * processes. */ if (running_tor && options->RunAsDaemon) { if (! start_daemon_has_been_called()) - crypto_prefork(); + subsystems_prefork(); /* No need to roll back, since you can't change the value. */ if (start_daemon()) - crypto_postfork(); + subsystems_postfork(); } #ifdef HAVE_SYSTEMD @@ -2031,9 +1997,6 @@ options_act(const or_options_t *old_options) finish_daemon(options->DataDirectory); } - /* See whether we need to enable/disable our once-a-second timer. */ - reschedule_per_second_timer(); - /* We want to reinit keys as needed before we do much of anything else: keys are important, and other things can depend on them. */ if (transition_affects_workers || @@ -3421,6 +3384,8 @@ options_validate(or_options_t *old_options, or_options_t *options, if (ContactInfo && !string_is_utf8(ContactInfo, strlen(ContactInfo))) REJECT("ContactInfo config option must be UTF-8."); + check_network_configuration(server_mode(options)); + /* Special case on first boot if no Log options are given. */ if (!options->Logs && !options->RunAsDaemon && !from_setconf) { if (quiet_level == 0) @@ -3877,6 +3842,10 @@ options_validate(or_options_t *old_options, or_options_t *options, "default."); } + if (options->DormantClientTimeout < 10*60 && !options->TestingTorNetwork) { + REJECT("DormantClientTimeout is too low. It must be at least 10 minutes."); + } + if (options->PathBiasNoticeRate > 1.0) { tor_asprintf(msg, "PathBiasNoticeRate is too high. " @@ -3928,7 +3897,8 @@ options_validate(or_options_t *old_options, or_options_t *options, } if (options->HeartbeatPeriod && - options->HeartbeatPeriod < MIN_HEARTBEAT_PERIOD) { + options->HeartbeatPeriod < MIN_HEARTBEAT_PERIOD && + !options->TestingTorNetwork) { log_warn(LD_CONFIG, "HeartbeatPeriod option is too short; " "raising to %d seconds.", MIN_HEARTBEAT_PERIOD); options->HeartbeatPeriod = MIN_HEARTBEAT_PERIOD; diff --git a/src/app/config/config.h b/src/app/config/config.h index a169cfd451..4c497b83a6 100644 --- a/src/app/config/config.h +++ b/src/app/config/config.h @@ -41,8 +41,6 @@ const char *escaped_safe_str_client(const char *address); const char *escaped_safe_str(const char *address); void init_protocol_warning_severity_level(void); int get_protocol_warning_severity_level(void); -const char *get_version(void); -const char *get_short_version(void); /** An error from options_trial_assign() or options_init_from_string(). */ typedef enum setopt_err_t { diff --git a/src/app/config/or_options_st.h b/src/app/config/or_options_st.h index 3524b99b53..c2bc1079a5 100644 --- a/src/app/config/or_options_st.h +++ b/src/app/config/or_options_st.h @@ -666,6 +666,9 @@ struct or_options_t { * accessing this value directly. */ int ClientPreferIPv6DirPort; + /** If true, prefer an IPv4 or IPv6 OR port at random. */ + int ClientAutoIPv6ORPort; + /** The length of time that we think a consensus should be fresh. */ int V3AuthVotingInterval; /** The length of time we think it will take to distribute votes. */ @@ -1072,6 +1075,20 @@ struct or_options_t { /** Autobool: Do we refuse single hop client rendezvous? */ int DoSRefuseSingleHopClientRendezvous; + + /** Interval: how long without activity does it take for a client + * to become dormant? + **/ + int DormantClientTimeout; + + /** Boolean: true if having an idle stream is sufficient to prevent a client + * from becoming dormant. + **/ + int DormantTimeoutDisabledByIdleStreams; + + /** Boolean: true if Tor should be dormant the first time it starts with + * a datadirectory; false otherwise. */ + int DormantOnFirstStartup; }; #endif diff --git a/src/app/config/or_state_st.h b/src/app/config/or_state_st.h index d95df6236b..00968d3731 100644 --- a/src/app/config/or_state_st.h +++ b/src/app/config/or_state_st.h @@ -87,6 +87,13 @@ struct or_state_t { /** When did we last rotate our onion key? "0" for 'no idea'. */ time_t LastRotatedOnionKey; + + /** Number of minutes since the last user-initiated request (as defined by + * the dormant net-status system.) Set to zero if we are dormant. */ + int MinutesSinceUserActivity; + /** True if we were dormant when we last wrote the file; false if we + * weren't. "auto" on initial startup. */ + int Dormant; }; #endif diff --git a/src/app/config/statefile.c b/src/app/config/statefile.c index 8a8b7ced01..97b96f1149 100644 --- a/src/app/config/statefile.c +++ b/src/app/config/statefile.c @@ -34,6 +34,7 @@ #include "app/config/config.h" #include "app/config/confparse.h" #include "core/mainloop/mainloop.h" +#include "core/mainloop/netstatus.h" #include "core/mainloop/connection.h" #include "feature/control/control.h" #include "feature/client/entrynodes.h" @@ -45,6 +46,7 @@ #include "app/config/statefile.h" #include "lib/encoding/confline.h" #include "lib/net/resolve.h" +#include "lib/version/torversion.h" #include "app/config/or_state_st.h" @@ -131,6 +133,9 @@ static config_var_t state_vars_[] = { VAR("CircuitBuildTimeBin", LINELIST_S, BuildtimeHistogram, NULL), VAR("BuildtimeHistogram", LINELIST_V, BuildtimeHistogram, NULL), + V(MinutesSinceUserActivity, UINT, NULL), + V(Dormant, AUTOBOOL, "auto"), + END_OF_CONFIG_VARS }; @@ -308,6 +313,8 @@ or_state_set(or_state_t *new_state) get_circuit_build_times_mutable(),global_state) < 0) { ret = -1; } + netstatus_load_from_state(global_state, time(NULL)); + return ret; } @@ -499,6 +506,8 @@ or_state_save(time_t now) entry_guards_update_state(global_state); rep_hist_update_state(global_state); circuit_build_times_update_state(get_circuit_build_times(), global_state); + netstatus_flush_to_state(global_state, now); + if (accounting_is_enabled(get_options())) accounting_run_housekeeping(now); diff --git a/src/app/main/main.c b/src/app/main/main.c index a2b7c08456..d71e43ec30 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -15,6 +15,7 @@ #include "app/config/statefile.h" #include "app/main/main.h" #include "app/main/ntmain.h" +#include "app/main/subsysmgr.h" #include "core/mainloop/connection.h" #include "core/mainloop/cpuworker.h" #include "core/mainloop/mainloop.h" @@ -33,6 +34,7 @@ #include "core/or/relay.h" #include "core/or/scheduler.h" #include "core/or/status.h" +#include "core/or/versions.h" #include "feature/api/tor_api.h" #include "feature/api/tor_api_internal.h" #include "feature/client/addressmap.h" @@ -65,11 +67,11 @@ #include "feature/stats/predict_ports.h" #include "feature/stats/rephist.h" #include "lib/compress/compress.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/crypt_ops/crypto_rand.h" #include "lib/crypt_ops/crypto_s2k.h" -#include "lib/err/backtrace.h" #include "lib/geoip/geoip.h" +#include "lib/net/resolve.h" #include "lib/process/waitpid.h" @@ -83,6 +85,7 @@ #include "lib/encoding/confline.h" #include "lib/evloop/timers.h" #include "lib/crypt_ops/crypto_init.h" +#include "lib/version/torversion.h" #include <event2/event.h> @@ -301,6 +304,19 @@ process_signal(int sig) log_heartbeat(time(NULL)); control_event_signal(sig); break; + case SIGACTIVE: + /* "SIGACTIVE" counts as ersatz user activity. */ + note_user_activity(approx_time()); + control_event_signal(sig); + break; + case SIGDORMANT: + /* "SIGDORMANT" means to ignore past user activity */ + log_notice(LD_GENERAL, "Going dormant because of controller request."); + reset_user_activity(0); + set_network_participation(false); + schedule_rescan_periodic_events(); + control_event_signal(sig); + break; } } @@ -426,18 +442,6 @@ dumpstats(int severity) rend_service_dump_stats(severity); } -/** Called by exit() as we shut down the process. - */ -static void -exit_function(void) -{ - /* NOTE: If we ever daemonize, this gets called immediately. That's - * okay for now, because we only use this on Windows. */ -#ifdef _WIN32 - WSACleanup(); -#endif -} - #ifdef _WIN32 #define UNIX_ONLY 0 #else @@ -482,6 +486,8 @@ static struct { { SIGNEWNYM, 0, NULL }, { SIGCLEARDNSCACHE, 0, NULL }, { SIGHEARTBEAT, 0, NULL }, + { SIGACTIVE, 0, NULL }, + { SIGDORMANT, 0, NULL }, { -1, -1, NULL } }; @@ -546,18 +552,13 @@ tor_init(int argc, char *argv[]) tor_snprintf(progname, sizeof(progname), "Tor %s", get_version()); log_set_application_name(progname); - /* Set up the crypto nice and early */ - if (crypto_early_init() < 0) { - log_err(LD_GENERAL, "Unable to initialize the crypto subsystem!"); - return -1; - } - /* Initialize the history structures. */ rep_hist_init(); /* Initialize the service cache. */ rend_cache_init(); addressmap_init(); /* Init the client dns cache. Do it always, since it's * cheap. */ + /* Initialize the HS subsystem. */ hs_init(); @@ -631,12 +632,6 @@ tor_init(int argc, char *argv[]) rust_log_welcome_string(); #endif /* defined(HAVE_RUST) */ - if (network_init()<0) { - log_err(LD_BUG,"Error initializing network; exiting."); - return -1; - } - atexit(exit_function); - int init_rv = options_init_from_torrc(argc,argv); if (init_rv < 0) { log_err(LD_CONFIG,"Reading config failed--see warnings above."); @@ -783,7 +778,6 @@ tor_free_all(int postfork) routerparse_free_all(); ext_orport_free_all(); control_free_all(); - tor_free_getaddrinfo_cache(); protover_free_all(); bridges_free_all(); consdiffmgr_free_all(); @@ -791,6 +785,7 @@ tor_free_all(int postfork) dos_free_all(); circuitmux_ewma_free_all(); accounting_free_all(); + protover_summary_cache_free_all(); if (!postfork) { config_free_all(); @@ -800,7 +795,6 @@ tor_free_all(int postfork) policies_free_all(); } if (!postfork) { - tor_tls_free_all(); #ifndef _WIN32 tor_getpwnam(NULL); #endif @@ -813,12 +807,12 @@ tor_free_all(int postfork) release_lockfile(); } tor_libevent_free_all(); + + subsystems_shutdown(); + /* Stuff in util.c and address.c*/ if (!postfork) { - escaped(NULL); esc_router_info(NULL); - clean_up_backtrace_handler(); - logs_free_all(); /* free log strings. do this last so logs keep working. */ } } @@ -877,7 +871,6 @@ tor_cleanup(void) later, if it makes shutdown unacceptably slow. But for now, leave it here: it's helped us catch bugs in the past. */ - crypto_global_cleanup(); } /** Read/create keys as needed, and echo our fingerprint to stdout. */ @@ -1273,7 +1266,6 @@ int run_tor_main_loop(void) { handle_signals(); - monotime_init(); timers_initialize(); initialize_mainloop_events(); @@ -1385,54 +1377,13 @@ tor_run_main(const tor_main_configuration_t *tor_cfg) { int result = 0; -#ifdef _WIN32 -#ifndef HeapEnableTerminationOnCorruption -#define HeapEnableTerminationOnCorruption 1 -#endif - /* On heap corruption, just give up; don't try to play along. */ - HeapSetInformation(NULL, HeapEnableTerminationOnCorruption, NULL, 0); - - /* SetProcessDEPPolicy is only supported on 32-bit Windows. - * (On 64-bit Windows it always fails, and some compilers don't like the - * PSETDEP cast.) - * 32-bit Windows defines _WIN32. - * 64-bit Windows defines _WIN32 and _WIN64. */ -#ifndef _WIN64 - /* Call SetProcessDEPPolicy to permanently enable DEP. - The function will not resolve on earlier versions of Windows, - and failure is not dangerous. */ - HMODULE hMod = GetModuleHandleA("Kernel32.dll"); - if (hMod) { - typedef BOOL (WINAPI *PSETDEP)(DWORD); - PSETDEP setdeppolicy = (PSETDEP)GetProcAddress(hMod, - "SetProcessDEPPolicy"); - if (setdeppolicy) { - /* PROCESS_DEP_ENABLE | PROCESS_DEP_DISABLE_ATL_THUNK_EMULATION */ - setdeppolicy(3); - } - } -#endif /* !defined(_WIN64) */ -#endif /* defined(_WIN32) */ - - { - int bt_err = configure_backtrace_handler(get_version()); - if (bt_err < 0) { - log_warn(LD_BUG, "Unable to install backtrace handler: %s", - strerror(-bt_err)); - } - } - #ifdef EVENT_SET_MEM_FUNCTIONS_IMPLEMENTED event_set_mem_functions(tor_malloc_, tor_realloc_, tor_free_); #endif - init_protocol_warning_severity_level(); + subsystems_init(); - update_approx_time(time(NULL)); - tor_threads_init(); - tor_compress_init(); - init_logging(0); - monotime_init(); + init_protocol_warning_severity_level(); int argc = tor_cfg->argc + tor_cfg->argc_owned; char **argv = tor_calloc(argc, sizeof(char*)); @@ -1468,6 +1419,7 @@ tor_run_main(const tor_main_configuration_t *tor_cfg) tor_free_all(0); return -1; } + tor_make_getaddrinfo_cache_active(); // registering libevent rng #ifdef HAVE_EVUTIL_SECURE_RNG_SET_URANDOM_DEVICE_FILE diff --git a/src/app/main/subsysmgr.c b/src/app/main/subsysmgr.c new file mode 100644 index 0000000000..abd2edd10b --- /dev/null +++ b/src/app/main/subsysmgr.c @@ -0,0 +1,202 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#include "app/main/subsysmgr.h" +#include "lib/err/torerr.h" + +#include "lib/log/log.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +/** + * True iff we have checked tor_subsystems for consistency. + **/ +static bool subsystem_array_validated = false; + +/** + * True if a given subsystem is initialized. Expand this array if there + * are more than this number of subsystems. (We'd rather not + * dynamically allocate in this module.) + **/ +static bool sys_initialized[128]; + +/** + * Exit with a raw assertion if the subsystems list is inconsistent; + * initialize the subsystem_initialized array. + **/ +static void +check_and_setup(void) +{ + if (subsystem_array_validated) + return; + + raw_assert(ARRAY_LENGTH(sys_initialized) >= n_tor_subsystems); + memset(sys_initialized, 0, sizeof(sys_initialized)); + + int last_level = MIN_SUBSYS_LEVEL; + + for (unsigned i = 0; i < n_tor_subsystems; ++i) { + const subsys_fns_t *sys = tor_subsystems[i]; + if (sys->level < MIN_SUBSYS_LEVEL || sys->level > MAX_SUBSYS_LEVEL) { + fprintf(stderr, "BUG: Subsystem %s (at %u) has an invalid level %d. " + "It is supposed to be between %d and %d (inclusive).\n", + sys->name, i, sys->level, MIN_SUBSYS_LEVEL, MAX_SUBSYS_LEVEL); + raw_assert_unreached_msg("There is a bug in subsystem_list.c"); + } + if (sys->level < last_level) { + fprintf(stderr, "BUG: Subsystem %s (at #%u) is in the wrong position. " + "Its level is %d; but the previous subsystem's level was %d.\n", + sys->name, i, sys->level, last_level); + raw_assert_unreached_msg("There is a bug in subsystem_list.c"); + } + last_level = sys->level; + } + + subsystem_array_validated = true; +} + +/** + * Initialize all the subsystems; exit on failure. + **/ +int +subsystems_init(void) +{ + return subsystems_init_upto(MAX_SUBSYS_LEVEL); +} + +/** + * Initialize all the subsystems whose level is less than or equal to + * <b>target_level</b>; exit on failure. + **/ +int +subsystems_init_upto(int target_level) +{ + check_and_setup(); + + for (unsigned i = 0; i < n_tor_subsystems; ++i) { + const subsys_fns_t *sys = tor_subsystems[i]; + if (!sys->supported) + continue; + if (sys->level > target_level) + break; + if (sys_initialized[i]) + continue; + int r = 0; + if (sys->initialize) { + // Note that the logging subsystem is designed so that it does no harm + // to log a message in an uninitialized state. These messages will be + // discarded for now, however. + log_debug(LD_GENERAL, "Initializing %s", sys->name); + r = sys->initialize(); + } + if (r < 0) { + fprintf(stderr, "BUG: subsystem %s (at %u) initialization failed.\n", + sys->name, i); + raw_assert_unreached_msg("A subsystem couldn't be initialized."); + } + sys_initialized[i] = true; + } + + return 0; +} + +/** + * Shut down all the subsystems. + **/ +void +subsystems_shutdown(void) +{ + subsystems_shutdown_downto(MIN_SUBSYS_LEVEL - 1); +} + +/** + * Shut down all the subsystems whose level is above <b>target_level</b>. + **/ +void +subsystems_shutdown_downto(int target_level) +{ + check_and_setup(); + + for (int i = (int)n_tor_subsystems - 1; i >= 0; --i) { + const subsys_fns_t *sys = tor_subsystems[i]; + if (!sys->supported) + continue; + if (sys->level <= target_level) + break; + if (! sys_initialized[i]) + continue; + if (sys->shutdown) { + log_debug(LD_GENERAL, "Shutting down %s", sys->name); + sys->shutdown(); + } + sys_initialized[i] = false; + } +} + +/** + * Run pre-fork code on all subsystems that declare any + **/ +void +subsystems_prefork(void) +{ + check_and_setup(); + + for (int i = (int)n_tor_subsystems - 1; i >= 0; --i) { + const subsys_fns_t *sys = tor_subsystems[i]; + if (!sys->supported) + continue; + if (! sys_initialized[i]) + continue; + if (sys->prefork) { + log_debug(LD_GENERAL, "Pre-fork: %s", sys->name); + sys->prefork(); + } + } +} + +/** + * Run post-fork code on all subsystems that declare any + **/ +void +subsystems_postfork(void) +{ + check_and_setup(); + + for (unsigned i = 0; i < n_tor_subsystems; ++i) { + const subsys_fns_t *sys = tor_subsystems[i]; + if (!sys->supported) + continue; + if (! sys_initialized[i]) + continue; + if (sys->postfork) { + log_debug(LD_GENERAL, "Post-fork: %s", sys->name); + sys->postfork(); + } + } +} + +/** + * Run thread-cleanup code on all subsystems that declare any + **/ +void +subsystems_thread_cleanup(void) +{ + check_and_setup(); + + for (int i = (int)n_tor_subsystems - 1; i >= 0; --i) { + const subsys_fns_t *sys = tor_subsystems[i]; + if (!sys->supported) + continue; + if (! sys_initialized[i]) + continue; + if (sys->thread_cleanup) { + log_debug(LD_GENERAL, "Thread cleanup: %s", sys->name); + sys->thread_cleanup(); + } + } +} diff --git a/src/app/main/subsysmgr.h b/src/app/main/subsysmgr.h new file mode 100644 index 0000000000..4b3cad62ad --- /dev/null +++ b/src/app/main/subsysmgr.h @@ -0,0 +1,24 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_SUBSYSMGR_T +#define TOR_SUBSYSMGR_T + +#include "lib/subsys/subsys.h" + +extern const struct subsys_fns_t *tor_subsystems[]; +extern const unsigned n_tor_subsystems; + +int subsystems_init(void); +int subsystems_init_upto(int level); + +void subsystems_shutdown(void); +void subsystems_shutdown_downto(int level); + +void subsystems_prefork(void); +void subsystems_postfork(void); +void subsystems_thread_cleanup(void); + +#endif diff --git a/src/app/main/subsystem_list.c b/src/app/main/subsystem_list.c new file mode 100644 index 0000000000..9f9bf10866 --- /dev/null +++ b/src/app/main/subsystem_list.c @@ -0,0 +1,49 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#include "app/main/subsysmgr.h" +#include "lib/cc/compat_compiler.h" +#include "lib/cc/torint.h" + +#include "core/or/ocirc_event_sys.h" +#include "core/or/orconn_event_sys.h" +#include "feature/control/btrack_sys.h" +#include "lib/compress/compress_sys.h" +#include "lib/crypt_ops/crypto_sys.h" +#include "lib/err/torerr_sys.h" +#include "lib/log/log_sys.h" +#include "lib/net/network_sys.h" +#include "lib/process/winprocess_sys.h" +#include "lib/thread/thread_sys.h" +#include "lib/time/time_sys.h" +#include "lib/tls/tortls_sys.h" +#include "lib/wallclock/wallclock_sys.h" +#include "lib/process/process_sys.h" + +#include <stddef.h> + +/** + * Global list of the subsystems in Tor, in the order of their initialization. + **/ +const subsys_fns_t *tor_subsystems[] = { + &sys_winprocess, /* -100 */ + &sys_torerr, /* -100 */ + &sys_wallclock, /* -99 */ + &sys_threads, /* -95 */ + &sys_logging, /* -90 */ + &sys_time, /* -90 */ + &sys_network, /* -90 */ + &sys_compress, /* -70 */ + &sys_crypto, /* -60 */ + &sys_tortls, /* -50 */ + &sys_process, /* -35 */ + + &sys_orconn_event, /* -33 */ + &sys_ocirc_event, /* -32 */ + &sys_btrack, /* -30 */ +}; + +const unsigned n_tor_subsystems = ARRAY_LENGTH(tor_subsystems); diff --git a/src/config/mmdb-convert.py b/src/config/mmdb-convert.py index 3a454a3fc1..706a8b03cc 100644 --- a/src/config/mmdb-convert.py +++ b/src/config/mmdb-convert.py @@ -77,7 +77,7 @@ def to_int32(s): def to_int28(s): "Parse a pair of big-endian 28-bit integers from bytestring s." - a, b = unpack("!LL", s + b'\x00') + a, b = struct.unpack("!LL", s + b'\x00') return (((a & 0xf0) << 20) + (a >> 8)), ((a & 0x0f) << 24) + (b >> 8) class Tree(object): diff --git a/src/core/include.am b/src/core/include.am index 1b8ef2ac58..5e69cb9ada 100644 --- a/src/core/include.am +++ b/src/core/include.am @@ -11,6 +11,8 @@ LIBTOR_APP_A_SOURCES = \ src/app/config/confparse.c \ src/app/config/statefile.c \ src/app/main/main.c \ + src/app/main/subsystem_list.c \ + src/app/main/subsysmgr.c \ src/core/crypto/hs_ntor.c \ src/core/crypto/onion_crypto.c \ src/core/crypto/onion_fast.c \ @@ -37,6 +39,8 @@ LIBTOR_APP_A_SOURCES = \ src/core/or/connection_or.c \ src/core/or/dos.c \ src/core/or/onion.c \ + src/core/or/ocirc_event.c \ + src/core/or/orconn_event.c \ src/core/or/policies.c \ src/core/or/protover.c \ src/core/or/protover_rust.c \ @@ -59,7 +63,13 @@ LIBTOR_APP_A_SOURCES = \ src/feature/client/dnsserv.c \ src/feature/client/entrynodes.c \ src/feature/client/transports.c \ + src/feature/control/btrack.c \ + src/feature/control/btrack_circuit.c \ + src/feature/control/btrack_orconn.c \ + src/feature/control/btrack_orconn_cevent.c \ + src/feature/control/btrack_orconn_maps.c \ src/feature/control/control.c \ + src/feature/control/control_bootstrap.c \ src/feature/control/fmt_serverstatus.c \ src/feature/control/getinfo_geoip.c \ src/feature/dirauth/keypin.c \ @@ -106,6 +116,7 @@ LIBTOR_APP_A_SOURCES = \ src/feature/nodelist/microdesc.c \ src/feature/nodelist/networkstatus.c \ src/feature/nodelist/nickname.c \ + src/feature/nodelist/nodefamily.c \ src/feature/nodelist/nodelist.c \ src/feature/nodelist/node_select.c \ src/feature/nodelist/routerinfo.c \ @@ -191,6 +202,7 @@ noinst_HEADERS += \ src/app/config/statefile.h \ src/app/main/main.h \ src/app/main/ntmain.h \ + src/app/main/subsysmgr.h \ src/core/crypto/hs_ntor.h \ src/core/crypto/onion_crypto.h \ src/core/crypto/onion_fast.h \ @@ -233,10 +245,14 @@ noinst_HEADERS += \ src/core/or/listener_connection_st.h \ src/core/or/onion.h \ src/core/or/or.h \ + src/core/or/orconn_event.h \ + src/core/or/orconn_event_sys.h \ src/core/or/or_circuit_st.h \ src/core/or/or_connection_st.h \ src/core/or/or_handshake_certs_st.h \ src/core/or/or_handshake_state_st.h \ + src/core/or/ocirc_event.h \ + src/core/or/ocirc_event_sys.h \ src/core/or/origin_circuit_st.h \ src/core/or/policies.h \ src/core/or/port_cfg_st.h \ @@ -263,6 +279,11 @@ noinst_HEADERS += \ src/feature/client/dnsserv.h \ src/feature/client/entrynodes.h \ src/feature/client/transports.h \ + src/feature/control/btrack_circuit.h \ + src/feature/control/btrack_orconn.h \ + src/feature/control/btrack_orconn_cevent.h \ + src/feature/control/btrack_orconn_maps.h \ + src/feature/control/btrack_sys.h \ src/feature/control/control.h \ src/feature/control/control_connection_st.h \ src/feature/control/fmt_serverstatus.h \ @@ -340,6 +361,8 @@ noinst_HEADERS += \ src/feature/nodelist/networkstatus_voter_info_st.h \ src/feature/nodelist/nickname.h \ src/feature/nodelist/node_st.h \ + src/feature/nodelist/nodefamily.h \ + src/feature/nodelist/nodefamily_st.h \ src/feature/nodelist/nodelist.h \ src/feature/nodelist/node_select.h \ src/feature/nodelist/routerinfo.h \ diff --git a/src/core/mainloop/connection.c b/src/core/mainloop/connection.c index 1198a01ad9..6cc6122702 100644 --- a/src/core/mainloop/connection.c +++ b/src/core/mainloop/connection.c @@ -57,7 +57,7 @@ #define CONNECTION_PRIVATE #include "core/or/or.h" #include "feature/client/bridges.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/tls/buffers_tls.h" #include "lib/err/backtrace.h" @@ -1460,6 +1460,20 @@ connection_listener_new(const struct sockaddr *listensockaddr, tor_socket_strerror(tor_socket_errno(s))); goto err; } + +#ifndef __APPLE__ + /* This code was introduced to help debug #28229. */ + int value; + socklen_t len = sizeof(value); + + if (!getsockopt(s, SOL_SOCKET, SO_ACCEPTCONN, &value, &len)) { + if (value == 0) { + log_err(LD_NET, "Could not listen on %s - " + "getsockopt(.,SO_ACCEPTCONN,.) yields 0.", address); + goto err; + } + } +#endif /* __APPLE__ */ #endif /* defined(HAVE_SYS_UN_H) */ } else { log_err(LD_BUG, "Got unexpected address family %d.", @@ -1861,7 +1875,7 @@ connection_init_accepted_conn(connection_t *conn, /* Initiate Extended ORPort authentication. */ return connection_ext_or_start_auth(TO_OR_CONN(conn)); case CONN_TYPE_OR: - control_event_or_conn_status(TO_OR_CONN(conn), OR_CONN_EVENT_NEW, 0); + connection_or_event_status(TO_OR_CONN(conn), OR_CONN_EVENT_NEW, 0); rv = connection_tls_start_handshake(TO_OR_CONN(conn), 1); if (rv < 0) { connection_or_close_for_error(TO_OR_CONN(conn), 0); @@ -1874,6 +1888,9 @@ connection_init_accepted_conn(connection_t *conn, TO_ENTRY_CONN(conn)->nym_epoch = get_signewnym_epoch(); TO_ENTRY_CONN(conn)->socks_request->listener_type = listener->base_.type; + /* Any incoming connection on an entry port counts as user activity. */ + note_user_activity(approx_time()); + switch (TO_CONN(listener)->type) { case CONN_TYPE_AP_LISTENER: conn->state = AP_CONN_STATE_SOCKS_WAIT; @@ -2073,6 +2090,11 @@ connection_connect_log_client_use_ip_version(const connection_t *conn) return; } + if (fascist_firewall_use_ipv6(options)) { + log_info(LD_NET, "Our outgoing connection is using IPv%d.", + tor_addr_family(&real_addr) == AF_INET6 ? 6 : 4); + } + /* Check if we couldn't satisfy an address family preference */ if ((!pref_ipv6 && tor_addr_family(&real_addr) == AF_INET6) || (pref_ipv6 && tor_addr_family(&real_addr) == AF_INET)) { @@ -2886,6 +2908,10 @@ retry_all_listeners(smartlist_t *new_conns, int close_all_noncontrol) retval = -1; #ifdef ENABLE_LISTENER_REBIND + if (smartlist_len(replacements)) + log_debug(LD_NET, "%d replacements - starting rebinding loop.", + smartlist_len(replacements)); + SMARTLIST_FOREACH_BEGIN(replacements, listener_replacement_t *, r) { int addr_in_use = 0; int skip = 0; @@ -2897,8 +2923,11 @@ retry_all_listeners(smartlist_t *new_conns, int close_all_noncontrol) connection_listener_new_for_port(r->new_port, &skip, &addr_in_use); connection_t *old_conn = r->old_conn; - if (skip) + if (skip) { + log_debug(LD_NET, "Skipping creating new listener for %s:%d", + old_conn->address, old_conn->port); continue; + } connection_close_immediate(old_conn); connection_mark_for_close(old_conn); @@ -4426,6 +4455,16 @@ connection_get_by_type_state(int type, int state) CONN_GET_TEMPLATE(conn, conn->type == type && conn->state == state); } +/** + * Return a connection of type <b>type</b> that is not an internally linked + * connection, and is not marked for close. + **/ +MOCK_IMPL(connection_t *, +connection_get_by_type_nonlinked,(int type)) +{ + CONN_GET_TEMPLATE(conn, conn->type == type && !conn->linked); +} + /** Return a connection of type <b>type</b> that has rendquery equal * to <b>rendquery</b>, and that is not marked for close. If state * is non-zero, conn must be of that state too. diff --git a/src/core/mainloop/connection.h b/src/core/mainloop/connection.h index b569bb038e..07b8df4138 100644 --- a/src/core/mainloop/connection.h +++ b/src/core/mainloop/connection.h @@ -240,6 +240,7 @@ size_t connection_get_outbuf_len(connection_t *conn); connection_t *connection_get_by_global_id(uint64_t id); connection_t *connection_get_by_type(int type); +MOCK_DECL(connection_t *,connection_get_by_type_nonlinked,(int type)); MOCK_DECL(connection_t *,connection_get_by_type_addr_port_purpose,(int type, const tor_addr_t *addr, uint16_t port, int purpose)); diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index 413c149105..a2e1080720 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -95,7 +95,7 @@ #include "feature/stats/geoip_stats.h" #include "feature/stats/predict_ports.h" #include "feature/stats/rephist.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/crypt_ops/crypto_rand.h" #include "lib/err/backtrace.h" #include "lib/tls/buffers_tls.h" @@ -200,12 +200,10 @@ static int can_complete_circuits = 0; #define LAZY_DESCRIPTOR_RETRY_INTERVAL (60) static int conn_close_if_marked(int i); -static int run_main_loop_until_done(void); static void connection_start_reading_from_linked_conn(connection_t *conn); static int connection_should_read_from_linked_conn(connection_t *conn); static void conn_read_callback(evutil_socket_t fd, short event, void *_conn); static void conn_write_callback(evutil_socket_t fd, short event, void *_conn); -static void second_elapsed_callback(periodic_timer_t *timer, void *args); static void shutdown_did_not_work_callback(evutil_socket_t fd, short event, void *arg) ATTR_NORETURN; @@ -1365,6 +1363,7 @@ CALLBACK(heartbeat); CALLBACK(hs_service); CALLBACK(launch_descriptor_fetches); CALLBACK(launch_reachability_tests); +CALLBACK(prune_old_routers); CALLBACK(reachability_warnings); CALLBACK(record_bridge_stats); CALLBACK(rend_cache_failure_clean); @@ -1377,78 +1376,93 @@ CALLBACK(save_stability); CALLBACK(save_state); CALLBACK(write_bridge_ns); CALLBACK(write_stats_file); +CALLBACK(control_per_second_events); +CALLBACK(second_elapsed); #undef CALLBACK /* Now we declare an array of periodic_event_item_t for each periodic event */ -#define CALLBACK(name, r, f) PERIODIC_EVENT(name, r, f) +#define CALLBACK(name, r, f) \ + PERIODIC_EVENT(name, PERIODIC_EVENT_ROLE_ ## r, f) +#define FL(name) (PERIODIC_EVENT_FLAG_ ## name) STATIC periodic_event_item_t periodic_events[] = { - /* Everyone needs to run those. */ - CALLBACK(add_entropy, PERIODIC_EVENT_ROLE_ALL, 0), - CALLBACK(check_expired_networkstatus, PERIODIC_EVENT_ROLE_ALL, 0), - CALLBACK(clean_caches, PERIODIC_EVENT_ROLE_ALL, 0), - CALLBACK(fetch_networkstatus, PERIODIC_EVENT_ROLE_ALL, - PERIODIC_EVENT_FLAG_NEED_NET), - CALLBACK(heartbeat, PERIODIC_EVENT_ROLE_ALL, 0), - CALLBACK(launch_descriptor_fetches, PERIODIC_EVENT_ROLE_ALL, - PERIODIC_EVENT_FLAG_NEED_NET), - CALLBACK(reset_padding_counts, PERIODIC_EVENT_ROLE_ALL, 0), - CALLBACK(retry_listeners, PERIODIC_EVENT_ROLE_ALL, - PERIODIC_EVENT_FLAG_NEED_NET), - CALLBACK(save_state, PERIODIC_EVENT_ROLE_ALL, 0), - CALLBACK(rotate_x509_certificate, PERIODIC_EVENT_ROLE_ALL, 0), - CALLBACK(write_stats_file, PERIODIC_EVENT_ROLE_ALL, 0), + + /* Everyone needs to run these. They need to have very long timeouts for + * that to be safe. */ + CALLBACK(add_entropy, ALL, 0), + CALLBACK(heartbeat, ALL, 0), + CALLBACK(reset_padding_counts, ALL, 0), + + /* This is a legacy catch-all callback that runs once per second if + * we are online and active. */ + CALLBACK(second_elapsed, NET_PARTICIPANT, + FL(NEED_NET)|FL(RUN_ON_DISABLE)), + + /* XXXX Do we have a reason to do this on a callback? Does it do any good at + * all? For now, if we're dormant, we can let our listeners decay. */ + CALLBACK(retry_listeners, NET_PARTICIPANT, FL(NEED_NET)), + + /* We need to do these if we're participating in the Tor network. */ + CALLBACK(check_expired_networkstatus, NET_PARTICIPANT, 0), + CALLBACK(fetch_networkstatus, NET_PARTICIPANT, 0), + CALLBACK(launch_descriptor_fetches, NET_PARTICIPANT, FL(NEED_NET)), + CALLBACK(rotate_x509_certificate, NET_PARTICIPANT, 0), + CALLBACK(check_network_participation, NET_PARTICIPANT, 0), + + /* We need to do these if we're participating in the Tor network, and + * immediately before we stop. */ + CALLBACK(clean_caches, NET_PARTICIPANT, FL(RUN_ON_DISABLE)), + CALLBACK(save_state, NET_PARTICIPANT, FL(RUN_ON_DISABLE)), + CALLBACK(write_stats_file, NET_PARTICIPANT, FL(RUN_ON_DISABLE)), + CALLBACK(prune_old_routers, NET_PARTICIPANT, FL(RUN_ON_DISABLE)), /* Routers (bridge and relay) only. */ - CALLBACK(check_descriptor, PERIODIC_EVENT_ROLE_ROUTER, - PERIODIC_EVENT_FLAG_NEED_NET), - CALLBACK(check_ed_keys, PERIODIC_EVENT_ROLE_ROUTER, 0), - CALLBACK(check_for_reachability_bw, PERIODIC_EVENT_ROLE_ROUTER, - PERIODIC_EVENT_FLAG_NEED_NET), - CALLBACK(check_onion_keys_expiry_time, PERIODIC_EVENT_ROLE_ROUTER, 0), - CALLBACK(expire_old_ciruits_serverside, PERIODIC_EVENT_ROLE_ROUTER, - PERIODIC_EVENT_FLAG_NEED_NET), - CALLBACK(reachability_warnings, PERIODIC_EVENT_ROLE_ROUTER, - PERIODIC_EVENT_FLAG_NEED_NET), - CALLBACK(retry_dns, PERIODIC_EVENT_ROLE_ROUTER, 0), - CALLBACK(rotate_onion_key, PERIODIC_EVENT_ROLE_ROUTER, 0), + CALLBACK(check_descriptor, ROUTER, FL(NEED_NET)), + CALLBACK(check_ed_keys, ROUTER, 0), + CALLBACK(check_for_reachability_bw, ROUTER, FL(NEED_NET)), + CALLBACK(check_onion_keys_expiry_time, ROUTER, 0), + CALLBACK(expire_old_ciruits_serverside, ROUTER, FL(NEED_NET)), + CALLBACK(reachability_warnings, ROUTER, FL(NEED_NET)), + CALLBACK(retry_dns, ROUTER, 0), + CALLBACK(rotate_onion_key, ROUTER, 0), /* Authorities (bridge and directory) only. */ - CALLBACK(downrate_stability, PERIODIC_EVENT_ROLE_AUTHORITIES, 0), - CALLBACK(launch_reachability_tests, PERIODIC_EVENT_ROLE_AUTHORITIES, - PERIODIC_EVENT_FLAG_NEED_NET), - CALLBACK(save_stability, PERIODIC_EVENT_ROLE_AUTHORITIES, 0), + CALLBACK(downrate_stability, AUTHORITIES, 0), + CALLBACK(launch_reachability_tests, AUTHORITIES, FL(NEED_NET)), + CALLBACK(save_stability, AUTHORITIES, 0), /* Directory authority only. */ - CALLBACK(check_authority_cert, PERIODIC_EVENT_ROLE_DIRAUTH, 0), - CALLBACK(dirvote, PERIODIC_EVENT_ROLE_DIRAUTH, PERIODIC_EVENT_FLAG_NEED_NET), + CALLBACK(check_authority_cert, DIRAUTH, 0), + CALLBACK(dirvote, DIRAUTH, FL(NEED_NET)), /* Relay only. */ - CALLBACK(check_canonical_channels, PERIODIC_EVENT_ROLE_RELAY, - PERIODIC_EVENT_FLAG_NEED_NET), - CALLBACK(check_dns_honesty, PERIODIC_EVENT_ROLE_RELAY, - PERIODIC_EVENT_FLAG_NEED_NET), + CALLBACK(check_canonical_channels, RELAY, FL(NEED_NET)), + CALLBACK(check_dns_honesty, RELAY, FL(NEED_NET)), /* Hidden Service service only. */ - CALLBACK(hs_service, PERIODIC_EVENT_ROLE_HS_SERVICE, - PERIODIC_EVENT_FLAG_NEED_NET), + CALLBACK(hs_service, HS_SERVICE, FL(NEED_NET)), // XXXX break this down more /* Bridge only. */ - CALLBACK(record_bridge_stats, PERIODIC_EVENT_ROLE_BRIDGE, 0), + CALLBACK(record_bridge_stats, BRIDGE, 0), /* Client only. */ - CALLBACK(rend_cache_failure_clean, PERIODIC_EVENT_ROLE_CLIENT, 0), + /* XXXX this could be restricted to CLIENT+NET_PARTICIPANT */ + CALLBACK(rend_cache_failure_clean, NET_PARTICIPANT, FL(RUN_ON_DISABLE)), /* Bridge Authority only. */ - CALLBACK(write_bridge_ns, PERIODIC_EVENT_ROLE_BRIDGEAUTH, 0), + CALLBACK(write_bridge_ns, BRIDGEAUTH, 0), /* Directory server only. */ - CALLBACK(clean_consdiffmgr, PERIODIC_EVENT_ROLE_DIRSERVER, 0), + CALLBACK(clean_consdiffmgr, DIRSERVER, 0), + + /* Controller with per-second events only. */ + CALLBACK(control_per_second_events, CONTROLEV, 0), END_OF_PERIODIC_EVENTS }; #undef CALLBACK +#undef FL /* These are pointers to members of periodic_events[] that are used to * implement particular callbacks. We keep them separate here so that we @@ -1460,6 +1474,7 @@ static periodic_event_item_t *fetch_networkstatus_event=NULL; static periodic_event_item_t *launch_descriptor_fetches_event=NULL; static periodic_event_item_t *check_dns_honesty_event=NULL; static periodic_event_item_t *save_state_event=NULL; +static periodic_event_item_t *prune_old_routers_event=NULL; /** Reset all the periodic events so we'll do all our actions again as if we * just started up. @@ -1496,7 +1511,7 @@ get_my_roles(const or_options_t *options) { tor_assert(options); - int roles = 0; + int roles = PERIODIC_EVENT_ROLE_ALL; int is_bridge = options->BridgeRelay; int is_relay = server_mode(options); int is_dirauth = authdir_mode_v3(options); @@ -1504,6 +1519,8 @@ get_my_roles(const or_options_t *options) int is_hidden_service = !!hs_service_get_num_services() || !!rend_num_services(); int is_dirserver = dir_server_mode(options); + int sending_control_events = control_any_per_second_event_enabled(); + /* We also consider tor to have the role of a client if the ControlPort is * set because a lot of things can be done over the control port which * requires tor to have basic functionnalities. */ @@ -1511,6 +1528,9 @@ get_my_roles(const or_options_t *options) options->ControlPort_set || options->OwningControllerFD != UINT64_MAX; + int is_net_participant = is_participating_on_network() || + is_relay || is_hidden_service; + if (is_bridge) roles |= PERIODIC_EVENT_ROLE_BRIDGE; if (is_client) roles |= PERIODIC_EVENT_ROLE_CLIENT; if (is_relay) roles |= PERIODIC_EVENT_ROLE_RELAY; @@ -1518,6 +1538,8 @@ get_my_roles(const or_options_t *options) if (is_bridgeauth) roles |= PERIODIC_EVENT_ROLE_BRIDGEAUTH; if (is_hidden_service) roles |= PERIODIC_EVENT_ROLE_HS_SERVICE; if (is_dirserver) roles |= PERIODIC_EVENT_ROLE_DIRSERVER; + if (is_net_participant) roles |= PERIODIC_EVENT_ROLE_NET_PARTICIPANT; + if (sending_control_events) roles |= PERIODIC_EVENT_ROLE_CONTROLEV; return roles; } @@ -1562,6 +1584,7 @@ initialize_periodic_events(void) STMT_BEGIN name ## _event = find_periodic_event( #name ); STMT_END NAMED_CALLBACK(check_descriptor); + NAMED_CALLBACK(prune_old_routers); NAMED_CALLBACK(dirvote); NAMED_CALLBACK(fetch_networkstatus); NAMED_CALLBACK(launch_descriptor_fetches); @@ -1585,6 +1608,30 @@ teardown_periodic_events(void) periodic_events_initialized = 0; } +static mainloop_event_t *rescan_periodic_events_ev = NULL; + +/** Callback: rescan the periodic event list. */ +static void +rescan_periodic_events_cb(mainloop_event_t *event, void *arg) +{ + (void)event; + (void)arg; + rescan_periodic_events(get_options()); +} + +/** + * Schedule an event that will rescan which periodic events should run. + **/ +MOCK_IMPL(void, +schedule_rescan_periodic_events,(void)) +{ + if (!rescan_periodic_events_ev) { + rescan_periodic_events_ev = + mainloop_event_new(rescan_periodic_events_cb, NULL); + } + mainloop_event_activate(rescan_periodic_events_ev); +} + /** Do a pass at all our periodic events, disable those we don't need anymore * and enable those we need now using the given options. */ void @@ -1619,7 +1666,11 @@ rescan_periodic_events(const or_options_t *options) periodic_event_enable(item); } else { log_debug(LD_GENERAL, "Disabling periodic event %s", item->name); - periodic_event_disable(item); + if (item->flags & PERIODIC_EVENT_FLAG_RUN_ON_DISABLE) { + periodic_event_schedule_and_disable(item); + } else { + periodic_event_disable(item); + } } } } @@ -1694,6 +1745,30 @@ mainloop_schedule_postloop_cleanup(void) mainloop_event_activate(postloop_cleanup_ev); } +/** Event to run 'scheduled_shutdown_cb' */ +static mainloop_event_t *scheduled_shutdown_ev=NULL; + +/** Callback: run a scheduled shutdown */ +static void +scheduled_shutdown_cb(mainloop_event_t *ev, void *arg) +{ + (void)ev; + (void)arg; + log_notice(LD_GENERAL, "Clean shutdown finished. Exiting."); + tor_shutdown_event_loop_and_exit(0); +} + +/** Schedule the mainloop to exit after <b>delay_sec</b> seconds. */ +void +mainloop_schedule_shutdown(int delay_sec) +{ + const struct timeval delay_tv = { delay_sec, 0 }; + if (! scheduled_shutdown_ev) { + scheduled_shutdown_ev = mainloop_event_new(scheduled_shutdown_cb, NULL); + } + mainloop_event_schedule(scheduled_shutdown_ev, &delay_tv); +} + #define LONGEST_TIMER_PERIOD (30 * 86400) /** Helper: Return the number of seconds between <b>now</b> and <b>next</b>, * clipped to the range [1 second, LONGEST_TIMER_PERIOD]. */ @@ -1718,16 +1793,16 @@ safe_timer_diff(time_t now, time_t next) } /** Perform regular maintenance tasks. This function gets run once per - * second by second_elapsed_callback(). + * second. */ -static void -run_scheduled_events(time_t now) +static int +second_elapsed_callback(time_t now, const or_options_t *options) { - const or_options_t *options = get_options(); - - /* 0. See if we've been asked to shut down and our timeout has - * expired; or if our bandwidth limits are exhausted and we - * should hibernate; or if it's time to wake up from hibernation. + /* 0. See if our bandwidth limits are exhausted and we should hibernate + * + * Note: we have redundant mechanisms to handle the case where it's + * time to wake up from hibernation; or where we have a scheduled + * shutdown and it's time to run it, but this will also handle those. */ consider_hibernation(now); @@ -1737,10 +1812,13 @@ run_scheduled_events(time_t now) if (options->UseBridges && !net_is_disabled()) { /* Note: this check uses net_is_disabled(), not should_delay_dir_fetches() * -- the latter is only for fetching consensus-derived directory info. */ + // TODO: client + // Also, schedule this rather than probing 1x / sec fetch_bridge_descriptors(options, now); } if (accounting_is_enabled(options)) { + // TODO: refactor or rewrite? accounting_run_housekeeping(now); } @@ -1751,6 +1829,7 @@ run_scheduled_events(time_t now) */ /* (If our circuit build timeout can ever become lower than a second (which * it can't, currently), we should do this more often.) */ + // TODO: All expire stuff can become NET_PARTICIPANT, RUN_ON_DISABLE circuit_expire_building(); circuit_expire_waiting_for_better_guard(); @@ -1784,9 +1863,8 @@ run_scheduled_events(time_t now) run_connection_housekeeping(i, now); } - /* 11b. check pending unconfigured managed proxies */ - if (!net_is_disabled() && pt_proxies_configuration_pending()) - pt_configure_remaining_proxies(); + /* Run again in a second. */ + return 1; } /* Periodic callback: rotate the onion keys after the period defined by the @@ -1933,6 +2011,55 @@ add_entropy_callback(time_t now, const or_options_t *options) return ENTROPY_INTERVAL; } +/** Periodic callback: if there has been no network usage in a while, + * enter a dormant state. */ +STATIC int +check_network_participation_callback(time_t now, const or_options_t *options) +{ + /* If we're a server, we can't become dormant. */ + if (server_mode(options)) { + goto found_activity; + } + + /* If we're running an onion service, we can't become dormant. */ + /* XXXX this would be nice to change, so that we can be dormant with a + * service. */ + if (hs_service_get_num_services() || rend_num_services()) { + goto found_activity; + } + + /* If we have any currently open entry streams other than "linked" + * connections used for directory requests, those count as user activity. + */ + if (options->DormantTimeoutDisabledByIdleStreams) { + if (connection_get_by_type_nonlinked(CONN_TYPE_AP) != NULL) { + goto found_activity; + } + } + + /* XXXX Make this configurable? */ +/** How often do we check whether we have had network activity? */ +#define CHECK_PARTICIPATION_INTERVAL (5*60) + + /* Become dormant if there has been no user activity in a long time. + * (The funny checks below are in order to prevent overflow.) */ + time_t time_since_last_activity = 0; + if (get_last_user_activity_time() < now) + time_since_last_activity = now - get_last_user_activity_time(); + if (time_since_last_activity >= options->DormantClientTimeout) { + log_notice(LD_GENERAL, "No user activity in a long time: becoming" + " dormant."); + set_network_participation(false); + rescan_periodic_events(options); + } + + return CHECK_PARTICIPATION_INTERVAL; + + found_activity: + note_user_activity(now); + return CHECK_PARTICIPATION_INTERVAL; +} + /** * Periodic callback: if we're an authority, make sure we test * the routers on the network for reachability. @@ -2226,6 +2353,27 @@ retry_dns_callback(time_t now, const or_options_t *options) return RETRY_DNS_INTERVAL; } +/** + * Periodic callback: prune routerlist of old information about Tor network. + */ +static int +prune_old_routers_callback(time_t now, const or_options_t *options) +{ +#define ROUTERLIST_PRUNING_INTERVAL (60*60) // 1 hour. + (void)now; + (void)options; + + if (!net_is_disabled()) { + /* If any networkstatus documents are no longer recent, we need to + * update all the descriptors' running status. */ + /* Remove dead routers. */ + log_debug(LD_GENERAL, "Pruning routerlist..."); + routerlist_remove_old_routers(); + } + + return ROUTERLIST_PRUNING_INTERVAL; +} + /** Periodic callback: consider rebuilding or and re-uploading our descriptor * (if we've passed our internal checks). */ static int @@ -2245,12 +2393,6 @@ check_descriptor_callback(time_t now, const or_options_t *options) check_descriptor_ipaddress_changed(now); mark_my_descriptor_dirty_if_too_old(now); consider_publishable_server(0); - /* If any networkstatus documents are no longer recent, we need to - * update all the descriptors' running status. */ - /* Remove dead routers. */ - /* XXXX This doesn't belong here, but it was here in the pre- - * XXXX refactoring code. */ - routerlist_remove_old_routers(); } return CHECK_DESCRIPTOR_INTERVAL; @@ -2508,36 +2650,19 @@ hs_service_callback(time_t now, const or_options_t *options) return 1; } -/** Timer: used to invoke second_elapsed_callback() once per second. */ -static periodic_timer_t *second_timer = NULL; - -/** - * Enable or disable the per-second timer as appropriate, creating it if - * necessary. +/* + * Periodic callback: Send once-per-second events to the controller(s). + * This is called every second. */ -void -reschedule_per_second_timer(void) +static int +control_per_second_events_callback(time_t now, const or_options_t *options) { - struct timeval one_second; - one_second.tv_sec = 1; - one_second.tv_usec = 0; - - if (! second_timer) { - second_timer = periodic_timer_new(tor_libevent_get_base(), - &one_second, - second_elapsed_callback, - NULL); - tor_assert(second_timer); - } + (void) options; + (void) now; - const bool run_per_second_events = - control_any_per_second_event_enabled() || ! net_is_completely_disabled(); + control_per_second_events(); - if (run_per_second_events) { - periodic_timer_launch(second_timer, &one_second); - } else { - periodic_timer_disable(second_timer); - } + return 1; } /** Last time that update_current_time was called. */ @@ -2567,6 +2692,17 @@ update_current_time(time_t now) memcpy(&last_updated, ¤t_second_last_changed, sizeof(last_updated)); monotime_coarse_get(¤t_second_last_changed); + /** How much clock jumping means that we should adjust our idea of when + * to go dormant? */ +#define NUM_JUMPED_SECONDS_BEFORE_NETSTATUS_UPDATE 20 + + /* Don't go dormant early or late just because we jumped in time. */ + if (ABS(seconds_elapsed) >= NUM_JUMPED_SECONDS_BEFORE_NETSTATUS_UPDATE) { + if (is_participating_on_network()) { + netstatus_note_clock_jumped(seconds_elapsed); + } + } + /** How much clock jumping do we tolerate? */ #define NUM_JUMPED_SECONDS_BEFORE_WARN 100 @@ -2576,6 +2712,7 @@ update_current_time(time_t now) if (seconds_elapsed < -NUM_JUMPED_SECONDS_BEFORE_WARN) { // moving back in time is always a bad sign. circuit_note_clock_jumped(seconds_elapsed, false); + } else if (seconds_elapsed >= NUM_JUMPED_SECONDS_BEFORE_WARN) { /* Compare the monotonic clock to the result of time(). */ const int32_t monotime_msec_passed = @@ -2605,31 +2742,6 @@ update_current_time(time_t now) current_second = now; } -/** Libevent callback: invoked once every second. */ -static void -second_elapsed_callback(periodic_timer_t *timer, void *arg) -{ - /* XXXX This could be sensibly refactored into multiple callbacks, and we - * could use Libevent's timers for this rather than checking the current - * time against a bunch of timeouts every second. */ - time_t now; - (void)timer; - (void)arg; - - now = time(NULL); - - /* We don't need to do this once-per-second any more: time-updating is - * only in this callback _because it is a callback_. It should be fine - * to disable this callback, and the time will still get updated. - */ - update_current_time(now); - - /* Maybe some controller events are ready to fire */ - control_per_second_events(); - - run_scheduled_events(now); -} - #ifdef HAVE_SYSTEMD_209 static periodic_timer_t *systemd_watchdog_timer = NULL; @@ -2720,9 +2832,6 @@ do_main_loop(void) initialize_periodic_events(); initialize_mainloop_events(); - /* set up once-a-second callback. */ - reschedule_per_second_timer(); - #ifdef HAVE_SYSTEMD_209 uint64_t watchdog_delay; /* set up systemd watchdog notification. */ @@ -2744,10 +2853,6 @@ do_main_loop(void) } } #endif /* defined(HAVE_SYSTEMD_209) */ - - main_loop_should_exit = 0; - main_loop_exit_value = 0; - #ifdef ENABLE_RESTART_DEBUGGING { static int first_time = 1; @@ -2873,10 +2978,14 @@ run_main_loop_once(void) * * Shadow won't invoke this function, so don't fill it up with things. */ -static int +STATIC int run_main_loop_until_done(void) { int loop_result = 1; + + main_loop_should_exit = 0; + main_loop_exit_value = 0; + do { loop_result = run_main_loop_once(); } while (loop_result == 1); @@ -2907,7 +3016,6 @@ tor_mainloop_free_all(void) smartlist_free(connection_array); smartlist_free(closeable_connection_lst); smartlist_free(active_linked_connection_lst); - periodic_timer_free(second_timer); teardown_periodic_events(); tor_event_free(shutdown_did_not_work_event); tor_event_free(initialize_periodic_events_event); @@ -2915,6 +3023,8 @@ tor_mainloop_free_all(void) mainloop_event_free(schedule_active_linked_connections_event); mainloop_event_free(postloop_cleanup_ev); mainloop_event_free(handle_deferred_signewnym_ev); + mainloop_event_free(scheduled_shutdown_ev); + mainloop_event_free(rescan_periodic_events_ev); #ifdef HAVE_SYSTEMD_209 periodic_timer_free(systemd_watchdog_timer); diff --git a/src/core/mainloop/mainloop.h b/src/core/mainloop/mainloop.h index 632733d9a6..730234857d 100644 --- a/src/core/mainloop/mainloop.h +++ b/src/core/mainloop/mainloop.h @@ -65,6 +65,7 @@ void reschedule_or_state_save(void); void reschedule_dirvote(const or_options_t *options); void mainloop_schedule_postloop_cleanup(void); void rescan_periodic_events(const or_options_t *options); +MOCK_DECL(void, schedule_rescan_periodic_events,(void)); void update_current_time(time_t now); @@ -81,11 +82,12 @@ uint64_t get_main_loop_error_count(void); uint64_t get_main_loop_idle_count(void); void periodic_events_on_new_options(const or_options_t *options); -void reschedule_per_second_timer(void); void do_signewnym(time_t); time_t get_last_signewnym_time(void); +void mainloop_schedule_shutdown(int delay_sec); + void tor_init_connection_lists(void); void initialize_mainloop_events(void); void tor_mainloop_free_all(void); @@ -98,10 +100,14 @@ extern struct token_bucket_rw_t global_bucket; extern struct token_bucket_rw_t global_relayed_bucket; #ifdef MAINLOOP_PRIVATE +STATIC int run_main_loop_until_done(void); STATIC void close_closeable_connections(void); STATIC void initialize_periodic_events(void); STATIC void teardown_periodic_events(void); STATIC int get_my_roles(const or_options_t *); +STATIC int check_network_participation_callback(time_t now, + const or_options_t *options); + #ifdef TOR_UNIT_TESTS extern smartlist_t *connection_array; diff --git a/src/core/mainloop/netstatus.c b/src/core/mainloop/netstatus.c index f026474494..d1989cb839 100644 --- a/src/core/mainloop/netstatus.c +++ b/src/core/mainloop/netstatus.c @@ -6,9 +6,12 @@ #include "core/or/or.h" #include "core/mainloop/netstatus.h" +#include "core/mainloop/mainloop.h" #include "app/config/config.h" #include "feature/hibernate/hibernate.h" +#include "app/config/or_state_st.h" + /** Return true iff our network is in some sense disabled or shutting down: * either we're hibernating, entering hibernation, or the network is turned * off with DisableNetwork. */ @@ -26,3 +29,132 @@ net_is_completely_disabled(void) { return get_options()->DisableNetwork || we_are_fully_hibernating(); } + +/** + * The time at which we've last seen "user activity" -- that is, any activity + * that should keep us as a participant on the network. + * + * This is not actually the true time. We will adjust this forward if + * our clock jumps, or if Tor is shut down for a while, so that the time + * since our last activity remains as it was before the jump or shutdown. + */ +static time_t last_user_activity_seen = 0; + +/** + * True iff we are currently a "network participant" -- that is, we + * are building circuits, fetching directory information, and so on. + **/ +static bool participating_on_network = false; + +/** + * Record the fact that we have seen "user activity" at the time now. Move + * "last activity seen" time forwards, but never backwards. + * + * If we were previously not participating on the network, set our + * participation status to true, and launch periodic events as appropriate. + **/ +void +note_user_activity(time_t now) +{ + last_user_activity_seen = MAX(now, last_user_activity_seen); + + if (! participating_on_network) { + log_notice(LD_GENERAL, "Tor is no longer dormant."); + set_network_participation(true); + schedule_rescan_periodic_events(); + } +} + +/** + * Change the time at which "user activitiy" was last seen to <b>now</b>. + * + * Unlike note_user_actity, this function sets the time without checking + * whether it is in the past, and without causing any rescan of periodic events + * or change in participation status. + */ +void +reset_user_activity(time_t now) +{ + last_user_activity_seen = now; +} + +/** + * Return the most recent time at which we recorded "user activity". + **/ +time_t +get_last_user_activity_time(void) +{ + return last_user_activity_seen; +} + +/** + * Set the field that remembers whether we are currently participating on the + * network. Does not schedule or un-schedule periodic events. + **/ +void +set_network_participation(bool participation) +{ + participating_on_network = participation; +} + +/** + * Return true iff we are currently participating on the network. + **/ +bool +is_participating_on_network(void) +{ + return participating_on_network; +} + +/** + * Update 'state' with the last time at which we were active on the network. + **/ +void +netstatus_flush_to_state(or_state_t *state, time_t now) +{ + state->Dormant = ! participating_on_network; + if (participating_on_network) { + time_t sec_since_activity = MAX(0, now - last_user_activity_seen); + state->MinutesSinceUserActivity = (int)(sec_since_activity / 60); + } else { + state->MinutesSinceUserActivity = 0; + } +} + +/** + * Update our current view of network participation from an or_state_t object. + **/ +void +netstatus_load_from_state(const or_state_t *state, time_t now) +{ + time_t last_activity; + if (state->Dormant == -1) { // Initial setup. + if (get_options()->DormantOnFirstStartup) { + last_activity = 0; + participating_on_network = false; + } else { + // Start up as active, treat activity as happening now. + last_activity = now; + participating_on_network = true; + } + } else if (state->Dormant) { + last_activity = 0; + participating_on_network = false; + } else { + last_activity = now - 60 * state->MinutesSinceUserActivity; + participating_on_network = true; + } + reset_user_activity(last_activity); +} + +/** + * Adjust the time at which the user was last active by <b>seconds_diff</b> + * in response to a clock jump. + */ +void +netstatus_note_clock_jumped(time_t seconds_diff) +{ + time_t last_active = get_last_user_activity_time(); + if (last_active) + reset_user_activity(last_active + seconds_diff); +} diff --git a/src/core/mainloop/netstatus.h b/src/core/mainloop/netstatus.h index e9310c2929..9a0fa410fd 100644 --- a/src/core/mainloop/netstatus.h +++ b/src/core/mainloop/netstatus.h @@ -10,4 +10,15 @@ int net_is_disabled(void); int net_is_completely_disabled(void); +void note_user_activity(time_t now); +void reset_user_activity(time_t now); +time_t get_last_user_activity_time(void); + +void set_network_participation(bool participation); +bool is_participating_on_network(void); + +void netstatus_flush_to_state(or_state_t *state, time_t now); +void netstatus_load_from_state(const or_state_t *state, time_t now); +void netstatus_note_clock_jumped(time_t seconds_diff); + #endif diff --git a/src/core/mainloop/periodic.c b/src/core/mainloop/periodic.c index c1785eb38f..9f9b178e43 100644 --- a/src/core/mainloop/periodic.c +++ b/src/core/mainloop/periodic.c @@ -45,10 +45,6 @@ periodic_event_dispatch(mainloop_event_t *ev, void *data) periodic_event_item_t *event = data; tor_assert(ev == event->ev); - if (BUG(!periodic_event_is_enabled(event))) { - return; - } - time_t now = time(NULL); update_current_time(now); const or_options_t *options = get_options(); @@ -57,7 +53,7 @@ periodic_event_dispatch(mainloop_event_t *ev, void *data) int next_interval = 0; if (!periodic_event_is_enabled(event)) { - /* The event got disabled from inside its callback; no need to + /* The event got disabled from inside its callback, or before: no need to * reschedule. */ return; } @@ -172,3 +168,19 @@ periodic_event_disable(periodic_event_item_t *event) mainloop_event_cancel(event->ev); event->enabled = 0; } + +/** + * Disable an event, then schedule it to run once. + * Do nothing if the event was already disabled. + */ +void +periodic_event_schedule_and_disable(periodic_event_item_t *event) +{ + tor_assert(event); + if (!periodic_event_is_enabled(event)) + return; + + periodic_event_disable(event); + + mainloop_event_activate(event->ev); +} diff --git a/src/core/mainloop/periodic.h b/src/core/mainloop/periodic.h index 4c8c3c96cc..05ba4297f3 100644 --- a/src/core/mainloop/periodic.h +++ b/src/core/mainloop/periodic.h @@ -15,6 +15,10 @@ #define PERIODIC_EVENT_ROLE_BRIDGEAUTH (1U << 4) #define PERIODIC_EVENT_ROLE_HS_SERVICE (1U << 5) #define PERIODIC_EVENT_ROLE_DIRSERVER (1U << 6) +#define PERIODIC_EVENT_ROLE_CONTROLEV (1U << 7) + +#define PERIODIC_EVENT_ROLE_NET_PARTICIPANT (1U << 8) +#define PERIODIC_EVENT_ROLE_ALL (1U << 9) /* Helper macro to make it a bit less annoying to defined groups of roles that * are often used. */ @@ -25,10 +29,6 @@ /* Authorities that is both bridge and directory. */ #define PERIODIC_EVENT_ROLE_AUTHORITIES \ (PERIODIC_EVENT_ROLE_BRIDGEAUTH | PERIODIC_EVENT_ROLE_DIRAUTH) -/* All roles. */ -#define PERIODIC_EVENT_ROLE_ALL \ - (PERIODIC_EVENT_ROLE_AUTHORITIES | PERIODIC_EVENT_ROLE_CLIENT | \ - PERIODIC_EVENT_ROLE_HS_SERVICE | PERIODIC_EVENT_ROLE_ROUTER) /* * Event flags which can change the behavior of an event. @@ -39,6 +39,11 @@ * the net_is_disabled() check. */ #define PERIODIC_EVENT_FLAG_NEED_NET (1U << 0) +/* Indicate that if the event is enabled, it needs to be run once before + * it becomes disabled. + */ +#define PERIODIC_EVENT_FLAG_RUN_ON_DISABLE (1U << 1) + /** Callback function for a periodic event to take action. The return value * influences the next time the function will get called. Return * PERIODIC_EVENT_NO_UPDATE to not update <b>last_action_time</b> and be polled @@ -83,6 +88,6 @@ void periodic_event_destroy(periodic_event_item_t *event); void periodic_event_reschedule(periodic_event_item_t *event); void periodic_event_enable(periodic_event_item_t *event); void periodic_event_disable(periodic_event_item_t *event); +void periodic_event_schedule_and_disable(periodic_event_item_t *event); #endif /* !defined(TOR_PERIODIC_H) */ - diff --git a/src/core/or/channeltls.c b/src/core/or/channeltls.c index 883acacf37..cf33feec01 100644 --- a/src/core/or/channeltls.c +++ b/src/core/or/channeltls.c @@ -59,6 +59,7 @@ #include "feature/nodelist/torcert.h" #include "feature/nodelist/networkstatus.h" #include "trunnel/channelpadding_negotiation.h" +#include "trunnel/netinfo.h" #include "core/or/channelpadding.h" #include "core/or/cell_st.h" @@ -949,7 +950,6 @@ channel_tls_listener_describe_transport_method(channel_listener_t *chan_l) void channel_tls_handle_state_change_on_orconn(channel_tls_t *chan, or_connection_t *conn, - uint8_t old_state, uint8_t state) { channel_t *base_chan; @@ -958,8 +958,6 @@ channel_tls_handle_state_change_on_orconn(channel_tls_t *chan, tor_assert(conn); tor_assert(conn->chan == chan); tor_assert(chan->conn == conn); - /* Shut the compiler up without triggering -Wtautological-compare */ - (void)old_state; base_chan = TLS_CHAN_TO_BASE(chan); @@ -1637,6 +1635,35 @@ channel_tls_process_padding_negotiate_cell(cell_t *cell, channel_tls_t *chan) } /** + * Convert <b>netinfo_addr</b> into corresponding <b>tor_addr</b>. + * Return 0 on success; on failure, return -1 and log a warning. + */ +static int +tor_addr_from_netinfo_addr(tor_addr_t *tor_addr, + const netinfo_addr_t *netinfo_addr) { + tor_assert(tor_addr); + tor_assert(netinfo_addr); + + uint8_t type = netinfo_addr_get_addr_type(netinfo_addr); + uint8_t len = netinfo_addr_get_len(netinfo_addr); + + if (type == NETINFO_ADDR_TYPE_IPV4 && len == 4) { + uint32_t ipv4 = netinfo_addr_get_addr_ipv4(netinfo_addr); + tor_addr_from_ipv4h(tor_addr, ipv4); + } else if (type == NETINFO_ADDR_TYPE_IPV6 && len == 16) { + const uint8_t *ipv6_bytes = netinfo_addr_getconstarray_addr_ipv6( + netinfo_addr); + tor_addr_from_ipv6_bytes(tor_addr, (const char *)ipv6_bytes); + } else { + log_fn(LOG_PROTOCOL_WARN, LD_OR, "Cannot read address from NETINFO " + "- wrong type/length."); + return -1; + } + + return 0; +} + +/** * Process a 'netinfo' cell. * * This function is called to handle an incoming NETINFO cell; read and act @@ -1648,8 +1675,6 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan) time_t timestamp; uint8_t my_addr_type; uint8_t my_addr_len; - const uint8_t *my_addr_ptr; - const uint8_t *cp, *end; uint8_t n_other_addrs; time_t now = time(NULL); const routerinfo_t *me = router_get_my_routerinfo(); @@ -1720,34 +1745,48 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan) } /* Decode the cell. */ - timestamp = ntohl(get_uint32(cell->payload)); - if (labs(now - chan->conn->handshake_state->sent_versions_at) < 180) { - apparent_skew = now - timestamp; + netinfo_cell_t *netinfo_cell = NULL; + + ssize_t parsed = netinfo_cell_parse(&netinfo_cell, cell->payload, + CELL_PAYLOAD_SIZE); + + if (parsed < 0) { + log_fn(LOG_PROTOCOL_WARN, LD_OR, + "Failed to parse NETINFO cell - closing connection."); + connection_or_close_for_error(chan->conn, 0); + return; } - my_addr_type = (uint8_t) cell->payload[4]; - my_addr_len = (uint8_t) cell->payload[5]; - my_addr_ptr = (uint8_t*) cell->payload + 6; - end = cell->payload + CELL_PAYLOAD_SIZE; - cp = cell->payload + 6 + my_addr_len; + timestamp = netinfo_cell_get_timestamp(netinfo_cell); + const netinfo_addr_t *my_addr = + netinfo_cell_getconst_other_addr(netinfo_cell); + + my_addr_type = netinfo_addr_get_addr_type(my_addr); + my_addr_len = netinfo_addr_get_len(my_addr); + + if (labs(now - chan->conn->handshake_state->sent_versions_at) < 180) { + apparent_skew = now - timestamp; + } /* We used to check: * if (my_addr_len >= CELL_PAYLOAD_SIZE - 6) { * * This is actually never going to happen, since my_addr_len is at most 255, * and CELL_PAYLOAD_LEN - 6 is 503. So we know that cp is < end. */ - if (my_addr_type == RESOLVED_TYPE_IPV4 && my_addr_len == 4) { - tor_addr_from_ipv4n(&my_apparent_addr, get_uint32(my_addr_ptr)); + if (tor_addr_from_netinfo_addr(&my_apparent_addr, my_addr) == -1) { + connection_or_close_for_error(chan->conn, 0); + netinfo_cell_free(netinfo_cell); + return; + } + if (my_addr_type == NETINFO_ADDR_TYPE_IPV4 && my_addr_len == 4) { if (!get_options()->BridgeRelay && me && - get_uint32(my_addr_ptr) == htonl(me->addr)) { + tor_addr_eq_ipv4h(&my_apparent_addr, me->addr)) { TLS_CHAN_TO_BASE(chan)->is_canonical_to_peer = 1; } - - } else if (my_addr_type == RESOLVED_TYPE_IPV6 && my_addr_len == 16) { - tor_addr_from_ipv6_bytes(&my_apparent_addr, (const char *) my_addr_ptr); - + } else if (my_addr_type == NETINFO_ADDR_TYPE_IPV6 && + my_addr_len == 16) { if (!get_options()->BridgeRelay && me && !tor_addr_is_null(&me->ipv6_addr) && tor_addr_eq(&my_apparent_addr, &me->ipv6_addr)) { @@ -1755,18 +1794,20 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan) } } - n_other_addrs = (uint8_t) *cp++; - while (n_other_addrs && cp < end-2) { + n_other_addrs = netinfo_cell_get_n_my_addrs(netinfo_cell); + for (uint8_t i = 0; i < n_other_addrs; i++) { /* Consider all the other addresses; if any matches, this connection is * "canonical." */ + + const netinfo_addr_t *netinfo_addr = + netinfo_cell_getconst_my_addrs(netinfo_cell, i); + tor_addr_t addr; - const uint8_t *next = - decode_address_from_payload(&addr, cp, (int)(end-cp)); - if (next == NULL) { + + if (tor_addr_from_netinfo_addr(&addr, netinfo_addr) == -1) { log_fn(LOG_PROTOCOL_WARN, LD_OR, - "Bad address in netinfo cell; closing connection."); - connection_or_close_for_error(chan->conn, 0); - return; + "Bad address in netinfo cell; Skipping."); + continue; } /* A relay can connect from anywhere and be canonical, so * long as it tells you from where it came. This may sound a bit @@ -1779,10 +1820,10 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan) connection_or_set_canonical(chan->conn, 1); break; } - cp = next; - --n_other_addrs; } + netinfo_cell_free(netinfo_cell); + if (me && !TLS_CHAN_TO_BASE(chan)->is_canonical_to_peer && channel_is_canonical(TLS_CHAN_TO_BASE(chan))) { const char *descr = diff --git a/src/core/or/channeltls.h b/src/core/or/channeltls.h index 12715450b9..2ec7fe5453 100644 --- a/src/core/or/channeltls.h +++ b/src/core/or/channeltls.h @@ -49,7 +49,6 @@ channel_tls_t * channel_tls_from_base(channel_t *chan); void channel_tls_handle_cell(cell_t *cell, or_connection_t *conn); void channel_tls_handle_state_change_on_orconn(channel_tls_t *chan, or_connection_t *conn, - uint8_t old_state, uint8_t state); void channel_tls_handle_var_cell(var_cell_t *var_cell, or_connection_t *conn); diff --git a/src/core/or/circuitbuild.c b/src/core/or/circuitbuild.c index da137accc3..b89ec09a99 100644 --- a/src/core/or/circuitbuild.c +++ b/src/core/or/circuitbuild.c @@ -26,6 +26,7 @@ **/ #define CIRCUITBUILD_PRIVATE +#define OCIRC_EVENT_PRIVATE #include "core/or/or.h" #include "app/config/config.h" @@ -46,6 +47,7 @@ #include "core/or/connection_edge.h" #include "core/or/connection_or.h" #include "core/or/onion.h" +#include "core/or/ocirc_event.h" #include "core/or/policies.h" #include "core/or/relay.h" #include "feature/client/bridges.h" @@ -492,7 +494,7 @@ circuit_establish_circuit(uint8_t purpose, extend_info_t *exit_ei, int flags) return NULL; } - control_event_circuit_status(circ, CIRC_EVENT_LAUNCHED, 0); + circuit_event_status(circ, CIRC_EVENT_LAUNCHED, 0); if ((err_reason = circuit_handle_first_hop(circ)) < 0) { circuit_mark_for_close(TO_CIRCUIT(circ), -err_reason); @@ -508,6 +510,28 @@ origin_circuit_get_guard_state(origin_circuit_t *circ) return circ->guard_state; } +/** + * Helper function to publish a channel association message + * + * circuit_handle_first_hop() calls this to notify subscribers about a + * channel launch event, which associates a circuit with a channel. + * This doesn't always correspond to an assignment of the circuit's + * n_chan field, because that seems to be only for fully-open + * channels. + **/ +static void +circuit_chan_publish(const origin_circuit_t *circ, const channel_t *chan) +{ + ocirc_event_msg_t msg; + + msg.type = OCIRC_MSGTYPE_CHAN; + msg.u.chan.gid = circ->global_identifier; + msg.u.chan.chan = chan->global_identifier; + msg.u.chan.onehop = circ->build_state->onehop_tunnel; + + ocirc_event_publish(&msg); +} + /** Start establishing the first hop of our circuit. Figure out what * OR we should connect to, and if necessary start the connection to * it. If we're already connected, then send the 'create' cell. @@ -559,8 +583,6 @@ circuit_handle_first_hop(origin_circuit_t *circ) circ->base_.n_hop = extend_info_dup(firsthop->extend_info); if (should_launch) { - if (circ->build_state->onehop_tunnel) - control_event_bootstrap(BOOTSTRAP_STATUS_CONN_DIR, 0); n_chan = channel_connect_for_circuit( &firsthop->extend_info->addr, firsthop->extend_info->port, @@ -570,6 +592,7 @@ circuit_handle_first_hop(origin_circuit_t *circ) log_info(LD_CIRC,"connect to firsthop failed. Closing."); return -END_CIRC_REASON_CONNECTFAILED; } + circuit_chan_publish(circ, n_chan); } log_debug(LD_CIRC,"connecting in progress (or finished). Good."); @@ -581,6 +604,7 @@ circuit_handle_first_hop(origin_circuit_t *circ) } else { /* it's already open. use it. */ tor_assert(!circ->base_.n_hop); circ->base_.n_chan = n_chan; + circuit_chan_publish(circ, n_chan); log_debug(LD_CIRC,"Conn open. Delivering first onion skin."); if ((err_reason = circuit_send_next_onion_skin(circ)) < 0) { log_info(LD_CIRC,"circuit_send_next_onion_skin failed."); @@ -1416,7 +1440,7 @@ circuit_finish_handshake(origin_circuit_t *circ, hop->state = CPATH_STATE_OPEN; log_info(LD_CIRC,"Finished building circuit hop:"); circuit_log_path(LOG_INFO,LD_CIRC,circ); - control_event_circuit_status(circ, CIRC_EVENT_EXTENDED, 0); + circuit_event_status(circ, CIRC_EVENT_EXTENDED, 0); return 0; } @@ -1657,22 +1681,25 @@ route_len_for_purpose(uint8_t purpose, extend_info_t *exit_ei) STATIC int new_route_len(uint8_t purpose, extend_info_t *exit_ei, smartlist_t *nodes) { - int num_acceptable_routers; int routelen; tor_assert(nodes); routelen = route_len_for_purpose(purpose, exit_ei); - num_acceptable_routers = count_acceptable_nodes(nodes); + int num_acceptable_direct = count_acceptable_nodes(nodes, 1); + int num_acceptable_indirect = count_acceptable_nodes(nodes, 0); - log_debug(LD_CIRC,"Chosen route length %d (%d/%d routers suitable).", - routelen, num_acceptable_routers, smartlist_len(nodes)); + log_debug(LD_CIRC,"Chosen route length %d (%d direct and %d indirect " + "routers suitable).", routelen, num_acceptable_direct, + num_acceptable_indirect); - if (num_acceptable_routers < routelen) { + if (num_acceptable_direct < 1 || num_acceptable_indirect < routelen - 1) { log_info(LD_CIRC, - "Not enough acceptable routers (%d/%d). Discarding this circuit.", - num_acceptable_routers, routelen); + "Not enough acceptable routers (%d/%d direct and %d/%d " + "indirect routers suitable). Discarding this circuit.", + num_acceptable_direct, routelen, + num_acceptable_indirect, routelen); return -1; } @@ -2314,7 +2341,7 @@ circuit_extend_to_new_exit(origin_circuit_t *circ, extend_info_t *exit_ei) * particular router. See bug #25885.) */ MOCK_IMPL(STATIC int, -count_acceptable_nodes, (smartlist_t *nodes)) +count_acceptable_nodes, (smartlist_t *nodes, int direct)) { int num=0; @@ -2328,7 +2355,7 @@ count_acceptable_nodes, (smartlist_t *nodes)) if (! node->is_valid) // log_debug(LD_CIRC,"Nope, the directory says %d is not valid.",i); continue; - if (! node_has_any_descriptor(node)) + if (! node_has_preferred_descriptor(node, direct)) continue; /* The node has a descriptor, so we can just check the ntor key directly */ if (!node_has_curve25519_onion_key(node)) diff --git a/src/core/or/circuitbuild.h b/src/core/or/circuitbuild.h index cee71b297b..93f903f060 100644 --- a/src/core/or/circuitbuild.h +++ b/src/core/or/circuitbuild.h @@ -84,7 +84,8 @@ void circuit_upgrade_circuits_from_guard_wait(void); STATIC circid_t get_unique_circ_id_by_chan(channel_t *chan); STATIC int new_route_len(uint8_t purpose, extend_info_t *exit_ei, smartlist_t *nodes); -MOCK_DECL(STATIC int, count_acceptable_nodes, (smartlist_t *nodes)); +MOCK_DECL(STATIC int, count_acceptable_nodes, (smartlist_t *nodes, + int direct)); STATIC int onion_extend_cpath(origin_circuit_t *circ); diff --git a/src/core/or/circuitlist.c b/src/core/or/circuitlist.c index 35efc6541f..c4b5f7ee3e 100644 --- a/src/core/or/circuitlist.c +++ b/src/core/or/circuitlist.c @@ -51,6 +51,7 @@ * logic, which was originally circuit-focused. **/ #define CIRCUITLIST_PRIVATE +#define OCIRC_EVENT_PRIVATE #include "lib/cc/torint.h" /* TOR_PRIuSZ */ #include "core/or/or.h" @@ -94,7 +95,10 @@ #include "lib/compress/compress_lzma.h" #include "lib/compress/compress_zlib.h" #include "lib/compress/compress_zstd.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" + +#define OCIRC_EVENT_PRIVATE +#include "core/or/ocirc_event.h" #include "ht.h" @@ -481,6 +485,56 @@ circuit_set_n_circid_chan(circuit_t *circ, circid_t id, } } +/** + * Helper function to publish a message about events on an origin circuit + * + * Publishes a message to subscribers of origin circuit events, and + * sends the control event. + **/ +int +circuit_event_status(origin_circuit_t *circ, circuit_status_event_t tp, + int reason_code) +{ + ocirc_event_msg_t msg; + + tor_assert(circ); + + msg.type = OCIRC_MSGTYPE_CEVENT; + msg.u.cevent.gid = circ->global_identifier; + msg.u.cevent.evtype = tp; + msg.u.cevent.reason = reason_code; + msg.u.cevent.onehop = circ->build_state->onehop_tunnel; + + ocirc_event_publish(&msg); + return control_event_circuit_status(circ, tp, reason_code); +} + +/** + * Helper function to publish a state change message + * + * circuit_set_state() calls this to notify subscribers about a change + * of the state of an origin circuit. + **/ +static void +circuit_state_publish(const circuit_t *circ) +{ + ocirc_event_msg_t msg; + const origin_circuit_t *ocirc; + + if (!CIRCUIT_IS_ORIGIN(circ)) + return; + ocirc = CONST_TO_ORIGIN_CIRCUIT(circ); + /* Only inbound OR circuits can be in this state, not origin circuits. */ + tor_assert(circ->state != CIRCUIT_STATE_ONIONSKIN_PENDING); + + msg.type = OCIRC_MSGTYPE_STATE; + msg.u.state.gid = ocirc->global_identifier; + msg.u.state.state = circ->state; + msg.u.state.onehop = ocirc->build_state->onehop_tunnel; + + ocirc_event_publish(&msg); +} + /** Change the state of <b>circ</b> to <b>state</b>, adding it to or removing * it from lists as appropriate. */ void @@ -510,6 +564,7 @@ circuit_set_state(circuit_t *circ, uint8_t state) if (state == CIRCUIT_STATE_GUARD_WAIT || state == CIRCUIT_STATE_OPEN) tor_assert(!circ->n_chan_create_cell); circ->state = state; + circuit_state_publish(circ); } /** Append to <b>out</b> all circuits in state CHAN_WAIT waiting for @@ -2270,7 +2325,7 @@ circuit_about_to_free(circuit_t *circ) smartlist_remove(circuits_pending_other_guards, circ); } if (CIRCUIT_IS_ORIGIN(circ)) { - control_event_circuit_status(TO_ORIGIN_CIRCUIT(circ), + circuit_event_status(TO_ORIGIN_CIRCUIT(circ), (circ->state == CIRCUIT_STATE_OPEN || circ->state == CIRCUIT_STATE_GUARD_WAIT) ? CIRC_EVENT_CLOSED:CIRC_EVENT_FAILED, diff --git a/src/core/or/circuitlist.h b/src/core/or/circuitlist.h index cb89d1820d..37d37a089d 100644 --- a/src/core/or/circuitlist.h +++ b/src/core/or/circuitlist.h @@ -14,6 +14,7 @@ #include "lib/testsupport/testsupport.h" #include "feature/hs/hs_ident.h" +#include "core/or/ocirc_event.h" /** Circuit state: I'm the origin, still haven't done all my handshakes. */ #define CIRCUIT_STATE_BUILDING 0 @@ -184,6 +185,8 @@ void channel_mark_circid_unusable(channel_t *chan, circid_t id); void channel_mark_circid_usable(channel_t *chan, circid_t id); time_t circuit_id_when_marked_unusable_on_channel(circid_t circ_id, channel_t *chan); +int circuit_event_status(origin_circuit_t *circ, circuit_status_event_t tp, + int reason_code); void circuit_set_state(circuit_t *circ, uint8_t state); void circuit_close_all_marked(void); int32_t circuit_initial_package_window(void); diff --git a/src/core/or/circuitstats.c b/src/core/or/circuitstats.c index 0429f2c86e..61d5e18a45 100644 --- a/src/core/or/circuitstats.c +++ b/src/core/or/circuitstats.c @@ -639,9 +639,9 @@ circuit_build_times_rewind_history(circuit_build_times_t *cbt, int n) void circuit_build_times_mark_circ_as_measurement_only(origin_circuit_t *circ) { - control_event_circuit_status(circ, - CIRC_EVENT_FAILED, - END_CIRC_REASON_TIMEOUT); + circuit_event_status(circ, + CIRC_EVENT_FAILED, + END_CIRC_REASON_TIMEOUT); circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT); /* Record this event to check for too many timeouts diff --git a/src/core/or/circuituse.c b/src/core/or/circuituse.c index 088358a4d6..e230ad1005 100644 --- a/src/core/or/circuituse.c +++ b/src/core/or/circuituse.c @@ -1664,7 +1664,7 @@ circuit_testing_failed(origin_circuit_t *circ, int at_last_hop) void circuit_has_opened(origin_circuit_t *circ) { - control_event_circuit_status(circ, CIRC_EVENT_BUILT, 0); + circuit_event_status(circ, CIRC_EVENT_BUILT, 0); /* Remember that this circuit has finished building. Now if we start * it building again later (e.g. by extending it), we will know not diff --git a/src/core/or/connection_edge.c b/src/core/or/connection_edge.c index 58aefcf8f2..9f76929e53 100644 --- a/src/core/or/connection_edge.c +++ b/src/core/or/connection_edge.c @@ -62,6 +62,7 @@ #include "app/config/config.h" #include "core/mainloop/connection.h" #include "core/mainloop/mainloop.h" +#include "core/mainloop/netstatus.h" #include "core/or/channel.h" #include "core/or/circuitbuild.h" #include "core/or/circuitlist.h" @@ -97,7 +98,7 @@ #include "feature/rend/rendservice.h" #include "feature/stats/predict_ports.h" #include "feature/stats/rephist.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/crypt_ops/crypto_util.h" #include "core/or/cell_st.h" @@ -297,6 +298,11 @@ connection_edge_process_inbuf(edge_connection_t *conn, int package_partial) } return 0; case AP_CONN_STATE_OPEN: + if (! conn->base_.linked) { + note_user_activity(approx_time()); + } + + /* falls through. */ case EXIT_CONN_STATE_OPEN: if (connection_edge_package_raw_inbuf(conn, package_partial, NULL) < 0) { /* (We already sent an end cell if possible) */ @@ -751,6 +757,11 @@ connection_edge_flushed_some(edge_connection_t *conn) { switch (conn->base_.state) { case AP_CONN_STATE_OPEN: + if (! conn->base_.linked) { + note_user_activity(approx_time()); + } + + /* falls through. */ case EXIT_CONN_STATE_OPEN: connection_edge_consider_sending_sendme(conn); break; diff --git a/src/core/or/connection_or.c b/src/core/or/connection_or.c index e6f04259f5..c8667fc399 100644 --- a/src/core/or/connection_or.c +++ b/src/core/or/connection_or.c @@ -22,13 +22,14 @@ **/ #include "core/or/or.h" #include "feature/client/bridges.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" /* * Define this so we get channel internal functions, since we're implementing * part of a subclass (channel_tls_t). */ #define TOR_CHANNEL_INTERNAL_ #define CONNECTION_OR_PRIVATE +#define ORCONN_EVENT_PRIVATE #include "core/or/channel.h" #include "core/or/channeltls.h" #include "core/or/circuitbuild.h" @@ -46,6 +47,7 @@ #include "lib/geoip/geoip.h" #include "core/mainloop/mainloop.h" #include "trunnel/link_handshake.h" +#include "trunnel/netinfo.h" #include "feature/nodelist/microdesc.h" #include "feature/nodelist/networkstatus.h" #include "feature/nodelist/nodelist.h" @@ -78,6 +80,8 @@ #include "lib/tls/tortls.h" #include "lib/tls/x509.h" +#include "core/or/orconn_event.h" + static int connection_tls_finish_handshake(or_connection_t *conn); static int connection_or_launch_v3_or_handshake(or_connection_t *conn); static int connection_or_process_cells_from_inbuf(or_connection_t *conn); @@ -400,6 +404,49 @@ connection_or_report_broken_states(int severity, int domain) smartlist_free(items); } +/** + * Helper function to publish an OR connection status event + * + * Publishes a messages to subscribers of ORCONN messages, and sends + * the control event. + **/ +void +connection_or_event_status(or_connection_t *conn, or_conn_status_event_t tp, + int reason) +{ + orconn_event_msg_t msg; + + msg.type = ORCONN_MSGTYPE_STATUS; + msg.u.status.gid = conn->base_.global_identifier; + msg.u.status.status = tp; + msg.u.status.reason = reason; + orconn_event_publish(&msg); + control_event_or_conn_status(conn, tp, reason); +} + +/** + * Helper function to publish a state change message + * + * connection_or_change_state() calls this to notify subscribers about + * a change of an OR connection state. + **/ +static void +connection_or_state_publish(const or_connection_t *conn, uint8_t state) +{ + orconn_event_msg_t msg; + + msg.type = ORCONN_MSGTYPE_STATE; + msg.u.state.gid = conn->base_.global_identifier; + msg.u.state.proxy_type = conn->proxy_type; + msg.u.state.state = state; + if (conn->chan) { + msg.u.state.chan = TLS_CHAN_TO_BASE(conn->chan)->global_identifier; + } else { + msg.u.state.chan = 0; + } + orconn_event_publish(&msg); +} + /** Call this to change or_connection_t states, so the owning channel_tls_t can * be notified. */ @@ -407,16 +454,13 @@ connection_or_report_broken_states(int severity, int domain) static void connection_or_change_state(or_connection_t *conn, uint8_t state) { - uint8_t old_state; - tor_assert(conn); - old_state = conn->base_.state; conn->base_.state = state; + connection_or_state_publish(conn, state); if (conn->chan) - channel_tls_handle_state_change_on_orconn(conn->chan, conn, - old_state, state); + channel_tls_handle_state_change_on_orconn(conn->chan, conn, state); } /** Return the number of circuits using an or_connection_t; this used to @@ -707,8 +751,6 @@ connection_or_finished_connecting(or_connection_t *or_conn) log_debug(LD_HANDSHAKE,"OR connect() to router at %s:%u finished.", conn->address,conn->port); - control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE, 0); - control_event_boot_first_orconn(); if (proxy_type != PROXY_NONE) { /* start proxy handshake */ @@ -758,8 +800,8 @@ connection_or_about_to_close(or_connection_t *or_conn) entry_guard_chan_failed(TLS_CHAN_TO_BASE(or_conn->chan)); if (conn->state >= OR_CONN_STATE_TLS_HANDSHAKING) { int reason = tls_error_to_orconn_end_reason(or_conn->tls_error); - control_event_or_conn_status(or_conn, OR_CONN_EVENT_FAILED, - reason); + connection_or_event_status(or_conn, OR_CONN_EVENT_FAILED, + reason); if (!authdir_mode_tests_reachability(options)) control_event_bootstrap_prob_or( orconn_end_reason_to_control_string(reason), @@ -769,10 +811,10 @@ connection_or_about_to_close(or_connection_t *or_conn) } else if (conn->hold_open_until_flushed) { /* We only set hold_open_until_flushed when we're intentionally * closing a connection. */ - control_event_or_conn_status(or_conn, OR_CONN_EVENT_CLOSED, + connection_or_event_status(or_conn, OR_CONN_EVENT_CLOSED, tls_error_to_orconn_end_reason(or_conn->tls_error)); } else if (!tor_digest_is_zero(or_conn->identity_digest)) { - control_event_or_conn_status(or_conn, OR_CONN_EVENT_CLOSED, + connection_or_event_status(or_conn, OR_CONN_EVENT_CLOSED, tls_error_to_orconn_end_reason(or_conn->tls_error)); } } @@ -1364,7 +1406,7 @@ void connection_or_connect_failed(or_connection_t *conn, int reason, const char *msg) { - control_event_or_conn_status(conn, OR_CONN_EVENT_FAILED, reason); + connection_or_event_status(conn, OR_CONN_EVENT_FAILED, reason); if (!authdir_mode_tests_reachability(get_options())) control_event_bootstrap_prob_or(msg, reason, conn); note_or_connect_failed(conn); @@ -1471,9 +1513,6 @@ connection_or_connect, (const tor_addr_t *_addr, uint16_t port, return NULL; } - connection_or_change_state(conn, OR_CONN_STATE_CONNECTING); - control_event_or_conn_status(conn, OR_CONN_EVENT_LAUNCHED, 0); - conn->is_outgoing = 1; /* If we are using a proxy server, find it and use it. */ @@ -1485,7 +1524,14 @@ connection_or_connect, (const tor_addr_t *_addr, uint16_t port, port = proxy_port; conn->base_.proxy_state = PROXY_INFANT; } + connection_or_change_state(conn, OR_CONN_STATE_CONNECTING); + connection_or_event_status(conn, OR_CONN_EVENT_LAUNCHED, 0); } else { + /* This duplication of state change calls is necessary in case we + * run into an error condition below */ + connection_or_change_state(conn, OR_CONN_STATE_CONNECTING); + connection_or_event_status(conn, OR_CONN_EVENT_LAUNCHED, 0); + /* get_proxy_addrport() might fail if we have a Bridge line that references a transport, but no ClientTransportPlugin lines defining its transport proxy. If this is the case, let's try to @@ -1981,8 +2027,8 @@ connection_or_client_learned_peer_id(or_connection_t *conn, /* Tell the new guard API about the channel failure */ entry_guard_chan_failed(TLS_CHAN_TO_BASE(conn->chan)); - control_event_or_conn_status(conn, OR_CONN_EVENT_FAILED, - END_OR_CONN_REASON_OR_IDENTITY); + connection_or_event_status(conn, OR_CONN_EVENT_FAILED, + END_OR_CONN_REASON_OR_IDENTITY); if (!authdir_mode_tests_reachability(options)) control_event_bootstrap_prob_or( "Unexpected identity in router certificate", @@ -2222,7 +2268,7 @@ int connection_or_set_state_open(or_connection_t *conn) { connection_or_change_state(conn, OR_CONN_STATE_OPEN); - control_event_or_conn_status(conn, OR_CONN_EVENT_CONNECTED, 0); + connection_or_event_status(conn, OR_CONN_EVENT_CONNECTED, 0); /* Link protocol 3 appeared in Tor 0.2.3.6-alpha, so any connection * that uses an earlier link protocol should not be treated as a relay. */ @@ -2428,6 +2474,31 @@ connection_or_send_versions(or_connection_t *conn, int v3_plus) return 0; } +static netinfo_addr_t * +netinfo_addr_from_tor_addr(const tor_addr_t *tor_addr) +{ + sa_family_t addr_family = tor_addr_family(tor_addr); + + if (BUG(addr_family != AF_INET && addr_family != AF_INET6)) + return NULL; + + netinfo_addr_t *netinfo_addr = netinfo_addr_new(); + + if (addr_family == AF_INET) { + netinfo_addr_set_addr_type(netinfo_addr, NETINFO_ADDR_TYPE_IPV4); + netinfo_addr_set_len(netinfo_addr, 4); + netinfo_addr_set_addr_ipv4(netinfo_addr, tor_addr_to_ipv4h(tor_addr)); + } else if (addr_family == AF_INET6) { + netinfo_addr_set_addr_type(netinfo_addr, NETINFO_ADDR_TYPE_IPV6); + netinfo_addr_set_len(netinfo_addr, 16); + uint8_t *ipv6_buf = netinfo_addr_getarray_addr_ipv6(netinfo_addr); + const uint8_t *in6_addr = tor_addr_to_in6_addr8(tor_addr); + memcpy(ipv6_buf, in6_addr, 16); + } + + return netinfo_addr; +} + /** Send a NETINFO cell on <b>conn</b>, telling the other server what we know * about their address, our address, and the current time. */ MOCK_IMPL(int, @@ -2436,8 +2507,7 @@ connection_or_send_netinfo,(or_connection_t *conn)) cell_t cell; time_t now = time(NULL); const routerinfo_t *me; - int len; - uint8_t *out; + int r = -1; tor_assert(conn->handshake_state); @@ -2450,20 +2520,21 @@ connection_or_send_netinfo,(or_connection_t *conn)) memset(&cell, 0, sizeof(cell_t)); cell.command = CELL_NETINFO; + netinfo_cell_t *netinfo_cell = netinfo_cell_new(); + /* Timestamp, if we're a relay. */ if (public_server_mode(get_options()) || ! conn->is_outgoing) - set_uint32(cell.payload, htonl((uint32_t)now)); + netinfo_cell_set_timestamp(netinfo_cell, (uint32_t)now); /* Their address. */ - out = cell.payload + 4; + const tor_addr_t *remote_tor_addr = + !tor_addr_is_null(&conn->real_addr) ? &conn->real_addr : &conn->base_.addr; /* We use &conn->real_addr below, unless it hasn't yet been set. If it * hasn't yet been set, we know that base_.addr hasn't been tampered with * yet either. */ - len = append_address_to_payload(out, !tor_addr_is_null(&conn->real_addr) - ? &conn->real_addr : &conn->base_.addr); - if (len<0) - return -1; - out += len; + netinfo_addr_t *their_addr = netinfo_addr_from_tor_addr(remote_tor_addr); + + netinfo_cell_set_other_addr(netinfo_cell, their_addr); /* My address -- only include it if I'm a public relay, or if I'm a * bridge and this is an incoming connection. If I'm a bridge and this @@ -2471,28 +2542,42 @@ connection_or_send_netinfo,(or_connection_t *conn)) if ((public_server_mode(get_options()) || !conn->is_outgoing) && (me = router_get_my_routerinfo())) { tor_addr_t my_addr; - *out++ = 1 + !tor_addr_is_null(&me->ipv6_addr); - tor_addr_from_ipv4h(&my_addr, me->addr); - len = append_address_to_payload(out, &my_addr); - if (len < 0) - return -1; - out += len; + + uint8_t n_my_addrs = 1 + !tor_addr_is_null(&me->ipv6_addr); + netinfo_cell_set_n_my_addrs(netinfo_cell, n_my_addrs); + + netinfo_cell_add_my_addrs(netinfo_cell, + netinfo_addr_from_tor_addr(&my_addr)); if (!tor_addr_is_null(&me->ipv6_addr)) { - len = append_address_to_payload(out, &me->ipv6_addr); - if (len < 0) - return -1; + netinfo_cell_add_my_addrs(netinfo_cell, + netinfo_addr_from_tor_addr(&me->ipv6_addr)); } - } else { - *out = 0; + } + + const char *errmsg = NULL; + if ((errmsg = netinfo_cell_check(netinfo_cell))) { + log_warn(LD_OR, "Failed to validate NETINFO cell with error: %s", + errmsg); + goto cleanup; + } + + if (netinfo_cell_encode(cell.payload, CELL_PAYLOAD_SIZE, + netinfo_cell) < 0) { + log_warn(LD_OR, "Failed generating NETINFO cell"); + goto cleanup; } conn->handshake_state->digest_sent_data = 0; conn->handshake_state->sent_netinfo = 1; connection_or_write_cell_to_buf(&cell, conn); - return 0; + r = 0; + cleanup: + netinfo_cell_free(netinfo_cell); + + return r; } /** Helper used to add an encoded certs to a cert cell */ diff --git a/src/core/or/connection_or.h b/src/core/or/connection_or.h index d4bcdd93e5..5f4856d51f 100644 --- a/src/core/or/connection_or.h +++ b/src/core/or/connection_or.h @@ -17,32 +17,7 @@ struct ed25519_keypair_t; or_connection_t *TO_OR_CONN(connection_t *); -#define OR_CONN_STATE_MIN_ 1 -/** State for a connection to an OR: waiting for connect() to finish. */ -#define OR_CONN_STATE_CONNECTING 1 -/** State for a connection to an OR: waiting for proxy handshake to complete */ -#define OR_CONN_STATE_PROXY_HANDSHAKING 2 -/** State for an OR connection client: SSL is handshaking, not done - * yet. */ -#define OR_CONN_STATE_TLS_HANDSHAKING 3 -/** State for a connection to an OR: We're doing a second SSL handshake for - * renegotiation purposes. (V2 handshake only.) */ -#define OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING 4 -/** State for a connection at an OR: We're waiting for the client to - * renegotiate (to indicate a v2 handshake) or send a versions cell (to - * indicate a v3 handshake) */ -#define OR_CONN_STATE_TLS_SERVER_RENEGOTIATING 5 -/** State for an OR connection: We're done with our SSL handshake, we've done - * renegotiation, but we haven't yet negotiated link protocol versions and - * sent a netinfo cell. */ -#define OR_CONN_STATE_OR_HANDSHAKING_V2 6 -/** State for an OR connection: We're done with our SSL handshake, but we - * haven't yet negotiated link protocol versions, done a V3 handshake, and - * sent a netinfo cell. */ -#define OR_CONN_STATE_OR_HANDSHAKING_V3 7 -/** State for an OR connection: Ready to send/receive cells. */ -#define OR_CONN_STATE_OPEN 8 -#define OR_CONN_STATE_MAX_ 8 +#include "core/or/orconn_event.h" void connection_or_clear_identity(or_connection_t *conn); void connection_or_clear_identity_map(void); @@ -81,6 +56,9 @@ MOCK_DECL(void,connection_or_close_for_error, void connection_or_report_broken_states(int severity, int domain); +void connection_or_event_status(or_connection_t *conn, + or_conn_status_event_t tp, int reason); + MOCK_DECL(int,connection_tls_start_handshake,(or_connection_t *conn, int receiving)); int connection_tls_continue_handshake(or_connection_t *conn); diff --git a/src/core/or/ocirc_event.c b/src/core/or/ocirc_event.c new file mode 100644 index 0000000000..b400022bb7 --- /dev/null +++ b/src/core/or/ocirc_event.c @@ -0,0 +1,84 @@ +/* Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file ocirc_event.c + * \brief Publish state change messages for origin circuits + * + * Implements a basic publish-subscribe framework for messages about + * the state of origin circuits. The publisher calls the subscriber + * callback functions synchronously. + * + * Although the synchronous calls might not simplify the call graph, + * this approach improves data isolation because the publisher doesn't + * need knowledge about the internals of subscribing subsystems. It + * also avoids race conditions that might occur in asynchronous + * frameworks. + **/ + +#include "core/or/or.h" + +#define OCIRC_EVENT_PRIVATE + +#include "core/or/cpath_build_state_st.h" +#include "core/or/ocirc_event.h" +#include "core/or/ocirc_event_sys.h" +#include "core/or/origin_circuit_st.h" +#include "lib/subsys/subsys.h" + +/** List of subscribers */ +static smartlist_t *ocirc_event_rcvrs; + +/** Initialize subscriber list */ +static int +ocirc_event_init(void) +{ + ocirc_event_rcvrs = smartlist_new(); + return 0; +} + +/** Free subscriber list */ +static void +ocirc_event_fini(void) +{ + smartlist_free(ocirc_event_rcvrs); +} + +/** + * Subscribe to messages about origin circuit events + * + * Register a callback function to receive messages about origin + * circuits. The publisher calls this function synchronously. + **/ +void +ocirc_event_subscribe(ocirc_event_rcvr_t fn) +{ + tor_assert(fn); + /* Don't duplicate subscriptions. */ + if (smartlist_contains(ocirc_event_rcvrs, fn)) + return; + + smartlist_add(ocirc_event_rcvrs, fn); +} + +/** + * Publish a message about OR connection events + * + * This calls the subscriber receiver function synchronously. + **/ +void +ocirc_event_publish(const ocirc_event_msg_t *msg) +{ + SMARTLIST_FOREACH_BEGIN(ocirc_event_rcvrs, ocirc_event_rcvr_t, fn) { + tor_assert(fn); + (*fn)(msg); + } SMARTLIST_FOREACH_END(fn); +} + +const subsys_fns_t sys_ocirc_event = { + .name = "ocirc_event", + .supported = true, + .level = -32, + .initialize = ocirc_event_init, + .shutdown = ocirc_event_fini, +}; diff --git a/src/core/or/ocirc_event.h b/src/core/or/ocirc_event.h new file mode 100644 index 0000000000..19a237d7df --- /dev/null +++ b/src/core/or/ocirc_event.h @@ -0,0 +1,89 @@ +/* Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file ocirc_event.h + * \brief Header file for ocirc_event.c + **/ + +#ifndef TOR_OCIRC_EVENT_H +#define TOR_OCIRC_EVENT_H + +#include <stdbool.h> + +#include "lib/cc/torint.h" + +/** Used to indicate the type of a circuit event passed to the controller. + * The various types are defined in control-spec.txt */ +typedef enum circuit_status_event_t { + CIRC_EVENT_LAUNCHED = 0, + CIRC_EVENT_BUILT = 1, + CIRC_EVENT_EXTENDED = 2, + CIRC_EVENT_FAILED = 3, + CIRC_EVENT_CLOSED = 4, +} circuit_status_event_t; + +/** Message for origin circuit state update */ +typedef struct ocirc_state_msg_t { + uint32_t gid; /**< global ID (only origin circuits have them) */ + int state; /**< new circuit state */ + bool onehop; /**< one-hop circuit? */ +} ocirc_state_msg_t; + +/** + * Message when a channel gets associated to a circuit. + * + * This doesn't always correspond to something in circuitbuild.c + * setting the n_chan field in the circuit. For some reason, if + * circuit_handle_first_hop() launches a new circuit, it doesn't set + * the n_chan field. + */ +typedef struct ocirc_chan_msg_t { + uint32_t gid; /**< global ID */ + uint64_t chan; /**< channel ID */ + bool onehop; /**< one-hop circuit? */ +} ocirc_chan_msg_t; + +/** + * Message for origin circuit status event + * + * This contains information that ends up in CIRC control protocol events. + */ +typedef struct ocirc_cevent_msg_t { + uint32_t gid; /**< global ID */ + int evtype; /**< event type */ + int reason; /**< reason */ + bool onehop; /**< one-hop circuit? */ +} ocirc_cevent_msg_t; + +/** Discriminant values for origin circuit event message */ +typedef enum ocirc_msgtype_t { + OCIRC_MSGTYPE_STATE, + OCIRC_MSGTYPE_CHAN, + OCIRC_MSGTYPE_CEVENT, +} ocirc_msgtype_t; + +/** Discriminated union for the actual message */ +typedef struct ocirc_event_msg_t { + int type; + union { + ocirc_state_msg_t state; + ocirc_chan_msg_t chan; + ocirc_cevent_msg_t cevent; + } u; +} ocirc_event_msg_t; + +/** + * Receiver function pointer for origin circuit subscribers + * + * This function gets called synchronously by the publisher. + **/ +typedef void (*ocirc_event_rcvr_t)(const ocirc_event_msg_t *); + +void ocirc_event_subscribe(ocirc_event_rcvr_t fn); + +#ifdef OCIRC_EVENT_PRIVATE +void ocirc_event_publish(const ocirc_event_msg_t *msg); +#endif + +#endif /* defined(TOR_OCIRC_EVENT_STATE_H) */ diff --git a/src/core/or/ocirc_event_sys.h b/src/core/or/ocirc_event_sys.h new file mode 100644 index 0000000000..0bc135ffaf --- /dev/null +++ b/src/core/or/ocirc_event_sys.h @@ -0,0 +1,13 @@ +/* Copyright (c) 2007-2018, The Tor Project, Inc. */ + +/** + * \file ocirc_event_sys.h + * \brief Declare subsystem object for the origin circuit event module. + **/ + +#ifndef TOR_OCIRC_EVENT_SYS_H +#define TOR_OCIRC_EVENT_SYS_H + +extern const struct subsys_fns_t sys_ocirc_event; + +#endif /* defined(TOR_OCIRC_EVENT_H) */ diff --git a/src/core/or/or.h b/src/core/or/or.h index acf092c8dc..ca373d8ed5 100644 --- a/src/core/or/or.h +++ b/src/core/or/or.h @@ -26,7 +26,7 @@ #include "lib/cc/compat_compiler.h" #include "lib/cc/torint.h" #include "lib/container/map.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/container/smartlist.h" #include "lib/crypt_ops/crypto_cipher.h" #include "lib/crypt_ops/crypto_rsa.h" @@ -97,6 +97,8 @@ struct curve25519_public_key_t; #define SIGNEWNYM 129 #define SIGCLEARDNSCACHE 130 #define SIGHEARTBEAT 131 +#define SIGACTIVE 132 +#define SIGDORMANT 133 #if (SIZEOF_CELL_T != 0) /* On Irix, stdlib.h defines a cell_t type, so we need to make sure diff --git a/src/core/or/orconn_event.c b/src/core/or/orconn_event.c new file mode 100644 index 0000000000..d81f7b5a0c --- /dev/null +++ b/src/core/or/orconn_event.c @@ -0,0 +1,81 @@ +/* Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file orconn_event.c + * \brief Publish state change messages for OR connections + * + * Implements a basic publish-subscribe framework for messages about + * the state of OR connections. The publisher calls the subscriber + * callback functions synchronously. + * + * Although the synchronous calls might not simplify the call graph, + * this approach improves data isolation because the publisher doesn't + * need knowledge about the internals of subscribing subsystems. It + * also avoids race conditions that might occur in asynchronous + * frameworks. + **/ + +#include "core/or/or.h" +#include "lib/subsys/subsys.h" + +#define ORCONN_EVENT_PRIVATE +#include "core/or/orconn_event.h" +#include "core/or/orconn_event_sys.h" + +/** List of subscribers */ +static smartlist_t *orconn_event_rcvrs; + +/** Initialize subscriber list */ +static int +orconn_event_init(void) +{ + orconn_event_rcvrs = smartlist_new(); + return 0; +} + +/** Free subscriber list */ +static void +orconn_event_fini(void) +{ + smartlist_free(orconn_event_rcvrs); +} + +/** + * Subscribe to messages about OR connection events + * + * Register a callback function to receive messages about ORCONNs. + * The publisher calls this function synchronously. + **/ +void +orconn_event_subscribe(orconn_event_rcvr_t fn) +{ + tor_assert(fn); + /* Don't duplicate subscriptions. */ + if (smartlist_contains(orconn_event_rcvrs, fn)) + return; + + smartlist_add(orconn_event_rcvrs, fn); +} + +/** + * Publish a message about OR connection events + * + * This calls the subscriber receiver function synchronously. + **/ +void +orconn_event_publish(const orconn_event_msg_t *msg) +{ + SMARTLIST_FOREACH_BEGIN(orconn_event_rcvrs, orconn_event_rcvr_t, fn) { + tor_assert(fn); + (*fn)(msg); + } SMARTLIST_FOREACH_END(fn); +} + +const subsys_fns_t sys_orconn_event = { + .name = "orconn_event", + .supported = true, + .level = -33, + .initialize = orconn_event_init, + .shutdown = orconn_event_fini, +}; diff --git a/src/core/or/orconn_event.h b/src/core/or/orconn_event.h new file mode 100644 index 0000000000..4c999e53be --- /dev/null +++ b/src/core/or/orconn_event.h @@ -0,0 +1,120 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file orconn_event.h + * \brief Header file for orconn_event.c + * + * The OR_CONN_STATE_* symbols are here to make it easier for + * subscribers to make decisions based on the messages that they + * receive. + **/ + +#ifndef TOR_ORCONN_EVENT_H +#define TOR_ORCONN_EVENT_H + +/** + * @name States of OR connections + * + * These must be in a partial ordering such that usually no OR + * connection will transition from a higher-numbered state to a + * lower-numbered one. Code such as bto_update_best() depends on this + * ordering to determine the best state it's seen so far. + * @{ */ +#define OR_CONN_STATE_MIN_ 1 +/** State for a connection to an OR: waiting for connect() to finish. */ +#define OR_CONN_STATE_CONNECTING 1 +/** State for a connection to an OR: waiting for proxy handshake to complete */ +#define OR_CONN_STATE_PROXY_HANDSHAKING 2 +/** State for an OR connection client: SSL is handshaking, not done + * yet. */ +#define OR_CONN_STATE_TLS_HANDSHAKING 3 +/** State for a connection to an OR: We're doing a second SSL handshake for + * renegotiation purposes. (V2 handshake only.) */ +#define OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING 4 +/** State for a connection at an OR: We're waiting for the client to + * renegotiate (to indicate a v2 handshake) or send a versions cell (to + * indicate a v3 handshake) */ +#define OR_CONN_STATE_TLS_SERVER_RENEGOTIATING 5 +/** State for an OR connection: We're done with our SSL handshake, we've done + * renegotiation, but we haven't yet negotiated link protocol versions and + * sent a netinfo cell. */ +#define OR_CONN_STATE_OR_HANDSHAKING_V2 6 +/** State for an OR connection: We're done with our SSL handshake, but we + * haven't yet negotiated link protocol versions, done a V3 handshake, and + * sent a netinfo cell. */ +#define OR_CONN_STATE_OR_HANDSHAKING_V3 7 +/** State for an OR connection: Ready to send/receive cells. */ +#define OR_CONN_STATE_OPEN 8 +#define OR_CONN_STATE_MAX_ 8 +/** @} */ + +/** Used to indicate the type of an OR connection event passed to the + * controller. The various types are defined in control-spec.txt */ +typedef enum or_conn_status_event_t { + OR_CONN_EVENT_LAUNCHED = 0, + OR_CONN_EVENT_CONNECTED = 1, + OR_CONN_EVENT_FAILED = 2, + OR_CONN_EVENT_CLOSED = 3, + OR_CONN_EVENT_NEW = 4, +} or_conn_status_event_t; + +/** Discriminant values for orconn event message */ +typedef enum orconn_msgtype_t { + ORCONN_MSGTYPE_STATE, + ORCONN_MSGTYPE_STATUS, +} orconn_msgtype_t; + +/** + * Message for orconn state update + * + * This contains information about internal state changes of + * or_connection_t objects. The chan and proxy_type fields are + * additional information that a subscriber may need to make + * decisions. + **/ +typedef struct orconn_state_msg_t { + uint64_t gid; /**< connection's global ID */ + uint64_t chan; /**< associated channel ID */ + int proxy_type; /**< connection's proxy type */ + uint8_t state; /**< new connection state */ +} orconn_state_msg_t; + +/** + * Message for orconn status event + * + * This contains information that ends up in ORCONN control protocol + * events. + **/ +typedef struct orconn_status_msg_t { + uint64_t gid; /**< connection's global ID */ + int status; /**< or_conn_status_event_t */ + int reason; /**< reason */ +} orconn_status_msg_t; + +/** Discriminated union for the actual message */ +typedef struct orconn_event_msg_t { + int type; + union { + orconn_state_msg_t state; + orconn_status_msg_t status; + } u; +} orconn_event_msg_t; + +/** + * Receiver function pointer for OR subscribers + * + * This function gets called synchronously by the publisher. + **/ +typedef void (*orconn_event_rcvr_t)(const orconn_event_msg_t *); + +void orconn_event_subscribe(orconn_event_rcvr_t); + +#ifdef ORCONN_EVENT_PRIVATE +void orconn_event_publish(const orconn_event_msg_t *); +#endif + +#endif /* defined(TOR_ORCONN_EVENT_H) */ diff --git a/src/core/or/orconn_event_sys.h b/src/core/or/orconn_event_sys.h new file mode 100644 index 0000000000..7639023386 --- /dev/null +++ b/src/core/or/orconn_event_sys.h @@ -0,0 +1,12 @@ +/* Copyright (c) 2007-2018, The Tor Project, Inc. */ + +/** + * \file orconn_event_sys.h + * \brief Declare subsystem object for the OR connection event module. + **/ +#ifndef TOR_ORCONN_EVENT_SYS_H +#define TOR_ORCONN_EVENT_SYS_H + +extern const struct subsys_fns_t sys_orconn_event; + +#endif /* defined(TOR_ORCONN_SYS_H) */ diff --git a/src/core/or/policies.c b/src/core/or/policies.c index 0eda93c5f3..bffdb1fddd 100644 --- a/src/core/or/policies.c +++ b/src/core/or/policies.c @@ -29,6 +29,7 @@ #include "feature/relay/routermode.h" #include "lib/geoip/geoip.h" #include "ht.h" +#include "lib/crypt_ops/crypto_rand.h" #include "lib/encoding/confline.h" #include "core/or/addr_policy_st.h" @@ -461,7 +462,8 @@ fascist_firewall_use_ipv6(const or_options_t *options) * ClientPreferIPv6DirPort is deprecated, but check it anyway. */ return (options->ClientUseIPv6 == 1 || options->ClientUseIPv4 == 0 || options->ClientPreferIPv6ORPort == 1 || - options->ClientPreferIPv6DirPort == 1 || options->UseBridges == 1); + options->ClientPreferIPv6DirPort == 1 || options->UseBridges == 1 || + options->ClientAutoIPv6ORPort == 1); } /** Do we prefer to connect to IPv6, ignoring ClientPreferIPv6ORPort and @@ -488,6 +490,15 @@ fascist_firewall_prefer_ipv6_impl(const or_options_t *options) return -1; } +/* Choose whether we prefer IPv4 or IPv6 by randomly choosing an address + * family. Return 0 for IPv4, and 1 for IPv6. */ +MOCK_IMPL(int, +fascist_firewall_rand_prefer_ipv6_addr, (void)) +{ + /* TODO: Check for failures, and infer our preference based on this. */ + return crypto_rand_int(2); +} + /** Do we prefer to connect to IPv6 ORPorts? * Use node_ipv6_or_preferred() whenever possible: it supports bridge client * per-node IPv6 preferences. @@ -502,7 +513,10 @@ fascist_firewall_prefer_ipv6_orport(const or_options_t *options) } /* We can use both IPv4 and IPv6 - which do we prefer? */ - if (options->ClientPreferIPv6ORPort == 1) { + if (options->ClientAutoIPv6ORPort == 1) { + /* If ClientAutoIPv6ORPort is 1, we prefer IPv4 or IPv6 at random. */ + return fascist_firewall_rand_prefer_ipv6_addr(); + } else if (options->ClientPreferIPv6ORPort == 1) { return 1; } @@ -2706,7 +2720,7 @@ parse_short_policy(const char *summary) int is_accept; int n_entries; short_policy_entry_t entries[MAX_EXITPOLICY_SUMMARY_LEN]; /* overkill */ - const char *next; + char *next; if (!strcmpstart(summary, "accept ")) { is_accept = 1; @@ -2721,57 +2735,56 @@ parse_short_policy(const char *summary) n_entries = 0; for ( ; *summary; summary = next) { - const char *comma = strchr(summary, ','); - unsigned low, high; - char dummy; - char ent_buf[32]; - size_t len; - - next = comma ? comma+1 : strchr(summary, '\0'); - len = comma ? (size_t)(comma - summary) : strlen(summary); - if (n_entries == MAX_EXITPOLICY_SUMMARY_LEN) { log_fn(LOG_PROTOCOL_WARN, LD_DIR, "Impossibly long policy summary %s", escaped(orig_summary)); return NULL; } - if (! TOR_ISDIGIT(*summary) || len > (sizeof(ent_buf)-1)) { - /* unrecognized entry format. skip it. */ - continue; - } - if (len < 1) { - /* empty; skip it. */ - /* XXX This happens to be unreachable, since if len==0, then *summary is - * ',' or '\0', and the TOR_ISDIGIT test above would have failed. */ - continue; + unsigned low, high; + int ok; + low = (unsigned) tor_parse_ulong(summary, 10, 1, 65535, &ok, &next); + if (!ok) { + if (! TOR_ISDIGIT(*summary) || *summary == ',') { + /* Unrecognized format: skip it. */ + goto skip_ent; + } else { + goto bad_ent; + } } - memcpy(ent_buf, summary, len); - ent_buf[len] = '\0'; + switch (*next) { + case ',': + ++next; + /* fall through */ + case '\0': + high = low; + break; + case '-': + high = (unsigned) tor_parse_ulong(next+1, 10, low, 65535, &ok, &next); + if (!ok) + goto bad_ent; - if (tor_sscanf(ent_buf, "%u-%u%c", &low, &high, &dummy) == 2) { - if (low<1 || low>65535 || high<1 || high>65535 || low>high) { - log_fn(LOG_PROTOCOL_WARN, LD_DIR, - "Found bad entry in policy summary %s", escaped(orig_summary)); - return NULL; - } - } else if (tor_sscanf(ent_buf, "%u%c", &low, &dummy) == 1) { - if (low<1 || low>65535) { - log_fn(LOG_PROTOCOL_WARN, LD_DIR, - "Found bad entry in policy summary %s", escaped(orig_summary)); - return NULL; - } - high = low; - } else { - log_fn(LOG_PROTOCOL_WARN, LD_DIR,"Found bad entry in policy summary %s", - escaped(orig_summary)); - return NULL; + if (*next == ',') + ++next; + else if (*next != '\0') + goto bad_ent; + + break; + default: + goto bad_ent; } entries[n_entries].min_port = low; entries[n_entries].max_port = high; n_entries++; + + continue; + skip_ent: + next = strchr(next, ','); + if (!next) + break; + ++next; } if (n_entries == 0) { @@ -2792,6 +2805,11 @@ parse_short_policy(const char *summary) result->n_entries = n_entries; memcpy(result->entries, entries, sizeof(short_policy_entry_t)*n_entries); return result; + + bad_ent: + log_fn(LOG_PROTOCOL_WARN, LD_DIR,"Found bad entry in policy summary %s", + escaped(orig_summary)); + return NULL; } /** Write <b>policy</b> back out into a string. */ diff --git a/src/core/or/policies.h b/src/core/or/policies.h index 7da3ba031f..0c64ecf378 100644 --- a/src/core/or/policies.h +++ b/src/core/or/policies.h @@ -70,6 +70,7 @@ typedef struct short_policy_t { int firewall_is_fascist_or(void); int firewall_is_fascist_dir(void); int fascist_firewall_use_ipv6(const or_options_t *options); +MOCK_DECL(int, fascist_firewall_rand_prefer_ipv6_addr, (void)); int fascist_firewall_prefer_ipv6_orport(const or_options_t *options); int fascist_firewall_prefer_ipv6_dirport(const or_options_t *options); diff --git a/src/core/or/relay.c b/src/core/or/relay.c index 510d96c648..2e92f2a55d 100644 --- a/src/core/or/relay.c +++ b/src/core/or/relay.c @@ -49,7 +49,7 @@ #include "core/or/or.h" #include "feature/client/addressmap.h" #include "lib/err/backtrace.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/or/channel.h" #include "feature/client/circpathbias.h" #include "core/or/circuitbuild.h" diff --git a/src/core/or/scheduler.c b/src/core/or/scheduler.c index 326e0d65d9..937e7e45db 100644 --- a/src/core/or/scheduler.c +++ b/src/core/or/scheduler.c @@ -9,7 +9,7 @@ #define SCHEDULER_KIST_PRIVATE #include "core/or/scheduler.h" #include "core/mainloop/mainloop.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #define TOR_CHANNEL_INTERNAL_ #include "core/or/channeltls.h" #include "lib/evloop/compat_libevent.h" diff --git a/src/core/or/scheduler_kist.c b/src/core/or/scheduler_kist.c index f112ea6357..3ed0f1a5e2 100644 --- a/src/core/or/scheduler_kist.c +++ b/src/core/or/scheduler_kist.c @@ -4,7 +4,7 @@ #define SCHEDULER_KIST_PRIVATE #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "app/config/config.h" #include "core/mainloop/connection.h" #include "feature/nodelist/networkstatus.h" diff --git a/src/core/or/versions.c b/src/core/or/versions.c index 06274996a7..7bd1f5899f 100644 --- a/src/core/or/versions.c +++ b/src/core/or/versions.c @@ -16,6 +16,25 @@ #include "core/or/tor_version_st.h" +/** + * Return the approximate date when this release came out, or was + * scheduled to come out, according to the APPROX_RELEASE_DATE set in + * configure.ac + **/ +time_t +tor_get_approx_release_date(void) +{ + char tbuf[ISO_TIME_LEN+1]; + tor_snprintf(tbuf, sizeof(tbuf), + "%s 00:00:00", APPROX_RELEASE_DATE); + time_t result = 0; + int r = parse_iso_time(tbuf, &result); + if (BUG(r < 0)) { + result = 0; + } + return result; +} + /** Return VS_RECOMMENDED if <b>myversion</b> is contained in * <b>versionlist</b>. Else, return VS_EMPTY if versionlist has no * entries. Else, return VS_OLD if every member of @@ -377,6 +396,64 @@ sort_version_list(smartlist_t *versions, int remove_duplicates) smartlist_uniq(versions, compare_tor_version_str_ptr_, tor_free_); } +/** If there are more than this many entries, we're probably under + * some kind of weird DoS. */ +static const int MAX_PROTOVER_SUMMARY_MAP_LEN = 1024; + +/** + * Map from protover string to protover_summary_flags_t. + */ +static strmap_t *protover_summary_map = NULL; + +/** + * Helper. Given a non-NULL protover string <b>protocols</b>, set <b>out</b> + * to its summary, and memoize the result in <b>protover_summary_map</b>. + */ +static void +memoize_protover_summary(protover_summary_flags_t *out, + const char *protocols) +{ + if (!protover_summary_map) + protover_summary_map = strmap_new(); + + if (strmap_size(protover_summary_map) >= MAX_PROTOVER_SUMMARY_MAP_LEN) { + protover_summary_cache_free_all(); + tor_assert(protover_summary_map == NULL); + protover_summary_map = strmap_new(); + } + + const protover_summary_flags_t *cached = + strmap_get(protover_summary_map, protocols); + + if (cached != NULL) { + /* We found a cached entry; no need to parse this one. */ + memcpy(out, cached, sizeof(protover_summary_flags_t)); + tor_assert(out->protocols_known); + return; + } + + memset(out, 0, sizeof(*out)); + out->protocols_known = 1; + out->supports_extend2_cells = + protocol_list_supports_protocol(protocols, PRT_RELAY, 2); + out->supports_ed25519_link_handshake_compat = + protocol_list_supports_protocol(protocols, PRT_LINKAUTH, 3); + out->supports_ed25519_link_handshake_any = + protocol_list_supports_protocol_or_later(protocols, PRT_LINKAUTH, 3); + out->supports_ed25519_hs_intro = + protocol_list_supports_protocol(protocols, PRT_HSINTRO, 4); + out->supports_v3_hsdir = + protocol_list_supports_protocol(protocols, PRT_HSDIR, + PROTOVER_HSDIR_V3); + out->supports_v3_rendezvous_point = + protocol_list_supports_protocol(protocols, PRT_HSREND, + PROTOVER_HS_RENDEZVOUS_POINT_V3); + + protover_summary_flags_t *new_cached = tor_memdup(out, sizeof(*out)); + cached = strmap_set(protover_summary_map, protocols, new_cached); + tor_assert(!cached); +} + /** Summarize the protocols listed in <b>protocols</b> into <b>out</b>, * falling back or correcting them based on <b>version</b> as appropriate. */ @@ -388,21 +465,7 @@ summarize_protover_flags(protover_summary_flags_t *out, tor_assert(out); memset(out, 0, sizeof(*out)); if (protocols) { - out->protocols_known = 1; - out->supports_extend2_cells = - protocol_list_supports_protocol(protocols, PRT_RELAY, 2); - out->supports_ed25519_link_handshake_compat = - protocol_list_supports_protocol(protocols, PRT_LINKAUTH, 3); - out->supports_ed25519_link_handshake_any = - protocol_list_supports_protocol_or_later(protocols, PRT_LINKAUTH, 3); - out->supports_ed25519_hs_intro = - protocol_list_supports_protocol(protocols, PRT_HSINTRO, 4); - out->supports_v3_hsdir = - protocol_list_supports_protocol(protocols, PRT_HSDIR, - PROTOVER_HSDIR_V3); - out->supports_v3_rendezvous_point = - protocol_list_supports_protocol(protocols, PRT_HSREND, - PROTOVER_HS_RENDEZVOUS_POINT_V3); + memoize_protover_summary(out, protocols); } if (version && !strcmpstart(version, "Tor ")) { if (!out->protocols_known) { @@ -420,3 +483,13 @@ summarize_protover_flags(protover_summary_flags_t *out, } } } + +/** + * Free all space held in the protover_summary_map. + */ +void +protover_summary_cache_free_all(void) +{ + strmap_free(protover_summary_map, tor_free_); + protover_summary_map = NULL; +} diff --git a/src/core/or/versions.h b/src/core/or/versions.h index 0c773f3f4c..acd8998918 100644 --- a/src/core/or/versions.h +++ b/src/core/or/versions.h @@ -26,6 +26,8 @@ typedef enum version_status_t { VS_UNKNOWN, /**< We have no idea. */ } version_status_t; +time_t tor_get_approx_release_date(void); + version_status_t tor_version_is_obsolete(const char *myversion, const char *versionlist); int tor_version_parse_platform(const char *platform, @@ -41,4 +43,6 @@ void summarize_protover_flags(protover_summary_flags_t *out, const char *protocols, const char *version); +void protover_summary_cache_free_all(void); + #endif /* !defined(TOR_VERSIONS_H) */ diff --git a/src/core/proto/proto_cell.c b/src/core/proto/proto_cell.c index 49b07663d7..70278cd488 100644 --- a/src/core/proto/proto_cell.c +++ b/src/core/proto/proto_cell.c @@ -5,7 +5,7 @@ /* See LICENSE for licensing information */ #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/proto/proto_cell.h" #include "core/or/connection_or.h" diff --git a/src/core/proto/proto_control0.c b/src/core/proto/proto_control0.c index d26c69753a..a770a061a7 100644 --- a/src/core/proto/proto_control0.c +++ b/src/core/proto/proto_control0.c @@ -5,7 +5,7 @@ /* See LICENSE for licensing information */ #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/proto/proto_control0.h" /** Return 1 iff buf looks more like it has an (obsolete) v0 controller diff --git a/src/core/proto/proto_ext_or.c b/src/core/proto/proto_ext_or.c index 04bec4e6b9..fe36f6b396 100644 --- a/src/core/proto/proto_ext_or.c +++ b/src/core/proto/proto_ext_or.c @@ -5,7 +5,7 @@ /* See LICENSE for licensing information */ #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "feature/relay/ext_orport.h" #include "core/proto/proto_ext_or.h" diff --git a/src/core/proto/proto_http.c b/src/core/proto/proto_http.c index 37298d1284..4ce9ba02f5 100644 --- a/src/core/proto/proto_http.c +++ b/src/core/proto/proto_http.c @@ -6,7 +6,7 @@ #define PROTO_HTTP_PRIVATE #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/proto/proto_http.h" /** Return true if <b>cmd</b> looks like a HTTP (proxy) request. */ diff --git a/src/core/proto/proto_socks.c b/src/core/proto/proto_socks.c index e23da7730b..4071f34f0d 100644 --- a/src/core/proto/proto_socks.c +++ b/src/core/proto/proto_socks.c @@ -6,7 +6,7 @@ #include "core/or/or.h" #include "feature/client/addressmap.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/mainloop/connection.h" #include "feature/control/control.h" #include "app/config/config.h" diff --git a/src/ext/.may_include b/src/ext/.may_include new file mode 100644 index 0000000000..1eafff2eeb --- /dev/null +++ b/src/ext/.may_include @@ -0,0 +1,10 @@ + +orconfig.h + +lib/err/*.h +lib/cc/*.h + +tinytest*.h +ext/siphash.h +ext/byteorder.h +ext/tor_readpassphrase.h
\ No newline at end of file diff --git a/src/ext/csiphash.c b/src/ext/csiphash.c index a6a9846db4..af8559a476 100644 --- a/src/ext/csiphash.c +++ b/src/ext/csiphash.c @@ -30,12 +30,12 @@ */ #include "lib/cc/torint.h" -#include "lib/log/util_bug.h" +#include "lib/err/torerr.h" -#include "siphash.h" +#include "ext/siphash.h" #include <string.h> #include <stdlib.h> -#include "byteorder.h" +#include "ext/byteorder.h" #define ROTATE(x, b) (uint64_t)( ((x) << (b)) | ( (x) >> (64 - (b))) ) @@ -112,13 +112,13 @@ static int the_siphash_key_is_set = 0; static struct sipkey the_siphash_key; uint64_t siphash24g(const void *src, unsigned long src_sz) { - tor_assert(the_siphash_key_is_set); + raw_assert(the_siphash_key_is_set); return siphash24(src, src_sz, &the_siphash_key); } void siphash_set_global_key(const struct sipkey *key) { - tor_assert(! the_siphash_key_is_set); + raw_assert(! the_siphash_key_is_set); the_siphash_key.k0 = key->k0; the_siphash_key.k1 = key->k1; the_siphash_key_is_set = 1; diff --git a/src/ext/readpassphrase.c b/src/ext/readpassphrase.c index e0df05d7b7..16611af1e2 100644 --- a/src/ext/readpassphrase.c +++ b/src/ext/readpassphrase.c @@ -30,7 +30,7 @@ #include <signal.h> #include <ctype.h> #include <fcntl.h> -#include "tor_readpassphrase.h" +#include "ext/tor_readpassphrase.h" #include <errno.h> #include <string.h> #include <unistd.h> diff --git a/src/ext/timeouts/.may_include b/src/ext/timeouts/.may_include new file mode 100644 index 0000000000..42f44befd4 --- /dev/null +++ b/src/ext/timeouts/.may_include @@ -0,0 +1,6 @@ +orconfig.h + +ext/tor_queue.h +timeout-bitops.c +timeout-debug.h +timeout.h diff --git a/src/ext/timeouts/timeout.c b/src/ext/timeouts/timeout.c index d4b514d2c5..07d06772c5 100644 --- a/src/ext/timeouts/timeout.c +++ b/src/ext/timeouts/timeout.c @@ -38,7 +38,7 @@ #include <errno.h> /* errno */ -#include "tor_queue.h" /* TAILQ(3) */ +#include "ext/tor_queue.h" /* TAILQ(3) */ #include "timeout.h" @@ -531,7 +531,7 @@ static timeout_t timeouts_int(struct timeouts *T) { timeout = MIN(_timeout, timeout); } - relmask <<= WHEEL_BIT; + relmask <<= WHEEL_BIT; relmask |= WHEEL_MASK; } @@ -751,4 +751,3 @@ TIMEOUT_PUBLIC int timeout_v_abi(void) { TIMEOUT_PUBLIC int timeout_v_api(void) { return TIMEOUT_V_API; } /* timeout_version() */ - diff --git a/src/ext/timeouts/timeout.h b/src/ext/timeouts/timeout.h index b35874e153..1ed309fd08 100644 --- a/src/ext/timeouts/timeout.h +++ b/src/ext/timeouts/timeout.h @@ -31,7 +31,7 @@ #include <inttypes.h> /* PRIu64 PRIx64 PRIX64 uint64_t */ -#include "tor_queue.h" /* TAILQ(3) */ +#include "ext/tor_queue.h" /* TAILQ(3) */ /* @@ -147,7 +147,7 @@ TIMEOUT_PUBLIC struct timeout *timeout_init(struct timeout *, int); #ifndef TIMEOUT_DISABLE_RELATIVE_ACCESS TIMEOUT_PUBLIC bool timeout_pending(struct timeout *); /* true if on timing wheel, false otherwise */ - + TIMEOUT_PUBLIC bool timeout_expired(struct timeout *); /* true if on expired queue, false otherwise */ diff --git a/src/feature/client/bridges.c b/src/feature/client/bridges.c index 47a4249556..5b537eb16b 100644 --- a/src/feature/client/bridges.c +++ b/src/feature/client/bridges.c @@ -844,7 +844,8 @@ rewrite_node_address_for_bridge(const bridge_info_t *bridge, node_t *node) } } - if (options->ClientPreferIPv6ORPort == -1) { + if (options->ClientPreferIPv6ORPort == -1 || + options->ClientAutoIPv6ORPort == 0) { /* Mark which address to use based on which bridge_t we got. */ node->ipv6_preferred = (tor_addr_family(&bridge->addr) == AF_INET6 && !tor_addr_is_null(&node->ri->ipv6_addr)); diff --git a/src/feature/client/dnsserv.c b/src/feature/client/dnsserv.c index ea4951f915..e5abe5c6aa 100644 --- a/src/feature/client/dnsserv.c +++ b/src/feature/client/dnsserv.c @@ -28,6 +28,7 @@ #include "core/or/connection_edge.h" #include "feature/control/control.h" #include "core/mainloop/mainloop.h" +#include "core/mainloop/netstatus.h" #include "core/or/policies.h" #include "feature/control/control_connection_st.h" @@ -213,6 +214,9 @@ dnsserv_launch_request(const char *name, int reverse, edge_connection_t *conn; char *q_name; + /* Launching a request for a user counts as user activity. */ + note_user_activity(approx_time()); + /* Make a new dummy AP connection, and attach the request to it. */ entry_conn = entry_connection_new(CONN_TYPE_AP, AF_INET); entry_conn->entry_cfg.dns_request = 1; diff --git a/src/feature/client/entrynodes.c b/src/feature/client/entrynodes.c index 20208ef613..e0fe1b9a58 100644 --- a/src/feature/client/entrynodes.c +++ b/src/feature/client/entrynodes.c @@ -287,7 +287,9 @@ create_initial_guard_context(void) guard_selection_type_t type = GS_TYPE_INFER; const char *name = choose_guard_selection( get_options(), - networkstatus_get_live_consensus(approx_time()), + networkstatus_get_reasonably_live_consensus( + approx_time(), + usable_consensus_flavor()), NULL, &type); tor_assert(name); // "name" can only be NULL if we had an old name. @@ -726,7 +728,9 @@ update_guard_selection_choice(const or_options_t *options) guard_selection_type_t type = GS_TYPE_INFER; const char *new_name = choose_guard_selection( options, - networkstatus_get_live_consensus(approx_time()), + networkstatus_get_reasonably_live_consensus( + approx_time(), + usable_consensus_flavor()), curr_guard_context, &type); tor_assert(new_name); @@ -1125,14 +1129,16 @@ select_and_add_guard_item_for_sample(guard_selection_t *gs, * or if we don't need a consensus because we're using bridges.) */ static int -live_consensus_is_missing(const guard_selection_t *gs) +reasonably_live_consensus_is_missing(const guard_selection_t *gs) { tor_assert(gs); if (gs->type == GS_TYPE_BRIDGE) { /* We don't update bridges from the consensus; they aren't there. */ return 0; } - return networkstatus_get_live_consensus(approx_time()) == NULL; + return networkstatus_get_reasonably_live_consensus( + approx_time(), + usable_consensus_flavor()) == NULL; } /** @@ -1147,9 +1153,9 @@ entry_guards_expand_sample(guard_selection_t *gs) tor_assert(gs); const or_options_t *options = get_options(); - if (live_consensus_is_missing(gs)) { + if (reasonably_live_consensus_is_missing(gs)) { log_info(LD_GUARD, "Not expanding the sample guard set; we have " - "no live consensus."); + "no reasonably live consensus."); return NULL; } @@ -1395,11 +1401,12 @@ sampled_guards_update_from_consensus(guard_selection_t *gs) { tor_assert(gs); - // It's important to use only a live consensus here; we don't want to - // make changes based on anything expired or old. - if (live_consensus_is_missing(gs)) { + // It's important to use a reasonably live consensus here; we want clients + // to bootstrap even if their clock is skewed by more than 2-3 hours. + // But we don't want to make changes based on anything that's really old. + if (reasonably_live_consensus_is_missing(gs)) { log_info(LD_GUARD, "Not updating the sample guard set; we have " - "no live consensus."); + "no reasonably live consensus."); return; } log_info(LD_GUARD, "Updating sampled guard status based on received " diff --git a/src/feature/client/transports.c b/src/feature/client/transports.c index 85d4d2e080..8a8bcd9f7f 100644 --- a/src/feature/client/transports.c +++ b/src/feature/client/transports.c @@ -101,11 +101,13 @@ #include "core/or/connection_or.h" #include "feature/relay/ext_orport.h" #include "feature/control/control.h" +#include "lib/encoding/confline.h" +#include "lib/encoding/kvline.h" +#include "lib/process/process.h" #include "lib/process/env.h" -#include "lib/process/subprocess.h" -static process_environment_t * +static smartlist_t * create_managed_proxy_environment(const managed_proxy_t *mp); static inline int proxy_configuration_finished(const managed_proxy_t *mp); @@ -127,6 +129,8 @@ static void parse_method_error(const char *line, int is_server_method); #define PROTO_SMETHODS_DONE "SMETHODS DONE" #define PROTO_PROXY_DONE "PROXY DONE" #define PROTO_PROXY_ERROR "PROXY-ERROR" +#define PROTO_LOG "LOG" +#define PROTO_STATUS "STATUS" /** The first and only supported - at the moment - configuration protocol version. */ @@ -490,8 +494,8 @@ proxy_prepare_for_restart(managed_proxy_t *mp) tor_assert(mp->conf_state == PT_PROTO_COMPLETED); /* destroy the process handle and terminate the process. */ - tor_process_handle_destroy(mp->process_handle, 1); - mp->process_handle = NULL; + process_set_data(mp->process, NULL); + process_terminate(mp->process); /* destroy all its registered transports, since we will no longer use them. */ @@ -520,34 +524,35 @@ proxy_prepare_for_restart(managed_proxy_t *mp) static int launch_managed_proxy(managed_proxy_t *mp) { - int retval; - - process_environment_t *env = create_managed_proxy_environment(mp); - -#ifdef _WIN32 - /* Passing NULL as lpApplicationName makes Windows search for the .exe */ - retval = tor_spawn_background(NULL, - (const char **)mp->argv, - env, - &mp->process_handle); -#else /* !(defined(_WIN32)) */ - retval = tor_spawn_background(mp->argv[0], - (const char **)mp->argv, - env, - &mp->process_handle); -#endif /* defined(_WIN32) */ - - process_environment_free(env); - - if (retval == PROCESS_STATUS_ERROR) { - log_warn(LD_GENERAL, "Managed proxy at '%s' failed at launch.", + tor_assert(mp); + + smartlist_t *env = create_managed_proxy_environment(mp); + + /* Configure our process. */ + process_set_data(mp->process, mp); + process_set_stdout_read_callback(mp->process, managed_proxy_stdout_callback); + process_set_stderr_read_callback(mp->process, managed_proxy_stderr_callback); + process_set_exit_callback(mp->process, managed_proxy_exit_callback); + process_set_protocol(mp->process, PROCESS_PROTOCOL_LINE); + process_reset_environment(mp->process, env); + + /* Cleanup our env. */ + SMARTLIST_FOREACH(env, char *, x, tor_free(x)); + smartlist_free(env); + + /* Skip the argv[0] as we get that from process_new(argv[0]). */ + for (int i = 1; mp->argv[i] != NULL; ++i) + process_append_argument(mp->process, mp->argv[i]); + + if (process_exec(mp->process) != PROCESS_STATUS_RUNNING) { + log_warn(LD_CONFIG, "Managed proxy at '%s' failed at launch.", mp->argv[0]); return -1; } - log_info(LD_CONFIG, "Managed proxy at '%s' has spawned with PID '%d'.", - mp->argv[0], tor_process_get_pid(mp->process_handle)); - + log_info(LD_CONFIG, + "Managed proxy at '%s' has spawned with PID '%" PRIu64 "'.", + mp->argv[0], process_get_pid(mp->process)); mp->conf_state = PT_PROTO_LAUNCHED; return 0; @@ -615,10 +620,6 @@ pt_configure_remaining_proxies(void) STATIC int configure_proxy(managed_proxy_t *mp) { - int configuration_finished = 0; - smartlist_t *proxy_output = NULL; - enum stream_status stream_status = 0; - /* if we haven't launched the proxy yet, do it now */ if (mp->conf_state == PT_PROTO_INFANT) { if (launch_managed_proxy(mp) < 0) { /* launch fail */ @@ -629,45 +630,8 @@ configure_proxy(managed_proxy_t *mp) } tor_assert(mp->conf_state != PT_PROTO_INFANT); - tor_assert(mp->process_handle); - - proxy_output = - tor_get_lines_from_handle(tor_process_get_stdout_pipe(mp->process_handle), - &stream_status); - if (!proxy_output) { /* failed to get input from proxy */ - if (stream_status != IO_STREAM_EAGAIN) { /* bad stream status! */ - mp->conf_state = PT_PROTO_BROKEN; - log_warn(LD_GENERAL, "The communication stream of managed proxy '%s' " - "is '%s'. Most probably the managed proxy stopped running. " - "This might be a bug of the managed proxy, a bug of Tor, or " - "a misconfiguration. Please enable logging on your managed " - "proxy and check the logs for errors.", - mp->argv[0], stream_status_to_string(stream_status)); - } - - goto done; - } - - /* Handle lines. */ - SMARTLIST_FOREACH_BEGIN(proxy_output, const char *, line) { - handle_proxy_line(line, mp); - if (proxy_configuration_finished(mp)) - goto done; - } SMARTLIST_FOREACH_END(line); - - done: - /* if the proxy finished configuring, exit the loop. */ - if (proxy_configuration_finished(mp)) { - handle_finished_proxy(mp); - configuration_finished = 1; - } - - if (proxy_output) { - SMARTLIST_FOREACH(proxy_output, char *, cp, tor_free(cp)); - smartlist_free(proxy_output); - } - - return configuration_finished; + tor_assert(mp->process); + return mp->conf_state == PT_PROTO_COMPLETED; } /** Register server managed proxy <b>mp</b> transports to state */ @@ -748,8 +712,11 @@ managed_proxy_destroy(managed_proxy_t *mp, /* free the outgoing proxy URI */ tor_free(mp->proxy_uri); - tor_process_handle_destroy(mp->process_handle, also_terminate_process); - mp->process_handle = NULL; + /* do we want to terminate our process if it's still running? */ + if (also_terminate_process && mp->process) + process_terminate(mp->process); + + process_free(mp->process); tor_free(mp); } @@ -945,21 +912,16 @@ handle_proxy_line(const char *line, managed_proxy_t *mp) parse_proxy_error(line); goto err; - } else if (!strcmpstart(line, SPAWN_ERROR_MESSAGE)) { - /* managed proxy launch failed: parse error message to learn why. */ - int retval, child_state, saved_errno; - retval = tor_sscanf(line, SPAWN_ERROR_MESSAGE "%x/%x", - &child_state, &saved_errno); - if (retval == 2) { - log_warn(LD_GENERAL, - "Could not launch managed proxy executable at '%s' ('%s').", - mp->argv[0], strerror(saved_errno)); - } else { /* failed to parse error message */ - log_warn(LD_GENERAL,"Could not launch managed proxy executable at '%s'.", - mp->argv[0]); - } - mp->conf_state = PT_PROTO_FAILED_LAUNCH; + /* We check for the additional " " after the PROTO_LOG * PROTO_STATUS + * string to make sure we can later extend this big if/else-if table with + * something that begins with "LOG" without having to get the order right. + * */ + } else if (!strcmpstart(line, PROTO_LOG " ")) { + parse_log_line(line, mp); + return; + } else if (!strcmpstart(line, PROTO_STATUS " ")) { + parse_status_line(line, mp); return; } @@ -1182,6 +1144,121 @@ parse_proxy_error(const char *line) line+strlen(PROTO_PROXY_ERROR)+1); } +/** Parses a LOG <b>line</b> and emit log events accordingly. */ +STATIC void +parse_log_line(const char *line, managed_proxy_t *mp) +{ + tor_assert(line); + tor_assert(mp); + + config_line_t *values = NULL; + char *log_message = NULL; + + if (strlen(line) < (strlen(PROTO_LOG) + 1)) { + log_warn(LD_PT, "Managed proxy sent us a %s line " + "with missing argument.", PROTO_LOG); + goto done; + } + + const char *data = line + strlen(PROTO_LOG) + 1; + values = kvline_parse(data, KV_QUOTED); + + if (! values) { + log_warn(LD_PT, "Managed proxy \"%s\" wrote an invalid LOG message: %s", + mp->argv[0], data); + goto done; + } + + const config_line_t *severity = config_line_find(values, "SEVERITY"); + const config_line_t *message = config_line_find(values, "MESSAGE"); + + /* Check if we got a message. */ + if (! message) { + log_warn(LD_PT, "Managed proxy \"%s\" wrote a LOG line without " + "MESSAGE: %s", mp->argv[0], escaped(data)); + goto done; + } + + /* Check if severity is there and whether it's valid. */ + if (! severity) { + log_warn(LD_PT, "Managed proxy \"%s\" wrote a LOG line without " + "SEVERITY: %s", mp->argv[0], escaped(data)); + goto done; + } + + int log_severity = managed_proxy_severity_parse(severity->value); + + if (log_severity == -1) { + log_warn(LD_PT, "Managed proxy \"%s\" wrote a LOG line with an " + "invalid severity level: %s", + mp->argv[0], severity->value); + goto done; + } + + tor_log(log_severity, LD_PT, "Managed proxy \"%s\": %s", + mp->argv[0], message->value); + + /* Prepend the PT name. */ + config_line_prepend(&values, "PT", mp->argv[0]); + log_message = kvline_encode(values, KV_QUOTED); + + /* Emit control port event. */ + control_event_pt_log(log_message); + + done: + config_free_lines(values); + tor_free(log_message); +} + +/** Parses a STATUS <b>line</b> and emit control events accordingly. */ +STATIC void +parse_status_line(const char *line, managed_proxy_t *mp) +{ + tor_assert(line); + tor_assert(mp); + + config_line_t *values = NULL; + char *status_message = NULL; + + if (strlen(line) < (strlen(PROTO_STATUS) + 1)) { + log_warn(LD_PT, "Managed proxy sent us a %s line " + "with missing argument.", PROTO_STATUS); + goto done; + } + + const char *data = line + strlen(PROTO_STATUS) + 1; + + values = kvline_parse(data, KV_QUOTED); + + if (! values) { + log_warn(LD_PT, "Managed proxy \"%s\" wrote an invalid " + "STATUS message: %s", mp->argv[0], escaped(data)); + goto done; + } + + /* We check if we received the TYPE parameter, which is the only *required* + * value. */ + const config_line_t *type = config_line_find(values, "TYPE"); + + if (! type) { + log_warn(LD_PT, "Managed proxy \"%s\" wrote a STATUS line without " + "TYPE: %s", mp->argv[0], escaped(data)); + goto done; + } + + /* Prepend the PT name. */ + config_line_prepend(&values, "PT", mp->argv[0]); + status_message = kvline_encode(values, KV_QUOTED); + + /* We have checked that TYPE is there, we can now emit the STATUS event via + * the control port. */ + control_event_pt_status(status_message); + + done: + config_free_lines(values); + tor_free(status_message); +} + /** Return a newly allocated string that tor should place in * TOR_PT_SERVER_TRANSPORT_OPTIONS while configuring the server * manged proxy in <b>mp</b>. Return NULL if no such options are found. */ @@ -1257,7 +1334,7 @@ get_bindaddr_for_server_proxy(const managed_proxy_t *mp) /** Return a newly allocated process_environment_t * for <b>mp</b>'s * process. */ -static process_environment_t * +static smartlist_t * create_managed_proxy_environment(const managed_proxy_t *mp) { const or_options_t *options = get_options(); @@ -1272,8 +1349,6 @@ create_managed_proxy_environment(const managed_proxy_t *mp) /* The final environment to be passed to mp. */ smartlist_t *merged_env_vars = get_current_process_environment_variables(); - process_environment_t *env; - { char *state_tmp = get_datadir_fname("pt_state/"); /* XXX temp */ smartlist_add_asprintf(envs, "TOR_PT_STATE_LOCATION=%s", state_tmp); @@ -1366,14 +1441,9 @@ create_managed_proxy_environment(const managed_proxy_t *mp) tor_free_, 1); } SMARTLIST_FOREACH_END(env_var); - env = process_environment_make(merged_env_vars); - smartlist_free(envs); - SMARTLIST_FOREACH(merged_env_vars, void *, x, tor_free(x)); - smartlist_free(merged_env_vars); - - return env; + return merged_env_vars; } /** Create and return a new managed proxy for <b>transport</b> using @@ -1392,6 +1462,7 @@ managed_proxy_create(const smartlist_t *with_transport_list, mp->argv = proxy_argv; mp->transports = smartlist_new(); mp->proxy_uri = get_pt_proxy_uri(); + mp->process = process_new(proxy_argv[0]); mp->transports_to_launch = smartlist_new(); SMARTLIST_FOREACH(with_transport_list, const char *, transport, @@ -1736,3 +1807,99 @@ tor_escape_str_for_pt_args(const char *string, const char *chars_to_escape) return new_string; } + +/** Callback function that is called when our PT process have data on its + * stdout. Our process can be found in <b>process</b>, the data can be found in + * <b>line</b> and the length of our line is given in <b>size</b>. */ +STATIC void +managed_proxy_stdout_callback(process_t *process, + const char *line, + size_t size) +{ + tor_assert(process); + tor_assert(line); + + (void)size; + + managed_proxy_t *mp = process_get_data(process); + + handle_proxy_line(line, mp); + + if (proxy_configuration_finished(mp)) { + handle_finished_proxy(mp); + tor_assert(mp->conf_state == PT_PROTO_COMPLETED); + } +} + +/** Callback function that is called when our PT process have data on its + * stderr. Our process can be found in <b>process</b>, the data can be found in + * <b>line</b> and the length of our line is given in <b>size</b>. */ +STATIC void +managed_proxy_stderr_callback(process_t *process, + const char *line, + size_t size) +{ + tor_assert(process); + tor_assert(line); + + (void)size; + + managed_proxy_t *mp = process_get_data(process); + + log_warn(LD_PT, "Managed proxy at '%s' reported: %s", mp->argv[0], line); +} + +/** Callback function that is called when our PT process terminates. The + * process exit code can be found in <b>exit_code</b> and our process can be + * found in <b>process</b>. Returns true iff we want the process subsystem to + * free our process_t handle for us. */ +STATIC bool +managed_proxy_exit_callback(process_t *process, process_exit_code_t exit_code) +{ + tor_assert(process); + + log_warn(LD_PT, + "Pluggable Transport process terminated with status code %" PRIu64, + exit_code); + + /* We detach ourself from the MP (if we are attached) and free ourself. */ + managed_proxy_t *mp = process_get_data(process); + + /* If we are still attached to the process, it is probably because our PT + * process crashed before we got to call process_set_data(p, NULL); */ + if (BUG(mp != NULL)) { + /* FIXME(ahf): Our process stopped without us having told it to stop + * (crashed). Should we restart it here? */ + mp->process = NULL; + process_set_data(process, NULL); + } + + return true; +} + +/** Returns a valid integer log severity level from <b>severity</b> that + * is compatible with Tor's logging functions. Returns <b>-1</b> on + * error. */ +STATIC int +managed_proxy_severity_parse(const char *severity) +{ + tor_assert(severity); + + /* Slightly different than log.c's parse_log_level :-( */ + if (! strcmp(severity, "debug")) + return LOG_DEBUG; + + if (! strcmp(severity, "info")) + return LOG_INFO; + + if (! strcmp(severity, "notice")) + return LOG_NOTICE; + + if (! strcmp(severity, "warning")) + return LOG_WARN; + + if (! strcmp(severity, "error")) + return LOG_ERR; + + return -1; +} diff --git a/src/feature/client/transports.h b/src/feature/client/transports.h index d304dcd485..1a910ae82c 100644 --- a/src/feature/client/transports.h +++ b/src/feature/client/transports.h @@ -11,6 +11,8 @@ #ifndef TOR_TRANSPORTS_H #define TOR_TRANSPORTS_H +#include "lib/process/process.h" + /** Represents a pluggable transport used by a bridge. */ typedef struct transport_t { /** SOCKS version: One of PROXY_SOCKS4, PROXY_SOCKS5. */ @@ -81,7 +83,7 @@ enum pt_proto_state { PT_PROTO_FAILED_LAUNCH /* failed while launching */ }; -struct process_handle_t; +struct process_t; /** Structure containing information of a managed proxy. */ typedef struct { @@ -94,10 +96,8 @@ typedef struct { int is_server; /* is it a server proxy? */ - /* A pointer to the process handle of this managed proxy. */ - struct process_handle_t *process_handle; - - int pid; /* The Process ID this managed proxy is using. */ + /* A pointer to the process of this managed proxy. */ + struct process_t *process; /** Boolean: We are re-parsing our config, and we are going to * remove this managed proxy if we don't find it any transport @@ -128,6 +128,8 @@ STATIC int parse_version(const char *line, managed_proxy_t *mp); STATIC void parse_env_error(const char *line); STATIC void parse_proxy_error(const char *line); STATIC void handle_proxy_line(const char *line, managed_proxy_t *mp); +STATIC void parse_log_line(const char *line, managed_proxy_t *mp); +STATIC void parse_status_line(const char *line, managed_proxy_t *mp); STATIC char *get_transport_options_for_server_proxy(const managed_proxy_t *mp); STATIC void managed_proxy_destroy(managed_proxy_t *mp, @@ -142,6 +144,12 @@ STATIC char* get_pt_proxy_uri(void); STATIC void free_execve_args(char **arg); +STATIC void managed_proxy_stdout_callback(process_t *, const char *, size_t); +STATIC void managed_proxy_stderr_callback(process_t *, const char *, size_t); +STATIC bool managed_proxy_exit_callback(process_t *, process_exit_code_t); + +STATIC int managed_proxy_severity_parse(const char *); + #endif /* defined(PT_PRIVATE) */ #endif /* !defined(TOR_TRANSPORTS_H) */ diff --git a/src/feature/control/btrack.c b/src/feature/control/btrack.c new file mode 100644 index 0000000000..14220faad1 --- /dev/null +++ b/src/feature/control/btrack.c @@ -0,0 +1,53 @@ +/* Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file btrack.c + * \brief Bootstrap trackers + * + * Initializes and shuts down the specific bootstrap trackers. These + * trackers help the reporting of bootstrap progress by maintaining + * state information about various subsystems within tor. When the + * correct state changes happen, these trackers emit controller + * events. + * + * These trackers avoid referring directly to the internals of state + * objects of other subsystems. + * + * btrack_circuit.c contains the tracker for origin circuits. + * + * btrack_orconn.c contains the tracker for OR connections. + * + * Eventually there will be a tracker for directory downloads as well. + **/ + +#include "feature/control/btrack_circuit.h" +#include "feature/control/btrack_orconn.h" +#include "feature/control/btrack_sys.h" +#include "lib/subsys/subsys.h" + +static int +btrack_init(void) +{ + if (btrack_orconn_init()) + return -1; + if (btrack_circ_init()) + return -1; + + return 0; +} + +static void +btrack_fini(void) +{ + btrack_orconn_fini(); + btrack_circ_fini(); +} + +const subsys_fns_t sys_btrack = { + .name = "btrack", + .supported = true, + .level = -30, + .initialize = btrack_init, + .shutdown = btrack_fini, +}; diff --git a/src/feature/control/btrack_circuit.c b/src/feature/control/btrack_circuit.c new file mode 100644 index 0000000000..bf09e0b99c --- /dev/null +++ b/src/feature/control/btrack_circuit.c @@ -0,0 +1,164 @@ +/* Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file btrack_circuit.c + * \brief Bootstrap tracker for origin circuits + * + * Track state changes of origin circuits, as published by the circuit + * subsystem. + **/ + +#include "core/or/or.h" + +#include "core/or/ocirc_event.h" + +#include "feature/control/btrack_circuit.h" +#include "feature/control/control.h" +#include "lib/log/log.h" + +/** Pair of a best origin circuit GID with its state or status */ +typedef struct btc_best_t { + uint32_t gid; + int val; +} btc_best_t; + +/** GID and state of the best origin circuit we've seen so far */ +static btc_best_t best_any_state = { 0, -1 }; +/** GID and state of the best application circuit we've seen so far */ +static btc_best_t best_ap_state = { 0, -1 }; +/** GID and status of the best origin circuit we've seen so far */ +static btc_best_t best_any_evtype = { 0, -1 }; +/** GID and status of the best application circuit we've seen so far */ +static btc_best_t best_ap_evtype = { 0, -1 }; + +/** Reset cached "best" values */ +static void +btc_reset_bests(void) +{ + best_any_state.gid = best_ap_state.gid = 0; + best_any_state.val = best_ap_state.val = -1; + best_any_evtype.gid = best_ap_state.gid = 0; + best_any_evtype.val = best_ap_evtype.val = -1; +} + +/** True if @a state is a "better" origin circuit state than @a best->val */ +static bool +btc_state_better(int state, const btc_best_t *best) +{ + return state > best->val; +} + +/** + * Definine an ordering on circuit status events + * + * The CIRC_EVENT_ constants aren't sorted in a useful order, so this + * array helps to decode them. This approach depends on the statuses + * being nonnegative and dense. + **/ +static int circ_event_order[] = { + [CIRC_EVENT_FAILED] = -1, + [CIRC_EVENT_CLOSED] = -1, + [CIRC_EVENT_LAUNCHED] = 1, + [CIRC_EVENT_EXTENDED] = 2, + [CIRC_EVENT_BUILT] = 3, +}; +#define N_CIRC_EVENT_ORDER \ + (sizeof(circ_event_order) / sizeof(circ_event_order[0])) + +/** True if @a state is a "better" origin circuit event status than @a + best->val */ +static bool +btc_evtype_better(int state, const btc_best_t *best) +{ + if (state < 0) + return false; + if (best->val < 0) + return true; + + tor_assert(state >= 0 && (unsigned)state < N_CIRC_EVENT_ORDER); + tor_assert(best->val >= 0 && (unsigned)best->val < N_CIRC_EVENT_ORDER); + return circ_event_order[state] > circ_event_order[best->val]; +} + +static bool +btc_update_state(const ocirc_state_msg_t *msg, btc_best_t *best, + const char *type) +{ + if (btc_state_better(msg->state, best)) { + log_info(LD_BTRACK, "CIRC BEST_%s state %d->%d gid=%"PRIu32, type, + best->val, msg->state, msg->gid); + best->gid = msg->gid; + best->val = msg->state; + return true; + } + return false; +} + +static bool +btc_update_evtype(const ocirc_cevent_msg_t *msg, btc_best_t *best, + const char *type) +{ + if (btc_evtype_better(msg->evtype, best)) { + log_info(LD_BTRACK, "CIRC BEST_%s evtype %d->%d gid=%"PRIu32, type, + best->val, msg->evtype, msg->gid); + best->gid = msg->gid; + best->val = msg->evtype; + return true; + } + return false; +} + +static void +btc_state_rcvr(const ocirc_state_msg_t *msg) +{ + log_debug(LD_BTRACK, "CIRC gid=%"PRIu32" state=%d onehop=%d", + msg->gid, msg->state, msg->onehop); + + btc_update_state(msg, &best_any_state, "ANY"); + if (msg->onehop) + return; + btc_update_state(msg, &best_ap_state, "AP"); +} + +static void +btc_cevent_rcvr(const ocirc_cevent_msg_t *msg) +{ + log_debug(LD_BTRACK, "CIRC gid=%"PRIu32" evtype=%d reason=%d onehop=%d", + msg->gid, msg->evtype, msg->reason, msg->onehop); + + btc_update_evtype(msg, &best_any_evtype, "ANY"); + if (msg->onehop) + return; + btc_update_evtype(msg, &best_ap_evtype, "AP"); +} + +static void +btc_event_rcvr(const ocirc_event_msg_t *msg) +{ + switch (msg->type) { + case OCIRC_MSGTYPE_STATE: + return btc_state_rcvr(&msg->u.state); + case OCIRC_MSGTYPE_CHAN: + log_debug(LD_BTRACK, "CIRC gid=%"PRIu32" chan=%"PRIu64" onehop=%d", + msg->u.chan.gid, msg->u.chan.chan, msg->u.chan.onehop); + break; + case OCIRC_MSGTYPE_CEVENT: + return btc_cevent_rcvr(&msg->u.cevent); + default: + break; + } +} + +int +btrack_circ_init(void) +{ + ocirc_event_subscribe(btc_event_rcvr); + return 0; +} + +void +btrack_circ_fini(void) +{ + btc_reset_bests(); +} diff --git a/src/feature/control/btrack_circuit.h b/src/feature/control/btrack_circuit.h new file mode 100644 index 0000000000..ab8b8b652c --- /dev/null +++ b/src/feature/control/btrack_circuit.h @@ -0,0 +1,15 @@ +/* Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file btrack_circuit.h + * \brief Header file for btrack_circuit.c + **/ + +#ifndef TOR_BTRACK_CIRCUIT_H +#define TOR_BTRACK_CIRCUIT_H + +int btrack_circ_init(void); +void btrack_circ_fini(void); + +#endif /* defined(TOR_BTRACK_CIRCUIT_H) */ diff --git a/src/feature/control/btrack_orconn.c b/src/feature/control/btrack_orconn.c new file mode 100644 index 0000000000..0fbf521000 --- /dev/null +++ b/src/feature/control/btrack_orconn.c @@ -0,0 +1,206 @@ +/* Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file btrack_orconn.c + * \brief Bootstrap tracker for OR connections + * + * Track state changes of OR connections, as published by the + * connection subsystem. Also track circuit launch events, because + * they're one of the few ways to discover the association between a + * channel (and OR connection) and a circuit. + * + * We track all OR connections that we receive events for, whether or + * not they're carrying origin circuits. (An OR connection might + * carry origin circuits only after we first find out about that + * connection.) + * + * All origin ORCONN events update the "any" state variables, while + * only application ORCONN events update the "ap" state variables (and + * also update the "any") variables. + * + * We do this because we want to report the first increments of + * connection progress as the earliest bootstrap phases. This results + * in a better user experience because failures here translate into + * zero or very small amounts of displayed progress, instead of + * progress stuck near completion. The first connection to a relay + * might be a one-hop circuit for directory lookups, or it might be a + * connection for an application circuit because we already have + * enough directory info to build an application circuit. + * + * We call functions in btrack_orconn_cevent.c to generate the actual + * controller events, because some of the state decoding we need to do + * is complicated. + **/ + +#include <stdbool.h> + +#include "core/or/or.h" + +#define BTRACK_ORCONN_PRIVATE + +#include "core/or/ocirc_event.h" +#include "core/or/orconn_event.h" +#include "feature/control/btrack_orconn.h" +#include "feature/control/btrack_orconn_cevent.h" +#include "feature/control/btrack_orconn_maps.h" +#include "lib/log/log.h" + +/** Pair of a best ORCONN GID and with its state */ +typedef struct bto_best_t { + uint64_t gid; + int state; +} bto_best_t; + +/** GID and state of the best ORCONN we've seen so far */ +static bto_best_t best_any = { 0, -1 }; +/** GID and state of the best application circuit ORCONN we've seen so far */ +static bto_best_t best_ap = { 0, -1 }; + +/** + * Update a cached state of a best ORCONN progress we've seen so far. + * + * Return true if the new state is better than the old. + **/ +static bool +bto_update_best(const bt_orconn_t *bto, bto_best_t *best, const char *type) +{ + if (bto->state < best->state) + return false; + /* Update even if we won't change best->state, because it's more + * recent information that a particular connection transitioned to + * that state. */ + best->gid = bto->gid; + if (bto->state > best->state) { + log_info(LD_BTRACK, "ORCONN BEST_%s state %d->%d gid=%"PRIu64, type, + best->state, bto->state, bto->gid); + best->state = bto->state; + return true; + } + return false; +} + +/** + * Update cached states of best ORCONN progress we've seen + * + * Only update the application ORCONN state if we know it's carrying + * an application circuit. + **/ +static void +bto_update_bests(const bt_orconn_t *bto) +{ + tor_assert(bto->is_orig); + + if (bto_update_best(bto, &best_any, "ANY")) + bto_cevent_anyconn(bto); + if (!bto->is_onehop && bto_update_best(bto, &best_ap, "AP")) + bto_cevent_apconn(bto); +} + +/** Reset cached "best" values */ +static void +bto_reset_bests(void) +{ + best_any.gid = best_ap.gid = 0; + best_any.state = best_ap.state = -1; +} + +/** + * Update cached states of ORCONNs from the incoming message. This + * message comes from code in connection_or.c. + **/ +static void +bto_state_rcvr(const orconn_state_msg_t *msg) +{ + bt_orconn_t *bto; + + bto = bto_find_or_new(msg->gid, msg->chan); + log_debug(LD_BTRACK, "ORCONN gid=%"PRIu64" chan=%"PRIu64 + " proxy_type=%d state=%d", + msg->gid, msg->chan, msg->proxy_type, msg->state); + bto->proxy_type = msg->proxy_type; + bto->state = msg->state; + if (bto->is_orig) + bto_update_bests(bto); +} + +/** + * Delete a cached ORCONN state if we get an incoming message saying + * the ORCONN is failed or closed. This message comes from code in + * control.c. + **/ +static void +bto_status_rcvr(const orconn_status_msg_t *msg) +{ + switch (msg->status) { + case OR_CONN_EVENT_FAILED: + case OR_CONN_EVENT_CLOSED: + log_info(LD_BTRACK, "ORCONN DELETE gid=%"PRIu64" status=%d reason=%d", + msg->gid, msg->status, msg->reason); + return bto_delete(msg->gid); + default: + break; + } +} + +/** Dispatch to individual ORCONN message handlers */ +static void +bto_event_rcvr(const orconn_event_msg_t *msg) +{ + switch (msg->type) { + case ORCONN_MSGTYPE_STATE: + return bto_state_rcvr(&msg->u.state); + case ORCONN_MSGTYPE_STATUS: + return bto_status_rcvr(&msg->u.status); + default: + tor_assert(false); + } +} + +/** + * Create or update a cached ORCONN state for a newly launched + * connection, including whether it's launched by an origin circuit + * and whether it's a one-hop circuit. + **/ +static void +bto_chan_rcvr(const ocirc_event_msg_t *msg) +{ + bt_orconn_t *bto; + + /* Ignore other kinds of origin circuit events; we don't need them */ + if (msg->type != OCIRC_MSGTYPE_CHAN) + return; + + bto = bto_find_or_new(0, msg->u.chan.chan); + if (!bto->is_orig || (bto->is_onehop && !msg->u.chan.onehop)) { + log_debug(LD_BTRACK, "ORCONN LAUNCH chan=%"PRIu64" onehop=%d", + msg->u.chan.chan, msg->u.chan.onehop); + } + bto->is_orig = true; + if (!msg->u.chan.onehop) + bto->is_onehop = false; + bto_update_bests(bto); +} + +/** + * Initialize the hash maps and subscribe to ORCONN and origin + * circuit events. + **/ +int +btrack_orconn_init(void) +{ + bto_init_maps(); + orconn_event_subscribe(bto_event_rcvr); + ocirc_event_subscribe(bto_chan_rcvr); + + return 0; +} + +/** Clear the hash maps and reset the "best" states */ +void +btrack_orconn_fini(void) +{ + bto_clear_maps(); + bto_reset_bests(); + bto_cevent_reset(); +} diff --git a/src/feature/control/btrack_orconn.h b/src/feature/control/btrack_orconn.h new file mode 100644 index 0000000000..4e514d4b04 --- /dev/null +++ b/src/feature/control/btrack_orconn.h @@ -0,0 +1,38 @@ +/* Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file btrack_orconn.h + * \brief Header file for btrack_orconn.c + **/ + +#ifndef TOR_BTRACK_ORCONN_H +#define TOR_BTRACK_ORCONN_H + +#ifdef BTRACK_ORCONN_PRIVATE + +#include "ht.h" + +/** + * Structure for tracking OR connection states + * + * This gets linked into two hash maps: one with connection IDs, and + * another with channel IDs. + **/ +typedef struct bt_orconn_t { + HT_ENTRY(bt_orconn_t) node; /**< Hash map entry indexed by gid */ + HT_ENTRY(bt_orconn_t) chan_node; /**< Hash map entry indexed by channel ID */ + uint64_t gid; /**< Global ID of this ORCONN */ + uint64_t chan; /**< Channel ID, if known */ + int proxy_type; /**< Proxy type */ + uint8_t state; /**< State of this ORCONN */ + bool is_orig; /**< Does this carry an origin circuit? */ + bool is_onehop; /**< Is this for a one-hop circuit? */ +} bt_orconn_t; + +#endif /* defined(BTRACK_ORCONN_PRIVATE) */ + +int btrack_orconn_init(void); +void btrack_orconn_fini(void); + +#endif /* defined(TOR_BTRACK_ORCONN_H) */ diff --git a/src/feature/control/btrack_orconn_cevent.c b/src/feature/control/btrack_orconn_cevent.c new file mode 100644 index 0000000000..c7970dca4d --- /dev/null +++ b/src/feature/control/btrack_orconn_cevent.c @@ -0,0 +1,159 @@ +/* Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file btrack_orconn_cevent.c + * \brief Emit bootstrap status events for OR connections + * + * We do some decoding of the raw OR_CONN_STATE_* values. For + * example, OR_CONN_STATE_CONNECTING means the first TCP connect() + * completing, regardless of whether it's directly to a relay instead + * of a proxy or a PT. + **/ + +#include <stdbool.h> + +#include "core/or/or.h" + +#define BTRACK_ORCONN_PRIVATE + +#include "core/or/orconn_event.h" +#include "feature/control/btrack_orconn.h" +#include "feature/control/btrack_orconn_cevent.h" +#include "feature/control/control.h" + +/** + * Have we completed our first OR connection? + * + * Block display of application circuit progress until we do, to avoid + * some misleading behavior of jumping to high progress. + **/ +static bool bto_first_orconn = false; + +/** Is the ORCONN using a pluggable transport? */ +static bool +using_pt(const bt_orconn_t *bto) +{ + return bto->proxy_type == PROXY_PLUGGABLE; +} + +/** Is the ORCONN using a non-PT proxy? */ +static bool +using_proxy(const bt_orconn_t *bto) +{ + switch (bto->proxy_type) { + case PROXY_CONNECT: + case PROXY_SOCKS4: + case PROXY_SOCKS5: + return true; + default: + return false; + } +} + +/** + * Emit control events when we have updated our idea of the best state + * that any OR connection has reached. + * + * Do some decoding of the ORCONN states depending on whether a PT or + * a proxy is in use. + **/ +void +bto_cevent_anyconn(const bt_orconn_t *bto) +{ + switch (bto->state) { + case OR_CONN_STATE_CONNECTING: + /* Exactly what kind of thing we're connecting to isn't + * information we directly get from the states in connection_or.c, + * so decode it here. */ + if (using_pt(bto)) + control_event_bootstrap(BOOTSTRAP_STATUS_CONN_PT, 0); + else if (using_proxy(bto)) + control_event_bootstrap(BOOTSTRAP_STATUS_CONN_PROXY, 0); + else + control_event_bootstrap(BOOTSTRAP_STATUS_CONN, 0); + break; + case OR_CONN_STATE_PROXY_HANDSHAKING: + /* Similarly, starting a proxy handshake means the TCP connect() + * succeeded to the proxy. Let's be specific about what kind of + * proxy. */ + if (using_pt(bto)) + control_event_bootstrap(BOOTSTRAP_STATUS_CONN_DONE_PT, 0); + else if (using_proxy(bto)) + control_event_bootstrap(BOOTSTRAP_STATUS_CONN_DONE_PROXY, 0); + break; + case OR_CONN_STATE_TLS_HANDSHAKING: + control_event_bootstrap(BOOTSTRAP_STATUS_CONN_DONE, 0); + break; + case OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING: + case OR_CONN_STATE_OR_HANDSHAKING_V2: + case OR_CONN_STATE_OR_HANDSHAKING_V3: + control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE, 0); + break; + case OR_CONN_STATE_OPEN: + control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE_DONE, 0); + /* Unblock directory progress display */ + control_event_boot_first_orconn(); + /* Unblock apconn progress display */ + bto_first_orconn = true; + break; + default: + break; + } +} + +/** + * Emit control events when we have updated our idea of the best state + * that any application circuit OR connection has reached. + * + * Do some decoding of the ORCONN states depending on whether a PT or + * a proxy is in use. + **/ +void +bto_cevent_apconn(const bt_orconn_t *bto) +{ + if (!bto_first_orconn) + return; + + switch (bto->state) { + case OR_CONN_STATE_CONNECTING: + /* Exactly what kind of thing we're connecting to isn't + * information we directly get from the states in connection_or.c, + * so decode it here. */ + if (using_pt(bto)) + control_event_bootstrap(BOOTSTRAP_STATUS_AP_CONN_PT, 0); + else if (using_proxy(bto)) + control_event_bootstrap(BOOTSTRAP_STATUS_AP_CONN_PROXY, 0); + else + control_event_bootstrap(BOOTSTRAP_STATUS_AP_CONN, 0); + break; + case OR_CONN_STATE_PROXY_HANDSHAKING: + /* Similarly, starting a proxy handshake means the TCP connect() + * succeeded to the proxy. Let's be specific about what kind of + * proxy. */ + if (using_pt(bto)) + control_event_bootstrap(BOOTSTRAP_STATUS_AP_CONN_DONE_PT, 0); + else if (using_proxy(bto)) + control_event_bootstrap(BOOTSTRAP_STATUS_AP_CONN_DONE_PROXY, 0); + break; + case OR_CONN_STATE_TLS_HANDSHAKING: + control_event_bootstrap(BOOTSTRAP_STATUS_AP_CONN_DONE, 0); + break; + case OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING: + case OR_CONN_STATE_OR_HANDSHAKING_V2: + case OR_CONN_STATE_OR_HANDSHAKING_V3: + control_event_bootstrap(BOOTSTRAP_STATUS_AP_HANDSHAKE, 0); + break; + case OR_CONN_STATE_OPEN: + control_event_bootstrap(BOOTSTRAP_STATUS_AP_HANDSHAKE_DONE, 0); + default: + break; + } +} + +/** Forget that we completed our first OR connection */ +void +bto_cevent_reset(void) +{ + bto_first_orconn = false; +} diff --git a/src/feature/control/btrack_orconn_cevent.h b/src/feature/control/btrack_orconn_cevent.h new file mode 100644 index 0000000000..165ff69cdb --- /dev/null +++ b/src/feature/control/btrack_orconn_cevent.h @@ -0,0 +1,17 @@ +/* Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file btrack_orconn_cevent.h + * \brief Header file for btrack_orconn_cevent.c + **/ + +#ifndef TOR_BTRACK_ORCONN_CEVENT_H + +#include "feature/control/btrack_orconn.h" + +void bto_cevent_anyconn(const bt_orconn_t *); +void bto_cevent_apconn(const bt_orconn_t *); +void bto_cevent_reset(void); + +#endif /* defined(TOR_BTRACK_ORCONN_CEVENT_H) */ diff --git a/src/feature/control/btrack_orconn_maps.c b/src/feature/control/btrack_orconn_maps.c new file mode 100644 index 0000000000..b6bb23804c --- /dev/null +++ b/src/feature/control/btrack_orconn_maps.c @@ -0,0 +1,223 @@ +/* Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file btrack_orconn_maps.c + * \brief Hash map implementation for btrack_orconn.c + * + * These functions manipulate the hash maps that contain bt_orconn + * objects. + **/ + +#include <stdbool.h> + +#include "core/or/or.h" + +#include "ht.h" +#include "siphash.h" + +#define BTRACK_ORCONN_PRIVATE + +#include "feature/control/btrack_orconn.h" +#include "feature/control/btrack_orconn_maps.h" +#include "lib/log/log.h" + +static inline unsigned int +bto_gid_hash_(bt_orconn_t *elm) +{ + return (unsigned)siphash24g(&elm->gid, sizeof(elm->gid)); +} + +static inline int +bto_gid_eq_(bt_orconn_t *a, bt_orconn_t *b) +{ + return a->gid == b->gid; +} + +static inline unsigned int +bto_chan_hash_(bt_orconn_t *elm) +{ + return (unsigned)siphash24g(&elm->chan, sizeof(elm->chan)); +} + +static inline int +bto_chan_eq_(bt_orconn_t *a, bt_orconn_t *b) +{ + return a->chan == b->chan; +} + +HT_HEAD(bto_gid_ht, bt_orconn_t); +HT_PROTOTYPE(bto_gid_ht, bt_orconn_t, node, bto_gid_hash_, bto_gid_eq_) +HT_GENERATE2(bto_gid_ht, bt_orconn_t, node, + bto_gid_hash_, bto_gid_eq_, 0.6, + tor_reallocarray_, tor_free_) +static struct bto_gid_ht *bto_gid_map; + +HT_HEAD(bto_chan_ht, bt_orconn_t); +HT_PROTOTYPE(bto_chan_ht, bt_orconn_t, chan_node, bto_chan_hash_, bto_chan_eq_) +HT_GENERATE2(bto_chan_ht, bt_orconn_t, chan_node, + bto_chan_hash_, bto_chan_eq_, 0.6, + tor_reallocarray_, tor_free_) +static struct bto_chan_ht *bto_chan_map; + +/** Clear the GID hash map, freeing any bt_orconn_t objects that become + * unreferenced */ +static void +bto_gid_clear_map(void) +{ + bt_orconn_t **elt, **next, *c; + + for (elt = HT_START(bto_gid_ht, bto_gid_map); + elt; + elt = next) { + c = *elt; + next = HT_NEXT_RMV(bto_gid_ht, bto_gid_map, elt); + + c->gid = 0; + /* Don't delete if chan ID isn't zero: it's still in the chan hash map */ + if (!c->chan) + tor_free(c); + } + HT_CLEAR(bto_gid_ht, bto_gid_map); + tor_free(bto_gid_map); +} + +/** Clear the chan ID hash map, freeing any bt_orconn_t objects that + * become unreferenced */ +static void +bto_chan_clear_map(void) +{ + bt_orconn_t **elt, **next, *c; + + for (elt = HT_START(bto_chan_ht, bto_chan_map); + elt; + elt = next) { + c = *elt; + next = HT_NEXT_RMV(bto_chan_ht, bto_chan_map, elt); + + c->chan = 0; + /* Don't delete if GID isn't zero, it's still in the GID hash map */ + if (!c->gid) + tor_free(c); + } + HT_CLEAR(bto_chan_ht, bto_chan_map); + tor_free(bto_chan_map); +} + +/** Delete a bt_orconn from the hash maps by GID */ +void +bto_delete(uint64_t gid) +{ + bt_orconn_t key, *bto; + + key.gid = gid; + key.chan = 0; + bto = HT_FIND(bto_gid_ht, bto_gid_map, &key); + if (!bto) { + /* The orconn might be unregistered because it's an EXT_OR_CONN? */ + log_debug(LD_BTRACK, "tried to delete unregistered ORCONN gid=%"PRIu64, + gid); + return; + } + HT_REMOVE(bto_gid_ht, bto_gid_map, &key); + if (bto->chan) { + key.chan = bto->chan; + HT_REMOVE(bto_chan_ht, bto_chan_map, &key); + } + tor_free(bto); +} + +/** + * Helper for bto_find_or_new(). + * + * Update GID and chan ID of an existing bt_orconn object if needed, + * given a search key previously used within bto_find_or_new(). + **/ +static bt_orconn_t * +bto_update(bt_orconn_t *bto, const bt_orconn_t *key) +{ + /* ORCONN GIDs shouldn't change once assigned */ + tor_assert(!bto->gid || !key->gid || bto->gid == key->gid); + if (!bto->gid && key->gid) { + /* Got a gid when we didn't already have one; insert into gid map */ + log_debug(LD_BTRACK, "ORCONN chan=%"PRIu64" newgid=%"PRIu64, key->chan, + key->gid); + bto->gid = key->gid; + HT_INSERT(bto_gid_ht, bto_gid_map, bto); + } + /* association of ORCONN with channel shouldn't change */ + tor_assert(!bto->chan || !key->chan || bto->chan == key->chan); + if (!bto->chan && key->chan) { + /* Got a chan when we didn't already have one; insert into chan map */ + log_debug(LD_BTRACK, "ORCONN gid=%"PRIu64" newchan=%"PRIu64, + bto->gid, key->chan); + bto->chan = key->chan; + HT_INSERT(bto_chan_ht, bto_chan_map, bto); + } + return bto; +} + +/** Helper for bto_find_or_new() */ +static bt_orconn_t * +bto_new(const bt_orconn_t *key) +{ + struct bt_orconn_t *bto = tor_malloc(sizeof(*bto)); + + bto->gid = key->gid; + bto->chan = key->chan; + bto->state = 0; + bto->proxy_type = 0; + bto->is_orig = false; + bto->is_onehop = true; + + if (bto->gid) + HT_INSERT(bto_gid_ht, bto_gid_map, bto); + if (bto->chan) + HT_INSERT(bto_chan_ht, bto_chan_map, bto); + + return bto; +} + +/** + * Insert a new bt_orconn with the given GID and chan ID, or update + * the GID and chan ID if one already exists. + * + * Return the found or allocated bt_orconn. + **/ +bt_orconn_t * +bto_find_or_new(uint64_t gid, uint64_t chan) +{ + bt_orconn_t key, *bto = NULL; + + tor_assert(gid || chan); + key.gid = gid; + key.chan = chan; + if (key.gid) + bto = HT_FIND(bto_gid_ht, bto_gid_map, &key); + if (!bto && key.chan) { + /* Not found by GID; look up by chan ID */ + bto = HT_FIND(bto_chan_ht, bto_chan_map, &key); + } + if (bto) + return bto_update(bto, &key); + else + return bto_new(&key); +} + +/** Initialize the hash maps */ +void +bto_init_maps(void) +{ + bto_gid_map = tor_malloc(sizeof(*bto_gid_map)); + HT_INIT(bto_gid_ht, bto_gid_map); + bto_chan_map = tor_malloc(sizeof(*bto_chan_map)); + HT_INIT(bto_chan_ht, bto_chan_map); +} + +/** Clear the hash maps, freeing all associated storage */ +void +bto_clear_maps(void) +{ + bto_gid_clear_map(); + bto_chan_clear_map(); +} diff --git a/src/feature/control/btrack_orconn_maps.h b/src/feature/control/btrack_orconn_maps.h new file mode 100644 index 0000000000..b1c2c7aa08 --- /dev/null +++ b/src/feature/control/btrack_orconn_maps.h @@ -0,0 +1,17 @@ +/* Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file btrack_orconn_maps.h + * \brief Header file for btrack_orconn_maps.c + **/ + +#ifndef TOR_BTRACK_ORCONN_MAPS_H + +void bto_delete(uint64_t); +bt_orconn_t *bto_find_or_new(uint64_t, uint64_t); + +void bto_init_maps(void); +void bto_clear_maps(void); + +#endif /* defined(TOR_BTRACK_ORCONN_MAPS_H) */ diff --git a/src/feature/control/btrack_sys.h b/src/feature/control/btrack_sys.h new file mode 100644 index 0000000000..f80cf342e7 --- /dev/null +++ b/src/feature/control/btrack_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file btrack_sys.h + * \brief Declare subsystem object for the bootstrap tracker susbystem. + **/ + +#ifndef TOR_BTRACK_SYS_H +#define TOR_BTRACK_SYS_H + +extern const struct subsys_fns_t sys_btrack; + +#endif /* defined(TOR_BTRACK_SYS_H) */ diff --git a/src/feature/control/control.c b/src/feature/control/control.c index 795902e6f4..da62c94981 100644 --- a/src/feature/control/control.c +++ b/src/feature/control/control.c @@ -34,6 +34,7 @@ **/ #define CONTROL_PRIVATE +#define OCIRC_EVENT_PRIVATE #include "core/or/or.h" #include "app/config/config.h" @@ -50,6 +51,7 @@ #include "core/or/command.h" #include "core/or/connection_edge.h" #include "core/or/connection_or.h" +#include "core/or/ocirc_event.h" #include "core/or/policies.h" #include "core/or/reasons.h" #include "core/or/versions.h" @@ -87,11 +89,12 @@ #include "feature/rend/rendservice.h" #include "feature/stats/geoip_stats.h" #include "feature/stats/predict_ports.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/crypt_ops/crypto_rand.h" #include "lib/crypt_ops/crypto_util.h" #include "lib/encoding/confline.h" #include "lib/evloop/compat_libevent.h" +#include "lib/version/torversion.h" #include "feature/dircache/cached_dir_st.h" #include "feature/control/control_connection_st.h" @@ -178,13 +181,6 @@ static uint8_t *authentication_cookie = NULL; */ static smartlist_t *detached_onion_services = NULL; -/** A sufficiently large size to record the last bootstrap phase string. */ -#define BOOTSTRAP_MSG_LEN 1024 - -/** What was the last bootstrap phase message we sent? We keep track - * of this so we can respond to getinfo status/bootstrap-phase queries. */ -static char last_sent_bootstrap_message[BOOTSTRAP_MSG_LEN]; - static void connection_printf_to_buf(control_connection_t *conn, const char *format, ...) CHECK_PRINTF(2,3); @@ -367,7 +363,7 @@ control_update_global_event_mask(void) control_get_bytes_rw_last_sec(&r, &w); } if (any_old_per_sec_events != control_any_per_second_event_enabled()) { - reschedule_per_second_timer(); + rescan_periodic_events(get_options()); } #undef NEWLY_ENABLED @@ -1680,6 +1676,8 @@ static const struct signal_t signal_table[] = { { SIGNEWNYM, "NEWNYM" }, { SIGCLEARDNSCACHE, "CLEARDNSCACHE"}, { SIGHEARTBEAT, "HEARTBEAT"}, + { SIGACTIVE, "ACTIVE" }, + { SIGDORMANT, "DORMANT" }, { 0, NULL }, }; @@ -2352,7 +2350,11 @@ getinfo_helper_dir(control_connection_t *control_conn, *answer = tor_strdup(consensus->dir); } if (!*answer) { /* try loading it from disk */ - *answer = networkstatus_read_cached_consensus("ns"); + tor_mmap_t *mapped = networkstatus_map_cached_consensus("ns"); + if (mapped) { + *answer = tor_memdup_nulterm(mapped->data, mapped->size); + tor_munmap_file(mapped); + } if (!*answer) { /* generate an error */ *errmsg = "Could not open cached consensus. " "Make sure FetchUselessDescriptors is set to 1."; @@ -3037,7 +3039,7 @@ getinfo_helper_events(control_connection_t *control_conn, check_whether_orport_reachable(options) ? 1 : 0, check_whether_dirport_reachable(options) ? 1 : 0); } else if (!strcmp(question, "status/bootstrap-phase")) { - *answer = tor_strdup(last_sent_bootstrap_message); + *answer = control_event_boot_last_msg(); } else if (!strcmpstart(question, "status/version/")) { int is_server = server_mode(options); networkstatus_t *c = networkstatus_get_latest_consensus(); @@ -3749,7 +3751,7 @@ handle_control_extendcircuit(control_connection_t *conn, uint32_t len, connection_printf_to_buf(conn, "250 EXTENDED %lu\r\n", (unsigned long)circ->global_identifier); if (zero_circ) /* send a 'launched' event, for completeness */ - control_event_circuit_status(circ, CIRC_EVENT_LAUNCHED, 0); + circuit_event_status(circ, CIRC_EVENT_LAUNCHED, 0); done: SMARTLIST_FOREACH(router_nicknames, char *, n, tor_free(n)); smartlist_free(router_nicknames); @@ -5625,6 +5627,7 @@ control_event_circuit_status(origin_circuit_t *circ, circuit_status_event_t tp, { const char *status; char reasons[64] = ""; + if (!EVENT_IS_INTERESTING(EVENT_CIRCUIT_STATUS)) return 0; tor_assert(circ); @@ -7008,361 +7011,6 @@ monitor_owning_controller_process(const char *process_spec) } } -/** Convert the name of a bootstrapping phase <b>s</b> into strings - * <b>tag</b> and <b>summary</b> suitable for display by the controller. */ -static int -bootstrap_status_to_string(bootstrap_status_t s, const char **tag, - const char **summary) -{ - switch (s) { - case BOOTSTRAP_STATUS_UNDEF: - *tag = "undef"; - *summary = "Undefined"; - break; - case BOOTSTRAP_STATUS_STARTING: - *tag = "starting"; - *summary = "Starting"; - break; - case BOOTSTRAP_STATUS_CONN_DIR: - *tag = "conn_dir"; - *summary = "Connecting to directory server"; - break; - case BOOTSTRAP_STATUS_HANDSHAKE: - *tag = "status_handshake"; - *summary = "Finishing handshake"; - break; - case BOOTSTRAP_STATUS_HANDSHAKE_DIR: - *tag = "handshake_dir"; - *summary = "Finishing handshake with directory server"; - break; - case BOOTSTRAP_STATUS_ONEHOP_CREATE: - *tag = "onehop_create"; - *summary = "Establishing an encrypted directory connection"; - break; - case BOOTSTRAP_STATUS_REQUESTING_STATUS: - *tag = "requesting_status"; - *summary = "Asking for networkstatus consensus"; - break; - case BOOTSTRAP_STATUS_LOADING_STATUS: - *tag = "loading_status"; - *summary = "Loading networkstatus consensus"; - break; - case BOOTSTRAP_STATUS_LOADING_KEYS: - *tag = "loading_keys"; - *summary = "Loading authority key certs"; - break; - case BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS: - *tag = "requesting_descriptors"; - /* XXXX this appears to incorrectly report internal on most loads */ - *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ? - "Asking for relay descriptors for internal paths" : - "Asking for relay descriptors"; - break; - /* If we're sure there are no exits in the consensus, - * inform the controller by adding "internal" - * to the status summaries. - * (We only check this while loading descriptors, - * so we may not know in the earlier stages.) - * But if there are exits, we can't be sure whether - * we're creating internal or exit paths/circuits. - * XXXX Or should be use different tags or statuses - * for internal and exit/all? */ - case BOOTSTRAP_STATUS_LOADING_DESCRIPTORS: - *tag = "loading_descriptors"; - *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ? - "Loading relay descriptors for internal paths" : - "Loading relay descriptors"; - break; - case BOOTSTRAP_STATUS_CONN_OR: - *tag = "conn_or"; - *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ? - "Connecting to the Tor network internally" : - "Connecting to the Tor network"; - break; - case BOOTSTRAP_STATUS_HANDSHAKE_OR: - *tag = "handshake_or"; - *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ? - "Finishing handshake with first hop of internal circuit" : - "Finishing handshake with first hop"; - break; - case BOOTSTRAP_STATUS_CIRCUIT_CREATE: - *tag = "circuit_create"; - *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ? - "Establishing an internal Tor circuit" : - "Establishing a Tor circuit"; - break; - case BOOTSTRAP_STATUS_DONE: - *tag = "done"; - *summary = "Done"; - break; - default: -// log_warn(LD_BUG, "Unrecognized bootstrap status code %d", s); - *tag = *summary = "unknown"; - return -1; - } - return 0; -} - -/** What percentage through the bootstrap process are we? We remember - * this so we can avoid sending redundant bootstrap status events, and - * so we can guess context for the bootstrap messages which are - * ambiguous. It starts at 'undef', but gets set to 'starting' while - * Tor initializes. */ -static int bootstrap_percent = BOOTSTRAP_STATUS_UNDEF; - -/** Like bootstrap_percent, but only takes on the enumerated values in - * bootstrap_status_t. - */ -static int bootstrap_phase = BOOTSTRAP_STATUS_UNDEF; - -/** As bootstrap_percent, but holds the bootstrapping level at which we last - * logged a NOTICE-level message. We use this, plus BOOTSTRAP_PCT_INCREMENT, - * to avoid flooding the log with a new message every time we get a few more - * microdescriptors */ -static int notice_bootstrap_percent = 0; - -/** How many problems have we had getting to the next bootstrapping phase? - * These include failure to establish a connection to a Tor relay, - * failures to finish the TLS handshake, failures to validate the - * consensus document, etc. */ -static int bootstrap_problems = 0; - -/** We only tell the controller once we've hit a threshold of problems - * for the current phase. */ -#define BOOTSTRAP_PROBLEM_THRESHOLD 10 - -/** When our bootstrapping progress level changes, but our bootstrapping - * status has not advanced, we only log at NOTICE when we have made at least - * this much progress. - */ -#define BOOTSTRAP_PCT_INCREMENT 5 - -/** Do the actual logging and notifications for - * control_event_bootstrap(). Doesn't change any state beyond that. - */ -static void -control_event_bootstrap_core(int loglevel, bootstrap_status_t status, - int progress) -{ - char buf[BOOTSTRAP_MSG_LEN]; - const char *tag, *summary; - - bootstrap_status_to_string(status, &tag, &summary); - /* Locally reset status if there's incremental progress */ - if (progress) - status = progress; - - tor_log(loglevel, LD_CONTROL, - "Bootstrapped %d%%: %s", status, summary); - tor_snprintf(buf, sizeof(buf), - "BOOTSTRAP PROGRESS=%d TAG=%s SUMMARY=\"%s\"", - status, tag, summary); - tor_snprintf(last_sent_bootstrap_message, - sizeof(last_sent_bootstrap_message), - "NOTICE %s", buf); - control_event_client_status(LOG_NOTICE, "%s", buf); -} - -/** Called when Tor has made progress at bootstrapping its directory - * information and initial circuits. - * - * <b>status</b> is the new status, that is, what task we will be doing - * next. <b>progress</b> is zero if we just started this task, else it - * represents progress on the task. - */ -void -control_event_bootstrap(bootstrap_status_t status, int progress) -{ - int loglevel = LOG_NOTICE; - - if (bootstrap_percent == BOOTSTRAP_STATUS_DONE) - return; /* already bootstrapped; nothing to be done here. */ - - /* special case for handshaking status, since our TLS handshaking code - * can't distinguish what the connection is going to be for. */ - if (status == BOOTSTRAP_STATUS_HANDSHAKE) { - if (bootstrap_percent < BOOTSTRAP_STATUS_CONN_OR) { - status = BOOTSTRAP_STATUS_HANDSHAKE_DIR; - } else { - status = BOOTSTRAP_STATUS_HANDSHAKE_OR; - } - } - - if (status <= bootstrap_percent) { - /* If there's no new progress, return early. */ - if (!progress || progress <= bootstrap_percent) - return; - /* Log at INFO if not enough progress happened. */ - if (progress < notice_bootstrap_percent + BOOTSTRAP_PCT_INCREMENT) - loglevel = LOG_INFO; - } - - control_event_bootstrap_core(loglevel, status, progress); - - if (status > bootstrap_percent) { - bootstrap_phase = status; /* new milestone reached */ - bootstrap_percent = status; - } - if (progress > bootstrap_percent) { - /* incremental progress within a milestone */ - bootstrap_percent = progress; - bootstrap_problems = 0; /* Progress! Reset our problem counter. */ - } - if (loglevel == LOG_NOTICE && - bootstrap_percent > notice_bootstrap_percent) { - /* Remember that we gave a notice at this level. */ - notice_bootstrap_percent = bootstrap_percent; - } -} - -/** Flag whether we've opened an OR_CONN yet */ -static int bootstrap_first_orconn = 0; - -/** Like bootstrap_phase, but for (possibly deferred) directory progress */ -static int bootstrap_dir_phase = BOOTSTRAP_STATUS_UNDEF; - -/** Like bootstrap_problems, but for (possibly deferred) directory progress */ -static int bootstrap_dir_progress = BOOTSTRAP_STATUS_UNDEF; - -/** Defer directory info bootstrap events until we have successfully - * completed our first connection to a router. */ -void -control_event_boot_dir(bootstrap_status_t status, int progress) -{ - if (status > bootstrap_dir_progress) { - bootstrap_dir_progress = status; - bootstrap_dir_phase = status; - } - if (progress && progress >= bootstrap_dir_progress) { - bootstrap_dir_progress = progress; - } - - /* Don't report unless we have successfully opened at least one OR_CONN */ - if (!bootstrap_first_orconn) - return; - - control_event_bootstrap(status, progress); -} - -/** Set a flag to allow reporting of directory bootstrap progress. - * (Code that reports completion of an OR_CONN calls this.) Also, - * report directory progress so far. */ -void -control_event_boot_first_orconn(void) -{ - bootstrap_first_orconn = 1; - control_event_bootstrap(bootstrap_dir_phase, bootstrap_dir_progress); -} - -/** Called when Tor has failed to make bootstrapping progress in a way - * that indicates a problem. <b>warn</b> gives a human-readable hint - * as to why, and <b>reason</b> provides a controller-facing short - * tag. <b>conn</b> is the connection that caused this problem and - * can be NULL if a connection cannot be easily identified. - */ -void -control_event_bootstrap_problem(const char *warn, const char *reason, - const connection_t *conn, int dowarn) -{ - int status = bootstrap_percent; - const char *tag = "", *summary = ""; - char buf[BOOTSTRAP_MSG_LEN]; - const char *recommendation = "ignore"; - int severity; - char *or_id = NULL, *hostaddr = NULL; - or_connection_t *or_conn = NULL; - - /* bootstrap_percent must not be in "undefined" state here. */ - tor_assert(status >= 0); - - if (bootstrap_percent == 100) - return; /* already bootstrapped; nothing to be done here. */ - - bootstrap_problems++; - - if (bootstrap_problems >= BOOTSTRAP_PROBLEM_THRESHOLD) - dowarn = 1; - - /* Don't warn about our bootstrapping status if we are hibernating or - * shutting down. */ - if (we_are_hibernating()) - dowarn = 0; - - tor_assert(bootstrap_status_to_string(bootstrap_phase, &tag, &summary) == 0); - - severity = dowarn ? LOG_WARN : LOG_INFO; - - if (dowarn) - recommendation = "warn"; - - if (conn && conn->type == CONN_TYPE_OR) { - /* XXX TO_OR_CONN can't deal with const */ - or_conn = TO_OR_CONN((connection_t *)conn); - or_id = tor_strdup(hex_str(or_conn->identity_digest, DIGEST_LEN)); - } else { - or_id = tor_strdup("?"); - } - - if (conn) - tor_asprintf(&hostaddr, "%s:%d", conn->address, (int)conn->port); - else - hostaddr = tor_strdup("?"); - - log_fn(severity, - LD_CONTROL, "Problem bootstrapping. Stuck at %d%%: %s. (%s; %s; " - "count %d; recommendation %s; host %s at %s)", - status, summary, warn, reason, - bootstrap_problems, recommendation, - or_id, hostaddr); - - connection_or_report_broken_states(severity, LD_HANDSHAKE); - - tor_snprintf(buf, sizeof(buf), - "BOOTSTRAP PROGRESS=%d TAG=%s SUMMARY=\"%s\" WARNING=\"%s\" REASON=%s " - "COUNT=%d RECOMMENDATION=%s HOSTID=\"%s\" HOSTADDR=\"%s\"", - bootstrap_percent, tag, summary, warn, reason, bootstrap_problems, - recommendation, - or_id, hostaddr); - - tor_snprintf(last_sent_bootstrap_message, - sizeof(last_sent_bootstrap_message), - "WARN %s", buf); - control_event_client_status(LOG_WARN, "%s", buf); - - tor_free(hostaddr); - tor_free(or_id); -} - -/** Called when Tor has failed to make bootstrapping progress in a way - * that indicates a problem. <b>warn</b> gives a hint as to why, and - * <b>reason</b> provides an "or_conn_end_reason" tag. <b>or_conn</b> - * is the connection that caused this problem. - */ -MOCK_IMPL(void, -control_event_bootstrap_prob_or, (const char *warn, int reason, - or_connection_t *or_conn)) -{ - int dowarn = 0; - - if (or_conn->have_noted_bootstrap_problem) - return; - - or_conn->have_noted_bootstrap_problem = 1; - - if (reason == END_OR_CONN_REASON_NO_ROUTE) - dowarn = 1; - - /* If we are using bridges and all our OR connections are now - closed, it means that we totally failed to connect to our - bridges. Throw a warning. */ - if (get_options()->UseBridges && !any_other_active_or_conns(or_conn)) - dowarn = 1; - - control_event_bootstrap_problem(warn, - orconn_end_reason_to_control_string(reason), - TO_CONN(or_conn), dowarn); -} - /** We just generated a new summary of which countries we've seen clients * from recently. Send a copy to the controller in case it wants to * display it for the user. */ @@ -7388,6 +7036,26 @@ control_event_transport_launched(const char *mode, const char *transport_name, mode, transport_name, fmt_addr(addr), port); } +/** A pluggable transport called <b>pt_name</b> has emitted a log message + * found in <b>message</b> at <b>severity</b> log level. */ +void +control_event_pt_log(const char *log) +{ + send_control_event(EVENT_PT_LOG, + "650 PT_LOG %s\r\n", + log); +} + +/** A pluggable transport has emitted a STATUS message found in + * <b>status</b>. */ +void +control_event_pt_status(const char *status) +{ + send_control_event(EVENT_PT_STATUS, + "650 PT_STATUS %s\r\n", + status); +} + /** Convert rendezvous auth type to string for HS_DESC control events */ const char * @@ -7879,17 +7547,10 @@ control_free_all(void) mainloop_event_free(flush_queued_events_event); flush_queued_events_event = NULL; } - bootstrap_percent = BOOTSTRAP_STATUS_UNDEF; - bootstrap_phase = BOOTSTRAP_STATUS_UNDEF; - notice_bootstrap_percent = 0; - bootstrap_problems = 0; - bootstrap_first_orconn = 0; - bootstrap_dir_progress = BOOTSTRAP_STATUS_UNDEF; - bootstrap_dir_phase = BOOTSTRAP_STATUS_UNDEF; + control_event_bootstrap_reset(); authentication_cookie_is_set = 0; global_event_mask = 0; disable_log_messages = 0; - memset(last_sent_bootstrap_message, 0, sizeof(last_sent_bootstrap_message)); } #ifdef TOR_UNIT_TESTS diff --git a/src/feature/control/control.h b/src/feature/control/control.h index cd5402d455..08d8924e2d 100644 --- a/src/feature/control/control.h +++ b/src/feature/control/control.h @@ -12,15 +12,7 @@ #ifndef TOR_CONTROL_H #define TOR_CONTROL_H -/** Used to indicate the type of a circuit event passed to the controller. - * The various types are defined in control-spec.txt */ -typedef enum circuit_status_event_t { - CIRC_EVENT_LAUNCHED = 0, - CIRC_EVENT_BUILT = 1, - CIRC_EVENT_EXTENDED = 2, - CIRC_EVENT_FAILED = 3, - CIRC_EVENT_CLOSED = 4, -} circuit_status_event_t; +#include "core/or/ocirc_event.h" /** Used to indicate the type of a CIRC_MINOR event passed to the controller. * The various types are defined in control-spec.txt . */ @@ -29,6 +21,8 @@ typedef enum circuit_status_minor_event_t { CIRC_MINOR_EVENT_CANNIBALIZED, } circuit_status_minor_event_t; +#include "core/or/orconn_event.h" + /** Used to indicate the type of a stream event passed to the controller. * The various types are defined in control-spec.txt */ typedef enum stream_status_event_t { @@ -43,16 +37,6 @@ typedef enum stream_status_event_t { STREAM_EVENT_REMAP = 8 } stream_status_event_t; -/** Used to indicate the type of an OR connection event passed to the - * controller. The various types are defined in control-spec.txt */ -typedef enum or_conn_status_event_t { - OR_CONN_EVENT_LAUNCHED = 0, - OR_CONN_EVENT_CONNECTED = 1, - OR_CONN_EVENT_FAILED = 2, - OR_CONN_EVENT_CLOSED = 3, - OR_CONN_EVENT_NEW = 4, -} or_conn_status_event_t; - /** Used to indicate the type of a buildtime event */ typedef enum buildtimeout_set_event_t { BUILDTIMEOUT_SET_EVENT_COMPUTED = 0, @@ -67,18 +51,42 @@ typedef enum buildtimeout_set_event_t { typedef enum { BOOTSTRAP_STATUS_UNDEF=-1, BOOTSTRAP_STATUS_STARTING=0, - BOOTSTRAP_STATUS_CONN_DIR=5, - BOOTSTRAP_STATUS_HANDSHAKE=-2, - BOOTSTRAP_STATUS_HANDSHAKE_DIR=10, - BOOTSTRAP_STATUS_ONEHOP_CREATE=15, - BOOTSTRAP_STATUS_REQUESTING_STATUS=20, - BOOTSTRAP_STATUS_LOADING_STATUS=25, + + /* Initial connection to any relay */ + + BOOTSTRAP_STATUS_CONN_PT=1, + BOOTSTRAP_STATUS_CONN_DONE_PT=2, + BOOTSTRAP_STATUS_CONN_PROXY=3, + BOOTSTRAP_STATUS_CONN_DONE_PROXY=4, + BOOTSTRAP_STATUS_CONN=5, + BOOTSTRAP_STATUS_CONN_DONE=10, + BOOTSTRAP_STATUS_HANDSHAKE=14, + BOOTSTRAP_STATUS_HANDSHAKE_DONE=15, + + /* Loading directory info */ + + BOOTSTRAP_STATUS_ONEHOP_CREATE=20, + BOOTSTRAP_STATUS_REQUESTING_STATUS=25, + BOOTSTRAP_STATUS_LOADING_STATUS=30, BOOTSTRAP_STATUS_LOADING_KEYS=40, BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS=45, BOOTSTRAP_STATUS_LOADING_DESCRIPTORS=50, - BOOTSTRAP_STATUS_CONN_OR=80, - BOOTSTRAP_STATUS_HANDSHAKE_OR=85, - BOOTSTRAP_STATUS_CIRCUIT_CREATE=90, + BOOTSTRAP_STATUS_ENOUGH_DIRINFO=75, + + /* Connecting to a relay for AP circuits */ + + BOOTSTRAP_STATUS_AP_CONN_PT=76, + BOOTSTRAP_STATUS_AP_CONN_DONE_PT=77, + BOOTSTRAP_STATUS_AP_CONN_PROXY=78, + BOOTSTRAP_STATUS_AP_CONN_DONE_PROXY=79, + BOOTSTRAP_STATUS_AP_CONN=80, + BOOTSTRAP_STATUS_AP_CONN_DONE=85, + BOOTSTRAP_STATUS_AP_HANDSHAKE=89, + BOOTSTRAP_STATUS_AP_HANDSHAKE_DONE=90, + + /* Creating AP circuits */ + + BOOTSTRAP_STATUS_CIRCUIT_CREATE=95, BOOTSTRAP_STATUS_DONE=100 } bootstrap_status_t; @@ -200,11 +208,15 @@ void control_event_boot_dir(bootstrap_status_t status, int progress); void control_event_boot_first_orconn(void); void control_event_bootstrap_problem(const char *warn, const char *reason, const connection_t *conn, int dowarn); +char *control_event_boot_last_msg(void); +void control_event_bootstrap_reset(void); void control_event_clients_seen(const char *controller_str); void control_event_transport_launched(const char *mode, const char *transport_name, tor_addr_t *addr, uint16_t port); +void control_event_pt_log(const char *log); +void control_event_pt_status(const char *status); const char *rend_auth_type_to_string(rend_auth_type_t auth_type); MOCK_DECL(const char *, node_describe_longname_by_id,(const char *id_digest)); void control_event_hs_descriptor_requested(const char *onion_address, @@ -293,7 +305,9 @@ void control_free_all(void); #define EVENT_HS_DESC 0x0021 #define EVENT_HS_DESC_CONTENT 0x0022 #define EVENT_NETWORK_LIVENESS 0x0023 -#define EVENT_MAX_ 0x0023 +#define EVENT_PT_LOG 0x0024 +#define EVENT_PT_STATUS 0x0025 +#define EVENT_MAX_ 0x0025 /* sizeof(control_connection_t.event_mask) in bits, currently a uint64_t */ #define EVENT_CAPACITY_ 0x0040 diff --git a/src/feature/control/control_bootstrap.c b/src/feature/control/control_bootstrap.c new file mode 100644 index 0000000000..49e190dc51 --- /dev/null +++ b/src/feature/control/control_bootstrap.c @@ -0,0 +1,383 @@ +/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file control_bootstrap.c + * \brief Provide bootstrap progress events for the control port. + */ +#include "core/or/or.h" + +#include "app/config/config.h" +#include "core/mainloop/connection.h" +#include "core/or/connection_or.h" +#include "core/or/connection_st.h" +#include "core/or/or_connection_st.h" +#include "core/or/reasons.h" +#include "feature/control/control.h" +#include "feature/hibernate/hibernate.h" +#include "lib/malloc/malloc.h" + +/** A sufficiently large size to record the last bootstrap phase string. */ +#define BOOTSTRAP_MSG_LEN 1024 + +/** What was the last bootstrap phase message we sent? We keep track + * of this so we can respond to getinfo status/bootstrap-phase queries. */ +static char last_sent_bootstrap_message[BOOTSTRAP_MSG_LEN]; + +/** Table to convert bootstrap statuses to strings. */ +static const struct { + bootstrap_status_t status; + const char *tag; + const char *summary; +} boot_to_str_tab[] = { + { BOOTSTRAP_STATUS_UNDEF, "undef", "Undefined" }, + { BOOTSTRAP_STATUS_STARTING, "starting", "Starting" }, + + /* Initial connection to any relay */ + + { BOOTSTRAP_STATUS_CONN_PT, "conn_pt", "Connecting to pluggable transport" }, + { BOOTSTRAP_STATUS_CONN_DONE_PT, "conn_done_pt", + "Connected to pluggable transport" }, + { BOOTSTRAP_STATUS_CONN_PROXY, "conn_proxy", "Connecting to proxy" }, + { BOOTSTRAP_STATUS_CONN_DONE_PROXY, "conn_done_proxy", + "Connected to proxy" }, + { BOOTSTRAP_STATUS_CONN, "conn", "Connecting to a relay" }, + { BOOTSTRAP_STATUS_CONN_DONE, "conn_done", "Connected to a relay" }, + { BOOTSTRAP_STATUS_HANDSHAKE, "handshake", + "Handshaking with a relay" }, + { BOOTSTRAP_STATUS_HANDSHAKE_DONE, "handshake_done", + "Handshake with a relay done" }, + + /* Loading directory info */ + + { BOOTSTRAP_STATUS_ONEHOP_CREATE, "onehop_create", + "Establishing an encrypted directory connection" }, + { BOOTSTRAP_STATUS_REQUESTING_STATUS, "requesting_status", + "Asking for networkstatus consensus" }, + { BOOTSTRAP_STATUS_LOADING_STATUS, "loading_status", + "Loading networkstatus consensus" }, + { BOOTSTRAP_STATUS_LOADING_KEYS, "loading_keys", + "Loading authority key certs" }, + { BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS, "requesting_descriptors", + "Asking for relay descriptors" }, + { BOOTSTRAP_STATUS_LOADING_DESCRIPTORS, "loading_descriptors", + "Loading relay descriptors" }, + { BOOTSTRAP_STATUS_ENOUGH_DIRINFO, "enough_dirinfo", + "Loaded enough directory info to build circuits" }, + + /* Connecting to a relay for AP circuits */ + + { BOOTSTRAP_STATUS_AP_CONN_PT, "ap_conn_pt", + "Connecting to pluggable transport to build circuits" }, + { BOOTSTRAP_STATUS_AP_CONN_DONE_PT, "ap_conn_done_pt", + "Connected to pluggable transport to build circuits" }, + { BOOTSTRAP_STATUS_AP_CONN_PROXY, "ap_conn_proxy", + "Connecting to proxy " }, + { BOOTSTRAP_STATUS_AP_CONN_DONE_PROXY, "ap_conn_done_proxy", + "Connected to proxy to build circuits" }, + { BOOTSTRAP_STATUS_AP_CONN, "ap_conn", + "Connecting to a relay to build circuits" }, + { BOOTSTRAP_STATUS_AP_CONN_DONE, "ap_conn_done", + "Connected to a relay to build circuits" }, + { BOOTSTRAP_STATUS_AP_HANDSHAKE, "ap_handshake", + "Finishing handshake with a relay to build circuits" }, + { BOOTSTRAP_STATUS_AP_HANDSHAKE_DONE, "ap_handshake_done", + "Handshake fininshed with a relay to build circuits" }, + + /* Creating AP circuits */ + + { BOOTSTRAP_STATUS_CIRCUIT_CREATE, "circuit_create", + "Establishing a Tor circuit" }, + { BOOTSTRAP_STATUS_DONE, "done", "Done" }, +}; +#define N_BOOT_TO_STR (sizeof(boot_to_str_tab)/sizeof(boot_to_str_tab[0])) + +/** Convert the name of a bootstrapping phase <b>s</b> into strings + * <b>tag</b> and <b>summary</b> suitable for display by the controller. */ +static int +bootstrap_status_to_string(bootstrap_status_t s, const char **tag, + const char **summary) +{ + for (size_t i = 0; i < N_BOOT_TO_STR; i++) { + if (s == boot_to_str_tab[i].status) { + *tag = boot_to_str_tab[i].tag; + *summary = boot_to_str_tab[i].summary; + return 0; + } + } + + *tag = *summary = "unknown"; + return -1; +} + +/** What percentage through the bootstrap process are we? We remember + * this so we can avoid sending redundant bootstrap status events, and + * so we can guess context for the bootstrap messages which are + * ambiguous. It starts at 'undef', but gets set to 'starting' while + * Tor initializes. */ +static int bootstrap_percent = BOOTSTRAP_STATUS_UNDEF; + +/** Like bootstrap_percent, but only takes on the enumerated values in + * bootstrap_status_t. + */ +static int bootstrap_phase = BOOTSTRAP_STATUS_UNDEF; + +/** As bootstrap_percent, but holds the bootstrapping level at which we last + * logged a NOTICE-level message. We use this, plus BOOTSTRAP_PCT_INCREMENT, + * to avoid flooding the log with a new message every time we get a few more + * microdescriptors */ +static int notice_bootstrap_percent = 0; + +/** How many problems have we had getting to the next bootstrapping phase? + * These include failure to establish a connection to a Tor relay, + * failures to finish the TLS handshake, failures to validate the + * consensus document, etc. */ +static int bootstrap_problems = 0; + +/** We only tell the controller once we've hit a threshold of problems + * for the current phase. */ +#define BOOTSTRAP_PROBLEM_THRESHOLD 10 + +/** When our bootstrapping progress level changes, but our bootstrapping + * status has not advanced, we only log at NOTICE when we have made at least + * this much progress. + */ +#define BOOTSTRAP_PCT_INCREMENT 5 + +/** Do the actual logging and notifications for + * control_event_bootstrap(). Doesn't change any state beyond that. + */ +static void +control_event_bootstrap_core(int loglevel, bootstrap_status_t status, + int progress) +{ + char buf[BOOTSTRAP_MSG_LEN]; + const char *tag, *summary; + + bootstrap_status_to_string(status, &tag, &summary); + /* Locally reset status if there's incremental progress */ + if (progress) + status = progress; + + tor_log(loglevel, LD_CONTROL, + "Bootstrapped %d%% (%s): %s", status, tag, summary); + tor_snprintf(buf, sizeof(buf), + "BOOTSTRAP PROGRESS=%d TAG=%s SUMMARY=\"%s\"", + status, tag, summary); + tor_snprintf(last_sent_bootstrap_message, + sizeof(last_sent_bootstrap_message), + "NOTICE %s", buf); + control_event_client_status(LOG_NOTICE, "%s", buf); +} + +/** Called when Tor has made progress at bootstrapping its directory + * information and initial circuits. + * + * <b>status</b> is the new status, that is, what task we will be doing + * next. <b>progress</b> is zero if we just started this task, else it + * represents progress on the task. + */ +void +control_event_bootstrap(bootstrap_status_t status, int progress) +{ + int loglevel = LOG_NOTICE; + + if (bootstrap_percent == BOOTSTRAP_STATUS_DONE) + return; /* already bootstrapped; nothing to be done here. */ + + if (status <= bootstrap_percent) { + /* If there's no new progress, return early. */ + if (!progress || progress <= bootstrap_percent) + return; + /* Log at INFO if not enough progress happened. */ + if (progress < notice_bootstrap_percent + BOOTSTRAP_PCT_INCREMENT) + loglevel = LOG_INFO; + } + + control_event_bootstrap_core(loglevel, status, progress); + + if (status > bootstrap_percent) { + bootstrap_phase = status; /* new milestone reached */ + bootstrap_percent = status; + } + if (progress > bootstrap_percent) { + /* incremental progress within a milestone */ + bootstrap_percent = progress; + bootstrap_problems = 0; /* Progress! Reset our problem counter. */ + } + if (loglevel == LOG_NOTICE && + bootstrap_percent > notice_bootstrap_percent) { + /* Remember that we gave a notice at this level. */ + notice_bootstrap_percent = bootstrap_percent; + } +} + +/** Flag whether we've opened an OR_CONN yet */ +static int bootstrap_first_orconn = 0; + +/** Like bootstrap_phase, but for (possibly deferred) directory progress */ +static int bootstrap_dir_phase = BOOTSTRAP_STATUS_UNDEF; + +/** Like bootstrap_problems, but for (possibly deferred) directory progress */ +static int bootstrap_dir_progress = BOOTSTRAP_STATUS_UNDEF; + +/** Defer directory info bootstrap events until we have successfully + * completed our first connection to a router. */ +void +control_event_boot_dir(bootstrap_status_t status, int progress) +{ + if (status > bootstrap_dir_progress) { + bootstrap_dir_progress = status; + bootstrap_dir_phase = status; + } + if (progress && progress >= bootstrap_dir_progress) { + bootstrap_dir_progress = progress; + } + + /* Don't report unless we have successfully opened at least one OR_CONN */ + if (!bootstrap_first_orconn) + return; + + control_event_bootstrap(status, progress); +} + +/** Set a flag to allow reporting of directory bootstrap progress. + * (Code that reports completion of an OR_CONN calls this.) Also, + * report directory progress so far. */ +void +control_event_boot_first_orconn(void) +{ + bootstrap_first_orconn = 1; + control_event_bootstrap(bootstrap_dir_phase, bootstrap_dir_progress); +} + +/** Called when Tor has failed to make bootstrapping progress in a way + * that indicates a problem. <b>warn</b> gives a human-readable hint + * as to why, and <b>reason</b> provides a controller-facing short + * tag. <b>conn</b> is the connection that caused this problem and + * can be NULL if a connection cannot be easily identified. + */ +void +control_event_bootstrap_problem(const char *warn, const char *reason, + const connection_t *conn, int dowarn) +{ + int status = bootstrap_percent; + const char *tag = "", *summary = ""; + char buf[BOOTSTRAP_MSG_LEN]; + const char *recommendation = "ignore"; + int severity; + char *or_id = NULL, *hostaddr = NULL; + or_connection_t *or_conn = NULL; + + /* bootstrap_percent must not be in "undefined" state here. */ + tor_assert(status >= 0); + + if (bootstrap_percent == 100) + return; /* already bootstrapped; nothing to be done here. */ + + bootstrap_problems++; + + if (bootstrap_problems >= BOOTSTRAP_PROBLEM_THRESHOLD) + dowarn = 1; + + /* Don't warn about our bootstrapping status if we are hibernating or + * shutting down. */ + if (we_are_hibernating()) + dowarn = 0; + + tor_assert(bootstrap_status_to_string(bootstrap_phase, &tag, &summary) == 0); + + severity = dowarn ? LOG_WARN : LOG_INFO; + + if (dowarn) + recommendation = "warn"; + + if (conn && conn->type == CONN_TYPE_OR) { + /* XXX TO_OR_CONN can't deal with const */ + or_conn = TO_OR_CONN((connection_t *)conn); + or_id = tor_strdup(hex_str(or_conn->identity_digest, DIGEST_LEN)); + } else { + or_id = tor_strdup("?"); + } + + if (conn) + tor_asprintf(&hostaddr, "%s:%d", conn->address, (int)conn->port); + else + hostaddr = tor_strdup("?"); + + log_fn(severity, + LD_CONTROL, "Problem bootstrapping. Stuck at %d%% (%s): %s. (%s; %s; " + "count %d; recommendation %s; host %s at %s)", + status, tag, summary, warn, reason, + bootstrap_problems, recommendation, + or_id, hostaddr); + + connection_or_report_broken_states(severity, LD_HANDSHAKE); + + tor_snprintf(buf, sizeof(buf), + "BOOTSTRAP PROGRESS=%d TAG=%s SUMMARY=\"%s\" WARNING=\"%s\" REASON=%s " + "COUNT=%d RECOMMENDATION=%s HOSTID=\"%s\" HOSTADDR=\"%s\"", + bootstrap_percent, tag, summary, warn, reason, bootstrap_problems, + recommendation, + or_id, hostaddr); + + tor_snprintf(last_sent_bootstrap_message, + sizeof(last_sent_bootstrap_message), + "WARN %s", buf); + control_event_client_status(LOG_WARN, "%s", buf); + + tor_free(hostaddr); + tor_free(or_id); +} + +/** Called when Tor has failed to make bootstrapping progress in a way + * that indicates a problem. <b>warn</b> gives a hint as to why, and + * <b>reason</b> provides an "or_conn_end_reason" tag. <b>or_conn</b> + * is the connection that caused this problem. + */ +MOCK_IMPL(void, +control_event_bootstrap_prob_or, (const char *warn, int reason, + or_connection_t *or_conn)) +{ + int dowarn = 0; + + if (or_conn->have_noted_bootstrap_problem) + return; + + or_conn->have_noted_bootstrap_problem = 1; + + if (reason == END_OR_CONN_REASON_NO_ROUTE) + dowarn = 1; + + /* If we are using bridges and all our OR connections are now + closed, it means that we totally failed to connect to our + bridges. Throw a warning. */ + if (get_options()->UseBridges && !any_other_active_or_conns(or_conn)) + dowarn = 1; + + control_event_bootstrap_problem(warn, + orconn_end_reason_to_control_string(reason), + TO_CONN(or_conn), dowarn); +} + +/** Return a copy of the last sent bootstrap message. */ +char * +control_event_boot_last_msg(void) +{ + return tor_strdup(last_sent_bootstrap_message); +} + +/** Reset bootstrap tracking state. */ +void +control_event_bootstrap_reset(void) +{ + bootstrap_percent = BOOTSTRAP_STATUS_UNDEF; + bootstrap_phase = BOOTSTRAP_STATUS_UNDEF; + notice_bootstrap_percent = 0; + bootstrap_problems = 0; + bootstrap_first_orconn = 0; + bootstrap_dir_progress = BOOTSTRAP_STATUS_UNDEF; + bootstrap_dir_phase = BOOTSTRAP_STATUS_UNDEF; + memset(last_sent_bootstrap_message, 0, sizeof(last_sent_bootstrap_message)); +} diff --git a/src/feature/dirauth/dirvote.c b/src/feature/dirauth/dirvote.c index 80a5b54737..d8a18835f2 100644 --- a/src/feature/dirauth/dirvote.c +++ b/src/feature/dirauth/dirvote.c @@ -28,6 +28,7 @@ #include "feature/nodelist/fmt_routerstatus.h" #include "feature/nodelist/microdesc.h" #include "feature/nodelist/networkstatus.h" +#include "feature/nodelist/nodefamily.h" #include "feature/nodelist/nodelist.h" #include "feature/nodelist/routerlist.h" #include "feature/relay/router.h" @@ -413,7 +414,8 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key, { networkstatus_t *v; - if (!(v = networkstatus_parse_vote_from_string(status, NULL, + if (!(v = networkstatus_parse_vote_from_string(status, strlen(status), + NULL, v3_ns->type))) { log_err(LD_BUG,"Generated a networkstatus %s we couldn't parse: " "<<%s>>", @@ -2410,7 +2412,8 @@ networkstatus_compute_consensus(smartlist_t *votes, { networkstatus_t *c; - if (!(c = networkstatus_parse_vote_from_string(result, NULL, + if (!(c = networkstatus_parse_vote_from_string(result, strlen(result), + NULL, NS_TYPE_CONSENSUS))) { log_err(LD_BUG, "Generated a networkstatus consensus we couldn't " "parse."); @@ -3133,7 +3136,8 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out) *msg_out = NULL; again: - vote = networkstatus_parse_vote_from_string(vote_body, &end_of_vote, + vote = networkstatus_parse_vote_from_string(vote_body, strlen(vote_body), + &end_of_vote, NS_TYPE_VOTE); if (!end_of_vote) end_of_vote = vote_body + strlen(vote_body); @@ -3391,7 +3395,9 @@ dirvote_compute_consensuses(void) flavor_name); continue; } - consensus = networkstatus_parse_vote_from_string(consensus_body, NULL, + consensus = networkstatus_parse_vote_from_string(consensus_body, + strlen(consensus_body), + NULL, NS_TYPE_CONSENSUS); if (!consensus) { log_warn(LD_DIR, "Couldn't parse %s consensus we generated!", @@ -3530,7 +3536,7 @@ dirvote_add_signatures_to_pending_consensus( * just in case we break detached signature processing at some point. */ { networkstatus_t *v = networkstatus_parse_vote_from_string( - pc->body, NULL, + pc->body, strlen(pc->body), NULL, NS_TYPE_CONSENSUS); tor_assert(v); networkstatus_vote_free(v); @@ -3655,7 +3661,9 @@ dirvote_publish_consensus(void) continue; } - if (networkstatus_set_current_consensus(pending->body, name, 0, NULL)) + if (networkstatus_set_current_consensus(pending->body, + strlen(pending->body), + name, 0, NULL)) log_warn(LD_DIR, "Error publishing %s consensus", name); else log_notice(LD_DIR, "Published %s consensus", name); @@ -3792,8 +3800,16 @@ dirvote_create_microdescriptor(const routerinfo_t *ri, int consensus_method) smartlist_add_asprintf(chunks, "a %s\n", fmt_addrport(&ri->ipv6_addr, ri->ipv6_orport)); - if (family) - smartlist_add_asprintf(chunks, "family %s\n", family); + if (family) { + if (consensus_method < MIN_METHOD_FOR_CANONICAL_FAMILIES_IN_MICRODESCS) { + smartlist_add_asprintf(chunks, "family %s\n", family); + } else { + const uint8_t *id = (const uint8_t *)ri->cache_info.identity_digest; + char *canonical_family = nodefamily_canonicalize(family, id, 0); + smartlist_add_asprintf(chunks, "family %s\n", canonical_family); + tor_free(canonical_family); + } + } if (summary && strcmp(summary, "reject 1-65535")) smartlist_add_asprintf(chunks, "p %s\n", summary); @@ -3891,7 +3907,10 @@ static const struct consensus_method_range_t { int high; } microdesc_consensus_methods[] = { {MIN_SUPPORTED_CONSENSUS_METHOD, MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC - 1}, - {MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC, MAX_SUPPORTED_CONSENSUS_METHOD}, + {MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC, + MIN_METHOD_FOR_CANONICAL_FAMILIES_IN_MICRODESCS - 1}, + {MIN_METHOD_FOR_CANONICAL_FAMILIES_IN_MICRODESCS, + MAX_SUPPORTED_CONSENSUS_METHOD}, {-1, -1} }; @@ -4365,6 +4384,23 @@ clear_status_flags_on_sybil(routerstatus_t *rs) * forget to add it to this clause. */ } +/** Space-separated list of all the flags that we will always vote on. */ +const char DIRVOTE_UNIVERSAL_FLAGS[] = + "Authority " + "Exit " + "Fast " + "Guard " + "HSDir " + "Stable " + "StaleDesc " + "V2Dir " + "Valid"; +/** Space-separated list of all flags that we may or may not vote on, + * depending on our configuration. */ +const char DIRVOTE_OPTIONAL_FLAGS[] = + "BadExit " + "Running"; + /** Return a new networkstatus_t* containing our current opinion. (For v3 * authorities) */ networkstatus_t * @@ -4613,7 +4649,7 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, v3_out->known_flags = smartlist_new(); smartlist_split_string(v3_out->known_flags, - "Authority Exit Fast Guard Stable V2Dir Valid HSDir", + DIRVOTE_UNIVERSAL_FLAGS, 0, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); if (vote_on_reachability) smartlist_add_strdup(v3_out->known_flags, "Running"); diff --git a/src/feature/dirauth/dirvote.h b/src/feature/dirauth/dirvote.h index a21e9f3455..85f4800700 100644 --- a/src/feature/dirauth/dirvote.h +++ b/src/feature/dirauth/dirvote.h @@ -57,7 +57,7 @@ #define MIN_SUPPORTED_CONSENSUS_METHOD 25 /** The highest consensus method that we currently support. */ -#define MAX_SUPPORTED_CONSENSUS_METHOD 28 +#define MAX_SUPPORTED_CONSENSUS_METHOD 29 /** Lowest consensus method where authorities vote on required/recommended * protocols. */ @@ -79,6 +79,12 @@ * addresses. See #23828 and #20916. */ #define MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC 28 +/** + * Lowest consensus method where microdescriptor lines are put in canonical + * form for improved compressibility and ease of storage. See proposal 298. + **/ +#define MIN_METHOD_FOR_CANONICAL_FAMILIES_IN_MICRODESCS 29 + /** Default bandwidth to clip unmeasured bandwidths to using method >= * MIN_METHOD_TO_CLIP_UNMEASURED_BW. (This is not a consensus method; do not * get confused with the above macros.) */ @@ -92,6 +98,9 @@ /** Maximum size of a line in a vote. */ #define MAX_BW_FILE_HEADERS_LINE_LEN 1024 +extern const char DIRVOTE_UNIVERSAL_FLAGS[]; +extern const char DIRVOTE_OPTIONAL_FLAGS[]; + /* * Public API. Used outside of the dirauth subsystem. * diff --git a/src/feature/dirauth/process_descs.c b/src/feature/dirauth/process_descs.c index c379f25bdd..dca87b3eaf 100644 --- a/src/feature/dirauth/process_descs.c +++ b/src/feature/dirauth/process_descs.c @@ -519,7 +519,8 @@ WRA_MORE_SEVERE(was_router_added_t a, was_router_added_t b) /** As for dirserv_add_descriptor(), but accepts multiple documents, and * returns the most severe error that occurred for any one of them. */ was_router_added_t -dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose, +dirserv_add_multiple_descriptors(const char *desc, size_t desclen, + uint8_t purpose, const char *source, const char **msg) { @@ -536,6 +537,11 @@ dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose, r=ROUTER_ADDED_SUCCESSFULLY; /*Least severe return value. */ + if (!string_is_utf8_no_bom(desc, desclen)) { + *msg = "descriptor(s) or extrainfo(s) not valid UTF-8 or had BOM."; + return ROUTER_AUTHDIR_REJECTS; + } + format_iso_time(time_buf, now); if (tor_snprintf(annotation_buf, sizeof(annotation_buf), "@uploaded-at %s\n" @@ -552,7 +558,7 @@ dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose, s = desc; list = smartlist_new(); - if (!router_parse_list_from_string(&s, NULL, list, SAVED_NOWHERE, 0, 0, + if (!router_parse_list_from_string(&s, s+desclen, list, SAVED_NOWHERE, 0, 0, annotation_buf, NULL)) { SMARTLIST_FOREACH(list, routerinfo_t *, ri, { msg_out = NULL; @@ -568,7 +574,7 @@ dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose, smartlist_clear(list); s = desc; - if (!router_parse_list_from_string(&s, NULL, list, SAVED_NOWHERE, 1, 0, + if (!router_parse_list_from_string(&s, s+desclen, list, SAVED_NOWHERE, 1, 0, NULL, NULL)) { SMARTLIST_FOREACH(list, extrainfo_t *, ei, { msg_out = NULL; diff --git a/src/feature/dirauth/process_descs.h b/src/feature/dirauth/process_descs.h index ad9d5c3d4c..5a0914acd8 100644 --- a/src/feature/dirauth/process_descs.h +++ b/src/feature/dirauth/process_descs.h @@ -17,7 +17,8 @@ void dirserv_free_fingerprint_list(void); int dirserv_add_own_fingerprint(crypto_pk_t *pk); enum was_router_added_t dirserv_add_multiple_descriptors( - const char *desc, uint8_t purpose, + const char *desc, size_t desclen, + uint8_t purpose, const char *source, const char **msg); enum was_router_added_t dirserv_add_descriptor(routerinfo_t *ri, diff --git a/src/feature/dirauth/shared_random_state.c b/src/feature/dirauth/shared_random_state.c index 7ae4a5dc86..8c5b28b7cd 100644 --- a/src/feature/dirauth/shared_random_state.c +++ b/src/feature/dirauth/shared_random_state.c @@ -22,6 +22,7 @@ #include "feature/dirauth/shared_random_state.h" #include "feature/dircommon/voting_schedule.h" #include "lib/encoding/confline.h" +#include "lib/version/torversion.h" #include "app/config/or_state_st.h" diff --git a/src/feature/dirauth/voteflags.c b/src/feature/dirauth/voteflags.c index 4a24dcb50d..aab322d96f 100644 --- a/src/feature/dirauth/voteflags.c +++ b/src/feature/dirauth/voteflags.c @@ -95,7 +95,7 @@ real_uptime(const routerinfo_t *router, time_t now) */ static int dirserv_thinks_router_is_unreliable(time_t now, - routerinfo_t *router, + const routerinfo_t *router, int need_uptime, int need_capacity) { if (need_uptime) { @@ -541,7 +541,7 @@ dirserv_set_router_is_running(routerinfo_t *router, time_t now) void set_routerstatus_from_routerinfo(routerstatus_t *rs, node_t *node, - routerinfo_t *ri, + const routerinfo_t *ri, time_t now, int listbadexits) { @@ -593,6 +593,10 @@ set_routerstatus_from_routerinfo(routerstatus_t *rs, rs->or_port = ri->or_port; rs->dir_port = ri->dir_port; rs->is_v2_dir = ri->supports_tunnelled_dir_requests; + + rs->is_staledesc = + (ri->cache_info.published_on + DESC_IS_STALE_INTERVAL) < now; + if (options->AuthDirHasIPv6Connectivity == 1 && !tor_addr_is_null(&ri->ipv6_addr) && node->last_reachable6 >= now - REACHABLE_TIMEOUT) { diff --git a/src/feature/dirauth/voteflags.h b/src/feature/dirauth/voteflags.h index 2f0e061ea4..8dce9fbb04 100644 --- a/src/feature/dirauth/voteflags.h +++ b/src/feature/dirauth/voteflags.h @@ -19,12 +19,16 @@ int running_long_enough_to_decide_unreachable(void); void set_routerstatus_from_routerinfo(routerstatus_t *rs, node_t *node, - routerinfo_t *ri, time_t now, + const routerinfo_t *ri, + time_t now, int listbadexits); void dirserv_compute_performance_thresholds(digestmap_t *omit_as_sybil); #ifdef VOTEFLAGS_PRIVATE +/** Any descriptor older than this age causes the authorities to set the + * StaleDesc flag. */ +#define DESC_IS_STALE_INTERVAL (18*60*60) STATIC void dirserv_set_routerstatus_testing(routerstatus_t *rs); #endif diff --git a/src/feature/dircache/consdiffmgr.c b/src/feature/dircache/consdiffmgr.c index e79aad6efb..8ecab5ca69 100644 --- a/src/feature/dircache/consdiffmgr.c +++ b/src/feature/dircache/consdiffmgr.c @@ -189,6 +189,7 @@ static consdiff_cfg_t consdiff_cfg = { static int consdiffmgr_ensure_space_for_files(int n); static int consensus_queue_compression_work(const char *consensus, + size_t consensus_len, const networkstatus_t *as_parsed); static int consensus_diff_queue_diff_work(consensus_cache_entry_t *diff_from, consensus_cache_entry_t *diff_to); @@ -509,8 +510,25 @@ get_max_age_to_cache(void) MAX_MAX_AGE_TO_CACHE); } +#ifdef TOR_UNIT_TESTS +/** As consdiffmgr_add_consensus, but requires a nul-terminated input. For + * testing. */ +int +consdiffmgr_add_consensus_nulterm(const char *consensus, + const networkstatus_t *as_parsed) +{ + size_t len = strlen(consensus); + /* make a non-nul-terminated copy so that we can have a better chance + * of catching errors. */ + char *ctmp = tor_memdup(consensus, len); + int r = consdiffmgr_add_consensus(ctmp, len, as_parsed); + tor_free(ctmp); + return r; +} +#endif + /** - * Given a string containing a networkstatus consensus, and the results of + * Given a buffer containing a networkstatus consensus, and the results of * having parsed that consensus, add that consensus to the cache if it is not * already present and not too old. Create new consensus diffs from or to * that consensus as appropriate. @@ -519,6 +537,7 @@ get_max_age_to_cache(void) */ int consdiffmgr_add_consensus(const char *consensus, + size_t consensus_len, const networkstatus_t *as_parsed) { if (BUG(consensus == NULL) || BUG(as_parsed == NULL)) @@ -544,7 +563,7 @@ consdiffmgr_add_consensus(const char *consensus, } /* We don't have it. Add it to the cache. */ - return consensus_queue_compression_work(consensus, as_parsed); + return consensus_queue_compression_work(consensus, consensus_len, as_parsed); } /** @@ -1387,19 +1406,21 @@ typedef struct consensus_diff_worker_job_t { } consensus_diff_worker_job_t; /** Given a consensus_cache_entry_t, check whether it has a label claiming - * that it was compressed. If so, uncompress its contents into <b>out</b> and - * set <b>outlen</b> to hold their size. If not, just copy the body into - * <b>out</b> and set <b>outlen</b> to its length. Return 0 on success, - * -1 on failure. - * - * In all cases, the output is nul-terminated. */ + * that it was compressed. If so, uncompress its contents into *<b>out</b> and + * set <b>outlen</b> to hold their size, and set *<b>owned_out</b> to a pointer + * that the caller will need to free. If not, just set *<b>out</b> and + * <b>outlen</b> to its extent in memory. Return 0 on success, -1 on failure. + **/ STATIC int -uncompress_or_copy(char **out, size_t *outlen, - consensus_cache_entry_t *ent) +uncompress_or_set_ptr(const char **out, size_t *outlen, + char **owned_out, + consensus_cache_entry_t *ent) { const uint8_t *body; size_t bodylen; + *owned_out = NULL; + if (consensus_cache_entry_get_body(ent, &body, &bodylen) < 0) return -1; @@ -1410,8 +1431,17 @@ uncompress_or_copy(char **out, size_t *outlen, if (lv_compression) method = compression_method_get_by_name(lv_compression); - return tor_uncompress(out, outlen, (const char *)body, bodylen, + int rv; + if (method == NO_METHOD) { + *out = (const char *)body; + *outlen = bodylen; + rv = 0; + } else { + rv = tor_uncompress(owned_out, outlen, (const char *)body, bodylen, method, 1, LOG_WARN); + *out = *owned_out; + } + return rv; } /** @@ -1478,16 +1508,17 @@ consensus_diff_worker_threadfn(void *state_, void *work_) char *consensus_diff; { - char *diff_from_nt = NULL, *diff_to_nt = NULL; + const char *diff_from_nt = NULL, *diff_to_nt = NULL; + char *owned1 = NULL, *owned2 = NULL; size_t diff_from_nt_len, diff_to_nt_len; - if (uncompress_or_copy(&diff_from_nt, &diff_from_nt_len, - job->diff_from) < 0) { + if (uncompress_or_set_ptr(&diff_from_nt, &diff_from_nt_len, &owned1, + job->diff_from) < 0) { return WQ_RPL_REPLY; } - if (uncompress_or_copy(&diff_to_nt, &diff_to_nt_len, - job->diff_to) < 0) { - tor_free(diff_from_nt); + if (uncompress_or_set_ptr(&diff_to_nt, &diff_to_nt_len, &owned2, + job->diff_to) < 0) { + tor_free(owned1); return WQ_RPL_REPLY; } tor_assert(diff_from_nt); @@ -1496,9 +1527,12 @@ consensus_diff_worker_threadfn(void *state_, void *work_) // XXXX ugh; this is going to calculate the SHA3 of both its // XXXX inputs again, even though we already have that. Maybe it's time // XXXX to change the API here? - consensus_diff = consensus_diff_generate(diff_from_nt, diff_to_nt); - tor_free(diff_from_nt); - tor_free(diff_to_nt); + consensus_diff = consensus_diff_generate(diff_from_nt, + diff_from_nt_len, + diff_to_nt, + diff_to_nt_len); + tor_free(owned1); + tor_free(owned2); } if (!consensus_diff) { /* Couldn't generate consensus; we'll leave the reply blank. */ @@ -1746,8 +1780,8 @@ consensus_compress_worker_threadfn(void *state_, void *work_) (const uint8_t *)consensus, bodylen); { const char *start, *end; - if (router_get_networkstatus_v3_signed_boundaries(consensus, - &start, &end) < 0) { + if (router_get_networkstatus_v3_signed_boundaries(consensus, bodylen, + &start, &end) < 0) { start = consensus; end = consensus+bodylen; } @@ -1811,14 +1845,15 @@ static int background_compression = 0; */ static int consensus_queue_compression_work(const char *consensus, + size_t consensus_len, const networkstatus_t *as_parsed) { tor_assert(consensus); tor_assert(as_parsed); consensus_compress_worker_job_t *job = tor_malloc_zero(sizeof(*job)); - job->consensus = tor_strdup(consensus); - job->consensus_len = strlen(consensus); + job->consensus = tor_memdup_nulterm(consensus, consensus_len); + job->consensus_len = strlen(job->consensus); job->flavor = as_parsed->flavor; char va_str[ISO_TIME_LEN+1]; diff --git a/src/feature/dircache/consdiffmgr.h b/src/feature/dircache/consdiffmgr.h index 66c3d65002..011c8799d6 100644 --- a/src/feature/dircache/consdiffmgr.h +++ b/src/feature/dircache/consdiffmgr.h @@ -22,6 +22,7 @@ typedef struct consdiff_cfg_t { struct consensus_cache_entry_t; // from conscache.h int consdiffmgr_add_consensus(const char *consensus, + size_t consensus_len, const networkstatus_t *as_parsed); consdiff_status_t consdiffmgr_find_consensus( @@ -68,8 +69,14 @@ STATIC consensus_cache_entry_t *cdm_cache_lookup_consensus( STATIC int cdm_entry_get_sha3_value(uint8_t *digest_out, consensus_cache_entry_t *ent, const char *label); -STATIC int uncompress_or_copy(char **out, size_t *outlen, - consensus_cache_entry_t *ent); +STATIC int uncompress_or_set_ptr(const char **out, size_t *outlen, + char **owned_out, + consensus_cache_entry_t *ent); #endif /* defined(CONSDIFFMGR_PRIVATE) */ +#ifdef TOR_UNIT_TESTS +int consdiffmgr_add_consensus_nulterm(const char *consensus, + const networkstatus_t *as_parsed); +#endif + #endif /* !defined(TOR_CONSDIFFMGR_H) */ diff --git a/src/feature/dircache/dircache.c b/src/feature/dircache/dircache.c index 872a88018f..4032223db4 100644 --- a/src/feature/dircache/dircache.c +++ b/src/feature/dircache/dircache.c @@ -166,22 +166,16 @@ write_http_response_header_impl(dir_connection_t *conn, ssize_t length, buf_free(buf); } -/** As write_http_response_header_impl, but sets encoding and content-typed - * based on whether the response will be <b>compressed</b> or not. */ +/** As write_http_response_header_impl, but translates method into + * encoding */ static void write_http_response_headers(dir_connection_t *conn, ssize_t length, compress_method_t method, const char *extra_headers, long cache_lifetime) { - const char *methodname = compression_method_get_name(method); - const char *doctype; - if (method == NO_METHOD) - doctype = "text/plain"; - else - doctype = "application/octet-stream"; write_http_response_header_impl(conn, length, - doctype, - methodname, + "text/plain", + compression_method_get_name(method), extra_headers, cache_lifetime); } @@ -1608,8 +1602,8 @@ directory_handle_command_post,(dir_connection_t *conn, const char *headers, const char *msg = "[None]"; uint8_t purpose = authdir_mode_bridge(options) ? ROUTER_PURPOSE_BRIDGE : ROUTER_PURPOSE_GENERAL; - was_router_added_t r = dirserv_add_multiple_descriptors(body, purpose, - conn->base_.address, &msg); + was_router_added_t r = dirserv_add_multiple_descriptors(body, body_len, + purpose, conn->base_.address, &msg); tor_assert(msg); if (r == ROUTER_ADDED_SUCCESSFULLY) { diff --git a/src/feature/dircache/dirserv.c b/src/feature/dircache/dirserv.c index 57178cd506..4366000e2e 100644 --- a/src/feature/dircache/dirserv.c +++ b/src/feature/dircache/dirserv.c @@ -234,6 +234,7 @@ free_cached_dir_(void *_d) * validation is performed. */ void dirserv_set_cached_consensus_networkstatus(const char *networkstatus, + size_t networkstatus_len, const char *flavor_name, const common_digests_t *digests, const uint8_t *sha3_as_signed, @@ -244,7 +245,9 @@ dirserv_set_cached_consensus_networkstatus(const char *networkstatus, if (!cached_consensuses) cached_consensuses = strmap_new(); - new_networkstatus = new_cached_dir(tor_strdup(networkstatus), published); + new_networkstatus = + new_cached_dir(tor_memdup_nulterm(networkstatus, networkstatus_len), + published); memcpy(&new_networkstatus->digests, digests, sizeof(common_digests_t)); memcpy(&new_networkstatus->digest_sha3_as_signed, sha3_as_signed, DIGEST256_LEN); diff --git a/src/feature/dircache/dirserv.h b/src/feature/dircache/dirserv.h index 41e1376688..aa1e2494ca 100644 --- a/src/feature/dircache/dirserv.h +++ b/src/feature/dircache/dirserv.h @@ -84,6 +84,7 @@ int directory_too_idle_to_fetch_descriptors(const or_options_t *options, cached_dir_t *dirserv_get_consensus(const char *flavor_name); void dirserv_set_cached_consensus_networkstatus(const char *consensus, + size_t consensus_len, const char *flavor_name, const common_digests_t *digests, const uint8_t *sha3_as_signed, diff --git a/src/feature/dirclient/dirclient.c b/src/feature/dirclient/dirclient.c index f6a712e429..cd88fa5ebf 100644 --- a/src/feature/dirclient/dirclient.c +++ b/src/feature/dirclient/dirclient.c @@ -2205,24 +2205,31 @@ handle_response_fetch_consensus(dir_connection_t *conn, if (looks_like_a_consensus_diff(body, body_len)) { /* First find our previous consensus. Maybe it's in ram, maybe not. */ cached_dir_t *cd = dirserv_get_consensus(flavname); - const char *consensus_body; - char *owned_consensus = NULL; + const char *consensus_body = NULL; + size_t consensus_body_len; + tor_mmap_t *mapped_consensus = NULL; if (cd) { consensus_body = cd->dir; + consensus_body_len = cd->dir_len; } else { - owned_consensus = networkstatus_read_cached_consensus(flavname); - consensus_body = owned_consensus; + mapped_consensus = networkstatus_map_cached_consensus(flavname); + if (mapped_consensus) { + consensus_body = mapped_consensus->data; + consensus_body_len = mapped_consensus->size; + } } if (!consensus_body) { log_warn(LD_DIR, "Received a consensus diff, but we can't find " "any %s-flavored consensus in our current cache.",flavname); + tor_munmap_file(mapped_consensus); networkstatus_consensus_download_failed(0, flavname); // XXXX if this happens too much, see below return -1; } - new_consensus = consensus_diff_apply(consensus_body, body); - tor_free(owned_consensus); + new_consensus = consensus_diff_apply(consensus_body, consensus_body_len, + body, body_len); + tor_munmap_file(mapped_consensus); if (new_consensus == NULL) { log_warn(LD_DIR, "Could not apply consensus diff received from server " "'%s:%d'", conn->base_.address, conn->base_.port); @@ -2244,7 +2251,9 @@ handle_response_fetch_consensus(dir_connection_t *conn, sourcename = "downloaded"; } - if ((r=networkstatus_set_current_consensus(consensus, flavname, 0, + if ((r=networkstatus_set_current_consensus(consensus, + strlen(consensus), + flavname, 0, conn->identity_digest))<0) { log_fn(r<-1?LOG_WARN:LOG_INFO, LD_DIR, "Unable to load %s consensus directory %s from " diff --git a/src/feature/dircommon/consdiff.c b/src/feature/dircommon/consdiff.c index f8ced8503f..15266f77bf 100644 --- a/src/feature/dircommon/consdiff.c +++ b/src/feature/dircommon/consdiff.c @@ -101,11 +101,11 @@ smartlist_add_linecpy(smartlist_t *lst, memarea_t *area, const char *s) /* This is a separate, mockable function so that we can override it when * fuzzing. */ MOCK_IMPL(STATIC int, -consensus_compute_digest,(const char *cons, +consensus_compute_digest,(const char *cons, size_t len, consensus_digest_t *digest_out)) { int r = crypto_digest256((char*)digest_out->sha3_256, - cons, strlen(cons), DIGEST_SHA3_256); + cons, len, DIGEST_SHA3_256); return r; } @@ -114,11 +114,11 @@ consensus_compute_digest,(const char *cons, /* This is a separate, mockable function so that we can override it when * fuzzing. */ MOCK_IMPL(STATIC int, -consensus_compute_digest_as_signed,(const char *cons, +consensus_compute_digest_as_signed,(const char *cons, size_t len, consensus_digest_t *digest_out)) { return router_get_networkstatus_v3_sha3_as_signed(digest_out->sha3_256, - cons); + cons, len); } /** Return true iff <b>d1</b> and <b>d2</b> contain the same digest */ @@ -1229,7 +1229,8 @@ consdiff_apply_diff(const smartlist_t *cons1, cons2_str = consensus_join_lines(cons2); consensus_digest_t cons2_digests; - if (consensus_compute_digest(cons2_str, &cons2_digests) < 0) { + if (consensus_compute_digest(cons2_str, strlen(cons2_str), + &cons2_digests) < 0) { /* LCOV_EXCL_START -- digest can't fail */ log_warn(LD_CONSDIFF, "Could not compute digests of the consensus " "resulting from applying a consensus diff."); @@ -1283,12 +1284,13 @@ consdiff_apply_diff(const smartlist_t *cons1, * generated cdlines will become invalid. */ STATIC int -consensus_split_lines(smartlist_t *out, const char *s, memarea_t *area) +consensus_split_lines(smartlist_t *out, + const char *s, size_t len, + memarea_t *area) { - const char *end_of_str = s + strlen(s); - tor_assert(*end_of_str == '\0'); + const char *end_of_str = s + len; - while (*s) { + while (s < end_of_str) { const char *eol = memchr(s, '\n', end_of_str - s); if (!eol) { /* File doesn't end with newline. */ @@ -1334,25 +1336,25 @@ consensus_join_lines(const smartlist_t *inp) * success, retun a newly allocated string containing that diff. On failure, * return NULL. */ char * -consensus_diff_generate(const char *cons1, - const char *cons2) +consensus_diff_generate(const char *cons1, size_t cons1len, + const char *cons2, size_t cons2len) { consensus_digest_t d1, d2; smartlist_t *lines1 = NULL, *lines2 = NULL, *result_lines = NULL; int r1, r2; char *result = NULL; - r1 = consensus_compute_digest_as_signed(cons1, &d1); - r2 = consensus_compute_digest(cons2, &d2); + r1 = consensus_compute_digest_as_signed(cons1, cons1len, &d1); + r2 = consensus_compute_digest(cons2, cons2len, &d2); if (BUG(r1 < 0 || r2 < 0)) return NULL; // LCOV_EXCL_LINE memarea_t *area = memarea_new(); lines1 = smartlist_new(); lines2 = smartlist_new(); - if (consensus_split_lines(lines1, cons1, area) < 0) + if (consensus_split_lines(lines1, cons1, cons1len, area) < 0) goto done; - if (consensus_split_lines(lines2, cons2, area) < 0) + if (consensus_split_lines(lines2, cons2, cons2len, area) < 0) goto done; result_lines = consdiff_gen_diff(lines1, lines2, &d1, &d2, area); @@ -1375,7 +1377,9 @@ consensus_diff_generate(const char *cons1, * consensus. On failure, return NULL. */ char * consensus_diff_apply(const char *consensus, - const char *diff) + size_t consensus_len, + const char *diff, + size_t diff_len) { consensus_digest_t d1; smartlist_t *lines1 = NULL, *lines2 = NULL; @@ -1383,15 +1387,15 @@ consensus_diff_apply(const char *consensus, char *result = NULL; memarea_t *area = memarea_new(); - r1 = consensus_compute_digest_as_signed(consensus, &d1); + r1 = consensus_compute_digest_as_signed(consensus, consensus_len, &d1); if (BUG(r1 < 0)) return NULL; // LCOV_EXCL_LINE lines1 = smartlist_new(); lines2 = smartlist_new(); - if (consensus_split_lines(lines1, consensus, area) < 0) + if (consensus_split_lines(lines1, consensus, consensus_len, area) < 0) goto done; - if (consensus_split_lines(lines2, diff, area) < 0) + if (consensus_split_lines(lines2, diff, diff_len, area) < 0) goto done; result = consdiff_apply_diff(lines1, lines2, &d1); diff --git a/src/feature/dircommon/consdiff.h b/src/feature/dircommon/consdiff.h index a5e4ba5cbf..eb7c9f9fe0 100644 --- a/src/feature/dircommon/consdiff.h +++ b/src/feature/dircommon/consdiff.h @@ -7,10 +7,10 @@ #include "core/or/or.h" -char *consensus_diff_generate(const char *cons1, - const char *cons2); -char *consensus_diff_apply(const char *consensus, - const char *diff); +char *consensus_diff_generate(const char *cons1, size_t cons1len, + const char *cons2, size_t cons2len); +char *consensus_diff_apply(const char *consensus, size_t consensus_len, + const char *diff, size_t diff_len); int looks_like_a_consensus_diff(const char *document, size_t len); @@ -78,7 +78,8 @@ STATIC int smartlist_slice_string_pos(const smartlist_slice_t *slice, STATIC void set_changed(bitarray_t *changed1, bitarray_t *changed2, const smartlist_slice_t *slice1, const smartlist_slice_t *slice2); -STATIC int consensus_split_lines(smartlist_t *out, const char *s, +STATIC int consensus_split_lines(smartlist_t *out, + const char *s, size_t len, struct memarea_t *area); STATIC void smartlist_add_linecpy(smartlist_t *lst, struct memarea_t *area, const char *s); @@ -86,10 +87,10 @@ STATIC int lines_eq(const cdline_t *a, const cdline_t *b); STATIC int line_str_eq(const cdline_t *a, const char *b); MOCK_DECL(STATIC int, - consensus_compute_digest,(const char *cons, + consensus_compute_digest,(const char *cons, size_t len, consensus_digest_t *digest_out)); MOCK_DECL(STATIC int, - consensus_compute_digest_as_signed,(const char *cons, + consensus_compute_digest_as_signed,(const char *cons, size_t len, consensus_digest_t *digest_out)); MOCK_DECL(STATIC int, consensus_digest_eq,(const uint8_t *d1, diff --git a/src/feature/dirparse/authcert_parse.c b/src/feature/dirparse/authcert_parse.c index 2ba46bb8fa..334baf8b1a 100644 --- a/src/feature/dirparse/authcert_parse.c +++ b/src/feature/dirparse/authcert_parse.c @@ -24,7 +24,8 @@ static token_rule_t dir_key_certificate_table[] = { /** Parse a key certificate from <b>s</b>; point <b>end-of-string</b> to * the first character after the certificate. */ authority_cert_t * -authority_cert_parse_from_string(const char *s, const char **end_of_string) +authority_cert_parse_from_string(const char *s, size_t maxlen, + const char **end_of_string) { /** Reject any certificate at least this big; it is probably an overflow, an * attack, a bug, or some other nonsense. */ @@ -35,24 +36,25 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) char digest[DIGEST_LEN]; directory_token_t *tok; char fp_declared[DIGEST_LEN]; - char *eos; + const char *eos; size_t len; int found; memarea_t *area = NULL; + const char *end_of_s = s + maxlen; const char *s_dup = s; - s = eat_whitespace(s); - eos = strstr(s, "\ndir-key-certification"); + s = eat_whitespace_eos(s, end_of_s); + eos = tor_memstr(s, end_of_s - s, "\ndir-key-certification"); if (! eos) { log_warn(LD_DIR, "No signature found on key certificate"); return NULL; } - eos = strstr(eos, "\n-----END SIGNATURE-----\n"); + eos = tor_memstr(eos, end_of_s - eos, "\n-----END SIGNATURE-----\n"); if (! eos) { log_warn(LD_DIR, "No end-of-signature found on key certificate"); return NULL; } - eos = strchr(eos+2, '\n'); + eos = memchr(eos+2, '\n', end_of_s - (eos+2)); tor_assert(eos); ++eos; len = eos - s; @@ -69,7 +71,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) log_warn(LD_DIR, "Error tokenizing key certificate"); goto err; } - if (router_get_hash_impl(s, strlen(s), digest, "dir-key-certificate-version", + if (router_get_hash_impl(s, eos - s, digest, "dir-key-certificate-version", "\ndir-key-certification", '\n', DIGEST_SHA1) < 0) goto err; tok = smartlist_get(tokens, 0); diff --git a/src/feature/dirparse/authcert_parse.h b/src/feature/dirparse/authcert_parse.h index f63525e04d..e4e9fec993 100644 --- a/src/feature/dirparse/authcert_parse.h +++ b/src/feature/dirparse/authcert_parse.h @@ -13,6 +13,7 @@ #define TOR_AUTHCERT_PARSE_H authority_cert_t *authority_cert_parse_from_string(const char *s, + size_t maxlen, const char **end_of_string); #endif /* !defined(TOR_AUTHCERT_PARSE_H) */ diff --git a/src/feature/dirparse/microdesc_parse.c b/src/feature/dirparse/microdesc_parse.c index aebff5a35f..165945e392 100644 --- a/src/feature/dirparse/microdesc_parse.c +++ b/src/feature/dirparse/microdesc_parse.c @@ -18,6 +18,7 @@ #include "feature/dirparse/routerparse.h" #include "feature/nodelist/microdesc.h" #include "feature/nodelist/nickname.h" +#include "feature/nodelist/nodefamily.h" #include "feature/relay/router.h" #include "lib/crypt_ops/crypto_curve25519.h" #include "lib/crypt_ops/crypto_ed25519.h" @@ -32,7 +33,7 @@ static token_rule_t microdesc_token_table[] = { T01("ntor-onion-key", K_ONION_KEY_NTOR, GE(1), NO_OBJ ), T0N("id", K_ID, GE(2), NO_OBJ ), T0N("a", K_A, GE(1), NO_OBJ ), - T01("family", K_FAMILY, ARGS, NO_OBJ ), + T01("family", K_FAMILY, CONCAT_ARGS, NO_OBJ ), T01("p", K_P, CONCAT_ARGS, NO_OBJ ), T01("p6", K_P6, CONCAT_ARGS, NO_OBJ ), A01("@last-listed", A_LAST_LISTED, CONCAT_ARGS, NO_OBJ ), @@ -176,8 +177,8 @@ microdescs_parse_from_string(const char *s, const char *eos, "Relay's onion key had invalid exponent."); goto next; } - router_set_rsa_onion_pkey(tok->key, &md->onion_pkey, - &md->onion_pkey_len); + md->onion_pkey = tor_memdup(tok->object_body, tok->object_size); + md->onion_pkey_len = tok->object_size; crypto_pk_free(tok->key); if ((tok = find_opt_by_keyword(tokens, K_ONION_KEY_NTOR))) { @@ -222,16 +223,9 @@ microdescs_parse_from_string(const char *s, const char *eos, } if ((tok = find_opt_by_keyword(tokens, K_FAMILY))) { - int i; - md->family = smartlist_new(); - for (i=0;i<tok->n_args;++i) { - if (!is_legal_nickname_or_hexdigest(tok->args[i])) { - log_warn(LD_DIR, "Illegal nickname %s in family line", - escaped(tok->args[i])); - goto next; - } - smartlist_add_strdup(md->family, tok->args[i]); - } + md->family = nodefamily_parse(tok->args[0], + NULL, + NF_WARN_MALFORMED); } if ((tok = find_opt_by_keyword(tokens, K_P))) { diff --git a/src/feature/dirparse/ns_parse.c b/src/feature/dirparse/ns_parse.c index 72299e8071..e0cdb2d46d 100644 --- a/src/feature/dirparse/ns_parse.c +++ b/src/feature/dirparse/ns_parse.c @@ -151,10 +151,11 @@ static token_rule_t networkstatus_vote_footer_token_table[] = { * -1. */ int router_get_networkstatus_v3_signed_boundaries(const char *s, + size_t len, const char **start_out, const char **end_out) { - return router_get_hash_impl_helper(s, strlen(s), + return router_get_hash_impl_helper(s, len, "network-status-version", "\ndirectory-signature", ' ', LOG_INFO, @@ -166,12 +167,13 @@ router_get_networkstatus_v3_signed_boundaries(const char *s, * signed portion can be identified. Return 0 on success, -1 on failure. */ int router_get_networkstatus_v3_sha3_as_signed(uint8_t *digest_out, - const char *s) + const char *s, size_t len) { const char *start, *end; - if (router_get_networkstatus_v3_signed_boundaries(s, &start, &end) < 0) { + if (router_get_networkstatus_v3_signed_boundaries(s, len, + &start, &end) < 0) { start = s; - end = s + strlen(s); + end = s + len; } tor_assert(start); tor_assert(end); @@ -182,9 +184,10 @@ router_get_networkstatus_v3_sha3_as_signed(uint8_t *digest_out, /** Set <b>digests</b> to all the digests of the consensus document in * <b>s</b> */ int -router_get_networkstatus_v3_hashes(const char *s, common_digests_t *digests) +router_get_networkstatus_v3_hashes(const char *s, size_t len, + common_digests_t *digests) { - return router_get_hashes_impl(s,strlen(s),digests, + return router_get_hashes_impl(s, len, digests, "network-status-version", "\ndirectory-signature", ' '); @@ -195,13 +198,13 @@ router_get_networkstatus_v3_hashes(const char *s, common_digests_t *digests) * return the start of the directory footer, or the next directory signature. * If none is found, return the end of the string. */ static inline const char * -find_start_of_next_routerstatus(const char *s) +find_start_of_next_routerstatus(const char *s, const char *s_eos) { const char *eos, *footer, *sig; - if ((eos = strstr(s, "\nr "))) + if ((eos = tor_memstr(s, s_eos - s, "\nr "))) ++eos; else - eos = s + strlen(s); + eos = s_eos; footer = tor_memstr(s, eos-s, "\ndirectory-footer"); sig = tor_memstr(s, eos-s, "\ndirectory-signature"); @@ -289,7 +292,8 @@ routerstatus_parse_guardfraction(const char *guardfraction_str, **/ STATIC routerstatus_t * routerstatus_parse_entry_from_string(memarea_t *area, - const char **s, smartlist_t *tokens, + const char **s, const char *s_eos, + smartlist_t *tokens, networkstatus_t *vote, vote_routerstatus_t *vote_rs, int consensus_method, @@ -308,7 +312,7 @@ routerstatus_parse_entry_from_string(memarea_t *area, flav = FLAV_NS; tor_assert(flav == FLAV_NS || flav == FLAV_MICRODESC); - eos = find_start_of_next_routerstatus(*s); + eos = find_start_of_next_routerstatus(*s, s_eos); if (tokenize_string(area,*s, eos, tokens, rtrstatus_token_table,0)) { log_warn(LD_DIR, "Error tokenizing router status"); @@ -430,6 +434,8 @@ routerstatus_parse_entry_from_string(memarea_t *area, rs->is_hs_dir = 1; } else if (!strcmp(tok->args[i], "V2Dir")) { rs->is_v2_dir = 1; + } else if (!strcmp(tok->args[i], "StaleDesc")) { + rs->is_staledesc = 1; } } /* These are implied true by having been included in a consensus made @@ -1051,7 +1057,9 @@ extract_shared_random_srvs(networkstatus_t *ns, smartlist_t *tokens) /** Parse a v3 networkstatus vote, opinion, or consensus (depending on * ns_type), from <b>s</b>, and return the result. Return NULL on failure. */ networkstatus_t * -networkstatus_parse_vote_from_string(const char *s, const char **eos_out, +networkstatus_parse_vote_from_string(const char *s, + size_t s_len, + const char **eos_out, networkstatus_type_t ns_type) { smartlist_t *tokens = smartlist_new(); @@ -1067,20 +1075,22 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, memarea_t *area = NULL, *rs_area = NULL; consensus_flavor_t flav = FLAV_NS; char *last_kwd=NULL; + const char *eos = s + s_len; tor_assert(s); if (eos_out) *eos_out = NULL; - if (router_get_networkstatus_v3_hashes(s, &ns_digests) || - router_get_networkstatus_v3_sha3_as_signed(sha3_as_signed, s)<0) { + if (router_get_networkstatus_v3_hashes(s, s_len, &ns_digests) || + router_get_networkstatus_v3_sha3_as_signed(sha3_as_signed, + s, s_len)<0) { log_warn(LD_DIR, "Unable to compute digest of network-status"); goto err; } area = memarea_new(); - end_of_header = find_start_of_next_routerstatus(s); + end_of_header = find_start_of_next_routerstatus(s, eos); if (tokenize_string(area, s, end_of_header, tokens, (ns_type == NS_TYPE_CONSENSUS) ? networkstatus_consensus_token_table : @@ -1111,10 +1121,12 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, if (ns_type != NS_TYPE_CONSENSUS) { const char *end_of_cert = NULL; - if (!(cert = strstr(s, "\ndir-key-certificate-version"))) + if (!(cert = tor_memstr(s, end_of_header - s, + "\ndir-key-certificate-version"))) goto err; ++cert; - ns->cert = authority_cert_parse_from_string(cert, &end_of_cert); + ns->cert = authority_cert_parse_from_string(cert, end_of_header - cert, + &end_of_cert); if (!ns->cert || !end_of_cert || end_of_cert > end_of_header) goto err; } @@ -1424,10 +1436,10 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, s = end_of_header; ns->routerstatus_list = smartlist_new(); - while (!strcmpstart(s, "r ")) { + while (eos - s >= 2 && fast_memeq(s, "r ", 2)) { if (ns->type != NS_TYPE_CONSENSUS) { vote_routerstatus_t *rs = tor_malloc_zero(sizeof(vote_routerstatus_t)); - if (routerstatus_parse_entry_from_string(rs_area, &s, rs_tokens, ns, + if (routerstatus_parse_entry_from_string(rs_area, &s, eos, rs_tokens, ns, rs, 0, 0)) { smartlist_add(ns->routerstatus_list, rs); } else { @@ -1435,7 +1447,8 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, } } else { routerstatus_t *rs; - if ((rs = routerstatus_parse_entry_from_string(rs_area, &s, rs_tokens, + if ((rs = routerstatus_parse_entry_from_string(rs_area, &s, eos, + rs_tokens, NULL, NULL, ns->consensus_method, flav))) { @@ -1480,10 +1493,10 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, /* Parse footer; check signature. */ footer_tokens = smartlist_new(); - if ((end_of_footer = strstr(s, "\nnetwork-status-version "))) + if ((end_of_footer = tor_memstr(s, eos-s, "\nnetwork-status-version "))) ++end_of_footer; else - end_of_footer = s + strlen(s); + end_of_footer = eos; if (tokenize_string(area,s, end_of_footer, footer_tokens, networkstatus_vote_footer_token_table, 0)) { log_warn(LD_DIR, "Error tokenizing network-status vote footer."); diff --git a/src/feature/dirparse/ns_parse.h b/src/feature/dirparse/ns_parse.h index 22438d73a7..85d9ded685 100644 --- a/src/feature/dirparse/ns_parse.h +++ b/src/feature/dirparse/ns_parse.h @@ -12,18 +12,19 @@ #ifndef TOR_NS_PARSE_H #define TOR_NS_PARSE_H -int router_get_networkstatus_v3_hashes(const char *s, +int router_get_networkstatus_v3_hashes(const char *s, size_t len, common_digests_t *digests); -int router_get_networkstatus_v3_signed_boundaries(const char *s, +int router_get_networkstatus_v3_signed_boundaries(const char *s, size_t len, const char **start_out, const char **end_out); int router_get_networkstatus_v3_sha3_as_signed(uint8_t *digest_out, - const char *s); + const char *s, size_t len); int compare_vote_routerstatus_entries(const void **_a, const void **_b); int networkstatus_verify_bw_weights(networkstatus_t *ns, int); enum networkstatus_type_t; networkstatus_t *networkstatus_parse_vote_from_string(const char *s, + size_t len, const char **eos_out, enum networkstatus_type_t ns_type); @@ -35,7 +36,8 @@ STATIC int routerstatus_parse_guardfraction(const char *guardfraction_str, struct memarea_t; STATIC routerstatus_t *routerstatus_parse_entry_from_string( struct memarea_t *area, - const char **s, smartlist_t *tokens, + const char **s, const char *eos, + smartlist_t *tokens, networkstatus_t *vote, vote_routerstatus_t *vote_rs, int consensus_method, diff --git a/src/feature/dirparse/parsecommon.c b/src/feature/dirparse/parsecommon.c index e00af0eea2..b53b0fe193 100644 --- a/src/feature/dirparse/parsecommon.c +++ b/src/feature/dirparse/parsecommon.c @@ -169,7 +169,6 @@ get_token_arguments(memarea_t *area, directory_token_t *tok, char *cp = mem; int j = 0; char *args[MAX_ARGS]; - memset(args, 0, sizeof(args)); while (*cp) { if (j == MAX_ARGS) return -1; @@ -266,7 +265,7 @@ get_next_token(memarea_t *area, * attack, a bug, or some other nonsense. */ #define MAX_LINE_LENGTH (128*1024) - const char *next, *eol, *obstart; + const char *next, *eol; size_t obname_len; int i; directory_token_t *tok; @@ -352,7 +351,6 @@ get_next_token(memarea_t *area, if (!eol || eol-*s<11 || strcmpstart(*s, "-----BEGIN ")) /* No object. */ goto check_object; - obstart = *s; /* Set obstart to start of object spec */ if (eol - *s <= 16 || memchr(*s+11,'\0',eol-*s-16) || /* no short lines, */ strcmp_len(eol-5, "-----", 5) || /* nuls or invalid endings */ (eol-*s) > MAX_UNPARSED_OBJECT_SIZE) { /* name too long */ @@ -383,22 +381,26 @@ get_next_token(memarea_t *area, if (next - *s > MAX_UNPARSED_OBJECT_SIZE) RET_ERR("Couldn't parse object: missing footer or object much too big."); - if (!strcmp(tok->object_type, "RSA PUBLIC KEY")) { /* If it's a public key */ - tok->key = crypto_pk_new(); - if (crypto_pk_read_public_key_from_string(tok->key, obstart, eol-obstart)) - RET_ERR("Couldn't parse public key."); - } else if (!strcmp(tok->object_type, "RSA PRIVATE KEY")) { /* private key */ - tok->key = crypto_pk_new(); - if (crypto_pk_read_private_key_from_string(tok->key, obstart, eol-obstart)) - RET_ERR("Couldn't parse private key."); - } else { /* If it's something else, try to base64-decode it */ + { int r; - tok->object_body = ALLOC(next-*s); /* really, this is too much RAM. */ - r = base64_decode(tok->object_body, next-*s, *s, next-*s); + size_t maxsize = base64_decode_maxsize(next-*s); + tok->object_body = ALLOC(maxsize); + r = base64_decode(tok->object_body, maxsize, *s, next-*s); if (r<0) RET_ERR("Malformed object: bad base64-encoded data"); tok->object_size = r; } + + if (!strcmp(tok->object_type, "RSA PUBLIC KEY")) { /* If it's a public key */ + tok->key = crypto_pk_asn1_decode(tok->object_body, tok->object_size); + if (! tok->key) + RET_ERR("Couldn't parse public key."); + } else if (!strcmp(tok->object_type, "RSA PRIVATE KEY")) { /* private key */ + tok->key = crypto_pk_asn1_decode_private(tok->object_body, + tok->object_size); + if (! tok->key) + RET_ERR("Couldn't parse private key."); + } *s = eol; check_object: diff --git a/src/feature/dirparse/routerparse.c b/src/feature/dirparse/routerparse.c index 2249ab7cbc..358f6e44e8 100644 --- a/src/feature/dirparse/routerparse.c +++ b/src/feature/dirparse/routerparse.c @@ -588,8 +588,8 @@ router_parse_entry_from_string(const char *s, const char *end, "Relay's onion key had invalid exponent."); goto err; } - router_set_rsa_onion_pkey(tok->key, &router->onion_pkey, - &router->onion_pkey_len); + router->onion_pkey = tor_memdup(tok->object_body, tok->object_size); + router->onion_pkey_len = tok->object_size; crypto_pk_free(tok->key); if ((tok = find_opt_by_keyword(tokens, K_ONION_KEY_NTOR))) { diff --git a/src/feature/hibernate/hibernate.c b/src/feature/hibernate/hibernate.c index 6f8795cecc..feeb3d92ef 100644 --- a/src/feature/hibernate/hibernate.c +++ b/src/feature/hibernate/hibernate.c @@ -66,8 +66,9 @@ static hibernate_state_t hibernate_state = HIBERNATE_STATE_INITIAL; /** If are hibernating, when do we plan to wake up? Set to 0 if we * aren't hibernating. */ static time_t hibernate_end_time = 0; -/** If we are shutting down, when do we plan finally exit? Set to 0 if - * we aren't shutting down. */ +/** If we are shutting down, when do we plan finally exit? Set to 0 if we + * aren't shutting down. (This is obsolete; scheduled shutdowns are supposed + * to happen from mainloop_schedule_shutdown() now.) */ static time_t shutdown_time = 0; /** A timed event that we'll use when it's time to wake up from @@ -867,7 +868,13 @@ hibernate_begin(hibernate_state_t new_state, time_t now) log_notice(LD_GENERAL,"Interrupt: we have stopped accepting new " "connections, and will shut down in %d seconds. Interrupt " "again to exit now.", options->ShutdownWaitLength); - shutdown_time = time(NULL) + options->ShutdownWaitLength; + /* We add an arbitrary delay here so that even if something goes wrong + * with the mainloop shutdown code, we can still shutdown from + * consider_hibernation() if we call it... but so that the + * mainloop_schedule_shutdown() mechanism will be the first one called. + */ + shutdown_time = time(NULL) + options->ShutdownWaitLength + 5; + mainloop_schedule_shutdown(options->ShutdownWaitLength); #ifdef HAVE_SYSTEMD /* tell systemd that we may need more than the default 90 seconds to shut * down so they don't kill us. add some extra time to actually finish @@ -1096,11 +1103,12 @@ consider_hibernation(time_t now) hibernate_state_t prev_state = hibernate_state; /* If we're in 'exiting' mode, then we just shut down after the interval - * elapses. */ + * elapses. The mainloop was supposed to catch this via + * mainloop_schedule_shutdown(), but apparently it didn't. */ if (hibernate_state == HIBERNATE_STATE_EXITING) { tor_assert(shutdown_time); if (shutdown_time <= now) { - log_notice(LD_GENERAL, "Clean shutdown finished. Exiting."); + log_notice(LD_BUG, "Mainloop did not catch shutdown event; exiting."); tor_shutdown_event_loop_and_exit(0); } return; /* if exiting soon, don't worry about bandwidth limits */ @@ -1112,7 +1120,7 @@ consider_hibernation(time_t now) if (hibernate_end_time > now && accounting_enabled) { /* If we're hibernating, don't wake up until it's time, regardless of * whether we're in a new interval. */ - return ; + return; } else { hibernate_end_time_elapsed(now); } @@ -1240,8 +1248,6 @@ on_hibernate_state_change(hibernate_state_t prev_state) if (prev_state != HIBERNATE_STATE_INITIAL) { rescan_periodic_events(get_options()); } - - reschedule_per_second_timer(); } /** Free all resources held by the accounting module */ diff --git a/src/feature/hs/hs_descriptor.c b/src/feature/hs/hs_descriptor.c index 8515314b38..1b2008c804 100644 --- a/src/feature/hs/hs_descriptor.c +++ b/src/feature/hs/hs_descriptor.c @@ -1400,6 +1400,50 @@ encrypted_data_length_is_valid(size_t len) return 0; } +/* Build the KEYS component for the authorized client computation. The format + * of the construction is: + * + * SECRET_SEED = x25519(sk, pk) + * KEYS = KDF(subcredential | SECRET_SEED, 40) + * + * Set the <b>keys_out</b> argument to point to the buffer containing the KEYS, + * and return the buffer's length. The caller should wipe and free its content + * once done with it. This function can't fail. */ +static size_t +build_descriptor_cookie_keys(const uint8_t *subcredential, + size_t subcredential_len, + const curve25519_secret_key_t *sk, + const curve25519_public_key_t *pk, + uint8_t **keys_out) +{ + uint8_t secret_seed[CURVE25519_OUTPUT_LEN]; + uint8_t *keystream; + size_t keystream_len = HS_DESC_CLIENT_ID_LEN + HS_DESC_COOKIE_KEY_LEN; + crypto_xof_t *xof; + + tor_assert(subcredential); + tor_assert(sk); + tor_assert(pk); + tor_assert(keys_out); + + keystream = tor_malloc_zero(keystream_len); + + /* Calculate x25519(sk, pk) to get the secret seed. */ + curve25519_handshake(secret_seed, sk, pk); + + /* Calculate KEYS = KDF(subcredential | SECRET_SEED, 40) */ + xof = crypto_xof_new(); + crypto_xof_add_bytes(xof, subcredential, subcredential_len); + crypto_xof_add_bytes(xof, secret_seed, sizeof(secret_seed)); + crypto_xof_squeeze_bytes(xof, keystream, keystream_len); + crypto_xof_free(xof); + + memwipe(secret_seed, 0, sizeof(secret_seed)); + + *keys_out = keystream; + return keystream_len; +} + /* Decrypt the descriptor cookie given the descriptor, the auth client, * and the client secret key. On sucess, return 0 and a newly allocated * descriptor cookie descriptor_cookie_out. On error or if the client id @@ -1412,12 +1456,11 @@ decrypt_descriptor_cookie(const hs_descriptor_t *desc, uint8_t **descriptor_cookie_out) { int ret = -1; - uint8_t secret_seed[CURVE25519_OUTPUT_LEN]; - uint8_t keystream[HS_DESC_CLIENT_ID_LEN + HS_DESC_COOKIE_KEY_LEN]; - uint8_t *cookie_key = NULL; + uint8_t *keystream = NULL; + size_t keystream_length = 0; uint8_t *descriptor_cookie = NULL; + const uint8_t *cookie_key = NULL; crypto_cipher_t *cipher = NULL; - crypto_xof_t *xof = NULL; tor_assert(desc); tor_assert(client); @@ -1429,16 +1472,13 @@ decrypt_descriptor_cookie(const hs_descriptor_t *desc, sizeof(*client_auth_sk))); tor_assert(!tor_mem_is_zero((char *) desc->subcredential, DIGEST256_LEN)); - /* Calculate x25519(client_x, hs_Y) */ - curve25519_handshake(secret_seed, client_auth_sk, - &desc->superencrypted_data.auth_ephemeral_pubkey); - - /* Calculate KEYS = KDF(subcredential | SECRET_SEED, 40) */ - xof = crypto_xof_new(); - crypto_xof_add_bytes(xof, desc->subcredential, DIGEST256_LEN); - crypto_xof_add_bytes(xof, secret_seed, sizeof(secret_seed)); - crypto_xof_squeeze_bytes(xof, keystream, sizeof(keystream)); - crypto_xof_free(xof); + /* Get the KEYS component to derive the CLIENT-ID and COOKIE-KEY. */ + keystream_length = + build_descriptor_cookie_keys(desc->subcredential, DIGEST256_LEN, + client_auth_sk, + &desc->superencrypted_data.auth_ephemeral_pubkey, + &keystream); + tor_assert(keystream_length > 0); /* If the client id of auth client is not the same as the calculcated * client id, it means that this auth client is invaild according to the @@ -1464,8 +1504,8 @@ decrypt_descriptor_cookie(const hs_descriptor_t *desc, if (cipher) { crypto_cipher_free(cipher); } - memwipe(secret_seed, 0, sizeof(secret_seed)); - memwipe(keystream, 0, sizeof(keystream)); + memwipe(keystream, 0, keystream_length); + tor_free(keystream); return ret; } @@ -2878,11 +2918,10 @@ hs_desc_build_authorized_client(const uint8_t *subcredential, const uint8_t *descriptor_cookie, hs_desc_authorized_client_t *client_out) { - uint8_t secret_seed[CURVE25519_OUTPUT_LEN]; - uint8_t keystream[HS_DESC_CLIENT_ID_LEN + HS_DESC_COOKIE_KEY_LEN]; - uint8_t *cookie_key; + uint8_t *keystream = NULL; + size_t keystream_length = 0; + const uint8_t *cookie_key; crypto_cipher_t *cipher; - crypto_xof_t *xof; tor_assert(client_auth_pk); tor_assert(auth_ephemeral_sk); @@ -2898,18 +2937,14 @@ hs_desc_build_authorized_client(const uint8_t *subcredential, tor_assert(!tor_mem_is_zero((char *) subcredential, DIGEST256_LEN)); - /* Calculate x25519(hs_y, client_X) */ - curve25519_handshake(secret_seed, - auth_ephemeral_sk, - client_auth_pk); - - /* Calculate KEYS = KDF(subcredential | SECRET_SEED, 40) */ - xof = crypto_xof_new(); - crypto_xof_add_bytes(xof, subcredential, DIGEST256_LEN); - crypto_xof_add_bytes(xof, secret_seed, sizeof(secret_seed)); - crypto_xof_squeeze_bytes(xof, keystream, sizeof(keystream)); - crypto_xof_free(xof); + /* Get the KEYS part so we can derive the CLIENT-ID and COOKIE-KEY. */ + keystream_length = + build_descriptor_cookie_keys(subcredential, DIGEST256_LEN, + auth_ephemeral_sk, client_auth_pk, + &keystream); + tor_assert(keystream_length > 0); + /* Extract the CLIENT-ID and COOKIE-KEY from the KEYS. */ memcpy(client_out->client_id, keystream, HS_DESC_CLIENT_ID_LEN); cookie_key = keystream + HS_DESC_CLIENT_ID_LEN; @@ -2924,8 +2959,8 @@ hs_desc_build_authorized_client(const uint8_t *subcredential, (const char *) descriptor_cookie, HS_DESC_DESCRIPTOR_COOKIE_LEN); - memwipe(secret_seed, 0, sizeof(secret_seed)); - memwipe(keystream, 0, sizeof(keystream)); + memwipe(keystream, 0, keystream_length); + tor_free(keystream); crypto_cipher_free(cipher); } diff --git a/src/feature/hs/hs_service.c b/src/feature/hs/hs_service.c index 3042428954..623a239d50 100644 --- a/src/feature/hs/hs_service.c +++ b/src/feature/hs/hs_service.c @@ -2931,8 +2931,8 @@ set_descriptor_revision_counter(hs_service_descriptor_t *hs_desc, time_t now, /* The OPE module returns CRYPTO_OPE_ERROR in case of errors. */ tor_assert_nonfatal(rev_counter < CRYPTO_OPE_ERROR); - log_info(LD_REND, "Encrypted revision counter %d to %ld", - (int) seconds_since_start_of_srv, (long int) rev_counter); + log_info(LD_REND, "Encrypted revision counter %d to %" PRIu64, + (int) seconds_since_start_of_srv, rev_counter); hs_desc->desc->plaintext_data.revision_counter = rev_counter; } @@ -3669,8 +3669,8 @@ hs_service_lookup_current_desc(const ed25519_public_key_t *pk) } /* Return the number of service we have configured and usable. */ -unsigned int -hs_service_get_num_services(void) +MOCK_IMPL(unsigned int, +hs_service_get_num_services,(void)) { if (hs_service_map == NULL) { return 0; diff --git a/src/feature/hs/hs_service.h b/src/feature/hs/hs_service.h index a8a9faaea9..be1155bcd1 100644 --- a/src/feature/hs/hs_service.h +++ b/src/feature/hs/hs_service.h @@ -310,7 +310,7 @@ hs_service_t *hs_service_new(const or_options_t *options); void hs_service_free_(hs_service_t *service); #define hs_service_free(s) FREE_AND_NULL(hs_service_t, hs_service_free_, (s)) -unsigned int hs_service_get_num_services(void); +MOCK_DECL(unsigned int, hs_service_get_num_services,(void)); void hs_service_stage_services(const smartlist_t *service_list); int hs_service_load_all_keys(void); int hs_service_get_version_from_key(const hs_service_t *service); diff --git a/src/feature/nodelist/authcert.c b/src/feature/nodelist/authcert.c index b111422d0d..2c4915e913 100644 --- a/src/feature/nodelist/authcert.c +++ b/src/feature/nodelist/authcert.c @@ -380,7 +380,8 @@ trusted_dirs_load_certs_from_string(const char *contents, int source, int added_trusted_cert = 0; for (s = contents; *s; s = eos) { - authority_cert_t *cert = authority_cert_parse_from_string(s, &eos); + authority_cert_t *cert = authority_cert_parse_from_string(s, strlen(s), + &eos); cert_list_t *cl; if (!cert) { failure_code = -1; diff --git a/src/feature/nodelist/fmt_routerstatus.c b/src/feature/nodelist/fmt_routerstatus.c index e70aeb2950..b1d4a48038 100644 --- a/src/feature/nodelist/fmt_routerstatus.c +++ b/src/feature/nodelist/fmt_routerstatus.c @@ -135,7 +135,7 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version, goto done; smartlist_add_asprintf(chunks, - "s%s%s%s%s%s%s%s%s%s%s\n", + "s%s%s%s%s%s%s%s%s%s%s%s\n", /* These must stay in alphabetical order. */ rs->is_authority?" Authority":"", rs->is_bad_exit?" BadExit":"", @@ -145,6 +145,7 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version, rs->is_hs_dir?" HSDir":"", rs->is_flagged_running?" Running":"", rs->is_stable?" Stable":"", + rs->is_staledesc?" StaleDesc":"", rs->is_v2_dir?" V2Dir":"", rs->is_valid?" Valid":""); diff --git a/src/feature/nodelist/microdesc.c b/src/feature/nodelist/microdesc.c index 146c772daf..eadca69548 100644 --- a/src/feature/nodelist/microdesc.c +++ b/src/feature/nodelist/microdesc.c @@ -23,6 +23,7 @@ #include "feature/nodelist/dirlist.h" #include "feature/nodelist/microdesc.h" #include "feature/nodelist/networkstatus.h" +#include "feature/nodelist/nodefamily.h" #include "feature/nodelist/nodelist.h" #include "feature/nodelist/routerlist.h" #include "feature/relay/router.h" @@ -108,10 +109,12 @@ microdesc_note_outdated_dirserver(const char *relay_digest) { char relay_hexdigest[HEX_DIGEST_LEN+1]; - /* Don't register outdated dirservers if we don't have a live consensus, - * since we might be trying to fetch microdescriptors that are not even - * currently active. */ - if (!networkstatus_get_live_consensus(approx_time())) { + /* If we have a reasonably live consensus, then most of our dirservers should + * still be caching all the microdescriptors in it. Reasonably live + * consensuses are up to a day old. But microdescriptors expire 7 days after + * the last consensus that referenced them. */ + if (!networkstatus_get_reasonably_live_consensus(approx_time(), + FLAV_MICRODESC)) { return; } @@ -882,10 +885,7 @@ microdesc_free_(microdesc_t *md, const char *fname, int lineno) if (md->body && md->saved_location != SAVED_IN_CACHE) tor_free(md->body); - if (md->family) { - SMARTLIST_FOREACH(md->family, char *, cp, tor_free(cp)); - smartlist_free(md->family); - } + nodefamily_free(md->family); short_policy_free(md->exit_policy); short_policy_free(md->ipv6_exit_policy); diff --git a/src/feature/nodelist/microdesc_st.h b/src/feature/nodelist/microdesc_st.h index d23da13137..30c896181d 100644 --- a/src/feature/nodelist/microdesc_st.h +++ b/src/feature/nodelist/microdesc_st.h @@ -9,6 +9,7 @@ struct curve25519_public_key_t; struct ed25519_public_key_t; +struct nodefamily_t; struct short_policy_t; /** A microdescriptor is the smallest amount of information needed to build a @@ -69,8 +70,8 @@ struct microdesc_t { tor_addr_t ipv6_addr; /** As routerinfo_t.ipv6_orport */ uint16_t ipv6_orport; - /** As routerinfo_t.family */ - smartlist_t *family; + /** As routerinfo_t.family, with readable members parsed. */ + struct nodefamily_t *family; /** IPv4 exit policy summary */ struct short_policy_t *exit_policy; /** IPv6 exit policy summary */ diff --git a/src/feature/nodelist/networkstatus.c b/src/feature/nodelist/networkstatus.c index de2451b79c..65ea3cc491 100644 --- a/src/feature/nodelist/networkstatus.c +++ b/src/feature/nodelist/networkstatus.c @@ -116,8 +116,6 @@ STATIC networkstatus_t *current_md_consensus = NULL; typedef struct consensus_waiting_for_certs_t { /** The consensus itself. */ networkstatus_t *consensus; - /** The encoded version of the consensus, nul-terminated. */ - char *body; /** When did we set the current value of consensus_waiting_for_certs? If * this is too recent, we shouldn't try to fetch a new consensus for a * little while, to give ourselves time to get certificates for this one. */ @@ -210,14 +208,11 @@ networkstatus_reset_download_failures(void) download_status_reset(&consensus_bootstrap_dl_status[i]); } -/** - * Read and and return the cached consensus of type <b>flavorname</b>. If - * <b>unverified</b> is true, get the one we haven't verified. Return NULL if - * the file isn't there. */ +/** Return the filename used to cache the consensus of a given flavor */ static char * -networkstatus_read_cached_consensus_impl(int flav, - const char *flavorname, - int unverified_consensus) +networkstatus_get_cache_fname(int flav, + const char *flavorname, + int unverified_consensus) { char buf[128]; const char *prefix; @@ -232,21 +227,35 @@ networkstatus_read_cached_consensus_impl(int flav, tor_snprintf(buf, sizeof(buf), "%s-%s-consensus", prefix, flavorname); } - char *filename = get_cachedir_fname(buf); - char *result = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL); + return get_cachedir_fname(buf); +} + +/** + * Read and and return the cached consensus of type <b>flavorname</b>. If + * <b>unverified</b> is false, get the one we haven't verified. Return NULL if + * the file isn't there. */ +static tor_mmap_t * +networkstatus_map_cached_consensus_impl(int flav, + const char *flavorname, + int unverified_consensus) +{ + char *filename = networkstatus_get_cache_fname(flav, + flavorname, + unverified_consensus); + tor_mmap_t *result = tor_mmap_file(filename); tor_free(filename); return result; } -/** Return a new string containing the current cached consensus of flavor - * <b>flavorname</b>. */ -char * -networkstatus_read_cached_consensus(const char *flavorname) - { +/** Map the file containing the current cached consensus of flavor + * <b>flavorname</b> */ +tor_mmap_t * +networkstatus_map_cached_consensus(const char *flavorname) +{ int flav = networkstatus_parse_flavor_name(flavorname); if (flav < 0) return NULL; - return networkstatus_read_cached_consensus_impl(flav, flavorname, 0); + return networkstatus_map_cached_consensus_impl(flav, flavorname, 0); } /** Read every cached v3 consensus networkstatus from the disk. */ @@ -259,24 +268,26 @@ router_reload_consensus_networkstatus(void) /* FFFF Suppress warnings if cached consensus is bad? */ for (flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) { const char *flavor = networkstatus_get_flavor_name(flav); - char *s = networkstatus_read_cached_consensus_impl(flav, flavor, 0); - if (s) { - if (networkstatus_set_current_consensus(s, flavor, flags, NULL) < -1) { + tor_mmap_t *m = networkstatus_map_cached_consensus_impl(flav, flavor, 0); + if (m) { + if (networkstatus_set_current_consensus(m->data, m->size, + flavor, flags, NULL) < -1) { log_warn(LD_FS, "Couldn't load consensus %s networkstatus from cache", flavor); } - tor_free(s); + tor_munmap_file(m); } - s = networkstatus_read_cached_consensus_impl(flav, flavor, 1); - if (s) { - if (networkstatus_set_current_consensus(s, flavor, + m = networkstatus_map_cached_consensus_impl(flav, flavor, 1); + if (m) { + if (networkstatus_set_current_consensus(m->data, m->size, + flavor, flags | NSSET_WAS_WAITING_FOR_CERTS, NULL)) { log_info(LD_FS, "Couldn't load unverified consensus %s networkstatus " "from cache", flavor); } - tor_free(s); + tor_munmap_file(m); } } @@ -713,8 +724,8 @@ networkstatus_vote_find_mutable_entry(networkstatus_t *ns, const char *digest) /** Return the entry in <b>ns</b> for the identity digest <b>digest</b>, or * NULL if none was found. */ -const routerstatus_t * -networkstatus_vote_find_entry(networkstatus_t *ns, const char *digest) +MOCK_IMPL(const routerstatus_t *, +networkstatus_vote_find_entry,(networkstatus_t *ns, const char *digest)) { return networkstatus_vote_find_mutable_entry(ns, digest); } @@ -1448,13 +1459,10 @@ networkstatus_valid_until_is_reasonably_live(time_t valid_until, return (now <= valid_until + REASONABLY_LIVE_TIME); } -/* XXXX remove this in favor of get_live_consensus. But actually, - * leave something like it for bridge users, who need to not totally - * lose if they spend a while fetching a new consensus. */ /** As networkstatus_get_live_consensus(), but is way more tolerant of expired * consensuses. */ -networkstatus_t * -networkstatus_get_reasonably_live_consensus(time_t now, int flavor) +MOCK_IMPL(networkstatus_t *, +networkstatus_get_reasonably_live_consensus,(time_t now, int flavor)) { networkstatus_t *consensus = networkstatus_get_latest_consensus_by_flavor(flavor); @@ -1844,6 +1852,7 @@ warn_early_consensus(const networkstatus_t *c, const char *flavor, */ int networkstatus_set_current_consensus(const char *consensus, + size_t consensus_len, const char *flavor, unsigned flags, const char *source_dir) @@ -1872,7 +1881,9 @@ networkstatus_set_current_consensus(const char *consensus, } /* Make sure it's parseable. */ - c = networkstatus_parse_vote_from_string(consensus, NULL, NS_TYPE_CONSENSUS); + c = networkstatus_parse_vote_from_string(consensus, + consensus_len, + NULL, NS_TYPE_CONSENSUS); if (!c) { log_warn(LD_DIR, "Unable to parse networkstatus consensus"); result = -2; @@ -1960,14 +1971,12 @@ networkstatus_set_current_consensus(const char *consensus, c->valid_after > current_valid_after) { waiting = &consensus_waiting_for_certs[flav]; networkstatus_vote_free(waiting->consensus); - tor_free(waiting->body); waiting->consensus = c; free_consensus = 0; - waiting->body = tor_strdup(consensus); waiting->set_at = now; waiting->dl_failed = 0; if (!from_cache) { - write_str_to_file(unverified_fname, consensus, 0); + write_bytes_to_file(unverified_fname, consensus, consensus_len, 0); } if (dl_certs) authority_certs_fetch_missing(c, now, source_dir); @@ -2058,10 +2067,6 @@ networkstatus_set_current_consensus(const char *consensus, waiting->consensus->valid_after <= c->valid_after) { networkstatus_vote_free(waiting->consensus); waiting->consensus = NULL; - if (consensus != waiting->body) - tor_free(waiting->body); - else - waiting->body = NULL; waiting->set_at = 0; waiting->dl_failed = 0; if (unlink(unverified_fname) != 0) { @@ -2111,17 +2116,18 @@ networkstatus_set_current_consensus(const char *consensus, if (we_want_to_fetch_flavor(options, flav)) { if (dir_server_mode(get_options())) { dirserv_set_cached_consensus_networkstatus(consensus, + consensus_len, flavor, &c->digests, c->digest_sha3_as_signed, c->valid_after); - consdiffmgr_add_consensus(consensus, c); + consdiffmgr_add_consensus(consensus, consensus_len, c); } } if (!from_cache) { - write_str_to_file(consensus_fname, consensus, 0); + write_bytes_to_file(consensus_fname, consensus, consensus_len, 0); } warn_early_consensus(c, flavor, now); @@ -2157,14 +2163,16 @@ networkstatus_note_certs_arrived(const char *source_dir) if (!waiting->consensus) continue; if (networkstatus_check_consensus_signature(waiting->consensus, 0)>=0) { - char *waiting_body = waiting->body; - if (!networkstatus_set_current_consensus( - waiting_body, - flavor_name, - NSSET_WAS_WAITING_FOR_CERTS, - source_dir)) { - tor_free(waiting_body); + tor_mmap_t *mapping = networkstatus_map_cached_consensus_impl( + i, flavor_name, 1); + if (mapping) { + networkstatus_set_current_consensus(mapping->data, + mapping->size, + flavor_name, + NSSET_WAS_WAITING_FOR_CERTS, + source_dir); } + tor_munmap_file(mapping); } } } @@ -2392,7 +2400,9 @@ networkstatus_dump_bridge_status_to_file(time_t now) published, thresholds, fingerprint_line ? fingerprint_line : "", status); fname = get_datadir_fname("networkstatus-bridges"); - write_str_to_file(fname,published_thresholds_and_status,0); + if (write_str_to_file(fname,published_thresholds_and_status,0)<0) { + log_warn(LD_DIRSERV, "Unable to write networkstatus-bridges file."); + } tor_free(thresholds); tor_free(published_thresholds_and_status); tor_free(fname); @@ -2671,6 +2681,9 @@ networkstatus_check_required_protocols(const networkstatus_t *ns, const char *required, *recommended; char *missing = NULL; + const bool consensus_postdates_this_release = + ns->valid_after >= tor_get_approx_release_date(); + tor_assert(warning_out); if (client_mode) { @@ -2688,7 +2701,7 @@ networkstatus_check_required_protocols(const networkstatus_t *ns, "%s on the Tor network. The missing protocols are: %s", func, missing); tor_free(missing); - return 1; + return consensus_postdates_this_release ? 1 : 0; } if (! protover_all_supported(recommended, &missing)) { @@ -2721,6 +2734,5 @@ networkstatus_free_all(void) networkstatus_vote_free(waiting->consensus); waiting->consensus = NULL; } - tor_free(waiting->body); } } diff --git a/src/feature/nodelist/networkstatus.h b/src/feature/nodelist/networkstatus.h index 6f8b2dc964..3d212dd304 100644 --- a/src/feature/nodelist/networkstatus.h +++ b/src/feature/nodelist/networkstatus.h @@ -16,7 +16,7 @@ void networkstatus_reset_warnings(void); void networkstatus_reset_download_failures(void); -char *networkstatus_read_cached_consensus(const char *flavorname); +tor_mmap_t *networkstatus_map_cached_consensus(const char *flavorname); int router_reload_consensus_networkstatus(void); void routerstatus_free_(routerstatus_t *rs); #define routerstatus_free(rs) \ @@ -40,8 +40,9 @@ int compare_digest_to_routerstatus_entry(const void *_key, const void **_member); int compare_digest_to_vote_routerstatus_entry(const void *_key, const void **_member); -const routerstatus_t *networkstatus_vote_find_entry(networkstatus_t *ns, - const char *digest); +MOCK_DECL(const routerstatus_t *,networkstatus_vote_find_entry,( + networkstatus_t *ns, + const char *digest)); routerstatus_t *networkstatus_vote_find_mutable_entry(networkstatus_t *ns, const char *digest); int networkstatus_vote_find_entry_idx(networkstatus_t *ns, @@ -89,8 +90,9 @@ int networkstatus_consensus_reasonably_live(const networkstatus_t *consensus, time_t now); int networkstatus_valid_until_is_reasonably_live(time_t valid_until, time_t now); -networkstatus_t *networkstatus_get_reasonably_live_consensus(time_t now, - int flavor); +MOCK_DECL(networkstatus_t *,networkstatus_get_reasonably_live_consensus, + (time_t now, + int flavor)); MOCK_DECL(int, networkstatus_consensus_is_bootstrapping,(time_t now)); int networkstatus_consensus_can_use_multiple_directories( const or_options_t *options); @@ -105,6 +107,7 @@ int networkstatus_consensus_has_ipv6(const or_options_t* options); #define NSSET_ACCEPT_OBSOLETE 8 #define NSSET_REQUIRE_FLAVOR 16 int networkstatus_set_current_consensus(const char *consensus, + size_t consensus_len, const char *flavor, unsigned flags, const char *source_dir); @@ -156,4 +159,3 @@ extern networkstatus_t *current_md_consensus; #endif /* defined(NETWORKSTATUS_PRIVATE) */ #endif /* !defined(TOR_NETWORKSTATUS_H) */ - diff --git a/src/feature/nodelist/nodefamily.c b/src/feature/nodelist/nodefamily.c new file mode 100644 index 0000000000..944ad54755 --- /dev/null +++ b/src/feature/nodelist/nodefamily.c @@ -0,0 +1,416 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file nodefamily.c + * \brief Code to manipulate encoded, reference-counted node families. We + * use these tricks to save space, since these families would otherwise + * require a large number of tiny allocations. + **/ + +#include "core/or/or.h" +#include "feature/nodelist/nickname.h" +#include "feature/nodelist/nodefamily.h" +#include "feature/nodelist/nodefamily_st.h" +#include "feature/nodelist/nodelist.h" +#include "feature/relay/router.h" +#include "feature/nodelist/routerlist.h" + +#include "ht.h" +#include "siphash.h" + +#include "lib/container/smartlist.h" +#include "lib/ctime/di_ops.h" +#include "lib/defs/digest_sizes.h" +#include "lib/log/util_bug.h" + +#include <stdlib.h> +#include <string.h> + +/** + * Allocate and return a blank node family with space to hold <b>n_members</b> + * members. + */ +static nodefamily_t * +nodefamily_alloc(int n_members) +{ + size_t alloc_len = offsetof(nodefamily_t, family_members) + + NODEFAMILY_ARRAY_SIZE(n_members); + nodefamily_t *nf = tor_malloc_zero(alloc_len); + nf->n_members = n_members; + return nf; +} + +/** + * Hashtable hash implementation. + */ +static inline unsigned int +nodefamily_hash(const nodefamily_t *nf) +{ + return (unsigned) siphash24g(nf->family_members, + NODEFAMILY_ARRAY_SIZE(nf->n_members)); +} + +/** + * Hashtable equality implementation. + */ +static inline unsigned int +nodefamily_eq(const nodefamily_t *a, const nodefamily_t *b) +{ + return (a->n_members == b->n_members) && + fast_memeq(a->family_members, b->family_members, + NODEFAMILY_ARRAY_SIZE(a->n_members)); +} + +static HT_HEAD(nodefamily_map, nodefamily_t) the_node_families + = HT_INITIALIZER(); + +HT_PROTOTYPE(nodefamily_map, nodefamily_t, ht_ent, nodefamily_hash, + nodefamily_eq) +HT_GENERATE2(nodefamily_map, nodefamily_t, ht_ent, nodefamily_hash, + node_family_eq, 0.6, tor_reallocarray_, tor_free_) + +/** + * Parse the family declaration in <b>s</b>, returning the canonical + * <b>nodefamily_t</b> for its members. Return NULL on error. + * + * If <b>rsa_id_self</b> is provided, it is a DIGEST_LEN-byte digest + * for the router that declared this family: insert it into the + * family declaration if it is not there already. + * + * If NF_WARN_MALFORMED is set in <b>flags</b>, warn about any + * elements that we can't parse. (By default, we log at info.) + * + * If NF_REJECT_MALFORMED is set in <b>flags</b>, treat any unparseable + * elements as an error. (By default, we simply omit them.) + **/ +nodefamily_t * +nodefamily_parse(const char *s, const uint8_t *rsa_id_self, + unsigned flags) +{ + smartlist_t *sl = smartlist_new(); + smartlist_split_string(sl, s, NULL, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + nodefamily_t *result = nodefamily_from_members(sl, rsa_id_self, flags, NULL); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_free(sl); + return result; +} + +/** + * Canonicalize the family list <b>s</b>, returning a newly allocated string. + * + * The canonicalization rules are fully specified in dir-spec.txt, but, + * briefly: $hexid entries are put in caps, $hexid[=~]foo entries are + * truncated, nicknames are put into lowercase, unrecognized entries are left + * alone, and everything is sorted. + **/ +char * +nodefamily_canonicalize(const char *s, const uint8_t *rsa_id_self, + unsigned flags) +{ + smartlist_t *sl = smartlist_new(); + smartlist_t *result_members = smartlist_new(); + smartlist_split_string(sl, s, NULL, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + nodefamily_t *nf = nodefamily_from_members(sl, rsa_id_self, flags, + result_members); + + char *formatted = nodefamily_format(nf); + smartlist_split_string(result_members, formatted, NULL, + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + smartlist_sort_strings(result_members); + char *combined = smartlist_join_strings(result_members, " ", 0, NULL); + + nodefamily_free(nf); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_free(sl); + SMARTLIST_FOREACH(result_members, char *, cp, tor_free(cp)); + smartlist_free(result_members); + tor_free(formatted); + + return combined; +} + +/** + * qsort helper for encoded nodefamily elements. + **/ +static int +compare_members(const void *a, const void *b) +{ + return fast_memcmp(a, b, NODEFAMILY_MEMBER_LEN); +} + +/** + * Parse the member strings in <b>members</b>, returning their canonical + * <b>nodefamily_t</b>. Return NULL on error. + * + * If <b>rsa_id_self</b> is provided, it is a DIGEST_LEN-byte digest + * for the router that declared this family: insert it into the + * family declaration if it is not there already. + * + * The <b>flags</b> element is interpreted as in nodefamily_parse(). + * + * If <b>unrecognized</b> is provided, fill it copies of any unrecognized + * members. (Note that malformed $hexids are not considered unrecognized.) + **/ +nodefamily_t * +nodefamily_from_members(const smartlist_t *members, + const uint8_t *rsa_id_self, + unsigned flags, + smartlist_t *unrecognized_out) +{ + const int n_self = rsa_id_self ? 1 : 0; + int n_bad_elements = 0; + int n_members = smartlist_len(members) + n_self; + nodefamily_t *tmp = nodefamily_alloc(n_members); + uint8_t *ptr = NODEFAMILY_MEMBER_PTR(tmp, 0); + + SMARTLIST_FOREACH_BEGIN(members, const char *, cp) { + bool bad_element = true; + if (is_legal_nickname(cp)) { + ptr[0] = NODEFAMILY_BY_NICKNAME; + tor_assert(strlen(cp) < DIGEST_LEN); // guaranteed by is_legal_nickname + memcpy(ptr+1, cp, strlen(cp)); + tor_strlower((char*) ptr+1); + bad_element = false; + } else if (is_legal_hexdigest(cp)) { + char digest_buf[DIGEST_LEN]; + char nn_buf[MAX_NICKNAME_LEN+1]; + char nn_char=0; + if (hex_digest_nickname_decode(cp, digest_buf, &nn_char, nn_buf)==0) { + bad_element = false; + ptr[0] = NODEFAMILY_BY_RSA_ID; + memcpy(ptr+1, digest_buf, DIGEST_LEN); + } + } else { + if (unrecognized_out) + smartlist_add_strdup(unrecognized_out, cp); + } + + if (bad_element) { + const int severity = (flags & NF_WARN_MALFORMED) ? LOG_WARN : LOG_INFO; + log_fn(severity, LD_GENERAL, + "Bad element %s while parsing a node family.", + escaped(cp)); + ++n_bad_elements; + } else { + ptr += NODEFAMILY_MEMBER_LEN; + } + } SMARTLIST_FOREACH_END(cp); + + if (n_bad_elements && (flags & NF_REJECT_MALFORMED)) + goto err; + + if (rsa_id_self) { + /* Add self. */ + ptr[0] = NODEFAMILY_BY_RSA_ID; + memcpy(ptr+1, rsa_id_self, DIGEST_LEN); + } + + n_members -= n_bad_elements; + + /* Sort tmp into canonical order. */ + qsort(tmp->family_members, n_members, NODEFAMILY_MEMBER_LEN, + compare_members); + + /* Remove duplicates. */ + int i; + for (i = 0; i < n_members-1; ++i) { + uint8_t *thisptr = NODEFAMILY_MEMBER_PTR(tmp, i); + uint8_t *nextptr = NODEFAMILY_MEMBER_PTR(tmp, i+1); + if (fast_memeq(thisptr, nextptr, NODEFAMILY_MEMBER_LEN)) { + memmove(thisptr, nextptr, (n_members-i-1)*NODEFAMILY_MEMBER_LEN); + --n_members; + --i; + } + } + int n_members_alloc = tmp->n_members; + tmp->n_members = n_members; + + /* See if we already allocated this family. */ + nodefamily_t *found = HT_FIND(nodefamily_map, &the_node_families, tmp); + if (found) { + /* If we did, great: incref it and return it. */ + ++found->refcnt; + tor_free(tmp); + return found; + } else { + /* If not, insert it into the hashtable. */ + if (n_members_alloc != n_members) { + /* Compact the family if needed */ + nodefamily_t *tmp2 = nodefamily_alloc(n_members); + memcpy(tmp2->family_members, tmp->family_members, + n_members * NODEFAMILY_MEMBER_LEN); + tor_free(tmp); + tmp = tmp2; + } + + tmp->refcnt = 1; + HT_INSERT(nodefamily_map, &the_node_families, tmp); + return tmp; + } + + err: + tor_free(tmp); + return NULL; +} + +/** + * Drop our reference to <b>family</b>, freeing it if there are no more + * references. + */ +void +nodefamily_free_(nodefamily_t *family) +{ + if (family == NULL) + return; + + --family->refcnt; + + if (family->refcnt == 0) { + HT_REMOVE(nodefamily_map, &the_node_families, family); + tor_free(family); + } +} + +/** + * Return true iff <b>family</b> contains the SHA1 RSA1024 identity + * <b>rsa_id</b>. + */ +bool +nodefamily_contains_rsa_id(const nodefamily_t *family, + const uint8_t *rsa_id) +{ + if (family == NULL) + return false; + + unsigned i; + for (i = 0; i < family->n_members; ++i) { + const uint8_t *ptr = NODEFAMILY_MEMBER_PTR(family, i); + if (ptr[0] == NODEFAMILY_BY_RSA_ID && + fast_memeq(ptr+1, rsa_id, DIGEST_LEN)) { + return true; + } + } + return false; +} + +/** + * Return true iff <b>family</b> contains the nickname <b>name</b>. + */ +bool +nodefamily_contains_nickname(const nodefamily_t *family, + const char *name) +{ + if (family == NULL) + return false; + + unsigned i; + for (i = 0; i < family->n_members; ++i) { + const uint8_t *ptr = NODEFAMILY_MEMBER_PTR(family, i); + // note that the strcasecmp() is safe because there is always at least one + // NUL in the encoded nickname, because all legal nicknames are less than + // DIGEST_LEN bytes long. + if (ptr[0] == NODEFAMILY_BY_NICKNAME && !strcasecmp((char*)ptr+1, name)) { + return true; + } + } + return false; +} + +/** + * Return true if <b>family</b> contains the nickname or the RSA ID for + * <b>node</b> + **/ +bool +nodefamily_contains_node(const nodefamily_t *family, + const node_t *node) +{ + return + nodefamily_contains_nickname(family, node_get_nickname(node)) + || + nodefamily_contains_rsa_id(family, node_get_rsa_id_digest(node)); +} + +/** + * Look up every entry in <b>family</b>, and add add the corresponding + * node_t to <b>out</b>. + **/ +void +nodefamily_add_nodes_to_smartlist(const nodefamily_t *family, + smartlist_t *out) +{ + if (!family) + return; + + unsigned i; + for (i = 0; i < family->n_members; ++i) { + const uint8_t *ptr = NODEFAMILY_MEMBER_PTR(family, i); + const node_t *node = NULL; + switch (ptr[0]) { + case NODEFAMILY_BY_NICKNAME: + node = node_get_by_nickname((char*)ptr+1, NNF_NO_WARN_UNNAMED); + break; + case NODEFAMILY_BY_RSA_ID: + node = node_get_by_id((char*)ptr+1); + break; + default: + /* LCOV_EXCL_START */ + tor_assert_nonfatal_unreached(); + break; + /* LCOV_EXCL_STOP */ + } + if (node) + smartlist_add(out, (void *)node); + } +} + +/** + * Encode <b>family</b> as a space-separated string. + */ +char * +nodefamily_format(const nodefamily_t *family) +{ + if (!family) + return tor_strdup(""); + + unsigned i; + smartlist_t *sl = smartlist_new(); + for (i = 0; i < family->n_members; ++i) { + const uint8_t *ptr = NODEFAMILY_MEMBER_PTR(family, i); + switch (ptr[0]) { + case NODEFAMILY_BY_NICKNAME: + smartlist_add_strdup(sl, (char*)ptr+1); + break; + case NODEFAMILY_BY_RSA_ID: { + char buf[HEX_DIGEST_LEN+2]; + buf[0]='$'; + base16_encode(buf+1, sizeof(buf)-1, (char*)ptr+1, DIGEST_LEN); + tor_strupper(buf); + smartlist_add_strdup(sl, buf); + break; + } + default: + /* LCOV_EXCL_START */ + tor_assert_nonfatal_unreached(); + break; + /* LCOV_EXCL_STOP */ + } + } + + char *result = smartlist_join_strings(sl, " ", 0, NULL); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_free(sl); + return result; +} + +/** + * Free all storage held in the nodefamily map. + **/ +void +nodefamily_free_all(void) +{ + HT_CLEAR(nodefamily_map, &the_node_families); +} diff --git a/src/feature/nodelist/nodefamily.h b/src/feature/nodelist/nodefamily.h new file mode 100644 index 0000000000..ea1076876d --- /dev/null +++ b/src/feature/nodelist/nodefamily.h @@ -0,0 +1,50 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file nodefamily.h + * \brief Header file for nodefamily.c. + **/ + +#ifndef TOR_NODEFAMILY_H +#define TOR_NODEFAMILY_H + +#include "lib/malloc/malloc.h" +#include <stdbool.h> + +typedef struct nodefamily_t nodefamily_t; +struct node_t; +struct smartlist_t; + +#define NF_WARN_MALFORMED (1u<<0) +#define NF_REJECT_MALFORMED (1u<<1) + +nodefamily_t *nodefamily_parse(const char *s, + const uint8_t *rsa_id_self, + unsigned flags); +nodefamily_t *nodefamily_from_members(const struct smartlist_t *members, + const uint8_t *rsa_id_self, + unsigned flags, + smartlist_t *unrecognized_out); +void nodefamily_free_(nodefamily_t *family); +#define nodefamily_free(family) \ + FREE_AND_NULL(nodefamily_t, nodefamily_free_, (family)) + +bool nodefamily_contains_rsa_id(const nodefamily_t *family, + const uint8_t *rsa_id); +bool nodefamily_contains_nickname(const nodefamily_t *family, + const char *name); +bool nodefamily_contains_node(const nodefamily_t *family, + const struct node_t *node); +void nodefamily_add_nodes_to_smartlist(const nodefamily_t *family, + struct smartlist_t *out); +char *nodefamily_format(const nodefamily_t *family); +char *nodefamily_canonicalize(const char *s, const uint8_t *rsa_id_self, + unsigned flags); + +void nodefamily_free_all(void); + +#endif diff --git a/src/feature/nodelist/nodefamily_st.h b/src/feature/nodelist/nodefamily_st.h new file mode 100644 index 0000000000..a498b4b3b9 --- /dev/null +++ b/src/feature/nodelist/nodefamily_st.h @@ -0,0 +1,48 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_NODEFAMILY_ST_H +#define TOR_NODEFAMILY_ST_H + +#include "orconfig.h" +#include "ht.h" + +struct nodefamily_t { + /** Entry for this nodefamily_t within the hashtable. */ + HT_ENTRY(nodefamily_t) ht_ent; + /** Reference count. (The hashtable is not treated as a reference */ + uint32_t refcnt; + /** Number of items encoded in <b>family_members</b>. */ + uint32_t n_members; + /* A byte-array encoding the members of this family. We encode each member + * as one byte to indicate whether it's a nickname or a fingerprint, plus + * DIGEST_LEN bytes of data. The entries are lexically sorted. + */ + uint8_t family_members[FLEXIBLE_ARRAY_MEMBER]; +}; + +#define NODEFAMILY_MEMBER_LEN (1+DIGEST_LEN) + +/** Tag byte, indicates that the following bytes are a RSA1024 SHA1 ID. + */ +#define NODEFAMILY_BY_RSA_ID 0 +/** Tag byte, indicates that the following bytes are a NUL-padded nickname. + */ +#define NODEFAMILY_BY_NICKNAME 1 + +/** + * Number of bytes to allocate in the array for a nodefamily_t with N members. + **/ +#define NODEFAMILY_ARRAY_SIZE(n) \ + ((n) * NODEFAMILY_MEMBER_LEN) + +/** + * Pointer to the i'th member of <b>nf</b>, as encoded. + */ +#define NODEFAMILY_MEMBER_PTR(nf, i) \ + (&((nf)->family_members[(i) * NODEFAMILY_MEMBER_LEN])) + +#endif diff --git a/src/feature/nodelist/nodelist.c b/src/feature/nodelist/nodelist.c index a39eb1d5b8..d94e73f48f 100644 --- a/src/feature/nodelist/nodelist.c +++ b/src/feature/nodelist/nodelist.c @@ -59,6 +59,7 @@ #include "feature/nodelist/microdesc.h" #include "feature/nodelist/networkstatus.h" #include "feature/nodelist/node_select.h" +#include "feature/nodelist/nodefamily.h" #include "feature/nodelist/nodelist.h" #include "feature/nodelist/routerlist.h" #include "feature/nodelist/routerset.h" @@ -1503,19 +1504,6 @@ node_is_me(const node_t *node) return router_digest_is_me(node->identity); } -/** Return <b>node</b> declared family (as a list of names), or NULL if - * the node didn't declare a family. */ -const smartlist_t * -node_get_declared_family(const node_t *node) -{ - if (node->ri && node->ri->declared_family) - return node->ri->declared_family; - else if (node->md && node->md->family) - return node->md->family; - else - return NULL; -} - /* Does this node have a valid IPv6 address? * Prefer node_has_ipv6_orport() or node_has_ipv6_dirport() for * checking specific ports. */ @@ -1866,6 +1854,9 @@ int addrs_in_same_network_family(const tor_addr_t *a1, const tor_addr_t *a2) { + if (tor_addr_is_null(a1) || tor_addr_is_null(a2)) + return 0; + switch (tor_addr_family(a1)) { case AF_INET: return 0 == tor_addr_compare_masked(a1, a2, 16, CMP_SEMANTIC); @@ -1881,7 +1872,7 @@ addrs_in_same_network_family(const tor_addr_t *a1, * (case-insensitive), or if <b>node's</b> identity key digest * matches a hexadecimal value stored in <b>nickname</b>. Return * false otherwise. */ -static int +STATIC int node_nickname_matches(const node_t *node, const char *nickname) { const char *n = node_get_nickname(node); @@ -1893,7 +1884,7 @@ node_nickname_matches(const node_t *node, const char *nickname) } /** Return true iff <b>node</b> is named by some nickname in <b>lst</b>. */ -static inline int +STATIC int node_in_nickname_smartlist(const smartlist_t *lst, const node_t *node) { if (!lst) return 0; @@ -1904,6 +1895,61 @@ node_in_nickname_smartlist(const smartlist_t *lst, const node_t *node) return 0; } +/** Return true iff n1's declared family contains n2. */ +STATIC int +node_family_contains(const node_t *n1, const node_t *n2) +{ + if (n1->ri && n1->ri->declared_family) { + return node_in_nickname_smartlist(n1->ri->declared_family, n2); + } else if (n1->md) { + return nodefamily_contains_node(n1->md->family, n2); + } else { + return 0; + } +} + +/** + * Return true iff <b>node</b> has declared a nonempty family. + **/ +STATIC bool +node_has_declared_family(const node_t *node) +{ + if (node->ri && node->ri->declared_family && + smartlist_len(node->ri->declared_family)) { + return true; + } + + if (node->md && node->md->family) { + return true; + } + + return false; +} + +/** + * Add to <b>out</b> every node_t that is listed by <b>node</b> as being in + * its family. (Note that these nodes are not in node's family unless they + * also agree that node is in their family.) + **/ +STATIC void +node_lookup_declared_family(smartlist_t *out, const node_t *node) +{ + if (node->ri && node->ri->declared_family && + smartlist_len(node->ri->declared_family)) { + SMARTLIST_FOREACH_BEGIN(node->ri->declared_family, const char *, name) { + const node_t *n2 = node_get_by_nickname(name, NNF_NO_WARN_UNNAMED); + if (n2) { + smartlist_add(out, (node_t *)n2); + } + } SMARTLIST_FOREACH_END(name); + return; + } + + if (node->md && node->md->family) { + nodefamily_add_nodes_to_smartlist(node->md->family, out); + } +} + /** Return true iff r1 and r2 are in the same family, but not the same * router. */ int @@ -1916,19 +1962,20 @@ nodes_in_same_family(const node_t *node1, const node_t *node2) tor_addr_t a1, a2; node_get_addr(node1, &a1); node_get_addr(node2, &a2); - if (addrs_in_same_network_family(&a1, &a2)) + + tor_addr_port_t ap6_1, ap6_2; + node_get_pref_ipv6_orport(node1, &ap6_1); + node_get_pref_ipv6_orport(node2, &ap6_2); + + if (addrs_in_same_network_family(&a1, &a2) || + addrs_in_same_network_family(&ap6_1.addr, &ap6_2.addr)) return 1; } /* Are they in the same family because the agree they are? */ - { - const smartlist_t *f1, *f2; - f1 = node_get_declared_family(node1); - f2 = node_get_declared_family(node2); - if (f1 && f2 && - node_in_nickname_smartlist(f1, node2) && - node_in_nickname_smartlist(f2, node1)) - return 1; + if (node_family_contains(node1, node2) && + node_family_contains(node2, node1)) { + return 1; } /* Are they in the same family because the user says they are? */ @@ -1956,13 +2003,10 @@ void nodelist_add_node_and_family(smartlist_t *sl, const node_t *node) { const smartlist_t *all_nodes = nodelist_get_list(); - const smartlist_t *declared_family; const or_options_t *options = get_options(); tor_assert(node); - declared_family = node_get_declared_family(node); - /* Let's make sure that we have the node itself, if it's a real node. */ { const node_t *real_node = node_get_by_id(node->identity); @@ -1973,35 +2017,35 @@ nodelist_add_node_and_family(smartlist_t *sl, const node_t *node) /* First, add any nodes with similar network addresses. */ if (options->EnforceDistinctSubnets) { tor_addr_t node_addr; + tor_addr_port_t node_ap6; node_get_addr(node, &node_addr); + node_get_pref_ipv6_orport(node, &node_ap6); SMARTLIST_FOREACH_BEGIN(all_nodes, const node_t *, node2) { tor_addr_t a; + tor_addr_port_t ap6; node_get_addr(node2, &a); - if (addrs_in_same_network_family(&a, &node_addr)) + node_get_pref_ipv6_orport(node2, &ap6); + if (addrs_in_same_network_family(&a, &node_addr) || + addrs_in_same_network_family(&ap6.addr, &node_ap6.addr)) smartlist_add(sl, (void*)node2); } SMARTLIST_FOREACH_END(node2); } - /* Now, add all nodes in the declared_family of this node, if they + /* Now, add all nodes in the declared family of this node, if they * also declare this node to be in their family. */ - if (declared_family) { + if (node_has_declared_family(node)) { + smartlist_t *declared_family = smartlist_new(); + node_lookup_declared_family(declared_family, node); + /* Add every r such that router declares familyness with node, and node * declares familyhood with router. */ - SMARTLIST_FOREACH_BEGIN(declared_family, const char *, name) { - const node_t *node2; - const smartlist_t *family2; - if (!(node2 = node_get_by_nickname(name, NNF_NO_WARN_UNNAMED))) - continue; - if (!(family2 = node_get_declared_family(node2))) - continue; - SMARTLIST_FOREACH_BEGIN(family2, const char *, name2) { - if (node_nickname_matches(node, name2)) { - smartlist_add(sl, (void*)node2); - break; - } - } SMARTLIST_FOREACH_END(name2); - } SMARTLIST_FOREACH_END(name); + SMARTLIST_FOREACH_BEGIN(declared_family, const node_t *, node2) { + if (node_family_contains(node2, node)) { + smartlist_add(sl, (void*)node2); + } + } SMARTLIST_FOREACH_END(node2); + smartlist_free(declared_family); } /* If the user declared any families locally, honor those too. */ @@ -2502,7 +2546,7 @@ count_loading_descriptors_progress(void) if (fraction > 1.0) return 0; /* it's not the number of descriptors holding us back */ return BOOTSTRAP_STATUS_LOADING_DESCRIPTORS + (int) - (fraction*(BOOTSTRAP_STATUS_CONN_OR-1 - + (fraction*(BOOTSTRAP_STATUS_ENOUGH_DIRINFO-1 - BOOTSTRAP_STATUS_LOADING_DESCRIPTORS)); } @@ -2589,7 +2633,7 @@ update_router_have_minimum_dir_info(void) /* If paths have just become available in this update. */ if (res && !have_min_dir_info) { control_event_client_status(LOG_NOTICE, "ENOUGH_DIR_INFO"); - control_event_boot_dir(BOOTSTRAP_STATUS_CONN_OR, 0); + control_event_boot_dir(BOOTSTRAP_STATUS_ENOUGH_DIRINFO, 0); log_info(LD_DIR, "We now have enough directory information to build circuits."); } diff --git a/src/feature/nodelist/nodelist.h b/src/feature/nodelist/nodelist.h index cf7658cf9d..32300eb00c 100644 --- a/src/feature/nodelist/nodelist.h +++ b/src/feature/nodelist/nodelist.h @@ -68,7 +68,6 @@ const char *node_get_platform(const node_t *node); uint32_t node_get_prim_addr_ipv4h(const node_t *node); void node_get_address_string(const node_t *node, char *cp, size_t len); long node_get_declared_uptime(const node_t *node); -const smartlist_t *node_get_declared_family(const node_t *node); const struct ed25519_public_key_t *node_get_ed25519_id(const node_t *node); int node_ed25519_id_matches(const node_t *node, const struct ed25519_public_key_t *id); @@ -155,10 +154,16 @@ int count_loading_descriptors_progress(void); #ifdef NODELIST_PRIVATE +STATIC int node_nickname_matches(const node_t *node, const char *nickname); +STATIC int node_in_nickname_smartlist(const smartlist_t *lst, + const node_t *node); +STATIC int node_family_contains(const node_t *n1, const node_t *n2); +STATIC bool node_has_declared_family(const node_t *node); +STATIC void node_lookup_declared_family(smartlist_t *out, const node_t *node); + #ifdef TOR_UNIT_TESTS -STATIC void -node_set_hsdir_index(node_t *node, const networkstatus_t *ns); +STATIC void node_set_hsdir_index(node_t *node, const networkstatus_t *ns); #endif /* defined(TOR_UNIT_TESTS) */ diff --git a/src/feature/nodelist/routerlist.c b/src/feature/nodelist/routerlist.c index a8c9003e68..b4d56459df 100644 --- a/src/feature/nodelist/routerlist.c +++ b/src/feature/nodelist/routerlist.c @@ -160,7 +160,7 @@ static time_t last_descriptor_download_attempted = 0; * * From time to time, we replace "cached-descriptors" with a new file * containing only the live, non-superseded descriptors, and clear - * cached-routers.new. + * cached-descriptors.new. * * On startup, we read both files. */ diff --git a/src/feature/nodelist/routerstatus_st.h b/src/feature/nodelist/routerstatus_st.h index 714aa27435..ea06587799 100644 --- a/src/feature/nodelist/routerstatus_st.h +++ b/src/feature/nodelist/routerstatus_st.h @@ -47,6 +47,8 @@ struct routerstatus_t { unsigned int is_v2_dir:1; /** True iff this router publishes an open DirPort * or it claims to accept tunnelled dir requests. */ + unsigned int is_staledesc:1; /** True iff the authorities think this router + * should upload a new descriptor soon. */ unsigned int has_bandwidth:1; /**< The vote/consensus had bw info */ unsigned int has_exitsummary:1; /**< The vote/consensus had exit summaries */ diff --git a/src/feature/relay/dns.c b/src/feature/relay/dns.c index cc213ff8d6..5213296992 100644 --- a/src/feature/relay/dns.c +++ b/src/feature/relay/dns.c @@ -1360,6 +1360,42 @@ evdns_err_is_transient(int err) } } +/** + * Return number of configured nameservers in <b>the_evdns_base</b>. + */ +size_t +number_of_configured_nameservers(void) +{ + return evdns_base_count_nameservers(the_evdns_base); +} + +#ifdef HAVE_EVDNS_BASE_GET_NAMESERVER_ADDR +/** + * Return address of configured nameserver in <b>the_evdns_base</b> + * at index <b>idx</b>. + */ +tor_addr_t * +configured_nameserver_address(const size_t idx) +{ + struct sockaddr_storage sa; + ev_socklen_t sa_len = sizeof(sa); + + if (evdns_base_get_nameserver_addr(the_evdns_base, (int)idx, + (struct sockaddr *)&sa, + sa_len) > 0) { + tor_addr_t *tor_addr = tor_malloc(sizeof(tor_addr_t)); + if (tor_addr_from_sockaddr(tor_addr, + (const struct sockaddr *)&sa, + NULL) == 0) { + return tor_addr; + } + tor_free(tor_addr); + } + + return NULL; +} +#endif + /** Configure eventdns nameservers if force is true, or if the configuration * has changed since the last time we called this function, or if we failed on * our last attempt. On Unix, this reads from /etc/resolv.conf or @@ -1391,16 +1427,23 @@ configure_nameservers(int force) evdns_set_log_fn(evdns_log_cb); if (conf_fname) { log_debug(LD_FS, "stat()ing %s", conf_fname); - if (stat(sandbox_intern_string(conf_fname), &st)) { + int missing_resolv_conf = 0; + int stat_res = stat(sandbox_intern_string(conf_fname), &st); + + if (stat_res) { log_warn(LD_EXIT, "Unable to stat resolver configuration in '%s': %s", conf_fname, strerror(errno)); - goto err; - } - if (!force && resolv_conf_fname && !strcmp(conf_fname,resolv_conf_fname) + missing_resolv_conf = 1; + } else if (!force && resolv_conf_fname && + !strcmp(conf_fname,resolv_conf_fname) && st.st_mtime == resolv_conf_mtime) { log_info(LD_EXIT, "No change to '%s'", conf_fname); return 0; } + + if (stat_res == 0 && st.st_size == 0) + missing_resolv_conf = 1; + if (nameservers_configured) { evdns_base_search_clear(the_evdns_base); evdns_base_clear_nameservers_and_suspend(the_evdns_base); @@ -1413,20 +1456,34 @@ configure_nameservers(int force) sandbox_intern_string("/etc/hosts")); } #endif /* defined(DNS_OPTION_HOSTSFILE) && defined(USE_LIBSECCOMP) */ - log_info(LD_EXIT, "Parsing resolver configuration in '%s'", conf_fname); - if ((r = evdns_base_resolv_conf_parse(the_evdns_base, flags, - sandbox_intern_string(conf_fname)))) { - log_warn(LD_EXIT, "Unable to parse '%s', or no nameservers in '%s' (%d)", - conf_fname, conf_fname, r); - goto err; - } - if (evdns_base_count_nameservers(the_evdns_base) == 0) { - log_warn(LD_EXIT, "Unable to find any nameservers in '%s'.", conf_fname); - goto err; + + if (!missing_resolv_conf) { + log_info(LD_EXIT, "Parsing resolver configuration in '%s'", conf_fname); + if ((r = evdns_base_resolv_conf_parse(the_evdns_base, flags, + sandbox_intern_string(conf_fname)))) { + log_warn(LD_EXIT, "Unable to parse '%s', or no nameservers " + "in '%s' (%d)", conf_fname, conf_fname, r); + + if (r != 6) // "r = 6" means "no DNS servers were in resolv.conf" - + goto err; // in which case we expect libevent to add 127.0.0.1 as + // fallback. + } + if (evdns_base_count_nameservers(the_evdns_base) == 0) { + log_warn(LD_EXIT, "Unable to find any nameservers in '%s'.", + conf_fname); + } + + tor_free(resolv_conf_fname); + resolv_conf_fname = tor_strdup(conf_fname); + resolv_conf_mtime = st.st_mtime; + } else { + log_warn(LD_EXIT, "Could not read your DNS config from '%s' - " + "please investigate your DNS configuration. " + "This is possibly a problem. Meanwhile, falling" + " back to local DNS at 127.0.0.1.", conf_fname); + evdns_base_nameserver_ip_add(the_evdns_base, "127.0.0.1"); } - tor_free(resolv_conf_fname); - resolv_conf_fname = tor_strdup(conf_fname); - resolv_conf_mtime = st.st_mtime; + if (nameservers_configured) evdns_base_resume(the_evdns_base); } diff --git a/src/feature/relay/dns.h b/src/feature/relay/dns.h index 1dd6f903d1..5758ea4363 100644 --- a/src/feature/relay/dns.h +++ b/src/feature/relay/dns.h @@ -45,6 +45,11 @@ size_t dns_cache_handle_oom(time_t now, size_t min_remove_bytes); #ifdef DNS_PRIVATE #include "feature/relay/dns_structs.h" +size_t number_of_configured_nameservers(void); +#ifdef HAVE_EVDNS_BASE_GET_NAMESERVER_ADDR +tor_addr_t *configured_nameserver_address(const size_t idx); +#endif + MOCK_DECL(STATIC int,dns_resolve_impl,(edge_connection_t *exitconn, int is_resolve,or_circuit_t *oncirc, char **hostname_out, int *made_connection_pending_out, cached_resolve_t **resolve_out)); diff --git a/src/feature/relay/ext_orport.c b/src/feature/relay/ext_orport.c index 3607bdede4..0a649f2743 100644 --- a/src/feature/relay/ext_orport.c +++ b/src/feature/relay/ext_orport.c @@ -90,7 +90,7 @@ connection_ext_or_transition(or_connection_t *conn) conn->base_.type = CONN_TYPE_OR; TO_CONN(conn)->state = 0; // set the state to a neutral value - control_event_or_conn_status(conn, OR_CONN_EVENT_NEW, 0); + connection_or_event_status(conn, OR_CONN_EVENT_NEW, 0); connection_tls_start_handshake(conn, 1); } diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c index 4afcddc675..be9ef3d2f3 100644 --- a/src/feature/relay/router.c +++ b/src/feature/relay/router.c @@ -30,6 +30,7 @@ #include "feature/nodelist/dirlist.h" #include "feature/nodelist/networkstatus.h" #include "feature/nodelist/nickname.h" +#include "feature/nodelist/nodefamily.h" #include "feature/nodelist/nodelist.h" #include "feature/nodelist/routerlist.h" #include "feature/nodelist/torcert.h" @@ -49,6 +50,7 @@ #include "lib/encoding/confline.h" #include "lib/osinfo/uname.h" #include "lib/tls/tortls.h" +#include "lib/version/torversion.h" #include "feature/dirauth/authmode.h" @@ -58,6 +60,7 @@ #include "feature/dircommon/dir_connection_st.h" #include "feature/nodelist/authority_cert_st.h" #include "feature/nodelist/extrainfo_st.h" +#include "feature/nodelist/networkstatus_st.h" #include "feature/nodelist/node_st.h" #include "feature/nodelist/routerinfo_st.h" #include "feature/nodelist/routerstatus_st.h" @@ -339,6 +342,16 @@ set_server_identity_key(crypto_pk_t *k) } } +#ifdef TOR_UNIT_TESTS +/** Testing only -- set the server's RSA identity digest to + * be <b>digest</b> */ +void +set_server_identity_key_digest_testing(const uint8_t *digest) +{ + memcpy(server_identitykey_digest, digest, DIGEST_LEN); +} +#endif + /** Make sure that we have set up our identity keys to match or not match as * appropriate, and die with an assertion if we have not. */ static void @@ -636,7 +649,7 @@ load_authority_keyset(int legacy, crypto_pk_t **key_out, fname); goto done; } - parsed = authority_cert_parse_from_string(cert, &eos); + parsed = authority_cert_parse_from_string(cert, strlen(cert), &eos); if (!parsed) { log_warn(LD_DIR, "Unable to parse certificate in %s", fname); goto done; @@ -1469,9 +1482,9 @@ static extrainfo_t *desc_extrainfo = NULL; static const char *desc_gen_reason = "uninitialized reason"; /** Since when has our descriptor been "clean"? 0 if we need to regenerate it * now. */ -static time_t desc_clean_since = 0; +STATIC time_t desc_clean_since = 0; /** Why did we mark the descriptor dirty? */ -static const char *desc_dirty_reason = "Tor just started"; +STATIC const char *desc_dirty_reason = "Tor just started"; /** Boolean: do we need to regenerate the above? */ static int desc_needs_upload = 0; @@ -1690,10 +1703,6 @@ router_get_descriptor_gen_reason(void) return desc_gen_reason; } -/** A list of nicknames that we've warned about including in our family - * declaration verbatim rather than as digests. */ -static smartlist_t *warned_nonexistent_family = NULL; - static int router_guess_address_from_dir_headers(uint32_t *guess); /** Make a current best guess at our address, either because @@ -1806,6 +1815,132 @@ router_check_descriptor_address_consistency(uint32_t ipv4h_desc_addr) CONN_TYPE_DIR_LISTENER); } +/** A list of nicknames that we've warned about including in our family, + * for one reason or another. */ +static smartlist_t *warned_family = NULL; + +/** + * Return a new smartlist containing the family members configured in + * <b>options</b>. Warn about invalid or missing entries. Return NULL + * if this relay should not declare a family. + **/ +STATIC smartlist_t * +get_my_declared_family(const or_options_t *options) +{ + if (!options->MyFamily) + return NULL; + + if (options->BridgeRelay) + return NULL; + + if (!warned_family) + warned_family = smartlist_new(); + + smartlist_t *declared_family = smartlist_new(); + config_line_t *family; + + /* First we try to get the whole family in the form of hexdigests. */ + for (family = options->MyFamily; family; family = family->next) { + char *name = family->value; + const node_t *member; + if (options->Nickname && !strcasecmp(name, options->Nickname)) + continue; /* Don't list ourself by nickname, that's redundant */ + else + member = node_get_by_nickname(name, 0); + + if (!member) { + /* This node doesn't seem to exist, so warn about it if it is not + * a hexdigest. */ + int is_legal = is_legal_nickname_or_hexdigest(name); + if (!smartlist_contains_string(warned_family, name) && + !is_legal_hexdigest(name)) { + if (is_legal) + log_warn(LD_CONFIG, + "There is a router named %s in my declared family, but " + "I have no descriptor for it. I'll use the nickname " + "as is, but this may confuse clients. Please list it " + "by identity digest instead.", escaped(name)); + else + log_warn(LD_CONFIG, "There is a router named %s in my declared " + "family, but that isn't a legal digest or nickname. " + "Skipping it.", escaped(name)); + smartlist_add_strdup(warned_family, name); + } + if (is_legal) { + smartlist_add_strdup(declared_family, name); + } + } else { + /* List the node by digest. */ + char *fp = tor_malloc(HEX_DIGEST_LEN+2); + fp[0] = '$'; + base16_encode(fp+1,HEX_DIGEST_LEN+1, + member->identity, DIGEST_LEN); + smartlist_add(declared_family, fp); + + if (! is_legal_hexdigest(name) && + !smartlist_contains_string(warned_family, name)) { + /* Warn if this node was not specified by hexdigest. */ + log_warn(LD_CONFIG, "There is a router named %s in my declared " + "family, but it wasn't listed by digest. Please consider " + "saying %s instead, if that's what you meant.", + escaped(name), fp); + smartlist_add_strdup(warned_family, name); + } + } + } + + /* Now declared_family should have the closest we can come to the + * identities that the user wanted. + * + * Unlike older versions of Tor, we _do_ include our own identity: this + * helps microdescriptor compression, and helps in-memory compression + * on clients. */ + nodefamily_t *nf = nodefamily_from_members(declared_family, + router_get_my_id_digest(), + NF_WARN_MALFORMED, + NULL); + SMARTLIST_FOREACH(declared_family, char *, s, tor_free(s)); + smartlist_free(declared_family); + if (!nf) { + return NULL; + } + + char *s = nodefamily_format(nf); + nodefamily_free(nf); + + smartlist_t *result = smartlist_new(); + smartlist_split_string(result, s, NULL, + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + tor_free(s); + + if (smartlist_len(result) == 1) { + /* This is a one-element list containing only ourself; instead return + * nothing */ + const char *singleton = smartlist_get(result, 0); + bool is_me = false; + if (singleton[0] == '$') { + char d[DIGEST_LEN]; + int n = base16_decode(d, sizeof(d), singleton+1, strlen(singleton+1)); + if (n == DIGEST_LEN && + fast_memeq(d, router_get_my_id_digest(), DIGEST_LEN)) { + is_me = true; + } + } + if (!is_me) { + // LCOV_EXCL_START + log_warn(LD_BUG, "Found a singleton family list with an element " + "that wasn't us! Element was %s", escaped(singleton)); + // LCOV_EXCL_STOP + } else { + SMARTLIST_FOREACH(result, char *, cp, tor_free(cp)); + smartlist_free(result); + return NULL; + } + } + + return result; +} + /** Build a fresh routerinfo, signed server descriptor, and extra-info document * for this OR. Set r to the generated routerinfo, e to the generated * extra-info document. Return 0 on success, -1 on temporary error. Failure to @@ -1920,54 +2055,7 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e) tor_free(p_tmp); } - if (options->MyFamily && ! options->BridgeRelay) { - if (!warned_nonexistent_family) - warned_nonexistent_family = smartlist_new(); - ri->declared_family = smartlist_new(); - config_line_t *family; - for (family = options->MyFamily; family; family = family->next) { - char *name = family->value; - const node_t *member; - if (!strcasecmp(name, options->Nickname)) - continue; /* Don't list ourself, that's redundant */ - else - member = node_get_by_nickname(name, 0); - if (!member) { - int is_legal = is_legal_nickname_or_hexdigest(name); - if (!smartlist_contains_string(warned_nonexistent_family, name) && - !is_legal_hexdigest(name)) { - if (is_legal) - log_warn(LD_CONFIG, - "I have no descriptor for the router named \"%s\" in my " - "declared family; I'll use the nickname as is, but " - "this may confuse clients.", name); - else - log_warn(LD_CONFIG, "There is a router named \"%s\" in my " - "declared family, but that isn't a legal nickname. " - "Skipping it.", escaped(name)); - smartlist_add_strdup(warned_nonexistent_family, name); - } - if (is_legal) { - smartlist_add_strdup(ri->declared_family, name); - } - } else if (router_digest_is_me(member->identity)) { - /* Don't list ourself in our own family; that's redundant */ - /* XXX shouldn't be possible */ - } else { - char *fp = tor_malloc(HEX_DIGEST_LEN+2); - fp[0] = '$'; - base16_encode(fp+1,HEX_DIGEST_LEN+1, - member->identity, DIGEST_LEN); - smartlist_add(ri->declared_family, fp); - if (smartlist_contains_string(warned_nonexistent_family, name)) - smartlist_string_remove(warned_nonexistent_family, name); - } - } - - /* remove duplicates from the list */ - smartlist_sort_strings(ri->declared_family); - smartlist_uniq_strings(ri->declared_family); - } + ri->declared_family = get_my_declared_family(options); /* Now generate the extrainfo. */ ei = tor_malloc_zero(sizeof(extrainfo_t)); @@ -2133,7 +2221,9 @@ mark_my_descriptor_dirty_if_too_old(time_t now) /* Now we see whether we want to be retrying frequently or no. The * rule here is that we'll retry frequently if we aren't listed in the * live consensus we have, or if the publication time of the - * descriptor listed for us in the consensus is very old. */ + * descriptor listed for us in the consensus is very old, or if the + * consensus lists us as "stale" and we haven't regenerated since the + * consensus was published. */ ns = networkstatus_get_live_consensus(now); if (ns) { rs = networkstatus_vote_find_entry(ns, server_identitykey_digest); @@ -2141,6 +2231,8 @@ mark_my_descriptor_dirty_if_too_old(time_t now) retry_fast_reason = "not listed in consensus"; else if (rs->published_on < slow_cutoff) retry_fast_reason = "version listed in consensus is quite old"; + else if (rs->is_staledesc && ns->valid_after > desc_clean_since) + retry_fast_reason = "listed as stale in consensus"; } if (retry_fast_reason && desc_clean_since < fast_cutoff) @@ -3058,9 +3150,9 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo, void router_reset_warnings(void) { - if (warned_nonexistent_family) { - SMARTLIST_FOREACH(warned_nonexistent_family, char *, cp, tor_free(cp)); - smartlist_clear(warned_nonexistent_family); + if (warned_family) { + SMARTLIST_FOREACH(warned_family, char *, cp, tor_free(cp)); + smartlist_clear(warned_family); } } @@ -3084,11 +3176,12 @@ router_free_all(void) memwipe(&curve25519_onion_key, 0, sizeof(curve25519_onion_key)); memwipe(&last_curve25519_onion_key, 0, sizeof(last_curve25519_onion_key)); - if (warned_nonexistent_family) { - SMARTLIST_FOREACH(warned_nonexistent_family, char *, cp, tor_free(cp)); - smartlist_free(warned_nonexistent_family); + if (warned_family) { + SMARTLIST_FOREACH(warned_family, char *, cp, tor_free(cp)); + smartlist_free(warned_family); } } + /* From the given RSA key object, convert it to ASN-1 encoded format and set * the newly allocated object in onion_pkey_out. The length of the key is set * in onion_pkey_len_out. */ diff --git a/src/feature/relay/router.h b/src/feature/relay/router.h index 4575172afb..7a63c33d32 100644 --- a/src/feature/relay/router.h +++ b/src/feature/relay/router.h @@ -117,6 +117,14 @@ void router_free_all(void); /* Used only by router.c and test.c */ STATIC void get_platform_str(char *platform, size_t len); STATIC int router_write_fingerprint(int hashed); +STATIC smartlist_t *get_my_declared_family(const or_options_t *options); + +#ifdef TOR_UNIT_TESTS +extern time_t desc_clean_since; +extern const char *desc_dirty_reason; +void set_server_identity_key_digest_testing(const uint8_t *digest); +#endif + #endif #endif /* !defined(TOR_ROUTER_H) */ diff --git a/src/feature/rend/rendcache.c b/src/feature/rend/rendcache.c index 848386b97d..b851e71959 100644 --- a/src/feature/rend/rendcache.c +++ b/src/feature/rend/rendcache.c @@ -45,7 +45,7 @@ STATIC digestmap_t *rend_cache_v2_dir = NULL; * looked up in this cache and if present, it is discarded from the fetched * descriptor. At the end, all IP(s) in the cache, for a specific service * ID, that were NOT present in the descriptor are removed from this cache. - * Which means that if at least one IP was not in this cache, thus usuable, + * Which means that if at least one IP was not in this cache, thus usable, * it's considered a new descriptor so we keep it. Else, if all IPs were in * this cache, we discard the descriptor as it's considered unusable. * diff --git a/src/feature/rend/rendmid.c b/src/feature/rend/rendmid.c index 8ca19a2522..67a8b5f8f4 100644 --- a/src/feature/rend/rendmid.c +++ b/src/feature/rend/rendmid.c @@ -236,8 +236,8 @@ rend_mid_establish_rendezvous(or_circuit_t *circ, const uint8_t *request, goto err; } - /* Check if we are configured to accept established rendezvous cells from - * client or in other words Tor2Web clients. */ + /* Check if we are configured to defend ourselves from clients that + * attempt to establish rendezvous points directly to us. */ if (channel_is_client(circ->p_chan) && dos_should_refuse_single_hop_client()) { /* Note it down for the heartbeat log purposes. */ diff --git a/src/feature/stats/geoip_stats.c b/src/feature/stats/geoip_stats.c index 1a4f8ddfb0..3106c6c82c 100644 --- a/src/feature/stats/geoip_stats.c +++ b/src/feature/stats/geoip_stats.c @@ -30,7 +30,7 @@ #include "core/or/or.h" #include "ht.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "app/config/config.h" #include "feature/control/control.h" #include "feature/client/dnsserv.h" diff --git a/src/include.am b/src/include.am index d2f83da814..9070a69a03 100644 --- a/src/include.am +++ b/src/include.am @@ -1,5 +1,6 @@ include src/ext/include.am include src/lib/arch/include.am +include src/lib/buf/include.am include src/lib/err/include.am include src/lib/cc/include.am include src/lib/ctime/include.am @@ -25,6 +26,7 @@ include src/lib/osinfo/include.am include src/lib/process/include.am include src/lib/sandbox/include.am include src/lib/string/include.am +include src/lib/subsys/include.am include src/lib/smartlist_core/include.am include src/lib/term/include.am include src/lib/testsupport/include.am @@ -32,6 +34,7 @@ include src/lib/thread/include.am include src/lib/time/include.am include src/lib/tls/include.am include src/lib/trace/include.am +include src/lib/version/include.am include src/lib/wallclock/include.am include src/trunnel/include.am diff --git a/src/lib/buf/.may_include b/src/lib/buf/.may_include new file mode 100644 index 0000000000..c4be73bce2 --- /dev/null +++ b/src/lib/buf/.may_include @@ -0,0 +1,10 @@ +orconfig.h + +lib/buf/*.h +lib/cc/*.h +lib/ctime/*.h +lib/malloc/*.h +lib/testsupport/*.h +lib/log/*.h +lib/string/*.h +lib/time/*.h diff --git a/src/lib/container/buffers.c b/src/lib/buf/buffers.c index 87d782d7ef..495c5ec453 100644 --- a/src/lib/container/buffers.c +++ b/src/lib/buf/buffers.c @@ -25,7 +25,7 @@ #define BUFFERS_PRIVATE #include "orconfig.h" #include <stddef.h> -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/cc/torint.h" #include "lib/log/log.h" #include "lib/log/util_bug.h" diff --git a/src/lib/container/buffers.h b/src/lib/buf/buffers.h index c48f83cfc7..c48f83cfc7 100644 --- a/src/lib/container/buffers.h +++ b/src/lib/buf/buffers.h diff --git a/src/lib/buf/include.am b/src/lib/buf/include.am new file mode 100644 index 0000000000..3338c3dbdb --- /dev/null +++ b/src/lib/buf/include.am @@ -0,0 +1,17 @@ + +noinst_LIBRARIES += src/lib/libtor-buf.a + +if UNITTESTS_ENABLED +noinst_LIBRARIES += src/lib/libtor-buf-testing.a +endif + +src_lib_libtor_buf_a_SOURCES = \ + src/lib/buf/buffers.c + +src_lib_libtor_buf_testing_a_SOURCES = \ + $(src_lib_libtor_buf_a_SOURCES) +src_lib_libtor_buf_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) +src_lib_libtor_buf_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) + +noinst_HEADERS += \ + src/lib/buf/buffers.h diff --git a/src/lib/cc/.may_include b/src/lib/cc/.may_include index 2b06e8519c..fa1478ce46 100644 --- a/src/lib/cc/.may_include +++ b/src/lib/cc/.may_include @@ -1 +1,2 @@ orconfig.h +lib/cc/*.h
\ No newline at end of file diff --git a/src/lib/cc/ctassert.h b/src/lib/cc/ctassert.h new file mode 100644 index 0000000000..e42976360f --- /dev/null +++ b/src/lib/cc/ctassert.h @@ -0,0 +1,53 @@ +/* Copyright (c) 2018 The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file ctassert.h + * + * \brief Compile-time assertions: CTASSERT(expression). + */ + +#ifndef TOR_CTASSERT_H +#define TOR_CTASSERT_H + +#include "lib/cc/compat_compiler.h" + +/** + * CTASSERT(expression) + * + * Trigger a compiler error if expression is false. + */ +#if __STDC_VERSION__ >= 201112L + +/* If C11 is available, just use _Static_assert. */ +#define CTASSERT(x) _Static_assert((x), #x) + +#else + +/* + * If C11 is not available, expand __COUNTER__, or __INCLUDE_LEVEL__ + * and __LINE__, or just __LINE__, with an intermediate preprocessor + * macro CTASSERT_EXPN, and then use CTASSERT_DECL to paste the + * expansions together into a unique name. + * + * We use this name as a typedef of an array type with a positive + * length if the assertion is true, and a negative length of the + * assertion is false, which is invalid and hence triggers a compiler + * error. + */ +#if defined(__COUNTER__) +#define CTASSERT(x) CTASSERT_EXPN((x), c, __COUNTER__) +#elif defined(__INCLUDE_LEVEL__) +#define CTASSERT(x) CTASSERT_EXPN((x), __INCLUDE_LEVEL__, __LINE__) +#else +/* hope it's unique enough */ +#define CTASSERT(x) CTASSERT_EXPN((x), l, __LINE__) +#endif + +#define CTASSERT_EXPN(x, a, b) CTASSERT_DECL(x, a, b) +#define CTASSERT_DECL(x, a, b) \ + typedef char tor_ctassert_##a##_##b[(x) ? 1 : -1] ATTR_UNUSED + +#endif + +#endif /* !defined(TOR_CTASSERT_H) */ diff --git a/src/lib/cc/include.am b/src/lib/cc/include.am index 2ae90f97dd..52cf8a9f72 100644 --- a/src/lib/cc/include.am +++ b/src/lib/cc/include.am @@ -1,4 +1,5 @@ noinst_HEADERS += \ src/lib/cc/compat_compiler.h \ + src/lib/cc/ctassert.h \ src/lib/cc/torint.h diff --git a/src/lib/compress/.may_include b/src/lib/compress/.may_include index 68fe9f1c54..6cd80086e6 100644 --- a/src/lib/compress/.may_include +++ b/src/lib/compress/.may_include @@ -1,5 +1,6 @@ orconfig.h lib/arch/*.h +lib/buf/*.h lib/cc/*.h lib/compress/*.h lib/container/*.h @@ -8,5 +9,6 @@ lib/intmath/*.h lib/log/*.h lib/malloc/*.h lib/string/*.h +lib/subsys/*.h lib/testsupport/*.h lib/thread/*.h diff --git a/src/lib/compress/compress.c b/src/lib/compress/compress.c index 2ad9b15b2e..6cb9bd492b 100644 --- a/src/lib/compress/compress.c +++ b/src/lib/compress/compress.c @@ -29,10 +29,12 @@ #include "lib/compress/compress.h" #include "lib/compress/compress_lzma.h" #include "lib/compress/compress_none.h" +#include "lib/compress/compress_sys.h" #include "lib/compress/compress_zlib.h" #include "lib/compress/compress_zstd.h" #include "lib/intmath/cmp.h" #include "lib/malloc/malloc.h" +#include "lib/subsys/subsys.h" #include "lib/thread/threads.h" /** Total number of bytes allocated for compression state overhead. */ @@ -660,7 +662,7 @@ tor_compress_state_size(const tor_compress_state_t *state) } /** Initialize all compression modules. */ -void +int tor_compress_init(void) { atomic_counter_init(&total_compress_allocation); @@ -668,6 +670,8 @@ tor_compress_init(void) tor_zlib_init(); tor_lzma_init(); tor_zstd_init(); + + return 0; } /** Warn if we had any problems while setting up our compression libraries. @@ -677,5 +681,20 @@ tor_compress_init(void) void tor_compress_log_init_warnings(void) { + // XXXX can we move this into tor_compress_init() after all? log.c queues + // XXXX log messages at startup. tor_zstd_warn_if_version_mismatched(); } + +static int +subsys_compress_initialize(void) +{ + return tor_compress_init(); +} + +const subsys_fns_t sys_compress = { + .name = "compress", + .supported = true, + .level = -70, + .initialize = subsys_compress_initialize, +}; diff --git a/src/lib/compress/compress.h b/src/lib/compress/compress.h index 4466e27c4d..4dd6506238 100644 --- a/src/lib/compress/compress.h +++ b/src/lib/compress/compress.h @@ -89,7 +89,7 @@ void tor_compress_free_(tor_compress_state_t *state); size_t tor_compress_state_size(const tor_compress_state_t *state); -void tor_compress_init(void); +int tor_compress_init(void); void tor_compress_log_init_warnings(void); struct buf_t; diff --git a/src/lib/compress/compress_buf.c b/src/lib/compress/compress_buf.c index 63ee9e0102..ecf76ee078 100644 --- a/src/lib/compress/compress_buf.c +++ b/src/lib/compress/compress_buf.c @@ -11,7 +11,7 @@ #define BUFFERS_PRIVATE #include "lib/cc/compat_compiler.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/compress/compress.h" #include "lib/log/util_bug.h" diff --git a/src/lib/compress/compress_sys.h b/src/lib/compress/compress_sys.h new file mode 100644 index 0000000000..a162140cfb --- /dev/null +++ b/src/lib/compress/compress_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_sys.h + * \brief Declare subsystem object for the compress module + **/ + +#ifndef TOR_COMPRESS_SYS_H +#define TOR_COMPRESS_SYS_H + +extern const struct subsys_fns_t sys_compress; + +#endif /* !defined(TOR_COMPRESS_SYS_H) */ diff --git a/src/lib/compress/include.am b/src/lib/compress/include.am index 75c9032bd2..b952779578 100644 --- a/src/lib/compress/include.am +++ b/src/lib/compress/include.am @@ -22,5 +22,6 @@ noinst_HEADERS += \ src/lib/compress/compress.h \ src/lib/compress/compress_lzma.h \ src/lib/compress/compress_none.h \ + src/lib/compress/compress_sys.h \ src/lib/compress/compress_zlib.h \ src/lib/compress/compress_zstd.h diff --git a/src/lib/container/.may_include b/src/lib/container/.may_include index 90de5eda40..81507527d3 100644 --- a/src/lib/container/.may_include +++ b/src/lib/container/.may_include @@ -7,12 +7,9 @@ lib/malloc/*.h lib/err/*.h lib/smartlist_core/*.h lib/string/*.h -lib/testsupport/testsupport.h +lib/testsupport/*.h lib/intmath/*.h lib/log/*.h -# XXXX I am unsure about this one. It's only here for buffers.c -lib/time/*.h - -ht.h -siphash.h +ext/ht.h +ext/siphash.h diff --git a/src/lib/container/bloomfilt.c b/src/lib/container/bloomfilt.c index ea2d2917c7..a64fcb9300 100644 --- a/src/lib/container/bloomfilt.c +++ b/src/lib/container/bloomfilt.c @@ -14,7 +14,7 @@ #include "lib/container/bloomfilt.h" #include "lib/intmath/bits.h" #include "lib/log/util_bug.h" -#include "siphash.h" +#include "ext/siphash.h" /** How many bloom-filter bits we set per address. This is twice the * BLOOMFILT_N_HASHES value, since we split the siphash output into two 32-bit diff --git a/src/lib/container/include.am b/src/lib/container/include.am index e6492098b5..032e4033da 100644 --- a/src/lib/container/include.am +++ b/src/lib/container/include.am @@ -7,7 +7,6 @@ endif src_lib_libtor_container_a_SOURCES = \ src/lib/container/bloomfilt.c \ - src/lib/container/buffers.c \ src/lib/container/map.c \ src/lib/container/order.c \ src/lib/container/smartlist.c @@ -20,7 +19,6 @@ src_lib_libtor_container_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) noinst_HEADERS += \ src/lib/container/bitarray.h \ src/lib/container/bloomfilt.h \ - src/lib/container/buffers.h \ src/lib/container/handles.h \ src/lib/container/map.h \ src/lib/container/order.h \ diff --git a/src/lib/container/map.c b/src/lib/container/map.c index 85e074e10a..137e316920 100644 --- a/src/lib/container/map.c +++ b/src/lib/container/map.c @@ -21,7 +21,7 @@ #include <stdlib.h> #include <string.h> -#include "ht.h" +#include "ext/ht.h" /** Helper: Declare an entry type and a map type to implement a mapping using * ht.h. The map type will be called <b>maptype</b>. The key part of each diff --git a/src/lib/container/map.h b/src/lib/container/map.h index ff71622682..4f427fe767 100644 --- a/src/lib/container/map.h +++ b/src/lib/container/map.h @@ -15,7 +15,7 @@ #include "lib/testsupport/testsupport.h" #include "lib/cc/torint.h" -#include "siphash.h" +#include "ext/siphash.h" #define DECLARE_MAP_FNS(maptype, keytype, prefix) \ typedef struct maptype maptype; \ diff --git a/src/lib/crypt_ops/.may_include b/src/lib/crypt_ops/.may_include index a0fa4ec05c..0739699686 100644 --- a/src/lib/crypt_ops/.may_include +++ b/src/lib/crypt_ops/.may_include @@ -12,7 +12,8 @@ lib/malloc/*.h lib/intmath/*.h lib/sandbox/*.h lib/string/*.h -lib/testsupport/testsupport.h +lib/subsys/*.h +lib/testsupport/*.h lib/thread/*.h lib/log/*.h @@ -21,4 +22,4 @@ trunnel/pwbox.h keccak-tiny/*.h ed25519/*.h -siphash.h +ext/siphash.h diff --git a/src/lib/crypt_ops/crypto_init.c b/src/lib/crypt_ops/crypto_init.c index 9d6e2da0d0..f9943939fb 100644 --- a/src/lib/crypt_ops/crypto_init.c +++ b/src/lib/crypt_ops/crypto_init.c @@ -20,8 +20,11 @@ #include "lib/crypt_ops/crypto_openssl_mgt.h" #include "lib/crypt_ops/crypto_nss_mgt.h" #include "lib/crypt_ops/crypto_rand.h" +#include "lib/crypt_ops/crypto_sys.h" -#include "siphash.h" +#include "lib/subsys/subsys.h" + +#include "ext/siphash.h" /** Boolean: has our crypto library been initialized? (early phase) */ static int crypto_early_initialized_ = 0; @@ -202,3 +205,47 @@ tor_is_using_nss(void) return 0; #endif } + +static int +subsys_crypto_initialize(void) +{ + if (crypto_early_init() < 0) + return -1; + crypto_dh_init(); + return 0; +} + +static void +subsys_crypto_shutdown(void) +{ + crypto_global_cleanup(); +} + +static void +subsys_crypto_prefork(void) +{ + crypto_prefork(); +} + +static void +subsys_crypto_postfork(void) +{ + crypto_postfork(); +} + +static void +subsys_crypto_thread_cleanup(void) +{ + crypto_thread_cleanup(); +} + +const struct subsys_fns_t sys_crypto = { + .name = "crypto", + .supported = true, + .level = -60, + .initialize = subsys_crypto_initialize, + .shutdown = subsys_crypto_shutdown, + .prefork = subsys_crypto_prefork, + .postfork = subsys_crypto_postfork, + .thread_cleanup = subsys_crypto_thread_cleanup, +}; diff --git a/src/lib/crypt_ops/crypto_sys.h b/src/lib/crypt_ops/crypto_sys.h new file mode 100644 index 0000000000..31644d088b --- /dev/null +++ b/src/lib/crypt_ops/crypto_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file log_crypto.h + * \brief Declare subsystem object for the crypto module. + **/ + +#ifndef TOR_CRYPTO_SYS_H +#define TOR_CRYPTO_SYS_H + +extern const struct subsys_fns_t sys_crypto; + +#endif /* !defined(TOR_CRYPTO_SYS_H) */ diff --git a/src/lib/crypt_ops/digestset.c b/src/lib/crypt_ops/digestset.c index 89dd377a9c..84516e0172 100644 --- a/src/lib/crypt_ops/digestset.c +++ b/src/lib/crypt_ops/digestset.c @@ -11,7 +11,7 @@ #include "lib/crypt_ops/crypto_rand.h" #include "lib/defs/digest_sizes.h" #include "lib/crypt_ops/digestset.h" -#include "siphash.h" +#include "ext/siphash.h" /* Wrap our hash function to have the signature that the bloom filter * needs. */ diff --git a/src/lib/crypt_ops/include.am b/src/lib/crypt_ops/include.am index 1022096fdc..d0ccc13bff 100644 --- a/src/lib/crypt_ops/include.am +++ b/src/lib/crypt_ops/include.am @@ -66,5 +66,6 @@ noinst_HEADERS += \ src/lib/crypt_ops/crypto_rand.h \ src/lib/crypt_ops/crypto_rsa.h \ src/lib/crypt_ops/crypto_s2k.h \ + src/lib/crypt_ops/crypto_sys.h \ src/lib/crypt_ops/crypto_util.h \ src/lib/crypt_ops/digestset.h diff --git a/src/lib/encoding/.may_include b/src/lib/encoding/.may_include index 7c2ef36929..c9bf4b1786 100644 --- a/src/lib/encoding/.may_include +++ b/src/lib/encoding/.may_include @@ -1,5 +1,6 @@ orconfig.h lib/cc/*.h +lib/container/*.h lib/ctime/*.h lib/encoding/*.h lib/intmath/*.h diff --git a/src/lib/encoding/binascii.c b/src/lib/encoding/binascii.c index e9140d9432..067db075ad 100644 --- a/src/lib/encoding/binascii.c +++ b/src/lib/encoding/binascii.c @@ -179,6 +179,18 @@ base64_encode_size(size_t srclen, int flags) return enclen; } +/** Return an upper bound on the number of bytes that might be needed to hold + * the data from decoding the base64 string <b>srclen</b>. This is only an + * upper bound, since some part of the base64 string might be padding or + * space. */ +size_t +base64_decode_maxsize(size_t srclen) +{ + tor_assert(srclen < INT_MAX / 3); + + return CEIL_DIV(srclen * 3, 4); +} + /** Internal table mapping 6 bit values to the Base64 alphabet. */ static const char base64_encode_table[64] = { 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', diff --git a/src/lib/encoding/binascii.h b/src/lib/encoding/binascii.h index 23cbaa7070..c71ba65dfb 100644 --- a/src/lib/encoding/binascii.h +++ b/src/lib/encoding/binascii.h @@ -42,6 +42,7 @@ const char *hex_str(const char *from, size_t fromlen); #define BASE64_ENCODE_MULTILINE 1 size_t base64_encode_size(size_t srclen, int flags); +size_t base64_decode_maxsize(size_t srclen); int base64_encode(char *dest, size_t destlen, const char *src, size_t srclen, int flags); int base64_decode(char *dest, size_t destlen, const char *src, size_t srclen); diff --git a/src/lib/encoding/include.am b/src/lib/encoding/include.am index 2d2aa3988a..83e9211b6f 100644 --- a/src/lib/encoding/include.am +++ b/src/lib/encoding/include.am @@ -9,6 +9,7 @@ src_lib_libtor_encoding_a_SOURCES = \ src/lib/encoding/confline.c \ src/lib/encoding/cstring.c \ src/lib/encoding/keyval.c \ + src/lib/encoding/kvline.c \ src/lib/encoding/pem.c \ src/lib/encoding/time_fmt.c @@ -22,5 +23,6 @@ noinst_HEADERS += \ src/lib/encoding/confline.h \ src/lib/encoding/cstring.h \ src/lib/encoding/keyval.h \ + src/lib/encoding/kvline.h \ src/lib/encoding/pem.h \ src/lib/encoding/time_fmt.h diff --git a/src/lib/encoding/kvline.c b/src/lib/encoding/kvline.c new file mode 100644 index 0000000000..11ff4f0f96 --- /dev/null +++ b/src/lib/encoding/kvline.c @@ -0,0 +1,239 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file kvline.c + * + * \brief Manipulating lines of key-value pairs. + **/ + +#include "orconfig.h" + +#include "lib/container/smartlist.h" +#include "lib/encoding/confline.h" +#include "lib/encoding/cstring.h" +#include "lib/encoding/kvline.h" +#include "lib/malloc/malloc.h" +#include "lib/string/compat_ctype.h" +#include "lib/string/printf.h" +#include "lib/string/util_string.h" +#include "lib/log/escape.h" +#include "lib/log/util_bug.h" + +#include <stdbool.h> +#include <stddef.h> +#include <string.h> + +/** Return true iff we need to quote and escape the string <b>s</b> to encode + * it. */ +static bool +needs_escape(const char *s, bool as_keyless_val) +{ + if (as_keyless_val && *s == 0) + return true; + + for (; *s; ++s) { + if (*s >= 127 || TOR_ISSPACE(*s) || ! TOR_ISPRINT(*s) || + *s == '\'' || *s == '\"') { + return true; + } + } + return false; +} + +/** + * Return true iff the key in <b>line</b> is not set. + **/ +static bool +line_has_no_key(const config_line_t *line) +{ + return line->key == NULL || strlen(line->key) == 0; +} + +/** + * Return true iff the all the lines in <b>line</b> can be encoded + * using <b>flags</b>. + **/ +static bool +kvline_can_encode_lines(const config_line_t *line, unsigned flags) +{ + for ( ; line; line = line->next) { + const bool keyless = line_has_no_key(line); + if (keyless) { + if (! (flags & KV_OMIT_KEYS)) { + /* If KV_OMIT_KEYS is not set, we can't encode a line with no key. */ + return false; + } + if (strchr(line->value, '=') && !( flags & KV_QUOTED)) { + /* We can't have a keyless value with = without quoting it. */ + return false; + } + } + + if (needs_escape(line->value, keyless) && ! (flags & KV_QUOTED)) { + /* If KV_QUOTED is false, we can't encode a value that needs quotes. */ + return false; + } + if (line->key && strlen(line->key) && + (needs_escape(line->key, false) || strchr(line->key, '='))) { + /* We can't handle keys that need quoting. */ + return false; + } + } + return true; +} + +/** + * Encode a linked list of lines in <b>line</b> as a series of 'Key=Value' + * pairs, using the provided <b>flags</b> to encode it. Return a newly + * allocated string on success, or NULL on failure. + * + * If KV_QUOTED is set in <b>flags</b>, then all values that contain + * spaces or unusual characters are escaped and quoted. Otherwise, such + * values are not allowed. + * + * If KV_OMIT_KEYS is set in <b>flags</b>, then pairs with empty keys are + * allowed, and are encoded as 'Value'. Otherwise, such pairs are not + * allowed. + */ +char * +kvline_encode(const config_line_t *line, + unsigned flags) +{ + if (!kvline_can_encode_lines(line, flags)) + return NULL; + + smartlist_t *elements = smartlist_new(); + + for (; line; line = line->next) { + + const char *k = ""; + const char *eq = "="; + const char *v = ""; + const bool keyless = line_has_no_key(line); + bool esc = needs_escape(line->value, keyless); + char *tmp = NULL; + + if (! keyless) { + k = line->key; + } else { + eq = ""; + if (strchr(line->value, '=')) { + esc = true; + } + } + + if (esc) { + tmp = esc_for_log(line->value); + v = tmp; + } else { + v = line->value; + } + + smartlist_add_asprintf(elements, "%s%s%s", k, eq, v); + tor_free(tmp); + } + + char *result = smartlist_join_strings(elements, " ", 0, NULL); + + SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp)); + smartlist_free(elements); + + return result; +} + +/** + * Decode a <b>line</b> containing a series of space-separated 'Key=Value' + * pairs, using the provided <b>flags</b> to decode it. Return a newly + * allocated list of pairs on success, or NULL on failure. + * + * If KV_QUOTED is set in <b>flags</b>, then (double-)quoted values are + * allowed. Otherwise, such values are not allowed. + * + * If KV_OMIT_KEYS is set in <b>flags</b>, then values without keys are + * allowed. Otherwise, such values are not allowed. + */ +config_line_t * +kvline_parse(const char *line, unsigned flags) +{ + const char *cp = line, *cplast = NULL; + bool omit_keys = (flags & KV_OMIT_KEYS) != 0; + bool quoted = (flags & KV_QUOTED) != 0; + + config_line_t *result = NULL; + config_line_t **next_line = &result; + + char *key = NULL; + char *val = NULL; + + while (*cp) { + key = val = NULL; + { + size_t idx = strspn(cp, " \t\r\v\n"); + cp += idx; + } + if (BUG(cp == cplast)) { + /* If we didn't parse anything, this code is broken. */ + goto err; // LCOV_EXCL_LINE + } + cplast = cp; + if (! *cp) + break; /* End of string; we're done. */ + + /* Possible formats are K=V, K="V", V, and "V", depending on flags. */ + + /* Find the key. */ + if (*cp != '\"') { + size_t idx = strcspn(cp, " \t\r\v\n="); + + if (cp[idx] == '=') { + key = tor_memdup_nulterm(cp, idx); + cp += idx + 1; + } else { + if (!omit_keys) + goto err; + } + } + + if (*cp == '\"') { + /* The type is "V". */ + if (!quoted) + goto err; + size_t len=0; + cp = unescape_string(cp, &val, &len); + if (cp == NULL || len != strlen(val)) { + // The string contains a NUL or is badly coded. + goto err; + } + } else { + size_t idx = strcspn(cp, " \t\r\v\n"); + val = tor_memdup_nulterm(cp, idx); + cp += idx; + } + + if (key && strlen(key) == 0) { + /* We don't allow empty keys. */ + goto err; + } + + *next_line = tor_malloc_zero(sizeof(config_line_t)); + (*next_line)->key = key ? key : tor_strdup(""); + (*next_line)->value = val; + next_line = &(*next_line)->next; + key = val = NULL; + } + + if (!kvline_can_encode_lines(result, flags)) { + goto err; + } + return result; + + err: + tor_free(key); + tor_free(val); + config_free_lines(result); + return NULL; +} diff --git a/src/lib/encoding/kvline.h b/src/lib/encoding/kvline.h new file mode 100644 index 0000000000..3272cc1754 --- /dev/null +++ b/src/lib/encoding/kvline.h @@ -0,0 +1,24 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file kvline.h + * + * \brief Header for kvline.c + **/ + +#ifndef TOR_KVLINE_H +#define TOR_KVLINE_H + +struct config_line_t; + +#define KV_QUOTED (1u<<0) +#define KV_OMIT_KEYS (1u<<1) + +struct config_line_t *kvline_parse(const char *line, unsigned flags); +char *kvline_encode(const struct config_line_t *line, unsigned flags); + +#endif /* !defined(TOR_KVLINE_H) */ diff --git a/src/lib/err/.may_include b/src/lib/err/.may_include index 48cc0ef088..daa1b6e4ca 100644 --- a/src/lib/err/.may_include +++ b/src/lib/err/.may_include @@ -1,3 +1,5 @@ orconfig.h lib/cc/*.h lib/err/*.h +lib/subsys/*.h +lib/version/*.h
\ No newline at end of file diff --git a/src/lib/err/include.am b/src/lib/err/include.am index f2a409c51e..43adcd2694 100644 --- a/src/lib/err/include.am +++ b/src/lib/err/include.am @@ -6,8 +6,9 @@ noinst_LIBRARIES += src/lib/libtor-err-testing.a endif src_lib_libtor_err_a_SOURCES = \ - src/lib/err/backtrace.c \ - src/lib/err/torerr.c + src/lib/err/backtrace.c \ + src/lib/err/torerr.c \ + src/lib/err/torerr_sys.c src_lib_libtor_err_testing_a_SOURCES = \ $(src_lib_libtor_err_a_SOURCES) @@ -16,4 +17,5 @@ src_lib_libtor_err_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) noinst_HEADERS += \ src/lib/err/backtrace.h \ - src/lib/err/torerr.h + src/lib/err/torerr.h \ + src/lib/err/torerr_sys.h diff --git a/src/lib/err/torerr.c b/src/lib/err/torerr.c index f9e139f967..e9de86837f 100644 --- a/src/lib/err/torerr.c +++ b/src/lib/err/torerr.c @@ -123,6 +123,16 @@ tor_log_set_sigsafe_err_fds(const int *fds, int n) } /** + * Reset the list of emergency error fds to its default. + */ +void +tor_log_reset_sigsafe_err_fds(void) +{ + int fds[] = { STDERR_FILENO }; + tor_log_set_sigsafe_err_fds(fds, 1); +} + +/** * Set the granularity (in ms) to use when reporting fatal errors outside * the logging system. */ diff --git a/src/lib/err/torerr.h b/src/lib/err/torerr.h index d4bba6916f..b415ef73ef 100644 --- a/src/lib/err/torerr.h +++ b/src/lib/err/torerr.h @@ -39,6 +39,7 @@ void tor_raw_assertion_failed_msg_(const char *file, int line, void tor_log_err_sigsafe(const char *m, ...); int tor_log_get_sigsafe_err_fds(const int **out); void tor_log_set_sigsafe_err_fds(const int *fds, int n); +void tor_log_reset_sigsafe_err_fds(void); void tor_log_sigsafe_err_set_granularity(int ms); int format_hex_number_sigsafe(unsigned long x, char *buf, int max_len); diff --git a/src/lib/err/torerr_sys.c b/src/lib/err/torerr_sys.c new file mode 100644 index 0000000000..96bb1308a4 --- /dev/null +++ b/src/lib/err/torerr_sys.c @@ -0,0 +1,40 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file torerr_sys.c + * \brief Subsystem object for the error handling subsystem. + **/ + +#include "orconfig.h" +#include "lib/err/backtrace.h" +#include "lib/err/torerr.h" +#include "lib/err/torerr_sys.h" +#include "lib/subsys/subsys.h" +#include "lib/version/torversion.h" + +#include <stddef.h> + +static int +subsys_torerr_initialize(void) +{ + if (configure_backtrace_handler(get_version()) < 0) + return -1; + tor_log_reset_sigsafe_err_fds(); + + return 0; +} +static void +subsys_torerr_shutdown(void) +{ + tor_log_reset_sigsafe_err_fds(); + clean_up_backtrace_handler(); +} + +const subsys_fns_t sys_torerr = { + .name = "err", + .level = -100, + .supported = true, + .initialize = subsys_torerr_initialize, + .shutdown = subsys_torerr_shutdown +}; diff --git a/src/lib/err/torerr_sys.h b/src/lib/err/torerr_sys.h new file mode 100644 index 0000000000..b56270d538 --- /dev/null +++ b/src/lib/err/torerr_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file torerr_sys.h + * \brief Declare subsystem object for torerr.c + **/ + +#ifndef TOR_TORERR_SYS_H +#define TOR_TORERR_SYS_H + +extern const struct subsys_fns_t sys_torerr; + +#endif /* !defined(TOR_TORERR_SYS_H) */ diff --git a/src/lib/evloop/.may_include b/src/lib/evloop/.may_include index 30af508914..273de7bb94 100644 --- a/src/lib/evloop/.may_include +++ b/src/lib/evloop/.may_include @@ -12,5 +12,5 @@ lib/testsupport/*.h lib/thread/*.h lib/time/*.h -src/ext/timeouts/timeout.c -tor_queue.h
\ No newline at end of file +ext/timeouts/timeout.c +ext/tor_queue.h
\ No newline at end of file diff --git a/src/lib/evloop/timers.c b/src/lib/evloop/timers.c index 3603bf1a7d..6743b6af51 100644 --- a/src/lib/evloop/timers.c +++ b/src/lib/evloop/timers.c @@ -80,7 +80,8 @@ struct timeout_cb { * use 32-bit math. */ #define WHEEL_BIT 5 #endif -#include "src/ext/timeouts/timeout.c" + +#include "ext/timeouts/timeout.c" static struct timeouts *global_timeouts = NULL; static struct mainloop_event_t *global_timer_event = NULL; diff --git a/src/lib/evloop/workqueue.c b/src/lib/evloop/workqueue.c index 931f65e710..b36a02da5e 100644 --- a/src/lib/evloop/workqueue.c +++ b/src/lib/evloop/workqueue.c @@ -15,7 +15,7 @@ * * The main thread informs the worker threads of pending work by using a * condition variable. The workers inform the main process of completed work - * by using an alert_sockets_t object, as implemented in compat_threads.c. + * by using an alert_sockets_t object, as implemented in net/alertsock.c. * * The main thread can also queue an "update" that will be handled by all the * workers. This is useful for updating state that all the workers share. @@ -36,7 +36,7 @@ #include "lib/net/socket.h" #include "lib/thread/threads.h" -#include "tor_queue.h" +#include "ext/tor_queue.h" #include <event2/event.h> #include <string.h> @@ -622,8 +622,8 @@ reply_event_cb(evutil_socket_t sock, short events, void *arg) tp->reply_cb(tp); } -/** Register the threadpool <b>tp</b>'s reply queue with the libevent - * mainloop of <b>base</b>. If <b>tp</b> is provided, it is run after +/** Register the threadpool <b>tp</b>'s reply queue with Tor's global + * libevent mainloop. If <b>cb</b> is provided, it is run after * each time there is work to process from the reply queue. Return 0 on * success, -1 on failure. */ diff --git a/src/lib/evloop/workqueue.h b/src/lib/evloop/workqueue.h index da292d1f05..10d5d47464 100644 --- a/src/lib/evloop/workqueue.h +++ b/src/lib/evloop/workqueue.h @@ -63,7 +63,6 @@ replyqueue_t *threadpool_get_replyqueue(threadpool_t *tp); replyqueue_t *replyqueue_new(uint32_t alertsocks_flags); void replyqueue_process(replyqueue_t *queue); -struct event_base; int threadpool_register_reply_event(threadpool_t *tp, void (*cb)(threadpool_t *tp)); diff --git a/src/lib/fs/.may_include b/src/lib/fs/.may_include index b1e49fc891..c192e6181c 100644 --- a/src/lib/fs/.may_include +++ b/src/lib/fs/.may_include @@ -13,4 +13,4 @@ lib/malloc/*.h lib/memarea/*.h lib/sandbox/*.h lib/string/*.h -lib/testsupport/testsupport.h +lib/testsupport/*.h diff --git a/src/lib/intmath/cmp.h b/src/lib/intmath/cmp.h index 16952bee3e..11b6fdf98e 100644 --- a/src/lib/intmath/cmp.h +++ b/src/lib/intmath/cmp.h @@ -36,4 +36,7 @@ ((v) > (max)) ? (max) : \ (v) ) +/** Give the absolute value of <b>x</b>, independent of its type. */ +#define ABS(x) ( ((x)<0) ? -(x) : (x) ) + #endif /* !defined(TOR_INTMATH_CMP_H) */ diff --git a/src/lib/log/.may_include b/src/lib/log/.may_include index 852173aab3..11c87f0a0d 100644 --- a/src/lib/log/.may_include +++ b/src/lib/log/.may_include @@ -9,7 +9,7 @@ lib/lock/*.h lib/log/*.h lib/malloc/*.h lib/string/*.h +lib/subsys/*.h lib/testsupport/*.h +lib/version/*.h lib/wallclock/*.h - -micro-revision.i
\ No newline at end of file diff --git a/src/lib/log/include.am b/src/lib/log/include.am index 4a6c9b3686..9d3dbe3104 100644 --- a/src/lib/log/include.am +++ b/src/lib/log/include.am @@ -7,9 +7,9 @@ endif src_lib_libtor_log_a_SOURCES = \ src/lib/log/escape.c \ - src/lib/log/git_revision.c \ src/lib/log/ratelim.c \ src/lib/log/log.c \ + src/lib/log/log_sys.c \ src/lib/log/util_bug.c if WIN32 @@ -21,16 +21,10 @@ src_lib_libtor_log_testing_a_SOURCES = \ src_lib_libtor_log_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_log_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) -# Declare that these object files depend on micro-revision.i. Without this -# rule, we could try to build them before micro-revision.i was created. -src/lib/log/git_revision.$(OBJEXT) \ - src/lib/log/src_lib_libtor_log_testing_a-git_revision.$(OBJEXT): \ - micro-revision.i - noinst_HEADERS += \ src/lib/log/escape.h \ - src/lib/log/git_revision.h \ src/lib/log/ratelim.h \ src/lib/log/log.h \ + src/lib/log/log_sys.h \ src/lib/log/util_bug.h \ src/lib/log/win32err.h diff --git a/src/lib/log/log.c b/src/lib/log/log.c index d60ce6308a..d032f57add 100644 --- a/src/lib/log/log.c +++ b/src/lib/log/log.c @@ -32,7 +32,8 @@ #define LOG_PRIVATE #include "lib/log/log.h" -#include "lib/log/git_revision.h" +#include "lib/log/log_sys.h" +#include "lib/version/git_revision.h" #include "lib/log/ratelim.h" #include "lib/lock/compat_mutex.h" #include "lib/smartlist_core/smartlist_core.h" @@ -1267,7 +1268,7 @@ static const char *domain_list[] = { "GENERAL", "CRYPTO", "NET", "CONFIG", "FS", "PROTOCOL", "MM", "HTTP", "APP", "CONTROL", "CIRC", "REND", "BUG", "DIR", "DIRSERV", "OR", "EDGE", "ACCT", "HIST", "HANDSHAKE", "HEARTBEAT", "CHANNEL", - "SCHED", "GUARD", "CONSDIFF", "DOS", NULL + "SCHED", "GUARD", "CONSDIFF", "DOS", "PROCESS", "PT", "BTRACK", NULL }; /** Return a bitmask for the log domain for which <b>domain</b> is the name, diff --git a/src/lib/log/log.h b/src/lib/log/log.h index 493409756f..423e58e11b 100644 --- a/src/lib/log/log.h +++ b/src/lib/log/log.h @@ -107,8 +107,14 @@ #define LD_CONSDIFF (1u<<24) /** Denial of Service mitigation. */ #define LD_DOS (1u<<25) +/** Processes */ +#define LD_PROCESS (1u<<26) +/** Pluggable Transports. */ +#define LD_PT (1u<<27) +/** Bootstrap tracker. */ +#define LD_BTRACK (1u<<28) /** Number of logging domains in the code. */ -#define N_LOGGING_DOMAINS 26 +#define N_LOGGING_DOMAINS 29 /** This log message is not safe to send to a callback-based logger * immediately. Used as a flag, not a log domain. */ diff --git a/src/lib/log/log_sys.c b/src/lib/log/log_sys.c new file mode 100644 index 0000000000..e20f3156ca --- /dev/null +++ b/src/lib/log/log_sys.c @@ -0,0 +1,35 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file log_sys.c + * \brief Setup and tear down the logging module. + **/ + +#include "orconfig.h" +#include "lib/subsys/subsys.h" +#include "lib/log/escape.h" +#include "lib/log/log.h" +#include "lib/log/log_sys.h" + +static int +subsys_logging_initialize(void) +{ + init_logging(0); + return 0; +} + +static void +subsys_logging_shutdown(void) +{ + logs_free_all(); + escaped(NULL); +} + +const subsys_fns_t sys_logging = { + .name = "log", + .supported = true, + .level = -90, + .initialize = subsys_logging_initialize, + .shutdown = subsys_logging_shutdown, +}; diff --git a/src/lib/log/log_sys.h b/src/lib/log/log_sys.h new file mode 100644 index 0000000000..f7afbb279d --- /dev/null +++ b/src/lib/log/log_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file log_sys.h + * \brief Declare subsystem object for the logging module. + **/ + +#ifndef TOR_LOG_SYS_H +#define TOR_LOG_SYS_H + +extern const struct subsys_fns_t sys_logging; + +#endif /* !defined(TOR_LOG_SYS_H) */ diff --git a/src/lib/malloc/.may_include b/src/lib/malloc/.may_include index cc62bb1013..7686bf862a 100644 --- a/src/lib/malloc/.may_include +++ b/src/lib/malloc/.may_include @@ -3,4 +3,4 @@ orconfig.h lib/cc/*.h lib/err/*.h lib/malloc/*.h -lib/testsupport/testsupport.h +lib/testsupport/*.h diff --git a/src/lib/memarea/.may_include b/src/lib/memarea/.may_include index 814652a93c..a1edaf2231 100644 --- a/src/lib/memarea/.may_include +++ b/src/lib/memarea/.may_include @@ -1,7 +1,7 @@ orconfig.h lib/arch/*.h lib/cc/*.h -lib/container/*.h lib/log/*.h lib/malloc/*.h lib/memarea/*.h +lib/smartlist_core/*.h
\ No newline at end of file diff --git a/src/lib/memarea/memarea.c b/src/lib/memarea/memarea.c index dd7c48c079..96d94c89d9 100644 --- a/src/lib/memarea/memarea.c +++ b/src/lib/memarea/memarea.c @@ -16,7 +16,8 @@ #include "lib/arch/bytes.h" #include "lib/cc/torint.h" -#include "lib/container/smartlist.h" +#include "lib/smartlist_core/smartlist_core.h" +#include "lib/smartlist_core/smartlist_foreach.h" #include "lib/log/log.h" #include "lib/log/util_bug.h" #include "lib/malloc/malloc.h" diff --git a/src/lib/net/.may_include b/src/lib/net/.may_include index 13b209bbed..e4368f799b 100644 --- a/src/lib/net/.may_include +++ b/src/lib/net/.may_include @@ -1,8 +1,9 @@ orconfig.h -siphash.h -ht.h +ext/siphash.h +ext/ht.h lib/arch/*.h +lib/buf/*.h lib/cc/*.h lib/container/*.h lib/ctime/*.h @@ -11,5 +12,6 @@ lib/lock/*.h lib/log/*.h lib/net/*.h lib/string/*.h +lib/subsys/*.h lib/testsupport/*.h lib/malloc/*.h
\ No newline at end of file diff --git a/src/lib/net/address.c b/src/lib/net/address.c index 03767e2950..240201d7b6 100644 --- a/src/lib/net/address.c +++ b/src/lib/net/address.c @@ -40,6 +40,7 @@ #include "lib/net/address.h" #include "lib/net/socket.h" +#include "lib/cc/ctassert.h" #include "lib/container/smartlist.h" #include "lib/ctime/di_ops.h" #include "lib/log/log.h" @@ -52,7 +53,7 @@ #include "lib/string/printf.h" #include "lib/string/util_string.h" -#include "siphash.h" +#include "ext/siphash.h" #ifdef HAVE_SYS_TIME_H #include <sys/time.h> @@ -98,6 +99,7 @@ #if AF_UNSPEC != 0 #error We rely on AF_UNSPEC being 0. Let us know about your platform, please! #endif +CTASSERT(AF_UNSPEC == 0); /** Convert the tor_addr_t in <b>a</b>, with port in <b>port</b>, into a * sockaddr object in *<b>sa_out</b> of object size <b>len</b>. If not enough @@ -1187,14 +1189,22 @@ tor_addr_parse(tor_addr_t *addr, const char *src) int result; struct in_addr in_tmp; struct in6_addr in6_tmp; + int brackets_detected = 0; + tor_assert(addr && src); - if (src[0] == '[' && src[1]) + + size_t len = strlen(src); + + if (len && src[0] == '[' && src[len - 1] == ']') { + brackets_detected = 1; src = tmp = tor_strndup(src+1, strlen(src)-2); + } if (tor_inet_pton(AF_INET6, src, &in6_tmp) > 0) { result = AF_INET6; tor_addr_from_in6(addr, &in6_tmp); - } else if (tor_inet_pton(AF_INET, src, &in_tmp) > 0) { + } else if (!brackets_detected && + tor_inet_pton(AF_INET, src, &in_tmp) > 0) { result = AF_INET; tor_addr_from_in(addr, &in_tmp); } else { diff --git a/src/lib/net/buffers_net.c b/src/lib/net/buffers_net.c index c52ea2784e..a4c0aca3b5 100644 --- a/src/lib/net/buffers_net.c +++ b/src/lib/net/buffers_net.c @@ -11,7 +11,7 @@ #define BUFFERS_PRIVATE #include "lib/net/buffers_net.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/log/log.h" #include "lib/log/util_bug.h" #include "lib/net/nettypes.h" @@ -22,6 +22,10 @@ #include <stdlib.h> +#ifdef HAVE_UNISTD_H +#include <unistd.h> +#endif + #ifdef PARANOIA /** Helper: If PARANOIA is defined, assert that the buffer in local variable * <b>buf</b> is well-formed. */ @@ -30,27 +34,36 @@ #define check() STMT_NIL #endif /* defined(PARANOIA) */ -/** Read up to <b>at_most</b> bytes from the socket <b>fd</b> into +/** Read up to <b>at_most</b> bytes from the file descriptor <b>fd</b> into * <b>chunk</b> (which must be on <b>buf</b>). If we get an EOF, set - * *<b>reached_eof</b> to 1. Return -1 on error, 0 on eof or blocking, - * and the number of bytes read otherwise. */ + * *<b>reached_eof</b> to 1. Uses <b>tor_socket_recv()</b> iff <b>is_socket</b> + * is true, otherwise it uses <b>read()</b>. Return -1 on error (and sets + * *<b>error</b> to errno), 0 on eof or blocking, and the number of bytes read + * otherwise. */ static inline int read_to_chunk(buf_t *buf, chunk_t *chunk, tor_socket_t fd, size_t at_most, - int *reached_eof, int *socket_error) + int *reached_eof, int *error, bool is_socket) { ssize_t read_result; if (at_most > CHUNK_REMAINING_CAPACITY(chunk)) at_most = CHUNK_REMAINING_CAPACITY(chunk); - read_result = tor_socket_recv(fd, CHUNK_WRITE_PTR(chunk), at_most, 0); + + if (is_socket) + read_result = tor_socket_recv(fd, CHUNK_WRITE_PTR(chunk), at_most, 0); + else + read_result = read(fd, CHUNK_WRITE_PTR(chunk), at_most); if (read_result < 0) { - int e = tor_socket_errno(fd); + int e = is_socket ? tor_socket_errno(fd) : errno; + if (!ERRNO_IS_EAGAIN(e)) { /* it's a real error */ #ifdef _WIN32 if (e == WSAENOBUFS) - log_warn(LD_NET,"recv() failed: WSAENOBUFS. Not enough ram?"); + log_warn(LD_NET, "%s() failed: WSAENOBUFS. Not enough ram?", + is_socket ? "recv" : "read"); #endif - *socket_error = e; + if (error) + *error = e; return -1; } return 0; /* would block. */ @@ -68,16 +81,17 @@ read_to_chunk(buf_t *buf, chunk_t *chunk, tor_socket_t fd, size_t at_most, } } -/** Read from socket <b>s</b>, writing onto end of <b>buf</b>. Read at most - * <b>at_most</b> bytes, growing the buffer as necessary. If recv() returns 0 - * (because of EOF), set *<b>reached_eof</b> to 1 and return 0. Return -1 on - * error; else return the number of bytes read. +/** Read from file descriptor <b>fd</b>, writing onto end of <b>buf</b>. Read + * at most <b>at_most</b> bytes, growing the buffer as necessary. If recv() + * returns 0 (because of EOF), set *<b>reached_eof</b> to 1 and return 0. + * Return -1 on error; else return the number of bytes read. */ /* XXXX indicate "read blocked" somehow? */ -int -buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most, - int *reached_eof, - int *socket_error) +static int +buf_read_from_fd(buf_t *buf, int fd, size_t at_most, + int *reached_eof, + int *socket_error, + bool is_socket) { /* XXXX It's stupid to overload the return values for these functions: * "error status" and "number of bytes read" are not mutually exclusive. @@ -87,7 +101,7 @@ buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most, check(); tor_assert(reached_eof); - tor_assert(SOCKET_OK(s)); + tor_assert(SOCKET_OK(fd)); if (BUG(buf->datalen >= INT_MAX)) return -1; @@ -108,7 +122,8 @@ buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most, readlen = cap; } - r = read_to_chunk(buf, chunk, s, readlen, reached_eof, socket_error); + r = read_to_chunk(buf, chunk, fd, readlen, + reached_eof, socket_error, is_socket); check(); if (r < 0) return r; /* Error */ @@ -122,22 +137,27 @@ buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most, } /** Helper for buf_flush_to_socket(): try to write <b>sz</b> bytes from chunk - * <b>chunk</b> of buffer <b>buf</b> onto socket <b>s</b>. On success, deduct - * the bytes written from *<b>buf_flushlen</b>. Return the number of bytes - * written on success, 0 on blocking, -1 on failure. + * <b>chunk</b> of buffer <b>buf</b> onto file descriptor <b>fd</b>. On + * success, deduct the bytes written from *<b>buf_flushlen</b>. Return the + * number of bytes written on success, 0 on blocking, -1 on failure. */ static inline int -flush_chunk(tor_socket_t s, buf_t *buf, chunk_t *chunk, size_t sz, - size_t *buf_flushlen) +flush_chunk(tor_socket_t fd, buf_t *buf, chunk_t *chunk, size_t sz, + size_t *buf_flushlen, bool is_socket) { ssize_t write_result; if (sz > chunk->datalen) sz = chunk->datalen; - write_result = tor_socket_send(s, chunk->data, sz, 0); + + if (is_socket) + write_result = tor_socket_send(fd, chunk->data, sz, 0); + else + write_result = write(fd, chunk->data, sz); if (write_result < 0) { - int e = tor_socket_errno(s); + int e = is_socket ? tor_socket_errno(fd) : errno; + if (!ERRNO_IS_EAGAIN(e)) { /* it's a real error */ #ifdef _WIN32 if (e == WSAENOBUFS) @@ -155,15 +175,15 @@ flush_chunk(tor_socket_t s, buf_t *buf, chunk_t *chunk, size_t sz, } } -/** Write data from <b>buf</b> to the socket <b>s</b>. Write at most +/** Write data from <b>buf</b> to the file descriptor <b>fd</b>. Write at most * <b>sz</b> bytes, decrement *<b>buf_flushlen</b> by * the number of bytes actually written, and remove the written bytes * from the buffer. Return the number of bytes written on success, * -1 on failure. Return 0 if write() would block. */ -int -buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz, - size_t *buf_flushlen) +static int +buf_flush_to_fd(buf_t *buf, int fd, size_t sz, + size_t *buf_flushlen, bool is_socket) { /* XXXX It's stupid to overload the return values for these functions: * "error status" and "number of bytes flushed" are not mutually exclusive. @@ -171,7 +191,7 @@ buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz, int r; size_t flushed = 0; tor_assert(buf_flushlen); - tor_assert(SOCKET_OK(s)); + tor_assert(SOCKET_OK(fd)); if (BUG(*buf_flushlen > buf->datalen)) { *buf_flushlen = buf->datalen; } @@ -188,7 +208,7 @@ buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz, else flushlen0 = buf->head->datalen; - r = flush_chunk(s, buf, buf->head, flushlen0, buf_flushlen); + r = flush_chunk(fd, buf, buf->head, flushlen0, buf_flushlen, is_socket); check(); if (r < 0) return r; @@ -200,3 +220,55 @@ buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz, tor_assert(flushed < INT_MAX); return (int)flushed; } + +/** Write data from <b>buf</b> to the socket <b>s</b>. Write at most + * <b>sz</b> bytes, decrement *<b>buf_flushlen</b> by + * the number of bytes actually written, and remove the written bytes + * from the buffer. Return the number of bytes written on success, + * -1 on failure. Return 0 if write() would block. + */ +int +buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz, + size_t *buf_flushlen) +{ + return buf_flush_to_fd(buf, s, sz, buf_flushlen, true); +} + +/** Read from socket <b>s</b>, writing onto end of <b>buf</b>. Read at most + * <b>at_most</b> bytes, growing the buffer as necessary. If recv() returns 0 + * (because of EOF), set *<b>reached_eof</b> to 1 and return 0. Return -1 on + * error; else return the number of bytes read. + */ +int +buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most, + int *reached_eof, + int *socket_error) +{ + return buf_read_from_fd(buf, s, at_most, reached_eof, socket_error, true); +} + +/** Write data from <b>buf</b> to the pipe <b>fd</b>. Write at most + * <b>sz</b> bytes, decrement *<b>buf_flushlen</b> by + * the number of bytes actually written, and remove the written bytes + * from the buffer. Return the number of bytes written on success, + * -1 on failure. Return 0 if write() would block. + */ +int +buf_flush_to_pipe(buf_t *buf, int fd, size_t sz, + size_t *buf_flushlen) +{ + return buf_flush_to_fd(buf, fd, sz, buf_flushlen, false); +} + +/** Read from pipe <b>fd</b>, writing onto end of <b>buf</b>. Read at most + * <b>at_most</b> bytes, growing the buffer as necessary. If read() returns 0 + * (because of EOF), set *<b>reached_eof</b> to 1 and return 0. Return -1 on + * error; else return the number of bytes read. + */ +int +buf_read_from_pipe(buf_t *buf, int fd, size_t at_most, + int *reached_eof, + int *socket_error) +{ + return buf_read_from_fd(buf, fd, at_most, reached_eof, socket_error, false); +} diff --git a/src/lib/net/buffers_net.h b/src/lib/net/buffers_net.h index 417f6f9413..8911b082a2 100644 --- a/src/lib/net/buffers_net.h +++ b/src/lib/net/buffers_net.h @@ -24,4 +24,11 @@ int buf_read_from_socket(struct buf_t *buf, tor_socket_t s, size_t at_most, int buf_flush_to_socket(struct buf_t *buf, tor_socket_t s, size_t sz, size_t *buf_flushlen); +int buf_read_from_pipe(struct buf_t *buf, int fd, size_t at_most, + int *reached_eof, + int *socket_error); + +int buf_flush_to_pipe(struct buf_t *buf, int fd, size_t sz, + size_t *buf_flushlen); + #endif /* !defined(TOR_BUFFERS_H) */ diff --git a/src/lib/net/inaddr.c b/src/lib/net/inaddr.c index dcd8fcdd65..0960d323c5 100644 --- a/src/lib/net/inaddr.c +++ b/src/lib/net/inaddr.c @@ -168,6 +168,13 @@ tor_inet_pton(int af, const char *src, void *dst) if (af == AF_INET) { return tor_inet_aton(src, dst); } else if (af == AF_INET6) { + ssize_t len = strlen(src); + + /* Reject if src has needless trailing ':'. */ + if (len > 2 && src[len - 1] == ':' && src[len - 2] != ':') { + return 0; + } + struct in6_addr *out = dst; uint16_t words[8]; int gapPos = -1, i, setWords=0; @@ -207,7 +214,6 @@ tor_inet_pton(int af, const char *src, void *dst) return 0; if (TOR_ISXDIGIT(*src)) { char *next; - ssize_t len; long r = strtol(src, &next, 16); if (next == NULL || next == src) { /* The 'next == src' error case can happen on versions of openbsd diff --git a/src/lib/net/include.am b/src/lib/net/include.am index ff0967e786..8a88f0f2ae 100644 --- a/src/lib/net/include.am +++ b/src/lib/net/include.am @@ -11,6 +11,7 @@ src_lib_libtor_net_a_SOURCES = \ src/lib/net/buffers_net.c \ src/lib/net/gethostname.c \ src/lib/net/inaddr.c \ + src/lib/net/network_sys.c \ src/lib/net/resolve.c \ src/lib/net/socket.c \ src/lib/net/socketpair.c @@ -28,6 +29,7 @@ noinst_HEADERS += \ src/lib/net/inaddr.h \ src/lib/net/inaddr_st.h \ src/lib/net/nettypes.h \ + src/lib/net/network_sys.h \ src/lib/net/resolve.h \ src/lib/net/socket.h \ src/lib/net/socketpair.h \ diff --git a/src/lib/net/network_sys.c b/src/lib/net/network_sys.c new file mode 100644 index 0000000000..ac49288ee6 --- /dev/null +++ b/src/lib/net/network_sys.c @@ -0,0 +1,44 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file network_sys.c + * \brief Subsystem object for networking setup. + **/ + +#include "orconfig.h" +#include "lib/subsys/subsys.h" +#include "lib/net/network_sys.h" +#include "lib/net/resolve.h" +#include "lib/net/socket.h" + +#ifdef _WIN32 +#include <winsock2.h> +#include <windows.h> +#endif + +static int +subsys_network_initialize(void) +{ + if (network_init() < 0) + return -1; + + return 0; +} + +static void +subsys_network_shutdown(void) +{ +#ifdef _WIN32 + WSACleanup(); +#endif + tor_free_getaddrinfo_cache(); +} + +const subsys_fns_t sys_network = { + .name = "network", + .level = -90, + .supported = true, + .initialize = subsys_network_initialize, + .shutdown = subsys_network_shutdown, +}; diff --git a/src/lib/net/network_sys.h b/src/lib/net/network_sys.h new file mode 100644 index 0000000000..62b778bb66 --- /dev/null +++ b/src/lib/net/network_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file log_network.h + * \brief Declare subsystem object for the network module. + **/ + +#ifndef TOR_NETWORK_SYS_H +#define TOR_NETWORK_SYS_H + +extern const struct subsys_fns_t sys_network; + +#endif /* !defined(TOR_NETWORK_SYS_H) */ diff --git a/src/lib/net/resolve.c b/src/lib/net/resolve.c index 7c8df3e307..95c1b171b5 100644 --- a/src/lib/net/resolve.c +++ b/src/lib/net/resolve.c @@ -16,8 +16,8 @@ #include "lib/string/parse_int.h" #include "lib/string/util_string.h" -#include "siphash.h" -#include "ht.h" +#include "ext/siphash.h" +#include "ext/ht.h" #ifdef HAVE_SYS_TYPES_H #include <sys/types.h> @@ -421,4 +421,13 @@ tor_make_getaddrinfo_cache_active(void) { sandbox_getaddrinfo_is_active = 1; } +#else +void +sandbox_disable_getaddrinfo_cache(void) +{ +} +void +tor_make_getaddrinfo_cache_active(void) +{ +} #endif diff --git a/src/lib/net/resolve.h b/src/lib/net/resolve.h index bf870c44c4..39157aaa67 100644 --- a/src/lib/net/resolve.h +++ b/src/lib/net/resolve.h @@ -42,7 +42,6 @@ int tor_getaddrinfo(const char *name, const char *servname, struct addrinfo **res); void tor_freeaddrinfo(struct addrinfo *addrinfo); void tor_free_getaddrinfo_cache(void); -void tor_make_getaddrinfo_cache_active(void); #else /* !(defined(USE_SANDBOX_GETADDRINFO)) */ #define tor_getaddrinfo(name, servname, hints, res) \ getaddrinfo((name),(servname), (hints),(res)) @@ -54,5 +53,6 @@ void tor_make_getaddrinfo_cache_active(void); #endif /* defined(USE_SANDBOX_GETADDRINFO) */ void sandbox_disable_getaddrinfo_cache(void); +void tor_make_getaddrinfo_cache_active(void); #endif diff --git a/src/lib/net/socket.c b/src/lib/net/socket.c index cd7c9685cd..8940e00591 100644 --- a/src/lib/net/socket.c +++ b/src/lib/net/socket.c @@ -31,6 +31,9 @@ #endif #include <stddef.h> #include <string.h> +#ifdef __FreeBSD__ +#include <sys/sysctl.h> +#endif /** Called before we make any calls to network-related functions. * (Some operating systems require their network libraries to be @@ -60,6 +63,32 @@ network_init(void) return 0; } +/** + * Warn the user if any system network parameters should be changed. + */ +void +check_network_configuration(bool server_mode) +{ +#ifdef __FreeBSD__ + if (server_mode) { + int random_id_state; + size_t state_size = sizeof(random_id_state); + + if (sysctlbyname("net.inet.ip.random_id", &random_id_state, + &state_size, NULL, 0)) { + log_warn(LD_CONFIG, + "Failed to figure out if IP ids are randomized."); + } else if (random_id_state == 0) { + log_warn(LD_CONFIG, "Looks like IP ids are not randomized. " + "Please consider setting the net.inet.ip.random_id sysctl, " + "so your relay makes it harder to figure out how busy it is."); + } + } +#else + (void) server_mode; +#endif +} + /* When set_max_file_sockets() is called, update this with the max file * descriptor value so we can use it to check the limit when opening a new * socket. Default value is what Debian sets as the default hard limit. */ diff --git a/src/lib/net/socket.h b/src/lib/net/socket.h index 2b87441fc6..822b9975e6 100644 --- a/src/lib/net/socket.h +++ b/src/lib/net/socket.h @@ -54,6 +54,7 @@ int tor_addr_from_getsockname(struct tor_addr_t *addr_out, tor_socket_t sock); int set_socket_nonblocking(tor_socket_t socket); int tor_socketpair(int family, int type, int protocol, tor_socket_t fd[2]); int network_init(void); +void check_network_configuration(bool server_mode); int get_max_sockets(void); void set_max_sockets(int); diff --git a/src/lib/process/.may_include b/src/lib/process/.may_include index 05414d2a96..ce1b6ecf59 100644 --- a/src/lib/process/.may_include +++ b/src/lib/process/.may_include @@ -1,17 +1,20 @@ orconfig.h +lib/buf/*.h lib/cc/*.h lib/container/*.h lib/ctime/*.h lib/err/*.h -lib/intmath/*.h +lib/evloop/*.h lib/fs/*.h +lib/intmath/*.h lib/log/*.h lib/malloc/*.h lib/net/*.h lib/process/*.h lib/string/*.h +lib/subsys/*.h lib/testsupport/*.h lib/thread/*.h -ht.h
\ No newline at end of file +ext/ht.h diff --git a/src/lib/process/include.am b/src/lib/process/include.am index c6cc3a6699..83b67bf029 100644 --- a/src/lib/process/include.am +++ b/src/lib/process/include.am @@ -9,10 +9,14 @@ src_lib_libtor_process_a_SOURCES = \ src/lib/process/daemon.c \ src/lib/process/env.c \ src/lib/process/pidfile.c \ + src/lib/process/process.c \ + src/lib/process/process_sys.c \ + src/lib/process/process_unix.c \ + src/lib/process/process_win32.c \ src/lib/process/restrict.c \ src/lib/process/setuid.c \ - src/lib/process/subprocess.c \ - src/lib/process/waitpid.c + src/lib/process/waitpid.c \ + src/lib/process/winprocess_sys.c src_lib_libtor_process_testing_a_SOURCES = \ $(src_lib_libtor_process_a_SOURCES) @@ -23,7 +27,11 @@ noinst_HEADERS += \ src/lib/process/daemon.h \ src/lib/process/env.h \ src/lib/process/pidfile.h \ + src/lib/process/process.h \ + src/lib/process/process_sys.h \ + src/lib/process/process_unix.h \ + src/lib/process/process_win32.h \ src/lib/process/restrict.h \ src/lib/process/setuid.h \ - src/lib/process/subprocess.h \ - src/lib/process/waitpid.h + src/lib/process/waitpid.h \ + src/lib/process/winprocess_sys.h diff --git a/src/lib/process/process.c b/src/lib/process/process.c new file mode 100644 index 0000000000..ae345ceeae --- /dev/null +++ b/src/lib/process/process.c @@ -0,0 +1,797 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process.c + * \brief Module for working with other processes. + **/ + +#define PROCESS_PRIVATE +#include "lib/buf/buffers.h" +#include "lib/net/buffers_net.h" +#include "lib/container/smartlist.h" +#include "lib/log/log.h" +#include "lib/log/util_bug.h" +#include "lib/process/process.h" +#include "lib/process/process_unix.h" +#include "lib/process/process_win32.h" +#include "lib/process/env.h" + +#ifdef HAVE_STDDEF_H +#include <stddef.h> +#endif + +/** A list of all <b>process_t</b> instances currently allocated. */ +static smartlist_t *processes; + +/** + * Boolean. If true, then Tor may call execve or CreateProcess via + * tor_spawn_background. + **/ +static int may_spawn_background_process = 1; + +/** Structure to represent a child process. */ +struct process_t { + /** Process status. */ + process_status_t status; + + /** Which protocol is the process using? */ + process_protocol_t protocol; + + /** Which function to call when we have data ready from stdout? */ + process_read_callback_t stdout_read_callback; + + /** Which function to call when we have data ready from stderr? */ + process_read_callback_t stderr_read_callback; + + /** Which function call when our process terminated? */ + process_exit_callback_t exit_callback; + + /** Our exit code when the process have terminated. */ + process_exit_code_t exit_code; + + /** Name of the command we want to execute (for example: /bin/ls). */ + char *command; + + /** The arguments used for the new process. The format here is one argument + * per element of the smartlist_t. On Windows these arguments are combined + * together using the <b>tor_join_win_cmdline</b> function. On Unix the + * process name (argv[0]) and the trailing NULL is added automatically before + * the process is executed. */ + smartlist_t *arguments; + + /** The environment used for the new process. */ + smartlist_t *environment; + + /** Buffer to store data from stdout when it is read. */ + buf_t *stdout_buffer; + + /** Buffer to store data from stderr when it is read. */ + buf_t *stderr_buffer; + + /** Buffer to store data to stdin before it is written. */ + buf_t *stdin_buffer; + + /** Do we need to store some custom data with the process? */ + void *data; + +#ifndef _WIN32 + /** Our Unix process handle. */ + process_unix_t *unix_process; +#else + /** Our Win32 process handle. */ + process_win32_t *win32_process; +#endif +}; + +/** Convert a given process status in <b>status</b> to its string + * representation. */ +const char * +process_status_to_string(process_status_t status) +{ + switch (status) { + case PROCESS_STATUS_NOT_RUNNING: + return "not running"; + case PROCESS_STATUS_RUNNING: + return "running"; + case PROCESS_STATUS_ERROR: + return "error"; + } + + /* LCOV_EXCL_START */ + tor_assert_unreached(); + return NULL; + /* LCOV_EXCL_STOP */ +} + +/** Convert a given process protocol in <b>protocol</b> to its string + * representation. */ +const char * +process_protocol_to_string(process_protocol_t protocol) +{ + switch (protocol) { + case PROCESS_PROTOCOL_LINE: + return "Line"; + case PROCESS_PROTOCOL_RAW: + return "Raw"; + } + + /* LCOV_EXCL_START */ + tor_assert_unreached(); + return NULL; + /* LCOV_EXCL_STOP */ +} + +/** + * Turn off may_spawn_background_process, so that all future calls to + * tor_spawn_background are guaranteed to fail. + **/ +void +tor_disable_spawning_background_processes(void) +{ + may_spawn_background_process = 0; +} + +/** Initialize the Process subsystem. This function initializes the Process + * subsystem's global state. For cleaning up, <b>process_free_all()</b> should + * be called. */ +void +process_init(void) +{ + processes = smartlist_new(); + +#ifdef _WIN32 + process_win32_init(); +#endif +} + +/** Free up all resources that is handled by the Process subsystem. Note that + * this call does not terminate already running processes. */ +void +process_free_all(void) +{ +#ifdef _WIN32 + process_win32_deinit(); +#endif + + SMARTLIST_FOREACH(processes, process_t *, x, process_free(x)); + smartlist_free(processes); +} + +/** Get a list of all processes. This function returns a smartlist of + * <b>process_t</b> containing all the currently allocated processes. */ +const smartlist_t * +process_get_all_processes(void) +{ + return processes; +} + +/** Allocate and initialize a new process. This function returns a newly + * allocated and initialized process data, which can be used to configure and + * later run a subprocess of Tor. Use the various <b>process_set_*()</b> + * methods to configure it and run the process using <b>process_exec()</b>. Use + * <b>command</b> to specify the path to the command to run. You can either + * specify an absolute path to the command or relative where Tor will use the + * underlying operating system's functionality for finding the command to run. + * */ +process_t * +process_new(const char *command) +{ + tor_assert(command); + + process_t *process; + process = tor_malloc_zero(sizeof(process_t)); + + /* Set our command. */ + process->command = tor_strdup(command); + + /* By default we are not running. */ + process->status = PROCESS_STATUS_NOT_RUNNING; + + /* Prepare process environment. */ + process->arguments = smartlist_new(); + process->environment = smartlist_new(); + + /* Prepare the buffers. */ + process->stdout_buffer = buf_new(); + process->stderr_buffer = buf_new(); + process->stdin_buffer = buf_new(); + +#ifndef _WIN32 + /* Prepare our Unix process handle. */ + process->unix_process = process_unix_new(); +#else + /* Prepare our Win32 process handle. */ + process->win32_process = process_win32_new(); +#endif + + smartlist_add(processes, process); + + return process; +} + +/** Deallocate the given process in <b>process</b>. */ +void +process_free_(process_t *process) +{ + if (! process) + return; + + /* Cleanup parameters. */ + tor_free(process->command); + + /* Cleanup arguments and environment. */ + SMARTLIST_FOREACH(process->arguments, char *, x, tor_free(x)); + smartlist_free(process->arguments); + + SMARTLIST_FOREACH(process->environment, char *, x, tor_free(x)); + smartlist_free(process->environment); + + /* Cleanup the buffers. */ + buf_free(process->stdout_buffer); + buf_free(process->stderr_buffer); + buf_free(process->stdin_buffer); + +#ifndef _WIN32 + /* Cleanup our Unix process handle. */ + process_unix_free(process->unix_process); +#else + /* Cleanup our Win32 process handle. */ + process_win32_free(process->win32_process); +#endif + + smartlist_remove(processes, process); + + tor_free(process); +} + +/** Execute the given process. This function executes the given process as a + * subprocess of Tor. Returns <b>PROCESS_STATUS_RUNNING</b> upon success. */ +process_status_t +process_exec(process_t *process) +{ + tor_assert(process); + + if (BUG(may_spawn_background_process == 0)) + return PROCESS_STATUS_ERROR; + + process_status_t status = PROCESS_STATUS_NOT_RUNNING; + + log_info(LD_PROCESS, "Starting new process: %s", process->command); + +#ifndef _WIN32 + status = process_unix_exec(process); +#else + status = process_win32_exec(process); +#endif + + /* Update our state. */ + process_set_status(process, status); + + if (status != PROCESS_STATUS_RUNNING) { + log_warn(LD_PROCESS, "Failed to start process: %s", + process_get_command(process)); + } + + return status; +} + +/** Terminate the given process. Returns true on success, + * otherwise false. */ +bool +process_terminate(process_t *process) +{ + tor_assert(process); + + /* Terminating a non-running process isn't going to work. */ + if (process_get_status(process) != PROCESS_STATUS_RUNNING) + return false; + + log_debug(LD_PROCESS, "Terminating process"); + +#ifndef _WIN32 + return process_unix_terminate(process); +#else + return process_win32_terminate(process); +#endif +} + +/** Returns the unique process identifier for the given <b>process</b>. */ +process_pid_t +process_get_pid(process_t *process) +{ + tor_assert(process); + +#ifndef _WIN32 + return process_unix_get_pid(process); +#else + return process_win32_get_pid(process); +#endif +} + +/** Set the callback function for output from the child process's standard out + * handle. This function sets the callback function which is called every time + * the child process have written output to its standard out file handle. + * + * Use <b>process_set_protocol(process, PROCESS_PROTOCOL_LINE)</b> if you want + * the callback to only contain complete "\n" or "\r\n" terminated lines. */ +void +process_set_stdout_read_callback(process_t *process, + process_read_callback_t callback) +{ + tor_assert(process); + process->stdout_read_callback = callback; +} + +/** Set the callback function for output from the child process's standard + * error handle. This function sets the callback function which is called + * every time the child process have written output to its standard error file + * handle. + * + * Use <b>process_set_protocol(process, PROCESS_PROTOCOL_LINE)</b> if you want + * the callback to only contain complete "\n" or "\r\n" terminated lines. */ +void +process_set_stderr_read_callback(process_t *process, + process_read_callback_t callback) +{ + tor_assert(process); + process->stderr_read_callback = callback; +} + +/** Set the callback function for process exit notification. The + * <b>callback</b> function will be called every time your child process have + * terminated. */ +void +process_set_exit_callback(process_t *process, + process_exit_callback_t callback) +{ + tor_assert(process); + process->exit_callback = callback; +} + +/** Get the current command of the given process. */ +const char * +process_get_command(const process_t *process) +{ + tor_assert(process); + return process->command; +} + +void +process_set_protocol(process_t *process, process_protocol_t protocol) +{ + tor_assert(process); + process->protocol = protocol; +} + +/** Get the currently used protocol of the given process. */ +process_protocol_t +process_get_protocol(const process_t *process) +{ + tor_assert(process); + return process->protocol; +} + +/** Set opague pointer to data. This function allows you to store a pointer to + * your own data in the given process. Use <b>process_get_data()</b> in the + * various callback functions to retrieve the data again. + * + * Note that the given process does NOT take ownership of the data and you are + * responsible for freeing up any resources allocated by the given data. + * */ +void +process_set_data(process_t *process, void *data) +{ + tor_assert(process); + process->data = data; +} + +/** Get the opaque pointer to callback data from the given process. This + * function allows you get the data you stored with <b>process_set_data()</b> + * in the different callback functions. */ +void * +process_get_data(const process_t *process) +{ + tor_assert(process); + return process->data; +} + +/** Set the status of a given process. */ +void +process_set_status(process_t *process, process_status_t status) +{ + tor_assert(process); + process->status = status; +} + +/** Get the status of the given process. */ +process_status_t +process_get_status(const process_t *process) +{ + tor_assert(process); + return process->status; +} + +/** Append an argument to the list of arguments in the given process. */ +void +process_append_argument(process_t *process, const char *argument) +{ + tor_assert(process); + tor_assert(argument); + + smartlist_add(process->arguments, tor_strdup(argument)); +} + +/** Returns a list of arguments (excluding the command itself) from the + * given process. */ +const smartlist_t * +process_get_arguments(const process_t *process) +{ + tor_assert(process); + return process->arguments; +} + +/** Returns a newly allocated Unix style argument vector. Use <b>tor_free()</b> + * to deallocate it after use. */ +char ** +process_get_argv(const process_t *process) +{ + tor_assert(process); + + /** Generate a Unix style process argument vector from our process's + * arguments smartlist_t. */ + char **argv = NULL; + + char *filename = process->command; + const smartlist_t *arguments = process->arguments; + const size_t size = smartlist_len(arguments); + + /* Make space for the process filename as argv[0] and a trailing NULL. */ + argv = tor_malloc_zero(sizeof(char *) * (size + 2)); + + /* Set our filename as first argument. */ + argv[0] = filename; + + /* Put in the rest of the values from arguments. */ + SMARTLIST_FOREACH_BEGIN(arguments, char *, arg_val) { + tor_assert(arg_val != NULL); + + argv[arg_val_sl_idx + 1] = arg_val; + } SMARTLIST_FOREACH_END(arg_val); + + return argv; +} + +/** This function clears the internal environment and copies over every string + * from <b>env</b> as the new environment. */ +void +process_reset_environment(process_t *process, const smartlist_t *env) +{ + tor_assert(process); + tor_assert(env); + + /* Cleanup old environment. */ + SMARTLIST_FOREACH(process->environment, char *, x, tor_free(x)); + smartlist_free(process->environment); + process->environment = smartlist_new(); + + SMARTLIST_FOREACH(env, char *, x, + smartlist_add(process->environment, tor_strdup(x))); +} + +/** Set the given <b>key</b>/<b>value</b> pair as environment variable in the + * given process. */ +void +process_set_environment(process_t *process, + const char *key, + const char *value) +{ + tor_assert(process); + tor_assert(key); + tor_assert(value); + + smartlist_add_asprintf(process->environment, "%s=%s", key, value); +} + +/** Returns a newly allocated <b>process_environment_t</b> containing the + * environment variables for the given process. */ +process_environment_t * +process_get_environment(const process_t *process) +{ + tor_assert(process); + return process_environment_make(process->environment); +} + +#ifndef _WIN32 +/** Get the internal handle for the Unix backend. */ +process_unix_t * +process_get_unix_process(const process_t *process) +{ + tor_assert(process); + tor_assert(process->unix_process); + return process->unix_process; +} +#else +/** Get the internal handle for Windows backend. */ +process_win32_t * +process_get_win32_process(const process_t *process) +{ + tor_assert(process); + tor_assert(process->win32_process); + return process->win32_process; +} +#endif + +/** Write <b>size</b> bytes of <b>data</b> to the given process's standard + * input. */ +void +process_write(process_t *process, + const uint8_t *data, size_t size) +{ + tor_assert(process); + tor_assert(data); + + buf_add(process->stdin_buffer, (char *)data, size); + process_write_stdin(process, process->stdin_buffer); +} + +/** As tor_vsnprintf(), but write the data to the given process's standard + * input. */ +void +process_vprintf(process_t *process, + const char *format, va_list args) +{ + tor_assert(process); + tor_assert(format); + + int size; + char *data; + + size = tor_vasprintf(&data, format, args); + process_write(process, (uint8_t *)data, size); + tor_free(data); +} + +/** As tor_snprintf(), but write the data to the given process's standard + * input. */ +void +process_printf(process_t *process, + const char *format, ...) +{ + tor_assert(process); + tor_assert(format); + + va_list ap; + va_start(ap, format); + process_vprintf(process, format, ap); + va_end(ap); +} + +/** This function is called by the Process backend when a given process have + * data that is ready to be read from the child process's standard output + * handle. */ +void +process_notify_event_stdout(process_t *process) +{ + tor_assert(process); + + int ret; + ret = process_read_stdout(process, process->stdout_buffer); + + if (ret > 0) + process_read_data(process, + process->stdout_buffer, + process->stdout_read_callback); +} + +/** This function is called by the Process backend when a given process have + * data that is ready to be read from the child process's standard error + * handle. */ +void +process_notify_event_stderr(process_t *process) +{ + tor_assert(process); + + int ret; + ret = process_read_stderr(process, process->stderr_buffer); + + if (ret > 0) + process_read_data(process, + process->stderr_buffer, + process->stderr_read_callback); +} + +/** This function is called by the Process backend when a given process is + * allowed to begin writing data to the standard input of the child process. */ +void +process_notify_event_stdin(process_t *process) +{ + tor_assert(process); + + process_write_stdin(process, process->stdin_buffer); +} + +/** This function is called by the Process backend when a given process have + * terminated. The exit status code is passed in <b>exit_code</b>. We mark the + * process as no longer running and calls the <b>exit_callback</b> with + * information about the process termination. The given <b>process</b> is + * free'd iff the exit_callback returns true. */ +void +process_notify_event_exit(process_t *process, process_exit_code_t exit_code) +{ + tor_assert(process); + + log_debug(LD_PROCESS, + "Process terminated with exit code: %"PRIu64, exit_code); + + /* Update our state. */ + process_set_status(process, PROCESS_STATUS_NOT_RUNNING); + process->exit_code = exit_code; + + /* Call our exit callback, if it exists. */ + bool free_process_handle = false; + + /* The exit callback will tell us if we should process_free() our handle. */ + if (process->exit_callback) + free_process_handle = process->exit_callback(process, exit_code); + + if (free_process_handle) + process_free(process); +} + +/** This function is called whenever the Process backend have notified us that + * there is data to be read from its standard out handle. Returns the number of + * bytes that have been put into the given buffer. */ +MOCK_IMPL(STATIC int, process_read_stdout, (process_t *process, buf_t *buffer)) +{ + tor_assert(process); + tor_assert(buffer); + +#ifndef _WIN32 + return process_unix_read_stdout(process, buffer); +#else + return process_win32_read_stdout(process, buffer); +#endif +} + +/** This function is called whenever the Process backend have notified us that + * there is data to be read from its standard error handle. Returns the number + * of bytes that have been put into the given buffer. */ +MOCK_IMPL(STATIC int, process_read_stderr, (process_t *process, buf_t *buffer)) +{ + tor_assert(process); + tor_assert(buffer); + +#ifndef _WIN32 + return process_unix_read_stderr(process, buffer); +#else + return process_win32_read_stderr(process, buffer); +#endif +} + +/** This function calls the backend function for the given process whenever + * there is data to be written to the backends' file handles. */ +MOCK_IMPL(STATIC void, process_write_stdin, + (process_t *process, buf_t *buffer)) +{ + tor_assert(process); + tor_assert(buffer); + +#ifndef _WIN32 + process_unix_write(process, buffer); +#else + process_win32_write(process, buffer); +#endif +} + +/** This function calls the protocol handlers based on the value of + * <b>process_get_protocol(process)</b>. Currently we call + * <b>process_read_buffer()</b> for <b>PROCESS_PROTOCOL_RAW</b> and + * <b>process_read_lines()</b> for <b>PROCESS_PROTOCOL_LINE</b>. */ +STATIC void +process_read_data(process_t *process, + buf_t *buffer, + process_read_callback_t callback) +{ + tor_assert(process); + tor_assert(buffer); + + switch (process_get_protocol(process)) { + case PROCESS_PROTOCOL_RAW: + process_read_buffer(process, buffer, callback); + break; + case PROCESS_PROTOCOL_LINE: + process_read_lines(process, buffer, callback); + break; + default: + /* LCOV_EXCL_START */ + tor_assert_unreached(); + return; + /* LCOV_EXCL_STOP */ + } +} + +/** This function takes the content of the given <b>buffer</b> and passes it to + * the given <b>callback</b> function, but ensures that an additional zero byte + * is added to the end of the data such that the given callback implementation + * can threat the content as a ASCIIZ string. */ +STATIC void +process_read_buffer(process_t *process, + buf_t *buffer, + process_read_callback_t callback) +{ + tor_assert(process); + tor_assert(buffer); + + const size_t size = buf_datalen(buffer); + + /* We allocate an extra byte for the zero byte in the end. */ + char *data = tor_malloc_zero(size + 1); + + buf_get_bytes(buffer, data, size); + log_debug(LD_PROCESS, "Read data from process"); + + if (callback) + callback(process, data, size); + + tor_free(data); +} + +/** This function tries to extract complete lines from the given <b>buffer</b> + * and calls the given <b>callback</b> function whenever it has a complete + * line. Before calling <b>callback</b> we remove the trailing "\n" or "\r\n" + * from the line. If we are unable to extract a complete line we leave the data + * in the buffer for next call. */ +STATIC void +process_read_lines(process_t *process, + buf_t *buffer, + process_read_callback_t callback) +{ + tor_assert(process); + tor_assert(buffer); + + const size_t size = buf_datalen(buffer) + 1; + size_t line_size = 0; + char *data = tor_malloc_zero(size); + int ret; + + while (true) { + line_size = size; + ret = buf_get_line(buffer, data, &line_size); + + /* A complete line should always be smaller than the size of our + * buffer. */ + tor_assert(ret != -1); + + /* Remove \n from the end of the line. */ + if (line_size >= 1 && data[line_size - 1] == '\n') { + data[line_size - 1] = '\0'; + --line_size; + } + + /* Remove \r from the end of the line. */ + if (line_size >= 1 && data[line_size - 1] == '\r') { + data[line_size - 1] = '\0'; + --line_size; + } + + if (ret == 1) { + log_debug(LD_PROCESS, "Read line from process: \"%s\"", data); + + if (callback) + callback(process, data, line_size); + + /* We have read a whole line, let's see if there is more lines to read. + * */ + continue; + } + + /* No complete line for us to read. We are done for now. */ + tor_assert_nonfatal(ret == 0); + break; + } + + tor_free(data); +} diff --git a/src/lib/process/process.h b/src/lib/process/process.h new file mode 100644 index 0000000000..956d34ab29 --- /dev/null +++ b/src/lib/process/process.h @@ -0,0 +1,145 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process.h + * \brief Header for process.c + **/ + +#ifndef TOR_PROCESS_H +#define TOR_PROCESS_H + +#include "orconfig.h" +#include "lib/malloc/malloc.h" +#include "lib/string/printf.h" + +/** Maximum number of bytes to write to a process' stdin. */ +#define PROCESS_MAX_WRITE (1024) + +/** Maximum number of bytes to read from a process' stdout/stderr. */ +#define PROCESS_MAX_READ (1024) + +typedef enum { + /** The process is not running. */ + PROCESS_STATUS_NOT_RUNNING, + + /** The process is running. */ + PROCESS_STATUS_RUNNING, + + /** The process is in an erroneous state. */ + PROCESS_STATUS_ERROR +} process_status_t; + +const char *process_status_to_string(process_status_t status); + +typedef enum { + /** Pass complete \n-terminated lines to the + * callback (with the \n or \r\n removed). */ + PROCESS_PROTOCOL_LINE, + + /** Pass the raw response from read() to the callback. */ + PROCESS_PROTOCOL_RAW +} process_protocol_t; + +const char *process_protocol_to_string(process_protocol_t protocol); + +void tor_disable_spawning_background_processes(void); + +struct smartlist_t; + +struct process_t; +typedef struct process_t process_t; + +typedef uint64_t process_exit_code_t; +typedef uint64_t process_pid_t; + +typedef void (*process_read_callback_t)(process_t *, + const char *, + size_t); +typedef bool +(*process_exit_callback_t)(process_t *, process_exit_code_t); + +void process_init(void); +void process_free_all(void); +const struct smartlist_t *process_get_all_processes(void); + +process_t *process_new(const char *command); +void process_free_(process_t *process); +#define process_free(s) FREE_AND_NULL(process_t, process_free_, (s)) + +process_status_t process_exec(process_t *process); +bool process_terminate(process_t *process); + +process_pid_t process_get_pid(process_t *process); + +void process_set_stdout_read_callback(process_t *, + process_read_callback_t); +void process_set_stderr_read_callback(process_t *, + process_read_callback_t); +void process_set_exit_callback(process_t *, + process_exit_callback_t); + +const char *process_get_command(const process_t *process); + +void process_append_argument(process_t *process, const char *argument); +const struct smartlist_t *process_get_arguments(const process_t *process); +char **process_get_argv(const process_t *process); + +void process_reset_environment(process_t *process, + const struct smartlist_t *env); +void process_set_environment(process_t *process, + const char *key, + const char *value); + +struct process_environment_t; +struct process_environment_t *process_get_environment(const process_t *); + +void process_set_protocol(process_t *process, process_protocol_t protocol); +process_protocol_t process_get_protocol(const process_t *process); + +void process_set_data(process_t *process, void *data); +void *process_get_data(const process_t *process); + +void process_set_status(process_t *process, process_status_t status); +process_status_t process_get_status(const process_t *process); + +#ifndef _WIN32 +struct process_unix_t; +struct process_unix_t *process_get_unix_process(const process_t *process); +#else +struct process_win32_t; +struct process_win32_t *process_get_win32_process(const process_t *process); +#endif + +void process_write(process_t *process, + const uint8_t *data, size_t size); +void process_vprintf(process_t *process, + const char *format, va_list args) CHECK_PRINTF(2, 0); +void process_printf(process_t *process, + const char *format, ...) CHECK_PRINTF(2, 3); + +void process_notify_event_stdout(process_t *process); +void process_notify_event_stderr(process_t *process); +void process_notify_event_stdin(process_t *process); +void process_notify_event_exit(process_t *process, + process_exit_code_t); + +#ifdef PROCESS_PRIVATE +MOCK_DECL(STATIC int, process_read_stdout, (process_t *, buf_t *)); +MOCK_DECL(STATIC int, process_read_stderr, (process_t *, buf_t *)); +MOCK_DECL(STATIC void, process_write_stdin, (process_t *, buf_t *)); + +STATIC void process_read_data(process_t *process, + buf_t *buffer, + process_read_callback_t callback); +STATIC void process_read_buffer(process_t *process, + buf_t *buffer, + process_read_callback_t callback); +STATIC void process_read_lines(process_t *process, + buf_t *buffer, + process_read_callback_t callback); +#endif /* defined(PROCESS_PRIVATE). */ + +#endif /* defined(TOR_PROCESS_H). */ diff --git a/src/lib/process/process_sys.c b/src/lib/process/process_sys.c new file mode 100644 index 0000000000..d0a94f35f7 --- /dev/null +++ b/src/lib/process/process_sys.c @@ -0,0 +1,33 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process_sys.c + * \brief Subsystem object for process setup. + **/ + +#include "orconfig.h" +#include "lib/subsys/subsys.h" +#include "lib/process/process_sys.h" +#include "lib/process/process.h" + +static int +subsys_process_initialize(void) +{ + process_init(); + return 0; +} + +static void +subsys_process_shutdown(void) +{ + process_free_all(); +} + +const subsys_fns_t sys_process = { + .name = "process", + .level = -35, + .supported = true, + .initialize = subsys_process_initialize, + .shutdown = subsys_process_shutdown +}; diff --git a/src/lib/process/process_sys.h b/src/lib/process/process_sys.h new file mode 100644 index 0000000000..b299334b6b --- /dev/null +++ b/src/lib/process/process_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process_sys.h + * \brief Declare subsystem object for the process module. + **/ + +#ifndef TOR_PROCESS_SYS_H +#define TOR_PROCESS_SYS_H + +extern const struct subsys_fns_t sys_process; + +#endif /* !defined(TOR_PROCESS_SYS_H) */ diff --git a/src/lib/process/process_unix.c b/src/lib/process/process_unix.c new file mode 100644 index 0000000000..57ca69a768 --- /dev/null +++ b/src/lib/process/process_unix.c @@ -0,0 +1,705 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process_unix.c + * \brief Module for working with Unix processes. + **/ + +#define PROCESS_UNIX_PRIVATE +#include "lib/intmath/cmp.h" +#include "lib/buf/buffers.h" +#include "lib/net/buffers_net.h" +#include "lib/container/smartlist.h" +#include "lib/evloop/compat_libevent.h" +#include "lib/log/log.h" +#include "lib/log/util_bug.h" +#include "lib/process/process.h" +#include "lib/process/process_unix.h" +#include "lib/process/waitpid.h" +#include "lib/process/env.h" + +#include <stdio.h> + +#ifdef HAVE_STRING_H +#include <string.h> +#endif + +#ifdef HAVE_ERRNO_H +#include <errno.h> +#endif + +#ifdef HAVE_UNISTD_H +#include <unistd.h> +#endif + +#ifdef HAVE_FCNTL_H +#include <fcntl.h> +#endif + +#if defined(HAVE_SYS_PRCTL_H) && defined(__linux__) +#include <sys/prctl.h> +#endif + +#if HAVE_SIGNAL_H +#include <signal.h> +#endif + +#ifndef _WIN32 + +/** Maximum number of file descriptors, if we cannot get it via sysconf() */ +#define DEFAULT_MAX_FD 256 + +/** Internal state for Unix handles. */ +struct process_unix_handle_t { + /** Unix File Descriptor. */ + int fd; + + /** Have we reached end of file? */ + bool reached_eof; + + /** Event structure for libevent. */ + struct event *event; + + /** Are we writing? */ + bool is_writing; +}; + +/** Internal state for our Unix process. */ +struct process_unix_t { + /** Standard in handle. */ + process_unix_handle_t stdin_handle; + + /** Standard out handle. */ + process_unix_handle_t stdout_handle; + + /** Standard error handle. */ + process_unix_handle_t stderr_handle; + + /** The process identifier of our process. */ + pid_t pid; + + /** Waitpid Callback structure. */ + waitpid_callback_t *waitpid; +}; + +/** Returns a newly allocated <b>process_unix_t</b>. */ +process_unix_t * +process_unix_new(void) +{ + process_unix_t *unix_process; + unix_process = tor_malloc_zero(sizeof(process_unix_t)); + + unix_process->stdin_handle.fd = -1; + unix_process->stderr_handle.fd = -1; + unix_process->stdout_handle.fd = -1; + + return unix_process; +} + +/** Deallocates the given <b>unix_process</b>. */ +void +process_unix_free_(process_unix_t *unix_process) +{ + if (! unix_process) + return; + + /* Clean up our waitpid callback. */ + clear_waitpid_callback(unix_process->waitpid); + + /* FIXME(ahf): Refactor waitpid code? */ + unix_process->waitpid = NULL; + + /* Close all our file descriptors. */ + process_unix_close_file_descriptors(unix_process); + + tor_event_free(unix_process->stdout_handle.event); + tor_event_free(unix_process->stderr_handle.event); + tor_event_free(unix_process->stdin_handle.event); + + tor_free(unix_process); +} + +/** Executes the given process as a child process of Tor. This function is + * responsible for setting up the child process and run it. This includes + * setting up pipes for interprocess communication, initialize the waitpid + * callbacks, and finally run fork() followed by execve(). Returns + * <b>PROCESS_STATUS_RUNNING</b> upon success. */ +process_status_t +process_unix_exec(process_t *process) +{ + static int max_fd = -1; + + process_unix_t *unix_process; + pid_t pid; + int stdin_pipe[2]; + int stdout_pipe[2]; + int stderr_pipe[2]; + int retval, fd; + + unix_process = process_get_unix_process(process); + + /* Create standard in pipe. */ + retval = pipe(stdin_pipe); + + if (-1 == retval) { + log_warn(LD_PROCESS, + "Unable to create pipe for stdin " + "communication with process: %s", + strerror(errno)); + + return PROCESS_STATUS_ERROR; + } + + /* Create standard out pipe. */ + retval = pipe(stdout_pipe); + + if (-1 == retval) { + log_warn(LD_PROCESS, + "Unable to create pipe for stdout " + "communication with process: %s", + strerror(errno)); + + /** Cleanup standard in pipe. */ + close(stdin_pipe[0]); + close(stdin_pipe[1]); + + return PROCESS_STATUS_ERROR; + } + + /* Create standard error pipe. */ + retval = pipe(stderr_pipe); + + if (-1 == retval) { + log_warn(LD_PROCESS, + "Unable to create pipe for stderr " + "communication with process: %s", + strerror(errno)); + + /** Cleanup standard in pipe. */ + close(stdin_pipe[0]); + close(stdin_pipe[1]); + + /** Cleanup standard out pipe. */ + close(stdin_pipe[0]); + close(stdin_pipe[1]); + + return PROCESS_STATUS_ERROR; + } + +#ifdef _SC_OPEN_MAX + if (-1 == max_fd) { + max_fd = (int)sysconf(_SC_OPEN_MAX); + + if (max_fd == -1) { + max_fd = DEFAULT_MAX_FD; + log_warn(LD_PROCESS, + "Cannot find maximum file descriptor, assuming: %d", max_fd); + } + } +#else /* !(defined(_SC_OPEN_MAX)) */ + max_fd = DEFAULT_MAX_FD; +#endif /* defined(_SC_OPEN_MAX) */ + + pid = fork(); + + if (0 == pid) { + /* This code is running in the child process context. */ + +#if defined(HAVE_SYS_PRCTL_H) && defined(__linux__) + /* Attempt to have the kernel issue a SIGTERM if the parent + * goes away. Certain attributes of the binary being execve()ed + * will clear this during the execve() call, but it's better + * than nothing. + */ + prctl(PR_SET_PDEATHSIG, SIGTERM); +#endif /* defined(HAVE_SYS_PRCTL_H) && defined(__linux__) */ + + /* Link process stdout to the write end of the pipe. */ + retval = dup2(stdout_pipe[1], STDOUT_FILENO); + if (-1 == retval) + goto error; + + /* Link process stderr to the write end of the pipe. */ + retval = dup2(stderr_pipe[1], STDERR_FILENO); + if (-1 == retval) + goto error; + + /* Link process stdin to the read end of the pipe */ + retval = dup2(stdin_pipe[0], STDIN_FILENO); + if (-1 == retval) + goto error; + + /* Close our pipes now after they have been dup2()'ed. */ + close(stderr_pipe[0]); + close(stderr_pipe[1]); + close(stdout_pipe[0]); + close(stdout_pipe[1]); + close(stdin_pipe[0]); + close(stdin_pipe[1]); + + /* Close all other fds, including the read end of the pipe. XXX: We should + * now be doing enough FD_CLOEXEC setting to make this needless. + */ + for (fd = STDERR_FILENO + 1; fd < max_fd; fd++) + close(fd); + + /* Create the argv value for our new process. */ + char **argv = process_get_argv(process); + + /* Create the env value for our new process. */ + process_environment_t *env = process_get_environment(process); + + /* Call the requested program. */ + retval = execve(argv[0], argv, env->unixoid_environment_block); + + /* If we made it here it is because execve failed :-( */ + if (-1 == retval) + fprintf(stderr, "Call to execve() failed: %s", strerror(errno)); + + tor_free(argv); + process_environment_free(env); + + tor_assert_unreached(); + + error: + /* LCOV_EXCL_START */ + fprintf(stderr, "Error from child process: %s", strerror(errno)); + _exit(1); + /* LCOV_EXCL_STOP */ + } + + /* We are in the parent process. */ + if (-1 == pid) { + log_warn(LD_PROCESS, + "Failed to create child process: %s", strerror(errno)); + + /** Cleanup standard in pipe. */ + close(stdin_pipe[0]); + close(stdin_pipe[1]); + + /** Cleanup standard out pipe. */ + close(stdin_pipe[0]); + close(stdin_pipe[1]); + + /** Cleanup standard error pipe. */ + close(stderr_pipe[0]); + close(stderr_pipe[1]); + + return PROCESS_STATUS_ERROR; + } + + /* Register our PID. */ + unix_process->pid = pid; + + /* Setup waitpid callbacks. */ + unix_process->waitpid = set_waitpid_callback(pid, + process_unix_waitpid_callback, + process); + + /* Handle standard out. */ + unix_process->stdout_handle.fd = stdout_pipe[0]; + retval = close(stdout_pipe[1]); + + if (-1 == retval) { + log_warn(LD_PROCESS, "Failed to close write end of standard out pipe: %s", + strerror(errno)); + } + + /* Handle standard error. */ + unix_process->stderr_handle.fd = stderr_pipe[0]; + retval = close(stderr_pipe[1]); + + if (-1 == retval) { + log_warn(LD_PROCESS, + "Failed to close write end of standard error pipe: %s", + strerror(errno)); + } + + /* Handle standard in. */ + unix_process->stdin_handle.fd = stdin_pipe[1]; + retval = close(stdin_pipe[0]); + + if (-1 == retval) { + log_warn(LD_PROCESS, "Failed to close read end of standard in pipe: %s", + strerror(errno)); + } + + /* Setup our handles. */ + process_unix_setup_handle(process, + &unix_process->stdout_handle, + EV_READ|EV_PERSIST, + stdout_read_callback); + + process_unix_setup_handle(process, + &unix_process->stderr_handle, + EV_READ|EV_PERSIST, + stderr_read_callback); + + process_unix_setup_handle(process, + &unix_process->stdin_handle, + EV_WRITE|EV_PERSIST, + stdin_write_callback); + + /* Start reading from standard out and standard error. */ + process_unix_start_reading(&unix_process->stdout_handle); + process_unix_start_reading(&unix_process->stderr_handle); + + return PROCESS_STATUS_RUNNING; +} + +/** Terminate the given process. Returns true on success, otherwise false. */ +bool +process_unix_terminate(process_t *process) +{ + tor_assert(process); + + process_unix_t *unix_process = process_get_unix_process(process); + + /* All running processes should have a waitpid. */ + if (BUG(unix_process->waitpid == NULL)) + return false; + + bool success = true; + + /* Send a SIGTERM to our child process. */ + int ret; + + ret = kill(unix_process->pid, SIGTERM); + + if (ret == -1) { + log_warn(LD_PROCESS, "Unable to terminate process: %s", + strerror(errno)); + success = false; + } + + /* Close all our FD's. */ + if (! process_unix_close_file_descriptors(unix_process)) + success = false; + + return success; +} + +/** Returns the unique process identifier for the given <b>process</b>. */ +process_pid_t +process_unix_get_pid(process_t *process) +{ + tor_assert(process); + + process_unix_t *unix_process = process_get_unix_process(process); + return (process_pid_t)unix_process->pid; +} + +/** Write the given <b>buffer</b> as input to the given <b>process</b>'s + * standard input. Returns the number of bytes written. */ +int +process_unix_write(process_t *process, buf_t *buffer) +{ + tor_assert(process); + tor_assert(buffer); + + process_unix_t *unix_process = process_get_unix_process(process); + + size_t buffer_flush_len = buf_datalen(buffer); + const size_t max_to_write = MIN(PROCESS_MAX_WRITE, buffer_flush_len); + + /* If we have data to write (when buffer_flush_len > 0) and we are not + * currently getting file descriptor events from the kernel, we tell the + * kernel to start notifying us about when we can write to our file + * descriptor and return. */ + if (buffer_flush_len > 0 && ! unix_process->stdin_handle.is_writing) { + process_unix_start_writing(&unix_process->stdin_handle); + return 0; + } + + /* We don't have any data to write, but the kernel is currently notifying us + * about whether we are able to write or not. Tell the kernel to stop + * notifying us until we have data to write. */ + if (buffer_flush_len == 0 && unix_process->stdin_handle.is_writing) { + process_unix_stop_writing(&unix_process->stdin_handle); + return 0; + } + + /* We have data to write and the kernel have told us to write it. */ + return buf_flush_to_pipe(buffer, + process_get_unix_process(process)->stdin_handle.fd, + max_to_write, &buffer_flush_len); +} + +/** Read data from the given process's standard output and put it into + * <b>buffer</b>. Returns the number of bytes read. */ +int +process_unix_read_stdout(process_t *process, buf_t *buffer) +{ + tor_assert(process); + tor_assert(buffer); + + process_unix_t *unix_process = process_get_unix_process(process); + + return process_unix_read_handle(process, + &unix_process->stdout_handle, + buffer); +} + +/** Read data from the given process's standard error and put it into + * <b>buffer</b>. Returns the number of bytes read. */ +int +process_unix_read_stderr(process_t *process, buf_t *buffer) +{ + tor_assert(process); + tor_assert(buffer); + + process_unix_t *unix_process = process_get_unix_process(process); + + return process_unix_read_handle(process, + &unix_process->stderr_handle, + buffer); +} + +/** This function is called whenever libevent thinks we have data that could be + * read from the child process's standard output. We notify the Process + * subsystem, which is then responsible for calling back to us for doing the + * actual reading of the data. */ +STATIC void +stdout_read_callback(evutil_socket_t fd, short event, void *data) +{ + (void)fd; + (void)event; + + process_t *process = data; + tor_assert(process); + + process_notify_event_stdout(process); +} + +/** This function is called whenever libevent thinks we have data that could be + * read from the child process's standard error. We notify the Process + * subsystem, which is then responsible for calling back to us for doing the + * actual reading of the data. */ +STATIC void +stderr_read_callback(evutil_socket_t fd, short event, void *data) +{ + (void)fd; + (void)event; + + process_t *process = data; + tor_assert(process); + + process_notify_event_stderr(process); +} + +/** This function is called whenever libevent thinks we have data that could be + * written the child process's standard input. We notify the Process subsystem, + * which is then responsible for calling back to us for doing the actual write + * of the data. */ +STATIC void +stdin_write_callback(evutil_socket_t fd, short event, void *data) +{ + (void)fd; + (void)event; + + process_t *process = data; + tor_assert(process); + + process_notify_event_stdin(process); +} + +/** This function tells libevent that we are interested in receiving read + * events from the given <b>handle</b>. */ +STATIC void +process_unix_start_reading(process_unix_handle_t *handle) +{ + tor_assert(handle); + + if (event_add(handle->event, NULL)) + log_warn(LD_PROCESS, + "Unable to add libevent event for handle."); +} + +/** This function tells libevent that we are no longer interested in receiving + * read events from the given <b>handle</b>. */ +STATIC void +process_unix_stop_reading(process_unix_handle_t *handle) +{ + tor_assert(handle); + + if (handle->event == NULL) + return; + + if (event_del(handle->event)) + log_warn(LD_PROCESS, + "Unable to delete libevent event for handle."); +} + +/** This function tells libevent that we are interested in receiving write + * events from the given <b>handle</b>. */ +STATIC void +process_unix_start_writing(process_unix_handle_t *handle) +{ + tor_assert(handle); + + if (event_add(handle->event, NULL)) + log_warn(LD_PROCESS, + "Unable to add libevent event for handle."); + + handle->is_writing = true; +} + +/** This function tells libevent that we are no longer interested in receiving + * write events from the given <b>handle</b>. */ +STATIC void +process_unix_stop_writing(process_unix_handle_t *handle) +{ + tor_assert(handle); + + if (handle->event == NULL) + return; + + if (event_del(handle->event)) + log_warn(LD_PROCESS, + "Unable to delete libevent event for handle."); + + handle->is_writing = false; +} + +/** This function is called when the waitpid system have detected that our + * process have terminated. We disable the waitpid system and notify the + * Process subsystem that we have terminated. */ +STATIC void +process_unix_waitpid_callback(int status, void *data) +{ + tor_assert(data); + + process_t *process = data; + process_unix_t *unix_process = process_get_unix_process(process); + + /* Remove our waitpid callback. */ + clear_waitpid_callback(unix_process->waitpid); + unix_process->waitpid = NULL; + + /* Notify our process. */ + process_notify_event_exit(process, status); + + /* Make sure you don't modify the process after we have called + * process_notify_event_exit() on it, to allow users to process_free() it in + * the exit callback. */ +} + +/** This function sets the file descriptor in the <b>handle</b> as non-blocking + * and configures the libevent event structure based on the given <b>flags</b> + * to ensure that <b>callback</b> is called whenever we have events on the + * given <b>handle</b>. */ +STATIC void +process_unix_setup_handle(process_t *process, + process_unix_handle_t *handle, + short flags, + event_callback_fn callback) +{ + tor_assert(process); + tor_assert(handle); + tor_assert(callback); + + /* Put our file descriptor into non-blocking mode. */ + if (fcntl(handle->fd, F_SETFL, O_NONBLOCK) < 0) { + log_warn(LD_PROCESS, "Unable mark Unix handle as non-blocking: %s", + strerror(errno)); + } + + /* Setup libevent event. */ + handle->event = tor_event_new(tor_libevent_get_base(), + handle->fd, + flags, + callback, + process); +} + +/** This function reads data from the given <b>handle</b> and puts it into + * <b>buffer</b>. Returns the number of bytes read this way. */ +STATIC int +process_unix_read_handle(process_t *process, + process_unix_handle_t *handle, + buf_t *buffer) +{ + tor_assert(process); + tor_assert(handle); + tor_assert(buffer); + + int ret = 0; + int eof = 0; + int error = 0; + + ret = buf_read_from_pipe(buffer, + handle->fd, + PROCESS_MAX_READ, + &eof, + &error); + + if (error) + log_warn(LD_PROCESS, + "Unable to read data: %s", strerror(error)); + + if (eof) { + handle->reached_eof = true; + process_unix_stop_reading(handle); + } + + return ret; +} + +/** Close the standard in, out, and error handles of the given + * <b>unix_process</b>. */ +STATIC bool +process_unix_close_file_descriptors(process_unix_t *unix_process) +{ + tor_assert(unix_process); + + int ret; + bool success = true; + + /* Stop reading and writing before we close() our + * file descriptors. */ + if (! unix_process->stdout_handle.reached_eof) + process_unix_stop_reading(&unix_process->stdout_handle); + + if (! unix_process->stderr_handle.reached_eof) + process_unix_stop_reading(&unix_process->stderr_handle); + + if (unix_process->stdin_handle.is_writing) + process_unix_stop_writing(&unix_process->stdin_handle); + + if (unix_process->stdin_handle.fd != -1) { + ret = close(unix_process->stdin_handle.fd); + if (ret == -1) { + log_warn(LD_PROCESS, "Unable to close standard in"); + success = false; + } + + unix_process->stdin_handle.fd = -1; + } + + if (unix_process->stdout_handle.fd != -1) { + ret = close(unix_process->stdout_handle.fd); + if (ret == -1) { + log_warn(LD_PROCESS, "Unable to close standard out"); + success = false; + } + + unix_process->stdout_handle.fd = -1; + } + + if (unix_process->stderr_handle.fd != -1) { + ret = close(unix_process->stderr_handle.fd); + if (ret == -1) { + log_warn(LD_PROCESS, "Unable to close standard error"); + success = false; + } + + unix_process->stderr_handle.fd = -1; + } + + return success; +} + +#endif /* defined(_WIN32). */ diff --git a/src/lib/process/process_unix.h b/src/lib/process/process_unix.h new file mode 100644 index 0000000000..86c10d7449 --- /dev/null +++ b/src/lib/process/process_unix.h @@ -0,0 +1,68 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process_unix.h + * \brief Header for process_unix.c + **/ + +#ifndef TOR_PROCESS_UNIX_H +#define TOR_PROCESS_UNIX_H + +#ifndef _WIN32 + +#include "orconfig.h" +#include "lib/malloc/malloc.h" + +#include <event2/event.h> + +struct process_t; + +struct process_unix_t; +typedef struct process_unix_t process_unix_t; + +process_unix_t *process_unix_new(void); +void process_unix_free_(process_unix_t *unix_process); +#define process_unix_free(s) \ + FREE_AND_NULL(process_unix_t, process_unix_free_, (s)) + +process_status_t process_unix_exec(struct process_t *process); +bool process_unix_terminate(struct process_t *process); + +process_pid_t process_unix_get_pid(struct process_t *process); + +int process_unix_write(struct process_t *process, buf_t *buffer); +int process_unix_read_stdout(struct process_t *process, buf_t *buffer); +int process_unix_read_stderr(struct process_t *process, buf_t *buffer); + +#ifdef PROCESS_UNIX_PRIVATE +struct process_unix_handle_t; +typedef struct process_unix_handle_t process_unix_handle_t; + +STATIC void stdout_read_callback(evutil_socket_t fd, short event, void *data); +STATIC void stderr_read_callback(evutil_socket_t fd, short event, void *data); +STATIC void stdin_write_callback(evutil_socket_t fd, short event, void *data); + +STATIC void process_unix_start_reading(process_unix_handle_t *); +STATIC void process_unix_stop_reading(process_unix_handle_t *); + +STATIC void process_unix_start_writing(process_unix_handle_t *); +STATIC void process_unix_stop_writing(process_unix_handle_t *); + +STATIC void process_unix_waitpid_callback(int status, void *data); + +STATIC void process_unix_setup_handle(process_t *process, + process_unix_handle_t *handle, + short flags, + event_callback_fn callback); +STATIC int process_unix_read_handle(process_t *, + process_unix_handle_t *, + buf_t *); +STATIC bool process_unix_close_file_descriptors(process_unix_t *); +#endif /* defined(PROCESS_UNIX_PRIVATE). */ + +#endif /* defined(_WIN32). */ + +#endif /* defined(TOR_PROCESS_UNIX_H). */ diff --git a/src/lib/process/process_win32.c b/src/lib/process/process_win32.c new file mode 100644 index 0000000000..d8a895875c --- /dev/null +++ b/src/lib/process/process_win32.c @@ -0,0 +1,1092 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process_win32.c + * \brief Module for working with Windows processes. + **/ + +#define PROCESS_WIN32_PRIVATE +#include "lib/intmath/cmp.h" +#include "lib/buf/buffers.h" +#include "lib/net/buffers_net.h" +#include "lib/container/smartlist.h" +#include "lib/log/log.h" +#include "lib/log/util_bug.h" +#include "lib/log/win32err.h" +#include "lib/process/process.h" +#include "lib/process/process_win32.h" +#include "lib/process/env.h" + +#ifdef HAVE_SYS_TIME_H +#include <sys/time.h> +#endif + +#ifdef HAVE_STRING_H +#include <string.h> +#endif + +#ifdef _WIN32 + +/** The size of our intermediate buffers. */ +#define BUFFER_SIZE (1024) + +/** Timer that ticks once a second and calls the process_win32_timer_callback() + * function. */ +static periodic_timer_t *periodic_timer; + +/** Structure to represent the state around the pipe HANDLE. + * + * This structure is used to store state about a given HANDLE, including + * whether we have reached end of file, its intermediate buffers, and how much + * data that is available in the intermediate buffer. */ +struct process_win32_handle_t { + /** Standard out pipe handle. */ + HANDLE pipe; + + /** True iff we have reached EOF from the pipe. */ + bool reached_eof; + + /** How much data is available in buffer. */ + size_t data_available; + + /** Intermediate buffer for ReadFileEx() and WriteFileEx(). */ + char buffer[BUFFER_SIZE]; + + /** Overlapped structure for ReadFileEx() and WriteFileEx(). */ + OVERLAPPED overlapped; + + /** Are we waiting for another I/O operation to complete? */ + bool busy; +}; + +/** Structure to represent the Windows specific implementation details of this + * Process backend. + * + * This structure is attached to <b>process_t</b> (see process.h) and is + * reachable from <b>process_t</b> via the <b>process_get_win32_process()</b> + * method. */ +struct process_win32_t { + /** Standard in state. */ + process_win32_handle_t stdin_handle; + + /** Standard out state. */ + process_win32_handle_t stdout_handle; + + /** Standard error state. */ + process_win32_handle_t stderr_handle; + + /** Process Information. */ + PROCESS_INFORMATION process_information; +}; + +/** Create a new <b>process_win32_t</b>. + * + * This function constructs a new <b>process_win32_t</b> and initializes the + * default values. */ +process_win32_t * +process_win32_new(void) +{ + process_win32_t *win32_process; + win32_process = tor_malloc_zero(sizeof(process_win32_t)); + + win32_process->stdin_handle.pipe = INVALID_HANDLE_VALUE; + win32_process->stdout_handle.pipe = INVALID_HANDLE_VALUE; + win32_process->stderr_handle.pipe = INVALID_HANDLE_VALUE; + + return win32_process; +} + +/** Free a given <b>process_win32_t</b>. + * + * This function deinitializes and frees up the resources allocated for the + * given <b>process_win32_t</b>. */ +void +process_win32_free_(process_win32_t *win32_process) +{ + if (! win32_process) + return; + + /* Cleanup our handles. */ + process_win32_cleanup_handle(&win32_process->stdin_handle); + process_win32_cleanup_handle(&win32_process->stdout_handle); + process_win32_cleanup_handle(&win32_process->stderr_handle); + + tor_free(win32_process); +} + +/** Initialize the Windows backend of the Process subsystem. */ +void +process_win32_init(void) +{ + /* We don't start the periodic timer here because it makes no sense to have + * the timer running until we have some processes that benefits from the + * timer timer ticks. */ +} + +/** Deinitialize the Windows backend of the Process subsystem. */ +void +process_win32_deinit(void) +{ + /* Stop our timer, but only if it's running. */ + if (process_win32_timer_running()) + process_win32_timer_stop(); +} + +/** Execute the given process. This function is responsible for setting up + * named pipes for I/O between the child process and the Tor process. Returns + * <b>PROCESS_STATUS_RUNNING</b> upon success. */ +process_status_t +process_win32_exec(process_t *process) +{ + tor_assert(process); + + process_win32_t *win32_process = process_get_win32_process(process); + + HANDLE stdout_pipe_read = NULL; + HANDLE stdout_pipe_write = NULL; + HANDLE stderr_pipe_read = NULL; + HANDLE stderr_pipe_write = NULL; + HANDLE stdin_pipe_read = NULL; + HANDLE stdin_pipe_write = NULL; + BOOL ret = FALSE; + const char *filename = process_get_command(process); + + /* Not much we can do if we haven't been told what to start. */ + if (BUG(filename == NULL)) + return PROCESS_STATUS_ERROR; + + /* Setup our security attributes. */ + SECURITY_ATTRIBUTES security_attributes; + security_attributes.nLength = sizeof(security_attributes); + security_attributes.bInheritHandle = TRUE; + /* FIXME: should we set explicit security attributes? + * (See Ticket #2046, comment 5) */ + security_attributes.lpSecurityDescriptor = NULL; + + /* Create our standard out pipe. */ + if (! process_win32_create_pipe(&stdout_pipe_read, + &stdout_pipe_write, + &security_attributes, + PROCESS_WIN32_PIPE_TYPE_READER)) { + return PROCESS_STATUS_ERROR; + } + + /* Create our standard error pipe. */ + if (! process_win32_create_pipe(&stderr_pipe_read, + &stderr_pipe_write, + &security_attributes, + PROCESS_WIN32_PIPE_TYPE_READER)) { + return PROCESS_STATUS_ERROR; + } + + /* Create out standard in pipe. */ + if (! process_win32_create_pipe(&stdin_pipe_read, + &stdin_pipe_write, + &security_attributes, + PROCESS_WIN32_PIPE_TYPE_WRITER)) { + return PROCESS_STATUS_ERROR; + } + + /* Configure startup info for our child process. */ + STARTUPINFOA startup_info; + + memset(&startup_info, 0, sizeof(startup_info)); + startup_info.cb = sizeof(startup_info); + startup_info.hStdError = stderr_pipe_write; + startup_info.hStdOutput = stdout_pipe_write; + startup_info.hStdInput = stdin_pipe_read; + startup_info.dwFlags |= STARTF_USESTDHANDLES; + + /* Create the env value for our new process. */ + process_environment_t *env = process_get_environment(process); + + /* Create the argv value for our new process. */ + char **argv = process_get_argv(process); + + /* Windows expects argv to be a whitespace delimited string, so join argv up + */ + char *joined_argv = tor_join_win_cmdline((const char **)argv); + + /* Create the child process */ + ret = CreateProcessA(filename, + joined_argv, + NULL, + NULL, + TRUE, + CREATE_NO_WINDOW, + env->windows_environment_block[0] == '\0' ? + NULL : env->windows_environment_block, + NULL, + &startup_info, + &win32_process->process_information); + + tor_free(argv); + tor_free(joined_argv); + process_environment_free(env); + + if (! ret) { + log_warn(LD_PROCESS, "CreateProcessA() failed: %s", + format_win32_error(GetLastError())); + + /* Cleanup our handles. */ + CloseHandle(stdout_pipe_read); + CloseHandle(stdout_pipe_write); + CloseHandle(stderr_pipe_read); + CloseHandle(stderr_pipe_write); + CloseHandle(stdin_pipe_read); + CloseHandle(stdin_pipe_write); + + return PROCESS_STATUS_ERROR; + } + + /* TODO: Should we close hProcess and hThread in + * process_handle->process_information? */ + win32_process->stdout_handle.pipe = stdout_pipe_read; + win32_process->stderr_handle.pipe = stderr_pipe_read; + win32_process->stdin_handle.pipe = stdin_pipe_write; + + /* Close our ends of the pipes that is now owned by the child process. */ + CloseHandle(stdout_pipe_write); + CloseHandle(stderr_pipe_write); + CloseHandle(stdin_pipe_read); + + /* Used by the callback functions from ReadFileEx() and WriteFileEx() such + * that we can figure out which process_t that was responsible for the event. + * + * Warning, here be dragons: + * + * MSDN says that the hEvent member of the overlapped structure is unused + * for ReadFileEx() and WriteFileEx, which allows us to store a pointer to + * our process state there. + */ + win32_process->stdout_handle.overlapped.hEvent = (HANDLE)process; + win32_process->stderr_handle.overlapped.hEvent = (HANDLE)process; + win32_process->stdin_handle.overlapped.hEvent = (HANDLE)process; + + /* Start our timer if it is not already running. */ + if (! process_win32_timer_running()) + process_win32_timer_start(); + + /* We use Windows Extended I/O functions, so our completion callbacks are + * called automatically for us when there is data to read. Because of this + * we start the read of standard out and error right away. */ + process_notify_event_stdout(process); + process_notify_event_stderr(process); + + return PROCESS_STATUS_RUNNING; +} + +/** Terminate the given process. Returns true on success, otherwise false. */ +bool +process_win32_terminate(process_t *process) +{ + tor_assert(process); + + process_win32_t *win32_process = process_get_win32_process(process); + + /* Terminate our process. */ + BOOL ret; + + ret = TerminateProcess(win32_process->process_information.hProcess, 0); + + if (! ret) { + log_warn(LD_PROCESS, "TerminateProcess() failed: %s", + format_win32_error(GetLastError())); + return false; + } + + /* Cleanup our handles. */ + process_win32_cleanup_handle(&win32_process->stdin_handle); + process_win32_cleanup_handle(&win32_process->stdout_handle); + process_win32_cleanup_handle(&win32_process->stderr_handle); + + return true; +} + +/** Returns the unique process identifier for the given <b>process</b>. */ +process_pid_t +process_win32_get_pid(process_t *process) +{ + tor_assert(process); + + process_win32_t *win32_process = process_get_win32_process(process); + return (process_pid_t)win32_process->process_information.dwProcessId; +} + +/** Schedule an async write of the data found in <b>buffer</b> for the given + * process. This function runs an async write operation of the content of + * buffer, if we are not already waiting for a pending I/O request. Returns the + * number of bytes that Windows will hopefully write for us in the background. + * */ +int +process_win32_write(struct process_t *process, buf_t *buffer) +{ + tor_assert(process); + tor_assert(buffer); + + process_win32_t *win32_process = process_get_win32_process(process); + BOOL ret = FALSE; + DWORD error_code = 0; + const size_t buffer_size = buf_datalen(buffer); + + /* Windows is still writing our buffer. */ + if (win32_process->stdin_handle.busy) + return 0; + + /* Nothing for us to do right now. */ + if (buffer_size == 0) + return 0; + + /* We have reached end of file already? */ + if (BUG(win32_process->stdin_handle.reached_eof)) + return 0; + + /* Figure out how much data we should read. */ + const size_t write_size = MIN(buffer_size, + sizeof(win32_process->stdin_handle.buffer)); + + /* Read data from the process_t buffer into our intermediate buffer. */ + buf_get_bytes(buffer, win32_process->stdin_handle.buffer, write_size); + + /* Because of the slightly weird API for WriteFileEx() we must set this to 0 + * before we call WriteFileEx() because WriteFileEx() does not reset the last + * error itself when it's succesful. See comment below after the call to + * GetLastError(). */ + SetLastError(0); + + /* Schedule our write. */ + ret = WriteFileEx(win32_process->stdin_handle.pipe, + win32_process->stdin_handle.buffer, + write_size, + &win32_process->stdin_handle.overlapped, + process_win32_stdin_write_done); + + if (! ret) { + error_code = GetLastError(); + + /* No need to log at warning level for these two. */ + if (error_code == ERROR_HANDLE_EOF || error_code == ERROR_BROKEN_PIPE) { + log_debug(LD_PROCESS, "WriteFileEx() returned EOF from pipe: %s", + format_win32_error(error_code)); + } else { + log_warn(LD_PROCESS, "WriteFileEx() failed: %s", + format_win32_error(error_code)); + } + + win32_process->stdin_handle.reached_eof = true; + return 0; + } + + /* Here be dragons: According to MSDN's documentation for WriteFileEx() we + * should check GetLastError() after a call to WriteFileEx() even though the + * `ret` return value was successful. If everything is good, GetLastError() + * returns `ERROR_SUCCESS` and nothing happens. + * + * XXX(ahf): I have not managed to trigger this code while stress-testing + * this code. */ + error_code = GetLastError(); + + if (error_code != ERROR_SUCCESS) { + /* LCOV_EXCL_START */ + log_warn(LD_PROCESS, "WriteFileEx() failed after returning success: %s", + format_win32_error(error_code)); + win32_process->stdin_handle.reached_eof = true; + return 0; + /* LCOV_EXCL_STOP */ + } + + /* This cast should be safe since our buffer can maximum be BUFFER_SIZE + * large. */ + return (int)write_size; +} + +/** This function is called from the Process subsystem whenever the Windows + * backend says it has data ready. This function also ensures that we are + * starting a new background read from the standard output of the child process + * and asks Windows to call process_win32_stdout_read_done() when that + * operation is finished. Returns the number of bytes moved into <b>buffer</b>. + * */ +int +process_win32_read_stdout(struct process_t *process, buf_t *buffer) +{ + tor_assert(process); + tor_assert(buffer); + + process_win32_t *win32_process = process_get_win32_process(process); + + return process_win32_read_from_handle(&win32_process->stdout_handle, + buffer, + process_win32_stdout_read_done); +} + +/** This function is called from the Process subsystem whenever the Windows + * backend says it has data ready. This function also ensures that we are + * starting a new background read from the standard error of the child process + * and asks Windows to call process_win32_stderr_read_done() when that + * operation is finished. Returns the number of bytes moved into <b>buffer</b>. + * */ +int +process_win32_read_stderr(struct process_t *process, buf_t *buffer) +{ + tor_assert(process); + tor_assert(buffer); + + process_win32_t *win32_process = process_get_win32_process(process); + + return process_win32_read_from_handle(&win32_process->stderr_handle, + buffer, + process_win32_stderr_read_done); +} + +/** This function is responsible for moving the Tor process into what Microsoft + * calls an "alertable" state. Once the process is in an alertable state the + * Windows kernel will notify us when our background I/O requests have finished + * and the callbacks will be executed. */ +void +process_win32_trigger_completion_callbacks(void) +{ + DWORD ret; + + /* The call to SleepEx(dwMilliseconds, dwAlertable) makes the process sleep + * for dwMilliseconds and if dwAlertable is set to TRUE it will also cause + * the process to enter alertable state, where the Windows kernel will notify + * us about completed I/O requests from ReadFileEx() and WriteFileEX(), which + * will cause our completion callbacks to be executed. + * + * This function returns 0 if the time interval expired or WAIT_IO_COMPLETION + * if one or more I/O callbacks were executed. */ + ret = SleepEx(0, TRUE); + + /* Warn us if the function returned something we did not anticipate. */ + if (ret != 0 && ret != WAIT_IO_COMPLETION) { + log_warn(LD_PROCESS, "SleepEx() returned %lu", ret); + } +} + +/** Start the periodic timer which is reponsible for checking whether processes + * are still alive and to make sure that the Tor process is periodically being + * moved into an alertable state. */ +void +process_win32_timer_start(void) +{ + /* Make sure we never start our timer if it's already running. */ + if (BUG(process_win32_timer_running())) + return; + + /* Wake up once a second. */ + static const struct timeval interval = {1, 0}; + + log_info(LD_PROCESS, "Starting Windows Process I/O timer"); + periodic_timer = periodic_timer_new(tor_libevent_get_base(), + &interval, + process_win32_timer_callback, + NULL); +} + +/** Stops the periodic timer. */ +void +process_win32_timer_stop(void) +{ + if (BUG(periodic_timer == NULL)) + return; + + log_info(LD_PROCESS, "Stopping Windows Process I/O timer"); + periodic_timer_free(periodic_timer); +} + +/** Returns true iff the periodic timer is running. */ +bool +process_win32_timer_running(void) +{ + return periodic_timer != NULL; +} + +/** This function is called whenever the periodic_timer ticks. The function is + * responsible for moving the Tor process into an alertable state once a second + * and checking for whether our child processes have terminated since the last + * tick. */ +STATIC void +process_win32_timer_callback(periodic_timer_t *timer, void *data) +{ + tor_assert(timer == periodic_timer); + tor_assert(data == NULL); + + /* Move the process into an alertable state. */ + process_win32_trigger_completion_callbacks(); + + /* Check if our processes are still alive. */ + + /* Since the call to process_win32_timer_test_process() might call + * process_notify_event_exit() which again might call process_free() which + * updates the list of processes returned by process_get_all_processes() it + * is important here that we make sure to not touch the list of processes if + * the call to process_win32_timer_test_process() returns true. */ + bool done; + + do { + const smartlist_t *processes = process_get_all_processes(); + done = true; + + SMARTLIST_FOREACH_BEGIN(processes, process_t *, process) { + /* If process_win32_timer_test_process() returns true, it means that + * smartlist_remove() might have been called on the list returned by + * process_get_all_processes(). We start the loop over again until we + * have a succesful run over the entire list where the list was not + * modified. */ + if (process_win32_timer_test_process(process)) { + done = false; + break; + } + } SMARTLIST_FOREACH_END(process); + } while (! done); +} + +/** Test whether a given process is still alive. Notify the Process subsystem + * if our process have died. Returns true iff the given process have + * terminated. */ +STATIC bool +process_win32_timer_test_process(process_t *process) +{ + tor_assert(process); + + /* No need to look at processes that don't claim they are running. */ + if (process_get_status(process) != PROCESS_STATUS_RUNNING) + return false; + + process_win32_t *win32_process = process_get_win32_process(process); + BOOL ret = FALSE; + DWORD exit_code = 0; + + /* Sometimes the Windows kernel wont give us the EOF/Broken Pipe error + * message until some time after the process have actually terminated. We + * make sure that our ReadFileEx() calls for the process have *all* returned + * and both standard out and error have been marked as EOF before we try to + * see if the process terminated. + * + * This ensures that we *never* call the exit callback of the `process_t`, + * which potentially ends up calling `process_free()` on our `process_t`, + * before all data have been received from the process. + * + * We do NOT have a check here for whether standard in reached EOF since + * standard in's WriteFileEx() function is only called on-demand when we have + * something to write and is thus usually not awaiting to finish any + * operations. If we WriteFileEx() to a file that has terminated we'll simply + * get an error from ReadFileEx() or its completion routine and move on with + * life. */ + if (! win32_process->stdout_handle.reached_eof) + return false; + + if (! win32_process->stderr_handle.reached_eof) + return false; + + /* We start by testing whether our process is still running. */ + ret = GetExitCodeProcess(win32_process->process_information.hProcess, + &exit_code); + + if (! ret) { + log_warn(LD_PROCESS, "GetExitCodeProcess() failed: %s", + format_win32_error(GetLastError())); + return false; + } + + /* Notify our process_t that our process have terminated. Since our + * exit_callback might decide to process_free() our process handle it is very + * important that we do not touch the process_t after the call to + * process_notify_event_exit(). */ + if (exit_code != STILL_ACTIVE) { + process_notify_event_exit(process, exit_code); + return true; + } + + return false; +} + +/** Create a new overlapped named pipe. This function creates a new connected, + * named, pipe in <b>*read_pipe</b> and <b>*write_pipe</b> if the function is + * succesful. Returns true on sucess, false on failure. */ +STATIC bool +process_win32_create_pipe(HANDLE *read_pipe, + HANDLE *write_pipe, + SECURITY_ATTRIBUTES *attributes, + process_win32_pipe_type_t pipe_type) +{ + tor_assert(read_pipe); + tor_assert(write_pipe); + tor_assert(attributes); + + BOOL ret = FALSE; + + /* Buffer size. */ + const size_t size = 4096; + + /* Our additional read/write modes that depends on which pipe type we are + * creating. */ + DWORD read_mode = 0; + DWORD write_mode = 0; + + /* Generate the unique pipe name. */ + char pipe_name[MAX_PATH]; + static DWORD process_id = 0; + static DWORD counter = 0; + + if (process_id == 0) + process_id = GetCurrentProcessId(); + + tor_snprintf(pipe_name, sizeof(pipe_name), + "\\\\.\\Pipe\\Tor-Process-Pipe-%lu-%lu", + process_id, counter++); + + /* Only one of our handles can be overlapped. */ + switch (pipe_type) { + case PROCESS_WIN32_PIPE_TYPE_READER: + read_mode = FILE_FLAG_OVERLAPPED; + break; + case PROCESS_WIN32_PIPE_TYPE_WRITER: + write_mode = FILE_FLAG_OVERLAPPED; + break; + default: + /* LCOV_EXCL_START */ + tor_assert_nonfatal_unreached_once(); + /* LCOV_EXCL_STOP */ + } + + /* Setup our read and write handles. */ + HANDLE read_handle; + HANDLE write_handle; + + /* Create our named pipe. */ + read_handle = CreateNamedPipeA(pipe_name, + (PIPE_ACCESS_INBOUND|read_mode), + (PIPE_TYPE_BYTE|PIPE_WAIT), + 1, + size, + size, + 1000, + attributes); + + if (read_handle == INVALID_HANDLE_VALUE) { + log_warn(LD_PROCESS, "CreateNamedPipeA() failed: %s", + format_win32_error(GetLastError())); + return false; + } + + /* Create our file in the pipe namespace. */ + write_handle = CreateFileA(pipe_name, + GENERIC_WRITE, + 0, + attributes, + OPEN_EXISTING, + (FILE_ATTRIBUTE_NORMAL|write_mode), + NULL); + + if (write_handle == INVALID_HANDLE_VALUE) { + log_warn(LD_PROCESS, "CreateFileA() failed: %s", + format_win32_error(GetLastError())); + + CloseHandle(read_handle); + + return false; + } + + /* Set the inherit flag for our pipe. */ + switch (pipe_type) { + case PROCESS_WIN32_PIPE_TYPE_READER: + ret = SetHandleInformation(read_handle, HANDLE_FLAG_INHERIT, 0); + break; + case PROCESS_WIN32_PIPE_TYPE_WRITER: + ret = SetHandleInformation(write_handle, HANDLE_FLAG_INHERIT, 0); + break; + default: + /* LCOV_EXCL_START */ + tor_assert_nonfatal_unreached_once(); + /* LCOV_EXCL_STOP */ + } + + if (! ret) { + log_warn(LD_PROCESS, "SetHandleInformation() failed: %s", + format_win32_error(GetLastError())); + + CloseHandle(read_handle); + CloseHandle(write_handle); + + return false; + } + + /* Everything is good. */ + *read_pipe = read_handle; + *write_pipe = write_handle; + + return true; +} + +/** Cleanup a given <b>handle</b>. */ +STATIC void +process_win32_cleanup_handle(process_win32_handle_t *handle) +{ + tor_assert(handle); + +#if 0 + BOOL ret; + DWORD error_code; + + /* Cancel any pending I/O requests: This means that instead of getting + * ERROR_BROKEN_PIPE we get ERROR_OPERATION_ABORTED, but it doesn't seem + * like this is needed. */ + ret = CancelIo(handle->pipe); + + if (! ret) { + error_code = GetLastError(); + + /* There was no pending I/O requests for our handle. */ + if (error_code != ERROR_NOT_FOUND) { + log_warn(LD_PROCESS, "CancelIo() failed: %s", + format_win32_error(error_code)); + } + } +#endif + + /* Close our handle. */ + if (handle->pipe != INVALID_HANDLE_VALUE) { + CloseHandle(handle->pipe); + handle->pipe = INVALID_HANDLE_VALUE; + handle->reached_eof = true; + } +} + +/** This function is called when ReadFileEx() completes its background read + * from the child process's standard output. We notify the Process subsystem if + * there is data available for it to read from us. */ +STATIC VOID WINAPI +process_win32_stdout_read_done(DWORD error_code, + DWORD byte_count, + LPOVERLAPPED overlapped) +{ + tor_assert(overlapped); + tor_assert(overlapped->hEvent); + + /* Extract our process_t from the hEvent member of OVERLAPPED. */ + process_t *process = (process_t *)overlapped->hEvent; + process_win32_t *win32_process = process_get_win32_process(process); + + if (process_win32_handle_read_completion(&win32_process->stdout_handle, + error_code, + byte_count)) { + /* Schedule our next read. */ + process_notify_event_stdout(process); + } +} + +/** This function is called when ReadFileEx() completes its background read + * from the child process's standard error. We notify the Process subsystem if + * there is data available for it to read from us. */ +STATIC VOID WINAPI +process_win32_stderr_read_done(DWORD error_code, + DWORD byte_count, + LPOVERLAPPED overlapped) +{ + tor_assert(overlapped); + tor_assert(overlapped->hEvent); + + /* Extract our process_t from the hEvent member of OVERLAPPED. */ + process_t *process = (process_t *)overlapped->hEvent; + process_win32_t *win32_process = process_get_win32_process(process); + + if (process_win32_handle_read_completion(&win32_process->stderr_handle, + error_code, + byte_count)) { + /* Schedule our next read. */ + process_notify_event_stderr(process); + } +} + +/** This function is called when WriteFileEx() completes its background write + * to the child process's standard input. We notify the Process subsystem that + * it can write data to us again. */ +STATIC VOID WINAPI +process_win32_stdin_write_done(DWORD error_code, + DWORD byte_count, + LPOVERLAPPED overlapped) +{ + tor_assert(overlapped); + tor_assert(overlapped->hEvent); + + (void)byte_count; + + process_t *process = (process_t *)overlapped->hEvent; + process_win32_t *win32_process = process_get_win32_process(process); + + /* Mark our handle as not having any outstanding I/O requests. */ + win32_process->stdin_handle.busy = false; + + /* Check if we have been asked to write to the handle that have been marked + * as having reached EOF. */ + if (BUG(win32_process->stdin_handle.reached_eof)) + return; + + if (error_code == 0) { + /** Our data have been succesfully written. Clear our state and schedule + * the next write. */ + win32_process->stdin_handle.data_available = 0; + memset(win32_process->stdin_handle.buffer, 0, + sizeof(win32_process->stdin_handle.buffer)); + + /* Schedule the next write. */ + process_notify_event_stdin(process); + } else if (error_code == ERROR_HANDLE_EOF || + error_code == ERROR_BROKEN_PIPE) { + /* Our WriteFileEx() call was succesful, but we reached the end of our + * file. We mark our handle as having reached EOF and returns. */ + tor_assert(byte_count == 0); + + win32_process->stdin_handle.reached_eof = true; + } else { + /* An error happened: We warn the user and mark our handle as having + * reached EOF */ + log_warn(LD_PROCESS, + "Error in I/O completion routine from WriteFileEx(): %s", + format_win32_error(error_code)); + win32_process->stdin_handle.reached_eof = true; + } +} + +/** This function reads data from the given <b>handle</b>'s internal buffer and + * moves it into the given <b>buffer</b>. Additionally, we start the next + * ReadFileEx() background operation with the given <b>callback</b> as + * completion callback. Returns the number of bytes written to the buffer. */ +STATIC int +process_win32_read_from_handle(process_win32_handle_t *handle, + buf_t *buffer, + LPOVERLAPPED_COMPLETION_ROUTINE callback) +{ + tor_assert(handle); + tor_assert(buffer); + tor_assert(callback); + + BOOL ret = FALSE; + int bytes_available = 0; + DWORD error_code = 0; + + /* We already have a request to read data that isn't complete yet. */ + if (BUG(handle->busy)) + return 0; + + /* Check if we have been asked to read from a handle that have already told + * us that we have reached the end of the file. */ + if (BUG(handle->reached_eof)) + return 0; + + /* This cast should be safe since our buffer can be at maximum up to + * BUFFER_SIZE in size. */ + bytes_available = (int)handle->data_available; + + if (handle->data_available > 0) { + /* Read data from our intermediate buffer into the process_t buffer. */ + buf_add(buffer, handle->buffer, handle->data_available); + + /* Reset our read state. */ + handle->data_available = 0; + memset(handle->buffer, 0, sizeof(handle->buffer)); + } + + /* Because of the slightly weird API for ReadFileEx() we must set this to 0 + * before we call ReadFileEx() because ReadFileEx() does not reset the last + * error itself when it's succesful. See comment below after the call to + * GetLastError(). */ + SetLastError(0); + + /* Ask the Windows kernel to read data from our pipe into our buffer and call + * the callback function when it is done. */ + ret = ReadFileEx(handle->pipe, + handle->buffer, + sizeof(handle->buffer), + &handle->overlapped, + callback); + + if (! ret) { + error_code = GetLastError(); + + /* No need to log at warning level for these two. */ + if (error_code == ERROR_HANDLE_EOF || error_code == ERROR_BROKEN_PIPE) { + log_debug(LD_PROCESS, "ReadFileEx() returned EOF from pipe: %s", + format_win32_error(error_code)); + } else { + log_warn(LD_PROCESS, "ReadFileEx() failed: %s", + format_win32_error(error_code)); + } + + handle->reached_eof = true; + return bytes_available; + } + + /* Here be dragons: According to MSDN's documentation for ReadFileEx() we + * should check GetLastError() after a call to ReadFileEx() even though the + * `ret` return value was successful. If everything is good, GetLastError() + * returns `ERROR_SUCCESS` and nothing happens. + * + * XXX(ahf): I have not managed to trigger this code while stress-testing + * this code. */ + error_code = GetLastError(); + + if (error_code != ERROR_SUCCESS) { + /* LCOV_EXCL_START */ + log_warn(LD_PROCESS, "ReadFileEx() failed after returning success: %s", + format_win32_error(error_code)); + handle->reached_eof = true; + return bytes_available; + /* LCOV_EXCL_STOP */ + } + + /* We mark our handle as having a pending I/O request. */ + handle->busy = true; + + return bytes_available; +} + +/** This function checks the callback values from ReadFileEx() in + * <b>error_code</b> and <b>byte_count</b> if we have read data. Returns true + * iff our caller should request more data from ReadFileEx(). */ +STATIC bool +process_win32_handle_read_completion(process_win32_handle_t *handle, + DWORD error_code, + DWORD byte_count) +{ + tor_assert(handle); + + /* Mark our handle as not having any outstanding I/O requests. */ + handle->busy = false; + + if (error_code == 0) { + /* Our ReadFileEx() call was succesful and there is data for us. */ + + /* This cast should be safe since byte_count should never be larger than + * BUFFER_SIZE. */ + tor_assert(byte_count <= BUFFER_SIZE); + handle->data_available = (size_t)byte_count; + + /* Tell our caller to schedule the next read. */ + return true; + } else if (error_code == ERROR_HANDLE_EOF || + error_code == ERROR_BROKEN_PIPE) { + /* Our ReadFileEx() finished, but we reached the end of our file. We mark + * our handle as having reached EOF and returns. */ + tor_assert(byte_count == 0); + + handle->reached_eof = true; + } else { + /* An error happened: We warn the user and mark our handle as having + * reached EOF */ + log_warn(LD_PROCESS, + "Error in I/O completion routine from ReadFileEx(): %s", + format_win32_error(error_code)); + + handle->reached_eof = true; + } + + /* Our caller should NOT schedule the next read. */ + return false; +} + +/** Format a single argument for being put on a Windows command line. + * Returns a newly allocated string */ +STATIC char * +format_win_cmdline_argument(const char *arg) +{ + char *formatted_arg; + char need_quotes; + const char *c; + int i; + int bs_counter = 0; + /* Backslash we can point to when one is inserted into the string */ + const char backslash = '\\'; + + /* Smartlist of *char */ + smartlist_t *arg_chars; + arg_chars = smartlist_new(); + + /* Quote string if it contains whitespace or is empty */ + need_quotes = (strchr(arg, ' ') || strchr(arg, '\t') || '\0' == arg[0]); + + /* Build up smartlist of *chars */ + for (c=arg; *c != '\0'; c++) { + if ('"' == *c) { + /* Double up backslashes preceding a quote */ + for (i=0; i<(bs_counter*2); i++) + smartlist_add(arg_chars, (void*)&backslash); + bs_counter = 0; + /* Escape the quote */ + smartlist_add(arg_chars, (void*)&backslash); + smartlist_add(arg_chars, (void*)c); + } else if ('\\' == *c) { + /* Count backslashes until we know whether to double up */ + bs_counter++; + } else { + /* Don't double up slashes preceding a non-quote */ + for (i=0; i<bs_counter; i++) + smartlist_add(arg_chars, (void*)&backslash); + bs_counter = 0; + smartlist_add(arg_chars, (void*)c); + } + } + /* Don't double up trailing backslashes */ + for (i=0; i<bs_counter; i++) + smartlist_add(arg_chars, (void*)&backslash); + + /* Allocate space for argument, quotes (if needed), and terminator */ + const size_t formatted_arg_len = smartlist_len(arg_chars) + + (need_quotes ? 2 : 0) + 1; + formatted_arg = tor_malloc_zero(formatted_arg_len); + + /* Add leading quote */ + i=0; + if (need_quotes) + formatted_arg[i++] = '"'; + + /* Add characters */ + SMARTLIST_FOREACH(arg_chars, char*, ch, + { + formatted_arg[i++] = *ch; + }); + + /* Add trailing quote */ + if (need_quotes) + formatted_arg[i++] = '"'; + formatted_arg[i] = '\0'; + + smartlist_free(arg_chars); + return formatted_arg; +} + +/** Format a command line for use on Windows, which takes the command as a + * string rather than string array. Follows the rules from "Parsing C++ + * Command-Line Arguments" in MSDN. Algorithm based on list2cmdline in the + * Python subprocess module. Returns a newly allocated string */ +STATIC char * +tor_join_win_cmdline(const char *argv[]) +{ + smartlist_t *argv_list; + char *joined_argv; + int i; + + /* Format each argument and put the result in a smartlist */ + argv_list = smartlist_new(); + for (i=0; argv[i] != NULL; i++) { + smartlist_add(argv_list, (void *)format_win_cmdline_argument(argv[i])); + } + + /* Join the arguments with whitespace */ + joined_argv = smartlist_join_strings(argv_list, " ", 0, NULL); + + /* Free the newly allocated arguments, and the smartlist */ + SMARTLIST_FOREACH(argv_list, char *, arg, + { + tor_free(arg); + }); + smartlist_free(argv_list); + + return joined_argv; +} + +#endif /* ! defined(_WIN32). */ diff --git a/src/lib/process/process_win32.h b/src/lib/process/process_win32.h new file mode 100644 index 0000000000..8ab4880fbd --- /dev/null +++ b/src/lib/process/process_win32.h @@ -0,0 +1,97 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process_win32.h + * \brief Header for process_win32.c + **/ + +#ifndef TOR_PROCESS_WIN32_H +#define TOR_PROCESS_WIN32_H + +#ifdef _WIN32 + +#include "orconfig.h" +#include "lib/malloc/malloc.h" +#include "lib/evloop/compat_libevent.h" + +#include <windows.h> + +struct process_t; + +struct process_win32_t; +typedef struct process_win32_t process_win32_t; + +process_win32_t *process_win32_new(void); +void process_win32_free_(process_win32_t *win32_process); +#define process_win32_free(s) \ + FREE_AND_NULL(process_win32_t, process_win32_free_, (s)) + +void process_win32_init(void); +void process_win32_deinit(void); + +process_status_t process_win32_exec(struct process_t *process); +bool process_win32_terminate(struct process_t *process); + +process_pid_t process_win32_get_pid(struct process_t *process); + +int process_win32_write(struct process_t *process, buf_t *buffer); +int process_win32_read_stdout(struct process_t *process, buf_t *buffer); +int process_win32_read_stderr(struct process_t *process, buf_t *buffer); + +void process_win32_trigger_completion_callbacks(void); + +/* Timer handling. */ +void process_win32_timer_start(void); +void process_win32_timer_stop(void); +bool process_win32_timer_running(void); + +#ifdef PROCESS_WIN32_PRIVATE +STATIC void process_win32_timer_callback(periodic_timer_t *, void *); +STATIC bool process_win32_timer_test_process(process_t *); + +/* I/O pipe handling. */ +struct process_win32_handle_t; +typedef struct process_win32_handle_t process_win32_handle_t; + +typedef enum process_win32_pipe_type_t { + /** This pipe is used for reading. */ + PROCESS_WIN32_PIPE_TYPE_READER, + + /** This pipe is used for writing. */ + PROCESS_WIN32_PIPE_TYPE_WRITER +} process_win32_pipe_type_t; + +STATIC bool process_win32_create_pipe(HANDLE *, + HANDLE *, + SECURITY_ATTRIBUTES *, + process_win32_pipe_type_t); + +STATIC void process_win32_cleanup_handle(process_win32_handle_t *handle); + +STATIC VOID WINAPI process_win32_stdout_read_done(DWORD, + DWORD, + LPOVERLAPPED); +STATIC VOID WINAPI process_win32_stderr_read_done(DWORD, + DWORD, + LPOVERLAPPED); +STATIC VOID WINAPI process_win32_stdin_write_done(DWORD, + DWORD, + LPOVERLAPPED); + +STATIC int process_win32_read_from_handle(process_win32_handle_t *, + buf_t *, + LPOVERLAPPED_COMPLETION_ROUTINE); +STATIC bool process_win32_handle_read_completion(process_win32_handle_t *, + DWORD, + DWORD); + +STATIC char *format_win_cmdline_argument(const char *arg); +STATIC char *tor_join_win_cmdline(const char *argv[]); +#endif /* defined(PROCESS_WIN32_PRIVATE). */ + +#endif /* ! defined(_WIN32). */ + +#endif /* defined(TOR_PROCESS_WIN32_H). */ diff --git a/src/lib/process/subprocess.c b/src/lib/process/subprocess.c deleted file mode 100644 index 70851c15e0..0000000000 --- a/src/lib/process/subprocess.c +++ /dev/null @@ -1,1236 +0,0 @@ -/* Copyright (c) 2003, Roger Dingledine - * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ -/* See LICENSE for licensing information */ - -/** - * \file subprocess.c - * \brief Launch and monitor other processes. - **/ - -#define SUBPROCESS_PRIVATE -#include "lib/process/subprocess.h" - -#include "lib/container/smartlist.h" -#include "lib/err/torerr.h" -#include "lib/log/log.h" -#include "lib/log/util_bug.h" -#include "lib/log/win32err.h" -#include "lib/malloc/malloc.h" -#include "lib/process/env.h" -#include "lib/process/waitpid.h" -#include "lib/string/compat_ctype.h" - -#ifdef HAVE_SYS_TYPES_H -#include <sys/types.h> -#endif -#ifdef HAVE_SYS_PRCTL_H -#include <sys/prctl.h> -#endif -#ifdef HAVE_UNISTD_H -#include <unistd.h> -#endif -#ifdef HAVE_SIGNAL_H -#include <signal.h> -#endif -#ifdef HAVE_FCNTL_H -#include <fcntl.h> -#endif -#ifdef HAVE_SYS_WAIT_H -#include <sys/wait.h> -#endif -#include <errno.h> -#include <string.h> - -/** Format a single argument for being put on a Windows command line. - * Returns a newly allocated string */ -static char * -format_win_cmdline_argument(const char *arg) -{ - char *formatted_arg; - char need_quotes; - const char *c; - int i; - int bs_counter = 0; - /* Backslash we can point to when one is inserted into the string */ - const char backslash = '\\'; - - /* Smartlist of *char */ - smartlist_t *arg_chars; - arg_chars = smartlist_new(); - - /* Quote string if it contains whitespace or is empty */ - need_quotes = (strchr(arg, ' ') || strchr(arg, '\t') || '\0' == arg[0]); - - /* Build up smartlist of *chars */ - for (c=arg; *c != '\0'; c++) { - if ('"' == *c) { - /* Double up backslashes preceding a quote */ - for (i=0; i<(bs_counter*2); i++) - smartlist_add(arg_chars, (void*)&backslash); - bs_counter = 0; - /* Escape the quote */ - smartlist_add(arg_chars, (void*)&backslash); - smartlist_add(arg_chars, (void*)c); - } else if ('\\' == *c) { - /* Count backslashes until we know whether to double up */ - bs_counter++; - } else { - /* Don't double up slashes preceding a non-quote */ - for (i=0; i<bs_counter; i++) - smartlist_add(arg_chars, (void*)&backslash); - bs_counter = 0; - smartlist_add(arg_chars, (void*)c); - } - } - /* Don't double up trailing backslashes */ - for (i=0; i<bs_counter; i++) - smartlist_add(arg_chars, (void*)&backslash); - - /* Allocate space for argument, quotes (if needed), and terminator */ - const size_t formatted_arg_len = smartlist_len(arg_chars) + - (need_quotes ? 2 : 0) + 1; - formatted_arg = tor_malloc_zero(formatted_arg_len); - - /* Add leading quote */ - i=0; - if (need_quotes) - formatted_arg[i++] = '"'; - - /* Add characters */ - SMARTLIST_FOREACH(arg_chars, char*, ch, - { - formatted_arg[i++] = *ch; - }); - - /* Add trailing quote */ - if (need_quotes) - formatted_arg[i++] = '"'; - formatted_arg[i] = '\0'; - - smartlist_free(arg_chars); - return formatted_arg; -} - -/** Format a command line for use on Windows, which takes the command as a - * string rather than string array. Follows the rules from "Parsing C++ - * Command-Line Arguments" in MSDN. Algorithm based on list2cmdline in the - * Python subprocess module. Returns a newly allocated string */ -char * -tor_join_win_cmdline(const char *argv[]) -{ - smartlist_t *argv_list; - char *joined_argv; - int i; - - /* Format each argument and put the result in a smartlist */ - argv_list = smartlist_new(); - for (i=0; argv[i] != NULL; i++) { - smartlist_add(argv_list, (void *)format_win_cmdline_argument(argv[i])); - } - - /* Join the arguments with whitespace */ - joined_argv = smartlist_join_strings(argv_list, " ", 0, NULL); - - /* Free the newly allocated arguments, and the smartlist */ - SMARTLIST_FOREACH(argv_list, char *, arg, - { - tor_free(arg); - }); - smartlist_free(argv_list); - - return joined_argv; -} - -#ifndef _WIN32 -/** Format <b>child_state</b> and <b>saved_errno</b> as a hex string placed in - * <b>hex_errno</b>. Called between fork and _exit, so must be signal-handler - * safe. - * - * <b>hex_errno</b> must have at least HEX_ERRNO_SIZE+1 bytes available. - * - * The format of <b>hex_errno</b> is: "CHILD_STATE/ERRNO\n", left-padded - * with spaces. CHILD_STATE indicates where - * in the process of starting the child process did the failure occur (see - * CHILD_STATE_* macros for definition), and SAVED_ERRNO is the value of - * errno when the failure occurred. - * - * On success return the number of characters added to hex_errno, not counting - * the terminating NUL; return -1 on error. - */ -STATIC int -format_helper_exit_status(unsigned char child_state, int saved_errno, - char *hex_errno) -{ - unsigned int unsigned_errno; - int written, left; - char *cur; - size_t i; - int res = -1; - - /* Fill hex_errno with spaces, and a trailing newline (memset may - not be signal handler safe, so we can't use it) */ - for (i = 0; i < (HEX_ERRNO_SIZE - 1); i++) - hex_errno[i] = ' '; - hex_errno[HEX_ERRNO_SIZE - 1] = '\n'; - - /* Convert errno to be unsigned for hex conversion */ - if (saved_errno < 0) { - // Avoid overflow on the cast to unsigned int when result is INT_MIN - // by adding 1 to the signed int negative value, - // then, after it has been negated and cast to unsigned, - // adding the original 1 back (the double-addition is intentional). - // Otherwise, the cast to signed could cause a temporary int - // to equal INT_MAX + 1, which is undefined. - unsigned_errno = ((unsigned int) -(saved_errno + 1)) + 1; - } else { - unsigned_errno = (unsigned int) saved_errno; - } - - /* - * Count how many chars of space we have left, and keep a pointer into the - * current point in the buffer. - */ - left = HEX_ERRNO_SIZE+1; - cur = hex_errno; - - /* Emit child_state */ - written = format_hex_number_sigsafe(child_state, cur, left); - - if (written <= 0) - goto err; - - /* Adjust left and cur */ - left -= written; - cur += written; - if (left <= 0) - goto err; - - /* Now the '/' */ - *cur = '/'; - - /* Adjust left and cur */ - ++cur; - --left; - if (left <= 0) - goto err; - - /* Need minus? */ - if (saved_errno < 0) { - *cur = '-'; - ++cur; - --left; - if (left <= 0) - goto err; - } - - /* Emit unsigned_errno */ - written = format_hex_number_sigsafe(unsigned_errno, cur, left); - - if (written <= 0) - goto err; - - /* Adjust left and cur */ - left -= written; - cur += written; - - /* Check that we have enough space left for a newline and a NUL */ - if (left <= 1) - goto err; - - /* Emit the newline and NUL */ - *cur++ = '\n'; - *cur++ = '\0'; - - res = (int)(cur - hex_errno - 1); - - goto done; - - err: - /* - * In error exit, just write a '\0' in the first char so whatever called - * this at least won't fall off the end. - */ - *hex_errno = '\0'; - - done: - return res; -} -#endif /* !defined(_WIN32) */ - -/* Maximum number of file descriptors, if we cannot get it via sysconf() */ -#define DEFAULT_MAX_FD 256 - -/** Terminate the process of <b>process_handle</b>, if that process has not - * already exited. - * - * Return 0 if we succeeded in terminating the process (or if the process - * already exited), and -1 if we tried to kill the process but failed. - * - * Based on code originally borrowed from Python's os.kill. */ -int -tor_terminate_process(process_handle_t *process_handle) -{ -#ifdef _WIN32 - if (tor_get_exit_code(process_handle, 0, NULL) == PROCESS_EXIT_RUNNING) { - HANDLE handle = process_handle->pid.hProcess; - - if (!TerminateProcess(handle, 0)) - return -1; - else - return 0; - } -#else /* !(defined(_WIN32)) */ - if (process_handle->waitpid_cb) { - /* We haven't got a waitpid yet, so we can just kill off the process. */ - return kill(process_handle->pid, SIGTERM); - } -#endif /* defined(_WIN32) */ - - return 0; /* We didn't need to kill the process, so report success */ -} - -/** Return the Process ID of <b>process_handle</b>. */ -int -tor_process_get_pid(process_handle_t *process_handle) -{ -#ifdef _WIN32 - return (int) process_handle->pid.dwProcessId; -#else - return (int) process_handle->pid; -#endif -} - -#ifdef _WIN32 -HANDLE -tor_process_get_stdout_pipe(process_handle_t *process_handle) -{ - return process_handle->stdout_pipe; -} -#else /* !(defined(_WIN32)) */ -/* DOCDOC tor_process_get_stdout_pipe */ -int -tor_process_get_stdout_pipe(process_handle_t *process_handle) -{ - return process_handle->stdout_pipe; -} -#endif /* defined(_WIN32) */ - -/* DOCDOC process_handle_new */ -static process_handle_t * -process_handle_new(void) -{ - process_handle_t *out = tor_malloc_zero(sizeof(process_handle_t)); - -#ifdef _WIN32 - out->stdin_pipe = INVALID_HANDLE_VALUE; - out->stdout_pipe = INVALID_HANDLE_VALUE; - out->stderr_pipe = INVALID_HANDLE_VALUE; -#else - out->stdin_pipe = -1; - out->stdout_pipe = -1; - out->stderr_pipe = -1; -#endif /* defined(_WIN32) */ - - return out; -} - -#ifndef _WIN32 -/** Invoked when a process that we've launched via tor_spawn_background() has - * been found to have terminated. - */ -static void -process_handle_waitpid_cb(int status, void *arg) -{ - process_handle_t *process_handle = arg; - - process_handle->waitpid_exit_status = status; - clear_waitpid_callback(process_handle->waitpid_cb); - if (process_handle->status == PROCESS_STATUS_RUNNING) - process_handle->status = PROCESS_STATUS_NOTRUNNING; - process_handle->waitpid_cb = 0; -} -#endif /* !defined(_WIN32) */ - -/** - * @name child-process states - * - * Each of these values represents a possible state that a child process can - * be in. They're used to determine what to say when telling the parent how - * far along we were before failure. - * - * @{ - */ -#define CHILD_STATE_INIT 0 -#define CHILD_STATE_PIPE 1 -#define CHILD_STATE_MAXFD 2 -#define CHILD_STATE_FORK 3 -#define CHILD_STATE_DUPOUT 4 -#define CHILD_STATE_DUPERR 5 -#define CHILD_STATE_DUPIN 6 -#define CHILD_STATE_CLOSEFD 7 -#define CHILD_STATE_EXEC 8 -#define CHILD_STATE_FAILEXEC 9 -/** @} */ -/** - * Boolean. If true, then Tor may call execve or CreateProcess via - * tor_spawn_background. - **/ -static int may_spawn_background_process = 1; -/** - * Turn off may_spawn_background_process, so that all future calls to - * tor_spawn_background are guaranteed to fail. - **/ -void -tor_disable_spawning_background_processes(void) -{ - may_spawn_background_process = 0; -} -/** Start a program in the background. If <b>filename</b> contains a '/', then - * it will be treated as an absolute or relative path. Otherwise, on - * non-Windows systems, the system path will be searched for <b>filename</b>. - * On Windows, only the current directory will be searched. Here, to search the - * system path (as well as the application directory, current working - * directory, and system directories), set filename to NULL. - * - * The strings in <b>argv</b> will be passed as the command line arguments of - * the child program (following convention, argv[0] should normally be the - * filename of the executable, and this must be the case if <b>filename</b> is - * NULL). The last element of argv must be NULL. A handle to the child process - * will be returned in process_handle (which must be non-NULL). Read - * process_handle.status to find out if the process was successfully launched. - * For convenience, process_handle.status is returned by this function. - * - * Some parts of this code are based on the POSIX subprocess module from - * Python, and example code from - * http://msdn.microsoft.com/en-us/library/ms682499%28v=vs.85%29.aspx. - */ -int -tor_spawn_background(const char *const filename, const char **argv, - process_environment_t *env, - process_handle_t **process_handle_out) -{ - if (BUG(may_spawn_background_process == 0)) { - /* We should never reach this point if we're forbidden to spawn - * processes. Instead we should have caught the attempt earlier. */ - return PROCESS_STATUS_ERROR; - } - -#ifdef _WIN32 - HANDLE stdout_pipe_read = NULL; - HANDLE stdout_pipe_write = NULL; - HANDLE stderr_pipe_read = NULL; - HANDLE stderr_pipe_write = NULL; - HANDLE stdin_pipe_read = NULL; - HANDLE stdin_pipe_write = NULL; - process_handle_t *process_handle; - int status; - - STARTUPINFOA siStartInfo; - BOOL retval = FALSE; - - SECURITY_ATTRIBUTES saAttr; - char *joined_argv; - - saAttr.nLength = sizeof(SECURITY_ATTRIBUTES); - saAttr.bInheritHandle = TRUE; - /* TODO: should we set explicit security attributes? (#2046, comment 5) */ - saAttr.lpSecurityDescriptor = NULL; - - /* Assume failure to start process */ - status = PROCESS_STATUS_ERROR; - - /* Set up pipe for stdout */ - if (!CreatePipe(&stdout_pipe_read, &stdout_pipe_write, &saAttr, 0)) { - log_warn(LD_GENERAL, - "Failed to create pipe for stdout communication with child process: %s", - format_win32_error(GetLastError())); - return status; - } - if (!SetHandleInformation(stdout_pipe_read, HANDLE_FLAG_INHERIT, 0)) { - log_warn(LD_GENERAL, - "Failed to configure pipe for stdout communication with child " - "process: %s", format_win32_error(GetLastError())); - return status; - } - - /* Set up pipe for stderr */ - if (!CreatePipe(&stderr_pipe_read, &stderr_pipe_write, &saAttr, 0)) { - log_warn(LD_GENERAL, - "Failed to create pipe for stderr communication with child process: %s", - format_win32_error(GetLastError())); - return status; - } - if (!SetHandleInformation(stderr_pipe_read, HANDLE_FLAG_INHERIT, 0)) { - log_warn(LD_GENERAL, - "Failed to configure pipe for stderr communication with child " - "process: %s", format_win32_error(GetLastError())); - return status; - } - - /* Set up pipe for stdin */ - if (!CreatePipe(&stdin_pipe_read, &stdin_pipe_write, &saAttr, 0)) { - log_warn(LD_GENERAL, - "Failed to create pipe for stdin communication with child process: %s", - format_win32_error(GetLastError())); - return status; - } - if (!SetHandleInformation(stdin_pipe_write, HANDLE_FLAG_INHERIT, 0)) { - log_warn(LD_GENERAL, - "Failed to configure pipe for stdin communication with child " - "process: %s", format_win32_error(GetLastError())); - return status; - } - - /* Create the child process */ - - /* Windows expects argv to be a whitespace delimited string, so join argv up - */ - joined_argv = tor_join_win_cmdline(argv); - - process_handle = process_handle_new(); - process_handle->status = status; - - ZeroMemory(&(process_handle->pid), sizeof(PROCESS_INFORMATION)); - ZeroMemory(&siStartInfo, sizeof(STARTUPINFO)); - siStartInfo.cb = sizeof(STARTUPINFO); - siStartInfo.hStdError = stderr_pipe_write; - siStartInfo.hStdOutput = stdout_pipe_write; - siStartInfo.hStdInput = stdin_pipe_read; - siStartInfo.dwFlags |= STARTF_USESTDHANDLES; - - /* Create the child process */ - - retval = CreateProcessA(filename, // module name - joined_argv, // command line - /* TODO: should we set explicit security attributes? (#2046, comment 5) */ - NULL, // process security attributes - NULL, // primary thread security attributes - TRUE, // handles are inherited - /*(TODO: set CREATE_NEW CONSOLE/PROCESS_GROUP to make GetExitCodeProcess() - * work?) */ - CREATE_NO_WINDOW, // creation flags - (env==NULL) ? NULL : env->windows_environment_block, - NULL, // use parent's current directory - &siStartInfo, // STARTUPINFO pointer - &(process_handle->pid)); // receives PROCESS_INFORMATION - - tor_free(joined_argv); - - if (!retval) { - log_warn(LD_GENERAL, - "Failed to create child process %s: %s", filename?filename:argv[0], - format_win32_error(GetLastError())); - tor_free(process_handle); - } else { - /* TODO: Close hProcess and hThread in process_handle->pid? */ - process_handle->stdout_pipe = stdout_pipe_read; - process_handle->stderr_pipe = stderr_pipe_read; - process_handle->stdin_pipe = stdin_pipe_write; - status = process_handle->status = PROCESS_STATUS_RUNNING; - } - - /* TODO: Close pipes on exit */ - *process_handle_out = process_handle; - return status; -#else /* !(defined(_WIN32)) */ - pid_t pid; - int stdout_pipe[2]; - int stderr_pipe[2]; - int stdin_pipe[2]; - int fd, retval; - process_handle_t *process_handle; - int status; - - const char *error_message = SPAWN_ERROR_MESSAGE; - size_t error_message_length; - - /* Represents where in the process of spawning the program is; - this is used for printing out the error message */ - unsigned char child_state = CHILD_STATE_INIT; - - char hex_errno[HEX_ERRNO_SIZE + 2]; /* + 1 should be sufficient actually */ - - static int max_fd = -1; - - status = PROCESS_STATUS_ERROR; - - /* We do the strlen here because strlen() is not signal handler safe, - and we are not allowed to use unsafe functions between fork and exec */ - error_message_length = strlen(error_message); - - // child_state = CHILD_STATE_PIPE; - - /* Set up pipe for redirecting stdout, stderr, and stdin of child */ - retval = pipe(stdout_pipe); - if (-1 == retval) { - log_warn(LD_GENERAL, - "Failed to set up pipe for stdout communication with child process: %s", - strerror(errno)); - return status; - } - - retval = pipe(stderr_pipe); - if (-1 == retval) { - log_warn(LD_GENERAL, - "Failed to set up pipe for stderr communication with child process: %s", - strerror(errno)); - - close(stdout_pipe[0]); - close(stdout_pipe[1]); - - return status; - } - - retval = pipe(stdin_pipe); - if (-1 == retval) { - log_warn(LD_GENERAL, - "Failed to set up pipe for stdin communication with child process: %s", - strerror(errno)); - - close(stdout_pipe[0]); - close(stdout_pipe[1]); - close(stderr_pipe[0]); - close(stderr_pipe[1]); - - return status; - } - - // child_state = CHILD_STATE_MAXFD; - -#ifdef _SC_OPEN_MAX - if (-1 == max_fd) { - max_fd = (int) sysconf(_SC_OPEN_MAX); - if (max_fd == -1) { - max_fd = DEFAULT_MAX_FD; - log_warn(LD_GENERAL, - "Cannot find maximum file descriptor, assuming %d", max_fd); - } - } -#else /* !(defined(_SC_OPEN_MAX)) */ - max_fd = DEFAULT_MAX_FD; -#endif /* defined(_SC_OPEN_MAX) */ - - // child_state = CHILD_STATE_FORK; - - pid = fork(); - if (0 == pid) { - /* In child */ - -#if defined(HAVE_SYS_PRCTL_H) && defined(__linux__) - /* Attempt to have the kernel issue a SIGTERM if the parent - * goes away. Certain attributes of the binary being execve()ed - * will clear this during the execve() call, but it's better - * than nothing. - */ - prctl(PR_SET_PDEATHSIG, SIGTERM); -#endif /* defined(HAVE_SYS_PRCTL_H) && defined(__linux__) */ - - child_state = CHILD_STATE_DUPOUT; - - /* Link child stdout to the write end of the pipe */ - retval = dup2(stdout_pipe[1], STDOUT_FILENO); - if (-1 == retval) - goto error; - - child_state = CHILD_STATE_DUPERR; - - /* Link child stderr to the write end of the pipe */ - retval = dup2(stderr_pipe[1], STDERR_FILENO); - if (-1 == retval) - goto error; - - child_state = CHILD_STATE_DUPIN; - - /* Link child stdin to the read end of the pipe */ - retval = dup2(stdin_pipe[0], STDIN_FILENO); - if (-1 == retval) - goto error; - - // child_state = CHILD_STATE_CLOSEFD; - - close(stderr_pipe[0]); - close(stderr_pipe[1]); - close(stdout_pipe[0]); - close(stdout_pipe[1]); - close(stdin_pipe[0]); - close(stdin_pipe[1]); - - /* Close all other fds, including the read end of the pipe */ - /* XXX: We should now be doing enough FD_CLOEXEC setting to make - * this needless. */ - for (fd = STDERR_FILENO + 1; fd < max_fd; fd++) { - close(fd); - } - - // child_state = CHILD_STATE_EXEC; - - /* Call the requested program. We need the cast because - execvp doesn't define argv as const, even though it - does not modify the arguments */ - if (env) - execve(filename, (char *const *) argv, env->unixoid_environment_block); - else { - static char *new_env[] = { NULL }; - execve(filename, (char *const *) argv, new_env); - } - - /* If we got here, the exec or open(/dev/null) failed */ - - child_state = CHILD_STATE_FAILEXEC; - - error: - { - /* XXX: are we leaking fds from the pipe? */ - int n, err=0; - ssize_t nbytes; - - n = format_helper_exit_status(child_state, errno, hex_errno); - - if (n >= 0) { - /* Write the error message. GCC requires that we check the return - value, but there is nothing we can do if it fails */ - /* TODO: Don't use STDOUT, use a pipe set up just for this purpose */ - nbytes = write(STDOUT_FILENO, error_message, error_message_length); - err = (nbytes < 0); - nbytes = write(STDOUT_FILENO, hex_errno, n); - err += (nbytes < 0); - } - - _exit(err?254:255); // exit ok: in child. - } - - /* Never reached, but avoids compiler warning */ - return status; // LCOV_EXCL_LINE - } - - /* In parent */ - - if (-1 == pid) { - log_warn(LD_GENERAL, "Failed to fork child process: %s", strerror(errno)); - close(stdin_pipe[0]); - close(stdin_pipe[1]); - close(stdout_pipe[0]); - close(stdout_pipe[1]); - close(stderr_pipe[0]); - close(stderr_pipe[1]); - return status; - } - - process_handle = process_handle_new(); - process_handle->status = status; - process_handle->pid = pid; - - /* TODO: If the child process forked but failed to exec, waitpid it */ - - /* Return read end of the pipes to caller, and close write end */ - process_handle->stdout_pipe = stdout_pipe[0]; - retval = close(stdout_pipe[1]); - - if (-1 == retval) { - log_warn(LD_GENERAL, - "Failed to close write end of stdout pipe in parent process: %s", - strerror(errno)); - } - - process_handle->waitpid_cb = set_waitpid_callback(pid, - process_handle_waitpid_cb, - process_handle); - - process_handle->stderr_pipe = stderr_pipe[0]; - retval = close(stderr_pipe[1]); - - if (-1 == retval) { - log_warn(LD_GENERAL, - "Failed to close write end of stderr pipe in parent process: %s", - strerror(errno)); - } - - /* Return write end of the stdin pipe to caller, and close the read end */ - process_handle->stdin_pipe = stdin_pipe[1]; - retval = close(stdin_pipe[0]); - - if (-1 == retval) { - log_warn(LD_GENERAL, - "Failed to close read end of stdin pipe in parent process: %s", - strerror(errno)); - } - - status = process_handle->status = PROCESS_STATUS_RUNNING; - /* Set stdin/stdout/stderr pipes to be non-blocking */ - if (fcntl(process_handle->stdout_pipe, F_SETFL, O_NONBLOCK) < 0 || - fcntl(process_handle->stderr_pipe, F_SETFL, O_NONBLOCK) < 0 || - fcntl(process_handle->stdin_pipe, F_SETFL, O_NONBLOCK) < 0) { - log_warn(LD_GENERAL, "Failed to set stderror/stdout/stdin pipes " - "nonblocking in parent process: %s", strerror(errno)); - } - - *process_handle_out = process_handle; - return status; -#endif /* defined(_WIN32) */ -} - -/** Destroy all resources allocated by the process handle in - * <b>process_handle</b>. - * If <b>also_terminate_process</b> is true, also terminate the - * process of the process handle. */ -MOCK_IMPL(void, -tor_process_handle_destroy,(process_handle_t *process_handle, - int also_terminate_process)) -{ - if (!process_handle) - return; - - if (also_terminate_process) { - if (tor_terminate_process(process_handle) < 0) { - const char *errstr = -#ifdef _WIN32 - format_win32_error(GetLastError()); -#else - strerror(errno); -#endif - log_notice(LD_GENERAL, "Failed to terminate process with " - "PID '%d' ('%s').", tor_process_get_pid(process_handle), - errstr); - } else { - log_info(LD_GENERAL, "Terminated process with PID '%d'.", - tor_process_get_pid(process_handle)); - } - } - - process_handle->status = PROCESS_STATUS_NOTRUNNING; - -#ifdef _WIN32 - if (process_handle->stdout_pipe) - CloseHandle(process_handle->stdout_pipe); - - if (process_handle->stderr_pipe) - CloseHandle(process_handle->stderr_pipe); - - if (process_handle->stdin_pipe) - CloseHandle(process_handle->stdin_pipe); -#else /* !(defined(_WIN32)) */ - close(process_handle->stdout_pipe); - close(process_handle->stderr_pipe); - close(process_handle->stdin_pipe); - - clear_waitpid_callback(process_handle->waitpid_cb); -#endif /* defined(_WIN32) */ - - memset(process_handle, 0x0f, sizeof(process_handle_t)); - tor_free(process_handle); -} - -/** Get the exit code of a process specified by <b>process_handle</b> and store - * it in <b>exit_code</b>, if set to a non-NULL value. If <b>block</b> is set - * to true, the call will block until the process has exited. Otherwise if - * the process is still running, the function will return - * PROCESS_EXIT_RUNNING, and exit_code will be left unchanged. Returns - * PROCESS_EXIT_EXITED if the process did exit. If there is a failure, - * PROCESS_EXIT_ERROR will be returned and the contents of exit_code (if - * non-NULL) will be undefined. N.B. Under *nix operating systems, this will - * probably not work in Tor, because waitpid() is called in main.c to reap any - * terminated child processes.*/ -int -tor_get_exit_code(process_handle_t *process_handle, - int block, int *exit_code) -{ -#ifdef _WIN32 - DWORD retval; - BOOL success; - - if (block) { - /* Wait for the process to exit */ - retval = WaitForSingleObject(process_handle->pid.hProcess, INFINITE); - if (retval != WAIT_OBJECT_0) { - log_warn(LD_GENERAL, "WaitForSingleObject() failed (%d): %s", - (int)retval, format_win32_error(GetLastError())); - return PROCESS_EXIT_ERROR; - } - } else { - retval = WaitForSingleObject(process_handle->pid.hProcess, 0); - if (WAIT_TIMEOUT == retval) { - /* Process has not exited */ - return PROCESS_EXIT_RUNNING; - } else if (retval != WAIT_OBJECT_0) { - log_warn(LD_GENERAL, "WaitForSingleObject() failed (%d): %s", - (int)retval, format_win32_error(GetLastError())); - return PROCESS_EXIT_ERROR; - } - } - - if (exit_code != NULL) { - success = GetExitCodeProcess(process_handle->pid.hProcess, - (PDWORD)exit_code); - if (!success) { - log_warn(LD_GENERAL, "GetExitCodeProcess() failed: %s", - format_win32_error(GetLastError())); - return PROCESS_EXIT_ERROR; - } - } -#else /* !(defined(_WIN32)) */ - int stat_loc; - int retval; - - if (process_handle->waitpid_cb) { - /* We haven't processed a SIGCHLD yet. */ - retval = waitpid(process_handle->pid, &stat_loc, block?0:WNOHANG); - if (retval == process_handle->pid) { - clear_waitpid_callback(process_handle->waitpid_cb); - process_handle->waitpid_cb = NULL; - process_handle->waitpid_exit_status = stat_loc; - } - } else { - /* We already got a SIGCHLD for this process, and handled it. */ - retval = process_handle->pid; - stat_loc = process_handle->waitpid_exit_status; - } - - if (!block && 0 == retval) { - /* Process has not exited */ - return PROCESS_EXIT_RUNNING; - } else if (retval != process_handle->pid) { - log_warn(LD_GENERAL, "waitpid() failed for PID %d: %s", - (int)process_handle->pid, strerror(errno)); - return PROCESS_EXIT_ERROR; - } - - if (!WIFEXITED(stat_loc)) { - log_warn(LD_GENERAL, "Process %d did not exit normally", - (int)process_handle->pid); - return PROCESS_EXIT_ERROR; - } - - if (exit_code != NULL) - *exit_code = WEXITSTATUS(stat_loc); -#endif /* defined(_WIN32) */ - - return PROCESS_EXIT_EXITED; -} - -#ifdef _WIN32 -/** Read from a handle <b>h</b> into <b>buf</b>, up to <b>count</b> bytes. If - * <b>hProcess</b> is NULL, the function will return immediately if there is - * nothing more to read. Otherwise <b>hProcess</b> should be set to the handle - * to the process owning the <b>h</b>. In this case, the function will exit - * only once the process has exited, or <b>count</b> bytes are read. Returns - * the number of bytes read, or -1 on error. */ -ssize_t -tor_read_all_handle(HANDLE h, char *buf, size_t count, - const process_handle_t *process) -{ - size_t numread = 0; - BOOL retval; - DWORD byte_count; - BOOL process_exited = FALSE; - - if (count > SIZE_T_CEILING || count > SSIZE_MAX) - return -1; - - while (numread < count) { - /* Check if there is anything to read */ - retval = PeekNamedPipe(h, NULL, 0, NULL, &byte_count, NULL); - if (!retval) { - log_warn(LD_GENERAL, - "Failed to peek from handle: %s", - format_win32_error(GetLastError())); - return -1; - } else if (0 == byte_count) { - /* Nothing available: process exited or it is busy */ - - /* Exit if we don't know whether the process is running */ - if (NULL == process) - break; - - /* The process exited and there's nothing left to read from it */ - if (process_exited) - break; - - /* If process is not running, check for output one more time in case - it wrote something after the peek was performed. Otherwise keep on - waiting for output */ - tor_assert(process != NULL); - byte_count = WaitForSingleObject(process->pid.hProcess, 0); - if (WAIT_TIMEOUT != byte_count) - process_exited = TRUE; - - continue; - } - - /* There is data to read; read it */ - retval = ReadFile(h, buf+numread, count-numread, &byte_count, NULL); - tor_assert(byte_count + numread <= count); - if (!retval) { - log_warn(LD_GENERAL, "Failed to read from handle: %s", - format_win32_error(GetLastError())); - return -1; - } else if (0 == byte_count) { - /* End of file */ - break; - } - numread += byte_count; - } - return (ssize_t)numread; -} -#else /* !(defined(_WIN32)) */ -/** Read from a handle <b>fd</b> into <b>buf</b>, up to <b>count</b> bytes. If - * <b>process</b> is NULL, the function will return immediately if there is - * nothing more to read. Otherwise data will be read until end of file, or - * <b>count</b> bytes are read. Returns the number of bytes read, or -1 on - * error. Sets <b>eof</b> to true if <b>eof</b> is not NULL and the end of the - * file has been reached. */ -ssize_t -tor_read_all_handle(int fd, char *buf, size_t count, - const process_handle_t *process, - int *eof) -{ - size_t numread = 0; - ssize_t result; - - if (eof) - *eof = 0; - - if (count > SIZE_T_CEILING || count > SSIZE_MAX) - return -1; - - while (numread < count) { - result = read(fd, buf+numread, count-numread); - - if (result == 0) { - log_debug(LD_GENERAL, "read() reached end of file"); - if (eof) - *eof = 1; - break; - } else if (result < 0 && errno == EAGAIN) { - if (process) - continue; - else - break; - } else if (result < 0) { - log_warn(LD_GENERAL, "read() failed: %s", strerror(errno)); - return -1; - } - - numread += result; - } - - log_debug(LD_GENERAL, "read() read %d bytes from handle", (int)numread); - return (ssize_t)numread; -} -#endif /* defined(_WIN32) */ - -/** Read from stdout of a process until the process exits. */ -ssize_t -tor_read_all_from_process_stdout(const process_handle_t *process_handle, - char *buf, size_t count) -{ -#ifdef _WIN32 - return tor_read_all_handle(process_handle->stdout_pipe, buf, count, - process_handle); -#else - return tor_read_all_handle(process_handle->stdout_pipe, buf, count, - process_handle, NULL); -#endif /* defined(_WIN32) */ -} - -/** Read from stdout of a process until the process exits. */ -ssize_t -tor_read_all_from_process_stderr(const process_handle_t *process_handle, - char *buf, size_t count) -{ -#ifdef _WIN32 - return tor_read_all_handle(process_handle->stderr_pipe, buf, count, - process_handle); -#else - return tor_read_all_handle(process_handle->stderr_pipe, buf, count, - process_handle, NULL); -#endif /* defined(_WIN32) */ -} - -/** Return a string corresponding to <b>stream_status</b>. */ -const char * -stream_status_to_string(enum stream_status stream_status) -{ - switch (stream_status) { - case IO_STREAM_OKAY: - return "okay"; - case IO_STREAM_EAGAIN: - return "temporarily unavailable"; - case IO_STREAM_TERM: - return "terminated"; - case IO_STREAM_CLOSED: - return "closed"; - default: - tor_fragile_assert(); - return "unknown"; - } -} - -/** Split buf into lines, and add to smartlist. The buffer <b>buf</b> will be - * modified. The resulting smartlist will consist of pointers to buf, so there - * is no need to free the contents of sl. <b>buf</b> must be a NUL-terminated - * string. <b>len</b> should be set to the length of the buffer excluding the - * NUL. Non-printable characters (including NUL) will be replaced with "." */ -int -tor_split_lines(smartlist_t *sl, char *buf, int len) -{ - /* Index in buf of the start of the current line */ - int start = 0; - /* Index in buf of the current character being processed */ - int cur = 0; - /* Are we currently in a line */ - char in_line = 0; - - /* Loop over string */ - while (cur < len) { - /* Loop until end of line or end of string */ - for (; cur < len; cur++) { - if (in_line) { - if ('\r' == buf[cur] || '\n' == buf[cur]) { - /* End of line */ - buf[cur] = '\0'; - /* Point cur to the next line */ - cur++; - /* Line starts at start and ends with a nul */ - break; - } else { - if (!TOR_ISPRINT(buf[cur])) - buf[cur] = '.'; - } - } else { - if ('\r' == buf[cur] || '\n' == buf[cur]) { - /* Skip leading vertical space */ - ; - } else { - in_line = 1; - start = cur; - if (!TOR_ISPRINT(buf[cur])) - buf[cur] = '.'; - } - } - } - /* We are at the end of the line or end of string. If in_line is true there - * is a line which starts at buf+start and ends at a NUL. cur points to - * the character after the NUL. */ - if (in_line) - smartlist_add(sl, (void *)(buf+start)); - in_line = 0; - } - return smartlist_len(sl); -} - -#ifdef _WIN32 - -/** Return a smartlist containing lines outputted from - * <b>handle</b>. Return NULL on error, and set - * <b>stream_status_out</b> appropriately. */ -MOCK_IMPL(smartlist_t *, -tor_get_lines_from_handle, (HANDLE *handle, - enum stream_status *stream_status_out)) -{ - int pos; - char stdout_buf[600] = {0}; - smartlist_t *lines = NULL; - - tor_assert(stream_status_out); - - *stream_status_out = IO_STREAM_TERM; - - pos = tor_read_all_handle(handle, stdout_buf, sizeof(stdout_buf) - 1, NULL); - if (pos < 0) { - *stream_status_out = IO_STREAM_TERM; - return NULL; - } - if (pos == 0) { - *stream_status_out = IO_STREAM_EAGAIN; - return NULL; - } - - /* End with a null even if there isn't a \r\n at the end */ - /* TODO: What if this is a partial line? */ - stdout_buf[pos] = '\0'; - - /* Split up the buffer */ - lines = smartlist_new(); - tor_split_lines(lines, stdout_buf, pos); - - /* Currently 'lines' is populated with strings residing on the - stack. Replace them with their exact copies on the heap: */ - SMARTLIST_FOREACH(lines, char *, line, - SMARTLIST_REPLACE_CURRENT(lines, line, tor_strdup(line))); - - *stream_status_out = IO_STREAM_OKAY; - - return lines; -} - -#else /* !(defined(_WIN32)) */ - -/** Return a smartlist containing lines outputted from - * <b>fd</b>. Return NULL on error, and set - * <b>stream_status_out</b> appropriately. */ -MOCK_IMPL(smartlist_t *, -tor_get_lines_from_handle, (int fd, enum stream_status *stream_status_out)) -{ - enum stream_status stream_status; - char stdout_buf[400]; - smartlist_t *lines = NULL; - - while (1) { - memset(stdout_buf, 0, sizeof(stdout_buf)); - - stream_status = get_string_from_pipe(fd, - stdout_buf, sizeof(stdout_buf) - 1); - if (stream_status != IO_STREAM_OKAY) - goto done; - - if (!lines) lines = smartlist_new(); - smartlist_split_string(lines, stdout_buf, "\n", 0, 0); - } - - done: - *stream_status_out = stream_status; - return lines; -} - -#endif /* defined(_WIN32) */ - -/** Reads from <b>fd</b> and stores input in <b>buf_out</b> making - * sure it's below <b>count</b> bytes. - * If the string has a trailing newline, we strip it off. - * - * This function is specifically created to handle input from managed - * proxies, according to the pluggable transports spec. Make sure it - * fits your needs before using it. - * - * Returns: - * IO_STREAM_CLOSED: If the stream is closed. - * IO_STREAM_EAGAIN: If there is nothing to read and we should check back - * later. - * IO_STREAM_TERM: If something is wrong with the stream. - * IO_STREAM_OKAY: If everything went okay and we got a string - * in <b>buf_out</b>. */ -enum stream_status -get_string_from_pipe(int fd, char *buf_out, size_t count) -{ - ssize_t ret; - - tor_assert(count <= INT_MAX); - - ret = read(fd, buf_out, count); - - if (ret == 0) - return IO_STREAM_CLOSED; - else if (ret < 0 && errno == EAGAIN) - return IO_STREAM_EAGAIN; - else if (ret < 0) - return IO_STREAM_TERM; - - if (buf_out[ret - 1] == '\n') { - /* Remove the trailing newline */ - buf_out[ret - 1] = '\0'; - } else - buf_out[ret] = '\0'; - - return IO_STREAM_OKAY; -} diff --git a/src/lib/process/subprocess.h b/src/lib/process/subprocess.h deleted file mode 100644 index 5b4318ef2b..0000000000 --- a/src/lib/process/subprocess.h +++ /dev/null @@ -1,134 +0,0 @@ -/* Copyright (c) 2003-2004, Roger Dingledine - * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ -/* See LICENSE for licensing information */ - -/** - * \file subprocess.h - * \brief Header for subprocess.c - **/ - -#ifndef TOR_SUBPROCESS_H -#define TOR_SUBPROCESS_H - -#include "lib/cc/torint.h" -#include "lib/testsupport/testsupport.h" -#include <stddef.h> -#ifdef _WIN32 -#include <windows.h> -#endif - -struct smartlist_t; - -void tor_disable_spawning_background_processes(void); - -typedef struct process_handle_t process_handle_t; -struct process_environment_t; -int tor_spawn_background(const char *const filename, const char **argv, - struct process_environment_t *env, - process_handle_t **process_handle_out); - -#define SPAWN_ERROR_MESSAGE "ERR: Failed to spawn background process - code " - -/** Status of an I/O stream. */ -enum stream_status { - IO_STREAM_OKAY, - IO_STREAM_EAGAIN, - IO_STREAM_TERM, - IO_STREAM_CLOSED -}; - -const char *stream_status_to_string(enum stream_status stream_status); - -enum stream_status get_string_from_pipe(int fd, char *buf, size_t count); - -/* Values of process_handle_t.status. */ -#define PROCESS_STATUS_NOTRUNNING 0 -#define PROCESS_STATUS_RUNNING 1 -#define PROCESS_STATUS_ERROR -1 - -#ifdef SUBPROCESS_PRIVATE -struct waitpid_callback_t; - -/** Structure to represent the state of a process with which Tor is - * communicating. The contents of this structure are private to util.c */ -struct process_handle_t { - /** One of the PROCESS_STATUS_* values */ - int status; -#ifdef _WIN32 - HANDLE stdin_pipe; - HANDLE stdout_pipe; - HANDLE stderr_pipe; - PROCESS_INFORMATION pid; -#else /* !(defined(_WIN32)) */ - int stdin_pipe; - int stdout_pipe; - int stderr_pipe; - pid_t pid; - /** If the process has not given us a SIGCHLD yet, this has the - * waitpid_callback_t that gets invoked once it has. Otherwise this - * contains NULL. */ - struct waitpid_callback_t *waitpid_cb; - /** The exit status reported by waitpid. */ - int waitpid_exit_status; -#endif /* defined(_WIN32) */ -}; -#endif /* defined(SUBPROCESS_PRIVATE) */ - -/* Return values of tor_get_exit_code() */ -#define PROCESS_EXIT_RUNNING 1 -#define PROCESS_EXIT_EXITED 0 -#define PROCESS_EXIT_ERROR -1 -int tor_get_exit_code(process_handle_t *process_handle, - int block, int *exit_code); -int tor_split_lines(struct smartlist_t *sl, char *buf, int len); -#ifdef _WIN32 -ssize_t tor_read_all_handle(HANDLE h, char *buf, size_t count, - const process_handle_t *process); -#else -ssize_t tor_read_all_handle(int fd, char *buf, size_t count, - const process_handle_t *process, - int *eof); -#endif /* defined(_WIN32) */ -ssize_t tor_read_all_from_process_stdout( - const process_handle_t *process_handle, char *buf, size_t count); -ssize_t tor_read_all_from_process_stderr( - const process_handle_t *process_handle, char *buf, size_t count); -char *tor_join_win_cmdline(const char *argv[]); - -int tor_process_get_pid(process_handle_t *process_handle); -#ifdef _WIN32 -HANDLE tor_process_get_stdout_pipe(process_handle_t *process_handle); -#else -int tor_process_get_stdout_pipe(process_handle_t *process_handle); -#endif - -#ifdef _WIN32 -MOCK_DECL(struct smartlist_t *, tor_get_lines_from_handle,(HANDLE *handle, - enum stream_status *stream_status)); -#else -MOCK_DECL(struct smartlist_t *, tor_get_lines_from_handle,(int fd, - enum stream_status *stream_status)); -#endif /* defined(_WIN32) */ - -int tor_terminate_process(process_handle_t *process_handle); - -MOCK_DECL(void, tor_process_handle_destroy,(process_handle_t *process_handle, - int also_terminate_process)); - -#ifdef SUBPROCESS_PRIVATE -/* Prototypes for private functions only used by util.c (and unit tests) */ - -#ifndef _WIN32 -STATIC int format_helper_exit_status(unsigned char child_state, - int saved_errno, char *hex_errno); - -/* Space for hex values of child state, a slash, saved_errno (with - leading minus) and newline (no null) */ -#define HEX_ERRNO_SIZE (sizeof(char) * 2 + 1 + \ - 1 + sizeof(int) * 2 + 1) -#endif /* !defined(_WIN32) */ - -#endif /* defined(SUBPROCESS_PRIVATE) */ - -#endif diff --git a/src/lib/process/waitpid.c b/src/lib/process/waitpid.c index 32ba4530da..46d30bf50e 100644 --- a/src/lib/process/waitpid.c +++ b/src/lib/process/waitpid.c @@ -16,7 +16,7 @@ #include "lib/log/log.h" #include "lib/log/util_bug.h" #include "lib/malloc/malloc.h" -#include "ht.h" +#include "ext/ht.h" #ifdef HAVE_SYS_WAIT_H #include <sys/wait.h> diff --git a/src/lib/process/winprocess_sys.c b/src/lib/process/winprocess_sys.c new file mode 100644 index 0000000000..ef66f8bfb1 --- /dev/null +++ b/src/lib/process/winprocess_sys.c @@ -0,0 +1,64 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file winprocess_sys.c + * \brief Subsystem object for windows process setup. + **/ + +#include "orconfig.h" +#include "lib/subsys/subsys.h" +#include "lib/process/winprocess_sys.h" + +#include <stdbool.h> +#include <stddef.h> + +#ifdef _WIN32 +#include <windows.h> + +#define WINPROCESS_SYS_ENABLED true + +static int +subsys_winprocess_initialize(void) +{ +#ifndef HeapEnableTerminationOnCorruption +#define HeapEnableTerminationOnCorruption 1 +#endif + + /* On heap corruption, just give up; don't try to play along. */ + HeapSetInformation(NULL, HeapEnableTerminationOnCorruption, NULL, 0); + + /* SetProcessDEPPolicy is only supported on 32-bit Windows. + * (On 64-bit Windows it always fails, and some compilers don't like the + * PSETDEP cast.) + * 32-bit Windows defines _WIN32. + * 64-bit Windows defines _WIN32 and _WIN64. */ +#ifndef _WIN64 + /* Call SetProcessDEPPolicy to permanently enable DEP. + The function will not resolve on earlier versions of Windows, + and failure is not dangerous. */ + HMODULE hMod = GetModuleHandleA("Kernel32.dll"); + if (hMod) { + typedef BOOL (WINAPI *PSETDEP)(DWORD); + PSETDEP setdeppolicy = (PSETDEP)GetProcAddress(hMod, + "SetProcessDEPPolicy"); + if (setdeppolicy) { + /* PROCESS_DEP_ENABLE | PROCESS_DEP_DISABLE_ATL_THUNK_EMULATION */ + setdeppolicy(3); + } + } +#endif /* !defined(_WIN64) */ + + return 0; +} +#else /* !defined(_WIN32) */ +#define WINPROCESS_SYS_ENABLED false +#define subsys_winprocess_initialize NULL +#endif /* defined(_WIN32) */ + +const subsys_fns_t sys_winprocess = { + .name = "winprocess", + .level = -100, + .supported = WINPROCESS_SYS_ENABLED, + .initialize = subsys_winprocess_initialize, +}; diff --git a/src/lib/process/winprocess_sys.h b/src/lib/process/winprocess_sys.h new file mode 100644 index 0000000000..cb096e0c92 --- /dev/null +++ b/src/lib/process/winprocess_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file winprocess_sys.h + * \brief Declare subsystem object for winprocess.c + **/ + +#ifndef TOR_WINPROCESS_SYS_H +#define TOR_WINPROCESS_SYS_H + +extern const struct subsys_fns_t sys_winprocess; + +#endif /* !defined(TOR_WINPROCESS_SYS_H) */ diff --git a/src/lib/sandbox/.may_include b/src/lib/sandbox/.may_include index 84906dfb3d..853dae7880 100644 --- a/src/lib/sandbox/.may_include +++ b/src/lib/sandbox/.may_include @@ -5,11 +5,10 @@ lib/container/*.h lib/err/*.h lib/log/*.h lib/malloc/*.h -lib/net/*.h lib/sandbox/*.h lib/sandbox/*.inc lib/string/*.h -ht.h -siphash.h -tor_queue.h +ext/ht.h +ext/siphash.h +ext/tor_queue.h diff --git a/src/lib/sandbox/sandbox.c b/src/lib/sandbox/sandbox.c index 6f074bb4e1..ea738b273e 100644 --- a/src/lib/sandbox/sandbox.c +++ b/src/lib/sandbox/sandbox.c @@ -38,13 +38,12 @@ #include "lib/err/torerr.h" #include "lib/log/log.h" #include "lib/cc/torint.h" -#include "lib/net/resolve.h" #include "lib/malloc/malloc.h" #include "lib/string/scanf.h" -#include "tor_queue.h" -#include "ht.h" -#include "siphash.h" +#include "ext/tor_queue.h" +#include "ext/ht.h" +#include "ext/siphash.h" #define DEBUGGING_CLOSE @@ -1553,7 +1552,6 @@ install_syscall_filter(sandbox_cfg_t* cfg) // marking the sandbox as active sandbox_active = 1; - tor_make_getaddrinfo_cache_active(); end: seccomp_release(ctx); @@ -1800,9 +1798,4 @@ sandbox_is_active(void) return 0; } -void -sandbox_disable_getaddrinfo_cache(void) -{ -} - #endif /* !defined(USE_LIBSECCOMP) */ diff --git a/src/lib/smartlist_core/.may_include b/src/lib/smartlist_core/.may_include index a8507761a4..2f0c8d341e 100644 --- a/src/lib/smartlist_core/.may_include +++ b/src/lib/smartlist_core/.may_include @@ -4,4 +4,4 @@ lib/malloc/*.h lib/err/*.h lib/string/*.h lib/smartlist_core/*.h -lib/testsupport/testsupport.h +lib/testsupport/*.h diff --git a/src/lib/string/.may_include b/src/lib/string/.may_include index ec5c769831..1fb9127f19 100644 --- a/src/lib/string/.may_include +++ b/src/lib/string/.may_include @@ -6,5 +6,5 @@ lib/malloc/*.h lib/ctime/*.h lib/string/*.h -strlcat.c -strlcpy.c +ext/strlcat.c +ext/strlcpy.c diff --git a/src/lib/string/compat_string.c b/src/lib/string/compat_string.c index eae82fdae0..b3f1e0fd96 100644 --- a/src/lib/string/compat_string.c +++ b/src/lib/string/compat_string.c @@ -14,10 +14,10 @@ /* Inline the strl functions if the platform doesn't have them. */ #ifndef HAVE_STRLCPY -#include "strlcpy.c" +#include "ext/strlcpy.c" #endif #ifndef HAVE_STRLCAT -#include "strlcat.c" +#include "ext/strlcat.c" #endif #include <stdlib.h> diff --git a/src/lib/string/util_string.c b/src/lib/string/util_string.c index b2b85d151d..e76e73046f 100644 --- a/src/lib/string/util_string.c +++ b/src/lib/string/util_string.c @@ -541,3 +541,16 @@ string_is_utf8(const char *str, size_t len) } return true; } + +/** As string_is_utf8(), but returns false if the string begins with a UTF-8 + * byte order mark (BOM). + */ +int +string_is_utf8_no_bom(const char *str, size_t len) +{ + if (len >= 3 && (!strcmpstart(str, "\uFEFF") || + !strcmpstart(str, "\uFFFE"))) { + return false; + } + return string_is_utf8(str, len); +} diff --git a/src/lib/string/util_string.h b/src/lib/string/util_string.h index 746ece0d33..99467a27c3 100644 --- a/src/lib/string/util_string.h +++ b/src/lib/string/util_string.h @@ -53,5 +53,6 @@ const char *find_str_at_start_of_line(const char *haystack, int string_is_C_identifier(const char *string); int string_is_utf8(const char *str, size_t len); +int string_is_utf8_no_bom(const char *str, size_t len); #endif /* !defined(TOR_UTIL_STRING_H) */ diff --git a/src/lib/subsys/.may_include b/src/lib/subsys/.may_include new file mode 100644 index 0000000000..2b06e8519c --- /dev/null +++ b/src/lib/subsys/.may_include @@ -0,0 +1 @@ +orconfig.h diff --git a/src/lib/subsys/include.am b/src/lib/subsys/include.am new file mode 100644 index 0000000000..4741126b14 --- /dev/null +++ b/src/lib/subsys/include.am @@ -0,0 +1,3 @@ + +noinst_HEADERS += \ + src/lib/subsys/subsys.h diff --git a/src/lib/subsys/subsys.h b/src/lib/subsys/subsys.h new file mode 100644 index 0000000000..2452ec6e2f --- /dev/null +++ b/src/lib/subsys/subsys.h @@ -0,0 +1,95 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_SUBSYS_T +#define TOR_SUBSYS_T + +#include <stdbool.h> + +struct dispatch_connector_t; + +/** + * A subsystem is a part of Tor that is initialized, shut down, configured, + * and connected to other parts of Tor. + * + * All callbacks are optional -- if a callback is set to NULL, the subsystem + * manager will treat it as a no-op. + * + * You should use c99 named-field initializers with this structure: we + * will be adding more fields, often in the middle of the structure. + **/ +typedef struct subsys_fns_t { + /** + * The name of this subsystem. It should be a programmer-readable + * identifier. + **/ + const char *name; + + /** + * Whether this subsystem is supported -- that is, whether it is compiled + * into Tor. For most subsystems, this should be true. + **/ + bool supported; + + /** + * The 'initialization level' for the subsystem. It should run from -100 + * through +100. The subsystems are initialized from lowest level to + * highest, and shut down from highest level to lowest. + **/ + int level; + + /** + * Initialize any global components of this subsystem. + * + * This function MAY rely on any lower-level subsystem being initialized. + * + * This function MUST NOT rely on any runtime configuration information; + * it is only for global state or pre-configuration state. + * + * (If you need to do any setup that depends on configuration, you'll need + * to declare a configuration callback. (Not yet designed)) + * + * This function MUST NOT have any parts that can fail. + **/ + int (*initialize)(void); + + /** + * Connect a subsystem to the message dispatch system. + **/ + int (*add_pubsub)(struct dispatch_connector_t *); + + /** + * Perform any necessary pre-fork cleanup. This function may not fail. + */ + void (*prefork)(void); + + /** + * Perform any necessary post-fork setup. This function may not fail. + */ + void (*postfork)(void); + + /** + * Free any thread-local resources held by this subsystem. Called before + * the thread exits. + */ + void (*thread_cleanup)(void); + + /** + * Free all resources held by this subsystem. + * + * This function is not allowed to fail. + **/ + void (*shutdown)(void); + +} subsys_fns_t; + +#define MIN_SUBSYS_LEVEL -100 +#define MAX_SUBSYS_LEVEL 100 + +/* All tor "libraries" (in src/libs) should have a subsystem level equal to or + * less than this value. */ +#define SUBSYS_LEVEL_LIBS -10 + +#endif diff --git a/src/lib/term/.may_include b/src/lib/term/.may_include index c93a06e59e..306fa57b7a 100644 --- a/src/lib/term/.may_include +++ b/src/lib/term/.may_include @@ -5,5 +5,4 @@ lib/log/*.h lib/term/*.h lib/malloc/*.h -# From src/ext -tor_readpassphrase.h +ext/tor_readpassphrase.h diff --git a/src/lib/term/getpass.c b/src/lib/term/getpass.c index 27a27179b6..a473fb765b 100644 --- a/src/lib/term/getpass.c +++ b/src/lib/term/getpass.c @@ -36,7 +36,7 @@ SecureZeroMemory(PVOID ptr, SIZE_T cnt) #elif defined(HAVE_READPASSPHRASE_H) #include <readpassphrase.h> #else -#include "tor_readpassphrase.h" +#include "ext/tor_readpassphrase.h" #endif /* defined(_WIN32) || ... */ #include <stdlib.h> diff --git a/src/lib/thread/.may_include b/src/lib/thread/.may_include index fc56f46836..02711348c5 100644 --- a/src/lib/thread/.may_include +++ b/src/lib/thread/.may_include @@ -2,6 +2,7 @@ orconfig.h lib/cc/*.h lib/lock/*.h lib/log/*.h +lib/subsys/*.h lib/testsupport/*.h lib/thread/*.h lib/wallclock/*.h diff --git a/src/lib/thread/compat_threads.c b/src/lib/thread/compat_threads.c index 7f1970af45..0b466da212 100644 --- a/src/lib/thread/compat_threads.c +++ b/src/lib/thread/compat_threads.c @@ -14,9 +14,11 @@ #include "orconfig.h" #include <stdlib.h> #include "lib/thread/threads.h" +#include "lib/thread/thread_sys.h" #include "lib/log/log.h" #include "lib/log/util_bug.h" +#include "lib/subsys/subsys.h" #include <string.h> @@ -109,3 +111,17 @@ atomic_counter_exchange(atomic_counter_t *counter, size_t newval) return oldval; } #endif /* !defined(HAVE_WORKING_STDATOMIC) */ + +static int +subsys_threads_initialize(void) +{ + tor_threads_init(); + return 0; +} + +const subsys_fns_t sys_threads = { + .name = "threads", + .supported = true, + .level = -95, + .initialize = subsys_threads_initialize, +}; diff --git a/src/lib/thread/include.am b/src/lib/thread/include.am index 9ec23d166e..695795a2c8 100644 --- a/src/lib/thread/include.am +++ b/src/lib/thread/include.am @@ -23,5 +23,6 @@ src_lib_libtor_thread_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_thread_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) noinst_HEADERS += \ - src/lib/thread/threads.h \ - src/lib/thread/numcpus.h + src/lib/thread/numcpus.h \ + src/lib/thread/thread_sys.h \ + src/lib/thread/threads.h diff --git a/src/lib/thread/thread_sys.h b/src/lib/thread/thread_sys.h new file mode 100644 index 0000000000..984abe88e8 --- /dev/null +++ b/src/lib/thread/thread_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file threads_sys.h + * \brief Declare subsystem object for threads library + **/ + +#ifndef TOR_THREADS_SYS_H +#define TOR_THREADS_SYS_H + +extern const struct subsys_fns_t sys_threads; + +#endif /* !defined(TOR_THREADS_SYS_H) */ diff --git a/src/lib/time/.may_include b/src/lib/time/.may_include index 2c7e37a836..40a18805ac 100644 --- a/src/lib/time/.may_include +++ b/src/lib/time/.may_include @@ -4,6 +4,7 @@ lib/cc/*.h lib/err/*.h lib/intmath/*.h lib/log/*.h +lib/subsys/*.h lib/time/*.h lib/wallclock/*.h diff --git a/src/lib/time/include.am b/src/lib/time/include.am index a3f93a3744..dae16f49ac 100644 --- a/src/lib/time/include.am +++ b/src/lib/time/include.am @@ -7,6 +7,7 @@ endif src_lib_libtor_time_a_SOURCES = \ src/lib/time/compat_time.c \ + src/lib/time/time_sys.c \ src/lib/time/tvdiff.c src_lib_libtor_time_testing_a_SOURCES = \ @@ -16,4 +17,5 @@ src_lib_libtor_time_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) noinst_HEADERS += \ src/lib/time/compat_time.h \ + src/lib/time/time_sys.h \ src/lib/time/tvdiff.h diff --git a/src/lib/time/time_sys.c b/src/lib/time/time_sys.c new file mode 100644 index 0000000000..b29ca35e69 --- /dev/null +++ b/src/lib/time/time_sys.c @@ -0,0 +1,26 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file time_sys.c + * \brief Subsystem object for monotime setup. + **/ + +#include "orconfig.h" +#include "lib/subsys/subsys.h" +#include "lib/time/time_sys.h" +#include "lib/time/compat_time.h" + +static int +subsys_time_initialize(void) +{ + monotime_init(); + return 0; +} + +const subsys_fns_t sys_time = { + .name = "time", + .level = -90, + .supported = true, + .initialize = subsys_time_initialize, +}; diff --git a/src/lib/time/time_sys.h b/src/lib/time/time_sys.h new file mode 100644 index 0000000000..5f5982a33b --- /dev/null +++ b/src/lib/time/time_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file time_sys.h + * \brief Declare subsystem object for the time module. + **/ + +#ifndef TOR_TIME_SYS_H +#define TOR_TIME_SYS_H + +extern const struct subsys_fns_t sys_time; + +#endif /* !defined(TOR_TIME_SYS_H) */ diff --git a/src/lib/tls/.may_include b/src/lib/tls/.may_include index 2840e590b8..c550bde024 100644 --- a/src/lib/tls/.may_include +++ b/src/lib/tls/.may_include @@ -1,6 +1,7 @@ orconfig.h lib/arch/*.h +lib/buf/*.h lib/cc/*.h lib/container/*.h lib/crypt_ops/*.h @@ -11,7 +12,7 @@ lib/log/*.h lib/malloc/*.h lib/net/*.h lib/string/*.h -lib/testsupport/testsupport.h +lib/subsys/*.h +lib/testsupport/*.h lib/tls/*.h - -ciphers.inc +lib/tls/*.inc diff --git a/src/lib/tls/buffers_tls.c b/src/lib/tls/buffers_tls.c index 69ae4f7fc0..b4059292ea 100644 --- a/src/lib/tls/buffers_tls.c +++ b/src/lib/tls/buffers_tls.c @@ -12,7 +12,7 @@ #define BUFFERS_PRIVATE #include "orconfig.h" #include <stddef.h> -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/tls/buffers_tls.h" #include "lib/cc/torint.h" #include "lib/log/log.h" diff --git a/src/lib/tls/include.am b/src/lib/tls/include.am index a664b29fb2..1817739eef 100644 --- a/src/lib/tls/include.am +++ b/src/lib/tls/include.am @@ -36,5 +36,6 @@ noinst_HEADERS += \ src/lib/tls/tortls.h \ src/lib/tls/tortls_internal.h \ src/lib/tls/tortls_st.h \ + src/lib/tls/tortls_sys.h \ src/lib/tls/x509.h \ src/lib/tls/x509_internal.h diff --git a/src/lib/tls/tortls.c b/src/lib/tls/tortls.c index 56f70bc371..654cacacf7 100644 --- a/src/lib/tls/tortls.c +++ b/src/lib/tls/tortls.c @@ -7,6 +7,7 @@ #define TOR_X509_PRIVATE #include "lib/tls/x509.h" #include "lib/tls/x509_internal.h" +#include "lib/tls/tortls_sys.h" #include "lib/tls/tortls.h" #include "lib/tls/tortls_st.h" #include "lib/tls/tortls_internal.h" @@ -15,6 +16,7 @@ #include "lib/crypt_ops/crypto_rsa.h" #include "lib/crypt_ops/crypto_rand.h" #include "lib/net/socket.h" +#include "lib/subsys/subsys.h" #ifdef _WIN32 #include <winsock2.h> @@ -440,3 +442,15 @@ tor_tls_verify(int severity, tor_tls_t *tls, crypto_pk_t **identity) return rv; } + +static void +subsys_tortls_shutdown(void) +{ + tor_tls_free_all(); +} + +const subsys_fns_t sys_tortls = { + .name = "tortls", + .level = -50, + .shutdown = subsys_tortls_shutdown +}; diff --git a/src/lib/tls/tortls_openssl.c b/src/lib/tls/tortls_openssl.c index 99371cfc40..c0ad74d908 100644 --- a/src/lib/tls/tortls_openssl.c +++ b/src/lib/tls/tortls_openssl.c @@ -464,7 +464,7 @@ static const char UNRESTRICTED_SERVER_CIPHER_LIST[] = /** List of ciphers that clients should advertise, omitting items that * our OpenSSL doesn't know about. */ static const char CLIENT_CIPHER_LIST[] = -#include "ciphers.inc" +#include "lib/tls/ciphers.inc" /* Tell it not to use SSLv2 ciphers, so that it can select an SSLv3 version * of any cipher we say. */ "!SSLv2" diff --git a/src/lib/tls/tortls_sys.h b/src/lib/tls/tortls_sys.h new file mode 100644 index 0000000000..fd909f6019 --- /dev/null +++ b/src/lib/tls/tortls_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file tortls_sys.h + * \brief Declare subsystem object for the tortls module + **/ + +#ifndef TOR_TORTLS_SYS_H +#define TOR_TORTLS_SYS_H + +extern const struct subsys_fns_t sys_tortls; + +#endif /* !defined(TOR_TORTLS_SYS_H) */ diff --git a/src/lib/version/.may_include b/src/lib/version/.may_include new file mode 100644 index 0000000000..d159ceb41f --- /dev/null +++ b/src/lib/version/.may_include @@ -0,0 +1,3 @@ +orconfig.h +micro-revision.i +lib/version/*.h
\ No newline at end of file diff --git a/src/lib/log/git_revision.c b/src/lib/version/git_revision.c index 9d29ecd2a2..e5b2ff534e 100644 --- a/src/lib/log/git_revision.c +++ b/src/lib/version/git_revision.c @@ -4,7 +4,7 @@ /* See LICENSE for licensing information */ #include "orconfig.h" -#include "lib/log/git_revision.h" +#include "lib/version/git_revision.h" /** String describing which Tor Git repository version the source was * built from. This string is generated by a bit of shell kludging in diff --git a/src/lib/log/git_revision.h b/src/lib/version/git_revision.h index 0ce1190795..0ce1190795 100644 --- a/src/lib/log/git_revision.h +++ b/src/lib/version/git_revision.h diff --git a/src/lib/version/include.am b/src/lib/version/include.am new file mode 100644 index 0000000000..6944eb05e3 --- /dev/null +++ b/src/lib/version/include.am @@ -0,0 +1,25 @@ + +noinst_LIBRARIES += src/lib/libtor-version.a + +if UNITTESTS_ENABLED +noinst_LIBRARIES += src/lib/libtor-version-testing.a +endif + +src_lib_libtor_version_a_SOURCES = \ + src/lib/version/git_revision.c \ + src/lib/version/version.c + +src_lib_libtor_version_testing_a_SOURCES = \ + $(src_lib_libtor_version_a_SOURCES) +src_lib_libtor_version_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) +src_lib_libtor_version_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) + +# Declare that these object files depend on micro-revision.i. Without this +# rule, we could try to build them before micro-revision.i was created. +src/lib/version/git_revision.$(OBJEXT) \ + src/lib/version/src_lib_libtor_version_testing_a-git_revision.$(OBJEXT): \ + micro-revision.i + +noinst_HEADERS += \ + src/lib/version/git_revision.h \ + src/lib/version/torversion.h diff --git a/src/lib/version/torversion.h b/src/lib/version/torversion.h new file mode 100644 index 0000000000..761d6f25ab --- /dev/null +++ b/src/lib/version/torversion.h @@ -0,0 +1,12 @@ +/* Copyright 2001-2004 Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_VERSION_H +#define TOR_VERSION_H + +const char *get_version(void); +const char *get_short_version(void); + +#endif /* !defined(TOR_VERSION_H) */ diff --git a/src/lib/version/version.c b/src/lib/version/version.c new file mode 100644 index 0000000000..29ada39c9d --- /dev/null +++ b/src/lib/version/version.c @@ -0,0 +1,50 @@ +/* Copyright 2001-2004 Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#include "lib/version/torversion.h" +#include "lib/version/git_revision.h" + +#include <stdio.h> +#include <string.h> + +/** A shorter version of this Tor process's version, for export in our router + * descriptor. (Does not include the git version, if any.) */ +static const char the_short_tor_version[] = + VERSION +#ifdef TOR_BUILD_TAG + " ("TOR_BUILD_TAG")" +#endif + ""; + +#define MAX_VERSION_LEN 128 + +/** The version of this Tor process, possibly including git version */ +static char the_tor_version[MAX_VERSION_LEN] = ""; + +/** Return the current Tor version. */ +const char * +get_version(void) +{ + if (the_tor_version[0] == 0) { + if (strlen(tor_git_revision)) { + snprintf(the_tor_version, sizeof(the_tor_version), + "%s (git-%s)", the_short_tor_version, tor_git_revision); + } else { + snprintf(the_tor_version, sizeof(the_tor_version), + "%s", the_short_tor_version); + } + the_tor_version[sizeof(the_tor_version)-1] = 0; + } + + return the_tor_version; +} + +/** Return the current Tor version, without any git tag. */ +const char * +get_short_version(void) +{ + return the_short_tor_version; +} diff --git a/src/lib/wallclock/.may_include b/src/lib/wallclock/.may_include index dc010da063..ce7a26472b 100644 --- a/src/lib/wallclock/.may_include +++ b/src/lib/wallclock/.may_include @@ -3,4 +3,5 @@ lib/cc/*.h lib/err/*.h lib/wallclock/*.h lib/string/*.h +lib/subsys/*.h lib/testsupport/*.h diff --git a/src/lib/wallclock/approx_time.c b/src/lib/wallclock/approx_time.c index bb9a292369..0b0ef382c2 100644 --- a/src/lib/wallclock/approx_time.c +++ b/src/lib/wallclock/approx_time.c @@ -9,7 +9,9 @@ **/ #include "orconfig.h" +#include "lib/subsys/subsys.h" #include "lib/wallclock/approx_time.h" +#include "lib/wallclock/wallclock_sys.h" /* ===== * Cached time @@ -41,3 +43,17 @@ update_approx_time(time_t now) cached_approx_time = now; } #endif /* !defined(TIME_IS_FAST) */ + +static int +subsys_wallclock_initialize(void) +{ + update_approx_time(time(NULL)); + return 0; +} + +const subsys_fns_t sys_wallclock = { + .name = "wallclock", + .supported = true, + .level = -99, + .initialize = subsys_wallclock_initialize, +}; diff --git a/src/lib/wallclock/include.am b/src/lib/wallclock/include.am index 1961639bd7..2351252e0c 100644 --- a/src/lib/wallclock/include.am +++ b/src/lib/wallclock/include.am @@ -19,4 +19,5 @@ noinst_HEADERS += \ src/lib/wallclock/approx_time.h \ src/lib/wallclock/timeval.h \ src/lib/wallclock/time_to_tm.h \ - src/lib/wallclock/tor_gettimeofday.h + src/lib/wallclock/tor_gettimeofday.h \ + src/lib/wallclock/wallclock_sys.h diff --git a/src/lib/wallclock/wallclock_sys.h b/src/lib/wallclock/wallclock_sys.h new file mode 100644 index 0000000000..e009578a83 --- /dev/null +++ b/src/lib/wallclock/wallclock_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file wallclock_sys.h + * \brief Declare subsystem object for the wallclock module. + **/ + +#ifndef TOR_WALLCLOCK_SYS_H +#define TOR_WALLCLOCK_SYS_H + +extern const struct subsys_fns_t sys_wallclock; + +#endif /* !defined(TOR_WALLCLOCK_SYS_H) */ diff --git a/src/rust/build.rs b/src/rust/build.rs index 123d5c0682..5626b35f75 100644 --- a/src/rust/build.rs +++ b/src/rust/build.rs @@ -149,8 +149,9 @@ pub fn main() { cfg.component("tor-sandbox-testing"); cfg.component("tor-encoding-testing"); cfg.component("tor-fs-testing"); - cfg.component("tor-time-testing"); cfg.component("tor-net-testing"); + cfg.component("tor-buf-testing"); + cfg.component("tor-time-testing"); cfg.component("tor-thread-testing"); cfg.component("tor-memarea-testing"); cfg.component("tor-log-testing"); @@ -162,6 +163,7 @@ pub fn main() { cfg.component("tor-malloc"); cfg.component("tor-wallclock"); cfg.component("tor-err-testing"); + cfg.component("tor-version-testing"); cfg.component("tor-intmath-testing"); cfg.component("tor-ctime-testing"); cfg.component("curve25519_donna"); diff --git a/src/rust/tor_util/strings.rs b/src/rust/tor_util/strings.rs index d64275e06b..71a908a58c 100644 --- a/src/rust/tor_util/strings.rs +++ b/src/rust/tor_util/strings.rs @@ -105,11 +105,7 @@ macro_rules! cstr { ($($bytes:expr),*) => ( ::std::ffi::CStr::from_bytes_with_nul( concat!($($bytes),*, "\0").as_bytes() - ).unwrap_or( - unsafe{ - ::std::ffi::CStr::from_bytes_with_nul_unchecked(b"\0") - } - ) + ).unwrap_or_default() ) } diff --git a/src/test/Makefile.nmake b/src/test/Makefile.nmake index cfbe281b94..aa16a22b52 100644 --- a/src/test/Makefile.nmake +++ b/src/test/Makefile.nmake @@ -1,4 +1,4 @@ -all: test.exe test-child.exe bench.exe +all: test.exe bench.exe CFLAGS = /I ..\win32 /I ..\..\..\build-alpha\include /I ..\common /I ..\or \ /I ..\ext @@ -30,8 +30,5 @@ test.exe: $(TEST_OBJECTS) bench.exe: bench.obj $(CC) $(CFLAGS) bench.obj $(LIBS) ..\common\*.lib /Fe$@ -test-child.exe: test-child.obj - $(CC) $(CFLAGS) test-child.obj /Fe$@ - clean: - del *.obj *.lib test.exe bench.exe test-child.exe + del *.obj *.lib test.exe bench.exe diff --git a/src/test/bench.c b/src/test/bench.c index 959d4374b1..f8680c3ab6 100644 --- a/src/test/bench.c +++ b/src/test/bench.c @@ -24,6 +24,7 @@ #include "core/or/circuitlist.h" #include "app/config/config.h" +#include "app/main/subsysmgr.h" #include "lib/crypt_ops/crypto_curve25519.h" #include "lib/crypt_ops/crypto_dh.h" #include "core/crypto/onion_ntor.h" @@ -38,6 +39,9 @@ #include "lib/crypt_ops/digestset.h" #include "lib/crypt_ops/crypto_init.h" +#include "feature/dirparse/microdesc_parse.h" +#include "feature/nodelist/microdesc.h" + #if defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_PROCESS_CPUTIME_ID) static uint64_t nanostart; static inline uint64_t @@ -638,6 +642,41 @@ bench_ecdh_p224(void) } #endif +static void +bench_md_parse(void) +{ + uint64_t start, end; + const int N = 100000; + // selected arbitrarily + const char md_text[] = + "@last-listed 2018-12-14 18:14:14\n" + "onion-key\n" + "-----BEGIN RSA PUBLIC KEY-----\n" + "MIGJAoGBAMHkZeXNDX/49JqM2BVLmh1Fnb5iMVnatvZZTLJyedqDLkbXZ1WKP5oh\n" + "7ec14dj/k3ntpwHD4s2o3Lb6nfagWbug4+F/rNJ7JuFru/PSyOvDyHGNAuegOXph\n" + "3gTGjdDpv/yPoiadGebbVe8E7n6hO+XxM2W/4dqheKimF0/s9B7HAgMBAAE=\n" + "-----END RSA PUBLIC KEY-----\n" + "ntor-onion-key QgF/EjqlNG1wRHLIop/nCekEH+ETGZSgYOhu26eiTF4=\n" + "family $00E9A86E7733240E60D8435A7BBD634A23894098 " + "$329BD7545DEEEBBDC8C4285F243916F248972102 " + "$69E06EBB2573A4F89330BDF8BC869794A3E10E4D " + "$DCA2A3FAE50B3729DAA15BC95FB21AF03389818B\n" + "p accept 53,80,443,5222-5223,25565\n" + "id ed25519 BzffzY99z6Q8KltcFlUTLWjNTBU7yKK+uQhyi1Ivb3A\n"; + + reset_perftime(); + start = perftime(); + for (int i = 0; i < N; ++i) { + smartlist_t *s = microdescs_parse_from_string(md_text, NULL, 1, + SAVED_IN_CACHE, NULL); + SMARTLIST_FOREACH(s, microdesc_t *, md, microdesc_free(md)); + smartlist_free(s); + } + + end = perftime(); + printf("Microdesc parse: %f nsec\n", NANOCOUNT(start, end, N)); +} + typedef void (*bench_fn)(void); typedef struct benchmark_t { @@ -665,6 +704,8 @@ static struct benchmark_t benchmarks[] = { ENT(ecdh_p256), ENT(ecdh_p224), #endif + + ENT(md_parse), {NULL,NULL,0} }; @@ -690,9 +731,10 @@ main(int argc, const char **argv) char *errmsg; or_options_t *options; - tor_threads_init(); + subsystems_init_upto(SUBSYS_LEVEL_LIBS); + flush_log_messages_from_startup(); + tor_compress_init(); - init_logging(1); if (argc == 4 && !strcmp(argv[1], "diff")) { const int N = 200; @@ -702,11 +744,13 @@ main(int argc, const char **argv) perror("X"); return 1; } + size_t f1len = strlen(f1); + size_t f2len = strlen(f2); for (i = 0; i < N; ++i) { - char *diff = consensus_diff_generate(f1, f2); + char *diff = consensus_diff_generate(f1, f1len, f2, f2len); tor_free(diff); } - char *diff = consensus_diff_generate(f1, f2); + char *diff = consensus_diff_generate(f1, f1len, f2, f2len); printf("%s", diff); tor_free(f1); tor_free(f2); @@ -737,7 +781,6 @@ main(int argc, const char **argv) init_protocol_warning_severity_level(); options = options_new(); - init_logging(1); options->command = CMD_RUN_UNITTESTS; options->DataDirectory = tor_strdup(""); options->KeyDirectory = tor_strdup(""); diff --git a/src/test/fuzz/fuzz_consensus.c b/src/test/fuzz/fuzz_consensus.c index b56702a650..1a4195b418 100644 --- a/src/test/fuzz/fuzz_consensus.c +++ b/src/test/fuzz/fuzz_consensus.c @@ -61,13 +61,13 @@ int fuzz_main(const uint8_t *data, size_t sz) { networkstatus_t *ns; - char *str = tor_memdup_nulterm(data, sz); const char *eos = NULL; networkstatus_type_t tp = NS_TYPE_CONSENSUS; if (tor_memstr(data, MIN(sz, 1024), "tus vote")) tp = NS_TYPE_VOTE; const char *what = (tp == NS_TYPE_CONSENSUS) ? "consensus" : "vote"; - ns = networkstatus_parse_vote_from_string(str, + ns = networkstatus_parse_vote_from_string((const char *)data, + sz, &eos, tp); if (ns) { @@ -76,6 +76,6 @@ fuzz_main(const uint8_t *data, size_t sz) } else { log_debug(LD_GENERAL, "Parsing as %s failed", what); } - tor_free(str); + return 0; } diff --git a/src/test/fuzz/fuzz_diff.c b/src/test/fuzz/fuzz_diff.c index 1079856fdb..64aecc8a64 100644 --- a/src/test/fuzz/fuzz_diff.c +++ b/src/test/fuzz/fuzz_diff.c @@ -10,9 +10,11 @@ #include "test/fuzz/fuzzing.h" static int -mock_consensus_compute_digest_(const char *c, consensus_digest_t *d) +mock_consensus_compute_digest_(const char *c, size_t len, + consensus_digest_t *d) { (void)c; + (void)len; memset(d->sha3_256, 3, sizeof(d->sha3_256)); return 0; } @@ -42,28 +44,34 @@ fuzz_main(const uint8_t *stdin_buf, size_t data_size) if (! separator) return 0; size_t c1_len = separator - stdin_buf; - char *c1 = tor_memdup_nulterm(stdin_buf, c1_len); + const char *c1 = (const char *)stdin_buf; size_t c2_len = data_size - c1_len - SEPLEN; - char *c2 = tor_memdup_nulterm(separator + SEPLEN, c2_len); + const char *c2 = (const char *)separator + SEPLEN; - char *c3 = consensus_diff_generate(c1, c2); + const char *cp = memchr(c1, 0, c1_len); + if (cp) + c1_len = cp - c1; + + cp = memchr(c2, 0, c2_len); + if (cp) + c2_len = cp - c2; + + char *c3 = consensus_diff_generate(c1, c1_len, c2, c2_len); if (c3) { - char *c4 = consensus_diff_apply(c1, c3); + char *c4 = consensus_diff_apply(c1, c1_len, c3, strlen(c3)); tor_assert(c4); - if (strcmp(c2, c4)) { - printf("%s\n", escaped(c1)); - printf("%s\n", escaped(c2)); + int equal = (c2_len == strlen(c4)) && fast_memeq(c2, c4, c2_len); + if (! equal) { + //printf("%s\n", escaped(c1)); + //printf("%s\n", escaped(c2)); printf("%s\n", escaped(c3)); printf("%s\n", escaped(c4)); } - tor_assert(! strcmp(c2, c4)); + tor_assert(equal); tor_free(c3); tor_free(c4); } - tor_free(c1); - tor_free(c2); return 0; } - diff --git a/src/test/fuzz/fuzz_diff_apply.c b/src/test/fuzz/fuzz_diff_apply.c index 165d0e6126..9b25185225 100644 --- a/src/test/fuzz/fuzz_diff_apply.c +++ b/src/test/fuzz/fuzz_diff_apply.c @@ -10,9 +10,11 @@ #include "test/fuzz/fuzzing.h" static int -mock_consensus_compute_digest_(const char *c, consensus_digest_t *d) +mock_consensus_compute_digest_(const char *c, size_t len, + consensus_digest_t *d) { (void)c; + (void)len; memset(d->sha3_256, 3, sizeof(d->sha3_256)); return 0; } @@ -50,16 +52,13 @@ fuzz_main(const uint8_t *stdin_buf, size_t data_size) if (! separator) return 0; size_t c1_len = separator - stdin_buf; - char *c1 = tor_memdup_nulterm(stdin_buf, c1_len); + const char *c1 = (const char *)stdin_buf; size_t c2_len = data_size - c1_len - SEPLEN; - char *c2 = tor_memdup_nulterm(separator + SEPLEN, c2_len); + const char *c2 = (const char *)separator + SEPLEN; - char *c3 = consensus_diff_apply(c1, c2); + char *c3 = consensus_diff_apply(c1, c1_len, c2, c2_len); - tor_free(c1); - tor_free(c2); tor_free(c3); return 0; } - diff --git a/src/test/fuzz/fuzz_http.c b/src/test/fuzz/fuzz_http.c index 06483282bc..4341bfabae 100644 --- a/src/test/fuzz/fuzz_http.c +++ b/src/test/fuzz/fuzz_http.c @@ -8,7 +8,7 @@ #include "core/or/or.h" #include "lib/err/backtrace.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "app/config/config.h" #include "core/mainloop/connection.h" #include "feature/dircache/dircache.h" diff --git a/src/test/fuzz/fuzz_http_connect.c b/src/test/fuzz/fuzz_http_connect.c index ca007a2c7f..e03d9e29d8 100644 --- a/src/test/fuzz/fuzz_http_connect.c +++ b/src/test/fuzz/fuzz_http_connect.c @@ -8,7 +8,7 @@ #include "core/or/or.h" #include "lib/err/backtrace.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "app/config/config.h" #include "core/mainloop/connection.h" #include "core/or/connection_edge.h" diff --git a/src/test/fuzz/fuzz_socks.c b/src/test/fuzz/fuzz_socks.c index 14c25304b1..2d93bea924 100644 --- a/src/test/fuzz/fuzz_socks.c +++ b/src/test/fuzz/fuzz_socks.c @@ -6,7 +6,7 @@ #define BUFFERS_PRIVATE #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/err/backtrace.h" #include "lib/log/log.h" #include "core/proto/proto_socks.h" diff --git a/src/test/fuzz/fuzz_strops.c b/src/test/fuzz/fuzz_strops.c new file mode 100644 index 0000000000..5da590acfa --- /dev/null +++ b/src/test/fuzz/fuzz_strops.c @@ -0,0 +1,247 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file fuzz_strops.c + * \brief Fuzzers for various string encoding/decoding operations + **/ + +#include "orconfig.h" + +#include "lib/cc/torint.h" +#include "lib/ctime/di_ops.h" +#include "lib/encoding/binascii.h" +#include "lib/encoding/cstring.h" +#include "lib/encoding/kvline.h" +#include "lib/encoding/confline.h" +#include "lib/malloc/malloc.h" +#include "lib/log/escape.h" +#include "lib/log/util_bug.h" +#include "lib/intmath/muldiv.h" + +#include "test/fuzz/fuzzing.h" + +#include <stdio.h> +#include <string.h> + +int +fuzz_init(void) +{ + return 0; +} + +int +fuzz_cleanup(void) +{ + return 0; +} + +typedef struct chunk_t { + uint8_t *buf; + size_t len; +} chunk_t; + +#define chunk_free(ch) \ + FREE_AND_NULL(chunk_t, chunk_free_, (ch)) + +static chunk_t * +chunk_new(size_t len) +{ + chunk_t *ch = tor_malloc(sizeof(chunk_t)); + ch->buf = tor_malloc(len); + ch->len = len; + return ch; +} +static void +chunk_free_(chunk_t *ch) +{ + if (!ch) + return; + tor_free(ch->buf); + tor_free(ch); +} +static bool +chunk_eq(const chunk_t *a, const chunk_t *b) +{ + return a->len == b->len && fast_memeq(a->buf, b->buf, a->len); +} + +static chunk_t * +b16_dec(const chunk_t *inp) +{ + chunk_t *ch = chunk_new(CEIL_DIV(inp->len, 2)); + int r = base16_decode((char *)ch->buf, ch->len, (char *)inp->buf, inp->len); + if (r >= 0) { + ch->len = r; + } else { + chunk_free(ch); + } + return ch; +} +static chunk_t * +b16_enc(const chunk_t *inp) +{ + chunk_t *ch = chunk_new(inp->len * 2 + 1); + base16_encode((char *)ch->buf, ch->len, (char*)inp->buf, inp->len); + return ch; +} + +#if 0 +static chunk_t * +b32_dec(const chunk_t *inp) +{ + chunk_t *ch = chunk_new(inp->len);//XXXX + int r = base32_decode((char *)ch->buf, ch->len, (char *)inp->buf, inp->len); + if (r >= 0) { + ch->len = r; // XXXX we need some way to get the actual length of + // XXXX the output here. + } else { + chunk_free(ch); + } + return ch; +} +static chunk_t * +b32_enc(const chunk_t *inp) +{ + chunk_t *ch = chunk_new(base32_encoded_size(inp->len)); + base32_encode((char *)ch->buf, ch->len, (char*)inp->buf, inp->len); + ch->len = strlen((char *) ch->buf); + return ch; +} +#endif + +static chunk_t * +b64_dec(const chunk_t *inp) +{ + chunk_t *ch = chunk_new(inp->len);//XXXX This could be shorter. + int r = base64_decode((char *)ch->buf, ch->len, (char *)inp->buf, inp->len); + if (r >= 0) { + ch->len = r; + } else { + chunk_free(ch); + } + return ch; +} +static chunk_t * +b64_enc(const chunk_t *inp) +{ + chunk_t *ch = chunk_new(BASE64_BUFSIZE(inp->len)); + base64_encode((char *)ch->buf, ch->len, (char *)inp->buf, inp->len, 0); + ch->len = strlen((char *) ch->buf); + return ch; +} + +static chunk_t * +c_dec(const chunk_t *inp) +{ + char *s = tor_memdup_nulterm(inp->buf, inp->len); + chunk_t *ch = tor_malloc(sizeof(chunk_t)); + char *r = NULL; + (void) unescape_string(s, &r, &ch->len); + tor_free(s); + ch->buf = (uint8_t*) r; + if (!ch->buf) { + tor_free(ch); + } + return ch; +} +static chunk_t * +c_enc(const chunk_t *inp) +{ + char *s = tor_memdup_nulterm(inp->buf, inp->len); + chunk_t *ch = tor_malloc(sizeof(chunk_t)); + ch->buf = (uint8_t*)esc_for_log(s); + tor_free(s); + ch->len = strlen((char*)ch->buf); + return ch; +} + +static int kv_flags = 0; +static config_line_t * +kv_dec(const chunk_t *inp) +{ + char *s = tor_memdup_nulterm(inp->buf, inp->len); + config_line_t *res = kvline_parse(s, kv_flags); + tor_free(s); + return res; +} +static chunk_t * +kv_enc(const config_line_t *inp) +{ + char *s = kvline_encode(inp, kv_flags); + if (!s) + return NULL; + chunk_t *res = tor_malloc(sizeof(chunk_t)); + res->buf = (uint8_t*)s; + res->len = strlen(s); + return res; +} + +/* Given an encoder function, a decoder function, and a function to free + * the decoded object, check whether any string that successfully decoded + * will then survive an encode-decode-encode round-trip unchanged. + */ +#define ENCODE_ROUNDTRIP(E,D,FREE) \ + STMT_BEGIN { \ + bool err = false; \ + a = D(&inp); \ + if (!a) \ + return 0; \ + b = E(a); \ + tor_assert(b); \ + c = D(b); \ + tor_assert(c); \ + d = E(c); \ + tor_assert(d); \ + if (!chunk_eq(b,d)) { \ + printf("Unequal chunks: %s\n", \ + hex_str((char*)b->buf, b->len)); \ + printf(" vs %s\n", \ + hex_str((char*)d->buf, d->len)); \ + err = true; \ + } \ + FREE(a); \ + chunk_free(b); \ + FREE(c); \ + chunk_free(d); \ + tor_assert(!err); \ + } STMT_END + +int +fuzz_main(const uint8_t *stdin_buf, size_t data_size) +{ + if (!data_size) + return 0; + + chunk_t inp = { (uint8_t*)stdin_buf, data_size }; + chunk_t *b=NULL,*d=NULL; + void *a=NULL,*c=NULL; + + switch (stdin_buf[0]) { + case 0: + ENCODE_ROUNDTRIP(b16_enc, b16_dec, chunk_free_); + break; + case 1: + /* + XXXX see notes above about our base-32 functions. + ENCODE_ROUNDTRIP(b32_enc, b32_dec, chunk_free_); + */ + break; + case 2: + ENCODE_ROUNDTRIP(b64_enc, b64_dec, chunk_free_); + break; + case 3: + ENCODE_ROUNDTRIP(c_enc, c_dec, chunk_free_); + break; + case 5: + kv_flags = KV_QUOTED|KV_OMIT_KEYS; + ENCODE_ROUNDTRIP(kv_enc, kv_dec, config_free_lines_); + break; + case 6: + kv_flags = 0; + ENCODE_ROUNDTRIP(kv_enc, kv_dec, config_free_lines_); + break; + } + + return 0; +} diff --git a/src/test/fuzz/fuzz_vrs.c b/src/test/fuzz/fuzz_vrs.c index 0b869aa5c0..f0d90d7cc6 100644 --- a/src/test/fuzz/fuzz_vrs.c +++ b/src/test/fuzz/fuzz_vrs.c @@ -3,6 +3,7 @@ #define NS_PARSE_PRIVATE #define NETWORKSTATUS_PRIVATE #include "core/or/or.h" +#include "feature/dirauth/dirvote.h" #include "feature/dirparse/ns_parse.h" #include "feature/dirparse/unparseable.h" #include "lib/memarea/memarea.h" @@ -35,9 +36,12 @@ fuzz_init(void) dummy_vote = tor_malloc_zero(sizeof(*dummy_vote)); dummy_vote->known_flags = smartlist_new(); smartlist_split_string(dummy_vote->known_flags, - "Authority BadExit Exit Fast Guard HSDir " - "NoEdConsensus Running Stable V2Dir Valid", + DIRVOTE_UNIVERSAL_FLAGS, " ", 0, 0); + smartlist_split_string(dummy_vote->known_flags, + DIRVOTE_OPTIONAL_FLAGS, + " ", 0, 0); + smartlist_sort_strings(dummy_vote->known_flags); return 0; } @@ -53,24 +57,24 @@ fuzz_cleanup(void) int fuzz_main(const uint8_t *data, size_t sz) { - char *str = tor_memdup_nulterm(data, sz); const char *s; routerstatus_t *rs_ns = NULL, *rs_md = NULL, *rs_vote = NULL; vote_routerstatus_t *vrs = tor_malloc_zero(sizeof(*vrs)); smartlist_t *tokens = smartlist_new(); + const char *eos = (const char *)data + sz; - s = str; - rs_ns = routerstatus_parse_entry_from_string(area, &s, tokens, + s = (const char *)data; + rs_ns = routerstatus_parse_entry_from_string(area, &s, eos, tokens, NULL, NULL, 26, FLAV_NS); tor_assert(smartlist_len(tokens) == 0); - s = str; - rs_md = routerstatus_parse_entry_from_string(area, &s, tokens, + s = (const char *)data; + rs_md = routerstatus_parse_entry_from_string(area, &s, eos, tokens, NULL, NULL, 26, FLAV_MICRODESC); tor_assert(smartlist_len(tokens) == 0); - s = str; - rs_vote = routerstatus_parse_entry_from_string(area, &s, tokens, + s = (const char *)data; + rs_vote = routerstatus_parse_entry_from_string(area, &s, eos, tokens, dummy_vote, vrs, 26, FLAV_NS); tor_assert(smartlist_len(tokens) == 0); @@ -82,6 +86,6 @@ fuzz_main(const uint8_t *data, size_t sz) vote_routerstatus_free(vrs); memarea_clear(area); smartlist_free(tokens); - tor_free(str); + return 0; } diff --git a/src/test/fuzz/fuzzing_common.c b/src/test/fuzz/fuzzing_common.c index 1401e4c28d..21aa07cfe2 100644 --- a/src/test/fuzz/fuzzing_common.c +++ b/src/test/fuzz/fuzzing_common.c @@ -3,12 +3,14 @@ #define CRYPTO_ED25519_PRIVATE #include "orconfig.h" #include "core/or/or.h" +#include "app/main/subsysmgr.h" #include "lib/err/backtrace.h" #include "app/config/config.h" #include "test/fuzz/fuzzing.h" #include "lib/compress/compress.h" #include "lib/crypt_ops/crypto_ed25519.h" #include "lib/crypt_ops/crypto_init.h" +#include "lib/version/torversion.h" static or_options_t *mock_options = NULL; static const or_options_t * @@ -94,12 +96,10 @@ disable_signature_checking(void) static void global_init(void) { - tor_threads_init(); - tor_compress_init(); + subsystems_init_upto(SUBSYS_LEVEL_LIBS); + flush_log_messages_from_startup(); - /* Initialise logging first */ - init_logging(1); - configure_backtrace_handler(get_version()); + tor_compress_init(); if (crypto_global_init(0, NULL, NULL) < 0) abort(); diff --git a/src/test/fuzz/include.am b/src/test/fuzz/include.am index 27eeced8c5..d0711f05d6 100644 --- a/src/test/fuzz/include.am +++ b/src/test/fuzz/include.am @@ -153,6 +153,16 @@ src_test_fuzz_fuzz_socks_LDADD = $(FUZZING_LIBS) endif if UNITTESTS_ENABLED +src_test_fuzz_fuzz_strops_SOURCES = \ + src/test/fuzz/fuzzing_common.c \ + src/test/fuzz/fuzz_strops.c +src_test_fuzz_fuzz_strops_CPPFLAGS = $(FUZZING_CPPFLAGS) +src_test_fuzz_fuzz_strops_CFLAGS = $(FUZZING_CFLAGS) +src_test_fuzz_fuzz_strops_LDFLAGS = $(FUZZING_LDFLAG) +src_test_fuzz_fuzz_strops_LDADD = $(FUZZING_LIBS) +endif + +if UNITTESTS_ENABLED src_test_fuzz_fuzz_vrs_SOURCES = \ src/test/fuzz/fuzzing_common.c \ src/test/fuzz/fuzz_vrs.c @@ -176,6 +186,7 @@ FUZZERS = \ src/test/fuzz/fuzz-iptsv2 \ src/test/fuzz/fuzz-microdesc \ src/test/fuzz/fuzz-socks \ + src/test/fuzz/fuzz-strops \ src/test/fuzz/fuzz-vrs endif @@ -291,6 +302,15 @@ src_test_fuzz_lf_fuzz_socks_LDADD = $(LIBFUZZER_LIBS) endif if UNITTESTS_ENABLED +src_test_fuzz_lf_fuzz_strops_SOURCES = \ + $(src_test_fuzz_fuzz_strops_SOURCES) +src_test_fuzz_lf_fuzz_strops_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) +src_test_fuzz_lf_fuzz_strops_CFLAGS = $(LIBFUZZER_CFLAGS) +src_test_fuzz_lf_fuzz_strops_LDFLAGS = $(LIBFUZZER_LDFLAG) +src_test_fuzz_lf_fuzz_strops_LDADD = $(LIBFUZZER_LIBS) +endif + +if UNITTESTS_ENABLED src_test_fuzz_lf_fuzz_vrs_SOURCES = \ $(src_test_fuzz_fuzz_vrs_SOURCES) src_test_fuzz_lf_fuzz_vrs_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) @@ -312,6 +332,7 @@ LIBFUZZER_FUZZERS = \ src/test/fuzz/lf-fuzz-iptsv2 \ src/test/fuzz/lf-fuzz-microdesc \ src/test/fuzz/lf-fuzz-socks \ + src/test/fuzz/lf-fuzz-strops \ src/test/fuzz/lf-fuzz-vrs else @@ -406,6 +427,13 @@ src_test_fuzz_liboss_fuzz_socks_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) endif if UNITTESTS_ENABLED +src_test_fuzz_liboss_fuzz_strops_a_SOURCES = \ + $(src_test_fuzz_fuzz_strops_SOURCES) +src_test_fuzz_liboss_fuzz_strops_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) +src_test_fuzz_liboss_fuzz_strops_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) +endif + +if UNITTESTS_ENABLED src_test_fuzz_liboss_fuzz_vrs_a_SOURCES = \ $(src_test_fuzz_fuzz_vrs_SOURCES) src_test_fuzz_liboss_fuzz_vrs_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) @@ -425,6 +453,7 @@ OSS_FUZZ_FUZZERS = \ src/test/fuzz/liboss-fuzz-iptsv2.a \ src/test/fuzz/liboss-fuzz-microdesc.a \ src/test/fuzz/liboss-fuzz-socks.a \ + src/test/fuzz/liboss-fuzz-strops.a \ src/test/fuzz/liboss-fuzz-vrs.a else diff --git a/src/test/include.am b/src/test/include.am index ecb7689579..4725e8cbaa 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -65,7 +65,7 @@ noinst_PROGRAMS+= \ src/test/test \ src/test/test-slow \ src/test/test-memwipe \ - src/test/test-child \ + src/test/test-process \ src/test/test_workqueue \ src/test/test-switch-id \ src/test/test-timers @@ -94,6 +94,7 @@ src_test_test_SOURCES += \ src/test/test_address.c \ src/test/test_address_set.c \ src/test/test_bridges.c \ + src/test/test_btrack.c \ src/test/test_buffers.c \ src/test/test_bwmgt.c \ src/test/test_cell_formats.c \ @@ -146,6 +147,7 @@ src_test_test_SOURCES += \ src/test/test_logging.c \ src/test/test_mainloop.c \ src/test/test_microdesc.c \ + src/test/test_netinfo.c \ src/test/test_nodelist.c \ src/test/test_oom.c \ src/test/test_oos.c \ @@ -153,6 +155,7 @@ src_test_test_SOURCES += \ src/test/test_pem.c \ src/test/test_periodic_event.c \ src/test/test_policy.c \ + src/test/test_process.c \ src/test/test_procmon.c \ src/test/test_proto_http.c \ src/test/test_proto_misc.c \ @@ -177,10 +180,12 @@ src_test_test_SOURCES += \ src/test/test_util.c \ src/test/test_util_format.c \ src/test/test_util_process.c \ + src/test/test_voting_flags.c \ src/test/test_voting_schedule.c \ src/test/test_x509.c \ src/test/test_helpers.c \ src/test/test_dns.c \ + src/test/test_parsecommon.c \ src/test/testing_common.c \ src/test/testing_rsakeys.c \ src/ext/tinytest.c @@ -200,7 +205,7 @@ if UNITTESTS_ENABLED src_test_test_slow_SOURCES += \ src/test/test_slow.c \ src/test/test_crypto_slow.c \ - src/test/test_util_slow.c \ + src/test/test_process_slow.c \ src/test/testing_common.c \ src/test/testing_rsakeys.c \ src/ext/tinytest.c diff --git a/src/test/test-child.c b/src/test/test-child.c deleted file mode 100644 index 14df1a9b76..0000000000 --- a/src/test/test-child.c +++ /dev/null @@ -1,61 +0,0 @@ -/* Copyright (c) 2011-2018, The Tor Project, Inc. */ -/* See LICENSE for licensing information */ - -#include "orconfig.h" -#include <stdio.h> -#ifdef _WIN32 -#define WINDOWS_LEAN_AND_MEAN -#include <windows.h> -#else -#include <unistd.h> -#endif /* defined(_WIN32) */ -#include <string.h> - -#ifdef _WIN32 -#define SLEEP(sec) Sleep((sec)*1000) -#else -#define SLEEP(sec) sleep(sec) -#endif - -/** Trivial test program which prints out its command line arguments so we can - * check if tor_spawn_background() works */ -int -main(int argc, char **argv) -{ - int i; - int delay = 1; - int fast = 0; - - if (argc > 1) { - if (!strcmp(argv[1], "--hang")) { - delay = 60; - } else if (!strcmp(argv[1], "--fast")) { - fast = 1; - delay = 0; - } - } - - fprintf(stdout, "OUT\n"); - fprintf(stderr, "ERR\n"); - for (i = 1; i < argc; i++) - fprintf(stdout, "%s\n", argv[i]); - if (!fast) - fprintf(stdout, "SLEEPING\n"); - /* We need to flush stdout so that test_util_spawn_background_partial_read() - succeed. Otherwise ReadFile() will get the entire output in one */ - // XXX: Can we make stdio flush on newline? - fflush(stdout); - if (!fast) - SLEEP(1); - fprintf(stdout, "DONE\n"); - fflush(stdout); - if (fast) - return 0; - - while (--delay) { - SLEEP(1); - } - - return 0; -} - diff --git a/src/test/test-process.c b/src/test/test-process.c new file mode 100644 index 0000000000..ec1b395002 --- /dev/null +++ b/src/test/test-process.c @@ -0,0 +1,85 @@ +/* Copyright (c) 2011-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#include <stdio.h> +#ifdef _WIN32 +#define WINDOWS_LEAN_AND_MEAN +#include <windows.h> +#else +#include <unistd.h> +#endif /* defined(_WIN32) */ +#include <string.h> +#include <stdlib.h> + +#ifdef _WIN32 +#define SLEEP(sec) Sleep((sec)*1000) +#else +#define SLEEP(sec) sleep(sec) +#endif + +/* Trivial test program to test process_t. */ +int +main(int argc, char **argv) +{ + /* Does our process get the right arguments? */ + for (int i = 0; i < argc; ++i) { + fprintf(stdout, "argv[%d] = '%s'\n", i, argv[i]); + fflush(stdout); + } + + /* Make sure our process got our environment variable. */ + fprintf(stdout, "Environment variable TOR_TEST_ENV = '%s'\n", + getenv("TOR_TEST_ENV")); + fflush(stdout); + + /* Test line handling on stdout and stderr. */ + fprintf(stdout, "Output on stdout\nThis is a new line\n"); + fflush(stdout); + + fprintf(stderr, "Output on stderr\nThis is a new line\n"); + fflush(stderr); + + fprintf(stdout, "Partial line on stdout ..."); + fflush(stdout); + + fprintf(stderr, "Partial line on stderr ..."); + fflush(stderr); + + SLEEP(2); + + fprintf(stdout, "end of partial line on stdout\n"); + fflush(stdout); + fprintf(stderr, "end of partial line on stderr\n"); + fflush(stderr); + + /* Echo input from stdin. */ + char buffer[1024]; + + int count = 0; + + while (fgets(buffer, sizeof(buffer), stdin)) { + /* Strip the newline. */ + size_t size = strlen(buffer); + + if (size >= 1 && buffer[size - 1] == '\n') { + buffer[size - 1] = '\0'; + --size; + } + + if (size >= 1 && buffer[size - 1] == '\r') { + buffer[size - 1] = '\0'; + --size; + } + + fprintf(stdout, "Read line from stdin: '%s'\n", buffer); + fflush(stdout); + + if (++count == 3) + break; + } + + fprintf(stdout, "We are done for here, thank you!\n"); + + return 0; +} diff --git a/src/test/test.c b/src/test/test.c index 70d91e3967..13e8c71709 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -37,7 +37,7 @@ #include "core/or/or.h" #include "lib/err/backtrace.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/or/circuitlist.h" #include "core/or/circuitstats.h" #include "lib/compress/compress.h" @@ -847,8 +847,8 @@ struct testgroup_t testgroups[] = { { "circuitbuild/", circuitbuild_tests }, { "circuitlist/", circuitlist_tests }, { "circuitmux/", circuitmux_tests }, - { "circuituse/", circuituse_tests }, { "circuitstats/", circuitstats_tests }, + { "circuituse/", circuituse_tests }, { "compat/libevent/", compat_libevent_tests }, { "config/", config_tests }, { "connection/", connection_tests }, @@ -857,6 +857,7 @@ struct testgroup_t testgroups[] = { { "consdiffmgr/", consdiffmgr_tests }, { "container/", container_tests }, { "control/", controller_tests }, + { "control/btrack/", btrack_tests }, { "control/event/", controller_event_tests }, { "crypto/", crypto_tests }, { "crypto/ope/", crypto_ope_tests }, @@ -865,37 +866,42 @@ struct testgroup_t testgroups[] = { #endif { "crypto/pem/", pem_tests }, { "dir/", dir_tests }, - { "dir_handle_get/", dir_handle_get_tests }, { "dir/md/", microdesc_tests }, - { "dir/voting-schedule/", voting_schedule_tests }, + { "dir/voting/flags/", voting_flags_tests }, + { "dir/voting/schedule/", voting_schedule_tests }, + { "dir_handle_get/", dir_handle_get_tests }, + { "dns/", dns_tests }, { "dos/", dos_tests }, { "entryconn/", entryconn_tests }, { "entrynodes/", entrynodes_tests }, - { "guardfraction/", guardfraction_tests }, { "extorport/", extorport_tests }, { "geoip/", geoip_tests }, - { "legacy_hs/", hs_tests }, + { "guardfraction/", guardfraction_tests }, { "hs_cache/", hs_cache }, { "hs_cell/", hs_cell_tests }, + { "hs_client/", hs_client_tests }, { "hs_common/", hs_common_tests }, { "hs_config/", hs_config_tests }, { "hs_control/", hs_control_tests }, { "hs_descriptor/", hs_descriptor }, + { "hs_intropoint/", hs_intropoint_tests }, { "hs_ntor/", hs_ntor_tests }, { "hs_service/", hs_service_tests }, - { "hs_client/", hs_client_tests }, - { "hs_intropoint/", hs_intropoint_tests }, { "introduce/", introduce_tests }, { "keypin/", keypin_tests }, + { "legacy_hs/", hs_tests }, { "link-handshake/", link_handshake_tests }, { "mainloop/", mainloop_tests }, + { "netinfo/", netinfo_tests }, { "nodelist/", nodelist_tests }, { "oom/", oom_tests }, { "oos/", oos_tests }, { "options/", options_tests }, + { "parsecommon/", parsecommon_tests }, { "periodic-event/" , periodic_event_tests }, { "policy/" , policy_tests }, { "procmon/", procmon_tests }, + { "process/", process_tests }, { "proto/http/", proto_http_tests }, { "proto/misc/", proto_misc_tests }, { "protover/", protover_tests }, @@ -910,8 +916,8 @@ struct testgroup_t testgroups[] = { { "routerlist/", routerlist_tests }, { "routerset/" , routerset_tests }, { "scheduler/", scheduler_tests }, - { "socks/", socks_tests }, { "shared-random/", sr_tests }, + { "socks/", socks_tests }, { "status/" , status_tests }, { "storagedir/", storagedir_tests }, { "tortls/", tortls_tests }, @@ -921,10 +927,9 @@ struct testgroup_t testgroups[] = { { "tortls/x509/", x509_tests }, { "util/", util_tests }, { "util/format/", util_format_tests }, + { "util/handle/", handle_tests }, { "util/logging/", logging_tests }, { "util/process/", util_process_tests }, { "util/thread/", thread_tests }, - { "util/handle/", handle_tests }, - { "dns/", dns_tests }, END_OF_GROUPS }; diff --git a/src/test/test.h b/src/test/test.h index a46fedf3e0..9f754469c8 100644 --- a/src/test/test.h +++ b/src/test/test.h @@ -177,11 +177,12 @@ extern const struct testcase_setup_t ed25519_test_setup; extern struct testcase_t accounting_tests[]; extern struct testcase_t addr_tests[]; -extern struct testcase_t address_tests[]; extern struct testcase_t address_set_tests[]; +extern struct testcase_t address_tests[]; extern struct testcase_t bridges_tests[]; -extern struct testcase_t bwmgt_tests[]; +extern struct testcase_t btrack_tests[]; extern struct testcase_t buffer_tests[]; +extern struct testcase_t bwmgt_tests[]; extern struct testcase_t cell_format_tests[]; extern struct testcase_t cell_queue_tests[]; extern struct testcase_t channel_tests[]; @@ -191,8 +192,8 @@ extern struct testcase_t checkdir_tests[]; extern struct testcase_t circuitbuild_tests[]; extern struct testcase_t circuitlist_tests[]; extern struct testcase_t circuitmux_tests[]; -extern struct testcase_t circuituse_tests[]; extern struct testcase_t circuitstats_tests[]; +extern struct testcase_t circuituse_tests[]; extern struct testcase_t compat_libevent_tests[]; extern struct testcase_t config_tests[]; extern struct testcase_t connection_tests[]; @@ -200,44 +201,49 @@ extern struct testcase_t conscache_tests[]; extern struct testcase_t consdiff_tests[]; extern struct testcase_t consdiffmgr_tests[]; extern struct testcase_t container_tests[]; -extern struct testcase_t controller_tests[]; extern struct testcase_t controller_event_tests[]; -extern struct testcase_t crypto_tests[]; +extern struct testcase_t controller_tests[]; extern struct testcase_t crypto_ope_tests[]; extern struct testcase_t crypto_openssl_tests[]; -extern struct testcase_t dir_tests[]; +extern struct testcase_t crypto_tests[]; extern struct testcase_t dir_handle_get_tests[]; +extern struct testcase_t dir_tests[]; +extern struct testcase_t dns_tests[]; extern struct testcase_t dos_tests[]; extern struct testcase_t entryconn_tests[]; extern struct testcase_t entrynodes_tests[]; -extern struct testcase_t guardfraction_tests[]; extern struct testcase_t extorport_tests[]; extern struct testcase_t geoip_tests[]; -extern struct testcase_t hs_tests[]; +extern struct testcase_t guardfraction_tests[]; +extern struct testcase_t handle_tests[]; extern struct testcase_t hs_cache[]; extern struct testcase_t hs_cell_tests[]; +extern struct testcase_t hs_client_tests[]; extern struct testcase_t hs_common_tests[]; extern struct testcase_t hs_config_tests[]; extern struct testcase_t hs_control_tests[]; extern struct testcase_t hs_descriptor[]; +extern struct testcase_t hs_intropoint_tests[]; extern struct testcase_t hs_ntor_tests[]; extern struct testcase_t hs_service_tests[]; -extern struct testcase_t hs_client_tests[]; -extern struct testcase_t hs_intropoint_tests[]; +extern struct testcase_t hs_tests[]; extern struct testcase_t introduce_tests[]; extern struct testcase_t keypin_tests[]; extern struct testcase_t link_handshake_tests[]; extern struct testcase_t logging_tests[]; extern struct testcase_t mainloop_tests[]; extern struct testcase_t microdesc_tests[]; +extern struct testcase_t netinfo_tests[]; extern struct testcase_t nodelist_tests[]; extern struct testcase_t oom_tests[]; extern struct testcase_t oos_tests[]; extern struct testcase_t options_tests[]; +extern struct testcase_t parsecommon_tests[]; extern struct testcase_t pem_tests[]; extern struct testcase_t periodic_event_tests[]; extern struct testcase_t policy_tests[]; extern struct testcase_t procmon_tests[]; +extern struct testcase_t process_tests[]; extern struct testcase_t proto_http_tests[]; extern struct testcase_t proto_misc_tests[]; extern struct testcase_t protover_tests[]; @@ -252,23 +258,22 @@ extern struct testcase_t routerkeys_tests[]; extern struct testcase_t routerlist_tests[]; extern struct testcase_t routerset_tests[]; extern struct testcase_t scheduler_tests[]; -extern struct testcase_t storagedir_tests[]; extern struct testcase_t socks_tests[]; +extern struct testcase_t sr_tests[]; extern struct testcase_t status_tests[]; +extern struct testcase_t storagedir_tests[]; extern struct testcase_t thread_tests[]; -extern struct testcase_t tortls_tests[]; extern struct testcase_t tortls_openssl_tests[]; -extern struct testcase_t util_tests[]; +extern struct testcase_t tortls_tests[]; extern struct testcase_t util_format_tests[]; extern struct testcase_t util_process_tests[]; +extern struct testcase_t util_tests[]; +extern struct testcase_t voting_flags_tests[]; extern struct testcase_t voting_schedule_tests[]; -extern struct testcase_t dns_tests[]; -extern struct testcase_t handle_tests[]; -extern struct testcase_t sr_tests[]; extern struct testcase_t x509_tests[]; extern struct testcase_t slow_crypto_tests[]; -extern struct testcase_t slow_util_tests[]; +extern struct testcase_t slow_process_tests[]; extern struct testgroup_t testgroups[]; diff --git a/src/test/test_addr.c b/src/test/test_addr.c index a9004048a5..1d97db52a6 100644 --- a/src/test/test_addr.c +++ b/src/test/test_addr.c @@ -723,7 +723,7 @@ test_addr_ip6_helpers(void *arg) ; } -/** Test tor_addr_port_parse(). */ +/** Test tor_addr_parse() and tor_addr_port_parse(). */ static void test_addr_parse(void *arg) { @@ -734,6 +734,60 @@ test_addr_parse(void *arg) /* Correct call. */ (void)arg; + r= tor_addr_parse(&addr, "192.0.2.1"); + tt_int_op(r,OP_EQ, AF_INET); + tor_addr_to_str(buf, &addr, sizeof(buf), 0); + tt_str_op(buf,OP_EQ, "192.0.2.1"); + + r= tor_addr_parse(&addr, "11:22::33:44"); + tt_int_op(r,OP_EQ, AF_INET6); + tor_addr_to_str(buf, &addr, sizeof(buf), 0); + tt_str_op(buf,OP_EQ, "11:22::33:44"); + + r= tor_addr_parse(&addr, "[11:22::33:44]"); + tt_int_op(r,OP_EQ, AF_INET6); + tor_addr_to_str(buf, &addr, sizeof(buf), 0); + tt_str_op(buf,OP_EQ, "11:22::33:44"); + + r= tor_addr_parse(&addr, "11:22:33:44:55:66:1.2.3.4"); + tt_int_op(r,OP_EQ, AF_INET6); + tor_addr_to_str(buf, &addr, sizeof(buf), 0); + tt_str_op(buf,OP_EQ, "11:22:33:44:55:66:102:304"); + + r= tor_addr_parse(&addr, "11:22::33:44:1.2.3.4"); + tt_int_op(r,OP_EQ, AF_INET6); + tor_addr_to_str(buf, &addr, sizeof(buf), 0); + tt_str_op(buf,OP_EQ, "11:22::33:44:102:304"); + + /* Empty string. */ + r= tor_addr_parse(&addr, ""); + tt_int_op(r,OP_EQ, -1); + + /* Square brackets around IPv4 address. */ + r= tor_addr_parse(&addr, "[192.0.2.1]"); + tt_int_op(r,OP_EQ, -1); + + /* Only left square bracket. */ + r= tor_addr_parse(&addr, "[11:22::33:44"); + tt_int_op(r,OP_EQ, -1); + + /* Only right square bracket. */ + r= tor_addr_parse(&addr, "11:22::33:44]"); + tt_int_op(r,OP_EQ, -1); + + /* Leading colon. */ + r= tor_addr_parse(&addr, ":11:22::33:44"); + tt_int_op(r,OP_EQ, -1); + + /* Trailing colon. */ + r= tor_addr_parse(&addr, "11:22::33:44:"); + tt_int_op(r,OP_EQ, -1); + + /* Too many hex words in IPv4-mapped IPv6 address. */ + r= tor_addr_parse(&addr, "11:22:33:44:55:66:77:88:1.2.3.4"); + tt_int_op(r,OP_EQ, -1); + + /* Correct call. */ r= tor_addr_port_parse(LOG_DEBUG, "192.0.2.1:1234", &addr, &port, -1); diff --git a/src/test/test_address.c b/src/test/test_address.c index e99220f838..a823fd9cd5 100644 --- a/src/test/test_address.c +++ b/src/test/test_address.c @@ -24,6 +24,8 @@ #endif /* defined(HAVE_IFCONF_TO_SMARTLIST) */ #include "core/or/or.h" +#include "feature/nodelist/routerinfo_st.h" +#include "feature/nodelist/node_st.h" #include "feature/nodelist/nodelist.h" #include "lib/net/address.h" #include "test/test.h" @@ -1170,6 +1172,78 @@ test_address_tor_addr_in_same_network_family(void *ignored) return; } +static node_t * +helper_create_mock_node(char id_char) +{ + node_t *node = tor_malloc_zero(sizeof(node_t)); + routerinfo_t *ri = tor_malloc_zero(sizeof(routerinfo_t)); + tor_addr_make_null(&ri->ipv6_addr, AF_INET6); + node->ri = ri; + memset(node->identity, id_char, sizeof(node->identity)); + return node; +} + +static void +helper_free_mock_node(node_t *node) +{ + if (!node) + return; + tor_free(node->ri); + tor_free(node); +} + +#define NODE_SET_IPV4(node, ipv4_addr, ipv4_port) { \ + tor_addr_t addr; \ + tor_addr_parse(&addr, ipv4_addr); \ + node->ri->addr = tor_addr_to_ipv4h(&addr); \ + node->ri->or_port = ipv4_port; \ + } + +#define NODE_CLEAR_IPV4(node) { \ + node->ri->addr = 0; \ + node->ri->or_port = 0; \ + } + +#define NODE_SET_IPV6(node, ipv6_addr_str, ipv6_port) { \ + tor_addr_parse(&node->ri->ipv6_addr, ipv6_addr_str); \ + node->ri->ipv6_orport = ipv6_port; \ + } + +static void +test_address_tor_node_in_same_network_family(void *ignored) +{ + (void)ignored; + node_t *node_a = helper_create_mock_node('a'); + node_t *node_b = helper_create_mock_node('b'); + + NODE_SET_IPV4(node_a, "8.8.8.8", 1); + NODE_SET_IPV4(node_b, "8.8.4.4", 1); + + tt_int_op(nodes_in_same_family(node_a, node_b), OP_EQ, 1); + + NODE_SET_IPV4(node_a, "8.8.8.8", 1); + NODE_SET_IPV4(node_b, "1.1.1.1", 1); + + tt_int_op(nodes_in_same_family(node_a, node_b), OP_EQ, 0); + + NODE_CLEAR_IPV4(node_a); + NODE_SET_IPV6(node_a, "2001:470:20::2", 1); + + tt_int_op(nodes_in_same_family(node_a, node_b), OP_EQ, 0); + + NODE_CLEAR_IPV4(node_b); + NODE_SET_IPV6(node_b, "2606:4700:4700::1111", 1); + + tt_int_op(nodes_in_same_family(node_a, node_b), OP_EQ, 0); + + NODE_SET_IPV6(node_a, "2606:4700:4700::1001", 1); + tt_int_op(nodes_in_same_family(node_a, node_b), OP_EQ, 1); + + done: + helper_free_mock_node(node_a); + helper_free_mock_node(node_b); +} + #define ADDRESS_TEST(name, flags) \ { #name, test_address_ ## name, flags, NULL, NULL } @@ -1202,6 +1276,6 @@ struct testcase_t address_tests[] = { ADDRESS_TEST(tor_addr_to_mapped_ipv4h, 0), ADDRESS_TEST(tor_addr_eq_ipv4h, 0), ADDRESS_TEST(tor_addr_in_same_network_family, 0), + ADDRESS_TEST(tor_node_in_same_network_family, 0), END_OF_TESTCASES }; - diff --git a/src/test/test_btrack.c b/src/test/test_btrack.c new file mode 100644 index 0000000000..7b5d108f98 --- /dev/null +++ b/src/test/test_btrack.c @@ -0,0 +1,100 @@ +/* Copyright (c) 2013-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "core/or/or.h" + +#include "test/test.h" +#include "test/log_test_helpers.h" + +#define OCIRC_EVENT_PRIVATE +#define ORCONN_EVENT_PRIVATE +#include "core/or/ocirc_event.h" +#include "core/or/orconn_event.h" + +static void +test_btrack_launch(void *arg) +{ + orconn_event_msg_t conn; + ocirc_event_msg_t circ; + + (void)arg; + conn.type = ORCONN_MSGTYPE_STATE; + conn.u.state.gid = 1; + conn.u.state.chan = 1; + conn.u.state.proxy_type = PROXY_NONE; + conn.u.state.state = OR_CONN_STATE_CONNECTING; + + setup_full_capture_of_logs(LOG_DEBUG); + orconn_event_publish(&conn); + expect_log_msg_containing("ORCONN gid=1 chan=1 proxy_type=0 state=1"); + expect_no_log_msg_containing("ORCONN BEST_"); + teardown_capture_of_logs(); + + circ.type = OCIRC_MSGTYPE_CHAN; + circ.u.chan.chan = 1; + circ.u.chan.onehop = true; + + setup_full_capture_of_logs(LOG_DEBUG); + ocirc_event_publish(&circ); + expect_log_msg_containing("ORCONN LAUNCH chan=1 onehop=1"); + expect_log_msg_containing("ORCONN BEST_ANY state -1->1 gid=1"); + teardown_capture_of_logs(); + + conn.u.state.gid = 2; + conn.u.state.chan = 2; + + setup_full_capture_of_logs(LOG_DEBUG); + orconn_event_publish(&conn); + expect_log_msg_containing("ORCONN gid=2 chan=2 proxy_type=0 state=1"); + expect_no_log_msg_containing("ORCONN BEST_"); + teardown_capture_of_logs(); + + circ.u.chan.chan = 2; + circ.u.chan.onehop = false; + + setup_full_capture_of_logs(LOG_DEBUG); + ocirc_event_publish(&circ); + expect_log_msg_containing("ORCONN LAUNCH chan=2 onehop=0"); + expect_log_msg_containing("ORCONN BEST_AP state -1->1 gid=2"); + teardown_capture_of_logs(); + + done: + ; +} + +static void +test_btrack_delete(void *arg) +{ + orconn_event_msg_t conn; + + (void)arg; + conn.type = ORCONN_MSGTYPE_STATE; + conn.u.state.gid = 1; + conn.u.state.chan = 1; + conn.u.state.proxy_type = PROXY_NONE; + conn.u.state.state = OR_CONN_STATE_CONNECTING; + + setup_full_capture_of_logs(LOG_DEBUG); + orconn_event_publish(&conn); + expect_log_msg_containing("ORCONN gid=1 chan=1 proxy_type=0"); + teardown_capture_of_logs(); + + conn.type = ORCONN_MSGTYPE_STATUS; + conn.u.status.gid = 1; + conn.u.status.status = OR_CONN_EVENT_CLOSED; + conn.u.status.reason = 0; + + setup_full_capture_of_logs(LOG_DEBUG); + orconn_event_publish(&conn); + expect_log_msg_containing("ORCONN DELETE gid=1 status=3 reason=0"); + teardown_capture_of_logs(); + + done: + ; +} + +struct testcase_t btrack_tests[] = { + { "launch", test_btrack_launch, TT_FORK, 0, NULL }, + { "delete", test_btrack_delete, TT_FORK, 0, NULL }, + END_OF_TESTCASES +}; diff --git a/src/test/test_buffers.c b/src/test/test_buffers.c index 477066f699..85e7b8d90a 100644 --- a/src/test/test_buffers.c +++ b/src/test/test_buffers.c @@ -6,7 +6,7 @@ #define BUFFERS_PRIVATE #define PROTO_HTTP_PRIVATE #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/tls/buffers_tls.h" #include "lib/tls/tortls.h" #include "lib/compress/compress.h" diff --git a/src/test/test_channelpadding.c b/src/test/test_channelpadding.c index 0fd60d0a92..bdd7c5f0a6 100644 --- a/src/test/test_channelpadding.c +++ b/src/test/test_channelpadding.c @@ -21,7 +21,7 @@ #include "test/log_test_helpers.h" #include "lib/tls/tortls.h" #include "lib/evloop/timers.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/or/cell_st.h" #include "feature/nodelist/networkstatus_st.h" diff --git a/src/test/test_channeltls.c b/src/test/test_channeltls.c index 787a30a85d..44d623561b 100644 --- a/src/test/test_channeltls.c +++ b/src/test/test_channeltls.c @@ -8,7 +8,7 @@ #define TOR_CHANNEL_INTERNAL_ #include "core/or/or.h" #include "lib/net/address.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/or/channel.h" #include "core/or/channeltls.h" #include "core/mainloop/connection.h" diff --git a/src/test/test_circuitbuild.c b/src/test/test_circuitbuild.c index 02eadecd98..dd47ad7689 100644 --- a/src/test/test_circuitbuild.c +++ b/src/test/test_circuitbuild.c @@ -21,11 +21,11 @@ static smartlist_t dummy_nodes; static extend_info_t dummy_ei; static int -mock_count_acceptable_nodes(smartlist_t *nodes) +mock_count_acceptable_nodes(smartlist_t *nodes, int direct) { (void)nodes; - return DEFAULT_ROUTE_LEN + 1; + return direct ? 1 : DEFAULT_ROUTE_LEN + 1; } /* Test route lengths when the caller of new_route_len() doesn't diff --git a/src/test/test_compat_libevent.c b/src/test/test_compat_libevent.c index 3f505d013b..ade76bdb07 100644 --- a/src/test/test_compat_libevent.c +++ b/src/test/test_compat_libevent.c @@ -187,4 +187,3 @@ struct testcase_t compat_libevent_tests[] = { TT_FORK, NULL, NULL }, END_OF_TESTCASES }; - diff --git a/src/test/test_config.c b/src/test/test_config.c index dae4d83766..67a43d669e 100644 --- a/src/test/test_config.c +++ b/src/test/test_config.c @@ -54,6 +54,7 @@ #include "lib/meminfo/meminfo.h" #include "lib/net/gethostname.h" #include "lib/encoding/confline.h" +#include "lib/encoding/kvline.h" #ifdef HAVE_UNISTD_H #include <unistd.h> @@ -5782,6 +5783,7 @@ test_config_extended_fmt(void *arg) tt_str_op(lp->value, OP_EQ, "is back here"); tt_int_op(lp->command, OP_EQ, CONFIG_LINE_NORMAL); lp = lp->next; + tt_assert(!lp); config_free_lines(lines); /* Try with the "extended" flag enabled. */ @@ -5808,11 +5810,88 @@ test_config_extended_fmt(void *arg) tt_str_op(lp->value, OP_EQ, ""); tt_int_op(lp->command, OP_EQ, CONFIG_LINE_CLEAR); lp = lp->next; + tt_assert(!lp); done: config_free_lines(lines); } +static void +test_config_kvline_parse(void *arg) +{ + (void)arg; + + config_line_t *lines = NULL; + char *enc = NULL; + + lines = kvline_parse("A=B CD=EF", 0); + tt_assert(lines); + tt_str_op(lines->key, OP_EQ, "A"); + tt_str_op(lines->value, OP_EQ, "B"); + tt_str_op(lines->next->key, OP_EQ, "CD"); + tt_str_op(lines->next->value, OP_EQ, "EF"); + enc = kvline_encode(lines, 0); + tt_str_op(enc, OP_EQ, "A=B CD=EF"); + tor_free(enc); + enc = kvline_encode(lines, KV_QUOTED|KV_OMIT_KEYS); + tt_str_op(enc, OP_EQ, "A=B CD=EF"); + tor_free(enc); + config_free_lines(lines); + + lines = kvline_parse("AB CDE=F", 0); + tt_assert(! lines); + + lines = kvline_parse("AB CDE=F", KV_OMIT_KEYS); + tt_assert(lines); + tt_str_op(lines->key, OP_EQ, ""); + tt_str_op(lines->value, OP_EQ, "AB"); + tt_str_op(lines->next->key, OP_EQ, "CDE"); + tt_str_op(lines->next->value, OP_EQ, "F"); + tt_assert(lines); + enc = kvline_encode(lines, 0); + tt_assert(!enc); + enc = kvline_encode(lines, KV_QUOTED|KV_OMIT_KEYS); + tt_str_op(enc, OP_EQ, "AB CDE=F"); + tor_free(enc); + config_free_lines(lines); + + lines = kvline_parse("AB=C CDE=\"F G\"", 0); + tt_assert(!lines); + + lines = kvline_parse("AB=C CDE=\"F G\" \"GHI\" ", KV_QUOTED|KV_OMIT_KEYS); + tt_assert(lines); + tt_str_op(lines->key, OP_EQ, "AB"); + tt_str_op(lines->value, OP_EQ, "C"); + tt_str_op(lines->next->key, OP_EQ, "CDE"); + tt_str_op(lines->next->value, OP_EQ, "F G"); + tt_str_op(lines->next->next->key, OP_EQ, ""); + tt_str_op(lines->next->next->value, OP_EQ, "GHI"); + enc = kvline_encode(lines, 0); + tt_assert(!enc); + enc = kvline_encode(lines, KV_QUOTED|KV_OMIT_KEYS); + tt_str_op(enc, OP_EQ, "AB=C CDE=\"F G\" GHI"); + tor_free(enc); + config_free_lines(lines); + + lines = kvline_parse("A\"B=C CDE=\"F\" \"GHI\" ", KV_QUOTED|KV_OMIT_KEYS); + tt_assert(! lines); + + lines = kvline_parse("AB=", KV_QUOTED); + tt_assert(lines); + tt_str_op(lines->key, OP_EQ, "AB"); + tt_str_op(lines->value, OP_EQ, ""); + config_free_lines(lines); + + lines = kvline_parse("AB=", 0); + tt_assert(lines); + tt_str_op(lines->key, OP_EQ, "AB"); + tt_str_op(lines->value, OP_EQ, ""); + + done: + config_free_lines(lines); + tor_free(enc); +} + #define CONFIG_TEST(name, flags) \ { #name, test_config_ ## name, flags, NULL, NULL } @@ -5864,5 +5943,6 @@ struct testcase_t config_tests[] = { CONFIG_TEST(include_opened_file_list, 0), CONFIG_TEST(compute_max_mem_in_queues, 0), CONFIG_TEST(extended_fmt, 0), + CONFIG_TEST(kvline_parse, 0), END_OF_TESTCASES }; diff --git a/src/test/test_consdiff.c b/src/test/test_consdiff.c index b836befd22..23e8f7167e 100644 --- a/src/test/test_consdiff.c +++ b/src/test/test_consdiff.c @@ -14,6 +14,39 @@ #define tt_str_eq_line(a,b) \ tt_assert(line_str_eq((b),(a))) +static int +consensus_split_lines_(smartlist_t *out, const char *s, memarea_t *area) +{ + size_t len = strlen(s); + return consensus_split_lines(out, s, len, area); +} + +static int +consensus_compute_digest_(const char *cons, + consensus_digest_t *digest_out) +{ + size_t len = strlen(cons); + char *tmp = tor_memdup(cons, len); + // We use memdup here to ensure that the input is NOT nul-terminated. + // This makes it likelier for us to spot bugs. + int r = consensus_compute_digest(tmp, len, digest_out); + tor_free(tmp); + return r; +} + +static int +consensus_compute_digest_as_signed_(const char *cons, + consensus_digest_t *digest_out) +{ + size_t len = strlen(cons); + char *tmp = tor_memdup(cons, len); + // We use memdup here to ensure that the input is NOT nul-terminated. + // This makes it likelier for us to spot bugs. + int r = consensus_compute_digest_as_signed(tmp, len, digest_out); + tor_free(tmp); + return r; +} + static void test_consdiff_smartlist_slice(void *arg) { @@ -58,7 +91,7 @@ test_consdiff_smartlist_slice_string_pos(void *arg) /* Create a regular smartlist. */ (void)arg; - consensus_split_lines(sl, "a\nd\nc\na\nb\n", area); + consensus_split_lines_(sl, "a\nd\nc\na\nb\n", area); /* See that smartlist_slice_string_pos respects the bounds of the slice. */ sls = smartlist_slice(sl, 2, 5); @@ -87,8 +120,8 @@ test_consdiff_lcs_lengths(void *arg) int e_lengths2[] = { 0, 1, 1, 2, 3, 4 }; (void)arg; - consensus_split_lines(sl1, "a\nb\nc\nd\ne\n", area); - consensus_split_lines(sl2, "a\nc\nd\ni\ne\n", area); + consensus_split_lines_(sl1, "a\nb\nc\nd\ne\n", area); + consensus_split_lines_(sl2, "a\nc\nd\ni\ne\n", area); sls1 = smartlist_slice(sl1, 0, -1); sls2 = smartlist_slice(sl2, 0, -1); @@ -119,10 +152,10 @@ test_consdiff_trim_slices(void *arg) memarea_t *area = memarea_new(); (void)arg; - consensus_split_lines(sl1, "a\nb\nb\nb\nd\n", area); - consensus_split_lines(sl2, "a\nc\nc\nc\nd\n", area); - consensus_split_lines(sl3, "a\nb\nb\nb\na\n", area); - consensus_split_lines(sl4, "c\nb\nb\nb\nc\n", area); + consensus_split_lines_(sl1, "a\nb\nb\nb\nd\n", area); + consensus_split_lines_(sl2, "a\nc\nc\nc\nd\n", area); + consensus_split_lines_(sl3, "a\nb\nb\nb\na\n", area); + consensus_split_lines_(sl4, "c\nb\nb\nb\nc\n", area); sls1 = smartlist_slice(sl1, 0, -1); sls2 = smartlist_slice(sl2, 0, -1); sls3 = smartlist_slice(sl3, 0, -1); @@ -165,8 +198,8 @@ test_consdiff_set_changed(void *arg) memarea_t *area = memarea_new(); (void)arg; - consensus_split_lines(sl1, "a\nb\na\na\n", area); - consensus_split_lines(sl2, "a\na\na\na\n", area); + consensus_split_lines_(sl1, "a\nb\na\na\n", area); + consensus_split_lines_(sl2, "a\na\na\na\n", area); /* Length of sls1 is 0. */ sls1 = smartlist_slice(sl1, 0, 0); @@ -240,8 +273,8 @@ test_consdiff_calc_changes(void *arg) memarea_t *area = memarea_new(); (void)arg; - consensus_split_lines(sl1, "a\na\na\na\n", area); - consensus_split_lines(sl2, "a\na\na\na\n", area); + consensus_split_lines_(sl1, "a\na\na\na\n", area); + consensus_split_lines_(sl2, "a\na\na\na\n", area); sls1 = smartlist_slice(sl1, 0, -1); sls2 = smartlist_slice(sl2, 0, -1); @@ -259,7 +292,7 @@ test_consdiff_calc_changes(void *arg) tt_assert(!bitarray_is_set(changed2, 3)); smartlist_clear(sl2); - consensus_split_lines(sl2, "a\nb\na\nb\n", area); + consensus_split_lines_(sl2, "a\nb\na\nb\n", area); tor_free(sls1); tor_free(sls2); sls1 = smartlist_slice(sl1, 0, -1); @@ -282,7 +315,7 @@ test_consdiff_calc_changes(void *arg) bitarray_clear(changed1, 3); smartlist_clear(sl2); - consensus_split_lines(sl2, "b\nb\nb\nb\n", area); + consensus_split_lines_(sl2, "b\nb\nb\nb\n", area); tor_free(sls1); tor_free(sls2); sls1 = smartlist_slice(sl1, 0, -1); @@ -610,8 +643,8 @@ test_consdiff_gen_ed_diff(void *arg) /* Test 'a', 'c' and 'd' together. See that it is done in reverse order. */ smartlist_clear(cons1); smartlist_clear(cons2); - consensus_split_lines(cons1, "A\nB\nC\nD\nE\n", area); - consensus_split_lines(cons2, "A\nC\nO\nE\nU\n", area); + consensus_split_lines_(cons1, "A\nB\nC\nD\nE\n", area); + consensus_split_lines_(cons2, "A\nC\nO\nE\nU\n", area); diff = gen_ed_diff(cons1, cons2, area); tt_ptr_op(NULL, OP_NE, diff); tt_int_op(7, OP_EQ, smartlist_len(diff)); @@ -627,8 +660,8 @@ test_consdiff_gen_ed_diff(void *arg) smartlist_clear(cons1); smartlist_clear(cons2); - consensus_split_lines(cons1, "B\n", area); - consensus_split_lines(cons2, "A\nB\n", area); + consensus_split_lines_(cons1, "B\n", area); + consensus_split_lines_(cons2, "A\nB\n", area); diff = gen_ed_diff(cons1, cons2, area); tt_ptr_op(NULL, OP_NE, diff); tt_int_op(3, OP_EQ, smartlist_len(diff)); @@ -656,7 +689,7 @@ test_consdiff_apply_ed_diff(void *arg) diff = smartlist_new(); setup_capture_of_logs(LOG_WARN); - consensus_split_lines(cons1, "A\nB\nC\nD\nE\n", area); + consensus_split_lines_(cons1, "A\nB\nC\nD\nE\n", area); /* Command without range. */ smartlist_add_linecpy(diff, area, "a"); @@ -829,7 +862,7 @@ test_consdiff_apply_ed_diff(void *arg) smartlist_clear(diff); /* Test appending text, 'a'. */ - consensus_split_lines(diff, "3a\nU\nO\n.\n0a\nV\n.\n", area); + consensus_split_lines_(diff, "3a\nU\nO\n.\n0a\nV\n.\n", area); cons2 = apply_ed_diff(cons1, diff, 0); tt_ptr_op(NULL, OP_NE, cons2); tt_int_op(8, OP_EQ, smartlist_len(cons2)); @@ -846,7 +879,7 @@ test_consdiff_apply_ed_diff(void *arg) smartlist_free(cons2); /* Test deleting text, 'd'. */ - consensus_split_lines(diff, "4d\n1,2d\n", area); + consensus_split_lines_(diff, "4d\n1,2d\n", area); cons2 = apply_ed_diff(cons1, diff, 0); tt_ptr_op(NULL, OP_NE, cons2); tt_int_op(2, OP_EQ, smartlist_len(cons2)); @@ -857,7 +890,7 @@ test_consdiff_apply_ed_diff(void *arg) smartlist_free(cons2); /* Test changing text, 'c'. */ - consensus_split_lines(diff, "4c\nT\nX\n.\n1,2c\nM\n.\n", area); + consensus_split_lines_(diff, "4c\nT\nX\n.\n1,2c\nM\n.\n", area); cons2 = apply_ed_diff(cons1, diff, 0); tt_ptr_op(NULL, OP_NE, cons2); tt_int_op(5, OP_EQ, smartlist_len(cons2)); @@ -871,7 +904,7 @@ test_consdiff_apply_ed_diff(void *arg) smartlist_free(cons2); /* Test 'a', 'd' and 'c' together. */ - consensus_split_lines(diff, "4c\nT\nX\n.\n2d\n0a\nM\n.\n", area); + consensus_split_lines_(diff, "4c\nT\nX\n.\n2d\n0a\nM\n.\n", area); cons2 = apply_ed_diff(cons1, diff, 0); tt_ptr_op(NULL, OP_NE, cons2); tt_int_op(6, OP_EQ, smartlist_len(cons2)); @@ -918,12 +951,12 @@ test_consdiff_gen_diff(void *arg) ); tt_int_op(0, OP_EQ, - consensus_compute_digest_as_signed(cons1_str, &digests1)); + consensus_compute_digest_as_signed_(cons1_str, &digests1)); tt_int_op(0, OP_EQ, - consensus_compute_digest(cons2_str, &digests2)); + consensus_compute_digest_(cons2_str, &digests2)); - consensus_split_lines(cons1, cons1_str, area); - consensus_split_lines(cons2, cons2_str, area); + consensus_split_lines_(cons1, cons1_str, area); + consensus_split_lines_(cons2, cons2_str, area); diff = consdiff_gen_diff(cons1, cons2, &digests1, &digests2, area); tt_ptr_op(NULL, OP_EQ, diff); @@ -937,9 +970,9 @@ test_consdiff_gen_diff(void *arg) "directory-signature foo bar\nbar\n" ); tt_int_op(0, OP_EQ, - consensus_compute_digest_as_signed(cons1_str, &digests1)); + consensus_compute_digest_as_signed_(cons1_str, &digests1)); smartlist_clear(cons1); - consensus_split_lines(cons1, cons1_str, area); + consensus_split_lines_(cons1, cons1_str, area); diff = consdiff_gen_diff(cons1, cons2, &digests1, &digests2, area); tt_ptr_op(NULL, OP_NE, diff); tt_int_op(11, OP_EQ, smartlist_len(diff)); @@ -991,8 +1024,8 @@ test_consdiff_apply_diff(void *arg) "directory-signature foo bar\nbar\n" ); tt_int_op(0, OP_EQ, - consensus_compute_digest(cons1_str, &digests1)); - consensus_split_lines(cons1, cons1_str, area); + consensus_compute_digest_(cons1_str, &digests1)); + consensus_split_lines_(cons1, cons1_str, area); /* diff doesn't have enough lines. */ cons2 = consdiff_apply_diff(cons1, diff, &digests1); @@ -1182,4 +1215,3 @@ struct testcase_t consdiff_tests[] = { CONSDIFF_LEGACY(apply_diff), END_OF_TESTCASES }; - diff --git a/src/test/test_consdiffmgr.c b/src/test/test_consdiffmgr.c index 6c451da685..b84753ff83 100644 --- a/src/test/test_consdiffmgr.c +++ b/src/test/test_consdiffmgr.c @@ -21,6 +21,23 @@ #include "test/test.h" #include "test/log_test_helpers.h" +#define consdiffmgr_add_consensus consdiffmgr_add_consensus_nulterm + +static char * +consensus_diff_apply_(const char *c, const char *d) +{ + size_t c_len = strlen(c); + size_t d_len = strlen(d); + // We use memdup here to ensure that the input is NOT nul-terminated. + // This makes it likelier for us to spot bugs. + char *c_tmp = tor_memdup(c, c_len); + char *d_tmp = tor_memdup(d, d_len); + char *result = consensus_diff_apply(c_tmp, c_len, d_tmp, d_len); + tor_free(c_tmp); + tor_free(d_tmp); + return result; +} + // ============================== Setup/teardown the consdiffmgr // These functions get run before/after each test in this module @@ -153,7 +170,8 @@ lookup_diff_from(consensus_cache_entry_t **out, const char *str1) { uint8_t digest[DIGEST256_LEN]; - if (router_get_networkstatus_v3_sha3_as_signed(digest, str1)<0) { + if (router_get_networkstatus_v3_sha3_as_signed(digest, + str1, strlen(str1))<0) { TT_FAIL(("Unable to compute sha3-as-signed")); return CONSDIFF_NOT_FOUND; } @@ -175,14 +193,15 @@ lookup_apply_and_verify_diff(consensus_flavor_t flav, consensus_cache_entry_incref(ent); size_t size; - char *diff_string = NULL; - int r = uncompress_or_copy(&diff_string, &size, ent); + const char *diff_string = NULL; + char *diff_owned = NULL; + int r = uncompress_or_set_ptr(&diff_string, &size, &diff_owned, ent); consensus_cache_entry_decref(ent); if (diff_string == NULL || r < 0) return -1; - char *applied = consensus_diff_apply(str1, diff_string); - tor_free(diff_string); + char *applied = consensus_diff_apply(str1, strlen(str1), diff_string, size); + tor_free(diff_owned); if (applied == NULL) return -1; @@ -282,7 +301,8 @@ test_consdiffmgr_add(void *arg) (void) arg; time_t now = approx_time(); - char *body = NULL; + const char *body = NULL; + char *body_owned = NULL; consensus_cache_entry_t *ent = NULL; networkstatus_t *ns_tmp = fake_ns_new(FLAV_NS, now); @@ -324,7 +344,7 @@ test_consdiffmgr_add(void *arg) tt_assert(ent); consensus_cache_entry_incref(ent); size_t s; - r = uncompress_or_copy(&body, &s, ent); + r = uncompress_or_set_ptr(&body, &s, &body_owned, ent); tt_int_op(r, OP_EQ, 0); tt_int_op(s, OP_EQ, 4); tt_mem_op(body, OP_EQ, "quux", 4); @@ -337,7 +357,7 @@ test_consdiffmgr_add(void *arg) networkstatus_vote_free(ns_tmp); teardown_capture_of_logs(); consensus_cache_entry_decref(ent); - tor_free(body); + tor_free(body_owned); } static void @@ -370,7 +390,8 @@ test_consdiffmgr_make_diffs(void *arg) ns = fake_ns_new(FLAV_MICRODESC, now-3600); md_ns_body = fake_ns_body_new(FLAV_MICRODESC, now-3600); r = consdiffmgr_add_consensus(md_ns_body, ns); - router_get_networkstatus_v3_sha3_as_signed(md_ns_sha3, md_ns_body); + router_get_networkstatus_v3_sha3_as_signed(md_ns_sha3, md_ns_body, + strlen(md_ns_body)); networkstatus_vote_free(ns); tt_int_op(r, OP_EQ, 0); @@ -414,7 +435,7 @@ test_consdiffmgr_make_diffs(void *arg) r = consensus_cache_entry_get_body(diff, &diff_body, &diff_size); tt_int_op(r, OP_EQ, 0); diff_text = tor_memdup_nulterm(diff_body, diff_size); - applied = consensus_diff_apply(md_ns_body, diff_text); + applied = consensus_diff_apply_(md_ns_body, diff_text); tt_assert(applied); tt_str_op(applied, OP_EQ, md_ns_body_2); diff --git a/src/test/test_controller_events.c b/src/test/test_controller_events.c index 4c404876b0..99e1eb7cb0 100644 --- a/src/test/test_controller_events.c +++ b/src/test/test_controller_events.c @@ -4,10 +4,14 @@ #define CONNECTION_PRIVATE #define TOR_CHANNEL_INTERNAL_ #define CONTROL_PRIVATE +#define OCIRC_EVENT_PRIVATE +#define ORCONN_EVENT_PRIVATE #include "core/or/or.h" #include "core/or/channel.h" #include "core/or/channeltls.h" #include "core/or/circuitlist.h" +#include "core/or/ocirc_event.h" +#include "core/or/orconn_event.h" #include "core/mainloop/connection.h" #include "feature/control/control.h" #include "test/test.h" @@ -351,10 +355,10 @@ test_cntev_dirboot_defer_desc(void *arg) /* This event should get deferred */ control_event_boot_dir(BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS, 0); assert_bootmsg("0 TAG=starting"); - control_event_bootstrap(BOOTSTRAP_STATUS_CONN_DIR, 0); - assert_bootmsg("5 TAG=conn_dir"); + control_event_bootstrap(BOOTSTRAP_STATUS_CONN, 0); + assert_bootmsg("5 TAG=conn"); control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE, 0); - assert_bootmsg("10 TAG=handshake_dir"); + assert_bootmsg("14 TAG=handshake"); /* The deferred event should appear */ control_event_boot_first_orconn(); assert_bootmsg("45 TAG=requesting_descriptors"); @@ -374,21 +378,159 @@ test_cntev_dirboot_defer_orconn(void *arg) control_event_bootstrap(BOOTSTRAP_STATUS_STARTING, 0); assert_bootmsg("0 TAG=starting"); /* This event should get deferred */ - control_event_boot_dir(BOOTSTRAP_STATUS_CONN_OR, 0); + control_event_boot_dir(BOOTSTRAP_STATUS_ENOUGH_DIRINFO, 0); assert_bootmsg("0 TAG=starting"); - control_event_bootstrap(BOOTSTRAP_STATUS_CONN_DIR, 0); - assert_bootmsg("5 TAG=conn_dir"); + control_event_bootstrap(BOOTSTRAP_STATUS_CONN, 0); + assert_bootmsg("5 TAG=conn"); control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE, 0); - assert_bootmsg("10 TAG=handshake_dir"); + assert_bootmsg("14 TAG=handshake"); /* The deferred event should appear */ control_event_boot_first_orconn(); - assert_bootmsg("80 TAG=conn_or"); + assert_bootmsg("75 TAG=enough_dirinfo"); done: tor_free(saved_event_str); UNMOCK(queue_control_event_string); } -#define TEST(name, flags) \ +static void +setup_orconn_state(orconn_event_msg_t *msg, uint64_t gid, uint64_t chan, + int proxy_type) +{ + msg->type = ORCONN_MSGTYPE_STATE; + msg->u.state.gid = gid; + msg->u.state.chan = chan; + msg->u.state.proxy_type = proxy_type; +} + +static void +send_orconn_state(orconn_event_msg_t *msg, uint8_t state) +{ + msg->u.state.state = state; + orconn_event_publish(msg); +} + +static void +send_ocirc_chan(uint32_t gid, uint64_t chan, bool onehop) +{ + ocirc_event_msg_t msg; + + msg.type = OCIRC_MSGTYPE_CHAN; + msg.u.chan.gid = gid; + msg.u.chan.chan = chan; + msg.u.chan.onehop = onehop; + ocirc_event_publish(&msg); +} + +static void +test_cntev_orconn_state(void *arg) +{ + orconn_event_msg_t conn; + + (void)arg; + MOCK(queue_control_event_string, mock_queue_control_event_string); + control_testing_set_global_event_mask(EVENT_MASK_(EVENT_STATUS_CLIENT)); + setup_orconn_state(&conn, 1, 1, PROXY_NONE); + + send_orconn_state(&conn, OR_CONN_STATE_CONNECTING); + send_ocirc_chan(1, 1, true); + assert_bootmsg("5 TAG=conn"); + send_orconn_state(&conn, OR_CONN_STATE_TLS_HANDSHAKING); + assert_bootmsg("10 TAG=conn_done"); + send_orconn_state(&conn, OR_CONN_STATE_OR_HANDSHAKING_V3); + assert_bootmsg("14 TAG=handshake"); + send_orconn_state(&conn, OR_CONN_STATE_OPEN); + assert_bootmsg("15 TAG=handshake_done"); + + conn.u.state.gid = 2; + conn.u.state.chan = 2; + send_orconn_state(&conn, OR_CONN_STATE_CONNECTING); + /* It doesn't know it's an origin circuit yet */ + assert_bootmsg("15 TAG=handshake_done"); + send_ocirc_chan(2, 2, false); + assert_bootmsg("80 TAG=ap_conn"); + send_orconn_state(&conn, OR_CONN_STATE_TLS_HANDSHAKING); + assert_bootmsg("85 TAG=ap_conn_done"); + send_orconn_state(&conn, OR_CONN_STATE_OR_HANDSHAKING_V3); + assert_bootmsg("89 TAG=ap_handshake"); + send_orconn_state(&conn, OR_CONN_STATE_OPEN); + assert_bootmsg("90 TAG=ap_handshake_done"); + + done: + tor_free(saved_event_str); + UNMOCK(queue_control_event_string); +} + +static void +test_cntev_orconn_state_pt(void *arg) +{ + orconn_event_msg_t conn; + + (void)arg; + MOCK(queue_control_event_string, mock_queue_control_event_string); + control_testing_set_global_event_mask(EVENT_MASK_(EVENT_STATUS_CLIENT)); + setup_orconn_state(&conn, 1, 1, PROXY_PLUGGABLE); + send_ocirc_chan(1, 1, true); + + send_orconn_state(&conn, OR_CONN_STATE_CONNECTING); + assert_bootmsg("1 TAG=conn_pt"); + send_orconn_state(&conn, OR_CONN_STATE_PROXY_HANDSHAKING); + assert_bootmsg("2 TAG=conn_done_pt"); + send_orconn_state(&conn, OR_CONN_STATE_TLS_HANDSHAKING); + assert_bootmsg("10 TAG=conn_done"); + send_orconn_state(&conn, OR_CONN_STATE_OR_HANDSHAKING_V3); + assert_bootmsg("14 TAG=handshake"); + send_orconn_state(&conn, OR_CONN_STATE_OPEN); + assert_bootmsg("15 TAG=handshake_done"); + + send_ocirc_chan(2, 2, false); + conn.u.state.gid = 2; + conn.u.state.chan = 2; + send_orconn_state(&conn, OR_CONN_STATE_CONNECTING); + assert_bootmsg("76 TAG=ap_conn_pt"); + send_orconn_state(&conn, OR_CONN_STATE_PROXY_HANDSHAKING); + assert_bootmsg("77 TAG=ap_conn_done_pt"); + + done: + tor_free(saved_event_str); + UNMOCK(queue_control_event_string); +} + +static void +test_cntev_orconn_state_proxy(void *arg) +{ + orconn_event_msg_t conn; + + (void)arg; + MOCK(queue_control_event_string, mock_queue_control_event_string); + control_testing_set_global_event_mask(EVENT_MASK_(EVENT_STATUS_CLIENT)); + setup_orconn_state(&conn, 1, 1, PROXY_CONNECT); + send_ocirc_chan(1, 1, true); + + send_orconn_state(&conn, OR_CONN_STATE_CONNECTING); + assert_bootmsg("3 TAG=conn_proxy"); + send_orconn_state(&conn, OR_CONN_STATE_PROXY_HANDSHAKING); + assert_bootmsg("4 TAG=conn_done_proxy"); + send_orconn_state(&conn, OR_CONN_STATE_TLS_HANDSHAKING); + assert_bootmsg("10 TAG=conn_done"); + send_orconn_state(&conn, OR_CONN_STATE_OR_HANDSHAKING_V3); + assert_bootmsg("14 TAG=handshake"); + send_orconn_state(&conn, OR_CONN_STATE_OPEN); + assert_bootmsg("15 TAG=handshake_done"); + + send_ocirc_chan(2, 2, false); + conn.u.state.gid = 2; + conn.u.state.chan = 2; + send_orconn_state(&conn, OR_CONN_STATE_CONNECTING); + assert_bootmsg("78 TAG=ap_conn_proxy"); + send_orconn_state(&conn, OR_CONN_STATE_PROXY_HANDSHAKING); + assert_bootmsg("79 TAG=ap_conn_done_proxy"); + + done: + tor_free(saved_event_str); + UNMOCK(queue_control_event_string); +} + +#define TEST(name, flags) \ { #name, test_cntev_ ## name, flags, 0, NULL } struct testcase_t controller_event_tests[] = { @@ -398,5 +540,8 @@ struct testcase_t controller_event_tests[] = { TEST(event_mask, TT_FORK), TEST(dirboot_defer_desc, TT_FORK), TEST(dirboot_defer_orconn, TT_FORK), + TEST(orconn_state, TT_FORK), + TEST(orconn_state_pt, TT_FORK), + TEST(orconn_state_proxy, TT_FORK), END_OF_TESTCASES }; diff --git a/src/test/test_dir.c b/src/test/test_dir.c index c18aa99fea..5cdbd877ce 100644 --- a/src/test/test_dir.c +++ b/src/test/test_dir.c @@ -94,6 +94,23 @@ #define NS_MODULE dir +static networkstatus_t * +networkstatus_parse_vote_from_string_(const char *s, + const char **eos_out, + enum networkstatus_type_t ns_type) +{ + size_t len = strlen(s); + // memdup so that it won't be nul-terminated. + char *tmp = tor_memdup(s, len); + networkstatus_t *result = + networkstatus_parse_vote_from_string(tmp, len, eos_out, ns_type); + if (eos_out && *eos_out) { + *eos_out = s + (*eos_out - tmp); + } + tor_free(tmp); + return result; +} + static void test_dir_nicknames(void *arg) { @@ -2806,11 +2823,17 @@ test_a_networkstatus( MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m); /* Parse certificates and keys. */ - cert1 = mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); + cert1 = mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, + strlen(AUTHORITY_CERT_1), + NULL); tt_assert(cert1); - cert2 = authority_cert_parse_from_string(AUTHORITY_CERT_2, NULL); + cert2 = authority_cert_parse_from_string(AUTHORITY_CERT_2, + strlen(AUTHORITY_CERT_2), + NULL); tt_assert(cert2); - cert3 = authority_cert_parse_from_string(AUTHORITY_CERT_3, NULL); + cert3 = authority_cert_parse_from_string(AUTHORITY_CERT_3, + strlen(AUTHORITY_CERT_3), + NULL); tt_assert(cert3); sign_skey_1 = crypto_pk_new(); sign_skey_2 = crypto_pk_new(); @@ -2912,7 +2935,7 @@ test_a_networkstatus( sign_skey_leg1, FLAV_NS); tt_assert(consensus_text); - con = networkstatus_parse_vote_from_string(consensus_text, NULL, + con = networkstatus_parse_vote_from_string_(consensus_text, NULL, NS_TYPE_CONSENSUS); tt_assert(con); //log_notice(LD_GENERAL, "<<%s>>\n<<%s>>\n<<%s>>\n", @@ -2924,7 +2947,7 @@ test_a_networkstatus( sign_skey_leg1, FLAV_MICRODESC); tt_assert(consensus_text_md); - con_md = networkstatus_parse_vote_from_string(consensus_text_md, NULL, + con_md = networkstatus_parse_vote_from_string_(consensus_text_md, NULL, NS_TYPE_CONSENSUS); tt_assert(con_md); tt_int_op(con_md->flavor,OP_EQ, FLAV_MICRODESC); @@ -3023,13 +3046,13 @@ test_a_networkstatus( tt_assert(consensus_text3); tt_assert(consensus_text_md2); tt_assert(consensus_text_md3); - con2 = networkstatus_parse_vote_from_string(consensus_text2, NULL, + con2 = networkstatus_parse_vote_from_string_(consensus_text2, NULL, NS_TYPE_CONSENSUS); - con3 = networkstatus_parse_vote_from_string(consensus_text3, NULL, + con3 = networkstatus_parse_vote_from_string_(consensus_text3, NULL, NS_TYPE_CONSENSUS); - con_md2 = networkstatus_parse_vote_from_string(consensus_text_md2, NULL, + con_md2 = networkstatus_parse_vote_from_string_(consensus_text_md2, NULL, NS_TYPE_CONSENSUS); - con_md3 = networkstatus_parse_vote_from_string(consensus_text_md3, NULL, + con_md3 = networkstatus_parse_vote_from_string_(consensus_text_md3, NULL, NS_TYPE_CONSENSUS); tt_assert(con2); tt_assert(con3); @@ -6030,6 +6053,80 @@ test_dir_find_dl_min_delay(void* data) } static void +test_dir_matching_flags(void *arg) +{ + (void) arg; + routerstatus_t *rs_noflags = NULL; + routerstatus_t *rs = NULL; + char *s = NULL; + + smartlist_t *tokens = smartlist_new(); + memarea_t *area = memarea_new(); + + int expected_val_when_unused = 0; + + const char *ex_noflags = + "r example hereiswhereyouridentitygoes 2015-08-30 12:00:00 " + "192.168.0.1 9001 0\n" + "m thisoneislongerbecauseitisa256bitmddigest33\n" + "s\n"; + const char *cp = ex_noflags; + rs_noflags = routerstatus_parse_entry_from_string( + area, &cp, + cp + strlen(cp), + tokens, NULL, NULL, + MAX_SUPPORTED_CONSENSUS_METHOD, FLAV_MICRODESC); + tt_assert(rs_noflags); + +#define FLAG(string, field) STMT_BEGIN { \ + tor_asprintf(&s,\ + "r example hereiswhereyouridentitygoes 2015-08-30 12:00:00 " \ + "192.168.0.1 9001 0\n" \ + "m thisoneislongerbecauseitisa256bitmddigest33\n" \ + "s %s\n", string); \ + cp = s; \ + rs = routerstatus_parse_entry_from_string( \ + area, &cp, \ + cp + strlen(cp), \ + tokens, NULL, NULL, \ + MAX_SUPPORTED_CONSENSUS_METHOD, FLAV_MICRODESC); \ + /* the field should usually be 0 when no flags are listed */ \ + tt_int_op(rs_noflags->field, OP_EQ, expected_val_when_unused); \ + /* the field should be 1 when this flags islisted */ \ + tt_int_op(rs->field, OP_EQ, 1); \ + tor_free(s); \ + routerstatus_free(rs); \ +} STMT_END + + FLAG("Authority", is_authority); + FLAG("BadExit", is_bad_exit); + FLAG("Exit", is_exit); + FLAG("Fast", is_fast); + FLAG("Guard", is_possible_guard); + FLAG("HSDir", is_hs_dir); + FLAG("Stable", is_stable); + FLAG("StaleDesc", is_staledesc); + FLAG("V2Dir", is_v2_dir); + + // These flags are assumed to be set whether they're declared or not. + expected_val_when_unused = 1; + FLAG("Running", is_flagged_running); + FLAG("Valid", is_valid); + expected_val_when_unused = 0; + + // These flags are no longer used, but still parsed. + FLAG("Named", is_named); + FLAG("Unnamed", is_unnamed); + + done: + tor_free(s); + routerstatus_free(rs); + routerstatus_free(rs_noflags); + memarea_drop_all(area); + smartlist_free(tokens); +} + +static void test_dir_assumed_flags(void *arg) { (void)arg; @@ -6044,9 +6141,10 @@ test_dir_assumed_flags(void *arg) "192.168.0.1 9001 0\n" "m thisoneislongerbecauseitisa256bitmddigest33\n" "s Fast Guard Stable\n"; + const char *eos = str1 + strlen(str1); const char *cp = str1; - rs = routerstatus_parse_entry_from_string(area, &cp, tokens, NULL, NULL, + rs = routerstatus_parse_entry_from_string(area, &cp, eos, tokens, NULL, NULL, 24, FLAV_MICRODESC); tt_assert(rs); tt_assert(rs->is_flagged_running); @@ -6353,6 +6451,7 @@ struct testcase_t dir_tests[] = { DIR_ARG(find_dl_min_delay, TT_FORK, "cfr"), DIR_ARG(find_dl_min_delay, TT_FORK, "car"), DIR(assumed_flags, 0), + DIR(matching_flags, 0), DIR(networkstatus_compute_bw_weights_v10, 0), DIR(platform_str, 0), DIR(networkstatus_consensus_has_ipv6, TT_FORK), diff --git a/src/test/test_dir_common.c b/src/test/test_dir_common.c index 86d2838944..eadeb11921 100644 --- a/src/test/test_dir_common.c +++ b/src/test/test_dir_common.c @@ -42,14 +42,20 @@ dir_common_authority_pk_init(authority_cert_t **cert1, { /* Parse certificates and keys. */ authority_cert_t *cert; - cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); + cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, + strlen(AUTHORITY_CERT_1), + NULL); tt_assert(cert); tt_assert(cert->identity_key); *cert1 = cert; tt_assert(*cert1); - *cert2 = authority_cert_parse_from_string(AUTHORITY_CERT_2, NULL); + *cert2 = authority_cert_parse_from_string(AUTHORITY_CERT_2, + strlen(AUTHORITY_CERT_2), + NULL); tt_assert(*cert2); - *cert3 = authority_cert_parse_from_string(AUTHORITY_CERT_3, NULL); + *cert3 = authority_cert_parse_from_string(AUTHORITY_CERT_3, + strlen(AUTHORITY_CERT_3), + NULL); tt_assert(*cert3); *sign_skey_1 = crypto_pk_new(); *sign_skey_2 = crypto_pk_new(); @@ -266,7 +272,9 @@ dir_common_add_rs_and_parse(networkstatus_t *vote, networkstatus_t **vote_out, /* dump the vote and try to parse it. */ v_text = format_networkstatus_vote(sign_skey, vote); tt_assert(v_text); - *vote_out = networkstatus_parse_vote_from_string(v_text, NULL, NS_TYPE_VOTE); + *vote_out = networkstatus_parse_vote_from_string(v_text, + strlen(v_text), + NULL, NS_TYPE_VOTE); done: if (v_text) @@ -424,4 +432,3 @@ dir_common_construct_vote_3(networkstatus_t **vote, authority_cert_t *cert, return 0; } - diff --git a/src/test/test_dir_handle_get.c b/src/test/test_dir_handle_get.c index 1d4a36d7fc..2ce98769af 100644 --- a/src/test/test_dir_handle_get.c +++ b/src/test/test_dir_handle_get.c @@ -72,6 +72,8 @@ ENABLE_GCC_WARNING(overlength-strings) #define NOT_ENOUGH_CONSENSUS_SIGNATURES "HTTP/1.0 404 " \ "Consensus not signed by sufficient number of requested authorities\r\n\r\n" +#define consdiffmgr_add_consensus consdiffmgr_add_consensus_nulterm + static dir_connection_t * new_dir_conn(void) { @@ -1275,7 +1277,9 @@ test_dir_handle_get_server_keys_authority(void* data) size_t body_used = 0; (void) data; - mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE, NULL); + mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE, + strlen(TEST_CERTIFICATE), + NULL); MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m); MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock); @@ -1425,7 +1429,9 @@ test_dir_handle_get_server_keys_sk(void* data) size_t body_used = 0; (void) data; - mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE, NULL); + mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE, + strlen(TEST_CERTIFICATE), + NULL); MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m); MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock); @@ -2393,7 +2399,9 @@ test_dir_handle_get_status_vote_next_authority(void* data) routerlist_free_all(); dirvote_free_all(); - mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE, NULL); + mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE, + strlen(TEST_CERTIFICATE), + NULL); /* create a trusted ds */ ds = trusted_dir_server_new("ds", "127.0.0.1", 9059, 9060, NULL, digest, @@ -2471,7 +2479,9 @@ test_dir_handle_get_status_vote_current_authority(void* data) routerlist_free_all(); dirvote_free_all(); - mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE, NULL); + mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE, + strlen(TEST_CERTIFICATE), + NULL); /* create a trusted ds */ ds = trusted_dir_server_new("ds", "127.0.0.1", 9059, 9060, NULL, digest, diff --git a/src/test/test_dns.c b/src/test/test_dns.c index 8369f844f6..ea0fcf8c5e 100644 --- a/src/test/test_dns.c +++ b/src/test/test_dns.c @@ -1,6 +1,7 @@ /* Copyright (c) 2015-2018, The Tor Project, Inc. */ /* See LICENSE for licensing information */ +#include "orconfig.h" #include "core/or/or.h" #include "test/test.h" @@ -13,9 +14,71 @@ #include "core/or/edge_connection_st.h" #include "core/or/or_circuit_st.h" +#include "app/config/or_options_st.h" +#include "app/config/config.h" + +#include <event2/event.h> +#include <event2/dns.h> #define NS_MODULE dns +#ifdef HAVE_EVDNS_BASE_GET_NAMESERVER_ADDR +#define NS_SUBMODULE configure_nameservers_fallback + +static or_options_t options = { + .ORPort_set = 1, +}; + +static const or_options_t * +mock_get_options(void) +{ + return &options; +} + +static void +NS(test_main)(void *arg) +{ + (void)arg; + tor_addr_t *nameserver_addr = NULL; + + MOCK(get_options, mock_get_options); + + options.ServerDNSResolvConfFile = (char *)"no_such_file!!!"; + + dns_init(); // calls configure_nameservers() + + tt_int_op(number_of_configured_nameservers(), OP_EQ, 1); + + nameserver_addr = configured_nameserver_address(0); + + tt_assert(tor_addr_family(nameserver_addr) == AF_INET); + tt_assert(tor_addr_eq_ipv4h(nameserver_addr, 0x7f000001)); + +#ifndef _WIN32 + tor_free(nameserver_addr); + + options.ServerDNSResolvConfFile = (char *)"/dev/null"; + + dns_init(); + + tt_int_op(number_of_configured_nameservers(), OP_EQ, 1); + + nameserver_addr = configured_nameserver_address(0); + + tt_assert(tor_addr_family(nameserver_addr) == AF_INET); + tt_assert(tor_addr_eq_ipv4h(nameserver_addr, 0x7f000001)); +#endif + + UNMOCK(get_options); + + done: + tor_free(nameserver_addr); + return; +} + +#undef NS_SUBMODULE +#endif + #define NS_SUBMODULE clip_ttl static void @@ -736,6 +799,9 @@ NS(test_main)(void *arg) #undef NS_SUBMODULE struct testcase_t dns_tests[] = { +#ifdef HAVE_EVDNS_BASE_GET_NAMESERVER_ADDR + TEST_CASE(configure_nameservers_fallback), +#endif TEST_CASE(clip_ttl), TEST_CASE(resolve), TEST_CASE_ASPECT(resolve_impl, addr_is_ip_no_need_to_resolve), diff --git a/src/test/test_entrynodes.c b/src/test/test_entrynodes.c index 4cd50463d4..16b14c0b73 100644 --- a/src/test/test_entrynodes.c +++ b/src/test/test_entrynodes.c @@ -74,9 +74,10 @@ bfn_mock_nodelist_get_list(void) } static networkstatus_t * -bfn_mock_networkstatus_get_live_consensus(time_t now) +bfn_mock_networkstatus_get_reasonably_live_consensus(time_t now, int flavor) { (void)now; + (void)flavor; return dummy_consensus; } @@ -118,7 +119,7 @@ big_fake_network_cleanup(const struct testcase_t *testcase, void *ptr) UNMOCK(nodelist_get_list); UNMOCK(node_get_by_id); UNMOCK(get_or_state); - UNMOCK(networkstatus_get_live_consensus); + UNMOCK(networkstatus_get_reasonably_live_consensus); or_state_free(dummy_state); dummy_state = NULL; tor_free(dummy_consensus); @@ -136,6 +137,12 @@ big_fake_network_setup(const struct testcase_t *testcase) * that we need for entrynodes.c. */ const int N_NODES = 271; + const char *argument = testcase->setup_data; + int reasonably_live_consensus = 0; + if (argument) { + reasonably_live_consensus = strstr(argument, "reasonably-live") != NULL; + } + big_fake_net_nodes = smartlist_new(); for (i = 0; i < N_NODES; ++i) { curve25519_secret_key_t curve25519_secret_key; @@ -191,15 +198,23 @@ big_fake_network_setup(const struct testcase_t *testcase) dummy_state = tor_malloc_zero(sizeof(or_state_t)); dummy_consensus = tor_malloc_zero(sizeof(networkstatus_t)); - dummy_consensus->valid_after = approx_time() - 3600; - dummy_consensus->valid_until = approx_time() + 3600; + if (reasonably_live_consensus) { + /* Make the dummy consensus valid from 4 hours ago, but expired an hour + * ago. */ + dummy_consensus->valid_after = approx_time() - 4*3600; + dummy_consensus->valid_until = approx_time() - 3600; + } else { + /* Make the dummy consensus valid for an hour either side of now. */ + dummy_consensus->valid_after = approx_time() - 3600; + dummy_consensus->valid_until = approx_time() + 3600; + } MOCK(nodelist_get_list, bfn_mock_nodelist_get_list); MOCK(node_get_by_id, bfn_mock_node_get_by_id); MOCK(get_or_state, get_or_state_replacement); - MOCK(networkstatus_get_live_consensus, - bfn_mock_networkstatus_get_live_consensus); + MOCK(networkstatus_get_reasonably_live_consensus, + bfn_mock_networkstatus_get_reasonably_live_consensus); /* Return anything but NULL (it's interpreted as test fail) */ return (void*)testcase; } @@ -2691,7 +2706,7 @@ test_entry_guard_upgrade_not_blocked_by_worse_circ_pending(void *arg) } static void -test_enty_guard_should_expire_waiting(void *arg) +test_entry_guard_should_expire_waiting(void *arg) { (void)arg; circuit_guard_state_t *fake_state = tor_malloc_zero(sizeof(*fake_state)); @@ -3012,39 +3027,42 @@ static const struct testcase_setup_t upgrade_circuits = { upgrade_circuits_setup, upgrade_circuits_cleanup }; +#define NO_PREFIX_TEST(name) \ + { #name, test_ ## name, 0, NULL, NULL } + +#define EN_TEST_BASE(name, fork, setup, arg) \ + { #name, test_entry_guard_ ## name, fork, setup, (void*)(arg) } + +#define EN_TEST(name) EN_TEST_BASE(name, 0, NULL, NULL) +#define EN_TEST_FORK(name) EN_TEST_BASE(name, TT_FORK, NULL, NULL) + #define BFN_TEST(name) \ - { #name, test_entry_guard_ ## name, TT_FORK, &big_fake_network, NULL } + EN_TEST_BASE(name, TT_FORK, &big_fake_network, NULL), \ + { #name "_reasonably_live", test_entry_guard_ ## name, TT_FORK, \ + &big_fake_network, (void*)("reasonably-live") } -#define UPGRADE_TEST(name, arg) \ - { #name, test_entry_guard_ ## name, TT_FORK, &upgrade_circuits, \ - (void*)(arg) } +#define UPGRADE_TEST(name, arg) \ + EN_TEST_BASE(name, TT_FORK, &upgrade_circuits, arg), \ + { #name "_reasonably_live", test_entry_guard_ ## name, TT_FORK, \ + &upgrade_circuits, (void*)(arg " reasonably-live") } struct testcase_t entrynodes_tests[] = { - { "node_preferred_orport", - test_node_preferred_orport, - 0, NULL, NULL }, - { "entry_guard_describe", test_entry_guard_describe, 0, NULL, NULL }, - { "randomize_time", test_entry_guard_randomize_time, 0, NULL, NULL }, - { "encode_for_state_minimal", - test_entry_guard_encode_for_state_minimal, 0, NULL, NULL }, - { "encode_for_state_maximal", - test_entry_guard_encode_for_state_maximal, 0, NULL, NULL }, - { "parse_from_state_minimal", - test_entry_guard_parse_from_state_minimal, 0, NULL, NULL }, - { "parse_from_state_maximal", - test_entry_guard_parse_from_state_maximal, 0, NULL, NULL }, - { "parse_from_state_failure", - test_entry_guard_parse_from_state_failure, 0, NULL, NULL }, - { "parse_from_state_partial_failure", - test_entry_guard_parse_from_state_partial_failure, 0, NULL, NULL }, - { "parse_from_state_full", - test_entry_guard_parse_from_state_full, TT_FORK, NULL, NULL }, - { "parse_from_state_broken", - test_entry_guard_parse_from_state_broken, TT_FORK, NULL, NULL }, - { "get_guard_selection_by_name", - test_entry_guard_get_guard_selection_by_name, TT_FORK, NULL, NULL }, - { "number_of_primaries", - test_entry_guard_number_of_primaries, TT_FORK, NULL, NULL }, + NO_PREFIX_TEST(node_preferred_orport), + NO_PREFIX_TEST(entry_guard_describe), + + EN_TEST(randomize_time), + EN_TEST(encode_for_state_minimal), + EN_TEST(encode_for_state_maximal), + EN_TEST(parse_from_state_minimal), + EN_TEST(parse_from_state_maximal), + EN_TEST(parse_from_state_failure), + EN_TEST(parse_from_state_partial_failure), + + EN_TEST_FORK(parse_from_state_full), + EN_TEST_FORK(parse_from_state_broken), + EN_TEST_FORK(get_guard_selection_by_name), + EN_TEST_FORK(number_of_primaries), + BFN_TEST(choose_selection_initial), BFN_TEST(add_single_guard), BFN_TEST(node_filter), @@ -3058,7 +3076,9 @@ struct testcase_t entrynodes_tests[] = { BFN_TEST(sample_reachable_filtered_empty), BFN_TEST(retry_unreachable), BFN_TEST(manage_primary), - { "guard_preferred", test_entry_guard_guard_preferred, TT_FORK, NULL, NULL }, + + EN_TEST_FORK(guard_preferred), + BFN_TEST(select_for_circuit_no_confirmed), BFN_TEST(select_for_circuit_confirmed), BFN_TEST(select_for_circuit_highlevel_primary), @@ -3081,8 +3101,8 @@ struct testcase_t entrynodes_tests[] = { UPGRADE_TEST(upgrade_not_blocked_by_restricted_circ_pending, "c2-done"), UPGRADE_TEST(upgrade_not_blocked_by_worse_circ_pending, "c1-done"), - { "should_expire_waiting", test_enty_guard_should_expire_waiting, TT_FORK, - NULL, NULL }, + + EN_TEST_FORK(should_expire_waiting), END_OF_TESTCASES }; diff --git a/src/test/test_extorport.c b/src/test/test_extorport.c index 71be9131c7..432a9ea5e3 100644 --- a/src/test/test_extorport.c +++ b/src/test/test_extorport.c @@ -5,7 +5,7 @@ #define EXT_ORPORT_PRIVATE #define MAINLOOP_PRIVATE #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/mainloop/connection.h" #include "core/or/connection_or.h" #include "app/config/config.h" diff --git a/src/test/test_helpers.c b/src/test/test_helpers.c index 6ac73bff5c..b7bda16494 100644 --- a/src/test/test_helpers.c +++ b/src/test/test_helpers.c @@ -14,7 +14,7 @@ #include "orconfig.h" #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "app/config/config.h" #include "app/config/confparse.h" #include "core/mainloop/connection.h" diff --git a/src/test/test_hs_common.c b/src/test/test_hs_common.c index 95f7ed14ba..6198573f22 100644 --- a/src/test/test_hs_common.c +++ b/src/test/test_hs_common.c @@ -630,7 +630,7 @@ test_disaster_srv(void *arg) get_disaster_srv(1, srv_one); get_disaster_srv(2, srv_two); - /* Check that the cached ones where updated */ + /* Check that the cached ones were updated */ tt_mem_op(cached_disaster_srv_one, OP_EQ, srv_one, DIGEST256_LEN); tt_mem_op(cached_disaster_srv_two, OP_EQ, srv_two, DIGEST256_LEN); diff --git a/src/test/test_hs_service.c b/src/test/test_hs_service.c index ee2d71aa75..b2aafc1cd6 100644 --- a/src/test/test_hs_service.c +++ b/src/test/test_hs_service.c @@ -1396,7 +1396,6 @@ static void test_build_update_descriptors(void *arg) { int ret; - time_t now = time(NULL); node_t *node; hs_service_t *service; hs_service_intro_point_t *ip_cur, *ip_next; @@ -1422,7 +1421,8 @@ test_build_update_descriptors(void *arg) voting_schedule_recalculate_timing(get_options(), mock_ns.valid_after); update_approx_time(mock_ns.valid_after+1); - now = mock_ns.valid_after+1; + + time_t now = mock_ns.valid_after+1; /* Create a service without a current descriptor to trigger a build. */ service = helper_create_service(); @@ -1654,6 +1654,9 @@ test_build_descriptors(void *arg) service->desc_current = NULL; build_all_descriptors(now); + tt_assert(service->desc_current); + tt_assert(service->desc_current->desc); + hs_desc_superencrypted_data_t *superencrypted; superencrypted = &service->desc_current->desc->superencrypted_data; tt_int_op(smartlist_len(superencrypted->clients), OP_EQ, 16); diff --git a/src/test/test_logging.c b/src/test/test_logging.c index cd685a4af7..2ecae461a3 100644 --- a/src/test/test_logging.c +++ b/src/test/test_logging.c @@ -9,7 +9,6 @@ #include "lib/err/torerr.h" #include "lib/log/log.h" #include "test/test.h" -#include "lib/process/subprocess.h" #ifdef HAVE_UNISTD_H #include <unistd.h> @@ -117,22 +116,27 @@ test_sigsafe_err(void *arg) content = read_file_to_str(fn, 0, NULL); tt_ptr_op(content, OP_NE, NULL); - tor_split_lines(lines, content, (int)strlen(content)); + smartlist_split_string(lines, content, "\n", 0, 0); tt_int_op(smartlist_len(lines), OP_GE, 5); - if (strstr(smartlist_get(lines, 0), "opening new log file")) + if (strstr(smartlist_get(lines, 0), "opening new log file")) { + void *item = smartlist_get(lines, 0); smartlist_del_keeporder(lines, 0); + tor_free(item); + } + tt_assert(strstr(smartlist_get(lines, 0), "Say, this isn't too cool")); - /* Next line is blank. */ - tt_assert(!strcmpstart(smartlist_get(lines, 1), "==============")); - tt_assert(!strcmpstart(smartlist_get(lines, 2), "Minimal.")); - /* Next line is blank. */ - tt_assert(!strcmpstart(smartlist_get(lines, 3), "==============")); - tt_str_op(smartlist_get(lines, 4), OP_EQ, + tt_str_op(smartlist_get(lines, 1), OP_EQ, ""); + tt_assert(!strcmpstart(smartlist_get(lines, 2), "==============")); + tt_assert(!strcmpstart(smartlist_get(lines, 3), "Minimal.")); + tt_str_op(smartlist_get(lines, 4), OP_EQ, ""); + tt_assert(!strcmpstart(smartlist_get(lines, 5), "==============")); + tt_str_op(smartlist_get(lines, 6), OP_EQ, "Testing any attempt to manually log from a signal."); done: tor_free(content); + SMARTLIST_FOREACH(lines, char *, x, tor_free(x)); smartlist_free(lines); } diff --git a/src/test/test_mainloop.c b/src/test/test_mainloop.c index 92ce2e9918..d797417912 100644 --- a/src/test/test_mainloop.c +++ b/src/test/test_mainloop.c @@ -6,11 +6,23 @@ * \brief Tests for functions closely related to the Tor main loop */ +#define CONFIG_PRIVATE +#define MAINLOOP_PRIVATE +#define STATEFILE_PRIVATE + #include "test/test.h" #include "test/log_test_helpers.h" #include "core/or/or.h" +#include "core/mainloop/connection.h" #include "core/mainloop/mainloop.h" +#include "core/mainloop/netstatus.h" + +#include "feature/hs/hs_service.h" + +#include "app/config/config.h" +#include "app/config/statefile.h" +#include "app/config/or_state_st.h" static const uint64_t BILLION = 1000000000; @@ -131,12 +143,219 @@ test_mainloop_update_time_jumps(void *arg) monotime_disable_test_mocking(); } +static int schedule_rescan_called = 0; +static void +mock_schedule_rescan_periodic_events(void) +{ + ++schedule_rescan_called; +} + +static void +test_mainloop_user_activity(void *arg) +{ + (void)arg; + const time_t start = 1542658829; + update_approx_time(start); + + MOCK(schedule_rescan_periodic_events, mock_schedule_rescan_periodic_events); + + reset_user_activity(start); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start); + + set_network_participation(false); + + // reset can move backwards and forwards, but does not change network + // participation. + reset_user_activity(start-10); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start-10); + reset_user_activity(start+10); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start+10); + + tt_int_op(schedule_rescan_called, OP_EQ, 0); + tt_int_op(false, OP_EQ, is_participating_on_network()); + + // "note" can only move forward. Calling it from a non-participating + // state makes us rescan the periodic callbacks and set participation. + note_user_activity(start+20); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start+20); + tt_int_op(true, OP_EQ, is_participating_on_network()); + tt_int_op(schedule_rescan_called, OP_EQ, 1); + + // Calling it again will move us forward, but not call rescan again. + note_user_activity(start+25); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start+25); + tt_int_op(true, OP_EQ, is_participating_on_network()); + tt_int_op(schedule_rescan_called, OP_EQ, 1); + + // We won't move backwards. + note_user_activity(start+20); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start+25); + tt_int_op(true, OP_EQ, is_participating_on_network()); + tt_int_op(schedule_rescan_called, OP_EQ, 1); + + // We _will_ adjust if the clock jumps though. + netstatus_note_clock_jumped(500); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start+525); + + netstatus_note_clock_jumped(-400); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start+125); + + done: + UNMOCK(schedule_rescan_periodic_events); +} + +static unsigned int +mock_get_num_services(void) +{ + return 1; +} + +static connection_t * +mock_connection_gbtu(int type) +{ + (void) type; + return (void *)"hello fellow connections"; +} + +static void +test_mainloop_check_participation(void *arg) +{ + (void)arg; + or_options_t *options = options_new(); + const time_t start = 1542658829; + const time_t ONE_DAY = 24*60*60; + + // Suppose we've been idle for a day or two + reset_user_activity(start - 2*ONE_DAY); + set_network_participation(true); + check_network_participation_callback(start, options); + tt_int_op(is_participating_on_network(), OP_EQ, false); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start-2*ONE_DAY); + + // suppose we've been idle for 2 days... but we are a server. + reset_user_activity(start - 2*ONE_DAY); + options->ORPort_set = 1; + set_network_participation(true); + check_network_participation_callback(start+2, options); + tt_int_op(is_participating_on_network(), OP_EQ, true); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start+2); + options->ORPort_set = 0; + + // idle for 2 days, but we have a hidden service. + reset_user_activity(start - 2*ONE_DAY); + set_network_participation(true); + MOCK(hs_service_get_num_services, mock_get_num_services); + check_network_participation_callback(start+3, options); + tt_int_op(is_participating_on_network(), OP_EQ, true); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start+3); + UNMOCK(hs_service_get_num_services); + + // idle for 2 days but we have at least one user connection + MOCK(connection_get_by_type_nonlinked, mock_connection_gbtu); + reset_user_activity(start - 2*ONE_DAY); + set_network_participation(true); + options->DormantTimeoutDisabledByIdleStreams = 1; + check_network_participation_callback(start+10, options); + tt_int_op(is_participating_on_network(), OP_EQ, true); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start+10); + + // as above, but DormantTimeoutDisabledByIdleStreams is not set + reset_user_activity(start - 2*ONE_DAY); + set_network_participation(true); + options->DormantTimeoutDisabledByIdleStreams = 0; + check_network_participation_callback(start+13, options); + tt_int_op(is_participating_on_network(), OP_EQ, false); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start-2*ONE_DAY); + UNMOCK(connection_get_by_type_nonlinked); + options->DormantTimeoutDisabledByIdleStreams = 1; + + // idle for 2 days but DormantClientTimeout is 3 days + reset_user_activity(start - 2*ONE_DAY); + set_network_participation(true); + options->DormantClientTimeout = ONE_DAY * 3; + check_network_participation_callback(start+30, options); + tt_int_op(is_participating_on_network(), OP_EQ, true); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start-2*ONE_DAY); + + done: + or_options_free(options); + UNMOCK(hs_service_get_num_services); + UNMOCK(connection_get_by_type_nonlinked); +} + +static void +test_mainloop_dormant_load_state(void *arg) +{ + (void)arg; + or_state_t *state = or_state_new(); + const time_t start = 1543956575; + + reset_user_activity(0); + set_network_participation(false); + + // When we construct a new state, it starts out in "auto" mode. + tt_int_op(state->Dormant, OP_EQ, -1); + + // Initializing from "auto" makes us start out (by default) non-Dormant, + // with activity right now. + netstatus_load_from_state(state, start); + tt_assert(is_participating_on_network()); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start); + + // Initializing from dormant clears the last user activity time, and + // makes us dormant. + state->Dormant = 1; + netstatus_load_from_state(state, start); + tt_assert(! is_participating_on_network()); + tt_i64_op(get_last_user_activity_time(), OP_EQ, 0); + + // Initializing from non-dormant sets the last user activity time, and + // makes us non-dormant. + state->Dormant = 0; + state->MinutesSinceUserActivity = 123; + netstatus_load_from_state(state, start); + tt_assert(is_participating_on_network()); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start - 123*60); + + done: + or_state_free(state); +} + +static void +test_mainloop_dormant_save_state(void *arg) +{ + (void)arg; + or_state_t *state = or_state_new(); + const time_t start = 1543956575; + + // Can we save a non-dormant state correctly? + reset_user_activity(start - 1000); + set_network_participation(true); + netstatus_flush_to_state(state, start); + + tt_int_op(state->Dormant, OP_EQ, 0); + tt_int_op(state->MinutesSinceUserActivity, OP_EQ, 1000 / 60); + + // Can we save a dormant state correctly? + set_network_participation(false); + netstatus_flush_to_state(state, start); + + tt_int_op(state->Dormant, OP_EQ, 1); + tt_int_op(state->MinutesSinceUserActivity, OP_EQ, 0); + + done: + or_state_free(state); +} + #define MAINLOOP_TEST(name) \ { #name, test_mainloop_## name , TT_FORK, NULL, NULL } struct testcase_t mainloop_tests[] = { MAINLOOP_TEST(update_time_normal), MAINLOOP_TEST(update_time_jumps), + MAINLOOP_TEST(user_activity), + MAINLOOP_TEST(check_participation), + MAINLOOP_TEST(dormant_load_state), + MAINLOOP_TEST(dormant_save_state), END_OF_TESTCASES }; - diff --git a/src/test/test_microdesc.c b/src/test/test_microdesc.c index 8ede2690ec..fd79aee6be 100644 --- a/src/test/test_microdesc.c +++ b/src/test/test_microdesc.c @@ -11,6 +11,7 @@ #include "feature/dirparse/routerparse.h" #include "feature/nodelist/microdesc.h" #include "feature/nodelist/networkstatus.h" +#include "feature/nodelist/nodefamily.h" #include "feature/nodelist/routerlist.h" #include "feature/nodelist/torcert.h" @@ -70,6 +71,7 @@ test_md_cache(void *data) const char *test_md3_noannotation = strchr(test_md3, '\n')+1; time_t time1, time2, time3; char *fn = NULL, *s = NULL; + char *encoded_family = NULL; (void)data; options = get_options_mutable(); @@ -172,8 +174,9 @@ test_md_cache(void *data) tt_ptr_op(md1->family, OP_EQ, NULL); tt_ptr_op(md3->family, OP_NE, NULL); - tt_int_op(smartlist_len(md3->family), OP_EQ, 3); - tt_str_op(smartlist_get(md3->family, 0), OP_EQ, "nodeX"); + + encoded_family = nodefamily_format(md3->family); + tt_str_op(encoded_family, OP_EQ, "nodex nodey nodez"); /* Now rebuild the cache! */ tt_int_op(microdesc_cache_rebuild(mc, 1), OP_EQ, 0); @@ -254,6 +257,7 @@ test_md_cache(void *data) smartlist_free(wanted); tor_free(s); tor_free(fn); + tor_free(encoded_family); } static const char truncated_md[] = @@ -417,6 +421,28 @@ static const char test_md2_21[] = "ntor-onion-key hbxdRnfVUJJY7+KcT4E3Rs7/zuClbN3hJrjSBiEGMgI=\n" "id ed25519 wqfLzgfCtRfYNg88LsL1QpzxS0itapJ1aj6TbnByx/Q\n"; +static const char test_md2_withfamily_28[] = + "onion-key\n" + "-----BEGIN RSA PUBLIC KEY-----\n" + "MIGJAoGBAL2R8EfubUcahxha4u02P4VAR0llQIMwFAmrHPjzcK7apcQgDOf2ovOA\n" + "+YQnJFxlpBmCoCZC6ssCi+9G0mqo650lFuTMP5I90BdtjotfzESfTykHLiChyvhd\n" + "l0dlqclb2SU/GKem/fLRXH16aNi72CdSUu/1slKs/70ILi34QixRAgMBAAE=\n" + "-----END RSA PUBLIC KEY-----\n" + "ntor-onion-key hbxdRnfVUJJY7+KcT4E3Rs7/zuClbN3hJrjSBiEGMgI=\n" + "family OtherNode !Strange\n" + "id ed25519 wqfLzgfCtRfYNg88LsL1QpzxS0itapJ1aj6TbnByx/Q\n"; + +static const char test_md2_withfamily_29[] = + "onion-key\n" + "-----BEGIN RSA PUBLIC KEY-----\n" + "MIGJAoGBAL2R8EfubUcahxha4u02P4VAR0llQIMwFAmrHPjzcK7apcQgDOf2ovOA\n" + "+YQnJFxlpBmCoCZC6ssCi+9G0mqo650lFuTMP5I90BdtjotfzESfTykHLiChyvhd\n" + "l0dlqclb2SU/GKem/fLRXH16aNi72CdSUu/1slKs/70ILi34QixRAgMBAAE=\n" + "-----END RSA PUBLIC KEY-----\n" + "ntor-onion-key hbxdRnfVUJJY7+KcT4E3Rs7/zuClbN3hJrjSBiEGMgI=\n" + "family !Strange $B7E27F104213C36F13E7E9829182845E495997A0 othernode\n" + "id ed25519 wqfLzgfCtRfYNg88LsL1QpzxS0itapJ1aj6TbnByx/Q\n"; + static void test_md_generate(void *arg) { @@ -447,6 +473,17 @@ test_md_generate(void *arg) tt_assert(ed25519_pubkey_eq(md->ed25519_identity_pkey, &ri->cache_info.signing_key_cert->signing_key)); + // Try family encoding. + microdesc_free(md); + ri->declared_family = smartlist_new(); + smartlist_add_strdup(ri->declared_family, "OtherNode !Strange"); + md = dirvote_create_microdescriptor(ri, 28); + tt_str_op(md->body, OP_EQ, test_md2_withfamily_28); + + microdesc_free(md); + md = dirvote_create_microdescriptor(ri, 29); + tt_str_op(md->body, OP_EQ, test_md2_withfamily_29); + done: microdesc_free(md); routerinfo_free(ri); diff --git a/src/test/test_netinfo.c b/src/test/test_netinfo.c new file mode 100644 index 0000000000..8fc5330a46 --- /dev/null +++ b/src/test/test_netinfo.c @@ -0,0 +1,48 @@ +/* Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#include "core/or/or.h" +#include "trunnel/netinfo.h" +#include "test/test.h" + +static void +test_netinfo_unsupported_addr(void *arg) +{ + const uint8_t wire_data[] = + { // TIME + 0x00, 0x00, 0x00, 0x01, + // OTHERADDR + 0x04, // ATYPE + 0x04, // ALEN + 0x08, 0x08, 0x08, 0x08, // AVAL + 0x01, // NMYADDR + 0x03, // ATYPE (unsupported) + 0x05, // ALEN + 'a', 'd', 'r', 'r', '!' // AVAL (unsupported) + }; + + (void)arg; + + netinfo_cell_t *parsed_cell = NULL; + + ssize_t parsed = netinfo_cell_parse(&parsed_cell, wire_data, + sizeof(wire_data)); + + tt_assert(parsed == sizeof(wire_data)); + + netinfo_addr_t *addr = netinfo_cell_get_my_addrs(parsed_cell, 0); + tt_assert(addr); + + tt_int_op(3, OP_EQ, netinfo_addr_get_addr_type(addr)); + tt_int_op(5, OP_EQ, netinfo_addr_get_len(addr)); + + done: + netinfo_cell_free(parsed_cell); +} + +struct testcase_t netinfo_tests[] = { + { "unsupported_addr", test_netinfo_unsupported_addr, 0, NULL, NULL }, + END_OF_TESTCASES +}; + diff --git a/src/test/test_nodelist.c b/src/test/test_nodelist.c index 1af6db68ec..ed919f4edf 100644 --- a/src/test/test_nodelist.c +++ b/src/test/test_nodelist.c @@ -6,15 +6,19 @@ * \brief Unit tests for nodelist related functions. **/ +#define NODELIST_PRIVATE + #include "core/or/or.h" #include "lib/crypt_ops/crypto_rand.h" #include "feature/nodelist/networkstatus.h" +#include "feature/nodelist/nodefamily.h" #include "feature/nodelist/nodelist.h" #include "feature/nodelist/torcert.h" #include "feature/nodelist/microdesc_st.h" #include "feature/nodelist/networkstatus_st.h" #include "feature/nodelist/node_st.h" +#include "feature/nodelist/nodefamily_st.h" #include "feature/nodelist/routerinfo_st.h" #include "feature/nodelist/routerstatus_st.h" @@ -231,6 +235,411 @@ test_nodelist_ed_id(void *arg) #undef N_NODES } +static void +test_nodelist_nodefamily(void *arg) +{ + (void)arg; + /* hex ID digests */ + const char h1[] = "5B435D6869206861206C65207363617270652070"; + const char h2[] = "75C3B220616E6461726520696E206769726F2061"; + const char h3[] = "2074726F766172206461206D616E67696172652C"; + const char h4[] = "206D656E747265206E6F6E2076616C65206C2769"; + const char h5[] = "6E766572736F2E202D2D5072696D6F204C657669"; + + /* binary ID digests */ + uint8_t d1[DIGEST_LEN], d2[DIGEST_LEN], d3[DIGEST_LEN], d4[DIGEST_LEN], + d5[DIGEST_LEN]; + base16_decode((char*)d1, sizeof(d1), h1, strlen(h1)); + base16_decode((char*)d2, sizeof(d2), h2, strlen(h2)); + base16_decode((char*)d3, sizeof(d3), h3, strlen(h3)); + base16_decode((char*)d4, sizeof(d4), h4, strlen(h4)); + base16_decode((char*)d5, sizeof(d5), h5, strlen(h5)); + + char *enc=NULL, *enc2=NULL; + + nodefamily_t *nf1 = NULL; + nodefamily_t *nf2 = NULL; + nodefamily_t *nf3 = NULL; + + enc = nodefamily_format(NULL); + tt_str_op(enc, OP_EQ, ""); + tor_free(enc); + + /* Make sure that sorting and de-duplication work. */ + tor_asprintf(&enc, "$%s hello", h1); + nf1 = nodefamily_parse(enc, NULL, 0); + tt_assert(nf1); + tor_free(enc); + + tor_asprintf(&enc, "hello hello $%s hello", h1); + nf2 = nodefamily_parse(enc, NULL, 0); + tt_assert(nf2); + tt_ptr_op(nf1, OP_EQ, nf2); + tor_free(enc); + + tor_asprintf(&enc, "%s $%s hello", h1, h1); + nf3 = nodefamily_parse(enc, NULL, 0); + tt_assert(nf3); + tt_ptr_op(nf1, OP_EQ, nf3); + tor_free(enc); + + tt_assert(nodefamily_contains_rsa_id(nf1, d1)); + tt_assert(! nodefamily_contains_rsa_id(nf1, d2)); + tt_assert(nodefamily_contains_nickname(nf1, "hello")); + tt_assert(nodefamily_contains_nickname(nf1, "HELLO")); + tt_assert(! nodefamily_contains_nickname(nf1, "goodbye")); + + tt_int_op(nf1->refcnt, OP_EQ, 3); + nodefamily_free(nf3); + tt_int_op(nf1->refcnt, OP_EQ, 2); + + /* Try parsing with a provided self RSA digest. */ + nf3 = nodefamily_parse("hello ", d1, 0); + tt_assert(nf3); + tt_ptr_op(nf1, OP_EQ, nf3); + + /* Do we get the expected result when we re-encode? */ + tor_asprintf(&enc, "$%s hello", h1); + enc2 = nodefamily_format(nf1); + tt_str_op(enc2, OP_EQ, enc); + tor_free(enc2); + tor_free(enc); + + /* Make sure that we get a different result if we give a different digest. */ + nodefamily_free(nf3); + tor_asprintf(&enc, "hello $%s hello", h3); + nf3 = nodefamily_parse(enc, NULL, 0); + tt_assert(nf3); + tt_ptr_op(nf1, OP_NE, nf3); + tor_free(enc); + + tt_assert(nodefamily_contains_rsa_id(nf3, d3)); + tt_assert(! nodefamily_contains_rsa_id(nf3, d2)); + tt_assert(! nodefamily_contains_rsa_id(nf3, d1)); + tt_assert(nodefamily_contains_nickname(nf3, "hello")); + tt_assert(! nodefamily_contains_nickname(nf3, "goodbye")); + + nodefamily_free(nf1); + nodefamily_free(nf2); + nodefamily_free(nf3); + + /* Try one with several digests, all with nicknames appended, in different + formats. */ + tor_asprintf(&enc, "%s $%s $%s=res $%s~ist", h1, h2, h3, h4); + nf1 = nodefamily_parse(enc, d5, 0); + tt_assert(nf1); + tt_assert(nodefamily_contains_rsa_id(nf1, d1)); + tt_assert(nodefamily_contains_rsa_id(nf1, d2)); + tt_assert(nodefamily_contains_rsa_id(nf1, d3)); + tt_assert(nodefamily_contains_rsa_id(nf1, d4)); + tt_assert(nodefamily_contains_rsa_id(nf1, d5)); + /* Nicknames aren't preserved when ids are present, since node naming is + * deprecated */ + tt_assert(! nodefamily_contains_nickname(nf3, "res")); + tor_free(enc); + tor_asprintf(&enc, "$%s $%s $%s $%s $%s", h4, h3, h1, h5, h2); + enc2 = nodefamily_format(nf1); + tt_str_op(enc, OP_EQ, enc2); + tor_free(enc); + tor_free(enc2); + + /* Try ones where we parse the empty string. */ + nf2 = nodefamily_parse("", NULL, 0); + nf3 = nodefamily_parse("", d4, 0); + tt_assert(nf2); + tt_assert(nf3); + tt_ptr_op(nf2, OP_NE, nf3); + + tt_assert(! nodefamily_contains_rsa_id(nf2, d4)); + tt_assert(nodefamily_contains_rsa_id(nf3, d4)); + tt_assert(! nodefamily_contains_rsa_id(nf2, d5)); + tt_assert(! nodefamily_contains_rsa_id(nf3, d5)); + tt_assert(! nodefamily_contains_nickname(nf2, "fred")); + tt_assert(! nodefamily_contains_nickname(nf3, "bosco")); + + /* The NULL family should contain nothing. */ + tt_assert(! nodefamily_contains_rsa_id(NULL, d4)); + tt_assert(! nodefamily_contains_rsa_id(NULL, d5)); + + done: + tor_free(enc); + tor_free(enc2); + nodefamily_free(nf1); + nodefamily_free(nf2); + nodefamily_free(nf3); + nodefamily_free_all(); +} + +static void +test_nodelist_nodefamily_parse_err(void *arg) +{ + (void)arg; + nodefamily_t *nf1 = NULL; + char *enc = NULL; + const char *semibogus = + "sdakljfdslkfjdsaklfjdkl9sdf " // too long for nickname + "$jkASDFLkjsadfjhkl " // not hex + "$7468696e67732d696e2d7468656d73656c766573 " // ok + "reticulatogranulate "// ok + "$73656d69616e7468726f706f6c6f676963616c6c79 " // too long for hex + "$616273656e746d696e6465646e6573736573" // too short for hex + ; + + setup_capture_of_logs(LOG_WARN); + + // We only get two items when we parse this. + for (int reject = 0; reject <= 1; ++reject) { + for (int log_at_warn = 0; log_at_warn <= 1; ++log_at_warn) { + unsigned flags = log_at_warn ? NF_WARN_MALFORMED : 0; + flags |= reject ? NF_REJECT_MALFORMED : 0; + nf1 = nodefamily_parse(semibogus, NULL, flags); + if (reject) { + tt_assert(nf1 == NULL); + } else { + tt_assert(nf1); + enc = nodefamily_format(nf1); + tt_str_op(enc, OP_EQ, + "$7468696E67732D696E2D7468656D73656C766573 " + "reticulatogranulate"); + tor_free(enc); + } + + if (log_at_warn) { + expect_log_msg_containing("$616273656e746d696e6465646e6573736573"); + expect_log_msg_containing("sdakljfdslkfjdsaklfjdkl9sdf"); + } else { + tt_int_op(mock_saved_log_n_entries(), OP_EQ, 0); + } + mock_clean_saved_logs(); + } + } + + done: + tor_free(enc); + nodefamily_free(nf1); + teardown_capture_of_logs(); +} + +static const node_t * +mock_node_get_by_id(const char *id) +{ + if (fast_memeq(id, "!!!!!!!!!!!!!!!!!!!!", DIGEST_LEN)) + return NULL; + + // use tor_free, not node_free. + node_t *fake_node = tor_malloc_zero(sizeof(node_t)); + memcpy(fake_node->identity, id, DIGEST_LEN); + return fake_node; +} + +static const node_t * +mock_node_get_by_nickname(const char *nn, unsigned flags) +{ + (void)flags; + if (!strcmp(nn, "nonesuch")) + return NULL; + + // use tor_free, not node_free. + node_t *fake_node = tor_malloc_zero(sizeof(node_t)); + strlcpy(fake_node->identity, nn, DIGEST_LEN); + return fake_node; +} + +static void +test_nodelist_nodefamily_lookup(void *arg) +{ + (void)arg; + MOCK(node_get_by_nickname, mock_node_get_by_nickname); + MOCK(node_get_by_id, mock_node_get_by_id); + smartlist_t *sl = smartlist_new(); + nodefamily_t *nf1 = NULL; + char *mem_op_hex_tmp = NULL; + + // 'null' is allowed. + nodefamily_add_nodes_to_smartlist(NULL, sl); + tt_int_op(smartlist_len(sl), OP_EQ, 0); + + // Try a real family + nf1 = nodefamily_parse("$EEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE " + "$2121212121212121212121212121212121212121 " + "$3333333333333333333333333333333333333333 " + "erewhon nonesuch", NULL, 0); + tt_assert(nf1); + nodefamily_add_nodes_to_smartlist(nf1, sl); + // There were 5 elements; 2 were dropped because the mocked lookup failed. + tt_int_op(smartlist_len(sl), OP_EQ, 3); + + const node_t *n = smartlist_get(sl, 0); + test_memeq_hex(n->identity, "3333333333333333333333333333333333333333"); + n = smartlist_get(sl, 1); + test_memeq_hex(n->identity, "EEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE"); + n = smartlist_get(sl, 2); + tt_str_op(n->identity, OP_EQ, "erewhon"); + + done: + UNMOCK(node_get_by_nickname); + UNMOCK(node_get_by_id); + SMARTLIST_FOREACH(sl, node_t *, fake_node, tor_free(fake_node)); + smartlist_free(sl); + nodefamily_free(nf1); + tor_free(mem_op_hex_tmp); +} + +static void +test_nodelist_nickname_matches(void *arg) +{ + (void)arg; + node_t mock_node; + routerstatus_t mock_rs; + memset(&mock_node, 0, sizeof(mock_node)); + memset(&mock_rs, 0, sizeof(mock_rs)); + + strlcpy(mock_rs.nickname, "evilgeniuses", sizeof(mock_rs.nickname)); + mock_node.rs = &mock_rs; + memcpy(mock_node.identity, ".forabettertomorrow.", DIGEST_LEN); + +#define match(x) tt_assert(node_nickname_matches(&mock_node, (x))) +#define no_match(x) tt_assert(! node_nickname_matches(&mock_node, (x))) + + match("evilgeniuses"); + match("EvilGeniuses"); + match("EvilGeniuses"); + match("2e666f7261626574746572746f6d6f72726f772e"); + match("2E666F7261626574746572746F6D6F72726F772E"); + match("$2e666f7261626574746572746f6d6f72726f772e"); + match("$2E666F7261626574746572746F6D6F72726F772E"); + match("$2E666F7261626574746572746F6D6F72726F772E~evilgeniuses"); + match("$2E666F7261626574746572746F6D6F72726F772E~EVILGENIUSES"); + + no_match("evilgenius"); + no_match("evilgeniuseses"); + no_match("evil.genius"); + no_match("$2E666F7261626574746572746F6D6F72726FFFFF"); + no_match("2E666F7261626574746572746F6D6F72726FFFFF"); + no_match("$2E666F7261626574746572746F6D6F72726F772E~fred"); + no_match("$2E666F7261626574746572746F6D6F72726F772E=EVILGENIUSES"); + done: + ; +} + +static void +test_nodelist_node_nodefamily(void *arg) +{ + (void)arg; + node_t mock_node1; + routerstatus_t mock_rs; + microdesc_t mock_md; + + node_t mock_node2; + routerinfo_t mock_ri; + + smartlist_t *nodes=smartlist_new(); + + memset(&mock_node1, 0, sizeof(mock_node1)); + memset(&mock_node2, 0, sizeof(mock_node2)); + memset(&mock_rs, 0, sizeof(mock_rs)); + memset(&mock_md, 0, sizeof(mock_md)); + memset(&mock_ri, 0, sizeof(mock_ri)); + + mock_node1.rs = &mock_rs; + mock_node1.md = &mock_md; + + mock_node2.ri = &mock_ri; + + strlcpy(mock_rs.nickname, "nodeone", sizeof(mock_rs.nickname)); + mock_ri.nickname = tor_strdup("nodetwo"); + + memcpy(mock_node1.identity, "NodeOneNode1NodeOne1", DIGEST_LEN); + memcpy(mock_node2.identity, "SecondNodeWe'reTestn", DIGEST_LEN); + + // empty families. + tt_assert(! node_family_contains(&mock_node1, &mock_node2)); + tt_assert(! node_family_contains(&mock_node2, &mock_node1)); + + // Families contain nodes, but not these nodes + mock_ri.declared_family = smartlist_new(); + smartlist_add(mock_ri.declared_family, (char*)"NodeThree"); + mock_md.family = nodefamily_parse("NodeFour", NULL, 0); + tt_assert(! node_family_contains(&mock_node1, &mock_node2)); + tt_assert(! node_family_contains(&mock_node2, &mock_node1)); + + // Families contain one another. + smartlist_add(mock_ri.declared_family, (char*) + "4e6f64654f6e654e6f6465314e6f64654f6e6531"); + tt_assert(! node_family_contains(&mock_node1, &mock_node2)); + tt_assert(node_family_contains(&mock_node2, &mock_node1)); + + nodefamily_free(mock_md.family); + mock_md.family = nodefamily_parse( + "NodeFour " + "5365636f6e644e6f64655765277265546573746e", NULL, 0); + tt_assert(node_family_contains(&mock_node1, &mock_node2)); + tt_assert(node_family_contains(&mock_node2, &mock_node1)); + + // Try looking up families now. + MOCK(node_get_by_nickname, mock_node_get_by_nickname); + MOCK(node_get_by_id, mock_node_get_by_id); + + node_lookup_declared_family(nodes, &mock_node1); + tt_int_op(smartlist_len(nodes), OP_EQ, 2); + const node_t *n = smartlist_get(nodes, 0); + tt_mem_op(n->identity, OP_EQ, "SecondNodeWe'reTestn", DIGEST_LEN); + n = smartlist_get(nodes, 1); + tt_str_op(n->identity, OP_EQ, "nodefour"); + + // free, try the other one. + SMARTLIST_FOREACH(nodes, node_t *, x, tor_free(x)); + smartlist_clear(nodes); + + node_lookup_declared_family(nodes, &mock_node2); + tt_int_op(smartlist_len(nodes), OP_EQ, 2); + n = smartlist_get(nodes, 0); + // This gets a truncated hex hex ID since it was looked up by name + tt_str_op(n->identity, OP_EQ, "NodeThree"); + n = smartlist_get(nodes, 1); + tt_str_op(n->identity, OP_EQ, "4e6f64654f6e654e6f6"); + + done: + UNMOCK(node_get_by_nickname); + UNMOCK(node_get_by_id); + smartlist_free(mock_ri.declared_family); + nodefamily_free(mock_md.family); + tor_free(mock_ri.nickname); + // use tor_free, these aren't real nodes + SMARTLIST_FOREACH(nodes, node_t *, x, tor_free(x)); + smartlist_free(nodes); +} + +static void +test_nodelist_nodefamily_canonicalize(void *arg) +{ + (void)arg; + char *c = NULL; + + c = nodefamily_canonicalize("", NULL, 0); + tt_str_op(c, OP_EQ, ""); + tor_free(c); + + uint8_t own_id[20]; + memset(own_id, 0, sizeof(own_id)); + c = nodefamily_canonicalize( + "alice BOB caroL %potrzebie !!!@#@# " + "$bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb=fred " + "ffffffffffffffffffffffffffffffffffffffff " + "$cccccccccccccccccccccccccccccccccccccccc ", own_id, 0); + tt_str_op(c, OP_EQ, + "!!!@#@# " + "$0000000000000000000000000000000000000000 " + "$BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB " + "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC " + "$FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF " + "%potrzebie " + "alice bob carol"); + + done: + tor_free(c); +} + #define NODE(name, flags) \ { #name, test_nodelist_##name, (flags), NULL, NULL } @@ -239,6 +648,11 @@ struct testcase_t nodelist_tests[] = { NODE(node_get_verbose_nickname_not_named, TT_FORK), NODE(node_is_dir, TT_FORK), NODE(ed_id, TT_FORK), + NODE(nodefamily, TT_FORK), + NODE(nodefamily_parse_err, TT_FORK), + NODE(nodefamily_lookup, TT_FORK), + NODE(nickname_matches, 0), + NODE(node_nodefamily, TT_FORK), + NODE(nodefamily_canonicalize, 0), END_OF_TESTCASES }; - diff --git a/src/test/test_oom.c b/src/test/test_oom.c index 313a6b3114..f84dc0764b 100644 --- a/src/test/test_oom.c +++ b/src/test/test_oom.c @@ -8,7 +8,7 @@ #define CIRCUITLIST_PRIVATE #define CONNECTION_PRIVATE #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/or/circuitlist.h" #include "lib/evloop/compat_libevent.h" #include "core/mainloop/connection.h" diff --git a/src/test/test_options.c b/src/test/test_options.c index f14e620eeb..376d77626f 100644 --- a/src/test/test_options.c +++ b/src/test/test_options.c @@ -425,6 +425,7 @@ get_options_test_data(const char *conf) // with options_init(), but about a dozen tests break when I do that. // Being kinda lame and just fixing the immedate breakage for now.. result->opt->ConnectionPadding = -1; // default must be "auto" + result->opt->DormantClientTimeout = 1800; // must be over 600. rv = config_get_lines(conf, &cl, 1); tt_int_op(rv, OP_EQ, 0); diff --git a/src/test/test_parsecommon.c b/src/test/test_parsecommon.c new file mode 100644 index 0000000000..8e74fcdb4d --- /dev/null +++ b/src/test/test_parsecommon.c @@ -0,0 +1,594 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "core/or/or.h" +#include "test/test.h" +#include "lib/memarea/memarea.h" +#include "lib/encoding/binascii.h" +#include "feature/dirparse/parsecommon.h" +#include "test/log_test_helpers.h" + +static void +test_parsecommon_tokenize_string_null(void *arg) +{ + + memarea_t *area = memarea_new(); + smartlist_t *tokens = smartlist_new(); + + (void)arg; + + const char *str_with_null = "a\0bccccccccc"; + + int retval = + tokenize_string(area, str_with_null, + str_with_null + 3, + tokens, NULL, 0); + + tt_int_op(retval, OP_EQ, -1); + + done: + memarea_drop_all(area); + smartlist_free(tokens); + return; +} + +static void +test_parsecommon_tokenize_string_multiple_lines(void *arg) +{ + memarea_t *area = memarea_new(); + smartlist_t *tokens = smartlist_new(); + + (void)arg; + + token_rule_t table[] = { + T01("uptime", K_UPTIME, GE(1), NO_OBJ), + T01("hibernating", K_HIBERNATING, GE(1), NO_OBJ), + T1( "published", K_PUBLISHED, CONCAT_ARGS, NO_OBJ), + END_OF_TABLE, + }; + + char *str = tor_strdup( + "hibernating 0\nuptime 1024\n" + "published 2018-10-15 10:00:00\n"); + + int retval = + tokenize_string(area, str, NULL, + tokens, table, 0); + + tt_int_op(smartlist_len(tokens), OP_EQ, 3); + directory_token_t *token = smartlist_get(tokens, 0); + + tt_int_op(token->tp, OP_EQ, K_HIBERNATING); + + token = smartlist_get(tokens, 1); + + tt_int_op(token->tp, OP_EQ, K_UPTIME); + + token = smartlist_get(tokens, 2); + + tt_int_op(token->tp, OP_EQ, K_PUBLISHED); + + tt_int_op(retval, OP_EQ, 0); + + done: + tor_free(str); + memarea_drop_all(area); + smartlist_free(tokens); + return; +} + +static void +test_parsecommon_tokenize_string_min_cnt(void *arg) +{ + memarea_t *area = memarea_new(); + smartlist_t *tokens = smartlist_new(); + + (void)arg; + + token_rule_t table[] = { + T01("uptime", K_UPTIME, EQ(2), NO_OBJ), + T01("hibernating", K_HIBERNATING, GE(1), NO_OBJ), + END_OF_TABLE, + }; + + // Missing "uptime" + char *str = tor_strdup("uptime 1024\nhibernating 0\n"); + + int retval = + tokenize_string(area, str, NULL, + tokens, table, 0); + + tt_int_op(retval, OP_EQ, -1); + + done: + tor_free(str); + memarea_drop_all(area); + smartlist_free(tokens); + return; +} + +static void +test_parsecommon_tokenize_string_max_cnt(void *arg) +{ + memarea_t *area = memarea_new(); + smartlist_t *tokens = smartlist_new(); + + (void)arg; + + token_rule_t table[] = { + T01("uptime", K_UPTIME, EQ(1), NO_OBJ), + T01("hibernating", K_HIBERNATING, GE(1), NO_OBJ), + END_OF_TABLE, + }; + + // "uptime" expected once, but occurs twice in input. + char *str = tor_strdup( + "uptime 1024\nuptime 2048\nhibernating 0\n"); + + int retval = + tokenize_string(area, str, NULL, + tokens, table, 0); + + tt_int_op(retval, OP_EQ, -1); + + done: + tor_free(str); + memarea_drop_all(area); + smartlist_free(tokens); + return; +} + +static void +test_parsecommon_tokenize_string_at_start(void *arg) +{ + memarea_t *area = memarea_new(); + smartlist_t *tokens = smartlist_new(); + + (void)arg; + + token_rule_t table[] = { + T1_START("client-name", C_CLIENT_NAME, CONCAT_ARGS, NO_OBJ), + T01("uptime", K_UPTIME, EQ(1), NO_OBJ), + END_OF_TABLE, + }; + + // "client-name" is not the first line. + char *str = tor_strdup( + "uptime 1024\nclient-name Alice\n"); + + int retval = + tokenize_string(area, str, NULL, tokens, table, 0); + + tt_int_op(retval, OP_EQ, -1); + + done: + tor_free(str); + memarea_drop_all(area); + smartlist_free(tokens); + return; +} + +static void +test_parsecommon_tokenize_string_at_end(void *arg) +{ + memarea_t *area = memarea_new(); + smartlist_t *tokens = smartlist_new(); + + (void)arg; + + token_rule_t table[] = { + T1_END("client-name", C_CLIENT_NAME, CONCAT_ARGS, NO_OBJ), + T01("uptime", K_UPTIME, EQ(1), NO_OBJ), + END_OF_TABLE, + }; + + // "client-name" is not the last line. + char *str = tor_strdup( + "client-name Alice\nuptime 1024\n"); + + int retval = + tokenize_string(area, str, NULL, tokens, table, 0); + + tt_int_op(retval, OP_EQ, -1); + + done: + tor_free(str); + memarea_drop_all(area); + smartlist_free(tokens); + return; +} + +static void +test_parsecommon_tokenize_string_no_annotations(void *arg) +{ + memarea_t *area = memarea_new(); + smartlist_t *tokens = smartlist_new(); + + (void)arg; + + token_rule_t table[] = { + A01("@last-listed", A_LAST_LISTED, CONCAT_ARGS, NO_OBJ), + END_OF_TABLE, + }; + + char *str = tor_strdup("@last-listed 2018-09-21 15:30:03\n"); + + int retval = + tokenize_string(area, str, NULL, tokens, table, 0); + + tt_int_op(retval, OP_EQ, -1); + + done: + tor_free(str); + memarea_drop_all(area); + smartlist_free(tokens); + return; +} + +static void +test_parsecommon_get_next_token_success(void *arg) +{ + memarea_t *area = memarea_new(); + const char *str = "uptime 1024"; + const char *end = str + strlen(str); + const char **s = &str; + token_rule_t table = T01("uptime", K_UPTIME, GE(1), NO_OBJ); + (void)arg; + + directory_token_t *token = get_next_token(area, s, end, &table); + + tt_int_op(token->tp, OP_EQ, K_UPTIME); + tt_int_op(token->n_args, OP_EQ, 1); + tt_str_op(*(token->args), OP_EQ, "1024"); + tt_assert(!token->object_type); + tt_int_op(token->object_size, OP_EQ, 0); + tt_assert(!token->object_body); + + tt_ptr_op(*s, OP_EQ, end); + + done: + memarea_drop_all(area); + return; +} + +static void +test_parsecommon_get_next_token_concat_args(void *arg) +{ + memarea_t *area = memarea_new(); + const char *str = "proto A=1 B=2"; + const char *end = str + strlen(str); + const char **s = &str; + token_rule_t rule = T01("proto", K_PROTO, CONCAT_ARGS, NO_OBJ); + (void)arg; + + directory_token_t *token = get_next_token(area, s, end, &rule); + + tt_int_op(token->tp, OP_EQ, K_PROTO); + tt_int_op(token->n_args, OP_EQ, 1); + tt_str_op(*(token->args), OP_EQ, "A=1 B=2"); + + done: + memarea_drop_all(area); +} + +static void +test_parsecommon_get_next_token_parse_keys(void *arg) +{ + (void)arg; + + memarea_t *area = memarea_new(); + const char *str = + "onion-key\n" + "-----BEGIN RSA PUBLIC KEY-----\n" + "MIGJAoGBAMDdIya33BfNlHOkzoTKSTT8EjD64waMfUr372syVHiFjHhObwKwGA5u\n" + "sHaMIe9r+Ij/4C1dKyuXkcz3DOl6gWNhTD7dZ89I+Okoh1jWe30jxCiAcywC22p5\n" + "XLhrDkX1A63Z7XCH9ltwU2WMqWsVM98N2GR6MTujP7wtqdLExYN1AgMBAAE=\n" + "-----END RSA PUBLIC KEY-----\n"; + + const char *end = str + strlen(str); + const char **s = (const char **)&str; + directory_token_t *token = NULL; + directory_token_t *token2 = NULL; + + token_rule_t rule = T1("onion-key", R_IPO_ONION_KEY, NO_ARGS, NEED_KEY_1024); + + token = get_next_token(area, s, end, &rule); + tt_assert(token); + + tt_int_op(token->tp, OP_EQ, R_IPO_ONION_KEY); + tt_int_op(token->n_args, OP_EQ, 0); + tt_str_op(token->object_type, OP_EQ, "RSA PUBLIC KEY"); + tt_int_op(token->object_size, OP_EQ, 140); + tt_assert(token->object_body); + tt_assert(token->key); + tt_assert(!token->error); + + const char *str2 = + "client-key\n" + "-----BEGIN RSA PRIVATE KEY-----\n" + "MIICXAIBAAKBgQCwS810a2auH2PQchOBz9smNgjlDu31aq0IYlUohSYbhcv5AJ+d\n" + "DY0nfZWzS+mZPwzL3UiEnTt6PVv7AgoZ5V9ZJWJTKIURjJpkK0mstfJKHKIZhf84\n" + "pmFfRej9GQViB6NLtp1obOXJgJixSlMfw9doDI4NoAnEISCyH/tD77Qs2wIDAQAB\n" + "AoGAbDg8CKkdQOnX9c7xFpCnsE8fKqz9eddgHHNwXw1NFTwOt+2gDWKSMZmv2X5S\n" + "CVZg3owZxf5W0nT0D6Ny2+6nliak7foYAvkD0BsCiBhgftwC0zAo6k5rIbUKB3PJ\n" + "QLFXgpJhqWuXkODyt/hS/GTernR437WVSEGp1bnALqiFabECQQDaqHOxzoWY/nvH\n" + "KrfUi8EhqCnqERlRHwrW0MQZ1RPvF16OPPma+xa+ht/amfh3vYN5tZY82Zm43gGl\n" + "XWL5cZhNAkEAzmdSootYVnqLLLRMfHKXnO1XbaEcA/08MDNKGlSclBJixFenE8jX\n" + "iQsUbHwMJuGONvzWpRGPBP2f8xBd28ZtxwJARY+LZshtpfNniz/ixYJESaHG28je\n" + "xfjbKOW3TQSFV+2WTifFvHEeljQwKMoMyoMGvYRwLCGJjs9JtMLVxsdFjQJBAKwD\n" + "3BBvBQ39TuPQ1zWX4tb7zjMlY83HTFP3Sriq71tP/1QWoL2SUl56B2lp8E6vB/C3\n" + "wsMK4SCNprHRYAd7VZ0CQDKn6Zhd11P94PLs0msybFEh1VXr6CEW/BrxBgbL4ls6\n" + "dbX5XO0z4Ra8gYXgObgimhyMDYO98Idt5+Z3HIdyrSc=\n" + "-----END RSA PRIVATE KEY-----\n"; + + const char *end2 = str2 + strlen(str2); + const char **s2 = (const char **)&str2; + + token_rule_t rule2 = T01("client-key", C_CLIENT_KEY, NO_ARGS, + NEED_SKEY_1024); + + token2 = get_next_token(area, s2, end2, &rule2); + tt_assert(token2); + + tt_int_op(token2->tp, OP_EQ, C_CLIENT_KEY); + tt_int_op(token2->n_args, OP_EQ, 0); + tt_str_op(token2->object_type, OP_EQ, "RSA PRIVATE KEY"); + tt_int_op(token2->object_size, OP_EQ, 608); + tt_assert(token2->object_body); + tt_assert(token2->key); + tt_assert(!token->error); + + done: + if (token) token_clear(token); + if (token2) token_clear(token2); + memarea_drop_all(area); +} + +static void +test_parsecommon_get_next_token_object(void *arg) +{ + memarea_t *area = memarea_new(); + + const char *str = + "directory-signature 0232AF901C31A04EE9848595AF9BB7620D4C5B2E " + "CD1FD971855430880D3C31E0331C5C55800C2F79\n" + "-----BEGIN SIGNATURE-----\n" + "dLTbc1Lad/OWKBJhA/dERzDHumswTAzBFAWAz2vnQhLsebs1SOm0W/vceEsiEkiF\n" + "A+JJSzIyfywJc6Mnk7aKMEIFjOO/MaxuAp4zv+q+JonJkF0ExjMqvKR0D6pSFmfN\n" + "cnemnxGHxNuPDnKl0imbWKmWDsHtwgi4zWeTq3MekfMOXKi6gIh+bDFzCs9/Vquh\n" + "uNKJI1jW/A2DEKeaSAODEv9VoCsYSvbVVEuHCBWjeNAurd5aL26BrAolW6m7pkD6\n" + "I+cQ8dQG6Wa/Zt6gLXtBbOP2o/iDI7ahDP9diNkBI/rm4nfp9j4piTwsqpi7xz9J\n" + "Ua9DEZB9KbJHVX1rGShrLA==\n" + "-----END SIGNATURE-----\n"; + + const char *end = str + strlen(str); + const char **s = &str; + token_rule_t rule = T("directory-signature", K_DIRECTORY_SIGNATURE, + GE(2), NEED_OBJ); + (void)arg; + + directory_token_t *token = get_next_token(area, s, end, &rule); + + tt_int_op(token->tp, OP_EQ, K_DIRECTORY_SIGNATURE); + tt_int_op(token->n_args, OP_EQ, 2); + tt_str_op(token->args[0], OP_EQ, + "0232AF901C31A04EE9848595AF9BB7620D4C5B2E"); + tt_str_op(token->args[1], OP_EQ, + "CD1FD971855430880D3C31E0331C5C55800C2F79"); + + tt_assert(!token->error); + + char decoded[256]; + const char *signature = + "dLTbc1Lad/OWKBJhA/dERzDHumswTAzBFAWAz2vnQhLsebs1SOm0W/vceEsiEkiF\n" + "A+JJSzIyfywJc6Mnk7aKMEIFjOO/MaxuAp4zv+q+JonJkF0ExjMqvKR0D6pSFmfN\n" + "cnemnxGHxNuPDnKl0imbWKmWDsHtwgi4zWeTq3MekfMOXKi6gIh+bDFzCs9/Vquh\n" + "uNKJI1jW/A2DEKeaSAODEv9VoCsYSvbVVEuHCBWjeNAurd5aL26BrAolW6m7pkD6\n" + "I+cQ8dQG6Wa/Zt6gLXtBbOP2o/iDI7ahDP9diNkBI/rm4nfp9j4piTwsqpi7xz9J\n" + "Ua9DEZB9KbJHVX1rGShrLA==\n"; + tt_assert(signature); + size_t signature_len = strlen(signature); + base64_decode(decoded, sizeof(decoded), signature, signature_len); + + tt_str_op(token->object_type, OP_EQ, "SIGNATURE"); + tt_int_op(token->object_size, OP_EQ, 256); + tt_mem_op(token->object_body, OP_EQ, decoded, 256); + + tt_assert(!token->key); + + done: + memarea_drop_all(area); +} + +static void +test_parsecommon_get_next_token_err_too_many_args(void *arg) +{ + memarea_t *area = memarea_new(); + const char *str = "uptime 1024 1024 1024"; + const char *end = str + strlen(str); + const char **s = &str; + token_rule_t table = T01("uptime", K_UPTIME, EQ(1), NO_OBJ); + (void)arg; + + directory_token_t *token = get_next_token(area, s, end, &table); + + tt_int_op(token->tp, OP_EQ, ERR_); + tt_str_op(token->error, OP_EQ, "Too many arguments to uptime"); + + done: + memarea_drop_all(area); + return; +} + +static void +test_parsecommon_get_next_token_err_too_few_args(void *arg) +{ + memarea_t *area = memarea_new(); + const char *str = "uptime"; + const char *end = str + strlen(str); + const char **s = &str; + token_rule_t table = T01("uptime", K_UPTIME, EQ(1), NO_OBJ); + (void)arg; + + directory_token_t *token = get_next_token(area, s, end, &table); + + tt_int_op(token->tp, OP_EQ, ERR_); + tt_str_op(token->error, OP_EQ, "Too few arguments to uptime"); + + done: + memarea_drop_all(area); + return; +} + +static void +test_parsecommon_get_next_token_err_obj_missing_endline(void *arg) +{ + memarea_t *area = memarea_new(); + + const char *str = + "directory-signature 0232AF901C31A04EE9848595AF9BB7620D4C5B2E " + "CD1FD971855430880D3C31E0331C5C55800C2F79\n" + "-----BEGIN SIGNATURE-----\n" + "dLTbc1Lad/OWKBJhA/dERzDHumswTAzBFAWAz2vnQhLsebs1SOm0W/vceEsiEkiF\n" + "A+JJSzIyfywJc6Mnk7aKMEIFjOO/MaxuAp4zv+q+JonJkF0ExjMqvKR0D6pSFmfN\n" + "cnemnxGHxNuPDnKl0imbWKmWDsHtwgi4zWeTq3MekfMOXKi6gIh+bDFzCs9/Vquh\n" + "uNKJI1jW/A2DEKeaSAODEv9VoCsYSvbVVEuHCBWjeNAurd5aL26BrAolW6m7pkD6\n" + "I+cQ8dQG6Wa/Zt6gLXtBbOP2o/iDI7ahDP9diNkBI/rm4nfp9j4piTwsqpi7xz9J\n" + "Ua9DEZB9KbJHVX1rGShrLA==\n"; + + const char *end = str + strlen(str); + const char **s = &str; + token_rule_t rule = T("directory-signature", K_DIRECTORY_SIGNATURE, + GE(2), NEED_OBJ); + (void)arg; + + directory_token_t *token = get_next_token(area, s, end, &rule); + + tt_int_op(token->tp, OP_EQ, ERR_); + tt_str_op(token->error, OP_EQ, "Malformed object: missing object end line"); + + done: + memarea_drop_all(area); + return; +} + +static void +test_parsecommon_get_next_token_err_bad_beginline(void *arg) +{ + memarea_t *area = memarea_new(); + + const char *str = + "directory-signature 0232AF901C31A04EE9848595AF9BB7620D4C5B2E " + "CD1FD971855430880D3C31E0331C5C55800C2F79\n" + "-----BEGIN SIGNATURE-Z---\n" + "dLTbc1Lad/OWKBJhA/dERzDHumswTAzBFAWAz2vnQhLsebs1SOm0W/vceEsiEkiF\n" + "A+JJSzIyfywJc6Mnk7aKMEIFjOO/MaxuAp4zv+q+JonJkF0ExjMqvKR0D6pSFmfN\n" + "cnemnxGHxNuPDnKl0imbWKmWDsHtwgi4zWeTq3MekfMOXKi6gIh+bDFzCs9/Vquh\n" + "uNKJI1jW/A2DEKeaSAODEv9VoCsYSvbVVEuHCBWjeNAurd5aL26BrAolW6m7pkD6\n" + "I+cQ8dQG6Wa/Zt6gLXtBbOP2o/iDI7ahDP9diNkBI/rm4nfp9j4piTwsqpi7xz9J\n" + "Ua9DEZB9KbJHVX1rGShrLA==\n" + "-----END SIGNATURE-----\n"; + + const char *end = str + strlen(str); + const char **s = &str; + token_rule_t rule = T("directory-signature", K_DIRECTORY_SIGNATURE, + GE(2), NEED_OBJ); + (void)arg; + + directory_token_t *token = get_next_token(area, s, end, &rule); + + tt_int_op(token->tp, OP_EQ, ERR_); + tt_str_op(token->error, OP_EQ, "Malformed object: bad begin line"); + + done: + memarea_drop_all(area); + return; +} + +static void +test_parsecommon_get_next_token_err_tag_mismatch(void *arg) +{ + memarea_t *area = memarea_new(); + + const char *str = + "directory-signature 0232AF901C31A04EE9848595AF9BB7620D4C5B2E " + "CD1FD971855430880D3C31E0331C5C55800C2F79\n" + "-----BEGIN SIGNATURE-----\n" + "dLTbc1Lad/OWKBJhA/dERzDHumswTAzBFAWAz2vnQhLsebs1SOm0W/vceEsiEkiF\n" + "A+JJSzIyfywJc6Mnk7aKMEIFjOO/MaxuAp4zv+q+JonJkF0ExjMqvKR0D6pSFmfN\n" + "cnemnxGHxNuPDnKl0imbWKmWDsHtwgi4zWeTq3MekfMOXKi6gIh+bDFzCs9/Vquh\n" + "uNKJI1jW/A2DEKeaSAODEv9VoCsYSvbVVEuHCBWjeNAurd5aL26BrAolW6m7pkD6\n" + "I+cQ8dQG6Wa/Zt6gLXtBbOP2o/iDI7ahDP9diNkBI/rm4nfp9j4piTwsqpi7xz9J\n" + "Ua9DEZB9KbJHVX1rGShrLA==\n" + "-----END SOMETHINGELSE-----\n"; + + const char *end = str + strlen(str); + const char **s = &str; + token_rule_t rule = T("directory-signature", K_DIRECTORY_SIGNATURE, + GE(2), NEED_OBJ); + (void)arg; + + directory_token_t *token = get_next_token(area, s, end, &rule); + + tt_int_op(token->tp, OP_EQ, ERR_); + tt_str_op(token->error, OP_EQ, + "Malformed object: mismatched end tag SIGNATURE"); + + done: + memarea_drop_all(area); + return; +} + +static void +test_parsecommon_get_next_token_err_bad_base64(void *arg) +{ + memarea_t *area = memarea_new(); + + const char *str = + "directory-signature 0232AF901C31A04EE9848595AF9BB7620D4C5B2E " + "CD1FD971855430880D3C31E0331C5C55800C2F79\n" + "-----BEGIN SIGNATURE-----\n" + "%%@%%%%%%%!!!'\n" + "-----END SIGNATURE-----\n"; + + const char *end = str + strlen(str); + const char **s = &str; + token_rule_t rule = T("directory-signature", K_DIRECTORY_SIGNATURE, + GE(2), NEED_OBJ); + (void)arg; + + directory_token_t *token = get_next_token(area, s, end, &rule); + + tt_int_op(token->tp, OP_EQ, ERR_); + tt_str_op(token->error, OP_EQ, "Malformed object: bad base64-encoded data"); + + done: + memarea_drop_all(area); + return; +} + +#define PARSECOMMON_TEST(name) \ + { #name, test_parsecommon_ ## name, 0, NULL, NULL } + +struct testcase_t parsecommon_tests[] = { + PARSECOMMON_TEST(tokenize_string_null), + PARSECOMMON_TEST(tokenize_string_multiple_lines), + PARSECOMMON_TEST(tokenize_string_min_cnt), + PARSECOMMON_TEST(tokenize_string_max_cnt), + PARSECOMMON_TEST(tokenize_string_at_start), + PARSECOMMON_TEST(tokenize_string_at_end), + PARSECOMMON_TEST(tokenize_string_no_annotations), + PARSECOMMON_TEST(get_next_token_success), + PARSECOMMON_TEST(get_next_token_concat_args), + PARSECOMMON_TEST(get_next_token_parse_keys), + PARSECOMMON_TEST(get_next_token_object), + PARSECOMMON_TEST(get_next_token_err_too_many_args), + PARSECOMMON_TEST(get_next_token_err_too_few_args), + PARSECOMMON_TEST(get_next_token_err_obj_missing_endline), + PARSECOMMON_TEST(get_next_token_err_bad_beginline), + PARSECOMMON_TEST(get_next_token_err_tag_mismatch), + PARSECOMMON_TEST(get_next_token_err_bad_base64), + END_OF_TESTCASES +}; diff --git a/src/test/test_periodic_event.c b/src/test/test_periodic_event.c index 86dedd85d8..f3d518eb7b 100644 --- a/src/test/test_periodic_event.c +++ b/src/test/test_periodic_event.c @@ -19,6 +19,7 @@ #include "feature/hibernate/hibernate.h" #include "feature/hs/hs_service.h" #include "core/mainloop/mainloop.h" +#include "core/mainloop/netstatus.h" #include "core/mainloop/periodic.h" /** Helper function: This is replaced in some tests for the event callbacks so @@ -50,6 +51,8 @@ test_pe_initialize(void *arg) * need to run the main loop and then wait for a second delaying the unit * tests. Instead, we'll test the callback work indepedently elsewhere. */ initialize_periodic_events(); + set_network_participation(false); + rescan_periodic_events(get_options()); /* Validate that all events have been set up. */ for (int i = 0; periodic_events[i].name; ++i) { @@ -59,7 +62,9 @@ test_pe_initialize(void *arg) tt_u64_op(item->last_action_time, OP_EQ, 0); /* Every event must have role(s) assign to it. This is done statically. */ tt_u64_op(item->roles, OP_NE, 0); - tt_uint_op(periodic_event_is_enabled(item), OP_EQ, 0); + int should_be_enabled = (item->roles & PERIODIC_EVENT_ROLE_ALL) && + !(item->flags & PERIODIC_EVENT_FLAG_NEED_NET); + tt_uint_op(periodic_event_is_enabled(item), OP_EQ, should_be_enabled); } done: @@ -79,6 +84,8 @@ test_pe_launch(void *arg) * network gets enabled. */ consider_hibernation(time(NULL)); + set_network_participation(true); + /* Hack: We'll set a dumb fn() of each events so they don't get called when * dispatching them. We just want to test the state of the callbacks, not * the whole code path. */ @@ -90,6 +97,7 @@ test_pe_launch(void *arg) options = get_options_mutable(); options->SocksPort_set = 1; periodic_events_on_new_options(options); + #if 0 /* Lets make sure that before intialization, we can't scan the periodic * events list and launch them. Lets try by being a Client. */ @@ -106,13 +114,12 @@ test_pe_launch(void *arg) /* Now that we've initialized, rescan the list to launch. */ periodic_events_on_new_options(options); + int mask = PERIODIC_EVENT_ROLE_CLIENT|PERIODIC_EVENT_ROLE_ALL| + PERIODIC_EVENT_ROLE_NET_PARTICIPANT; for (int i = 0; periodic_events[i].name; ++i) { periodic_event_item_t *item = &periodic_events[i]; - if (item->roles & PERIODIC_EVENT_ROLE_CLIENT) { - tt_int_op(periodic_event_is_enabled(item), OP_EQ, 1); - } else { - tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0); - } + int should_be_enabled = !!(item->roles & mask); + tt_int_op(periodic_event_is_enabled(item), OP_EQ, should_be_enabled); // enabled or not, the event has not yet been run. tt_u64_op(item->last_action_time, OP_EQ, 0); } @@ -124,7 +131,8 @@ test_pe_launch(void *arg) unsigned roles = get_my_roles(options); tt_uint_op(roles, OP_EQ, - PERIODIC_EVENT_ROLE_RELAY|PERIODIC_EVENT_ROLE_DIRSERVER); + PERIODIC_EVENT_ROLE_RELAY|PERIODIC_EVENT_ROLE_DIRSERVER| + PERIODIC_EVENT_ROLE_ALL|PERIODIC_EVENT_ROLE_NET_PARTICIPANT); for (int i = 0; periodic_events[i].name; ++i) { periodic_event_item_t *item = &periodic_events[i]; @@ -144,17 +152,23 @@ test_pe_launch(void *arg) /* Disable everything and we'll enable them ALL. */ options->SocksPort_set = 0; options->ORPort_set = 0; + options->DisableNetwork = 1; + set_network_participation(false); periodic_events_on_new_options(options); for (int i = 0; periodic_events[i].name; ++i) { periodic_event_item_t *item = &periodic_events[i]; - tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0); + int should_be_enabled = (item->roles & PERIODIC_EVENT_ROLE_ALL) && + !(item->flags & PERIODIC_EVENT_FLAG_NEED_NET); + tt_int_op(periodic_event_is_enabled(item), OP_EQ, should_be_enabled); } /* Enable everything. */ options->SocksPort_set = 1; options->ORPort_set = 1; options->BridgeRelay = 1; options->AuthoritativeDir = 1; options->V3AuthoritativeDir = 1; options->BridgeAuthoritativeDir = 1; + options->DisableNetwork = 0; + set_network_participation(true); register_dummy_hidden_service(&service); periodic_events_on_new_options(options); /* Note down the reference because we need to remove this service from the @@ -165,7 +179,8 @@ test_pe_launch(void *arg) for (int i = 0; periodic_events[i].name; ++i) { periodic_event_item_t *item = &periodic_events[i]; - tt_int_op(periodic_event_is_enabled(item), OP_EQ, 1); + tt_int_op(periodic_event_is_enabled(item), OP_EQ, + (item->roles != PERIODIC_EVENT_ROLE_CONTROLEV)); } done: @@ -187,42 +202,49 @@ test_pe_get_roles(void *arg) or_options_t *options = get_options_mutable(); tt_assert(options); + set_network_participation(true); + + const int ALL = PERIODIC_EVENT_ROLE_ALL | + PERIODIC_EVENT_ROLE_NET_PARTICIPANT; /* Nothing configured, should be no roles. */ + tt_assert(net_is_disabled()); roles = get_my_roles(options); - tt_int_op(roles, OP_EQ, 0); + tt_int_op(roles, OP_EQ, ALL); /* Indicate we have a SocksPort, roles should be come Client. */ options->SocksPort_set = 1; roles = get_my_roles(options); - tt_int_op(roles, OP_EQ, PERIODIC_EVENT_ROLE_CLIENT); + tt_int_op(roles, OP_EQ, PERIODIC_EVENT_ROLE_CLIENT|ALL); /* Now, we'll add a ORPort so should now be a Relay + Client. */ options->ORPort_set = 1; roles = get_my_roles(options); tt_int_op(roles, OP_EQ, (PERIODIC_EVENT_ROLE_CLIENT | PERIODIC_EVENT_ROLE_RELAY | - PERIODIC_EVENT_ROLE_DIRSERVER)); + PERIODIC_EVENT_ROLE_DIRSERVER | ALL)); /* Now add a Bridge. */ options->BridgeRelay = 1; roles = get_my_roles(options); tt_int_op(roles, OP_EQ, (PERIODIC_EVENT_ROLE_CLIENT | PERIODIC_EVENT_ROLE_RELAY | - PERIODIC_EVENT_ROLE_BRIDGE | PERIODIC_EVENT_ROLE_DIRSERVER)); + PERIODIC_EVENT_ROLE_BRIDGE | PERIODIC_EVENT_ROLE_DIRSERVER | + ALL)); tt_assert(roles & PERIODIC_EVENT_ROLE_ROUTER); /* Unset client so we can solely test Router role. */ options->SocksPort_set = 0; roles = get_my_roles(options); tt_int_op(roles, OP_EQ, - PERIODIC_EVENT_ROLE_ROUTER | PERIODIC_EVENT_ROLE_DIRSERVER); + PERIODIC_EVENT_ROLE_ROUTER | PERIODIC_EVENT_ROLE_DIRSERVER | + ALL); /* Reset options so we can test authorities. */ options->SocksPort_set = 0; options->ORPort_set = 0; options->BridgeRelay = 0; roles = get_my_roles(options); - tt_int_op(roles, OP_EQ, 0); + tt_int_op(roles, OP_EQ, ALL); /* Now upgrade to Dirauth. */ options->DirPort_set = 1; @@ -230,7 +252,7 @@ test_pe_get_roles(void *arg) options->V3AuthoritativeDir = 1; roles = get_my_roles(options); tt_int_op(roles, OP_EQ, - PERIODIC_EVENT_ROLE_DIRAUTH|PERIODIC_EVENT_ROLE_DIRSERVER); + PERIODIC_EVENT_ROLE_DIRAUTH|PERIODIC_EVENT_ROLE_DIRSERVER|ALL); tt_assert(roles & PERIODIC_EVENT_ROLE_AUTHORITIES); /* Now Bridge Authority. */ @@ -238,7 +260,7 @@ test_pe_get_roles(void *arg) options->BridgeAuthoritativeDir = 1; roles = get_my_roles(options); tt_int_op(roles, OP_EQ, - PERIODIC_EVENT_ROLE_BRIDGEAUTH|PERIODIC_EVENT_ROLE_DIRSERVER); + PERIODIC_EVENT_ROLE_BRIDGEAUTH|PERIODIC_EVENT_ROLE_DIRSERVER|ALL); tt_assert(roles & PERIODIC_EVENT_ROLE_AUTHORITIES); /* Move that bridge auth to become a relay. */ @@ -246,7 +268,7 @@ test_pe_get_roles(void *arg) roles = get_my_roles(options); tt_int_op(roles, OP_EQ, (PERIODIC_EVENT_ROLE_BRIDGEAUTH | PERIODIC_EVENT_ROLE_RELAY - | PERIODIC_EVENT_ROLE_DIRSERVER)); + | PERIODIC_EVENT_ROLE_DIRSERVER|ALL)); tt_assert(roles & PERIODIC_EVENT_ROLE_AUTHORITIES); /* And now an Hidden service. */ @@ -257,7 +279,8 @@ test_pe_get_roles(void *arg) remove_service(get_hs_service_map(), &service); tt_int_op(roles, OP_EQ, (PERIODIC_EVENT_ROLE_BRIDGEAUTH | PERIODIC_EVENT_ROLE_RELAY | - PERIODIC_EVENT_ROLE_HS_SERVICE | PERIODIC_EVENT_ROLE_DIRSERVER)); + PERIODIC_EVENT_ROLE_HS_SERVICE | PERIODIC_EVENT_ROLE_DIRSERVER | + ALL)); tt_assert(roles & PERIODIC_EVENT_ROLE_AUTHORITIES); done: diff --git a/src/test/test_policy.c b/src/test/test_policy.c index afe608f5f7..3820c6c1db 100644 --- a/src/test/test_policy.c +++ b/src/test/test_policy.c @@ -2024,6 +2024,20 @@ test_policies_fascist_firewall_allows_address(void *arg) expect_ap); \ STMT_END +/** Mock the preferred address function to return zero (prefer IPv4). */ +static int +mock_fascist_firewall_rand_prefer_ipv6_addr_use_ipv4(void) +{ + return 0; +} + +/** Mock the preferred address function to return one (prefer IPv6). */ +static int +mock_fascist_firewall_rand_prefer_ipv6_addr_use_ipv6(void) +{ + return 1; +} + /** Run unit tests for fascist_firewall_choose_address */ static void test_policies_fascist_firewall_choose_address(void *arg) @@ -2422,6 +2436,42 @@ test_policies_fascist_firewall_choose_address(void *arg) CHECK_CHOSEN_ADDR_RN(fake_rs, fake_node, FIREWALL_DIR_CONNECTION, 1, 1, ipv4_dir_ap); + /* Test ClientAutoIPv6ORPort and pretend we prefer IPv4. */ + memset(&mock_options, 0, sizeof(or_options_t)); + mock_options.ClientAutoIPv6ORPort = 1; + mock_options.ClientUseIPv4 = 1; + mock_options.ClientUseIPv6 = 1; + MOCK(fascist_firewall_rand_prefer_ipv6_addr, + mock_fascist_firewall_rand_prefer_ipv6_addr_use_ipv4); + /* Simulate the initialisation of fake_node.ipv6_preferred */ + fake_node.ipv6_preferred = fascist_firewall_prefer_ipv6_orport( + &mock_options); + + CHECK_CHOSEN_ADDR_RN(fake_rs, fake_node, FIREWALL_OR_CONNECTION, 0, 1, + ipv4_or_ap); + CHECK_CHOSEN_ADDR_RN(fake_rs, fake_node, FIREWALL_OR_CONNECTION, 1, 1, + ipv4_or_ap); + + UNMOCK(fascist_firewall_rand_prefer_ipv6_addr); + + /* Test ClientAutoIPv6ORPort and pretend we prefer IPv6. */ + memset(&mock_options, 0, sizeof(or_options_t)); + mock_options.ClientAutoIPv6ORPort = 1; + mock_options.ClientUseIPv4 = 1; + mock_options.ClientUseIPv6 = 1; + MOCK(fascist_firewall_rand_prefer_ipv6_addr, + mock_fascist_firewall_rand_prefer_ipv6_addr_use_ipv6); + /* Simulate the initialisation of fake_node.ipv6_preferred */ + fake_node.ipv6_preferred = fascist_firewall_prefer_ipv6_orport( + &mock_options); + + CHECK_CHOSEN_ADDR_RN(fake_rs, fake_node, FIREWALL_OR_CONNECTION, 0, 1, + ipv6_or_ap); + CHECK_CHOSEN_ADDR_RN(fake_rs, fake_node, FIREWALL_OR_CONNECTION, 1, 1, + ipv6_or_ap); + + UNMOCK(fascist_firewall_rand_prefer_ipv6_addr); + done: UNMOCK(get_options); } diff --git a/src/test/test_process.c b/src/test/test_process.c new file mode 100644 index 0000000000..27c9935e0d --- /dev/null +++ b/src/test/test_process.c @@ -0,0 +1,669 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file test_process.c + * \brief Test cases for the Process API. + */ + +#include "orconfig.h" +#include "core/or/or.h" +#include "test/test.h" +#include "lib/process/env.h" + +#define PROCESS_PRIVATE +#include "lib/process/process.h" +#define PROCESS_UNIX_PRIVATE +#include "lib/process/process_unix.h" +#define PROCESS_WIN32_PRIVATE +#include "lib/process/process_win32.h" + +static const char *stdout_read_buffer; +static const char *stderr_read_buffer; + +struct process_data_t { + smartlist_t *stdout_data; + smartlist_t *stderr_data; + smartlist_t *stdin_data; + process_exit_code_t exit_code; +}; + +typedef struct process_data_t process_data_t; + +static process_data_t * +process_data_new(void) +{ + process_data_t *process_data = tor_malloc_zero(sizeof(process_data_t)); + process_data->stdout_data = smartlist_new(); + process_data->stderr_data = smartlist_new(); + process_data->stdin_data = smartlist_new(); + return process_data; +} + +static void +process_data_free(process_data_t *process_data) +{ + if (process_data == NULL) + return; + + SMARTLIST_FOREACH(process_data->stdout_data, char *, x, tor_free(x)); + SMARTLIST_FOREACH(process_data->stderr_data, char *, x, tor_free(x)); + SMARTLIST_FOREACH(process_data->stdin_data, char *, x, tor_free(x)); + + smartlist_free(process_data->stdout_data); + smartlist_free(process_data->stderr_data); + smartlist_free(process_data->stdin_data); + tor_free(process_data); +} + +static int +process_mocked_read_stdout(process_t *process, buf_t *buffer) +{ + (void)process; + + if (stdout_read_buffer != NULL) { + buf_add_string(buffer, stdout_read_buffer); + stdout_read_buffer = NULL; + } + + return (int)buf_datalen(buffer); +} + +static int +process_mocked_read_stderr(process_t *process, buf_t *buffer) +{ + (void)process; + + if (stderr_read_buffer != NULL) { + buf_add_string(buffer, stderr_read_buffer); + stderr_read_buffer = NULL; + } + + return (int)buf_datalen(buffer); +} + +static void +process_mocked_write_stdin(process_t *process, buf_t *buffer) +{ + const size_t size = buf_datalen(buffer); + + if (size == 0) + return; + + char *data = tor_malloc_zero(size + 1); + process_data_t *process_data = process_get_data(process); + + buf_get_bytes(buffer, data, size); + smartlist_add(process_data->stdin_data, data); +} + +static void +process_stdout_callback(process_t *process, const char *data, size_t size) +{ + tt_ptr_op(process, OP_NE, NULL); + tt_ptr_op(data, OP_NE, NULL); + tt_int_op(strlen(data), OP_EQ, size); + + process_data_t *process_data = process_get_data(process); + smartlist_add(process_data->stdout_data, tor_strdup(data)); + + done: + return; +} + +static void +process_stderr_callback(process_t *process, const char *data, size_t size) +{ + tt_ptr_op(process, OP_NE, NULL); + tt_ptr_op(data, OP_NE, NULL); + tt_int_op(strlen(data), OP_EQ, size); + + process_data_t *process_data = process_get_data(process); + smartlist_add(process_data->stderr_data, tor_strdup(data)); + + done: + return; +} + +static bool +process_exit_callback(process_t *process, process_exit_code_t exit_code) +{ + tt_ptr_op(process, OP_NE, NULL); + + process_data_t *process_data = process_get_data(process); + process_data->exit_code = exit_code; + + done: + /* Do not free up our process_t. */ + return false; +} + +static void +test_default_values(void *arg) +{ + (void)arg; + process_t *process = process_new("/path/to/nothing"); + + /* We are not running by default. */ + tt_int_op(PROCESS_STATUS_NOT_RUNNING, OP_EQ, process_get_status(process)); + + /* We use the line protocol by default. */ + tt_int_op(PROCESS_PROTOCOL_LINE, OP_EQ, process_get_protocol(process)); + + /* We don't set any custom data by default. */ + tt_ptr_op(NULL, OP_EQ, process_get_data(process)); + + /* Our command was given to the process_t's constructor in process_new(). */ + tt_str_op("/path/to/nothing", OP_EQ, process_get_command(process)); + + /* Make sure we are listed in the list of proccesses. */ + tt_assert(smartlist_contains(process_get_all_processes(), + process)); + + /* Default PID is 0. */ + tt_int_op(0, OP_EQ, process_get_pid(process)); + + /* Our arguments should be empty. */ + tt_int_op(0, OP_EQ, + smartlist_len(process_get_arguments(process))); + + done: + process_free(process); +} + +static void +test_environment(void *arg) +{ + (void)arg; + + process_t *process = process_new(""); + process_environment_t *env = NULL; + + process_set_environment(process, "E", "F"); + process_set_environment(process, "C", "D"); + process_set_environment(process, "A", "B"); + + env = process_get_environment(process); + tt_mem_op(env->windows_environment_block, OP_EQ, + "A=B\0C=D\0E=F\0", 12); + tt_str_op(env->unixoid_environment_block[0], OP_EQ, + "A=B"); + tt_str_op(env->unixoid_environment_block[1], OP_EQ, + "C=D"); + tt_str_op(env->unixoid_environment_block[2], OP_EQ, + "E=F"); + tt_ptr_op(env->unixoid_environment_block[3], OP_EQ, + NULL); + process_environment_free(env); + + /* Reset our environment. */ + smartlist_t *new_env = smartlist_new(); + smartlist_add(new_env, (char *)"FOO=bar"); + smartlist_add(new_env, (char *)"HELLO=world"); + + process_reset_environment(process, new_env); + smartlist_free(new_env); + + env = process_get_environment(process); + tt_mem_op(env->windows_environment_block, OP_EQ, + "FOO=bar\0HELLO=world\0", 20); + tt_str_op(env->unixoid_environment_block[0], OP_EQ, + "FOO=bar"); + tt_str_op(env->unixoid_environment_block[1], OP_EQ, + "HELLO=world"); + tt_ptr_op(env->unixoid_environment_block[2], OP_EQ, + NULL); + + done: + process_environment_free(env); + process_free(process); +} + +static void +test_stringified_types(void *arg) +{ + (void)arg; + + /* process_protocol_t values. */ + tt_str_op("Raw", OP_EQ, process_protocol_to_string(PROCESS_PROTOCOL_RAW)); + tt_str_op("Line", OP_EQ, process_protocol_to_string(PROCESS_PROTOCOL_LINE)); + + /* process_status_t values. */ + tt_str_op("not running", OP_EQ, + process_status_to_string(PROCESS_STATUS_NOT_RUNNING)); + tt_str_op("running", OP_EQ, + process_status_to_string(PROCESS_STATUS_RUNNING)); + tt_str_op("error", OP_EQ, + process_status_to_string(PROCESS_STATUS_ERROR)); + + done: + return; +} + +static void +test_line_protocol_simple(void *arg) +{ + (void)arg; + + process_data_t *process_data = process_data_new(); + + process_t *process = process_new(""); + process_set_data(process, process_data); + + process_set_stdout_read_callback(process, process_stdout_callback); + process_set_stderr_read_callback(process, process_stderr_callback); + + MOCK(process_read_stdout, process_mocked_read_stdout); + MOCK(process_read_stderr, process_mocked_read_stderr); + + /* Make sure we are running with the line protocol. */ + tt_int_op(PROCESS_PROTOCOL_LINE, OP_EQ, process_get_protocol(process)); + + tt_int_op(0, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(0, OP_EQ, smartlist_len(process_data->stderr_data)); + + stdout_read_buffer = "Hello stdout\n"; + process_notify_event_stdout(process); + tt_ptr_op(NULL, OP_EQ, stdout_read_buffer); + + stderr_read_buffer = "Hello stderr\r\n"; + process_notify_event_stderr(process); + tt_ptr_op(NULL, OP_EQ, stderr_read_buffer); + + /* Data should be ready. */ + tt_int_op(1, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(1, OP_EQ, smartlist_len(process_data->stderr_data)); + + /* Check if the data is correct. */ + tt_str_op(smartlist_get(process_data->stdout_data, 0), OP_EQ, + "Hello stdout"); + tt_str_op(smartlist_get(process_data->stderr_data, 0), OP_EQ, + "Hello stderr"); + + done: + process_data_free(process_data); + process_free(process); + + UNMOCK(process_read_stdout); + UNMOCK(process_read_stderr); +} + +static void +test_line_protocol_multi(void *arg) +{ + (void)arg; + + process_data_t *process_data = process_data_new(); + + process_t *process = process_new(""); + process_set_data(process, process_data); + process_set_stdout_read_callback(process, process_stdout_callback); + process_set_stderr_read_callback(process, process_stderr_callback); + + MOCK(process_read_stdout, process_mocked_read_stdout); + MOCK(process_read_stderr, process_mocked_read_stderr); + + /* Make sure we are running with the line protocol. */ + tt_int_op(PROCESS_PROTOCOL_LINE, OP_EQ, process_get_protocol(process)); + + tt_int_op(0, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(0, OP_EQ, smartlist_len(process_data->stderr_data)); + + stdout_read_buffer = "Hello stdout\r\nOnion Onion Onion\nA B C D\r\n\r\n"; + process_notify_event_stdout(process); + tt_ptr_op(NULL, OP_EQ, stdout_read_buffer); + + stderr_read_buffer = "Hello stderr\nFoo bar baz\nOnion Onion Onion\n"; + process_notify_event_stderr(process); + tt_ptr_op(NULL, OP_EQ, stderr_read_buffer); + + /* Data should be ready. */ + tt_int_op(4, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(3, OP_EQ, smartlist_len(process_data->stderr_data)); + + /* Check if the data is correct. */ + tt_str_op(smartlist_get(process_data->stdout_data, 0), OP_EQ, + "Hello stdout"); + tt_str_op(smartlist_get(process_data->stdout_data, 1), OP_EQ, + "Onion Onion Onion"); + tt_str_op(smartlist_get(process_data->stdout_data, 2), OP_EQ, + "A B C D"); + tt_str_op(smartlist_get(process_data->stdout_data, 3), OP_EQ, + ""); + + tt_str_op(smartlist_get(process_data->stderr_data, 0), OP_EQ, + "Hello stderr"); + tt_str_op(smartlist_get(process_data->stderr_data, 1), OP_EQ, + "Foo bar baz"); + tt_str_op(smartlist_get(process_data->stderr_data, 2), OP_EQ, + "Onion Onion Onion"); + + done: + process_data_free(process_data); + process_free(process); + + UNMOCK(process_read_stdout); + UNMOCK(process_read_stderr); +} + +static void +test_line_protocol_partial(void *arg) +{ + (void)arg; + + process_data_t *process_data = process_data_new(); + + process_t *process = process_new(""); + process_set_data(process, process_data); + process_set_stdout_read_callback(process, process_stdout_callback); + process_set_stderr_read_callback(process, process_stderr_callback); + + MOCK(process_read_stdout, process_mocked_read_stdout); + MOCK(process_read_stderr, process_mocked_read_stderr); + + /* Make sure we are running with the line protocol. */ + tt_int_op(PROCESS_PROTOCOL_LINE, OP_EQ, process_get_protocol(process)); + + tt_int_op(0, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(0, OP_EQ, smartlist_len(process_data->stderr_data)); + + stdout_read_buffer = "Hello stdout this is a partial line ..."; + process_notify_event_stdout(process); + tt_ptr_op(NULL, OP_EQ, stdout_read_buffer); + + stderr_read_buffer = "Hello stderr this is a partial line ..."; + process_notify_event_stderr(process); + tt_ptr_op(NULL, OP_EQ, stderr_read_buffer); + + /* Data should NOT be ready. */ + tt_int_op(0, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(0, OP_EQ, smartlist_len(process_data->stderr_data)); + + stdout_read_buffer = " the end\nAnother partial string goes here ..."; + process_notify_event_stdout(process); + tt_ptr_op(NULL, OP_EQ, stdout_read_buffer); + + stderr_read_buffer = " the end\nAnother partial string goes here ..."; + process_notify_event_stderr(process); + tt_ptr_op(NULL, OP_EQ, stderr_read_buffer); + + /* Some data should be ready. */ + tt_int_op(1, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(1, OP_EQ, smartlist_len(process_data->stderr_data)); + + stdout_read_buffer = " the end\nFoo bar baz\n"; + process_notify_event_stdout(process); + tt_ptr_op(NULL, OP_EQ, stdout_read_buffer); + + stderr_read_buffer = " the end\nFoo bar baz\n"; + process_notify_event_stderr(process); + tt_ptr_op(NULL, OP_EQ, stderr_read_buffer); + + /* Some data should be ready. */ + tt_int_op(3, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(3, OP_EQ, smartlist_len(process_data->stderr_data)); + + /* Check if the data is correct. */ + tt_str_op(smartlist_get(process_data->stdout_data, 0), OP_EQ, + "Hello stdout this is a partial line ... the end"); + tt_str_op(smartlist_get(process_data->stdout_data, 1), OP_EQ, + "Another partial string goes here ... the end"); + tt_str_op(smartlist_get(process_data->stdout_data, 2), OP_EQ, + "Foo bar baz"); + + tt_str_op(smartlist_get(process_data->stderr_data, 0), OP_EQ, + "Hello stderr this is a partial line ... the end"); + tt_str_op(smartlist_get(process_data->stderr_data, 1), OP_EQ, + "Another partial string goes here ... the end"); + tt_str_op(smartlist_get(process_data->stderr_data, 2), OP_EQ, + "Foo bar baz"); + + done: + process_data_free(process_data); + process_free(process); + + UNMOCK(process_read_stdout); + UNMOCK(process_read_stderr); +} + +static void +test_raw_protocol_simple(void *arg) +{ + (void)arg; + + process_data_t *process_data = process_data_new(); + + process_t *process = process_new(""); + process_set_data(process, process_data); + process_set_protocol(process, PROCESS_PROTOCOL_RAW); + + process_set_stdout_read_callback(process, process_stdout_callback); + process_set_stderr_read_callback(process, process_stderr_callback); + + MOCK(process_read_stdout, process_mocked_read_stdout); + MOCK(process_read_stderr, process_mocked_read_stderr); + + /* Make sure we are running with the raw protocol. */ + tt_int_op(PROCESS_PROTOCOL_RAW, OP_EQ, process_get_protocol(process)); + + tt_int_op(0, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(0, OP_EQ, smartlist_len(process_data->stderr_data)); + + stdout_read_buffer = "Hello stdout\n"; + process_notify_event_stdout(process); + tt_ptr_op(NULL, OP_EQ, stdout_read_buffer); + + stderr_read_buffer = "Hello stderr\n"; + process_notify_event_stderr(process); + tt_ptr_op(NULL, OP_EQ, stderr_read_buffer); + + /* Data should be ready. */ + tt_int_op(1, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(1, OP_EQ, smartlist_len(process_data->stderr_data)); + + stdout_read_buffer = "Hello, again, stdout\nThis contains multiple lines"; + process_notify_event_stdout(process); + tt_ptr_op(NULL, OP_EQ, stdout_read_buffer); + + stderr_read_buffer = "Hello, again, stderr\nThis contains multiple lines"; + process_notify_event_stderr(process); + tt_ptr_op(NULL, OP_EQ, stderr_read_buffer); + + /* Data should be ready. */ + tt_int_op(2, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(2, OP_EQ, smartlist_len(process_data->stderr_data)); + + /* Check if the data is correct. */ + tt_str_op(smartlist_get(process_data->stdout_data, 0), OP_EQ, + "Hello stdout\n"); + tt_str_op(smartlist_get(process_data->stdout_data, 1), OP_EQ, + "Hello, again, stdout\nThis contains multiple lines"); + + tt_str_op(smartlist_get(process_data->stderr_data, 0), OP_EQ, + "Hello stderr\n"); + tt_str_op(smartlist_get(process_data->stderr_data, 1), OP_EQ, + "Hello, again, stderr\nThis contains multiple lines"); + + done: + process_data_free(process_data); + process_free(process); + + UNMOCK(process_read_stdout); + UNMOCK(process_read_stderr); +} + +static void +test_write_simple(void *arg) +{ + (void)arg; + + process_data_t *process_data = process_data_new(); + + process_t *process = process_new(""); + process_set_data(process, process_data); + + MOCK(process_write_stdin, process_mocked_write_stdin); + + process_write(process, (uint8_t *)"Hello world\n", 12); + process_notify_event_stdin(process); + tt_int_op(1, OP_EQ, smartlist_len(process_data->stdin_data)); + + process_printf(process, "Hello %s !\n", "moon"); + process_notify_event_stdin(process); + tt_int_op(2, OP_EQ, smartlist_len(process_data->stdin_data)); + + done: + process_data_free(process_data); + process_free(process); + + UNMOCK(process_write_stdin); +} + +static void +test_exit_simple(void *arg) +{ + (void)arg; + + process_data_t *process_data = process_data_new(); + + process_t *process = process_new(""); + process_set_data(process, process_data); + process_set_exit_callback(process, process_exit_callback); + + /* Our default is 0. */ + tt_int_op(0, OP_EQ, process_data->exit_code); + + /* Fake that we are a running process. */ + process_set_status(process, PROCESS_STATUS_RUNNING); + tt_int_op(process_get_status(process), OP_EQ, PROCESS_STATUS_RUNNING); + + /* Fake an exit. */ + process_notify_event_exit(process, 1337); + + /* Check if our state changed and if our callback fired. */ + tt_int_op(process_get_status(process), OP_EQ, PROCESS_STATUS_NOT_RUNNING); + tt_int_op(1337, OP_EQ, process_data->exit_code); + + done: + process_set_data(process, process_data); + process_data_free(process_data); + process_free(process); +} + +static void +test_argv_simple(void *arg) +{ + (void)arg; + + process_t *process = process_new("/bin/cat"); + char **argv = NULL; + + /* Setup some arguments. */ + process_append_argument(process, "foo"); + process_append_argument(process, "bar"); + process_append_argument(process, "baz"); + + /* Check the number of elements. */ + tt_int_op(3, OP_EQ, + smartlist_len(process_get_arguments(process))); + + /* Let's try to convert it into a Unix style char **argv. */ + argv = process_get_argv(process); + + /* Check our values. */ + tt_str_op(argv[0], OP_EQ, "/bin/cat"); + tt_str_op(argv[1], OP_EQ, "foo"); + tt_str_op(argv[2], OP_EQ, "bar"); + tt_str_op(argv[3], OP_EQ, "baz"); + tt_ptr_op(argv[4], OP_EQ, NULL); + + done: + tor_free(argv); + process_free(process); +} + +static void +test_unix(void *arg) +{ + (void)arg; +#ifndef _WIN32 + process_t *process = process_new(""); + + /* On Unix all processes should have a Unix process handle. */ + tt_ptr_op(NULL, OP_NE, process_get_unix_process(process)); + + done: + process_free(process); +#endif +} + +static void +test_win32(void *arg) +{ + (void)arg; +#ifdef _WIN32 + process_t *process = process_new(""); + char *joined_argv = NULL; + + /* On Win32 all processes should have a Win32 process handle. */ + tt_ptr_op(NULL, OP_NE, process_get_win32_process(process)); + + /* Based on some test cases from "Parsing C++ Command-Line Arguments" in + * MSDN but we don't exercise all quoting rules because tor_join_win_cmdline + * will try to only generate simple cases for the child process to parse; + * i.e. we never embed quoted strings in arguments. */ + + const char *argvs[][4] = { + {"a", "bb", "CCC", NULL}, // Normal + {NULL, NULL, NULL, NULL}, // Empty argument list + {"", NULL, NULL, NULL}, // Empty argument + {"\"a", "b\"b", "CCC\"", NULL}, // Quotes + {"a\tbc", "dd dd", "E", NULL}, // Whitespace + {"a\\\\\\b", "de fg", "H", NULL}, // Backslashes + {"a\\\"b", "\\c", "D\\", NULL}, // Backslashes before quote + {"a\\\\b c", "d", "E", NULL}, // Backslashes not before quote + { NULL } // Terminator + }; + + const char *cmdlines[] = { + "a bb CCC", + "", + "\"\"", + "\\\"a b\\\"b CCC\\\"", + "\"a\tbc\" \"dd dd\" E", + "a\\\\\\b \"de fg\" H", + "a\\\\\\\"b \\c D\\", + "\"a\\\\b c\" d E", + NULL // Terminator + }; + + int i; + + for (i=0; cmdlines[i]!=NULL; i++) { + log_info(LD_GENERAL, "Joining argvs[%d], expecting <%s>", i, cmdlines[i]); + joined_argv = tor_join_win_cmdline(argvs[i]); + tt_str_op(cmdlines[i],OP_EQ, joined_argv); + tor_free(joined_argv); + } + + done: + tor_free(joined_argv); + process_free(process); +#endif +} + +struct testcase_t process_tests[] = { + { "default_values", test_default_values, TT_FORK, NULL, NULL }, + { "environment", test_environment, TT_FORK, NULL, NULL }, + { "stringified_types", test_stringified_types, TT_FORK, NULL, NULL }, + { "line_protocol_simple", test_line_protocol_simple, TT_FORK, NULL, NULL }, + { "line_protocol_multi", test_line_protocol_multi, TT_FORK, NULL, NULL }, + { "line_protocol_partial", test_line_protocol_partial, TT_FORK, NULL, NULL }, + { "raw_protocol_simple", test_raw_protocol_simple, TT_FORK, NULL, NULL }, + { "write_simple", test_write_simple, TT_FORK, NULL, NULL }, + { "exit_simple", test_exit_simple, TT_FORK, NULL, NULL }, + { "argv_simple", test_argv_simple, TT_FORK, NULL, NULL }, + { "unix", test_unix, TT_FORK, NULL, NULL }, + { "win32", test_win32, TT_FORK, NULL, NULL }, + END_OF_TESTCASES +}; diff --git a/src/test/test_process_slow.c b/src/test/test_process_slow.c new file mode 100644 index 0000000000..c28ade033c --- /dev/null +++ b/src/test/test_process_slow.c @@ -0,0 +1,335 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file test_process_slow.c + * \brief Slow test cases for the Process API. + */ + +#define MAINLOOP_PRIVATE +#include "orconfig.h" +#include "core/or/or.h" +#include "core/mainloop/mainloop.h" +#include "lib/evloop/compat_libevent.h" +#include "lib/process/process.h" +#include "lib/process/waitpid.h" +#include "test/test.h" + +#ifndef BUILDDIR +#define BUILDDIR "." +#endif + +#ifdef _WIN32 +#define TEST_PROCESS "test-process.exe" +#else +#define TEST_PROCESS BUILDDIR "/src/test/test-process" +#endif /* defined(_WIN32) */ + +/** Timer that ticks once a second and stop the event loop after 5 ticks. */ +static periodic_timer_t *main_loop_timeout_timer; + +/** How many times have our timer ticked? */ +static int timer_tick_count; + +struct process_data_t { + smartlist_t *stdout_data; + smartlist_t *stderr_data; + smartlist_t *stdin_data; + process_exit_code_t exit_code; + bool did_exit; +}; + +typedef struct process_data_t process_data_t; + +static process_data_t * +process_data_new(void) +{ + process_data_t *process_data = tor_malloc_zero(sizeof(process_data_t)); + process_data->stdout_data = smartlist_new(); + process_data->stderr_data = smartlist_new(); + process_data->stdin_data = smartlist_new(); + return process_data; +} + +static void +process_data_free(process_data_t *process_data) +{ + if (process_data == NULL) + return; + + SMARTLIST_FOREACH(process_data->stdout_data, char *, x, tor_free(x)); + SMARTLIST_FOREACH(process_data->stderr_data, char *, x, tor_free(x)); + SMARTLIST_FOREACH(process_data->stdin_data, char *, x, tor_free(x)); + + smartlist_free(process_data->stdout_data); + smartlist_free(process_data->stderr_data); + smartlist_free(process_data->stdin_data); + tor_free(process_data); +} + +static void +process_stdout_callback(process_t *process, const char *data, size_t size) +{ + tt_ptr_op(process, OP_NE, NULL); + tt_ptr_op(data, OP_NE, NULL); + tt_int_op(strlen(data), OP_EQ, size); + + process_data_t *process_data = process_get_data(process); + smartlist_add(process_data->stdout_data, tor_strdup(data)); + + done: + return; +} + +static void +process_stderr_callback(process_t *process, const char *data, size_t size) +{ + tt_ptr_op(process, OP_NE, NULL); + tt_ptr_op(data, OP_NE, NULL); + tt_int_op(strlen(data), OP_EQ, size); + + process_data_t *process_data = process_get_data(process); + smartlist_add(process_data->stderr_data, tor_strdup(data)); + + done: + return; +} + +static bool +process_exit_callback(process_t *process, process_exit_code_t exit_code) +{ + process_status_t status; + + tt_ptr_op(process, OP_NE, NULL); + + process_data_t *process_data = process_get_data(process); + process_data->exit_code = exit_code; + process_data->did_exit = true; + + /* Check if our process is still running? */ + status = process_get_status(process); + tt_int_op(status, OP_EQ, PROCESS_STATUS_NOT_RUNNING); + + done: + /* Do not free up our process_t. */ + return false; +} + +#ifdef _WIN32 +static const char * +get_win32_test_binary_path(void) +{ + static char buffer[MAX_PATH]; + + /* Get the absolute path of our binary: \path\to\test-slow.exe. */ + GetModuleFileNameA(GetModuleHandle(0), buffer, sizeof(buffer)); + + /* Find our process name. */ + char *offset = strstr(buffer, "test-slow.exe"); + tt_ptr_op(offset, OP_NE, NULL); + + /* Change test-slow.exe to test-process.exe. */ + memcpy(offset, TEST_PROCESS, strlen(TEST_PROCESS)); + + return buffer; + done: + return NULL; +} +#endif + +static void +main_loop_timeout_cb(periodic_timer_t *timer, void *data) +{ + /* Sanity check. */ + tt_ptr_op(timer, OP_EQ, main_loop_timeout_timer); + tt_ptr_op(data, OP_NE, NULL); + + /* Our process data. */ + process_data_t *process_data = data; + + /* Our process did exit. */ + if (process_data->did_exit) + tor_shutdown_event_loop_and_exit(0); + + /* Have we been called 10 times we exit the main loop. */ + timer_tick_count++; + + tt_int_op(timer_tick_count, OP_LT, 10); + +#ifndef _WIN32 + /* Call waitpid callbacks. */ + notify_pending_waitpid_callbacks(); +#endif + + return; + done: + /* Exit with an error. */ + tor_shutdown_event_loop_and_exit(-1); +} + +static void +run_main_loop(process_data_t *process_data) +{ + int ret; + + /* Wake up after 1 seconds. */ + static const struct timeval interval = {1, 0}; + + timer_tick_count = 0; + main_loop_timeout_timer = periodic_timer_new(tor_libevent_get_base(), + &interval, + main_loop_timeout_cb, + process_data); + + /* Run our main loop. */ + ret = run_main_loop_until_done(); + + /* Clean up our main loop timeout timer. */ + tt_int_op(ret, OP_EQ, 0); + + done: + periodic_timer_free(main_loop_timeout_timer); +} + +static void +test_callbacks(void *arg) +{ + (void)arg; + const char *filename = NULL; + +#ifdef _WIN32 + filename = get_win32_test_binary_path(); +#else + filename = TEST_PROCESS; +#endif + + /* Process callback data. */ + process_data_t *process_data = process_data_new(); + + /* Setup our process. */ + process_t *process = process_new(filename); + process_set_data(process, process_data); + process_set_stdout_read_callback(process, process_stdout_callback); + process_set_stderr_read_callback(process, process_stderr_callback); + process_set_exit_callback(process, process_exit_callback); + + /* Set environment variable. */ + process_set_environment(process, "TOR_TEST_ENV", "Hello, from Tor!"); + + /* Add some arguments. */ + process_append_argument(process, "This is the first one"); + process_append_argument(process, "Second one"); + process_append_argument(process, "Third: Foo bar baz"); + + /* Run our process. */ + process_status_t status; + + status = process_exec(process); + tt_int_op(status, OP_EQ, PROCESS_STATUS_RUNNING); + + /* Write some lines to stdin. */ + process_printf(process, "Hi process!\r\n"); + process_printf(process, "Can you read more than one line?\n"); + process_printf(process, "Can you read partial ..."); + process_printf(process, " lines?\r\n"); + + /* Start our main loop. */ + run_main_loop(process_data); + + /* We returned. Let's see what our event loop said. */ + tt_int_op(smartlist_len(process_data->stdout_data), OP_EQ, 12); + tt_int_op(smartlist_len(process_data->stderr_data), OP_EQ, 3); + tt_assert(process_data->did_exit); + tt_int_op(process_data->exit_code, OP_EQ, 0); + + /* Check stdout output. */ + char argv0_expected[256]; + tor_snprintf(argv0_expected, sizeof(argv0_expected), + "argv[0] = '%s'", filename); + + tt_str_op(smartlist_get(process_data->stdout_data, 0), OP_EQ, + argv0_expected); + tt_str_op(smartlist_get(process_data->stdout_data, 1), OP_EQ, + "argv[1] = 'This is the first one'"); + tt_str_op(smartlist_get(process_data->stdout_data, 2), OP_EQ, + "argv[2] = 'Second one'"); + tt_str_op(smartlist_get(process_data->stdout_data, 3), OP_EQ, + "argv[3] = 'Third: Foo bar baz'"); + tt_str_op(smartlist_get(process_data->stdout_data, 4), OP_EQ, + "Environment variable TOR_TEST_ENV = 'Hello, from Tor!'"); + tt_str_op(smartlist_get(process_data->stdout_data, 5), OP_EQ, + "Output on stdout"); + tt_str_op(smartlist_get(process_data->stdout_data, 6), OP_EQ, + "This is a new line"); + tt_str_op(smartlist_get(process_data->stdout_data, 7), OP_EQ, + "Partial line on stdout ...end of partial line on stdout"); + tt_str_op(smartlist_get(process_data->stdout_data, 8), OP_EQ, + "Read line from stdin: 'Hi process!'"); + tt_str_op(smartlist_get(process_data->stdout_data, 9), OP_EQ, + "Read line from stdin: 'Can you read more than one line?'"); + tt_str_op(smartlist_get(process_data->stdout_data, 10), OP_EQ, + "Read line from stdin: 'Can you read partial ... lines?'"); + tt_str_op(smartlist_get(process_data->stdout_data, 11), OP_EQ, + "We are done for here, thank you!"); + + /* Check stderr output. */ + tt_str_op(smartlist_get(process_data->stderr_data, 0), OP_EQ, + "Output on stderr"); + tt_str_op(smartlist_get(process_data->stderr_data, 1), OP_EQ, + "This is a new line"); + tt_str_op(smartlist_get(process_data->stderr_data, 2), OP_EQ, + "Partial line on stderr ...end of partial line on stderr"); + + done: + process_data_free(process_data); + process_free(process); +} + +static void +test_callbacks_terminate(void *arg) +{ + (void)arg; + const char *filename = NULL; + +#ifdef _WIN32 + filename = get_win32_test_binary_path(); +#else + filename = TEST_PROCESS; +#endif + + /* Process callback data. */ + process_data_t *process_data = process_data_new(); + + /* Setup our process. */ + process_t *process = process_new(filename); + process_set_data(process, process_data); + process_set_exit_callback(process, process_exit_callback); + + /* Run our process. */ + process_status_t status; + + status = process_exec(process); + tt_int_op(status, OP_EQ, PROCESS_STATUS_RUNNING); + + /* Zap our process. */ + bool success; + + success = process_terminate(process); + tt_assert(success); + + /* Start our main loop. */ + run_main_loop(process_data); + + /* Check if we did exit. */ + tt_assert(process_data->did_exit); + + done: + process_data_free(process_data); + process_free(process); +} + +struct testcase_t slow_process_tests[] = { + { "callbacks", test_callbacks, 0, NULL, NULL }, + { "callbacks_terminate", test_callbacks_terminate, 0, NULL, NULL }, + END_OF_TESTCASES +}; diff --git a/src/test/test_proto_http.c b/src/test/test_proto_http.c index 1cfa0a752c..b4e8278423 100644 --- a/src/test/test_proto_http.c +++ b/src/test/test_proto_http.c @@ -8,7 +8,7 @@ #include "core/or/or.h" #include "test/test.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/proto/proto_http.h" #include "test/log_test_helpers.h" diff --git a/src/test/test_proto_misc.c b/src/test/test_proto_misc.c index 1fcb763421..f7f6f69667 100644 --- a/src/test/test_proto_misc.c +++ b/src/test/test_proto_misc.c @@ -8,7 +8,7 @@ #include "core/or/or.h" #include "test/test.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/or/connection_or.h" #include "feature/relay/ext_orport.h" #include "core/proto/proto_cell.h" diff --git a/src/test/test_pt.c b/src/test/test_pt.c index d0160d1148..60a044aeca 100644 --- a/src/test/test_pt.c +++ b/src/test/test_pt.c @@ -8,7 +8,7 @@ #define UTIL_PRIVATE #define STATEFILE_PRIVATE #define CONTROL_PRIVATE -#define SUBPROCESS_PRIVATE +#define PROCESS_PRIVATE #include "core/or/or.h" #include "app/config/config.h" #include "app/config/confparse.h" @@ -17,9 +17,9 @@ #include "core/or/circuitbuild.h" #include "app/config/statefile.h" #include "test/test.h" -#include "lib/process/subprocess.h" #include "lib/encoding/confline.h" #include "lib/net/resolve.h" +#include "lib/process/process.h" #include "app/config/or_state_st.h" @@ -288,41 +288,35 @@ test_pt_get_extrainfo_string(void *arg) tor_free(s); } -#ifdef _WIN32 -#define STDIN_HANDLE HANDLE* -#else -#define STDIN_HANDLE int -#endif - -static smartlist_t * -tor_get_lines_from_handle_replacement(STDIN_HANDLE handle, - enum stream_status *stream_status_out) +static int +process_read_stdout_replacement(process_t *process, buf_t *buffer) { + (void)process; static int times_called = 0; - smartlist_t *retval_sl = smartlist_new(); - - (void) handle; - (void) stream_status_out; /* Generate some dummy CMETHOD lines the first 5 times. The 6th time, send 'CMETHODS DONE' to finish configuring the proxy. */ - if (times_called++ != 5) { - smartlist_add_asprintf(retval_sl, "SMETHOD mock%d 127.0.0.1:555%d", + times_called++; + + if (times_called <= 5) { + buf_add_printf(buffer, "SMETHOD mock%d 127.0.0.1:555%d\n", times_called, times_called); - } else { - smartlist_add_strdup(retval_sl, "SMETHODS DONE"); + } else if (times_called <= 6) { + buf_add_string(buffer, "SMETHODS DONE\n"); + } else if (times_called <= 7) { + buf_add_string(buffer, "LOG SEVERITY=error MESSAGE=\"Oh noes, something " + "bad happened. What do we do!?\"\n"); + buf_add_string(buffer, "LOG SEVERITY=warning MESSAGE=\"warning msg\"\n"); + buf_add_string(buffer, "LOG SEVERITY=notice MESSAGE=\"notice msg\"\n"); + buf_add_string(buffer, "LOG SEVERITY=info MESSAGE=\"info msg\"\n"); + buf_add_string(buffer, "LOG SEVERITY=debug MESSAGE=\"debug msg\"\n"); + } else if (times_called <= 8) { + buf_add_string(buffer, "STATUS TYPE=a K_1=a K_2=b K_3=\"foo bar\"\n"); + buf_add_string(buffer, "STATUS TYPE=b K_1=a K_2=b K_3=\"foo bar\"\n"); + buf_add_string(buffer, "STATUS TYPE=c K_1=a K_2=b K_3=\"foo bar\"\n"); } - return retval_sl; -} - -/* NOP mock */ -static void -tor_process_handle_destroy_replacement(process_handle_t *process_handle, - int also_terminate_process) -{ - (void) process_handle; - (void) also_terminate_process; + return (int)buf_datalen(buffer); } static or_state_t *dummy_state = NULL; @@ -357,10 +351,7 @@ test_pt_configure_proxy(void *arg) dummy_state = tor_malloc_zero(sizeof(or_state_t)); - MOCK(tor_get_lines_from_handle, - tor_get_lines_from_handle_replacement); - MOCK(tor_process_handle_destroy, - tor_process_handle_destroy_replacement); + MOCK(process_read_stdout, process_read_stdout_replacement); MOCK(get_or_state, get_or_state_replacement); MOCK(queue_control_event_string, @@ -372,24 +363,34 @@ test_pt_configure_proxy(void *arg) mp->conf_state = PT_PROTO_ACCEPTING_METHODS; mp->transports = smartlist_new(); mp->transports_to_launch = smartlist_new(); - mp->process_handle = tor_malloc_zero(sizeof(process_handle_t)); mp->argv = tor_malloc_zero(sizeof(char*)*2); mp->argv[0] = tor_strdup("<testcase>"); mp->is_server = 1; + /* Configure the process. */ + mp->process = process_new(""); + process_set_stdout_read_callback(mp->process, managed_proxy_stdout_callback); + process_set_data(mp->process, mp); + /* Test the return value of configure_proxy() by calling it some times while it is uninitialized and then finally finalizing its configuration. */ for (i = 0 ; i < 5 ; i++) { + /* force a read from our mocked stdout reader. */ + process_notify_event_stdout(mp->process); + /* try to configure our proxy. */ retval = configure_proxy(mp); /* retval should be zero because proxy hasn't finished configuring yet */ tt_int_op(retval, OP_EQ, 0); /* check the number of registered transports */ - tt_assert(smartlist_len(mp->transports) == i+1); + tt_int_op(smartlist_len(mp->transports), OP_EQ, i+1); /* check that the mp is still waiting for transports */ tt_assert(mp->conf_state == PT_PROTO_ACCEPTING_METHODS); } + /* Get the SMETHOD DONE written to the process. */ + process_notify_event_stdout(mp->process); + /* this last configure_proxy() should finalize the proxy configuration. */ retval = configure_proxy(mp); /* retval should be 1 since the proxy finished configuring */ @@ -412,6 +413,46 @@ test_pt_configure_proxy(void *arg) tt_str_op(smartlist_get(controlevent_msgs, 4), OP_EQ, "650 TRANSPORT_LAUNCHED server mock5 127.0.0.1 5555\r\n"); + /* Get the log message out. */ + process_notify_event_stdout(mp->process); + + tt_int_op(controlevent_n, OP_EQ, 10); + tt_int_op(controlevent_event, OP_EQ, EVENT_PT_LOG); + tt_int_op(smartlist_len(controlevent_msgs), OP_EQ, 10); + tt_str_op(smartlist_get(controlevent_msgs, 5), OP_EQ, + "650 PT_LOG PT=<testcase> SEVERITY=error " + "MESSAGE=\"Oh noes, " + "something bad happened. What do we do!?\"\r\n"); + tt_str_op(smartlist_get(controlevent_msgs, 6), OP_EQ, + "650 PT_LOG PT=<testcase> SEVERITY=warning " + "MESSAGE=\"warning msg\"\r\n"); + tt_str_op(smartlist_get(controlevent_msgs, 7), OP_EQ, + "650 PT_LOG PT=<testcase> SEVERITY=notice " + "MESSAGE=\"notice msg\"\r\n"); + tt_str_op(smartlist_get(controlevent_msgs, 8), OP_EQ, + "650 PT_LOG PT=<testcase> SEVERITY=info " + "MESSAGE=\"info msg\"\r\n"); + tt_str_op(smartlist_get(controlevent_msgs, 9), OP_EQ, + "650 PT_LOG PT=<testcase> SEVERITY=debug " + "MESSAGE=\"debug msg\"\r\n"); + + /* Get the STATUS messages out. */ + process_notify_event_stdout(mp->process); + + tt_int_op(controlevent_n, OP_EQ, 13); + tt_int_op(controlevent_event, OP_EQ, EVENT_PT_STATUS); + tt_int_op(smartlist_len(controlevent_msgs), OP_EQ, 13); + + tt_str_op(smartlist_get(controlevent_msgs, 10), OP_EQ, + "650 PT_STATUS " + "PT=<testcase> TYPE=a K_1=a K_2=b K_3=\"foo bar\"\r\n"); + tt_str_op(smartlist_get(controlevent_msgs, 11), OP_EQ, + "650 PT_STATUS " + "PT=<testcase> TYPE=b K_1=a K_2=b K_3=\"foo bar\"\r\n"); + tt_str_op(smartlist_get(controlevent_msgs, 12), OP_EQ, + "650 PT_STATUS " + "PT=<testcase> TYPE=c K_1=a K_2=b K_3=\"foo bar\"\r\n"); + { /* check that the transport info were saved properly in the tor state */ config_line_t *transport_in_state = NULL; smartlist_t *transport_info_sl = smartlist_new(); @@ -435,8 +476,7 @@ test_pt_configure_proxy(void *arg) done: or_state_free(dummy_state); - UNMOCK(tor_get_lines_from_handle); - UNMOCK(tor_process_handle_destroy); + UNMOCK(process_read_stdout); UNMOCK(get_or_state); UNMOCK(queue_control_event_string); if (controlevent_msgs) { @@ -449,7 +489,7 @@ test_pt_configure_proxy(void *arg) smartlist_free(mp->transports); } smartlist_free(mp->transports_to_launch); - tor_free(mp->process_handle); + process_free(mp->process); tor_free(mp->argv[0]); tor_free(mp->argv); tor_free(mp); diff --git a/src/test/test_rebind.py b/src/test/test_rebind.py index 00e5a08be7..65feedd243 100644 --- a/src/test/test_rebind.py +++ b/src/test/test_rebind.py @@ -18,9 +18,10 @@ def fail(msg): def try_connecting_to_socksport(): socks_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - if socks_socket.connect_ex(('127.0.0.1', socks_port)): + e = socks_socket.connect_ex(('127.0.0.1', socks_port)) + if e: tor_process.terminate() - fail('Cannot connect to SOCKSPort') + fail('Cannot connect to SOCKSPort: error ' + os.strerror(e)) socks_socket.close() def wait_for_log(s): diff --git a/src/test/test_rendcache.c b/src/test/test_rendcache.c index 394e28d785..2ace45d085 100644 --- a/src/test/test_rendcache.c +++ b/src/test/test_rendcache.c @@ -788,7 +788,9 @@ test_rend_cache_clean(void *data) desc_two->pk = pk_generate(1); strmap_set_lc(rend_cache, "foo1", one); + rend_cache_increment_allocation(rend_cache_entry_allocation(one)); strmap_set_lc(rend_cache, "foo2", two); + rend_cache_increment_allocation(rend_cache_entry_allocation(two)); rend_cache_clean(time(NULL), REND_CACHE_TYPE_CLIENT); tt_int_op(strmap_size(rend_cache), OP_EQ, 0); @@ -806,7 +808,9 @@ test_rend_cache_clean(void *data) desc_one->pk = pk_generate(0); desc_two->pk = pk_generate(1); + rend_cache_increment_allocation(rend_cache_entry_allocation(one)); strmap_set_lc(rend_cache, "foo1", one); + rend_cache_increment_allocation(rend_cache_entry_allocation(two)); strmap_set_lc(rend_cache, "foo2", two); rend_cache_clean(time(NULL), REND_CACHE_TYPE_CLIENT); diff --git a/src/test/test_router.c b/src/test/test_router.c index 921ec42904..91cdd2c064 100644 --- a/src/test/test_router.c +++ b/src/test/test_router.c @@ -7,16 +7,25 @@ * \brief Unittests for code in router.c **/ +#define CONFIG_PRIVATE +#define ROUTER_PRIVATE + #include "core/or/or.h" #include "app/config/config.h" #include "core/mainloop/mainloop.h" #include "feature/hibernate/hibernate.h" +#include "feature/nodelist/networkstatus.h" +#include "feature/nodelist/networkstatus_st.h" +#include "feature/nodelist/node_st.h" +#include "feature/nodelist/nodelist.h" #include "feature/nodelist/routerinfo_st.h" #include "feature/nodelist/routerlist.h" +#include "feature/nodelist/routerstatus_st.h" #include "feature/relay/router.h" #include "feature/stats/rephist.h" #include "lib/crypt_ops/crypto_curve25519.h" #include "lib/crypt_ops/crypto_ed25519.h" +#include "lib/encoding/confline.h" /* Test suite stuff */ #include "test/test.h" @@ -231,11 +240,254 @@ test_router_check_descriptor_bandwidth_changed(void *arg) UNMOCK(we_are_hibernating); } +static networkstatus_t *mock_ns = NULL; +static networkstatus_t * +mock_networkstatus_get_live_consensus(time_t now) +{ + (void)now; + return mock_ns; +} + +static routerstatus_t *mock_rs = NULL; +static const routerstatus_t * +mock_networkstatus_vote_find_entry(networkstatus_t *ns, const char *digest) +{ + (void)ns; + (void)digest; + return mock_rs; +} + +static void +test_router_mark_if_too_old(void *arg) +{ + (void)arg; + time_t now = approx_time(); + MOCK(networkstatus_get_live_consensus, + mock_networkstatus_get_live_consensus); + MOCK(networkstatus_vote_find_entry, mock_networkstatus_vote_find_entry); + + routerstatus_t rs; + networkstatus_t ns; + memset(&rs, 0, sizeof(rs)); + memset(&ns, 0, sizeof(ns)); + mock_ns = &ns; + mock_ns->valid_after = now-3600; + mock_rs = &rs; + mock_rs->published_on = now - 10; + + // no reason to mark this time. + desc_clean_since = now-10; + desc_dirty_reason = NULL; + mark_my_descriptor_dirty_if_too_old(now); + tt_i64_op(desc_clean_since, OP_EQ, now-10); + + // Doesn't appear in consensus? Still don't mark it. + mock_ns = NULL; + mark_my_descriptor_dirty_if_too_old(now); + tt_i64_op(desc_clean_since, OP_EQ, now-10); + mock_ns = &ns; + + // No new descriptor in a long time? Mark it. + desc_clean_since = now - 3600 * 96; + mark_my_descriptor_dirty_if_too_old(now); + tt_i64_op(desc_clean_since, OP_EQ, 0); + tt_str_op(desc_dirty_reason, OP_EQ, "time for new descriptor"); + + // Version in consensus published a long time ago? We won't mark it + // if it's been clean for only a short time. + desc_clean_since = now - 10; + desc_dirty_reason = NULL; + mock_rs->published_on = now - 3600 * 96; + mark_my_descriptor_dirty_if_too_old(now); + tt_i64_op(desc_clean_since, OP_EQ, now - 10); + + // ... but if it's been clean a while, we mark. + desc_clean_since = now - 2 * 3600; + mark_my_descriptor_dirty_if_too_old(now); + tt_i64_op(desc_clean_since, OP_EQ, 0); + tt_str_op(desc_dirty_reason, OP_EQ, + "version listed in consensus is quite old"); + + // same deal if we're marked stale. + desc_clean_since = now - 2 * 3600; + desc_dirty_reason = NULL; + mock_rs->published_on = now - 10; + mock_rs->is_staledesc = 1; + mark_my_descriptor_dirty_if_too_old(now); + tt_i64_op(desc_clean_since, OP_EQ, 0); + tt_str_op(desc_dirty_reason, OP_EQ, + "listed as stale in consensus"); + + // same deal if we're absent from the consensus. + desc_clean_since = now - 2 * 3600; + desc_dirty_reason = NULL; + mock_rs = NULL; + mark_my_descriptor_dirty_if_too_old(now); + tt_i64_op(desc_clean_since, OP_EQ, 0); + tt_str_op(desc_dirty_reason, OP_EQ, + "not listed in consensus"); + + done: + UNMOCK(networkstatus_get_live_consensus); + UNMOCK(networkstatus_vote_find_entry); +} + +static node_t fake_node; +static const node_t * +mock_node_get_by_nickname(const char *name, unsigned flags) +{ + (void)flags; + if (!strcasecmp(name, "crumpet")) + return &fake_node; + else + return NULL; +} + +static void +test_router_get_my_family(void *arg) +{ + (void)arg; + or_options_t *options = options_new(); + smartlist_t *sl = NULL; + char *join = NULL; + // Overwrite the result of router_get_my_identity_digest(). This + // happens to be okay, but only for testing. + set_server_identity_key_digest_testing( + (const uint8_t*)"holeinthebottomofthe"); + + setup_capture_of_logs(LOG_WARN); + + // No family listed -- so there's no list. + sl = get_my_declared_family(options); + tt_ptr_op(sl, OP_EQ, NULL); + expect_no_log_entry(); + +#define CLEAR() do { \ + if (sl) { \ + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); \ + smartlist_free(sl); \ + } \ + tor_free(join); \ + mock_clean_saved_logs(); \ + } while (0) + + // Add a single nice friendly hex member. This should be enough + // to have our own ID added. + tt_ptr_op(options->MyFamily, OP_EQ, NULL); + config_line_append(&options->MyFamily, "MyFamily", + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"); + + sl = get_my_declared_family(options); + tt_ptr_op(sl, OP_NE, NULL); + tt_int_op(smartlist_len(sl), OP_EQ, 2); + join = smartlist_join_strings(sl, " ", 0, NULL); + tt_str_op(join, OP_EQ, + "$686F6C65696E746865626F74746F6D6F66746865 " + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"); + expect_no_log_entry(); + CLEAR(); + + // Add a hex member with a ~. The ~ part should get removed. + config_line_append(&options->MyFamily, "MyFamily", + "$0123456789abcdef0123456789abcdef01234567~Muffin"); + sl = get_my_declared_family(options); + tt_ptr_op(sl, OP_NE, NULL); + tt_int_op(smartlist_len(sl), OP_EQ, 3); + join = smartlist_join_strings(sl, " ", 0, NULL); + tt_str_op(join, OP_EQ, + "$0123456789ABCDEF0123456789ABCDEF01234567 " + "$686F6C65696E746865626F74746F6D6F66746865 " + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"); + expect_no_log_entry(); + CLEAR(); + + // Nickname lookup will fail, so a nickname will appear verbatim. + config_line_append(&options->MyFamily, "MyFamily", + "BAGEL"); + sl = get_my_declared_family(options); + tt_ptr_op(sl, OP_NE, NULL); + tt_int_op(smartlist_len(sl), OP_EQ, 4); + join = smartlist_join_strings(sl, " ", 0, NULL); + tt_str_op(join, OP_EQ, + "$0123456789ABCDEF0123456789ABCDEF01234567 " + "$686F6C65696E746865626F74746F6D6F66746865 " + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA " + "bagel"); + expect_single_log_msg_containing( + "There is a router named \"BAGEL\" in my declared family, but " + "I have no descriptor for it."); + CLEAR(); + + // A bogus digest should fail entirely. + config_line_append(&options->MyFamily, "MyFamily", + "$painauchocolat"); + sl = get_my_declared_family(options); + tt_ptr_op(sl, OP_NE, NULL); + tt_int_op(smartlist_len(sl), OP_EQ, 4); + join = smartlist_join_strings(sl, " ", 0, NULL); + tt_str_op(join, OP_EQ, + "$0123456789ABCDEF0123456789ABCDEF01234567 " + "$686F6C65696E746865626F74746F6D6F66746865 " + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA " + "bagel"); + // "BAGEL" is still there, but it won't make a warning, because we already + // warned about it. + expect_single_log_msg_containing( + "There is a router named \"$painauchocolat\" in my declared " + "family, but that isn't a legal digest or nickname. Skipping it."); + CLEAR(); + + // Let's introduce a node we can look up by nickname + memset(&fake_node, 0, sizeof(fake_node)); + memcpy(fake_node.identity, "whydoyouasknonononon", DIGEST_LEN); + MOCK(node_get_by_nickname, mock_node_get_by_nickname); + + config_line_append(&options->MyFamily, "MyFamily", + "CRUmpeT"); + sl = get_my_declared_family(options); + tt_ptr_op(sl, OP_NE, NULL); + tt_int_op(smartlist_len(sl), OP_EQ, 5); + join = smartlist_join_strings(sl, " ", 0, NULL); + tt_str_op(join, OP_EQ, + "$0123456789ABCDEF0123456789ABCDEF01234567 " + "$686F6C65696E746865626F74746F6D6F66746865 " + "$776879646F796F7561736B6E6F6E6F6E6F6E6F6E " + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA " + "bagel"); + // "BAGEL" is still there, but it won't make a warning, because we already + // warned about it. Some with "$painauchocolat". + expect_single_log_msg_containing( + "There is a router named \"CRUmpeT\" in my declared " + "family, but it wasn't listed by digest. Please consider saying " + "$776879646F796F7561736B6E6F6E6F6E6F6E6F6E instead, if that's " + "what you meant."); + CLEAR(); + UNMOCK(node_get_by_nickname); + + // Try a singleton list containing only us: It should give us NULL. + config_free_lines(options->MyFamily); + config_line_append(&options->MyFamily, "MyFamily", + "$686F6C65696E746865626F74746F6D6F66746865"); + sl = get_my_declared_family(options); + tt_ptr_op(sl, OP_EQ, NULL); + expect_no_log_entry(); + + done: + or_options_free(options); + teardown_capture_of_logs(); + CLEAR(); + UNMOCK(node_get_by_nickname); + +#undef CLEAR +} + #define ROUTER_TEST(name, flags) \ { #name, test_router_ ## name, flags, NULL, NULL } struct testcase_t router_tests[] = { ROUTER_TEST(check_descriptor_bandwidth_changed, TT_FORK), ROUTER_TEST(dump_router_to_string_no_bridge_distribution_method, TT_FORK), + ROUTER_TEST(mark_if_too_old, TT_FORK), + ROUTER_TEST(get_my_family, TT_FORK), END_OF_TESTCASES }; diff --git a/src/test/test_routerlist.c b/src/test/test_routerlist.c index 67af2fd484..167b7a35ce 100644 --- a/src/test/test_routerlist.c +++ b/src/test/test_routerlist.c @@ -46,7 +46,7 @@ #include "feature/nodelist/routerstatus_st.h" #include "lib/encoding/confline.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "test/test.h" #include "test/test_dir_common.h" @@ -265,7 +265,9 @@ test_router_pick_directory_server_impl(void *arg) /* Init SR subsystem. */ MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m); - mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); + mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, + strlen(AUTHORITY_CERT_1), + NULL); sr_init(0); UNMOCK(get_my_v3_authority_cert); @@ -275,7 +277,9 @@ test_router_pick_directory_server_impl(void *arg) construct_consensus(&consensus_text_md, now); tt_assert(consensus_text_md); - con_md = networkstatus_parse_vote_from_string(consensus_text_md, NULL, + con_md = networkstatus_parse_vote_from_string(consensus_text_md, + strlen(consensus_text_md), + NULL, NS_TYPE_CONSENSUS); tt_assert(con_md); tt_int_op(con_md->flavor,OP_EQ, FLAV_MICRODESC); @@ -475,7 +479,9 @@ test_directory_guard_fetch_with_no_dirinfo(void *arg) /* Initialize the SRV subsystem */ MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m); - mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); + mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, + strlen(AUTHORITY_CERT_1), + NULL); sr_init(0); UNMOCK(get_my_v3_authority_cert); @@ -648,7 +654,9 @@ test_skew_common(void *arg, time_t now, unsigned long *offset) /* Initialize the SRV subsystem */ MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m); - mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); + mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, + strlen(AUTHORITY_CERT_1), + NULL); sr_init(0); UNMOCK(get_my_v3_authority_cert); @@ -662,7 +670,8 @@ test_skew_common(void *arg, time_t now, unsigned long *offset) MOCK(clock_skew_warning, mock_clock_skew_warning); /* Caller will call teardown_capture_of_logs() */ setup_capture_of_logs(LOG_WARN); - retval = networkstatus_set_current_consensus(consensus, "microdesc", 0, + retval = networkstatus_set_current_consensus(consensus, strlen(consensus), + "microdesc", 0, NULL); done: diff --git a/src/test/test_shared_random.c b/src/test/test_shared_random.c index 8a7fb95cc1..16e28afafe 100644 --- a/src/test/test_shared_random.c +++ b/src/test/test_shared_random.c @@ -65,7 +65,9 @@ init_authority_state(void) MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m); or_options_t *options = get_options_mutable(); - mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); + mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, + strlen(AUTHORITY_CERT_1), + NULL); tt_assert(mock_cert); options->AuthoritativeDir = 1; tt_int_op(load_ed_keys(options, time(NULL)), OP_GE, 0); @@ -307,6 +309,7 @@ test_get_start_time_of_current_run(void *arg) retval = parse_rfc1123_time("Mon, 19 Apr 2015 23:00:00 UTC", &mock_consensus.valid_after); + tt_int_op(retval, OP_EQ, 0); retval = parse_rfc1123_time("Mon, 20 Apr 2015 00:08:00 UTC", ¤t_time); @@ -421,7 +424,9 @@ test_sr_commit(void *arg) { /* Setup a minimal dirauth environment for this test */ or_options_t *options = get_options_mutable(); - auth_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); + auth_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, + strlen(AUTHORITY_CERT_1), + NULL); tt_assert(auth_cert); options->AuthoritativeDir = 1; @@ -824,7 +829,9 @@ test_sr_setup_commits(void) { /* Setup a minimal dirauth environment for this test */ or_options_t *options = get_options_mutable(); - auth_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); + auth_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, + strlen(AUTHORITY_CERT_1), + NULL); tt_assert(auth_cert); options->AuthoritativeDir = 1; diff --git a/src/test/test_slow.c b/src/test/test_slow.c index 0b665363ab..97c2912af6 100644 --- a/src/test/test_slow.c +++ b/src/test/test_slow.c @@ -20,7 +20,7 @@ struct testgroup_t testgroups[] = { { "slow/crypto/", slow_crypto_tests }, - { "slow/util/", slow_util_tests }, + { "slow/process/", slow_process_tests }, END_OF_GROUPS }; diff --git a/src/test/test_socks.c b/src/test/test_socks.c index 7f6d8a48f1..d430f4329b 100644 --- a/src/test/test_socks.c +++ b/src/test/test_socks.c @@ -4,7 +4,7 @@ /* See LICENSE for licensing information */ #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "app/config/config.h" #include "core/mainloop/connection.h" #include "core/proto/proto_socks.h" diff --git a/src/test/test_util.c b/src/test/test_util.c index 7bc1b7921a..b983cbb0bf 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -10,10 +10,10 @@ #define UTIL_PRIVATE #define UTIL_MALLOC_PRIVATE #define SOCKET_PRIVATE -#define SUBPROCESS_PRIVATE +#define PROCESS_WIN32_PRIVATE #include "lib/testsupport/testsupport.h" #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "app/config/config.h" #include "feature/control/control.h" #include "feature/client/transports.h" @@ -22,6 +22,7 @@ #include "test/test.h" #include "lib/memarea/memarea.h" #include "lib/process/waitpid.h" +#include "lib/process/process_win32.h" #include "test/log_test_helpers.h" #include "lib/compress/compress.h" #include "lib/compress/compress_zstd.h" @@ -30,7 +31,6 @@ #include "lib/fs/winlib.h" #include "lib/process/env.h" #include "lib/process/pidfile.h" -#include "lib/process/subprocess.h" #include "lib/intmath/weakrng.h" #include "lib/thread/numcpus.h" #include "lib/math/fp.h" @@ -4024,6 +4024,13 @@ test_util_string_is_utf8(void *ptr) tt_int_op(1, OP_EQ, string_is_utf8("ascii\x7f\n", 7)); tt_int_op(1, OP_EQ, string_is_utf8("Risqu\u00e9=1", 9)); + /* Test the utf8_no_bom function */ + tt_int_op(0, OP_EQ, string_is_utf8_no_bom("\uFEFF", 3)); + tt_int_op(0, OP_EQ, string_is_utf8_no_bom("\uFFFE", 3)); + tt_int_op(0, OP_EQ, string_is_utf8_no_bom("\uFEFFlove", 7)); + tt_int_op(1, OP_EQ, string_is_utf8_no_bom("loveandrespect", + strlen("loveandrespect"))); + // Validate exactly 'len' bytes. tt_int_op(0, OP_EQ, string_is_utf8("\0\x80", 2)); tt_int_op(0, OP_EQ, string_is_utf8("Risqu\u00e9=1", 6)); @@ -4294,204 +4301,6 @@ test_util_load_win_lib(void *ptr) } #endif /* defined(_WIN32) */ -#ifndef _WIN32 -static void -clear_hex_errno(char *hex_errno) -{ - memset(hex_errno, '\0', HEX_ERRNO_SIZE + 1); -} - -static void -test_util_exit_status(void *ptr) -{ - /* Leave an extra byte for a \0 so we can do string comparison */ - char hex_errno[HEX_ERRNO_SIZE + 1]; - int n; - - (void)ptr; - - clear_hex_errno(hex_errno); - tt_str_op("",OP_EQ, hex_errno); - - clear_hex_errno(hex_errno); - n = format_helper_exit_status(0, 0, hex_errno); - tt_str_op("0/0\n",OP_EQ, hex_errno); - tt_int_op(n,OP_EQ, strlen(hex_errno)); - -#if SIZEOF_INT == 4 - - clear_hex_errno(hex_errno); - n = format_helper_exit_status(0, 0x7FFFFFFF, hex_errno); - tt_str_op("0/7FFFFFFF\n",OP_EQ, hex_errno); - tt_int_op(n,OP_EQ, strlen(hex_errno)); - - clear_hex_errno(hex_errno); - n = format_helper_exit_status(0xFF, -0x80000000, hex_errno); - tt_str_op("FF/-80000000\n",OP_EQ, hex_errno); - tt_int_op(n,OP_EQ, strlen(hex_errno)); - tt_int_op(n,OP_EQ, HEX_ERRNO_SIZE); - -#elif SIZEOF_INT == 8 - - clear_hex_errno(hex_errno); - n = format_helper_exit_status(0, 0x7FFFFFFFFFFFFFFF, hex_errno); - tt_str_op("0/7FFFFFFFFFFFFFFF\n",OP_EQ, hex_errno); - tt_int_op(n,OP_EQ, strlen(hex_errno)); - - clear_hex_errno(hex_errno); - n = format_helper_exit_status(0xFF, -0x8000000000000000, hex_errno); - tt_str_op("FF/-8000000000000000\n",OP_EQ, hex_errno); - tt_int_op(n,OP_EQ, strlen(hex_errno)); - tt_int_op(n,OP_EQ, HEX_ERRNO_SIZE); - -#endif /* SIZEOF_INT == 4 || ... */ - - clear_hex_errno(hex_errno); - n = format_helper_exit_status(0x7F, 0, hex_errno); - tt_str_op("7F/0\n",OP_EQ, hex_errno); - tt_int_op(n,OP_EQ, strlen(hex_errno)); - - clear_hex_errno(hex_errno); - n = format_helper_exit_status(0x08, -0x242, hex_errno); - tt_str_op("8/-242\n",OP_EQ, hex_errno); - tt_int_op(n,OP_EQ, strlen(hex_errno)); - - clear_hex_errno(hex_errno); - tt_str_op("",OP_EQ, hex_errno); - - done: - ; -} -#endif /* !defined(_WIN32) */ - -#ifndef _WIN32 -static void -test_util_string_from_pipe(void *ptr) -{ - int test_pipe[2] = {-1, -1}; - int retval = 0; - enum stream_status status = IO_STREAM_TERM; - ssize_t retlen; - char buf[4] = { 0 }; - - (void)ptr; - - errno = 0; - - /* Set up a pipe to test on */ - retval = pipe(test_pipe); - tt_int_op(retval, OP_EQ, 0); - - /* Send in a string. */ - retlen = write(test_pipe[1], "ABC", 3); - tt_int_op(retlen, OP_EQ, 3); - - status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1); - tt_int_op(errno, OP_EQ, 0); - tt_int_op(status, OP_EQ, IO_STREAM_OKAY); - tt_str_op(buf, OP_EQ, "ABC"); - errno = 0; - - /* Send in a string that contains a nul. */ - retlen = write(test_pipe[1], "AB\0", 3); - tt_int_op(retlen, OP_EQ, 3); - - status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1); - tt_int_op(errno, OP_EQ, 0); - tt_int_op(status, OP_EQ, IO_STREAM_OKAY); - tt_str_op(buf, OP_EQ, "AB"); - errno = 0; - - /* Send in a string that contains a nul only. */ - retlen = write(test_pipe[1], "\0", 1); - tt_int_op(retlen, OP_EQ, 1); - - status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1); - tt_int_op(errno, OP_EQ, 0); - tt_int_op(status, OP_EQ, IO_STREAM_OKAY); - tt_str_op(buf, OP_EQ, ""); - errno = 0; - - /* Send in a string that contains a trailing newline. */ - retlen = write(test_pipe[1], "AB\n", 3); - tt_int_op(retlen, OP_EQ, 3); - - status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1); - tt_int_op(errno, OP_EQ, 0); - tt_int_op(status, OP_EQ, IO_STREAM_OKAY); - tt_str_op(buf, OP_EQ, "AB"); - errno = 0; - - /* Send in a string that contains a newline only. */ - retlen = write(test_pipe[1], "\n", 1); - tt_int_op(retlen, OP_EQ, 1); - - status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1); - tt_int_op(errno, OP_EQ, 0); - tt_int_op(status, OP_EQ, IO_STREAM_OKAY); - tt_str_op(buf, OP_EQ, ""); - errno = 0; - - /* Send in a string and check that we nul terminate return values. */ - retlen = write(test_pipe[1], "AAA", 3); - tt_int_op(retlen, OP_EQ, 3); - - status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1); - tt_int_op(errno, OP_EQ, 0); - tt_int_op(status, OP_EQ, IO_STREAM_OKAY); - tt_str_op(buf, OP_EQ, "AAA"); - tt_mem_op(buf, OP_EQ, "AAA\0", sizeof(buf)); - errno = 0; - - retlen = write(test_pipe[1], "B", 1); - tt_int_op(retlen, OP_EQ, 1); - - memset(buf, '\xff', sizeof(buf)); - status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1); - tt_int_op(errno, OP_EQ, 0); - tt_int_op(status, OP_EQ, IO_STREAM_OKAY); - tt_str_op(buf, OP_EQ, "B"); - tt_mem_op(buf, OP_EQ, "B\0\xff\xff", sizeof(buf)); - errno = 0; - - /* Send in multiple lines. */ - retlen = write(test_pipe[1], "A\nB", 3); - tt_int_op(retlen, OP_EQ, 3); - - status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1); - tt_int_op(errno, OP_EQ, 0); - tt_int_op(status, OP_EQ, IO_STREAM_OKAY); - tt_str_op(buf, OP_EQ, "A\nB"); - errno = 0; - - /* Send in a line and close */ - retlen = write(test_pipe[1], "AB", 2); - tt_int_op(retlen, OP_EQ, 2); - retval = close(test_pipe[1]); - tt_int_op(retval, OP_EQ, 0); - test_pipe[1] = -1; - - status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1); - tt_int_op(errno, OP_EQ, 0); - tt_int_op(status, OP_EQ, IO_STREAM_OKAY); - tt_str_op(buf, OP_EQ, "AB"); - errno = 0; - - /* Check for EOF */ - status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1); - tt_int_op(errno, OP_EQ, 0); - tt_int_op(status, OP_EQ, IO_STREAM_CLOSED); - errno = 0; - - done: - if (test_pipe[0] != -1) - close(test_pipe[0]); - if (test_pipe[1] != -1) - close(test_pipe[1]); -} - -#endif /* !defined(_WIN32) */ - /** * Test for format_hex_number_sigsafe() */ @@ -4586,57 +4395,6 @@ test_util_format_dec_number(void *ptr) return; } -/** - * Test that we can properly format a Windows command line - */ -static void -test_util_join_win_cmdline(void *ptr) -{ - /* Based on some test cases from "Parsing C++ Command-Line Arguments" in - * MSDN but we don't exercise all quoting rules because tor_join_win_cmdline - * will try to only generate simple cases for the child process to parse; - * i.e. we never embed quoted strings in arguments. */ - - const char *argvs[][4] = { - {"a", "bb", "CCC", NULL}, // Normal - {NULL, NULL, NULL, NULL}, // Empty argument list - {"", NULL, NULL, NULL}, // Empty argument - {"\"a", "b\"b", "CCC\"", NULL}, // Quotes - {"a\tbc", "dd dd", "E", NULL}, // Whitespace - {"a\\\\\\b", "de fg", "H", NULL}, // Backslashes - {"a\\\"b", "\\c", "D\\", NULL}, // Backslashes before quote - {"a\\\\b c", "d", "E", NULL}, // Backslashes not before quote - { NULL } // Terminator - }; - - const char *cmdlines[] = { - "a bb CCC", - "", - "\"\"", - "\\\"a b\\\"b CCC\\\"", - "\"a\tbc\" \"dd dd\" E", - "a\\\\\\b \"de fg\" H", - "a\\\\\\\"b \\c D\\", - "\"a\\\\b c\" d E", - NULL // Terminator - }; - - int i; - char *joined_argv = NULL; - - (void)ptr; - - for (i=0; cmdlines[i]!=NULL; i++) { - log_info(LD_GENERAL, "Joining argvs[%d], expecting <%s>", i, cmdlines[i]); - joined_argv = tor_join_win_cmdline(argvs[i]); - tt_str_op(cmdlines[i],OP_EQ, joined_argv); - tor_free(joined_argv); - } - - done: - tor_free(joined_argv); -} - #define MAX_SPLIT_LINE_COUNT 4 struct split_lines_test_t { const char *orig_line; // Line to be split (may contain \0's) @@ -4644,67 +4402,6 @@ struct split_lines_test_t { const char *split_line[MAX_SPLIT_LINE_COUNT]; // Split lines }; -/** - * Test that we properly split a buffer into lines - */ -static void -test_util_split_lines(void *ptr) -{ - /* Test cases. orig_line of last test case must be NULL. - * The last element of split_line[i] must be NULL. */ - struct split_lines_test_t tests[] = { - {"", 0, {NULL}}, - {"foo", 3, {"foo", NULL}}, - {"\n\rfoo\n\rbar\r\n", 12, {"foo", "bar", NULL}}, - {"fo o\r\nb\tar", 10, {"fo o", "b.ar", NULL}}, - {"\x0f""f\0o\0\n\x01""b\0r\0\r", 12, {".f.o.", ".b.r.", NULL}}, - {"line 1\r\nline 2", 14, {"line 1", "line 2", NULL}}, - {"line 1\r\n\r\nline 2", 16, {"line 1", "line 2", NULL}}, - {"line 1\r\n\r\r\r\nline 2", 18, {"line 1", "line 2", NULL}}, - {"line 1\r\n\n\n\n\rline 2", 18, {"line 1", "line 2", NULL}}, - {"line 1\r\n\r\t\r\nline 3", 18, {"line 1", ".", "line 3", NULL}}, - {"\n\t\r\t\nline 3", 11, {".", ".", "line 3", NULL}}, - {NULL, 0, { NULL }} - }; - - int i, j; - char *orig_line=NULL; - smartlist_t *sl=NULL; - - (void)ptr; - - for (i=0; tests[i].orig_line; i++) { - sl = smartlist_new(); - /* Allocate space for string and trailing NULL */ - orig_line = tor_memdup(tests[i].orig_line, tests[i].orig_length + 1); - tor_split_lines(sl, orig_line, tests[i].orig_length); - - j = 0; - log_info(LD_GENERAL, "Splitting test %d of length %d", - i, tests[i].orig_length); - SMARTLIST_FOREACH_BEGIN(sl, const char *, line) { - /* Check we have not got too many lines */ - tt_int_op(MAX_SPLIT_LINE_COUNT, OP_GT, j); - /* Check that there actually should be a line here */ - tt_ptr_op(tests[i].split_line[j], OP_NE, NULL); - log_info(LD_GENERAL, "Line %d of test %d, should be <%s>", - j, i, tests[i].split_line[j]); - /* Check that the line is as expected */ - tt_str_op(line,OP_EQ, tests[i].split_line[j]); - j++; - } SMARTLIST_FOREACH_END(line); - /* Check that we didn't miss some lines */ - tt_ptr_op(NULL,OP_EQ, tests[i].split_line[j]); - tor_free(orig_line); - smartlist_free(sl); - sl = NULL; - } - - done: - tor_free(orig_line); - smartlist_free(sl); -} - static void test_util_di_ops(void *arg) { @@ -5808,6 +5505,18 @@ test_util_ipv4_validation(void *arg) } static void +test_util_ipv6_validation(void *arg) +{ + (void)arg; + + tt_assert(string_is_valid_ipv6_address("2a00:1450:401b:800::200e")); + tt_assert(!string_is_valid_ipv6_address("11:22::33:44:")); + + done: + return; +} + +static void test_util_writepid(void *arg) { (void) arg; @@ -6464,12 +6173,8 @@ struct testcase_t util_tests[] = { UTIL_TEST(nowrap_math, 0), UTIL_TEST(num_cpus, 0), UTIL_TEST_WIN_ONLY(load_win_lib, 0), - UTIL_TEST_NO_WIN(exit_status, 0), - UTIL_TEST_NO_WIN(string_from_pipe, 0), UTIL_TEST(format_hex_number, 0), UTIL_TEST(format_dec_number, 0), - UTIL_TEST(join_win_cmdline, 0), - UTIL_TEST(split_lines, 0), UTIL_TEST(n_bits_set, 0), UTIL_TEST(eat_whitespace, 0), UTIL_TEST(sl_new_from_text_lines, 0), @@ -6498,6 +6203,7 @@ struct testcase_t util_tests[] = { UTIL_TEST(hostname_validation, 0), UTIL_TEST(dest_validation_edgecase, 0), UTIL_TEST(ipv4_validation, 0), + UTIL_TEST(ipv6_validation, 0), UTIL_TEST(writepid, 0), UTIL_TEST(get_avail_disk_space, 0), UTIL_TEST(touch_file, 0), diff --git a/src/test/test_util_format.c b/src/test/test_util_format.c index 85d8a8e62e..fd57125b86 100644 --- a/src/test/test_util_format.c +++ b/src/test/test_util_format.c @@ -392,10 +392,13 @@ test_util_format_encoded_size(void *arg) base64_encode(outbuf, sizeof(outbuf), (char *)inbuf, i, 0); tt_int_op(strlen(outbuf), OP_EQ, base64_encode_size(i, 0)); + tt_int_op(i, OP_LE, base64_decode_maxsize(strlen(outbuf))); + base64_encode(outbuf, sizeof(outbuf), (char *)inbuf, i, BASE64_ENCODE_MULTILINE); tt_int_op(strlen(outbuf), OP_EQ, base64_encode_size(i, BASE64_ENCODE_MULTILINE)); + tt_int_op(i, OP_LE, base64_decode_maxsize(strlen(outbuf))); } done: @@ -417,4 +420,3 @@ struct testcase_t util_format_tests[] = { { "encoded_size", test_util_format_encoded_size, 0, NULL, NULL }, END_OF_TESTCASES }; - diff --git a/src/test/test_util_slow.c b/src/test/test_util_slow.c deleted file mode 100644 index c7b3e3e2a4..0000000000 --- a/src/test/test_util_slow.c +++ /dev/null @@ -1,396 +0,0 @@ -/* Copyright (c) 2001-2004, Roger Dingledine. - * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ -/* See LICENSE for licensing information */ - -#include "orconfig.h" -#define UTIL_PRIVATE -#define SUBPROCESS_PRIVATE -#include "lib/crypt_ops/crypto_cipher.h" -#include "lib/log/log.h" -#include "lib/process/subprocess.h" -#include "lib/process/waitpid.h" -#include "lib/string/printf.h" -#include "lib/time/compat_time.h" -#include "test/test.h" - -#include <errno.h> -#include <string.h> - -#ifndef BUILDDIR -#define BUILDDIR "." -#endif - -#ifdef _WIN32 -#define notify_pending_waitpid_callbacks() STMT_NIL -#define TEST_CHILD "test-child.exe" -#define EOL "\r\n" -#else -#define TEST_CHILD (BUILDDIR "/src/test/test-child") -#define EOL "\n" -#endif /* defined(_WIN32) */ - -#ifdef _WIN32 -/* I've assumed Windows doesn't have the gap between fork and exec - * that causes the race condition on unix-like platforms */ -#define MATCH_PROCESS_STATUS(s1,s2) ((s1) == (s2)) - -#else /* !(defined(_WIN32)) */ -/* work around a race condition of the timing of SIGCHLD handler updates - * to the process_handle's fields, and checks of those fields - * - * TODO: Once we can signal failure to exec, change PROCESS_STATUS_RUNNING to - * PROCESS_STATUS_ERROR (and similarly with *_OR_NOTRUNNING) */ -#define PROCESS_STATUS_RUNNING_OR_NOTRUNNING (PROCESS_STATUS_RUNNING+1) -#define IS_RUNNING_OR_NOTRUNNING(s) \ - ((s) == PROCESS_STATUS_RUNNING || (s) == PROCESS_STATUS_NOTRUNNING) -/* well, this is ugly */ -#define MATCH_PROCESS_STATUS(s1,s2) \ - ( (s1) == (s2) \ - ||((s1) == PROCESS_STATUS_RUNNING_OR_NOTRUNNING \ - && IS_RUNNING_OR_NOTRUNNING(s2)) \ - ||((s2) == PROCESS_STATUS_RUNNING_OR_NOTRUNNING \ - && IS_RUNNING_OR_NOTRUNNING(s1))) - -#endif /* defined(_WIN32) */ - -/** Helper function for testing tor_spawn_background */ -static void -run_util_spawn_background(const char *argv[], const char *expected_out, - const char *expected_err, int expected_exit, - int expected_status) -{ - int retval, exit_code; - ssize_t pos; - process_handle_t *process_handle=NULL; - char stdout_buf[100], stderr_buf[100]; - int status; - - /* Start the program */ -#ifdef _WIN32 - status = tor_spawn_background(NULL, argv, NULL, &process_handle); -#else - status = tor_spawn_background(argv[0], argv, NULL, &process_handle); -#endif - - notify_pending_waitpid_callbacks(); - - /* the race condition doesn't affect status, - * because status isn't updated by the SIGCHLD handler, - * but we still need to handle PROCESS_STATUS_RUNNING_OR_NOTRUNNING */ - tt_assert(MATCH_PROCESS_STATUS(expected_status, status)); - if (status == PROCESS_STATUS_ERROR) { - tt_ptr_op(process_handle, OP_EQ, NULL); - return; - } - - tt_ptr_op(process_handle, OP_NE, NULL); - - /* When a spawned process forks, fails, then exits very quickly, - * (this typically occurs when exec fails) - * there is a race condition between the SIGCHLD handler - * updating the process_handle's fields, and this test - * checking the process status in those fields. - * The SIGCHLD update can occur before or after the code below executes. - * This causes intermittent failures in spawn_background_fail(), - * typically when the machine is under load. - * We use PROCESS_STATUS_RUNNING_OR_NOTRUNNING to avoid this issue. */ - - /* the race condition affects the change in - * process_handle->status from RUNNING to NOTRUNNING */ - tt_assert(MATCH_PROCESS_STATUS(expected_status, process_handle->status)); - -#ifndef _WIN32 - notify_pending_waitpid_callbacks(); - /* the race condition affects the change in - * process_handle->waitpid_cb to NULL, - * so we skip the check if expected_status is ambiguous, - * that is, PROCESS_STATUS_RUNNING_OR_NOTRUNNING */ - tt_assert(process_handle->waitpid_cb != NULL - || expected_status == PROCESS_STATUS_RUNNING_OR_NOTRUNNING); -#endif /* !defined(_WIN32) */ - -#ifdef _WIN32 - tt_assert(process_handle->stdout_pipe != INVALID_HANDLE_VALUE); - tt_assert(process_handle->stderr_pipe != INVALID_HANDLE_VALUE); - tt_assert(process_handle->stdin_pipe != INVALID_HANDLE_VALUE); -#else - tt_assert(process_handle->stdout_pipe >= 0); - tt_assert(process_handle->stderr_pipe >= 0); - tt_assert(process_handle->stdin_pipe >= 0); -#endif /* defined(_WIN32) */ - - /* Check stdout */ - pos = tor_read_all_from_process_stdout(process_handle, stdout_buf, - sizeof(stdout_buf) - 1); - tt_assert(pos >= 0); - stdout_buf[pos] = '\0'; - tt_int_op(strlen(expected_out),OP_EQ, pos); - tt_str_op(expected_out,OP_EQ, stdout_buf); - - notify_pending_waitpid_callbacks(); - - /* Check it terminated correctly */ - retval = tor_get_exit_code(process_handle, 1, &exit_code); - tt_int_op(PROCESS_EXIT_EXITED,OP_EQ, retval); - tt_int_op(expected_exit,OP_EQ, exit_code); - // TODO: Make test-child exit with something other than 0 - -#ifndef _WIN32 - notify_pending_waitpid_callbacks(); - tt_ptr_op(process_handle->waitpid_cb, OP_EQ, NULL); -#endif - - /* Check stderr */ - pos = tor_read_all_from_process_stderr(process_handle, stderr_buf, - sizeof(stderr_buf) - 1); - tt_assert(pos >= 0); - stderr_buf[pos] = '\0'; - tt_str_op(expected_err,OP_EQ, stderr_buf); - tt_int_op(strlen(expected_err),OP_EQ, pos); - - notify_pending_waitpid_callbacks(); - - done: - if (process_handle) - tor_process_handle_destroy(process_handle, 1); -} - -/** Check that we can launch a process and read the output */ -static void -test_util_spawn_background_ok(void *ptr) -{ - const char *argv[] = {TEST_CHILD, "--test", NULL}; - const char *expected_out = "OUT"EOL "--test"EOL "SLEEPING"EOL "DONE" EOL; - const char *expected_err = "ERR"EOL; - - (void)ptr; - - run_util_spawn_background(argv, expected_out, expected_err, 0, - PROCESS_STATUS_RUNNING); -} - -/** Check that failing to find the executable works as expected */ -static void -test_util_spawn_background_fail(void *ptr) -{ - const char *argv[] = {BUILDDIR "/src/test/no-such-file", "--test", NULL}; - const char *expected_err = ""; - char expected_out[1024]; - char code[32]; -#ifdef _WIN32 - const int expected_status = PROCESS_STATUS_ERROR; -#else - /* TODO: Once we can signal failure to exec, set this to be - * PROCESS_STATUS_RUNNING_OR_ERROR */ - const int expected_status = PROCESS_STATUS_RUNNING_OR_NOTRUNNING; -#endif /* defined(_WIN32) */ - - memset(expected_out, 0xf0, sizeof(expected_out)); - memset(code, 0xf0, sizeof(code)); - - (void)ptr; - - tor_snprintf(code, sizeof(code), "%x/%x", - 9 /* CHILD_STATE_FAILEXEC */ , ENOENT); - tor_snprintf(expected_out, sizeof(expected_out), - "ERR: Failed to spawn background process - code %s\n", code); - - run_util_spawn_background(argv, expected_out, expected_err, 255, - expected_status); -} - -/** Test that reading from a handle returns a partial read rather than - * blocking */ -static void -test_util_spawn_background_partial_read_impl(int exit_early) -{ - const int expected_exit = 0; - const int expected_status = PROCESS_STATUS_RUNNING; - - int retval, exit_code; - ssize_t pos = -1; - process_handle_t *process_handle=NULL; - int status; - char stdout_buf[100], stderr_buf[100]; - - const char *argv[] = {TEST_CHILD, "--test", NULL}; - const char *expected_out[] = { "OUT" EOL "--test" EOL "SLEEPING" EOL, - "DONE" EOL, - NULL }; - const char *expected_err = "ERR" EOL; - -#ifndef _WIN32 - int eof = 0; -#endif - int expected_out_ctr; - - if (exit_early) { - argv[1] = "--hang"; - expected_out[0] = "OUT"EOL "--hang"EOL "SLEEPING" EOL; - } - - /* Start the program */ -#ifdef _WIN32 - status = tor_spawn_background(NULL, argv, NULL, &process_handle); -#else - status = tor_spawn_background(argv[0], argv, NULL, &process_handle); -#endif - tt_int_op(expected_status,OP_EQ, status); - tt_assert(process_handle); - tt_int_op(expected_status,OP_EQ, process_handle->status); - - /* Check stdout */ - for (expected_out_ctr = 0; expected_out[expected_out_ctr] != NULL;) { -#ifdef _WIN32 - pos = tor_read_all_handle(process_handle->stdout_pipe, stdout_buf, - sizeof(stdout_buf) - 1, NULL); -#else - /* Check that we didn't read the end of file last time */ - tt_assert(!eof); - pos = tor_read_all_handle(process_handle->stdout_pipe, stdout_buf, - sizeof(stdout_buf) - 1, NULL, &eof); -#endif /* defined(_WIN32) */ - log_info(LD_GENERAL, "tor_read_all_handle() returned %d", (int)pos); - - /* We would have blocked, keep on trying */ - if (0 == pos) - continue; - - tt_assert(pos > 0); - stdout_buf[pos] = '\0'; - tt_str_op(expected_out[expected_out_ctr],OP_EQ, stdout_buf); - tt_int_op(strlen(expected_out[expected_out_ctr]),OP_EQ, pos); - expected_out_ctr++; - } - - if (exit_early) { - tor_process_handle_destroy(process_handle, 1); - process_handle = NULL; - goto done; - } - - /* The process should have exited without writing more */ -#ifdef _WIN32 - pos = tor_read_all_handle(process_handle->stdout_pipe, stdout_buf, - sizeof(stdout_buf) - 1, - process_handle); - tt_int_op(0,OP_EQ, pos); -#else /* !(defined(_WIN32)) */ - if (!eof) { - /* We should have got all the data, but maybe not the EOF flag */ - pos = tor_read_all_handle(process_handle->stdout_pipe, stdout_buf, - sizeof(stdout_buf) - 1, - process_handle, &eof); - tt_int_op(0,OP_EQ, pos); - tt_assert(eof); - } - /* Otherwise, we got the EOF on the last read */ -#endif /* defined(_WIN32) */ - - /* Check it terminated correctly */ - retval = tor_get_exit_code(process_handle, 1, &exit_code); - tt_int_op(PROCESS_EXIT_EXITED,OP_EQ, retval); - tt_int_op(expected_exit,OP_EQ, exit_code); - - // TODO: Make test-child exit with something other than 0 - - /* Check stderr */ - pos = tor_read_all_from_process_stderr(process_handle, stderr_buf, - sizeof(stderr_buf) - 1); - tt_assert(pos >= 0); - stderr_buf[pos] = '\0'; - tt_str_op(expected_err,OP_EQ, stderr_buf); - tt_int_op(strlen(expected_err),OP_EQ, pos); - - done: - tor_process_handle_destroy(process_handle, 1); -} - -static void -test_util_spawn_background_partial_read(void *arg) -{ - (void)arg; - test_util_spawn_background_partial_read_impl(0); -} - -static void -test_util_spawn_background_exit_early(void *arg) -{ - (void)arg; - test_util_spawn_background_partial_read_impl(1); -} - -static void -test_util_spawn_background_waitpid_notify(void *arg) -{ - int retval, exit_code; - process_handle_t *process_handle=NULL; - int status; - int ms_timer; - - const char *argv[] = {TEST_CHILD, "--fast", NULL}; - - (void) arg; - -#ifdef _WIN32 - status = tor_spawn_background(NULL, argv, NULL, &process_handle); -#else - status = tor_spawn_background(argv[0], argv, NULL, &process_handle); -#endif - - tt_int_op(status, OP_EQ, PROCESS_STATUS_RUNNING); - tt_ptr_op(process_handle, OP_NE, NULL); - - /* We're not going to look at the stdout/stderr output this time. Instead, - * we're testing whether notify_pending_waitpid_calbacks() can report the - * process exit (on unix) and/or whether tor_get_exit_code() can notice it - * (on windows) */ - -#ifndef _WIN32 - ms_timer = 30*1000; - tt_ptr_op(process_handle->waitpid_cb, OP_NE, NULL); - while (process_handle->waitpid_cb && ms_timer > 0) { - tor_sleep_msec(100); - ms_timer -= 100; - notify_pending_waitpid_callbacks(); - } - tt_int_op(ms_timer, OP_GT, 0); - tt_ptr_op(process_handle->waitpid_cb, OP_EQ, NULL); -#endif /* !defined(_WIN32) */ - - ms_timer = 30*1000; - while (((retval = tor_get_exit_code(process_handle, 0, &exit_code)) - == PROCESS_EXIT_RUNNING) && ms_timer > 0) { - tor_sleep_msec(100); - ms_timer -= 100; - } - tt_int_op(ms_timer, OP_GT, 0); - - tt_int_op(retval, OP_EQ, PROCESS_EXIT_EXITED); - - done: - tor_process_handle_destroy(process_handle, 1); -} - -#undef TEST_CHILD -#undef EOL - -#undef MATCH_PROCESS_STATUS - -#ifndef _WIN32 -#undef PROCESS_STATUS_RUNNING_OR_NOTRUNNING -#undef IS_RUNNING_OR_NOTRUNNING -#endif - -#define UTIL_TEST(name, flags) \ - { #name, test_util_ ## name, flags, NULL, NULL } - -struct testcase_t slow_util_tests[] = { - UTIL_TEST(spawn_background_ok, 0), - UTIL_TEST(spawn_background_fail, 0), - UTIL_TEST(spawn_background_partial_read, 0), - UTIL_TEST(spawn_background_exit_early, 0), - UTIL_TEST(spawn_background_waitpid_notify, 0), - END_OF_TESTCASES -}; diff --git a/src/test/test_voting_flags.c b/src/test/test_voting_flags.c new file mode 100644 index 0000000000..740b96b4c0 --- /dev/null +++ b/src/test/test_voting_flags.c @@ -0,0 +1,191 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" + +#define VOTEFLAGS_PRIVATE + +#include "core/or/or.h" + +#include "feature/dirauth/voteflags.h" +#include "feature/nodelist/node_st.h" +#include "feature/nodelist/routerstatus_st.h" +#include "feature/nodelist/routerinfo_st.h" + +#include "app/config/config.h" + +#include "test/test.h" + +typedef struct { + time_t now; + routerinfo_t ri; + node_t node; + + routerstatus_t expected; +} flag_vote_test_cfg_t; + +static void +setup_cfg(flag_vote_test_cfg_t *c) +{ + memset(c, 0, sizeof(*c)); + + c->now = approx_time(); + + c->ri.nickname = (char *) "testing100"; + strlcpy(c->expected.nickname, "testing100", sizeof(c->expected.nickname)); + + memset(c->ri.cache_info.identity_digest, 0xff, DIGEST_LEN); + memset(c->ri.cache_info.signed_descriptor_digest, 0xee, DIGEST_LEN); + + c->ri.cache_info.published_on = c->now - 100; + c->expected.published_on = c->now - 100; + + c->ri.addr = 0x7f010105; + c->expected.addr = 0x7f010105; + c->ri.or_port = 9090; + c->expected.or_port = 9090; + + tor_addr_make_null(&c->ri.ipv6_addr, AF_INET6); + tor_addr_make_null(&c->expected.ipv6_addr, AF_INET6); + + // By default we have no loaded information about stability or speed, + // so we'll default to voting "yeah sure." on these two. + c->expected.is_fast = 1; + c->expected.is_stable = 1; +} + +static bool +check_result(flag_vote_test_cfg_t *c) +{ + bool result = false; + routerstatus_t rs; + memset(&rs, 0, sizeof(rs)); + set_routerstatus_from_routerinfo(&rs, &c->node, &c->ri, c->now, 0); + + tt_i64_op(rs.published_on, OP_EQ, c->expected.published_on); + tt_str_op(rs.nickname, OP_EQ, c->expected.nickname); + + // identity_digest and descriptor_digest are not set here. + + tt_uint_op(rs.addr, OP_EQ, c->expected.addr); + tt_uint_op(rs.or_port, OP_EQ, c->expected.or_port); + tt_uint_op(rs.dir_port, OP_EQ, c->expected.dir_port); + + tt_assert(tor_addr_eq(&rs.ipv6_addr, &c->expected.ipv6_addr)); + tt_uint_op(rs.ipv6_orport, OP_EQ, c->expected.ipv6_orport); + +#define FLAG(flagname) \ + tt_uint_op(rs.flagname, OP_EQ, c->expected.flagname) + + FLAG(is_authority); + FLAG(is_exit); + FLAG(is_stable); + FLAG(is_fast); + FLAG(is_flagged_running); + FLAG(is_named); + FLAG(is_unnamed); + FLAG(is_valid); + FLAG(is_possible_guard); + FLAG(is_bad_exit); + FLAG(is_hs_dir); + FLAG(is_v2_dir); + FLAG(is_staledesc); + FLAG(has_bandwidth); + FLAG(has_exitsummary); + FLAG(bw_is_unmeasured); + + result = true; + + done: + return result; +} + +static void +test_voting_flags_minimal(void *arg) +{ + flag_vote_test_cfg_t *cfg = arg; + (void) check_result(cfg); +} + +static void +test_voting_flags_ipv6(void *arg) +{ + flag_vote_test_cfg_t *cfg = arg; + + tt_assert(tor_addr_parse(&cfg->ri.ipv6_addr, "f00::b42") == AF_INET6); + cfg->ri.ipv6_orport = 9091; + // no change in expected results, since we aren't set up with ipv6 + // connectivity. + if (!check_result(cfg)) + goto done; + + get_options_mutable()->AuthDirHasIPv6Connectivity = 1; + // no change in expected results, since last_reachable6 won't be set. + if (!check_result(cfg)) + goto done; + + cfg->node.last_reachable6 = cfg->now - 10; + // now that lastreachable6 is set, we expect to see the result. + tt_assert(tor_addr_parse(&cfg->expected.ipv6_addr, "f00::b42") == AF_INET6); + cfg->expected.ipv6_orport = 9091; + if (!check_result(cfg)) + goto done; + done: + ; +} + +static void +test_voting_flags_staledesc(void *arg) +{ + flag_vote_test_cfg_t *cfg = arg; + time_t now = cfg->now; + + cfg->ri.cache_info.published_on = now - DESC_IS_STALE_INTERVAL + 10; + cfg->expected.published_on = now - DESC_IS_STALE_INTERVAL + 10; + // no change in expectations for is_staledesc + if (!check_result(cfg)) + goto done; + + cfg->ri.cache_info.published_on = now - DESC_IS_STALE_INTERVAL - 10; + cfg->expected.published_on = now - DESC_IS_STALE_INTERVAL - 10; + cfg->expected.is_staledesc = 1; + if (!check_result(cfg)) + goto done; + + done: + ; +} + +static void * +setup_voting_flags_test(const struct testcase_t *testcase) +{ + (void)testcase; + flag_vote_test_cfg_t *cfg = tor_malloc_zero(sizeof(*cfg)); + setup_cfg(cfg); + return cfg; +} + +static int +teardown_voting_flags_test(const struct testcase_t *testcase, void *arg) +{ + (void)testcase; + flag_vote_test_cfg_t *cfg = arg; + tor_free(cfg); + return 1; +} + +static const struct testcase_setup_t voting_flags_setup = { + .setup_fn = setup_voting_flags_test, + .cleanup_fn = teardown_voting_flags_test, +}; + +#define T(name,flags) \ + { #name, test_voting_flags_##name, (flags), &voting_flags_setup, NULL } + +struct testcase_t voting_flags_tests[] = { + T(minimal, 0), + T(ipv6, TT_FORK), + // TODO: Add more of these tests. + T(staledesc, TT_FORK), + END_OF_TESTCASES +}; diff --git a/src/test/testing_common.c b/src/test/testing_common.c index c52683afca..6d2db28f15 100644 --- a/src/test/testing_common.c +++ b/src/test/testing_common.c @@ -25,6 +25,8 @@ #include "lib/compress/compress.h" #include "lib/evloop/compat_libevent.h" #include "lib/crypt_ops/crypto_init.h" +#include "lib/version/torversion.h" +#include "app/main/subsysmgr.h" #include <stdio.h> #ifdef HAVE_FCNTL_H @@ -230,12 +232,12 @@ void tinytest_prefork(void) { free_pregenerated_keys(); - crypto_prefork(); + subsystems_prefork(); } void tinytest_postfork(void) { - crypto_postfork(); + subsystems_postfork(); init_pregenerated_keys(); } @@ -250,24 +252,15 @@ main(int c, const char **v) int loglevel = LOG_ERR; int accel_crypto = 0; - /* We must initialise logs before we call tor_assert() */ - init_logging(1); + subsystems_init_upto(SUBSYS_LEVEL_LIBS); - update_approx_time(time(NULL)); options = options_new(); - tor_threads_init(); - tor_compress_init(); - - network_init(); - - monotime_init(); struct tor_libevent_cfg cfg; memset(&cfg, 0, sizeof(cfg)); tor_libevent_initialize(&cfg); control_initialize_event_queue(); - configure_backtrace_handler(get_version()); for (i_out = i = 1; i < c; ++i) { if (!strcmp(v[i], "--warn")) { @@ -294,6 +287,7 @@ main(int c, const char **v) s.masks[LOG_WARN-LOG_ERR] |= LD_BUG; add_stream_log(&s, "", fileno(stdout)); } + flush_log_messages_from_startup(); init_protocol_warning_severity_level(); options->command = CMD_RUN_UNITTESTS; @@ -334,8 +328,6 @@ main(int c, const char **v) free_pregenerated_keys(); - crypto_global_cleanup(); - if (have_failed) return 1; else diff --git a/src/tools/include.am b/src/tools/include.am index f7aa7e0d1e..72dfe6017c 100644 --- a/src/tools/include.am +++ b/src/tools/include.am @@ -5,9 +5,11 @@ noinst_PROGRAMS+= src/tools/tor-cov-resolve endif src_tools_tor_resolve_SOURCES = src/tools/tor-resolve.c -src_tools_tor_resolve_LDFLAGS = +src_tools_tor_resolve_LDFLAGS = @TOR_LDFLAGS_openssl@ src_tools_tor_resolve_LDADD = \ + src/trunnel/libor-trunnel.a \ $(TOR_UTIL_LIBS) \ + $(TOR_CRYPTO_LIBS) $(TOR_LIBS_CRYPTLIB)\ $(rust_ldadd) \ @TOR_LIB_MATH@ @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_USERENV@ @@ -15,8 +17,11 @@ if COVERAGE_ENABLED src_tools_tor_cov_resolve_SOURCES = src/tools/tor-resolve.c src_tools_tor_cov_resolve_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_tools_tor_cov_resolve_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +src_tools_tor_cov_resolve_LDFLAGS = @TOR_LDFLAGS_openssl@ src_tools_tor_cov_resolve_LDADD = \ + src/trunnel/libor-trunnel.a \ $(TOR_UTIL_TESTING_LIBS) \ + $(TOR_CRYPTO_TESTING_LIBS) $(TOR_LIBS_CRYPTLIB) \ @TOR_LIB_MATH@ @TOR_LIB_WS32@ endif diff --git a/src/tools/tor-resolve.c b/src/tools/tor-resolve.c index 95411d173d..803ed26b3b 100644 --- a/src/tools/tor-resolve.c +++ b/src/tools/tor-resolve.c @@ -15,6 +15,7 @@ #include "lib/string/util_string.h" #include "lib/net/socks5_status.h" +#include "trunnel/socks5.h" #include <stdio.h> #include <stdlib.h> @@ -49,68 +50,204 @@ static void usage(void) ATTR_NORETURN; +/** + * Set <b>out</b> to a pointer to newly allocated buffer containing + * SOCKS4a RESOLVE request with <b>username</b> and <b>hostname</b>. + * Return number of bytes in the buffer if succeeded or -1 if failed. + */ +static ssize_t +build_socks4a_resolve_request(uint8_t **out, + const char *username, + const char *hostname) +{ + tor_assert(out); + tor_assert(username); + tor_assert(hostname); + + const char *errmsg = NULL; + uint8_t *output = NULL; + socks4_client_request_t *rq = socks4_client_request_new(); + + socks4_client_request_set_version(rq, 4); + socks4_client_request_set_command(rq, CMD_RESOLVE); + socks4_client_request_set_port(rq, 0); + socks4_client_request_set_addr(rq, 0x00000001u); + socks4_client_request_set_username(rq, username); + socks4_client_request_set_socks4a_addr_hostname(rq, hostname); + + errmsg = socks4_client_request_check(rq); + if (errmsg) { + goto cleanup; + } + + ssize_t encoded_len = socks4_client_request_encoded_len(rq); + if (encoded_len <= 0) { + errmsg = "socks4_client_request_encoded_len failed"; + goto cleanup; + } + + output = tor_malloc(encoded_len); + memset(output, 0, encoded_len); + + encoded_len = socks4_client_request_encode(output, encoded_len, rq); + if (encoded_len <= 0) { + errmsg = "socks4_client_request_encode failed"; + goto cleanup; + } + + *out = output; + + cleanup: + socks4_client_request_free(rq); + if (errmsg) { + log_err(LD_NET, "build_socks4a_resolve_request failed: %s", errmsg); + *out = NULL; + tor_free(output); + } + return errmsg ? -1 : encoded_len; +} + +#define SOCKS5_ATYPE_HOSTNAME 0x03 +#define SOCKS5_ATYPE_IPV4 0x01 +#define SOCKS5_ATYPE_IPV6 0x04 + +/** + * Set <b>out</b> to pointer to newly allocated buffer containing + * SOCKS5 RESOLVE/RESOLVE_PTR request with given <b>hostname<b>. + * Generate a reverse request if <b>reverse</b> is true. + * Return the number of bytes in the buffer if succeeded or -1 if failed. + */ +static ssize_t +build_socks5_resolve_request(uint8_t **out, + const char *hostname, + int reverse) +{ + const char *errmsg = NULL; + uint8_t *outbuf = NULL; + int is_ip_address; + tor_addr_t addr; + size_t addrlen; + int ipv6; + is_ip_address = tor_addr_parse(&addr, hostname) != -1; + if (!is_ip_address && reverse) { + log_err(LD_GENERAL, "Tried to do a reverse lookup on a non-IP!"); + return -1; + } + ipv6 = reverse && tor_addr_family(&addr) == AF_INET6; + addrlen = reverse ? (ipv6 ? 16 : 4) : 1 + strlen(hostname); + if (addrlen > UINT8_MAX) { + log_err(LD_GENERAL, "Hostname is too long!"); + return -1; + } + + socks5_client_request_t *rq = socks5_client_request_new(); + + socks5_client_request_set_version(rq, 5); + /* RESOLVE_PTR or RESOLVE */ + socks5_client_request_set_command(rq, reverse ? CMD_RESOLVE_PTR : + CMD_RESOLVE); + socks5_client_request_set_reserved(rq, 0); + + uint8_t atype = SOCKS5_ATYPE_HOSTNAME; + if (reverse) + atype = ipv6 ? SOCKS5_ATYPE_IPV6 : SOCKS5_ATYPE_IPV4; + + socks5_client_request_set_atype(rq, atype); + + switch (atype) { + case SOCKS5_ATYPE_IPV4: { + socks5_client_request_set_dest_addr_ipv4(rq, + tor_addr_to_ipv4h(&addr)); + } break; + case SOCKS5_ATYPE_IPV6: { + uint8_t *ipv6_array = + socks5_client_request_getarray_dest_addr_ipv6(rq); + + tor_assert(ipv6_array); + + memcpy(ipv6_array, tor_addr_to_in6_addr8(&addr), 16); + } break; + + case SOCKS5_ATYPE_HOSTNAME: { + domainname_t *dn = domainname_new(); + domainname_set_len(dn, addrlen - 1); + domainname_setlen_name(dn, addrlen - 1); + char *dn_buf = domainname_getarray_name(dn); + memcpy(dn_buf, hostname, addrlen - 1); + + errmsg = domainname_check(dn); + + if (errmsg) { + domainname_free(dn); + goto cleanup; + } else { + socks5_client_request_set_dest_addr_domainname(rq, dn); + } + } break; + default: + tor_assert_unreached(); + break; + } + + socks5_client_request_set_dest_port(rq, 0); + + errmsg = socks5_client_request_check(rq); + if (errmsg) { + goto cleanup; + } + + ssize_t encoded_len = socks5_client_request_encoded_len(rq); + if (encoded_len < 0) { + errmsg = "Cannot predict encoded length"; + goto cleanup; + } + + outbuf = tor_malloc(encoded_len); + memset(outbuf, 0, encoded_len); + + encoded_len = socks5_client_request_encode(outbuf, encoded_len, rq); + if (encoded_len < 0) { + errmsg = "encoding failed"; + goto cleanup; + } + + *out = outbuf; + + cleanup: + socks5_client_request_free(rq); + if (errmsg) { + tor_free(outbuf); + log_err(LD_NET, "build_socks5_resolve_request failed with error: %s", + errmsg); + } + + return errmsg ? -1 : encoded_len; +} + /** Set *<b>out</b> to a newly allocated SOCKS4a resolve request with * <b>username</b> and <b>hostname</b> as provided. Return the number * of bytes in the request. */ static ssize_t -build_socks_resolve_request(char **out, +build_socks_resolve_request(uint8_t **out, const char *username, const char *hostname, int reverse, int version) { - size_t len = 0; tor_assert(out); tor_assert(username); tor_assert(hostname); + tor_assert(version == 4 || version == 5); + if (version == 4) { - len = 8 + strlen(username) + 1 + strlen(hostname) + 1; - *out = tor_malloc(len); - (*out)[0] = 4; /* SOCKS version 4 */ - (*out)[1] = '\xF0'; /* Command: resolve. */ - set_uint16((*out)+2, htons(0)); /* port: 0. */ - set_uint32((*out)+4, htonl(0x00000001u)); /* addr: 0.0.0.1 */ - memcpy((*out)+8, username, strlen(username)+1); - memcpy((*out)+8+strlen(username)+1, hostname, strlen(hostname)+1); + return build_socks4a_resolve_request(out, username, hostname); } else if (version == 5) { - int is_ip_address; - tor_addr_t addr; - size_t addrlen; - int ipv6; - is_ip_address = tor_addr_parse(&addr, hostname) != -1; - if (!is_ip_address && reverse) { - log_err(LD_GENERAL, "Tried to do a reverse lookup on a non-IP!"); - return -1; - } - ipv6 = reverse && tor_addr_family(&addr) == AF_INET6; - addrlen = reverse ? (ipv6 ? 16 : 4) : 1 + strlen(hostname); - if (addrlen > UINT8_MAX) { - log_err(LD_GENERAL, "Hostname is too long!"); - return -1; - } - len = 6 + addrlen; - *out = tor_malloc(len); - (*out)[0] = 5; /* SOCKS version 5 */ - (*out)[1] = reverse ? '\xF1' : '\xF0'; /* RESOLVE_PTR or RESOLVE */ - (*out)[2] = 0; /* reserved. */ - if (reverse) { - (*out)[3] = ipv6 ? 4 : 1; - if (ipv6) - memcpy((*out)+4, tor_addr_to_in6_addr8(&addr), 16); - else - set_uint32((*out)+4, tor_addr_to_ipv4n(&addr)); - } else { - (*out)[3] = 3; - (*out)[4] = (char)(uint8_t)(addrlen - 1); - memcpy((*out)+5, hostname, addrlen - 1); - } - set_uint16((*out)+4+addrlen, 0); /* port */ - } else { - tor_assert(0); + return build_socks5_resolve_request(out, hostname, reverse); } - return len; + tor_assert_unreached(); + return -1; } static void @@ -134,34 +271,50 @@ parse_socks4a_resolve_response(const char *hostname, const char *response, size_t len, tor_addr_t *addr_out) { + int result = 0; uint8_t status; tor_assert(response); tor_assert(addr_out); - if (len < RESPONSE_LEN_4) { + socks4_server_reply_t *reply; + + ssize_t parsed = socks4_server_reply_parse(&reply, + (const uint8_t *)response, + len); + + if (parsed == -1) { + log_warn(LD_PROTOCOL, "Failed parsing SOCKS4a response"); + result = -1; goto cleanup; + } + + if (parsed == -2) { log_warn(LD_PROTOCOL,"Truncated socks response."); - return -1; + result = -1; goto cleanup; } - if (((uint8_t)response[0])!=0) { /* version: 0 */ + + if (socks4_server_reply_get_version(reply) != 0) { /* version: 0 */ log_warn(LD_PROTOCOL,"Nonzero version in socks response: bad format."); - return -1; + result = -1; goto cleanup; } - status = (uint8_t)response[1]; - if (get_uint16(response+2)!=0) { /* port: 0 */ + if (socks4_server_reply_get_port(reply) != 0) { /* port: 0 */ log_warn(LD_PROTOCOL,"Nonzero port in socks response: bad format."); - return -1; + result = -1; goto cleanup; } + status = socks4_server_reply_get_status(reply); if (status != 90) { log_warn(LD_NET,"Got status response '%d': socks request failed.", status); if (!strcasecmpend(hostname, ".onion")) { onion_warning(hostname); - return -1; + result = -1; goto cleanup; } - return -1; + result = -1; goto cleanup; } - tor_addr_from_ipv4n(addr_out, get_uint32(response+4)); - return 0; + tor_addr_from_ipv4h(addr_out, socks4_server_reply_get_addr(reply)); + + cleanup: + socks4_server_reply_free(reply); + return result; } /* It would be nice to let someone know what SOCKS5 issue a user may have */ @@ -205,7 +358,7 @@ do_resolve(const char *hostname, int s = -1; struct sockaddr_storage ss; socklen_t socklen; - char *req = NULL; + uint8_t *req = NULL; ssize_t len = 0; tor_assert(hostname); @@ -230,23 +383,57 @@ do_resolve(const char *hostname, } if (version == 5) { - char method_buf[2]; - if (write_all_to_socket(s, "\x05\x01\x00", 3) != 3) { + socks5_client_version_t *v = socks5_client_version_new(); + + socks5_client_version_set_version(v, 5); + socks5_client_version_set_n_methods(v, 1); + socks5_client_version_setlen_methods(v, 1); + socks5_client_version_set_methods(v, 0, 0x00); + + tor_assert(!socks5_client_version_check(v)); + ssize_t encoded_len = socks5_client_version_encoded_len(v); + tor_assert(encoded_len > 0); + + uint8_t *buf = tor_malloc(encoded_len); + encoded_len = socks5_client_version_encode(buf, encoded_len, v); + tor_assert(encoded_len > 0); + + socks5_client_version_free(v); + + if (write_all_to_socket(s, (const char *)buf, + encoded_len) != encoded_len) { log_err(LD_NET, "Error sending SOCKS5 method list."); + tor_free(buf); + goto err; } - if (read_all_from_socket(s, method_buf, 2) != 2) { + + tor_free(buf); + + uint8_t method_buf[2]; + + if (read_all_from_socket(s, (char *)method_buf, 2) != 2) { log_err(LD_NET, "Error reading SOCKS5 methods."); goto err; } - if (method_buf[0] != '\x05') { - log_err(LD_NET, "Unrecognized socks version: %u", - (unsigned)method_buf[0]); + + socks5_server_method_t *m; + ssize_t parsed = socks5_server_method_parse(&m, method_buf, + sizeof(method_buf)); + + if (parsed < 2) { + log_err(LD_NET, "Failed to parse SOCKS5 method selection " + "message"); goto err; } - if (method_buf[1] != '\x00') { + + uint8_t method = socks5_server_method_get_method(m); + + socks5_server_method_free(m); + + if (method != 0x00) { log_err(LD_NET, "Unrecognized socks authentication method: %u", - (unsigned)method_buf[1]); + (unsigned)method); goto err; } } @@ -257,7 +444,7 @@ do_resolve(const char *hostname, tor_assert(!req); goto err; } - if (write_all_to_socket(s, req, len) != len) { + if (write_all_to_socket(s, (const char *)req, len) != len) { log_sock_error("sending SOCKS request", s); tor_free(req); goto err; @@ -276,55 +463,59 @@ do_resolve(const char *hostname, goto err; } } else { - char reply_buf[16]; - if (read_all_from_socket(s, reply_buf, 4) != 4) { - log_err(LD_NET, "Error reading SOCKS5 response."); + uint8_t reply_buf[512]; + + len = read_all_from_socket(s, (char *)reply_buf, + sizeof(reply_buf)); + + socks5_server_reply_t *reply; + + ssize_t parsed = socks5_server_reply_parse(&reply, + reply_buf, + len); + if (parsed == -1) { + log_err(LD_NET, "Failed parsing SOCKS5 response"); goto err; } - if (reply_buf[0] != 5) { - log_err(LD_NET, "Bad SOCKS5 reply version."); + + if (parsed == -2) { + log_err(LD_NET, "Truncated SOCKS5 response"); goto err; } + /* Give a user some useful feedback about SOCKS5 errors */ - if (reply_buf[1] != 0) { + uint8_t reply_field = socks5_server_reply_get_reply(reply); + if (reply_field != 0) { log_warn(LD_NET,"Got SOCKS5 status response '%u': %s", - (unsigned)reply_buf[1], - socks5_reason_to_string(reply_buf[1])); - if (reply_buf[1] == 4 && !strcasecmpend(hostname, ".onion")) { + (unsigned)reply_field, + socks5_reason_to_string(reply_field)); + if (reply_field == 4 && !strcasecmpend(hostname, ".onion")) { onion_warning(hostname); } + + socks5_server_reply_free(reply); goto err; } - if (reply_buf[3] == 1) { + + uint8_t atype = socks5_server_reply_get_atype(reply); + + if (atype == SOCKS5_ATYPE_IPV4) { /* IPv4 address */ - if (read_all_from_socket(s, reply_buf, 4) != 4) { - log_err(LD_NET, "Error reading address in socks5 response."); - goto err; - } - tor_addr_from_ipv4n(result_addr, get_uint32(reply_buf)); - } else if (reply_buf[3] == 4) { + tor_addr_from_ipv4h(result_addr, + socks5_server_reply_get_bind_addr_ipv4(reply)); + } else if (atype == SOCKS5_ATYPE_IPV6) { /* IPv6 address */ - if (read_all_from_socket(s, reply_buf, 16) != 16) { - log_err(LD_NET, "Error reading address in socks5 response."); - goto err; - } - tor_addr_from_ipv6_bytes(result_addr, reply_buf); - } else if (reply_buf[3] == 3) { + tor_addr_from_ipv6_bytes(result_addr, + (const char *)socks5_server_reply_getarray_bind_addr_ipv6(reply)); + } else if (atype == SOCKS5_ATYPE_HOSTNAME) { /* Domain name */ - size_t result_len; - if (read_all_from_socket(s, reply_buf, 1) != 1) { - log_err(LD_NET, "Error reading address_length in socks5 response."); - goto err; - } - result_len = *(uint8_t*)(reply_buf); - *result_hostname = tor_malloc(result_len+1); - if (read_all_from_socket(s, *result_hostname, result_len) - != (int) result_len) { - log_err(LD_NET, "Error reading hostname in socks5 response."); - goto err; - } - (*result_hostname)[result_len] = '\0'; + domainname_t *dn = + socks5_server_reply_get_bind_addr_domainname(reply); + + *result_hostname = tor_strdup(domainname_getstr_name(dn)); } + + socks5_server_reply_free(reply); } tor_close_socket(s); diff --git a/src/trunnel/include.am b/src/trunnel/include.am index 03c1753e96..b5db0609a8 100644 --- a/src/trunnel/include.am +++ b/src/trunnel/include.am @@ -23,7 +23,8 @@ TRUNNELSOURCES = \ src/trunnel/hs/cell_introduce1.c \ src/trunnel/hs/cell_rendezvous.c \ src/trunnel/channelpadding_negotiation.c \ - src/trunnel/socks5.c + src/trunnel/socks5.c \ + src/trunnel/netinfo.c TRUNNELHEADERS = \ src/ext/trunnel/trunnel.h \ @@ -37,7 +38,8 @@ TRUNNELHEADERS = \ src/trunnel/hs/cell_introduce1.h \ src/trunnel/hs/cell_rendezvous.h \ src/trunnel/channelpadding_negotiation.h \ - src/trunnel/socks5.h + src/trunnel/socks5.h \ + src/trunnel/netinfo.h src_trunnel_libor_trunnel_a_SOURCES = $(TRUNNELSOURCES) src_trunnel_libor_trunnel_a_CPPFLAGS = \ diff --git a/src/trunnel/netinfo.c b/src/trunnel/netinfo.c new file mode 100644 index 0000000000..5d815b9b12 --- /dev/null +++ b/src/trunnel/netinfo.c @@ -0,0 +1,723 @@ +/* netinfo.c -- generated by Trunnel v1.5.2. + * https://gitweb.torproject.org/trunnel.git + * You probably shouldn't edit this file. + */ +#include <stdlib.h> +#include "trunnel-impl.h" + +#include "netinfo.h" + +#define TRUNNEL_SET_ERROR_CODE(obj) \ + do { \ + (obj)->trunnel_error_code_ = 1; \ + } while (0) + +#if defined(__COVERITY__) || defined(__clang_analyzer__) +/* If we're running a static analysis tool, we don't want it to complain + * that some of our remaining-bytes checks are dead-code. */ +int netinfo_deadcode_dummy__ = 0; +#define OR_DEADCODE_DUMMY || netinfo_deadcode_dummy__ +#else +#define OR_DEADCODE_DUMMY +#endif + +#define CHECK_REMAINING(nbytes, label) \ + do { \ + if (remaining < (nbytes) OR_DEADCODE_DUMMY) { \ + goto label; \ + } \ + } while (0) + +netinfo_addr_t * +netinfo_addr_new(void) +{ + netinfo_addr_t *val = trunnel_calloc(1, sizeof(netinfo_addr_t)); + if (NULL == val) + return NULL; + return val; +} + +/** Release all storage held inside 'obj', but do not free 'obj'. + */ +static void +netinfo_addr_clear(netinfo_addr_t *obj) +{ + (void) obj; +} + +void +netinfo_addr_free(netinfo_addr_t *obj) +{ + if (obj == NULL) + return; + netinfo_addr_clear(obj); + trunnel_memwipe(obj, sizeof(netinfo_addr_t)); + trunnel_free_(obj); +} + +uint8_t +netinfo_addr_get_addr_type(const netinfo_addr_t *inp) +{ + return inp->addr_type; +} +int +netinfo_addr_set_addr_type(netinfo_addr_t *inp, uint8_t val) +{ + inp->addr_type = val; + return 0; +} +uint8_t +netinfo_addr_get_len(const netinfo_addr_t *inp) +{ + return inp->len; +} +int +netinfo_addr_set_len(netinfo_addr_t *inp, uint8_t val) +{ + inp->len = val; + return 0; +} +uint32_t +netinfo_addr_get_addr_ipv4(const netinfo_addr_t *inp) +{ + return inp->addr_ipv4; +} +int +netinfo_addr_set_addr_ipv4(netinfo_addr_t *inp, uint32_t val) +{ + inp->addr_ipv4 = val; + return 0; +} +size_t +netinfo_addr_getlen_addr_ipv6(const netinfo_addr_t *inp) +{ + (void)inp; return 16; +} + +uint8_t +netinfo_addr_get_addr_ipv6(netinfo_addr_t *inp, size_t idx) +{ + trunnel_assert(idx < 16); + return inp->addr_ipv6[idx]; +} + +uint8_t +netinfo_addr_getconst_addr_ipv6(const netinfo_addr_t *inp, size_t idx) +{ + return netinfo_addr_get_addr_ipv6((netinfo_addr_t*)inp, idx); +} +int +netinfo_addr_set_addr_ipv6(netinfo_addr_t *inp, size_t idx, uint8_t elt) +{ + trunnel_assert(idx < 16); + inp->addr_ipv6[idx] = elt; + return 0; +} + +uint8_t * +netinfo_addr_getarray_addr_ipv6(netinfo_addr_t *inp) +{ + return inp->addr_ipv6; +} +const uint8_t * +netinfo_addr_getconstarray_addr_ipv6(const netinfo_addr_t *inp) +{ + return (const uint8_t *)netinfo_addr_getarray_addr_ipv6((netinfo_addr_t*)inp); +} +const char * +netinfo_addr_check(const netinfo_addr_t *obj) +{ + if (obj == NULL) + return "Object was NULL"; + if (obj->trunnel_error_code_) + return "A set function failed on this object"; + switch (obj->addr_type) { + + case NETINFO_ADDR_TYPE_IPV4: + break; + + case NETINFO_ADDR_TYPE_IPV6: + break; + + default: + break; + } + return NULL; +} + +ssize_t +netinfo_addr_encoded_len(const netinfo_addr_t *obj) +{ + ssize_t result = 0; + + if (NULL != netinfo_addr_check(obj)) + return -1; + + + /* Length of u8 addr_type */ + result += 1; + + /* Length of u8 len */ + result += 1; + switch (obj->addr_type) { + + case NETINFO_ADDR_TYPE_IPV4: + + /* Length of u32 addr_ipv4 */ + result += 4; + break; + + case NETINFO_ADDR_TYPE_IPV6: + + /* Length of u8 addr_ipv6[16] */ + result += 16; + break; + + default: + break; + } + return result; +} +int +netinfo_addr_clear_errors(netinfo_addr_t *obj) +{ + int r = obj->trunnel_error_code_; + obj->trunnel_error_code_ = 0; + return r; +} +ssize_t +netinfo_addr_encode(uint8_t *output, const size_t avail, const netinfo_addr_t *obj) +{ + ssize_t result = 0; + size_t written = 0; + uint8_t *ptr = output; + const char *msg; +#ifdef TRUNNEL_CHECK_ENCODED_LEN + const ssize_t encoded_len = netinfo_addr_encoded_len(obj); +#endif + + uint8_t *backptr_len = NULL; + + if (NULL != (msg = netinfo_addr_check(obj))) + goto check_failed; + +#ifdef TRUNNEL_CHECK_ENCODED_LEN + trunnel_assert(encoded_len >= 0); +#endif + + /* Encode u8 addr_type */ + trunnel_assert(written <= avail); + if (avail - written < 1) + goto truncated; + trunnel_set_uint8(ptr, (obj->addr_type)); + written += 1; ptr += 1; + + /* Encode u8 len */ + backptr_len = ptr; + trunnel_assert(written <= avail); + if (avail - written < 1) + goto truncated; + trunnel_set_uint8(ptr, (obj->len)); + written += 1; ptr += 1; + { + size_t written_before_union = written; + + /* Encode union addr[addr_type] */ + trunnel_assert(written <= avail); + switch (obj->addr_type) { + + case NETINFO_ADDR_TYPE_IPV4: + + /* Encode u32 addr_ipv4 */ + trunnel_assert(written <= avail); + if (avail - written < 4) + goto truncated; + trunnel_set_uint32(ptr, trunnel_htonl(obj->addr_ipv4)); + written += 4; ptr += 4; + break; + + case NETINFO_ADDR_TYPE_IPV6: + + /* Encode u8 addr_ipv6[16] */ + trunnel_assert(written <= avail); + if (avail - written < 16) + goto truncated; + memcpy(ptr, obj->addr_ipv6, 16); + written += 16; ptr += 16; + break; + + default: + break; + } + /* Write the length field back to len */ + trunnel_assert(written >= written_before_union); +#if UINT8_MAX < SIZE_MAX + if (written - written_before_union > UINT8_MAX) + goto check_failed; +#endif + trunnel_set_uint8(backptr_len, (written - written_before_union)); + } + + + trunnel_assert(ptr == output + written); +#ifdef TRUNNEL_CHECK_ENCODED_LEN + { + trunnel_assert(encoded_len >= 0); + trunnel_assert((size_t)encoded_len == written); + } + +#endif + + return written; + + truncated: + result = -2; + goto fail; + check_failed: + (void)msg; + result = -1; + goto fail; + fail: + trunnel_assert(result < 0); + return result; +} + +/** As netinfo_addr_parse(), but do not allocate the output object. + */ +static ssize_t +netinfo_addr_parse_into(netinfo_addr_t *obj, const uint8_t *input, const size_t len_in) +{ + const uint8_t *ptr = input; + size_t remaining = len_in; + ssize_t result = 0; + (void)result; + + /* Parse u8 addr_type */ + CHECK_REMAINING(1, truncated); + obj->addr_type = (trunnel_get_uint8(ptr)); + remaining -= 1; ptr += 1; + + /* Parse u8 len */ + CHECK_REMAINING(1, truncated); + obj->len = (trunnel_get_uint8(ptr)); + remaining -= 1; ptr += 1; + { + size_t remaining_after; + CHECK_REMAINING(obj->len, truncated); + remaining_after = remaining - obj->len; + remaining = obj->len; + + /* Parse union addr[addr_type] */ + switch (obj->addr_type) { + + case NETINFO_ADDR_TYPE_IPV4: + + /* Parse u32 addr_ipv4 */ + CHECK_REMAINING(4, fail); + obj->addr_ipv4 = trunnel_ntohl(trunnel_get_uint32(ptr)); + remaining -= 4; ptr += 4; + break; + + case NETINFO_ADDR_TYPE_IPV6: + + /* Parse u8 addr_ipv6[16] */ + CHECK_REMAINING(16, fail); + memcpy(obj->addr_ipv6, ptr, 16); + remaining -= 16; ptr += 16; + break; + + default: + /* Skip to end of union */ + ptr += remaining; remaining = 0; + break; + } + if (remaining != 0) + goto fail; + remaining = remaining_after; + } + trunnel_assert(ptr + remaining == input + len_in); + return len_in - remaining; + + truncated: + return -2; + fail: + result = -1; + return result; +} + +ssize_t +netinfo_addr_parse(netinfo_addr_t **output, const uint8_t *input, const size_t len_in) +{ + ssize_t result; + *output = netinfo_addr_new(); + if (NULL == *output) + return -1; + result = netinfo_addr_parse_into(*output, input, len_in); + if (result < 0) { + netinfo_addr_free(*output); + *output = NULL; + } + return result; +} +netinfo_cell_t * +netinfo_cell_new(void) +{ + netinfo_cell_t *val = trunnel_calloc(1, sizeof(netinfo_cell_t)); + if (NULL == val) + return NULL; + return val; +} + +/** Release all storage held inside 'obj', but do not free 'obj'. + */ +static void +netinfo_cell_clear(netinfo_cell_t *obj) +{ + (void) obj; + netinfo_addr_free(obj->other_addr); + obj->other_addr = NULL; + { + + unsigned idx; + for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->my_addrs); ++idx) { + netinfo_addr_free(TRUNNEL_DYNARRAY_GET(&obj->my_addrs, idx)); + } + } + TRUNNEL_DYNARRAY_WIPE(&obj->my_addrs); + TRUNNEL_DYNARRAY_CLEAR(&obj->my_addrs); +} + +void +netinfo_cell_free(netinfo_cell_t *obj) +{ + if (obj == NULL) + return; + netinfo_cell_clear(obj); + trunnel_memwipe(obj, sizeof(netinfo_cell_t)); + trunnel_free_(obj); +} + +uint32_t +netinfo_cell_get_timestamp(const netinfo_cell_t *inp) +{ + return inp->timestamp; +} +int +netinfo_cell_set_timestamp(netinfo_cell_t *inp, uint32_t val) +{ + inp->timestamp = val; + return 0; +} +struct netinfo_addr_st * +netinfo_cell_get_other_addr(netinfo_cell_t *inp) +{ + return inp->other_addr; +} +const struct netinfo_addr_st * +netinfo_cell_getconst_other_addr(const netinfo_cell_t *inp) +{ + return netinfo_cell_get_other_addr((netinfo_cell_t*) inp); +} +int +netinfo_cell_set_other_addr(netinfo_cell_t *inp, struct netinfo_addr_st *val) +{ + if (inp->other_addr && inp->other_addr != val) + netinfo_addr_free(inp->other_addr); + return netinfo_cell_set0_other_addr(inp, val); +} +int +netinfo_cell_set0_other_addr(netinfo_cell_t *inp, struct netinfo_addr_st *val) +{ + inp->other_addr = val; + return 0; +} +uint8_t +netinfo_cell_get_n_my_addrs(const netinfo_cell_t *inp) +{ + return inp->n_my_addrs; +} +int +netinfo_cell_set_n_my_addrs(netinfo_cell_t *inp, uint8_t val) +{ + inp->n_my_addrs = val; + return 0; +} +size_t +netinfo_cell_getlen_my_addrs(const netinfo_cell_t *inp) +{ + return TRUNNEL_DYNARRAY_LEN(&inp->my_addrs); +} + +struct netinfo_addr_st * +netinfo_cell_get_my_addrs(netinfo_cell_t *inp, size_t idx) +{ + return TRUNNEL_DYNARRAY_GET(&inp->my_addrs, idx); +} + + const struct netinfo_addr_st * +netinfo_cell_getconst_my_addrs(const netinfo_cell_t *inp, size_t idx) +{ + return netinfo_cell_get_my_addrs((netinfo_cell_t*)inp, idx); +} +int +netinfo_cell_set_my_addrs(netinfo_cell_t *inp, size_t idx, struct netinfo_addr_st * elt) +{ + netinfo_addr_t *oldval = TRUNNEL_DYNARRAY_GET(&inp->my_addrs, idx); + if (oldval && oldval != elt) + netinfo_addr_free(oldval); + return netinfo_cell_set0_my_addrs(inp, idx, elt); +} +int +netinfo_cell_set0_my_addrs(netinfo_cell_t *inp, size_t idx, struct netinfo_addr_st * elt) +{ + TRUNNEL_DYNARRAY_SET(&inp->my_addrs, idx, elt); + return 0; +} +int +netinfo_cell_add_my_addrs(netinfo_cell_t *inp, struct netinfo_addr_st * elt) +{ +#if SIZE_MAX >= UINT8_MAX + if (inp->my_addrs.n_ == UINT8_MAX) + goto trunnel_alloc_failed; +#endif + TRUNNEL_DYNARRAY_ADD(struct netinfo_addr_st *, &inp->my_addrs, elt, {}); + return 0; + trunnel_alloc_failed: + TRUNNEL_SET_ERROR_CODE(inp); + return -1; +} + +struct netinfo_addr_st * * +netinfo_cell_getarray_my_addrs(netinfo_cell_t *inp) +{ + return inp->my_addrs.elts_; +} +const struct netinfo_addr_st * const * +netinfo_cell_getconstarray_my_addrs(const netinfo_cell_t *inp) +{ + return (const struct netinfo_addr_st * const *)netinfo_cell_getarray_my_addrs((netinfo_cell_t*)inp); +} +int +netinfo_cell_setlen_my_addrs(netinfo_cell_t *inp, size_t newlen) +{ + struct netinfo_addr_st * *newptr; +#if UINT8_MAX < SIZE_MAX + if (newlen > UINT8_MAX) + goto trunnel_alloc_failed; +#endif + newptr = trunnel_dynarray_setlen(&inp->my_addrs.allocated_, + &inp->my_addrs.n_, inp->my_addrs.elts_, newlen, + sizeof(inp->my_addrs.elts_[0]), (trunnel_free_fn_t) netinfo_addr_free, + &inp->trunnel_error_code_); + if (newlen != 0 && newptr == NULL) + goto trunnel_alloc_failed; + inp->my_addrs.elts_ = newptr; + return 0; + trunnel_alloc_failed: + TRUNNEL_SET_ERROR_CODE(inp); + return -1; +} +const char * +netinfo_cell_check(const netinfo_cell_t *obj) +{ + if (obj == NULL) + return "Object was NULL"; + if (obj->trunnel_error_code_) + return "A set function failed on this object"; + { + const char *msg; + if (NULL != (msg = netinfo_addr_check(obj->other_addr))) + return msg; + } + { + const char *msg; + + unsigned idx; + for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->my_addrs); ++idx) { + if (NULL != (msg = netinfo_addr_check(TRUNNEL_DYNARRAY_GET(&obj->my_addrs, idx)))) + return msg; + } + } + if (TRUNNEL_DYNARRAY_LEN(&obj->my_addrs) != obj->n_my_addrs) + return "Length mismatch for my_addrs"; + return NULL; +} + +ssize_t +netinfo_cell_encoded_len(const netinfo_cell_t *obj) +{ + ssize_t result = 0; + + if (NULL != netinfo_cell_check(obj)) + return -1; + + + /* Length of u32 timestamp */ + result += 4; + + /* Length of struct netinfo_addr other_addr */ + result += netinfo_addr_encoded_len(obj->other_addr); + + /* Length of u8 n_my_addrs */ + result += 1; + + /* Length of struct netinfo_addr my_addrs[n_my_addrs] */ + { + + unsigned idx; + for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->my_addrs); ++idx) { + result += netinfo_addr_encoded_len(TRUNNEL_DYNARRAY_GET(&obj->my_addrs, idx)); + } + } + return result; +} +int +netinfo_cell_clear_errors(netinfo_cell_t *obj) +{ + int r = obj->trunnel_error_code_; + obj->trunnel_error_code_ = 0; + return r; +} +ssize_t +netinfo_cell_encode(uint8_t *output, const size_t avail, const netinfo_cell_t *obj) +{ + ssize_t result = 0; + size_t written = 0; + uint8_t *ptr = output; + const char *msg; +#ifdef TRUNNEL_CHECK_ENCODED_LEN + const ssize_t encoded_len = netinfo_cell_encoded_len(obj); +#endif + + if (NULL != (msg = netinfo_cell_check(obj))) + goto check_failed; + +#ifdef TRUNNEL_CHECK_ENCODED_LEN + trunnel_assert(encoded_len >= 0); +#endif + + /* Encode u32 timestamp */ + trunnel_assert(written <= avail); + if (avail - written < 4) + goto truncated; + trunnel_set_uint32(ptr, trunnel_htonl(obj->timestamp)); + written += 4; ptr += 4; + + /* Encode struct netinfo_addr other_addr */ + trunnel_assert(written <= avail); + result = netinfo_addr_encode(ptr, avail - written, obj->other_addr); + if (result < 0) + goto fail; /* XXXXXXX !*/ + written += result; ptr += result; + + /* Encode u8 n_my_addrs */ + trunnel_assert(written <= avail); + if (avail - written < 1) + goto truncated; + trunnel_set_uint8(ptr, (obj->n_my_addrs)); + written += 1; ptr += 1; + + /* Encode struct netinfo_addr my_addrs[n_my_addrs] */ + { + + unsigned idx; + for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->my_addrs); ++idx) { + trunnel_assert(written <= avail); + result = netinfo_addr_encode(ptr, avail - written, TRUNNEL_DYNARRAY_GET(&obj->my_addrs, idx)); + if (result < 0) + goto fail; /* XXXXXXX !*/ + written += result; ptr += result; + } + } + + + trunnel_assert(ptr == output + written); +#ifdef TRUNNEL_CHECK_ENCODED_LEN + { + trunnel_assert(encoded_len >= 0); + trunnel_assert((size_t)encoded_len == written); + } + +#endif + + return written; + + truncated: + result = -2; + goto fail; + check_failed: + (void)msg; + result = -1; + goto fail; + fail: + trunnel_assert(result < 0); + return result; +} + +/** As netinfo_cell_parse(), but do not allocate the output object. + */ +static ssize_t +netinfo_cell_parse_into(netinfo_cell_t *obj, const uint8_t *input, const size_t len_in) +{ + const uint8_t *ptr = input; + size_t remaining = len_in; + ssize_t result = 0; + (void)result; + + /* Parse u32 timestamp */ + CHECK_REMAINING(4, truncated); + obj->timestamp = trunnel_ntohl(trunnel_get_uint32(ptr)); + remaining -= 4; ptr += 4; + + /* Parse struct netinfo_addr other_addr */ + result = netinfo_addr_parse(&obj->other_addr, ptr, remaining); + if (result < 0) + goto relay_fail; + trunnel_assert((size_t)result <= remaining); + remaining -= result; ptr += result; + + /* Parse u8 n_my_addrs */ + CHECK_REMAINING(1, truncated); + obj->n_my_addrs = (trunnel_get_uint8(ptr)); + remaining -= 1; ptr += 1; + + /* Parse struct netinfo_addr my_addrs[n_my_addrs] */ + TRUNNEL_DYNARRAY_EXPAND(netinfo_addr_t *, &obj->my_addrs, obj->n_my_addrs, {}); + { + netinfo_addr_t * elt; + unsigned idx; + for (idx = 0; idx < obj->n_my_addrs; ++idx) { + result = netinfo_addr_parse(&elt, ptr, remaining); + if (result < 0) + goto relay_fail; + trunnel_assert((size_t)result <= remaining); + remaining -= result; ptr += result; + TRUNNEL_DYNARRAY_ADD(netinfo_addr_t *, &obj->my_addrs, elt, {netinfo_addr_free(elt);}); + } + } + trunnel_assert(ptr + remaining == input + len_in); + return len_in - remaining; + + truncated: + return -2; + relay_fail: + trunnel_assert(result < 0); + return result; + trunnel_alloc_failed: + return -1; +} + +ssize_t +netinfo_cell_parse(netinfo_cell_t **output, const uint8_t *input, const size_t len_in) +{ + ssize_t result; + *output = netinfo_cell_new(); + if (NULL == *output) + return -1; + result = netinfo_cell_parse_into(*output, input, len_in); + if (result < 0) { + netinfo_cell_free(*output); + *output = NULL; + } + return result; +} diff --git a/src/trunnel/netinfo.h b/src/trunnel/netinfo.h new file mode 100644 index 0000000000..ac46e603ba --- /dev/null +++ b/src/trunnel/netinfo.h @@ -0,0 +1,226 @@ +/* netinfo.h -- generated by Trunnel v1.5.2. + * https://gitweb.torproject.org/trunnel.git + * You probably shouldn't edit this file. + */ +#ifndef TRUNNEL_NETINFO_H +#define TRUNNEL_NETINFO_H + +#include <stdint.h> +#include "trunnel.h" + +#define NETINFO_ADDR_TYPE_IPV4 4 +#define NETINFO_ADDR_TYPE_IPV6 6 +#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_NETINFO_ADDR) +struct netinfo_addr_st { + uint8_t addr_type; + uint8_t len; + uint32_t addr_ipv4; + uint8_t addr_ipv6[16]; + uint8_t trunnel_error_code_; +}; +#endif +typedef struct netinfo_addr_st netinfo_addr_t; +#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_NETINFO_CELL) +struct netinfo_cell_st { + uint32_t timestamp; + struct netinfo_addr_st *other_addr; + uint8_t n_my_addrs; + TRUNNEL_DYNARRAY_HEAD(, struct netinfo_addr_st *) my_addrs; + uint8_t trunnel_error_code_; +}; +#endif +typedef struct netinfo_cell_st netinfo_cell_t; +/** Return a newly allocated netinfo_addr with all elements set to + * zero. + */ +netinfo_addr_t *netinfo_addr_new(void); +/** Release all storage held by the netinfo_addr in 'victim'. (Do + * nothing if 'victim' is NULL.) + */ +void netinfo_addr_free(netinfo_addr_t *victim); +/** Try to parse a netinfo_addr from the buffer in 'input', using up + * to 'len_in' bytes from the input buffer. On success, return the + * number of bytes consumed and set *output to the newly allocated + * netinfo_addr_t. On failure, return -2 if the input appears + * truncated, and -1 if the input is otherwise invalid. + */ +ssize_t netinfo_addr_parse(netinfo_addr_t **output, const uint8_t *input, const size_t len_in); +/** Return the number of bytes we expect to need to encode the + * netinfo_addr in 'obj'. On failure, return a negative value. Note + * that this value may be an overestimate, and can even be an + * underestimate for certain unencodeable objects. + */ +ssize_t netinfo_addr_encoded_len(const netinfo_addr_t *obj); +/** Try to encode the netinfo_addr from 'input' into the buffer at + * 'output', using up to 'avail' bytes of the output buffer. On + * success, return the number of bytes used. On failure, return -2 if + * the buffer was not long enough, and -1 if the input was invalid. + */ +ssize_t netinfo_addr_encode(uint8_t *output, size_t avail, const netinfo_addr_t *input); +/** Check whether the internal state of the netinfo_addr in 'obj' is + * consistent. Return NULL if it is, and a short message if it is not. + */ +const char *netinfo_addr_check(const netinfo_addr_t *obj); +/** Clear any errors that were set on the object 'obj' by its setter + * functions. Return true iff errors were cleared. + */ +int netinfo_addr_clear_errors(netinfo_addr_t *obj); +/** Return the value of the addr_type field of the netinfo_addr_t in + * 'inp' + */ +uint8_t netinfo_addr_get_addr_type(const netinfo_addr_t *inp); +/** Set the value of the addr_type field of the netinfo_addr_t in + * 'inp' to 'val'. Return 0 on success; return -1 and set the error + * code on 'inp' on failure. + */ +int netinfo_addr_set_addr_type(netinfo_addr_t *inp, uint8_t val); +/** Return the value of the len field of the netinfo_addr_t in 'inp' + */ +uint8_t netinfo_addr_get_len(const netinfo_addr_t *inp); +/** Set the value of the len field of the netinfo_addr_t in 'inp' to + * 'val'. Return 0 on success; return -1 and set the error code on + * 'inp' on failure. + */ +int netinfo_addr_set_len(netinfo_addr_t *inp, uint8_t val); +/** Return the value of the addr_ipv4 field of the netinfo_addr_t in + * 'inp' + */ +uint32_t netinfo_addr_get_addr_ipv4(const netinfo_addr_t *inp); +/** Set the value of the addr_ipv4 field of the netinfo_addr_t in + * 'inp' to 'val'. Return 0 on success; return -1 and set the error + * code on 'inp' on failure. + */ +int netinfo_addr_set_addr_ipv4(netinfo_addr_t *inp, uint32_t val); +/** Return the (constant) length of the array holding the addr_ipv6 + * field of the netinfo_addr_t in 'inp'. + */ +size_t netinfo_addr_getlen_addr_ipv6(const netinfo_addr_t *inp); +/** Return the element at position 'idx' of the fixed array field + * addr_ipv6 of the netinfo_addr_t in 'inp'. + */ +uint8_t netinfo_addr_get_addr_ipv6(netinfo_addr_t *inp, size_t idx); +/** As netinfo_addr_get_addr_ipv6, but take and return a const pointer + */ +uint8_t netinfo_addr_getconst_addr_ipv6(const netinfo_addr_t *inp, size_t idx); +/** Change the element at position 'idx' of the fixed array field + * addr_ipv6 of the netinfo_addr_t in 'inp', so that it will hold the + * value 'elt'. + */ +int netinfo_addr_set_addr_ipv6(netinfo_addr_t *inp, size_t idx, uint8_t elt); +/** Return a pointer to the 16-element array field addr_ipv6 of 'inp'. + */ +uint8_t * netinfo_addr_getarray_addr_ipv6(netinfo_addr_t *inp); +/** As netinfo_addr_get_addr_ipv6, but take and return a const pointer + */ +const uint8_t * netinfo_addr_getconstarray_addr_ipv6(const netinfo_addr_t *inp); +/** Return a newly allocated netinfo_cell with all elements set to + * zero. + */ +netinfo_cell_t *netinfo_cell_new(void); +/** Release all storage held by the netinfo_cell in 'victim'. (Do + * nothing if 'victim' is NULL.) + */ +void netinfo_cell_free(netinfo_cell_t *victim); +/** Try to parse a netinfo_cell from the buffer in 'input', using up + * to 'len_in' bytes from the input buffer. On success, return the + * number of bytes consumed and set *output to the newly allocated + * netinfo_cell_t. On failure, return -2 if the input appears + * truncated, and -1 if the input is otherwise invalid. + */ +ssize_t netinfo_cell_parse(netinfo_cell_t **output, const uint8_t *input, const size_t len_in); +/** Return the number of bytes we expect to need to encode the + * netinfo_cell in 'obj'. On failure, return a negative value. Note + * that this value may be an overestimate, and can even be an + * underestimate for certain unencodeable objects. + */ +ssize_t netinfo_cell_encoded_len(const netinfo_cell_t *obj); +/** Try to encode the netinfo_cell from 'input' into the buffer at + * 'output', using up to 'avail' bytes of the output buffer. On + * success, return the number of bytes used. On failure, return -2 if + * the buffer was not long enough, and -1 if the input was invalid. + */ +ssize_t netinfo_cell_encode(uint8_t *output, size_t avail, const netinfo_cell_t *input); +/** Check whether the internal state of the netinfo_cell in 'obj' is + * consistent. Return NULL if it is, and a short message if it is not. + */ +const char *netinfo_cell_check(const netinfo_cell_t *obj); +/** Clear any errors that were set on the object 'obj' by its setter + * functions. Return true iff errors were cleared. + */ +int netinfo_cell_clear_errors(netinfo_cell_t *obj); +/** Return the value of the timestamp field of the netinfo_cell_t in + * 'inp' + */ +uint32_t netinfo_cell_get_timestamp(const netinfo_cell_t *inp); +/** Set the value of the timestamp field of the netinfo_cell_t in + * 'inp' to 'val'. Return 0 on success; return -1 and set the error + * code on 'inp' on failure. + */ +int netinfo_cell_set_timestamp(netinfo_cell_t *inp, uint32_t val); +/** Return the value of the other_addr field of the netinfo_cell_t in + * 'inp' + */ +struct netinfo_addr_st * netinfo_cell_get_other_addr(netinfo_cell_t *inp); +/** As netinfo_cell_get_other_addr, but take and return a const + * pointer + */ +const struct netinfo_addr_st * netinfo_cell_getconst_other_addr(const netinfo_cell_t *inp); +/** Set the value of the other_addr field of the netinfo_cell_t in + * 'inp' to 'val'. Free the old value if any. Steals the referenceto + * 'val'.Return 0 on success; return -1 and set the error code on + * 'inp' on failure. + */ +int netinfo_cell_set_other_addr(netinfo_cell_t *inp, struct netinfo_addr_st *val); +/** As netinfo_cell_set_other_addr, but does not free the previous + * value. + */ +int netinfo_cell_set0_other_addr(netinfo_cell_t *inp, struct netinfo_addr_st *val); +/** Return the value of the n_my_addrs field of the netinfo_cell_t in + * 'inp' + */ +uint8_t netinfo_cell_get_n_my_addrs(const netinfo_cell_t *inp); +/** Set the value of the n_my_addrs field of the netinfo_cell_t in + * 'inp' to 'val'. Return 0 on success; return -1 and set the error + * code on 'inp' on failure. + */ +int netinfo_cell_set_n_my_addrs(netinfo_cell_t *inp, uint8_t val); +/** Return the length of the dynamic array holding the my_addrs field + * of the netinfo_cell_t in 'inp'. + */ +size_t netinfo_cell_getlen_my_addrs(const netinfo_cell_t *inp); +/** Return the element at position 'idx' of the dynamic array field + * my_addrs of the netinfo_cell_t in 'inp'. + */ +struct netinfo_addr_st * netinfo_cell_get_my_addrs(netinfo_cell_t *inp, size_t idx); +/** As netinfo_cell_get_my_addrs, but take and return a const pointer + */ + const struct netinfo_addr_st * netinfo_cell_getconst_my_addrs(const netinfo_cell_t *inp, size_t idx); +/** Change the element at position 'idx' of the dynamic array field + * my_addrs of the netinfo_cell_t in 'inp', so that it will hold the + * value 'elt'. Free the previous value, if any. + */ +int netinfo_cell_set_my_addrs(netinfo_cell_t *inp, size_t idx, struct netinfo_addr_st * elt); +/** As netinfo_cell_set_my_addrs, but does not free the previous + * value. + */ +int netinfo_cell_set0_my_addrs(netinfo_cell_t *inp, size_t idx, struct netinfo_addr_st * elt); +/** Append a new element 'elt' to the dynamic array field my_addrs of + * the netinfo_cell_t in 'inp'. + */ +int netinfo_cell_add_my_addrs(netinfo_cell_t *inp, struct netinfo_addr_st * elt); +/** Return a pointer to the variable-length array field my_addrs of + * 'inp'. + */ +struct netinfo_addr_st * * netinfo_cell_getarray_my_addrs(netinfo_cell_t *inp); +/** As netinfo_cell_get_my_addrs, but take and return a const pointer + */ +const struct netinfo_addr_st * const * netinfo_cell_getconstarray_my_addrs(const netinfo_cell_t *inp); +/** Change the length of the variable-length array field my_addrs of + * 'inp' to 'newlen'.Fill extra elements with NULL; free removed + * elements. Return 0 on success; return -1 and set the error code on + * 'inp' on failure. + */ +int netinfo_cell_setlen_my_addrs(netinfo_cell_t *inp, size_t newlen); + + +#endif diff --git a/src/trunnel/netinfo.trunnel b/src/trunnel/netinfo.trunnel new file mode 100644 index 0000000000..2c4b7a7591 --- /dev/null +++ b/src/trunnel/netinfo.trunnel @@ -0,0 +1,24 @@ +// Warning: make sure these values are consistent with RESOLVED_TYPE_* +// constants in Tor code and numbers in Section 6.4 of tor-spec.txt. + +const NETINFO_ADDR_TYPE_IPV4 = 4; +const NETINFO_ADDR_TYPE_IPV6 = 6; + +struct netinfo_addr { + u8 addr_type; + u8 len; + union addr[addr_type] with length len { + NETINFO_ADDR_TYPE_IPV4: u32 ipv4; + NETINFO_ADDR_TYPE_IPV6: u8 ipv6[16]; + default: ignore; + }; + +} + +struct netinfo_cell { + u32 timestamp; + struct netinfo_addr other_addr; + u8 n_my_addrs; + struct netinfo_addr my_addrs[n_my_addrs]; +} + diff --git a/src/trunnel/socks5.c b/src/trunnel/socks5.c index 9e5f6fcfed..057a52b042 100644 --- a/src/trunnel/socks5.c +++ b/src/trunnel/socks5.c @@ -1694,7 +1694,6 @@ socks4_server_reply_new(void) socks4_server_reply_t *val = trunnel_calloc(1, sizeof(socks4_server_reply_t)); if (NULL == val) return NULL; - val->version = 4; return val; } @@ -1724,7 +1723,7 @@ socks4_server_reply_get_version(const socks4_server_reply_t *inp) int socks4_server_reply_set_version(socks4_server_reply_t *inp, uint8_t val) { - if (! ((val == 4))) { + if (! ((val == 0 || val == 4))) { TRUNNEL_SET_ERROR_CODE(inp); return -1; } @@ -1771,7 +1770,7 @@ socks4_server_reply_check(const socks4_server_reply_t *obj) return "Object was NULL"; if (obj->trunnel_error_code_) return "A set function failed on this object"; - if (! (obj->version == 4)) + if (! (obj->version == 0 || obj->version == 4)) return "Integer out of bounds"; return NULL; } @@ -1785,7 +1784,7 @@ socks4_server_reply_encoded_len(const socks4_server_reply_t *obj) return -1; - /* Length of u8 version IN [4] */ + /* Length of u8 version IN [0, 4] */ result += 1; /* Length of u8 status */ @@ -1823,7 +1822,7 @@ socks4_server_reply_encode(uint8_t *output, const size_t avail, const socks4_ser trunnel_assert(encoded_len >= 0); #endif - /* Encode u8 version IN [4] */ + /* Encode u8 version IN [0, 4] */ trunnel_assert(written <= avail); if (avail - written < 1) goto truncated; @@ -1886,11 +1885,11 @@ socks4_server_reply_parse_into(socks4_server_reply_t *obj, const uint8_t *input, ssize_t result = 0; (void)result; - /* Parse u8 version IN [4] */ + /* Parse u8 version IN [0, 4] */ CHECK_REMAINING(1, truncated); obj->version = (trunnel_get_uint8(ptr)); remaining -= 1; ptr += 1; - if (! (obj->version == 4)) + if (! (obj->version == 0 || obj->version == 4)) goto fail; /* Parse u8 status */ diff --git a/src/trunnel/socks5.trunnel b/src/trunnel/socks5.trunnel index b86ec03b9d..17fa2ed996 100644 --- a/src/trunnel/socks5.trunnel +++ b/src/trunnel/socks5.trunnel @@ -86,7 +86,7 @@ struct socks4_client_request { } struct socks4_server_reply { - u8 version IN [4]; + u8 version IN [0,4]; u8 status; u16 port; u32 addr; diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h index 4645d4a20f..cfc3bc9e9e 100644 --- a/src/win32/orconfig.h +++ b/src/win32/orconfig.h @@ -218,7 +218,7 @@ #define USING_TWOS_COMPLEMENT /* Version number of package */ -#define VERSION "0.3.5.6-rc-dev" +#define VERSION "0.4.0.0-alpha-dev" |