diff options
274 files changed, 9667 insertions, 2112 deletions
diff --git a/.gitignore b/.gitignore index f304a32ecf..2f94254c53 100644 --- a/.gitignore +++ b/.gitignore @@ -132,6 +132,9 @@ uptime-*.json /src/common/libor.a /src/common/libor-testing.a /src/common/libor.lib +/src/common/libor-ctime.a +/src/common/libor-ctime-testing.a +/src/common/libor-ctime.lib /src/common/libor-crypto.a /src/common/libor-crypto-testing.a /src/common/libor-crypto.lib @@ -180,6 +183,7 @@ uptime-*.json /src/test/test-memwipe /src/test/test-ntor-cl /src/test/test-switch-id +/src/test/test-timers /src/test/test_workqueue /src/test/test.exe /src/test/test-slow.exe @@ -188,6 +192,7 @@ uptime-*.json /src/test/test-ntor-cl.exe /src/test/test-memwipe.exe /src/test/test-switch-id.exe +/src/test/test-timers.exe /src/test/test_workqueue.exe # /src/tools/ @@ -1,3 +1,227 @@ +Changes in version 0.2.9.1-alpha - 2016-??-?? + + + +Changes in version 0.2.8.4-rc - 2016-06-15 + Tor 0.2.8.4-rc is the first release candidate in the Tor 0.2.8 series. + If we find no new bugs or regressions here, the first stable 0.2.8 + release will be identical to it. It has a few small bugfixes against + previous versions. + + o Major bugfixes (user interface): + - Correctly give a warning in the cases where a relay is specified + by nickname, and one such relay is found, but it is not officially + Named. Fixes bug 19203; bugfix on 0.2.3.1-alpha. + + o Minor features (build): + - Tor now builds once again with the recent OpenSSL 1.1 development + branch (tested against 1.1.0-pre5 and 1.1.0-pre6-dev). + + o Minor features (geoip): + - Update geoip and geoip6 to the June 7 2016 Maxmind GeoLite2 + Country database. + + o Minor bugfixes (compilation): + - Cause the unit tests to compile correctly on mingw64 versions that + lack sscanf. Fixes bug 19213; bugfix on 0.2.7.1-alpha. + + o Minor bugfixes (downloading): + - Predict more correctly whether we'll be downloading over HTTP when + we determine the maximum length of a URL. This should avoid a + "BUG" warning about the Squid HTTP proxy and its URL limits. Fixes + bug 19191. + + +Changes in version 0.2.8.3-alpha - 2016-05-26 + Tor 0.2.8.3-alpha resolves several bugs, most of them introduced over + the course of the 0.2.8 development cycle. It improves the behavior of + directory clients, fixes several crash bugs, fixes a gap in compiler + hardening, and allows the full integration test suite to run on + more platforms. + + o Major bugfixes (security, client, DNS proxy): + - Stop a crash that could occur when a client running with DNSPort + received a query with multiple address types, and the first + address type was not supported. Found and fixed by Scott Dial. + Fixes bug 18710; bugfix on 0.2.5.4-alpha. + + o Major bugfixes (security, compilation): + - Correctly detect compiler flags on systems where _FORTIFY_SOURCE + is predefined. Previously, our use of -D_FORTIFY_SOURCE would + cause a compiler warning, thereby making other checks fail, and + needlessly disabling compiler-hardening support. Fixes one case of + bug 18841; bugfix on 0.2.3.17-beta. Patch from "trudokal". + + o Major bugfixes (security, directory authorities): + - Fix a crash and out-of-bounds write during authority voting, when + the list of relays includes duplicate ed25519 identity keys. Fixes + bug 19032; bugfix on 0.2.8.2-alpha. + + o Major bugfixes (client, bootstrapping): + - Check if bootstrap consensus downloads are still needed when the + linked connection attaches. This prevents tor making unnecessary + begindir-style connections, which are the only directory + connections tor clients make since the fix for 18483 was merged. + - Fix some edge cases where consensus download connections may not + have been closed, even though they were not needed. Related to fix + for 18809. + - Make relays retry consensus downloads the correct number of times, + rather than the more aggressive client retry count. Fixes part of + ticket 18809. + - Stop downloading consensuses when we have a consensus, even if we + don't have all the certificates for it yet. Fixes bug 18809; + bugfix on 0.2.8.1-alpha. Patches by arma and teor. + + o Major bugfixes (directory mirrors): + - Decide whether to advertise begindir support in the the same way + we decide whether to advertise our DirPort. Allowing these + decisions to become out-of-sync led to surprising behavior like + advertising begindir support when hibernation made us not + advertise a DirPort. Resolves bug 18616; bugfix on 0.2.8.1-alpha. + Patch by teor. + + o Major bugfixes (IPv6 bridges, client): + - Actually use IPv6 addresses when selecting directory addresses for + IPv6 bridges. Fixes bug 18921; bugfix on 0.2.8.1-alpha. Patch + by "teor". + + o Major bugfixes (key management): + - If OpenSSL fails to generate an RSA key, do not retain a dangling + pointer to the previous (uninitialized) key value. The impact here + should be limited to a difficult-to-trigger crash, if OpenSSL is + running an engine that makes key generation failures possible, or + if OpenSSL runs out of memory. Fixes bug 19152; bugfix on + 0.2.1.10-alpha. Found by Yuan Jochen Kang, Suman Jana, and + Baishakhi Ray. + + o Major bugfixes (testing): + - Fix a bug that would block 'make test-network-all' on systems where + IPv6 packets were lost. Fixes bug 19008; bugfix on tor-0.2.7.3-rc. + - Avoid "WSANOTINITIALISED" warnings in the unit tests. Fixes bug 18668; + bugfix on 0.2.8.1-alpha. + + o Minor features (clients): + - Make clients, onion services, and bridge relays always use an + encrypted begindir connection for directory requests. Resolves + ticket 18483. Patch by "teor". + + o Minor features (fallback directory mirrors): + - Give each fallback the same weight for client selection; restrict + fallbacks to one per operator; report fallback directory detail + changes when rebuilding list; add new fallback directory mirrors + to the whitelist; update fallback directories based on the latest + OnionOO data; and any other minor simplifications and fixes. + Closes tasks 17158, 17905, 18749, bug 18689, and fixes part of bug + 18812 on 0.2.8.1-alpha; patch by "teor". + + o Minor features (geoip): + - Update geoip and geoip6 to the May 4 2016 Maxmind GeoLite2 + Country database. + + o Minor bugfixes (assert, portability): + - Fix an assertion failure in memarea.c on systems where "long" is + shorter than the size of a pointer. Fixes bug 18716; bugfix + on 0.2.1.1-alpha. + + o Minor bugfixes (bootstrap): + - Consistently use the consensus download schedule for authority + certificates. Fixes bug 18816; bugfix on 0.2.4.13-alpha. + + o Minor bugfixes (build): + - Remove a pair of redundant AM_CONDITIONAL declarations from + configure.ac. Fixes one final case of bug 17744; bugfix + on 0.2.8.2-alpha. + - Resolve warnings when building on systems that are concerned with + signed char. Fixes bug 18728; bugfix on 0.2.7.2-alpha + and 0.2.6.1-alpha. + - When libscrypt.h is found, but no libscrypt library can be linked, + treat libscrypt as absent. Fixes bug 19161; bugfix + on 0.2.6.1-alpha. + + o Minor bugfixes (client): + - Turn all TestingClientBootstrap* into non-testing torrc options. + This changes simply renames them by removing "Testing" in front of + them and they do not require TestingTorNetwork to be enabled + anymore. Fixes bug 18481; bugfix on 0.2.8.1-alpha. + - Make directory node selection more reliable, mainly for IPv6-only + clients and clients with few reachable addresses. Fixes bug 18929; + bugfix on 0.2.8.1-alpha. Patch by "teor". + + o Minor bugfixes (controller, microdescriptors): + - Make GETINFO dir/status-vote/current/consensus conform to the + control specification by returning "551 Could not open cached + consensus..." when not caching consensuses. Fixes bug 18920; + bugfix on 0.2.2.6-alpha. + + o Minor bugfixes (crypto, portability): + - The SHA3 and SHAKE routines now produce the correct output on Big + Endian systems. No code calls either algorithm yet, so this is + primarily a build fix. Fixes bug 18943; bugfix on 0.2.8.1-alpha. + - Tor now builds again with the recent OpenSSL 1.1 development + branch (tested against 1.1.0-pre4 and 1.1.0-pre5-dev). Closes + ticket 18286. + + o Minor bugfixes (directories): + - When fetching extrainfo documents, compare their SHA256 digests + and Ed25519 signing key certificates with the routerinfo that led + us to fetch them, rather than with the most recent routerinfo. + Otherwise we generate many spurious warnings about mismatches. + Fixes bug 17150; bugfix on 0.2.7.2-alpha. + + o Minor bugfixes (logging): + - When we can't generate a signing key because OfflineMasterKey is + set, do not imply that we should have been able to load it. Fixes + bug 18133; bugfix on 0.2.7.2-alpha. + - Stop periodic_event_dispatch() from blasting twelve lines per + second at loglevel debug. Fixes bug 18729; fix on 0.2.8.1-alpha. + - When rejecting a misformed INTRODUCE2 cell, only log at + PROTOCOL_WARN severity. Fixes bug 18761; bugfix on 0.2.8.2-alpha. + + o Minor bugfixes (pluggable transports): + - Avoid reporting a spurious error when we decide that we don't need + to terminate a pluggable transport because it has already exited. + Fixes bug 18686; bugfix on 0.2.5.5-alpha. + + o Minor bugfixes (pointer arithmetic): + - Fix a bug in memarea_alloc() that could have resulted in remote + heap write access, if Tor had ever passed an unchecked size to + memarea_alloc(). Fortunately, all the sizes we pass to + memarea_alloc() are pre-checked to be less than 128 kilobytes. + Fixes bug 19150; bugfix on 0.2.1.1-alpha. Bug found by + Guido Vranken. + + o Minor bugfixes (relays): + - Consider more config options when relays decide whether to + regenerate their descriptor. Fixes more of bug 12538; bugfix + on 0.2.8.1-alpha. + - Resolve some edge cases where we might launch an ORPort + reachability check even when DisableNetwork is set. Noticed while + fixing bug 18616; bugfix on 0.2.3.9-alpha. + + o Minor bugfixes (statistics): + - We now include consensus downloads via IPv6 in our directory- + request statistics. Fixes bug 18460; bugfix on 0.2.3.14-alpha. + + o Minor bugfixes (testing): + - Allow directories in small networks to bootstrap by skipping + DirPort checks when the consensus has no exits. Fixes bug 19003; + bugfix on 0.2.8.1-alpha. Patch by teor. + - Fix a small memory leak that would occur when the + TestingEnableCellStatsEvent option was turned on. Fixes bug 18673; + bugfix on 0.2.5.2-alpha. + + o Minor bugfixes (time handling): + - When correcting a corrupt 'struct tm' value, fill in the tm_wday + field. Otherwise, our unit tests crash on Windows. Fixes bug + 18977; bugfix on 0.2.2.25-alpha. + + o Documentation: + - Document the contents of the 'datadir/keys' subdirectory in the + manual page. Closes ticket 17621. + - Stop recommending use of nicknames to identify relays in our + MapAddress documentation. Closes ticket 18312. + + Changes in version 0.2.8.2-alpha - 2016-03-28 Tor 0.2.8.2-alpha is the second alpha in its series. It fixes numerous bugs in earlier versions of Tor, including some that prevented @@ -271,6 +271,110 @@ src/ext/readpassphrase.[ch] are distributed under this license: Materiel Command, USAF, under agreement number F39502-99-1-0512. =============================================================================== +src/ext/mulodi4.c is distributed under this license: + + ========================================================================= + compiler_rt License + ========================================================================= + + The compiler_rt library is dual licensed under both the + University of Illinois "BSD-Like" license and the MIT license. + As a user of this code you may choose to use it under either + license. As a contributor, you agree to allow your code to be + used under both. + + Full text of the relevant licenses is included below. + + ========================================================================= + + University of Illinois/NCSA + Open Source License + + Copyright (c) 2009-2016 by the contributors listed in CREDITS.TXT + + All rights reserved. + + Developed by: + + LLVM Team + + University of Illinois at Urbana-Champaign + + http://llvm.org + + Permission is hereby granted, free of charge, to any person + obtaining a copy of this software and associated documentation + files (the "Software"), to deal with the Software without + restriction, including without limitation the rights to use, + copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following + conditions: + + * Redistributions of source code must retain the above + copyright notice, this list of conditions and the following + disclaimers. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimers in the documentation and/or other materials + provided with the distribution. + + * Neither the names of the LLVM Team, University of Illinois + at Urbana-Champaign, nor the names of its contributors may + be used to endorse or promote products derived from this + Software without specific prior written permission. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE CONTRIBUTORS OR COPYRIGHT + HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + OTHER DEALINGS WITH THE SOFTWARE. + + ========================================================================= + + Copyright (c) 2009-2015 by the contributors listed in CREDITS.TXT + + Permission is hereby granted, free of charge, to any person + obtaining a copy of this software and associated documentation + files (the "Software"), to deal in the Software without + restriction, including without limitation the rights to use, + copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following + conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + OTHER DEALINGS IN THE SOFTWARE. + + ========================================================================= + Copyrights and Licenses for Third Party Software Distributed with LLVM: + ========================================================================= + + The LLVM software contains code written by third parties. Such + software will have its own individual LICENSE.TXT file in the + directory in which it appears. This file will describe the + copyrights, license, and restrictions which apply to that code. + + The disclaimer of warranty in the University of Illinois Open + Source License applies to all code in the LLVM Distribution, and + nothing in any of the other licenses gives permission to use the + names of the LLVM Team or the University of Illinois to endorse + or promote products derived from this Software. + +=============================================================================== If you got Tor as a static binary with OpenSSL included, then you should know: "This product includes software developed by the OpenSSL Project for use in the OpenSSL Toolkit (http://www.openssl.org/)" diff --git a/Makefile.am b/Makefile.am index cd88264264..a8aa3af40d 100644 --- a/Makefile.am +++ b/Makefile.am @@ -15,13 +15,13 @@ noinst_PROGRAMS= DISTCLEANFILES= bin_SCRIPTS= AM_CPPFLAGS= -AM_CFLAGS = @TOR_SYSTEMD_CFLAGS@ -SHELL = @SHELL@ +AM_CFLAGS=@TOR_SYSTEMD_CFLAGS@ @CFLAGS_BUGTRAP@ +SHELL=@SHELL@ if COVERAGE_ENABLED -TESTING_TOR_BINARY="$(top_builddir)/src/or/tor-cov" +TESTING_TOR_BINARY=$(top_builddir)/src/or/tor-cov$(EXEEXT) else -TESTING_TOR_BINARY="$(top_builddir)/src/or/tor" +TESTING_TOR_BINARY=$(top_builddir)/src/or/tor$(EXEEXT) endif include src/include.am @@ -93,7 +93,7 @@ need-chutney-path: # Note that test-network requires a copy of Chutney in $CHUTNEY_PATH. # Chutney can be cloned from https://git.torproject.org/chutney.git . -test-network: need-chutney-path all +test-network: need-chutney-path $(TESTING_TOR_BINARY) src/tools/tor-gencert $(top_srcdir)/src/test/test-network.sh $(TEST_NETWORK_FLAGS) # Run all available tests using automake's test-driver @@ -101,7 +101,7 @@ test-network: need-chutney-path all # some IPv6 tests will fail without an IPv6 DNS server (see #16971 and #17011) # only run mixed tests if we have a tor-stable binary # see #17015 for autodetection of different tor versions -test-network-all: need-chutney-path all test-driver +test-network-all: need-chutney-path test-driver $(TESTING_TOR_BINARY) src/tools/tor-gencert mkdir -p $(TEST_NETWORK_ALL_LOG_DIR) @flavors="$(TEST_CHUTNEY_FLAVORS)"; \ if ping6 -q -c 1 -o ::1 >/dev/null 2>&1; then \ @@ -134,11 +134,11 @@ need-stem-path: exit 1; \ fi -test-stem: need-stem-path all - @$(PYTHON) "$$STEM_SOURCE_DIR"/run_tests.py --tor $(TESTING_TOR_BINARY) --all --log notice --target RUN_ALL; +test-stem: need-stem-path $(TESTING_TOR_BINARY) + @$(PYTHON) "$$STEM_SOURCE_DIR"/run_tests.py --tor "$(TESTING_TOR_BINARY)" --all --log notice --target RUN_ALL; -test-stem-full: need-stem-path all - @$(PYTHON) "$$STEM_SOURCE_DIR"/run_tests.py --tor $(TESTING_TOR_BINARY) --all --log notice --target RUN_ALL,ONLINE -v; +test-stem-full: need-stem-path $(TESTING_TOR_BINARY) + @$(PYTHON) "$$STEM_SOURCE_DIR"/run_tests.py --tor "$(TESTING_TOR_BINARY)" --all --log notice --target RUN_ALL,ONLINE -v; test-full: need-stem-path need-chutney-path check test-network test-stem diff --git a/acinclude.m4 b/acinclude.m4 index 7b1aab2f99..ab12317139 100644 --- a/acinclude.m4 +++ b/acinclude.m4 @@ -42,10 +42,11 @@ AC_DEFUN([TOR_DEFINE_CODEPATH], AC_SUBST(TOR_LDFLAGS_$2) ]) -dnl 1:flags -dnl 2:also try to link (yes: non-empty string) -dnl will set yes or no in $tor_can_link_$1 (as modified by AS_VAR_PUSHDEF) -AC_DEFUN([TOR_CHECK_CFLAGS], [ +dnl 1: flags +dnl 2: try to link too if this is nonempty. +dnl 3: what to do on success compiling +dnl 4: what to do on failure compiling +AC_DEFUN([TOR_TRY_COMPILE_WITH_CFLAGS], [ AS_VAR_PUSHDEF([VAR],[tor_cv_cflags_$1]) AC_CACHE_CHECK([whether the compiler accepts $1], VAR, [ tor_saved_CFLAGS="$CFLAGS" @@ -63,12 +64,21 @@ AC_DEFUN([TOR_CHECK_CFLAGS], [ CFLAGS="$tor_saved_CFLAGS" ]) if test x$VAR = xyes; then - CFLAGS="$CFLAGS $1" + $3 + else + $4 fi AS_VAR_POPDEF([VAR]) ]) dnl 1:flags +dnl 2:also try to link (yes: non-empty string) +dnl will set yes or no in $tor_can_link_$1 (as modified by AS_VAR_PUSHDEF) +AC_DEFUN([TOR_CHECK_CFLAGS], [ + TOR_TRY_COMPILE_WITH_CFLAGS($1, $2, CFLAGS="$CFLAGS $1", true) +]) + +dnl 1:flags dnl 2:extra ldflags dnl 3:extra libraries AC_DEFUN([TOR_CHECK_LDFLAGS], [ diff --git a/changes/19044 b/changes/19044 new file mode 100644 index 0000000000..a7f938a248 --- /dev/null +++ b/changes/19044 @@ -0,0 +1,5 @@ + o Minor features (compilation): + - Our big list of extra GCC warnings is now enabled by default when + building with GCC (or with anything like Clang that claims to be + GCC-compatible). To make all warnings into fatal compilation errors, + pass --enable-fatal-warnings to configure. Closes ticket 19044. diff --git a/changes/assert_nonfatal b/changes/assert_nonfatal new file mode 100644 index 0000000000..0cbee4419b --- /dev/null +++ b/changes/assert_nonfatal @@ -0,0 +1,5 @@ + o Minor features (safety, debugging): + + * Add a set of macros to check nonfatal assertions, for internal + use. Migrating more of our checks to these should help us avoid + needless crash bugs. Closes ticket 18613. diff --git a/changes/bug13239 b/changes/bug13239 new file mode 100644 index 0000000000..17030c923a --- /dev/null +++ b/changes/bug13239 @@ -0,0 +1,4 @@ + o Minor bugfixes (hidden service client): + - Increase the minimum number of internal circuits we preemptively build + from 2 to 3 so they are available when a client connects to another + onion service. Fixes bug 13239; bugfix on tor-0.1.0.1-rc~460. diff --git a/changes/bug14334 b/changes/bug14334 new file mode 100644 index 0000000000..c53781ecf2 --- /dev/null +++ b/changes/bug14334 @@ -0,0 +1,4 @@ + o Minor bugfixes (guards): + - Don't mark guards as unreachable if connection_connect() fails. That + function fails for local reasons, so it shouldn't reveal anything about + the status of the guard. Fixes bug #14334; bugfix on 0.2.3.10-alpha. diff --git a/changes/bug15942 b/changes/bug15942 new file mode 100644 index 0000000000..0edc2b7980 --- /dev/null +++ b/changes/bug15942 @@ -0,0 +1,3 @@ + o Bugfixes (downloading): + - Use random exponential backoffs when retrying downloads from the dir + servers. Fixes bug 15942; bugfix on ?????. diff --git a/changes/bug17688 b/changes/bug17688 new file mode 100644 index 0000000000..aae60082c0 --- /dev/null +++ b/changes/bug17688 @@ -0,0 +1,3 @@ + o Minor bugfixes (guard selection): + - Use a single entry guard even if the NumEntryGuards consensus parameter + is not provided. Fixes bug #17688. diff --git a/changes/bug17744_redux b/changes/bug17744_redux deleted file mode 100644 index d61e17fec3..0000000000 --- a/changes/bug17744_redux +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (build): - - Remove a pair of redundant AM_CONDITIONAL declarations from - configure.ac. Fixes one final case of bug 17744; bugfix on - 0.2.8.2-alpha. - diff --git a/changes/bug17983 b/changes/bug17983 new file mode 100644 index 0000000000..db52a37615 --- /dev/null +++ b/changes/bug17983 @@ -0,0 +1,11 @@ + o Minor features (bug-finding): + - Tor now builds with -ftrapv by default on compilers that support it. + This option detects signed integer overflow, and turns it into a + hard-failure. We do not apply this option to code that needs to run + in constant time to avoid side-channels; instead, we use -fwrapv. + Closes ticket 17983. + - When --enable-expensive-hardening is selected, stop applying the clang/gcc + sanitizers to code that needs to run in constant-time to avoid side + channels: although we are aware of no introduced side-channels, we + are not able to prove that this is safe. Related to ticket 17983. + diff --git a/changes/bug18133 b/changes/bug18133 deleted file mode 100644 index 177d286495..0000000000 --- a/changes/bug18133 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (logging): - - When we can't generate a signing key because OfflineMasterKey is set, - do not imply that we should have been able to load it. - Fixes bug 18133; bugfix on 0.2.7.2-alpha. diff --git a/changes/bug18240 b/changes/bug18240 new file mode 100644 index 0000000000..6be7ba18de --- /dev/null +++ b/changes/bug18240 @@ -0,0 +1,5 @@ + o Minor bugfixes (build): + - Make the test-stem and test-network targets depend only on the + tor binary to be tested. Previously, they depended on "make all". + Fixes bug 18240; bugfix on 0.2.8.2-alpha. + Based on a patch from "cypherpunks". diff --git a/changes/bug18286 b/changes/bug18286 deleted file mode 100644 index e398fb004b..0000000000 --- a/changes/bug18286 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor features (build): - - Tor now builds again with the recent OpenSSL 1.1 development branch - (tested against 1.1.0-pre4 and 1.1.0-pre5-dev). Closes ticket 18286. - - diff --git a/changes/bug18300 b/changes/bug18300 new file mode 100644 index 0000000000..791752ae0b --- /dev/null +++ b/changes/bug18300 @@ -0,0 +1,3 @@ + o Minor features (logging): + - Provide a more useful warning message when configured with an + invalid Nickname. Closes ticket 18300; patch from "icanhasaccount". diff --git a/changes/bug18312 b/changes/bug18312 deleted file mode 100644 index 7dcb3266bf..0000000000 --- a/changes/bug18312 +++ /dev/null @@ -1,4 +0,0 @@ - o Documentation: - - Stop recommending use of nicknames to identify relays in our - MapAddress documentation. Closes ticket 18312. - diff --git a/changes/bug18460 b/changes/bug18460 deleted file mode 100644 index a8c1a19774..0000000000 --- a/changes/bug18460 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (statistics): - - We now include consensus downloads via IPv6 in our directory-request statistics. - Fixes bug 18460; bugfix on 0.2.3.14-alpha. - diff --git a/changes/bug18481 b/changes/bug18481 deleted file mode 100644 index 6fd882b36b..0000000000 --- a/changes/bug18481 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (client): - - Turn all TestingClientBootstrap* into non-testing torrc options. This - changes simply renames them by removing "Testing" in front of them and - they do not require TestingTorNetwork to be enabled anymore. Fixes - bug 18481; bugfix on 0.2.8.1-alpha. diff --git a/changes/bug18604 b/changes/bug18604 new file mode 100644 index 0000000000..114c67e1c0 --- /dev/null +++ b/changes/bug18604 @@ -0,0 +1,4 @@ + o Minor bugfixes (ephemeral hidden service) + - When deleting an ephemeral hidden service, close its intro points even + if not in the open state. Resolves ticket #18604; bugfix on + tor-0.2.7.1-alpha. diff --git a/changes/bug18616 b/changes/bug18616 deleted file mode 100644 index ec59e846ed..0000000000 --- a/changes/bug18616 +++ /dev/null @@ -1,14 +0,0 @@ - o Major bugfixes (directory mirrors): - - Decide whether to advertise begindir support the same way we decide - whether to advertise our DirPort. These decisions being out of sync - led to surprising behavior like advertising begindir support when - our hibernation config options made us not advertise a DirPort. - Resolves bug 18616; bugfix on 0.2.8.1-alpha. Patch by teor. - - o Minor bugfixes: - - Consider more config options when relays decide whether to regenerate - their descriptor. Fixes more of bug 12538; bugfix on 0.2.8.1-alpha. - - Resolve some edge cases where we might launch an ORPort reachability - check even when DisableNetwork is set. Noticed while fixing bug - 18616; bugfix on 0.2.3.9-alpha. - diff --git a/changes/bug18673 b/changes/bug18673 deleted file mode 100644 index 5d6161718a..0000000000 --- a/changes/bug18673 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (memory leak): - - Fix a small memory leak that would occur when the - TestingEnableCellStatsEvent option was turned on. Fixes bug 18673; - bugfix on 0.2.5.2-alpha. diff --git a/changes/bug18686 b/changes/bug18686 deleted file mode 100644 index 23547d211d..0000000000 --- a/changes/bug18686 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (pluggable transports): - - Avoid reporting a spurious error when we decide that we don't - need to terminate a pluggable transport because it has already - exited. Fixes bug 18686; bugfix on 0.2.5.5-alpha. - diff --git a/changes/bug18704 b/changes/bug18704 new file mode 100644 index 0000000000..966cc2a5f3 --- /dev/null +++ b/changes/bug18704 @@ -0,0 +1,5 @@ + o Major bugfixes (hidden service client) + - With FetchHidServDescriptors set to 0, there is no descriptor fetch + (which is intended) but also no descriptor cache lookup was done + making any Tor client not working with this option unset. Resolves + ticket #18704. Patch by "twim"; Bugfix on tor-0.2.0.20-rc. diff --git a/changes/bug18710 b/changes/bug18710 deleted file mode 100644 index 269395563d..0000000000 --- a/changes/bug18710 +++ /dev/null @@ -1,6 +0,0 @@ - o Major bugfixes (DNS proxy): - - Stop a crash that could occur when a client running with DNSPort - received a query with multiple address types, where the first - address type was not supported. Found and fixed by Scott Dial. - Fixes bug 18710; bugfix on 0.2.5.4-alpha. - diff --git a/changes/bug18716 b/changes/bug18716 deleted file mode 100644 index b15a343f4c..0000000000 --- a/changes/bug18716 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (assert, portability): - - Fix an assertion failure in memarea.c on systems where "long" is - shorter than the size of a pointer. - Fixes bug 18716; bugfix on 0.2.1.1-alpha diff --git a/changes/bug18728 b/changes/bug18728 deleted file mode 100644 index e181c17e65..0000000000 --- a/changes/bug18728 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (build): - - Resolve warnings when building on systems that are concerned with - signed char. Fixes bug 18728; bugfix on 0.2.7.2-alpha and - 0.2.6.1-alpha. diff --git a/changes/bug18729 b/changes/bug18729 deleted file mode 100644 index 4ec9ca3254..0000000000 --- a/changes/bug18729 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor features (logging): - - Stop blasting twelve lines per second from periodic_event_dispatch() - at loglevel debug. Resolves ticket 18729; fix on 0.2.8.1-alpha. diff --git a/changes/bug18761 b/changes/bug18761 deleted file mode 100644 index 78500a88ea..0000000000 --- a/changes/bug18761 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor feature (logging): - - When rejecting a misformed INTRODUCE2 cell, only log at PROTOCOL_WARN - severity. Closes ticket 18761. diff --git a/changes/bug18815 b/changes/bug18815 new file mode 100644 index 0000000000..cb504b2a8e --- /dev/null +++ b/changes/bug18815 @@ -0,0 +1,3 @@ + o Minor features (performance): + - When fetching a consensus for the first time, use optimistic data. + This saves a round-trip during startup. Closes ticket 18815. diff --git a/changes/bug18816 b/changes/bug18816 deleted file mode 100644 index 103f816962..0000000000 --- a/changes/bug18816 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfix (bootstrap): - - Consistently use the consensus download schedule for - authority certificates. - Fixes bug 18816; bugfix on 0.2.4.13-alpha. diff --git a/changes/bug18840 b/changes/bug18840 new file mode 100644 index 0000000000..b8de1aae88 --- /dev/null +++ b/changes/bug18840 @@ -0,0 +1,4 @@ + o Minor bugfixes: + - Authorities now sort the "package" lines in their votes, for ease + of debugging. (They are already sorted in the consensus documents.) + Fixes bug 18840; bugfix on 0.2.6.3-alpha. diff --git a/changes/bug18841.1 b/changes/bug18841.1 deleted file mode 100644 index 205ee5a425..0000000000 --- a/changes/bug18841.1 +++ /dev/null @@ -1,7 +0,0 @@ - o Major bugfixes (compilation): - - Correctly detect compiler flags on systems where _FORTIFY_SOURCE - is predefined. Previously, our use of -D_FORTIFY_SOURCE would - cause a compiler warning, thereby making other checks fail. - Fixes one case of bug 18841; bugfix on 0.2.3.17-beta. Patch from - "trudokal". - diff --git a/changes/bug18889 b/changes/bug18889 new file mode 100644 index 0000000000..45b09921d6 --- /dev/null +++ b/changes/bug18889 @@ -0,0 +1,2 @@ + o Code simplification and refactoring: + - Remove redundant declarations of the MIN macro. Closes ticket 18889. diff --git a/changes/bug18895 b/changes/bug18895 new file mode 100644 index 0000000000..9ad857c546 --- /dev/null +++ b/changes/bug18895 @@ -0,0 +1,6 @@ + o Minor features (build): + - When building on a system without runtime support for some of the + runtime hardening options, try to log a useful warning at configuration + time, rather than an incomprehensible warning at link time. + If expensive hardening was requested, this warning becomes an error. + Closes ticket 18895. diff --git a/changes/bug18920 b/changes/bug18920 deleted file mode 100644 index 1babfd6656..0000000000 --- a/changes/bug18920 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (controller, microdescriptors): - - Make GETINFO dir/status-vote/current/consensus conform to the control - specification by returning "551 Could not open cached consensus..." - when not caching consensuses. - Fixes bug 18920; bugfix on 0.2.2.6-alpha. diff --git a/changes/bug18921 b/changes/bug18921 deleted file mode 100644 index cdd868a005..0000000000 --- a/changes/bug18921 +++ /dev/null @@ -1,4 +0,0 @@ - o Major bugfixes (IPv6 bridges): - - Fix directory address selection for IPv6 bridges. - Fixes bug 18921; bugfix on 0.2.8.1-alpha. - Patch by "teor". diff --git a/changes/bug18929 b/changes/bug18929 deleted file mode 100644 index c607e630a6..0000000000 --- a/changes/bug18929 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (IPv6): - - Make directory node selection more reliable, mainly for - IPv6-only clients and clients with few reachable addresses. - Fixes bug 18929; bugfix on 0.2.8.1-alpha. - Patch by "teor". diff --git a/changes/bug18934 b/changes/bug18934 new file mode 100644 index 0000000000..fba703e5a4 --- /dev/null +++ b/changes/bug18934 @@ -0,0 +1,3 @@ + o Minor features (testing): + - Let backtrace tests work correctly under AddressSanitizer. + Fixes part of bug 18934. diff --git a/changes/bug18934_2 b/changes/bug18934_2 new file mode 100644 index 0000000000..039feafb89 --- /dev/null +++ b/changes/bug18934_2 @@ -0,0 +1,4 @@ + o Minor bugfixes (testing): + - Disable ASAN's detection of segmentation faults while running + test_bt.sh, so that we can make sure that our own backtrace generation + code works. Fixes another aspect of bug 18934. Patch from "cypherpunks". diff --git a/changes/bug18943 b/changes/bug18943 deleted file mode 100644 index 6bcd868460..0000000000 --- a/changes/bug18943 +++ /dev/null @@ -1,6 +0,0 @@ - o Major bugfixes (crypto, portability): - - The SHA3 and SHAKE routines now produce the correct output on - Big Endian systems, unbreaking the unit tests. No code calls - either algorithm family yet, so this is primarily a build fix. - Fixes bug 18943; bugfix on 0.2.8.1-alpha. - diff --git a/changes/bug18956 b/changes/bug18956 new file mode 100644 index 0000000000..0cf10e9224 --- /dev/null +++ b/changes/bug18956 @@ -0,0 +1,5 @@ + o Minor bugfixes (memory leaks): + - Fix a small, uncommon memory leak that could occur when reading a + truncated ed25519 key file. Fixes bug 18956; bugfix on 0.2.6.1-alpha. + + diff --git a/changes/bug18963 b/changes/bug18963 new file mode 100644 index 0000000000..f122288b1d --- /dev/null +++ b/changes/bug18963 @@ -0,0 +1,5 @@ + o Minor bugfix (bootstrap): + - Remember the directory we fetched the consensus or previous + certificates from, and use it to fetch future authority + certificates. + Resolves ticket 18963; fix on #4483 in 0.2.8.1-alpha. diff --git a/changes/bug18977 b/changes/bug18977 deleted file mode 100644 index 3f46b09fba..0000000000 --- a/changes/bug18977 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (time handling): - - When correcting a corrupt 'struct tm' value, fill in the tm_wday - field. Otherwise, our unit tests crash on Windows. - Fixes bug 18977; bugfix on 0.2.2.25-alpha. diff --git a/changes/bug19003 b/changes/bug19003 deleted file mode 100644 index ca94938ef9..0000000000 --- a/changes/bug19003 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (small networks): - - Allow directories in small networks to bootstrap by - skipping DirPort checks when the consensus has no exits. - Fixes bug 19003; bugfix on 0.2.8.1-alpha. - Patch by teor. diff --git a/changes/bug19008 b/changes/bug19008 deleted file mode 100644 index c51c98faa6..0000000000 --- a/changes/bug19008 +++ /dev/null @@ -1,3 +0,0 @@ - o Major bugfixes (testing): - - Fix a bug that would block 'make test-network-all' on systems - where IPv6 packets were lost. Fixes bug 19008; bugfix on tor-0.2.7.3-rc. diff --git a/changes/bug19032 b/changes/bug19032 deleted file mode 100644 index 93f17c2f91..0000000000 --- a/changes/bug19032 +++ /dev/null @@ -1,4 +0,0 @@ - o Major bugfixes (security, directory authorities): - - Fix a crash and out-of-bounds write during authority voting, when the - list of relays includes duplicate ed25519 identity keys. Fixes bug 19032; - bugfix on 0.2.8.2-alpha. diff --git a/changes/bug19063 b/changes/bug19063 new file mode 100644 index 0000000000..10020098bc --- /dev/null +++ b/changes/bug19063 @@ -0,0 +1,4 @@ + o Minor features (code safety): + - In our integer-parsing functions, check that the maxiumum + value given is no smaller than the minimum value. Closes ticket + 19063; patch from U+039b. diff --git a/changes/bug19066 b/changes/bug19066 new file mode 100644 index 0000000000..c3d1fc789a --- /dev/null +++ b/changes/bug19066 @@ -0,0 +1,5 @@ + o Minor bugfixes (directory authority): + - When parsing detached signature, make sure we use the length of the + digest algorithm instead of an hardcoded DIGEST256_LEN in order to + avoid comparing bytes out of bound with a smaller digest length such + as SHA1. Fixes #19066; bugfix on tor-0.2.2.6-alpha. diff --git a/changes/bug19079 b/changes/bug19079 new file mode 100644 index 0000000000..6cbc6cee3b --- /dev/null +++ b/changes/bug19079 @@ -0,0 +1,4 @@ + o Minor features (build): + - Detect and work around a libclang_rt problem that prevents clang from + finding __mulodi4() on some 32-bit platforms. This clang bug would keep + -ftrapv from linking on those systems. Closes ticket 19079. diff --git a/changes/bug19139 b/changes/bug19139 new file mode 100644 index 0000000000..2e53d7d445 --- /dev/null +++ b/changes/bug19139 @@ -0,0 +1,3 @@ + o Minor features (build): + - Use the Autoconf macro AC_USE_SYSTEM_EXTENSIONS to automatically + turn on C and POSIX extensions. Closes ticket 19139. diff --git a/changes/bug19153 b/changes/bug19153 new file mode 100644 index 0000000000..ac235c5e3a --- /dev/null +++ b/changes/bug19153 @@ -0,0 +1,3 @@ + o Documentation: + - Fix spelling of "--enable-tor2web-mode" in the manpage. Closes + ticket #19153. Patch from "U+039b". diff --git a/changes/bug19216 b/changes/bug19216 new file mode 100644 index 0000000000..0bca7d1cab --- /dev/null +++ b/changes/bug19216 @@ -0,0 +1,4 @@ + o Minor bugfixes: + - When building with Clang, include our full array of GCC warnings. + (Previously, we included only a subset, because of the way we + detected them.) Fixes bug 19216; bugfix on 0.2.0.1-alpha. diff --git a/changes/bug19428 b/changes/bug19428 new file mode 100644 index 0000000000..32d293eb8e --- /dev/null +++ b/changes/bug19428 @@ -0,0 +1,5 @@ + - Minor bugfixes (timing): + o When computing the difference between two times in milliseconds, + we now round to the nearest millisecond correctly. Previously, + we could sometimes round in the wrong direction. Fixes bug 19428; + bugfix on 0.2.2.2-alpha. diff --git a/changes/doc17621 b/changes/doc17621 deleted file mode 100644 index ab37d29b50..0000000000 --- a/changes/doc17621 +++ /dev/null @@ -1,3 +0,0 @@ - o Documentation: - - Document the contents of the 'datadir/keys' subdirectory in the manual - page. Closes ticket 17621. diff --git a/changes/fallbacks-201604 b/changes/fallbacks-201604 deleted file mode 100644 index 7acefaaf08..0000000000 --- a/changes/fallbacks-201604 +++ /dev/null @@ -1,9 +0,0 @@ - o Minor features (fallback directory mirrors): - - Give each fallback the same weight for client selection; - restrict fallbacks to one per operator; - report fallback directory detail changes when rebuilding list; - add new fallback directory mirrors to the whitelist; - update fallback directories based on the latest OnionOO data; - and any other minor simplifications and fixes. - Closes tasks 17158, 17905, 18749, bug 18689, and fixes part of - bug 18812 on 0.2.8.1-alpha; patch by "teor". diff --git a/changes/feature15588 b/changes/feature15588 new file mode 100644 index 0000000000..b5563079e1 --- /dev/null +++ b/changes/feature15588 @@ -0,0 +1,4 @@ + o Minor features (controller): + - Add support for configuring basic client authorization on hidden + services created with the ADD_ONION control command. + Implements ticket 15588. Patch by "special". diff --git a/changes/feature18483 b/changes/feature18483 deleted file mode 100644 index d0fa8df58d..0000000000 --- a/changes/feature18483 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (clients): - - Make clients, onion services, and bridge relays always - use an encrypted begindir connection for directory requests. - Resolves ticket 18483. Patch by "teor". diff --git a/changes/feature18624 b/changes/feature18624 new file mode 100644 index 0000000000..a3be90f745 --- /dev/null +++ b/changes/feature18624 @@ -0,0 +1,7 @@ + o Minor features: + - Directory authorities now only give the Guard flag to a relay if + they are also giving it the Stable flag. This change allows us to + simplify path selection for clients, and it should have minimal + effect in practice since >99% of Guards already have the Stable + flag. Implements ticket 18624. + diff --git a/changes/feature18685 b/changes/feature18685 new file mode 100644 index 0000000000..bc0d1be8e5 --- /dev/null +++ b/changes/feature18685 @@ -0,0 +1,3 @@ + o Minor features (controller): + - Fire a `STATUS_SERVER` event whenever the hibernation status changes + between "awake"/"soft"/"hard". Closes ticket 18685. diff --git a/changes/feature18760 b/changes/feature18760 new file mode 100644 index 0000000000..e6e8f6aad3 --- /dev/null +++ b/changes/feature18760 @@ -0,0 +1,6 @@ + o Minor features: + - When the directory authorities refuse a bad relay's descriptor, + encourage the relay operator to contact us. Many relay operators + won't notice this line in their logs, but it's a win if even a + few learn why we don't like what their relay was doing. Resolves + ticket 18760. diff --git a/changes/feature18998 b/changes/feature18998 new file mode 100644 index 0000000000..a2679c016c --- /dev/null +++ b/changes/feature18998 @@ -0,0 +1,5 @@ + o Minor features: + - Stop being so strict about the payload length of "rendezvous1" + cells. We used to be locked in to the "tap" handshake length, + and now we can handle better handshakes like "ntor". Resolves + ticket 18998. diff --git a/changes/feature19036 b/changes/feature19036 new file mode 100644 index 0000000000..98bcfca515 --- /dev/null +++ b/changes/feature19036 @@ -0,0 +1,4 @@ + o Minor features: + - Make directory authorities write the v3-status-votes file out + to disk earlier in the consensus process, so we have the votes + even if we abort the consensus process below. Resolves ticket 19036. diff --git a/changes/geoip-april2016 b/changes/geoip-april2016 deleted file mode 100644 index c55aa179b5..0000000000 --- a/changes/geoip-april2016 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (geoip): - - Update geoip and geoip6 to the April 5 2016 Maxmind GeoLite2 - Country database. - diff --git a/changes/geoip-may2016 b/changes/geoip-may2016 deleted file mode 100644 index cf78ab10c7..0000000000 --- a/changes/geoip-may2016 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (geoip): - - Update geoip and geoip6 to the May 4 2016 Maxmind GeoLite2 - Country database. - diff --git a/changes/lcov_excl b/changes/lcov_excl new file mode 100644 index 0000000000..474181cfa3 --- /dev/null +++ b/changes/lcov_excl @@ -0,0 +1,7 @@ + o Minor features (testing): + - Use the lcov convention for marking lines as unreachable, so that + we don't count them when we're generating test coverage data. + Update our coverage tools to understand this convention. + Closes ticket #16792. + + diff --git a/changes/task19035 b/changes/task19035 new file mode 100644 index 0000000000..15c376fb9a --- /dev/null +++ b/changes/task19035 @@ -0,0 +1,5 @@ + o Removed features: + - Remove support for "GET /tor/bytes.txt" DirPort request, and + "GETINFO dir-usage" controller request, which were only available + via a compile-time option in Tor anyway. Feature was added in + 0.2.2.1-alpha. Resolves ticket 19035. diff --git a/changes/test_zlib_bombs b/changes/test_zlib_bombs new file mode 100644 index 0000000000..26121ed8ce --- /dev/null +++ b/changes/test_zlib_bombs @@ -0,0 +1,3 @@ + o Testing: + - We now have unit tests for our code to reject zlib "compression bombs". + (Fortunately, the code works fine.) diff --git a/changes/ticket16698 b/changes/ticket16698 new file mode 100644 index 0000000000..5057050c16 --- /dev/null +++ b/changes/ticket16698 @@ -0,0 +1,3 @@ + o Code simplification and refactoring: + - Split the 600-line directory_handle_command_get function into + separate functions for different URL types. Closes ticket 16698. diff --git a/changes/ticket18462 b/changes/ticket18462 new file mode 100644 index 0000000000..04e7e60e0b --- /dev/null +++ b/changes/ticket18462 @@ -0,0 +1,3 @@ + o Code simplification and refactoring: + - Rename tor_dup_addr() to tor_addr_to_str_dup() to avoid confusion. + Closes ticket #18462; patch from "icanhasaccount". diff --git a/changes/timeouts b/changes/timeouts new file mode 100644 index 0000000000..dc8f724974 --- /dev/null +++ b/changes/timeouts @@ -0,0 +1,7 @@ + o Minor features (infrastructure): + - Tor now includes an improved timer backend, so that we can efficiently + support tens or hundreds of thousands of concurrent timers, as will be + needed for some of our planned anti-traffic-analysis work. This code + is based on William Ahern's "timeout.c" project, which implements + a "tickless hierarchical timing wheel". Closes ticket #18365. + diff --git a/changes/workqueue_tests b/changes/workqueue_tests new file mode 100644 index 0000000000..7c5d756f76 --- /dev/null +++ b/changes/workqueue_tests @@ -0,0 +1,4 @@ + o Testing: + - Run more workqueue tests as part of "make check". These had previously + been implemented, but you needed to know special command-line options + to enable them. diff --git a/changes/zlib_12 b/changes/zlib_12 new file mode 100644 index 0000000000..3344286861 --- /dev/null +++ b/changes/zlib_12 @@ -0,0 +1,4 @@ + o New system requirements: + - We now require zlib version 1.2 or later. (Back when we started, + zlib 1.1 and zlib 1.0 were still found in the wild. 1.2 was released in + 2003. We recommend the latest version.) diff --git a/configure.ac b/configure.ac index 7471f99e16..6d721a2fc5 100644 --- a/configure.ac +++ b/configure.ac @@ -4,7 +4,7 @@ dnl Copyright (c) 2007-2015, The Tor Project, Inc. dnl See LICENSE for licensing information AC_PREREQ([2.63]) -AC_INIT([tor],[0.2.8.4-rc-dev]) +AC_INIT([tor],[0.2.9.0-alpha-dev]) AC_CONFIG_SRCDIR([src/or/main.c]) AC_CONFIG_MACRO_DIR([m4]) @@ -16,6 +16,7 @@ AM_INIT_AUTOMAKE([foreign 1.11 subdir-objects]) m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) AC_CONFIG_HEADERS([orconfig.h]) +AC_USE_SYSTEM_EXTENSIONS AC_CANONICAL_HOST PKG_PROG_PKG_CONFIG @@ -30,11 +31,8 @@ fi # the += operator on it in src/or/Makefile.am CPPFLAGS="$CPPFLAGS -I\${top_srcdir}/src/common" -#XXXX020 We should make these enabled or not, before 0.2.0.x-final AC_ARG_ENABLE(openbsd-malloc, AS_HELP_STRING(--enable-openbsd-malloc, [use malloc code from OpenBSD. Linux only])) -AC_ARG_ENABLE(instrument-downloads, - AS_HELP_STRING(--enable-instrument-downloads, [instrument downloads of directory resources etc.])) AC_ARG_ENABLE(static-openssl, AS_HELP_STRING(--enable-static-openssl, [link against a static openssl library. Requires --with-openssl-dir])) AC_ARG_ENABLE(static-libevent, @@ -73,10 +71,6 @@ if test "$enable_system_torrc" = "no"; then fi AM_CONDITIONAL(USE_OPENBSD_MALLOC, test "x$enable_openbsd_malloc" = "xyes") -if test "x$enable_instrument_downloads" = "xyes"; then - AC_DEFINE(INSTRUMENT_DOWNLOADS, 1, - [Defined if we want to keep track of how much of each kind of resource we download.]) -fi AC_ARG_ENABLE(transparent, AS_HELP_STRING(--disable-transparent, [disable transparent proxy support]), @@ -143,9 +137,11 @@ case "$host" in esac AC_ARG_ENABLE(gcc-warnings, - AS_HELP_STRING(--enable-gcc-warnings, [enable verbose warnings])) + AS_HELP_STRING(--enable-gcc-warnings, [deprecated alias for enable-fatal-warnings])) +AC_ARG_ENABLE(fatal-warnings, + AS_HELP_STRING(--enable-fatal-warnings, [tell the compiler to treat all warnings as errors.])) AC_ARG_ENABLE(gcc-warnings-advisory, - AS_HELP_STRING(--enable-gcc-warnings-advisory, [enable verbose warnings, excluding -Werror])) + AS_HELP_STRING(--disable-gcc-warnings-advisory, [disable the regular verbose warnings])) dnl Others suggest '/gs /safeseh /nxcompat /dynamicbase' for non-gcc on Windows AC_ARG_ENABLE(gcc-hardening, @@ -432,6 +428,7 @@ AC_CHECK_FUNCS( strtoull \ sysconf \ sysctl \ + truncate \ uname \ usleep \ vasprintf \ @@ -755,6 +752,11 @@ dnl use it with a build of a library. all_ldflags_for_check="$TOR_LDFLAGS_zlib $TOR_LDFLAGS_openssl $TOR_LDFLAGS_libevent" all_libs_for_check="$TOR_ZLIB_LIBS $TOR_LIB_MATH $TOR_LIBEVENT_LIBS $TOR_OPENSSL_LIBS $TOR_SYSTEMD_LIBS $TOR_LIB_WS32 $TOR_LIB_GDI $TOR_CAP_LIBS" +CFLAGS_FTRAPV= +CFLAGS_FWRAPV= +CFLAGS_ASAN= +CFLAGS_UBSAN= + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([], [ #if !defined(__clang__) #error @@ -777,20 +779,88 @@ m4_ifdef([AS_VAR_IF],[ AS_VAR_POPDEF([can_link]) AS_VAR_POPDEF([can_compile]) TOR_CHECK_CFLAGS(-Wstack-protector) - TOR_CHECK_CFLAGS(-fwrapv) TOR_CHECK_CFLAGS(--param ssp-buffer-size=1) if test "$bwin32" = "false"; then TOR_CHECK_CFLAGS(-fPIE) TOR_CHECK_LDFLAGS(-pie, "$all_ldflags_for_check", "$all_libs_for_check") fi + TOR_TRY_COMPILE_WITH_CFLAGS(-ftrapv, also_link, CFLAGS_FTRAPV="-ftrapv", true) + TOR_TRY_COMPILE_WITH_CFLAGS(-fwrapv, also_link, CFLAGS_FWRAPV="-fwrapv", true) + if test "$tor_cv_cflags__ftrapv" = "yes" && test "$tor_can_link__ftrapv" != "yes"; then + AC_MSG_WARN([The compiler supports -ftrapv, but for some reason I was not able to link with -ftrapv. Are you missing run-time support? Run-time hardening will not work as well as it should.]) + fi fi if test "x$enable_expensive_hardening" = "xyes"; then - TOR_CHECK_CFLAGS([-fsanitize=address]) - TOR_CHECK_CFLAGS([-fsanitize=undefined]) - TOR_CHECK_CFLAGS([-fno-omit-frame-pointer]) + if test "$tor_cv_cflags__ftrapv" != "yes"; then + AC_MSG_ERROR([You requested expensive hardening, but the compiler does not seem to support -ftrapv.]) + fi + + TOR_TRY_COMPILE_WITH_CFLAGS([-fsanitize=address], also_link, CFLAGS_ASAN="-fsanitize=address", true) + if test "$tor_cv_cflags__fsanitize_address" = "yes" && test "$tor_can_link__fsanitize_address" != "yes"; then + AC_MSG_ERROR([The compiler supports -fsanitize=address, but for some reason I was not able to link when using it. Are you missing run-time support? With GCC you need libubsan.so, and with Clang you need libclang_rt.ubsan*]) + fi + + TOR_TRY_COMPILE_WITH_CFLAGS([-fsanitize=undefined], also_link, CFLAGS_UBSAN="-fsanitize=undefined", true) + if test "$tor_cv_cflags__fsanitize_address" = "yes" && test "$tor_can_link__fsanitize_address" != "yes"; then + AC_MSG_ERROR([The compiler supports -fsanitize=undefined, but for some reason I was not able to link when using it. Are you missing run-time support? With GCC you need libasan.so, and with Clang you need libclang_rt.ubsan*]) + fi + +TOR_CHECK_CFLAGS([-fno-omit-frame-pointer]) +fi + +CFLAGS_BUGTRAP="$CFLAGS_FTRAPV $CFLAGS_ASAN $CFLAGS_UBSAN" +CFLAGS_CONSTTIME="$CFLAGS_FWRAPV" + +mulodi_fixes_ftrapv=no +if test "$have_clang" = "yes"; then + saved_CFLAGS="$CFLAGS" + CFLAGS="$CFLAGS $CFLAGS_FTRAPV" + AC_MSG_CHECKING([whether clang -ftrapv can link a 64-bit int multiply]) + AC_LINK_IFELSE([ + AC_LANG_SOURCE([[ + #include <stdint.h> + #include <stdlib.h> + int main(int argc, char **argv) + { + int64_t x = ((int64_t)atoi(argv[1])) * (int64_t)atoi(argv[2]) + * (int64_t)atoi(argv[3]); + return x == 9; + } ]])], + [ftrapv_can_link=yes; AC_MSG_RESULT([yes])], + [ftrapv_can_link=no; AC_MSG_RESULT([no])]) + if test "$ftrapv_can_link" = "no"; then + AC_MSG_CHECKING([whether defining __mulodi4 fixes that]) + AC_LINK_IFELSE([ + AC_LANG_SOURCE([[ + #include <stdint.h> + #include <stdlib.h> + int64_t __mulodi4(int64_t a, int64_t b, int *overflow) { + *overflow=0; + return a; + } + int main(int argc, char **argv) + { + int64_t x = ((int64_t)atoi(argv[1])) * (int64_t)atoi(argv[2]) + * (int64_t)atoi(argv[3]); + return x == 9; + } ]])], + [mulodi_fixes_ftrapv=yes; AC_MSG_RESULT([yes])], + [mulodi_fixes_ftrapv=no; AC_MSG_RESULT([no])]) + fi + CFLAGS="$saved_CFLAGS" fi +AM_CONDITIONAL(ADD_MULODI4, test "$mulodi_fixes_ftrapv" = "yes") + +dnl These cflags add bunches of branches, and we haven't been able to +dnl persuade ourselves that they're suitable for code that needs to be +dnl constant time. +AC_SUBST(CFLAGS_BUGTRAP) +dnl These cflags are variant ones sutable for code that needs to be +dnl constant-time. +AC_SUBST(CFLAGS_CONSTTIME) + if test "x$enable_linker_hardening" != "xno"; then TOR_CHECK_LDFLAGS(-z relro -z now, "$all_ldflags_for_check", "$all_libs_for_check") fi @@ -943,64 +1013,57 @@ AC_SUBST(CURVE25519_LIBS) dnl Make sure to enable support for large off_t if available. AC_SYS_LARGEFILE -AC_CHECK_HEADERS( - assert.h \ - errno.h \ - fcntl.h \ - signal.h \ - string.h \ - sys/capability.h \ - sys/fcntl.h \ - sys/stat.h \ - sys/time.h \ - sys/types.h \ - time.h \ - unistd.h - , , AC_MSG_WARN(Some headers were not found, compilation may fail. If compilation succeeds, please send your orconfig.h to the developers so we can fix this warning.)) - -dnl These headers are not essential - -AC_CHECK_HEADERS( - arpa/inet.h \ - crt_externs.h \ - execinfo.h \ - grp.h \ - ifaddrs.h \ - inttypes.h \ - limits.h \ - linux/types.h \ - machine/limits.h \ - malloc.h \ - malloc/malloc.h \ - malloc_np.h \ - netdb.h \ - netinet/in.h \ - netinet/in6.h \ - pwd.h \ - readpassphrase.h \ - stdint.h \ - sys/eventfd.h \ - sys/file.h \ - sys/ioctl.h \ - sys/limits.h \ - sys/mman.h \ - sys/param.h \ - sys/prctl.h \ - sys/resource.h \ - sys/select.h \ - sys/socket.h \ - sys/statvfs.h \ - sys/syscall.h \ - sys/sysctl.h \ - sys/syslimits.h \ - sys/time.h \ - sys/types.h \ - sys/un.h \ - sys/utime.h \ - sys/wait.h \ - syslog.h \ - utime.h -) +AC_CHECK_HEADERS([assert.h \ + errno.h \ + fcntl.h \ + signal.h \ + string.h \ + sys/capability.h \ + sys/fcntl.h \ + sys/stat.h \ + sys/time.h \ + sys/types.h \ + time.h \ + unistd.h \ + arpa/inet.h \ + crt_externs.h \ + execinfo.h \ + grp.h \ + ifaddrs.h \ + inttypes.h \ + limits.h \ + linux/types.h \ + machine/limits.h \ + malloc.h \ + malloc/malloc.h \ + malloc_np.h \ + netdb.h \ + netinet/in.h \ + netinet/in6.h \ + pwd.h \ + readpassphrase.h \ + stdint.h \ + sys/eventfd.h \ + sys/file.h \ + sys/ioctl.h \ + sys/limits.h \ + sys/mman.h \ + sys/param.h \ + sys/prctl.h \ + sys/resource.h \ + sys/select.h \ + sys/socket.h \ + sys/statvfs.h \ + sys/syscall.h \ + sys/sysctl.h \ + sys/syslimits.h \ + sys/time.h \ + sys/types.h \ + sys/un.h \ + sys/utime.h \ + sys/wait.h \ + syslog.h \ + utime.h]) AC_CHECK_HEADERS(sys/param.h) @@ -1490,9 +1553,6 @@ int main(int c, char **v) { puts(__FUNCTION__); }])], AC_CACHE_CHECK([whether we have extern char **environ already declared], tor_cv_have_environ_declared, AC_COMPILE_IFELSE([AC_LANG_SOURCE([ -/* We define _GNU_SOURCE here because it is also defined in compat.c. - * Without it environ doesn't get declared. */ -#define _GNU_SOURCE #ifdef HAVE_UNISTD_H #include <unistd.h> #endif @@ -1577,6 +1637,12 @@ else enable_gcc_warnings_advisory=no fi +# Warnings implies advisory-warnings and -Werror. +if test "$enable_gcc_warnings" = "yes"; then + enable_gcc_warnings_advisory=yes + enable_fatal_warnings=yes +fi + # OS X Lion started deprecating the system openssl. Let's just disable # all deprecation warnings on OS X. Also, to potentially make the binary # a little smaller, let's enable dead_strip. @@ -1589,35 +1655,11 @@ esac # Add some more warnings which we use in development but not in the # released versions. (Some relevant gcc versions can't handle these.) -if test "x$enable_gcc_warnings" = "xyes" || - test "x$enable_gcc_warnings_advisory" = "xyes"; then - - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([], [ -#if !defined(__GNUC__) || (__GNUC__ < 4) -#error -#endif])], have_gcc4=yes, have_gcc4=no) - - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([], [ -#if !defined(__GNUC__) || (__GNUC__ < 4) || (__GNUC__ == 4 && __GNUC_MINOR__ < 2) -#error -#endif])], have_gcc42=yes, have_gcc42=no) - - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([], [ -#if !defined(__GNUC__) || (__GNUC__ < 4) || (__GNUC__ == 4 && __GNUC_MINOR__ < 3) -#error -#endif])], have_gcc43=yes, have_gcc43=no) - - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([], [ -#if !defined(__GNUC__) || (__GNUC__ < 4) || (__GNUC__ == 4 && __GNUC_MINOR__ < 6) -#error -#endif])], have_gcc46=yes, have_gcc46=no) - - - save_CFLAGS="$CFLAGS" - CFLAGS="$CFLAGS -Wshorten-64-to-32" - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([], [])], have_shorten64_flag=yes, - have_shorten64_flag=no) - CFLAGS="$save_CFLAGS" +# +# Note that we have to do this near the end of the autoconf process, or +# else we may run into problems when these warnings hit on the testing C +# programs that autoconf wants to build. +if test "x$enable_gcc_warnings_advisory" != "xno"; then case "$host" in *-*-openbsd* | *-*-bitrig*) @@ -1628,56 +1670,252 @@ if test "x$enable_gcc_warnings" = "xyes" || CFLAGS="$CFLAGS -Wno-system-headers" ;; esac + # GCC4.3 users once report trouble with -Wstrict-overflow=5. GCC5 users + # have it work better. + # CFLAGS="$CFLAGS -Wstrict-overflow=1" + + # This warning was added in gcc 4.3, but it appears to generate + # spurious warnings in gcc 4.4. I don't know if it works in 4.5. + #CFLAGS="$CFLAGS -Wlogical-op" + + m4_foreach_w([warning_flag], [ + -Waddress + -Waddress-of-array-temporary + -Waddress-of-temporary + -Wambiguous-macro + -Wanonymous-pack-parens + -Warc + -Warc-abi + -Warc-bridge-casts-disallowed-in-nonarc + -Warc-maybe-repeated-use-of-weak + -Warc-performSelector-leaks + -Warc-repeated-use-of-weak + -Warray-bounds + -Warray-bounds-pointer-arithmetic + -Wasm + -Wasm-operand-widths + -Watomic-properties + -Watomic-property-with-user-defined-accessor + -Wauto-import + -Wauto-storage-class + -Wauto-var-id + -Wavailability + -Wbackslash-newline-escape + -Wbad-array-new-length + -Wbind-to-temporary-copy + -Wbitfield-constant-conversion + -Wbool-conversion + -Wbool-conversions + -Wbuiltin-requires-header + -Wchar-align + -Wcompare-distinct-pointer-types + -Wcomplex-component-init + -Wconditional-type-mismatch + -Wconfig-macros + -Wconstant-conversion + -Wconstant-logical-operand + -Wconstexpr-not-const + -Wcustom-atomic-properties + -Wdangling-field + -Wdangling-initializer-list + -Wdate-time + -Wdelegating-ctor-cycles + -Wdeprecated-implementations + -Wdeprecated-register + -Wdirect-ivar-access + -Wdiscard-qual + -Wdistributed-object-modifiers + -Wdivision-by-zero + -Wdollar-in-identifier-extension + -Wdouble-promotion + -Wduplicate-decl-specifier + -Wduplicate-enum + -Wduplicate-method-arg + -Wduplicate-method-match + -Wduplicated-cond + -Wdynamic-class-memaccess + -Wembedded-directive + -Wempty-translation-unit + -Wenum-conversion + -Wexit-time-destructors + -Wexplicit-ownership-type + -Wextern-initializer + -Wextra + -Wextra-semi + -Wextra-tokens + -Wflexible-array-extensions + -Wfloat-conversion + -Wformat-non-iso + -Wfour-char-constants + -Wgcc-compat + -Wglobal-constructors + -Wgnu-array-member-paren-init + -Wgnu-designator + -Wgnu-static-float-init + -Wheader-guard + -Wheader-hygiene + -Widiomatic-parentheses + -Wignored-attributes + -Wimplicit-atomic-properties + -Wimplicit-conversion-floating-point-to-bool + -Wimplicit-exception-spec-mismatch + -Wimplicit-fallthrough + -Wimplicit-fallthrough-per-function + -Wimplicit-retain-self + -Wimport-preprocessor-directive-pedantic + -Wincompatible-library-redeclaration + -Wincompatible-pointer-types-discards-qualifiers + -Wincomplete-implementation + -Wincomplete-module + -Wincomplete-umbrella + -Winit-self + -Wint-conversions + -Wint-to-void-pointer-cast + -Winteger-overflow + -Winvalid-constexpr + -Winvalid-iboutlet + -Winvalid-noreturn + -Winvalid-pp-token + -Winvalid-source-encoding + -Winvalid-token-paste + -Wknr-promoted-parameter + -Wlanguage-extension-token + -Wlarge-by-value-copy + -Wliteral-conversion + -Wliteral-range + -Wlocal-type-template-args + -Wlogical-op + -Wloop-analysis + -Wmain-return-type + -Wmalformed-warning-check + -Wmethod-signatures + -Wmicrosoft + -Wmicrosoft-exists + -Wmismatched-parameter-types + -Wmismatched-return-types + -Wmissing-field-initializers + -Wmissing-format-attribute + -Wmissing-noreturn + -Wmissing-selector-name + -Wmissing-sysroot + -Wmissing-variable-declarations + -Wmodule-conflict + -Wnested-anon-types + -Wnewline-eof + -Wnon-literal-null-conversion + -Wnon-pod-varargs + -Wnonportable-cfstrings + -Wnormalized=id + -Wnull-arithmetic + -Wnull-character + -Wnull-conversion + -Wnull-dereference + -Wout-of-line-declaration + -Wover-aligned + -Woverlength-strings + -Woverride-init + -Woverriding-method-mismatch + -Wpointer-type-mismatch + -Wpredefined-identifier-outside-function + -Wprotocol-property-synthesis-ambiguity + -Wreadonly-iboutlet-property + -Wreadonly-setter-attrs + -Wreceiver-expr + -Wreceiver-forward-class + -Wreceiver-is-weak + -Wreinterpret-base-class + -Wrequires-super-attribute + -Wreserved-user-defined-literal + -Wreturn-stack-address + -Wsection + -Wselector-type-mismatch + -Wsentinel + -Wserialized-diagnostics + -Wshift-count-negative + -Wshift-count-overflow + -Wshift-negative-value + -Wshift-overflow=2 + -Wshift-sign-overflow + -Wshorten-64-to-32 + -Wsizeof-array-argument + -Wsource-uses-openmp + -Wstatic-float-init + -Wstatic-in-inline + -Wstatic-local-in-inline + -Wstrict-overflow=2 + -Wstring-compare + -Wstring-conversion + -Wstrlcpy-strlcat-size + -Wstrncat-size + -Wsuggest-attribute=format + -Wsuggest-attribute=noreturn + -Wsuper-class-method-mismatch + -Wswitch-bool + -Wsync-nand + -Wtautological-constant-out-of-range-compare + -Wtentative-definition-incomplete-type + -Wthread-safety + -Wthread-safety-analysis + -Wthread-safety-attributes + -Wthread-safety-beta + -Wthread-safety-precise + -Wtrampolines + -Wtype-safety + -Wtypedef-redefinition + -Wtypename-missing + -Wundefined-inline + -Wundefined-internal + -Wundefined-reinterpret-cast + -Wunicode + -Wunicode-whitespace + -Wunknown-warning-option + -Wunnamed-type-template-args + -Wunneeded-member-function + -Wunsequenced + -Wunsupported-visibility + -Wunused-but-set-parameter + -Wunused-but-set-variable + -Wunused-command-line-argument + -Wunused-const-variable=2 + -Wunused-exception-parameter + -Wunused-local-typedefs + -Wunused-member-function + -Wunused-sanitize-argument + -Wunused-volatile-lvalue + -Wuser-defined-literals + -Wvariadic-macros + -Wvector-conversion + -Wvector-conversions + -Wvexing-parse + -Wvisibility + -Wvla-extension + -Wzero-length-array + ], [ TOR_CHECK_CFLAGS([warning_flag]) ]) + CFLAGS="$CFLAGS -W -Wfloat-equal -Wundef -Wpointer-arith" CFLAGS="$CFLAGS -Wstrict-prototypes -Wmissing-prototypes -Wwrite-strings" CFLAGS="$CFLAGS -Wredundant-decls -Wchar-subscripts -Wcomment -Wformat=2" - CFLAGS="$CFLAGS -Wwrite-strings -Wmissing-declarations -Wredundant-decls" + CFLAGS="$CFLAGS -Wwrite-strings" CFLAGS="$CFLAGS -Wnested-externs -Wbad-function-cast -Wswitch-enum" - - if test "x$enable_gcc_warnings" = "xyes"; then - CFLAGS="$CFLAGS -Werror" + CFLAGS="$CFLAGS -Waggregate-return -Wpacked -Wunused" + CFLAGS="$CFLAGS -Wunused-parameter " + # These interfere with building main() { return 0; }, which autoconf + # likes to use as its default program. + CFLAGS="$CFLAGS -Wold-style-definition -Wmissing-declarations" + + if test "$tor_cv_cflags__Wnull_dereference" = "yes"; then + AC_DEFINE([HAVE_CFLAG_WNULL_DEREFERENCE], 1, [True if we have -Wnull-dereference]) fi - - # Disabled, so we can use mallinfo(): -Waggregate-return - - if test "x$have_gcc4" = "xyes"; then - # These warnings break gcc 3.3.5 and work on gcc 4.0.2 - CFLAGS="$CFLAGS -Winit-self -Wmissing-field-initializers -Wold-style-definition" + if test "$tor_cv_cflags__Woverlength_strings" = "yes"; then + AC_DEFINE([HAVE_CFLAG_WOVERLENGTH_STRINGS], 1, [True if we have -Woverlength-strings]) fi - if test "x$have_gcc42" = "xyes"; then - # These warnings break gcc 4.0.2 and work on gcc 4.2 - # XXXX020 See if any of these work with earlier versions. - CFLAGS="$CFLAGS -Waddress -Wmissing-noreturn -Wstrict-overflow=1" - - # We used to use -Wstrict-overflow=5, but that breaks us heavily under 4.3. - fi - - if test "x$have_gcc42" = "xyes" && test "x$have_clang" = "xno"; then - # These warnings break gcc 4.0.2 and clang, but work on gcc 4.2 - CFLAGS="$CFLAGS -Wnormalized=id -Woverride-init" - fi - - if test "x$have_gcc43" = "xyes"; then - # These warnings break gcc 4.2 and work on gcc 4.3 - # XXXX020 See if any of these work with earlier versions. - CFLAGS="$CFLAGS -Wextra -Warray-bounds" - fi - - if test "x$have_gcc46" = "xyes"; then - # This warning was added in gcc 4.3, but it appears to generate - # spurious warnings in gcc 4.4. I don't know if it works in 4.5. - CFLAGS="$CFLAGS -Wlogical-op" - fi - - if test "x$have_shorten64_flag" = "xyes"; then - CFLAGS="$CFLAGS -Wshorten-64-to-32" + if test "x$enable_fatal_warnings" = "xyes"; then + # I'd like to use TOR_CHECK_CFLAGS here, but I can't, since the + # default autoconf programs are full of errors. + CFLAGS="$CFLAGS -Werror" fi - - -##This will break the world on some 64-bit architectures -# CFLAGS="$CFLAGS -Winline" fi if test "$enable_coverage" = "yes" && test "$have_clang" = "no"; then diff --git a/contrib/clang/sanitize_blacklist.txt b/contrib/clang/sanitize_blacklist.txt index 03d1e70f31..c71cc4d878 100644 --- a/contrib/clang/sanitize_blacklist.txt +++ b/contrib/clang/sanitize_blacklist.txt @@ -2,6 +2,17 @@ # syntax specified in http://clang.llvm.org/docs/SanitizerSpecialCaseList.html # for more info see http://clang.llvm.org/docs/AddressSanitizer.html +# +# Tor notes: This file is obsolete! +# +# It was necessary in order to apply the sanitizers to all of tor. But +# we don't believe that's a good idea: some parts of tor need constant-time +# behavior that is hard to guarantee with these sanitizers. +# +# If you need this behavior, then please consider --enable-expensive-hardening, +# and report bugs as needed. +# + # usage: # 1. configure tor build: # ./configure \ diff --git a/contrib/win32build/tor-mingw.nsi.in b/contrib/win32build/tor-mingw.nsi.in index 7949c79aed..47fb7e0cd2 100644 --- a/contrib/win32build/tor-mingw.nsi.in +++ b/contrib/win32build/tor-mingw.nsi.in @@ -8,7 +8,7 @@ !include "LogicLib.nsh" !include "FileFunc.nsh" !insertmacro GetParameters -!define VERSION "0.2.8.4-rc-dev" +!define VERSION "0.2.9.0-alpha-dev" !define INSTALLER "tor-${VERSION}-win32.exe" !define WEBSITE "https://www.torproject.org/" !define LICENSE "LICENSE" diff --git a/doc/HACKING/CodingStandards.md b/doc/HACKING/CodingStandards.md index 4aafa5ddd4..f1c65850a4 100644 --- a/doc/HACKING/CodingStandards.md +++ b/doc/HACKING/CodingStandards.md @@ -3,7 +3,7 @@ Coding conventions for Tor tl;dr: - - Run configure with `--enable-gcc-warnings` + - Run configure with `--enable-fatal-warnings` - Run `make check-spaces` to catch whitespace errors - Document your functions - Write unit tests @@ -21,7 +21,7 @@ preference) Did you remember... - - To build your code while configured with `--enable-gcc-warnings`? + - To build your code while configured with `--enable-fatal-warnings`? - To run `make check-spaces` on your code? - To run `make check-docs` to see whether all new options are on the manpage? @@ -125,10 +125,10 @@ deviations from our C whitespace style. Generally, we use: `puts (x)`. - Function declarations at the start of the line. -We try hard to build without warnings everywhere. In particular, if you're -using gcc, you should invoke the configure script with the option -`--enable-gcc-warnings`. This will give a bunch of extra warning flags to -the compiler, and help us find divergences from our preferred C style. +We try hard to build without warnings everywhere. In particular, if +you're using gcc, you should invoke the configure script with the +option `--enable-fatal-warnings`. This will tell the compiler +to make all warnings into errors. Functions to use; functions not to use -------------------------------------- diff --git a/doc/HACKING/HowToReview.md b/doc/HACKING/HowToReview.md index de7891c923..d53318942f 100644 --- a/doc/HACKING/HowToReview.md +++ b/doc/HACKING/HowToReview.md @@ -15,7 +15,7 @@ Top-level smell-checks (Difficulty: easy) -- Does it compile with `--enable-gcc-warnings`? +- Does it compile with `--enable-fatal-warnings`? - Does `make check-spaces` pass? diff --git a/doc/HACKING/ReleasingTor.md b/doc/HACKING/ReleasingTor.md index 2378aef568..8f5a47d827 100644 --- a/doc/HACKING/ReleasingTor.md +++ b/doc/HACKING/ReleasingTor.md @@ -4,13 +4,42 @@ Putting out a new release Here are the steps Roger takes when putting out a new Tor release: +=== 0. Preliminaries + +1. Get at least three of weasel/arma/Sebastian/Sina to put the new + version number in their approved versions list. + + +=== I. Make sure it works + 1. Use it for a while, as a client, as a relay, as a hidden service, and as a directory authority. See if it has any obvious bugs, and resolve those. As applicable, merge the `maint-X` branch into the `release-X` branch. -2. Gather the `changes/*` files into a changelog entry, rewriting many +2. Are all of the jenkins builders happy? See jenkins.torproject.org. + + What about the bsd buildbots? + See http://buildbot.pixelminers.net/builders/ + + What about Coverity Scan? + + Is make check-spaces happy? + + Does 'make distcheck' compain? + + How about 'make test-stem' and 'make test-network'? + + - Are all those tests still happy with --enable-expensive-hardening ? + + Any memory leaks? + + +=== II. Write a changelog. + + +1. Gather the `changes/*` files into a changelog entry, rewriting many of them and reordering to focus on what users and funders would find interesting and understandable. @@ -62,13 +91,13 @@ Here are the steps Roger takes when putting out a new Tor release: 7. Run `./scripts/maint/format_changelog.py` to make it prettier. -3. Compose a short release blurb to highlight the user-facing +2. Compose a short release blurb to highlight the user-facing changes. Insert said release blurb into the ChangeLog stanza. If it's a stable release, add it to the ReleaseNotes file too. If we're adding to a release-0.2.x branch, manually commit the changelogs to the later git branches too. - If you're doing the first stable release in a series, you need to +3. If you're doing the first stable release in a series, you need to create a ReleaseNotes for the series as a whole. To get started there, copy all of the Changelog entries from the series into a new file, and run `./scripts/maint/sortChanges.py` on it. That will @@ -78,7 +107,10 @@ Here are the steps Roger takes when putting out a new Tor release: to start sorting and condensing entries. (Generally, we don't edit the text of existing entries, though.) -4. In `maint-0.2.x`, bump the version number in `configure.ac` and run + +=== III. Making the source release. + +1. In `maint-0.2.x`, bump the version number in `configure.ac` and run `scripts/maint/updateVersions.pl` to update version numbers in other places, and commit. Then merge `maint-0.2.x` into `release-0.2.x`. @@ -86,20 +118,19 @@ Here are the steps Roger takes when putting out a new Tor release: either `make`, or `perl scripts/maint/updateVersions.pl`, depending on your version.) -5. Make distcheck, put the tarball up somewhere, and tell `#tor` about +2. Make distcheck, put the tarball up somewhere, and tell `#tor` about it. Wait a while to see if anybody has problems building it. Try to get Sebastian or somebody to try building it on Windows. -6. Get at least two of weasel/arma/Sebastian to put the new version number - in their approved versions list. +=== IV. Commit, upload, announce -7. Sign the tarball, then sign and push the git tag: +1. Sign the tarball, then sign and push the git tag: gpg -ba <the_tarball> git tag -u <keyid> tor-0.2.x.y-status git push origin tag tor-0.2.x.y-status -8. scp the tarball and its sig to the dist website, i.e. +2. scp the tarball and its sig to the dist website, i.e. `/srv/dist-master.torproject.org/htdocs/` on dist-master. When you want it to go live, you run "static-update-component dist.torproject.org" on dist-master. @@ -110,34 +141,38 @@ Here are the steps Roger takes when putting out a new Tor release: once. Nonetheless, do not call your version "alpha" if it is stable, or people will get confused.) -9. Email the packagers (cc'ing tor-assistants) that a new tarball is up. +3. Email the packagers (cc'ing tor-assistants) that a new tarball is up. The current list of packagers is: - {weasel,gk,mikeperry} at torproject dot org - {blueness} at gentoo dot org - {paul} at invizbox dot io - - {ondrej.mikle} at gmail dot com - {lfleischer} at archlinux dot org - {tails-dev} at boum dot org -10. Add the version number to Trac. To do this, go to Trac, log in, +4. Add the version number to Trac. To do this, go to Trac, log in, select "Admin" near the top of the screen, then select "Versions" from the menu on the left. At the right, there will be an "Add version" box. By convention, we enter the version in the form "Tor: 0.2.2.23-alpha" (or whatever the version is), and we select the date as the date in the ChangeLog. -11. Forward-port the ChangeLog (and ReleaseNotes if appropriate). - -12. Wait up to a day or two (for a development release), or until most +5. Wait up to a day or two (for a development release), or until most packages are up (for a stable release), and mail the release blurb and changelog to tor-talk or tor-announce. (We might be moving to faster announcements, but don't announce until the website is at least updated.) -13. If it's a stable release, bump the version number in the `maint-x.y.z` + +=== V. Aftermath and cleanup + +1. If it's a stable release, bump the version number in the `maint-x.y.z` branch to "newversion-dev", and do a `merge -s ours` merge to avoid taking that change into master. Do a similar `merge -s theirs` merge to get the change (and only that change) into release. (Some of the build scripts require that maint merge cleanly into release.) + +2. Forward-port the ChangeLog (and ReleaseNotes if appropriate). + + diff --git a/doc/HACKING/WritingTests.md b/doc/HACKING/WritingTests.md index 4e98d3d645..7bcadc6087 100644 --- a/doc/HACKING/WritingTests.md +++ b/doc/HACKING/WritingTests.md @@ -109,6 +109,19 @@ To count new or modified uncovered lines in D2, you can run: ./scripts/test/cov-diff ${D1} ${D2}" | grep '^+ *\#' | wc -l +### Marking lines as unreachable by tests + +You can mark a specific line as unreachable by using the special +string LCOV_EXCL_LINE. You can mark a range of lines as unreachable +with LCOV_EXCL_START... LCOV_EXCL_STOP. Note that older versions of +lcov don't understand these lines. + +You can post-process .gcov files to make these lines 'unreached' by +running ./scripts/test/cov-exclude on them. + +Note: you should never do this unless the line is meant to 100% +unreachable by actual code. + What kinds of test should I write? ---------------------------------- @@ -417,18 +430,50 @@ makefile exports them. Writing integration tests with Stem ----------------------------------- -The 'stem' library includes extensive unit tests for the Tor controller -protocol. - -For more information on writing new tests for stem, have a look around -the `test/*` directory in stem, and find a good example to emulate. You -might want to start with -`https://gitweb.torproject.org/stem.git/tree/test/integ/control/controller.py` -to improve Tor's test coverage. - +The 'stem' library includes extensive tests for the Tor controller protocol. You can run stem tests from tor with `make test-stem`, or see `https://stem.torproject.org/faq.html#how-do-i-run-the-tests`. +To see what tests are available, have a look around the `test/*` directory in +stem. The first thing you'll notice is that there are both `unit` and `integ` +tests. The former are for tests of the facilities provided by stem itself that +can be tested on their own, without the need to hook up a tor process. These +are less relevant, unless you want to develop a new stem feature. The latter, +however, are a very useful tool to write tests for controller features. They +provide a default environment with a connected tor instance that can be +modified and queried. Adding more integration tests is a great way to increase +the test coverage inside Tor, especially for controller features. + +Let's assume you actually want to write a test for a previously untested +controller feature. I'm picking the `exit-policy/*` GETINFO queries. Since +these are a controller feature that we want to write an integration test for, +the right file to modify is +`https://gitweb.torproject.org/stem.git/tree/test/integ/control/controller.py`. + +First off we notice that there is an integration test called +`test_get_exit_policy()` that's already written. This exercises the interaction +of stem's `Controller.get_exit_policy()` method, and is not relevant for our +test since there are no stem methods to make use of all `exit-policy/*` +queries (if there were, likely they'd be tested already. Maybe you want to +write a stem feature, but I chose to just add tests). + +Our test requires a tor controller connection, so we'll use the +`@require_controller` annotation for our `test_exit_policy()` method. We need a +controller instance, which we get from +`test.runner.get_runner().get_tor_controller()`. The attached Tor instance is +configured as a client, but the exit-policy GETINFO queries need a relay to +work, so we have to change the config (using `controller.set_options()`). This +is OK for us to do, we just have to remember to set DisableNetwork so we don't +actually start an exit relay and also to undo the changes we made (by calling +`controller.reset_conf()` at the end of our test). Additionally, we have to +configure a static Address for Tor to use, because it refuses to build a +descriptor when it can't guess a suitable IP address. Unfortunately, these +kinds of tripwires are everywhere. Don't forget to file appropriate tickets if +you notice any strange behaviour that seems totally unreasonable. + +Check out the `test_exit_policy()` function in abovementioned file to see the +final implementation for this test. + System testing with Chutney --------------------------- diff --git a/doc/tor.1.txt b/doc/tor.1.txt index 74915b7119..65d3c2d5f5 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -1426,7 +1426,7 @@ The following options are useful only for clients (that is, if **non-anonymously**. This option also disables client connections to non-hidden-service hostnames through Tor. It **must only** be used when running a tor2web Hidden Service web proxy. - To enable this option the compile time flag --enable-tor2webmode must be + To enable this option the compile time flag --enable-tor2web-mode must be specified. (Default: 0) [[Tor2webRendezvousPoints]] **Tor2webRendezvousPoints** __node__,__node__,__...__:: @@ -2103,8 +2103,7 @@ on the public Tor network. server. Instead of caching the directory, it generates its own list of good servers, signs it, and sends that to the clients. Unless the clients already have you listed as a trusted directory, you probably do not want - to set this option. Please coordinate with the other admins at - tor-ops@torproject.org if you think you should be a directory. + to set this option. [[V3AuthoritativeDirectory]] **V3AuthoritativeDirectory** **0**|**1**:: When this option is set in addition to **AuthoritativeDirectory**, Tor diff --git a/m4/pc_from_ucontext.m4 b/m4/pc_from_ucontext.m4 index 6bedcbb2da..8a9dc459e6 100644 --- a/m4/pc_from_ucontext.m4 +++ b/m4/pc_from_ucontext.m4 @@ -79,24 +79,21 @@ AC_DEFUN([AC_PC_FROM_UCONTEXT], if ! $pc_field_found; then # Prefer sys/ucontext.h to ucontext.h, for OS X's sake. if test "x$ac_cv_header_cygwin_signal_h" = xyes; then - AC_TRY_COMPILE([#define _GNU_SOURCE 1 - #include <cygwin/signal.h>], + AC_TRY_COMPILE([#include <cygwin/signal.h>], [ucontext_t u; return u.$pc_field == 0;], AC_DEFINE_UNQUOTED(PC_FROM_UCONTEXT, $pc_field, How to access the PC from a struct ucontext) AC_MSG_RESULT([$pc_field]) pc_field_found=true) elif test "x$ac_cv_header_sys_ucontext_h" = xyes; then - AC_TRY_COMPILE([#define _GNU_SOURCE 1 - #include <sys/ucontext.h>], + AC_TRY_COMPILE([#include <sys/ucontext.h>], [ucontext_t u; return u.$pc_field == 0;], AC_DEFINE_UNQUOTED(PC_FROM_UCONTEXT, $pc_field, How to access the PC from a struct ucontext) AC_MSG_RESULT([$pc_field]) pc_field_found=true) elif test "x$ac_cv_header_ucontext_h" = xyes; then - AC_TRY_COMPILE([#define _GNU_SOURCE 1 - #include <ucontext.h>], + AC_TRY_COMPILE([#include <ucontext.h>], [ucontext_t u; return u.$pc_field == 0;], AC_DEFINE_UNQUOTED(PC_FROM_UCONTEXT, $pc_field, How to access the PC from a struct ucontext) diff --git a/scripts/maint/format_changelog.py b/scripts/maint/format_changelog.py index 5e4c8cac9a..e909fc550a 100755 --- a/scripts/maint/format_changelog.py +++ b/scripts/maint/format_changelog.py @@ -398,16 +398,24 @@ class ChangeLog(object): self.dumpEndOfSections() self.dumpEndOfChangelog() +# Let's turn bugs to html. +BUG_PAT = re.compile('(bug|ticket|feature)\s+(\d{4,5})', re.I) +def bug_html(m): + return "%s <a href='https://bugs.torproject.org/%s'>%s</a>" % (m.group(1), m.group(2), m.group(2)) + class HTMLChangeLog(ChangeLog): def __init__(self, *args, **kwargs): ChangeLog.__init__(self, *args, **kwargs) def htmlText(self, graf): + output = [] for line in graf: line = line.rstrip().replace("&","&") line = line.rstrip().replace("<","<").replace(">",">") - sys.stdout.write(line.strip()) - sys.stdout.write(" ") + output.append(line.strip()) + output = " ".join(output) + output = BUG_PAT.sub(bug_html, output) + sys.stdout.write(output) def htmlPar(self, graf): sys.stdout.write("<p>") diff --git a/scripts/test/cov-diff b/scripts/test/cov-diff index 48dbec9d54..7da7f0be9d 100755 --- a/scripts/test/cov-diff +++ b/scripts/test/cov-diff @@ -9,8 +9,8 @@ DIRB="$2" for A in $DIRA/*; do B=$DIRB/`basename $A` - perl -pe 's/^\s*\d+:/ 1:/; s/^([^:]+:)[\d\s]+:/$1/; s/^ *-:(Runs|Programs):.*//;' "$A" > "$A.tmp" - perl -pe 's/^\s*\d+:/ 1:/; s/^([^:]+:)[\d\s]+:/$1/; s/^ *-:(Runs|Programs):.*//;' "$B" > "$B.tmp" + perl -pe 's/^\s*\!*\d+:/ 1:/; s/^([^:]+:)[\d\s]+:/$1/; s/^ *-:(Runs|Programs):.*//;' "$A" > "$A.tmp" + perl -pe 's/^\s*\!*\d+:/ 1:/; s/^([^:]+:)[\d\s]+:/$1/; s/^ *-:(Runs|Programs):.*//;' "$B" > "$B.tmp" diff -u "$A.tmp" "$B.tmp" rm "$A.tmp" "$B.tmp" done diff --git a/scripts/test/cov-exclude b/scripts/test/cov-exclude new file mode 100755 index 0000000000..5117f11ec4 --- /dev/null +++ b/scripts/test/cov-exclude @@ -0,0 +1,28 @@ +#!/usr/bin/perl -p -i + +use warnings; +use strict; +our $excluding; + +# This script is meant to post-process a .gcov file for an input source +# that was annotated with LCOV_EXCL_START, LCOV_EXCL_STOP, and LCOV_EXCL_LINE +# entries. It doesn't understand the LCOV_EXCL_BR* variations. +# +# It replaces unreached reached lines with x:, and reached excluded lines +# with !!!num:. + +BEGIN { our $excluding = 0; } + +if (m/LCOV_EXCL_START/) { + $excluding = 1; +} +if ($excluding and m/LCOV_EXCL_STOP/) { + $excluding = 0; +} + +my $exclude_this = (m/LCOV_EXCL_LINE/); + +if ($excluding or $exclude_this) { + s{^\s*\#\#+:}{ x:}; + s{^ (\s*)(\d+):}{$1!!!$2:}; +} diff --git a/src/common/address.c b/src/common/address.c index 793a40effc..5cd7a8df67 100644 --- a/src/common/address.c +++ b/src/common/address.c @@ -131,7 +131,8 @@ tor_addr_to_sockaddr(const tor_addr_t *a, #endif sin6->sin6_family = AF_INET6; sin6->sin6_port = htons(port); - memcpy(&sin6->sin6_addr, tor_addr_to_in6(a), sizeof(struct in6_addr)); + memcpy(&sin6->sin6_addr, tor_addr_to_in6_assert(a), + sizeof(struct in6_addr)); return sizeof(struct sockaddr_in6); } else { return 0; @@ -334,7 +335,7 @@ tor_addr_lookup(const char *name, uint16_t family, tor_addr_t *addr) } else if (ent->h_addrtype == AF_INET6) { tor_addr_from_in6(addr, (struct in6_addr*) ent->h_addr); } else { - tor_assert(0); /* gethostbyname() returned a bizarre addrtype */ + tor_assert(0); // LCOV_EXCL_LINE: gethostbyname() returned bizarre type } return 0; } @@ -905,8 +906,10 @@ tor_addr_is_loopback(const tor_addr_t *addr) case AF_UNSPEC: return 0; default: + /* LCOV_EXCL_START */ tor_fragile_assert(); return 0; + /* LCOV_EXCL_STOP */ } } @@ -1027,7 +1030,7 @@ tor_addr_copy_tight(tor_addr_t *dest, const tor_addr_t *src) case AF_UNSPEC: break; default: - tor_fragile_assert(); + tor_fragile_assert(); // LCOV_EXCL_LINE } } @@ -1096,6 +1099,7 @@ tor_addr_compare_masked(const tor_addr_t *addr1, const tor_addr_t *addr2, case AF_INET6: { if (mbits > 128) mbits = 128; + const uint8_t *a1 = tor_addr_to_in6_addr8(addr1); const uint8_t *a2 = tor_addr_to_in6_addr8(addr2); const int bytes = mbits >> 3; @@ -1111,8 +1115,10 @@ tor_addr_compare_masked(const tor_addr_t *addr1, const tor_addr_t *addr2, } } default: + /* LCOV_EXCL_START */ tor_fragile_assert(); return 0; + /* LCOV_EXCL_STOP */ } } else if (how == CMP_EXACT) { /* Unequal families and an exact comparison? Stop now! */ @@ -1165,14 +1171,16 @@ tor_addr_hash(const tor_addr_t *addr) case AF_INET6: return siphash24g(&addr->addr.in6_addr.s6_addr, 16); default: + /* LCOV_EXCL_START */ tor_fragile_assert(); return 0; + /* LCOV_EXCL_END */ } } /** Return a newly allocated string with a representation of <b>addr</b>. */ char * -tor_dup_addr(const tor_addr_t *addr) +tor_addr_to_str_dup(const tor_addr_t *addr) { char buf[TOR_ADDR_BUF_LEN]; if (tor_addr_to_str(buf, addr, sizeof(buf), 0)) { @@ -1809,7 +1817,7 @@ MOCK_IMPL(smartlist_t *,get_interface_address6_list,(int severity, /* ====== * IPv4 helpers - * XXXX024 IPv6 deprecate some of these. + * XXXX IPv6 deprecate some of these. */ /** Given an address of the form "ip:port", try to divide it into its diff --git a/src/common/address.h b/src/common/address.h index 53712bde02..51db42c315 100644 --- a/src/common/address.h +++ b/src/common/address.h @@ -74,6 +74,8 @@ typedef struct tor_addr_port_t #define TOR_ADDR_NULL {AF_UNSPEC, {0}} static inline const struct in6_addr *tor_addr_to_in6(const tor_addr_t *a); +static inline const struct in6_addr *tor_addr_to_in6_assert( + const tor_addr_t *a); static inline uint32_t tor_addr_to_ipv4n(const tor_addr_t *a); static inline uint32_t tor_addr_to_ipv4h(const tor_addr_t *a); static inline uint32_t tor_addr_to_mapped_ipv4h(const tor_addr_t *a); @@ -97,21 +99,31 @@ tor_addr_to_in6(const tor_addr_t *a) return a->family == AF_INET6 ? &a->addr.in6_addr : NULL; } +/** As tor_addr_to_in6, but assert that the address truly is an IPv6 + * address. */ +static inline const struct in6_addr * +tor_addr_to_in6_assert(const tor_addr_t *a) +{ + tor_assert(a->family == AF_INET6); + return &a->addr.in6_addr; +} + /** Given an IPv6 address <b>x</b>, yield it as an array of uint8_t. * * Requires that <b>x</b> is actually an IPv6 address. */ -#define tor_addr_to_in6_addr8(x) tor_addr_to_in6(x)->s6_addr +#define tor_addr_to_in6_addr8(x) tor_addr_to_in6_assert(x)->s6_addr + /** Given an IPv6 address <b>x</b>, yield it as an array of uint16_t. * * Requires that <b>x</b> is actually an IPv6 address. */ -#define tor_addr_to_in6_addr16(x) S6_ADDR16(*tor_addr_to_in6(x)) +#define tor_addr_to_in6_addr16(x) S6_ADDR16(*tor_addr_to_in6_assert(x)) /** Given an IPv6 address <b>x</b>, yield it as an array of uint32_t. * * Requires that <b>x</b> is actually an IPv6 address. */ -#define tor_addr_to_in6_addr32(x) S6_ADDR32(*tor_addr_to_in6(x)) +#define tor_addr_to_in6_addr32(x) S6_ADDR32(*tor_addr_to_in6_assert(x)) /** Return an IPv4 address in network order for <b>a</b>, or 0 if * <b>a</b> is not an IPv4 address. */ @@ -179,7 +191,7 @@ tor_addr_eq_ipv4h(const tor_addr_t *a, uint32_t u) #define TOR_ADDR_BUF_LEN 48 int tor_addr_lookup(const char *name, uint16_t family, tor_addr_t *addr_out); -char *tor_dup_addr(const tor_addr_t *addr) ATTR_MALLOC; +char *tor_addr_to_str_dup(const tor_addr_t *addr) ATTR_MALLOC; /** Wrapper function of fmt_addr_impl(). It does not decorate IPv6 * addresses. */ diff --git a/src/common/aes.c b/src/common/aes.c index 15970a73f0..2b8a68c4a2 100644 --- a/src/common/aes.c +++ b/src/common/aes.c @@ -23,18 +23,7 @@ #error "We require OpenSSL >= 1.0.0" #endif -#ifdef __GNUC__ -#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -#endif - -#if __GNUC__ && GCC_VERSION >= 402 -#if GCC_VERSION >= 406 -#pragma GCC diagnostic push -#endif -/* Some versions of OpenSSL declare SSL_get_selected_srtp_profile twice in - * srtp.h. Suppress the GCC warning so we can build with -Wredundant-decl. */ -#pragma GCC diagnostic ignored "-Wredundant-decls" -#endif +DISABLE_GCC_WARNING(redundant-decls) #include <assert.h> #include <stdlib.h> @@ -44,13 +33,7 @@ #include <openssl/engine.h> #include <openssl/modes.h> -#if __GNUC__ && GCC_VERSION >= 402 -#if GCC_VERSION >= 406 -#pragma GCC diagnostic pop -#else -#pragma GCC diagnostic warning "-Wredundant-decls" -#endif -#endif +ENABLE_GCC_WARNING(redundant-decls) #include "compat.h" #include "aes.h" @@ -255,9 +238,11 @@ evaluate_ctr_for_aes(void) if (fast_memneq(output, encrypt_zero, 16)) { /* Counter mode is buggy */ + /* LCOV_EXCL_START */ log_err(LD_CRYPTO, "This OpenSSL has a buggy version of counter mode; " "quitting tor."); exit(1); + /* LCOV_EXCL_STOP */ } return 0; } @@ -300,7 +285,7 @@ aes_set_key(aes_cnt_cipher_t *cipher, const char *key, int key_bits) case 128: c = EVP_aes_128_ecb(); break; case 192: c = EVP_aes_192_ecb(); break; case 256: c = EVP_aes_256_ecb(); break; - default: tor_assert(0); + default: tor_assert(0); // LCOV_EXCL_LINE } EVP_EncryptInit(&cipher->key.evp, c, (const unsigned char*)key, NULL); cipher->using_evp = 1; diff --git a/src/common/backtrace.c b/src/common/backtrace.c index 3b762b68e3..2841281927 100644 --- a/src/common/backtrace.c +++ b/src/common/backtrace.c @@ -13,9 +13,6 @@ * detect crashes. */ -#define __USE_GNU -#define _GNU_SOURCE 1 - #include "orconfig.h" #include "compat.h" #include "util.h" @@ -112,8 +109,10 @@ log_backtrace(int severity, int domain, const char *msg) tor_log(severity, domain, "%s. Stack trace:", msg); if (!symbols) { + /* LCOV_EXCL_START -- we can't provoke this. */ tor_log(severity, domain, " Unable to generate backtrace."); goto done; + /* LCOV_EXCL_STOP */ } for (i=0; i < depth; ++i) { tor_log(severity, domain, " %s", symbols[i]); @@ -176,8 +175,10 @@ install_bt_handler(void) for (i = 0; trap_signals[i] >= 0; ++i) { if (sigaction(trap_signals[i], &sa, NULL) == -1) { + /* LCOV_EXCL_START */ log_warn(LD_BUG, "Sigaction failed: %s", strerror(errno)); rv = -1; + /* LCOV_EXCL_STOP */ } } diff --git a/src/common/compat.c b/src/common/compat.c index 23eaa134cf..bae76f45db 100644 --- a/src/common/compat.c +++ b/src/common/compat.c @@ -12,17 +12,6 @@ * the platform. **/ -/* This is required on rh7 to make strptime not complain. - * We also need it to make memmem get defined (where available) - */ -/* XXXX024 We should just use AC_USE_SYSTEM_EXTENSIONS in our autoconf, - * and get this (and other important stuff!) automatically. Once we do that, - * make sure to also change the extern char **environ detection in - * configure.ac, because whether that is declared or not depends on whether - * we have _GNU_SOURCE defined! Maybe that means that once we take this out, - * we can also take out the configure check. */ -#define _GNU_SOURCE - #define COMPAT_PRIVATE #include "compat.h" @@ -525,8 +514,10 @@ tor_asprintf(char **strp, const char *fmt, ...) r = tor_vasprintf(strp, fmt, args); va_end(args); if (!*strp || r < 0) { + /* LCOV_EXCL_START */ log_err(LD_BUG, "Internal error in asprintf"); tor_assert(0); + /* LCOV_EXCL_STOP */ } return r; } @@ -1154,14 +1145,12 @@ tor_close_socket(tor_socket_t s) --n_sockets_open; #else if (r != EBADF) - --n_sockets_open; + --n_sockets_open; // LCOV_EXCL_LINE -- EIO and EINTR too hard to force. #endif r = -1; } - if (n_sockets_open < 0) - log_warn(LD_BUG, "Our socket count is below zero: %d. Please submit a " - "bug report.", n_sockets_open); + tor_assert_nonfatal(n_sockets_open >= 0); socket_accounting_unlock(); return r; } @@ -1941,7 +1930,7 @@ tor_getpwnam(const char *username) return NULL; if (! strcmp(username, passwd_cached->pw_name)) - return passwd_cached; + return passwd_cached; // LCOV_EXCL_LINE - would need to make getpwnam flaky return NULL; } @@ -1967,7 +1956,7 @@ tor_getpwuid(uid_t uid) return NULL; if (uid == passwd_cached->pw_uid) - return passwd_cached; + return passwd_cached; // LCOV_EXCL_LINE - would need to make getpwnam flaky return NULL; } @@ -2350,28 +2339,15 @@ get_parent_directory(char *fname) static char * alloc_getcwd(void) { - int saved_errno = errno; -/* We use this as a starting path length. Not too large seems sane. */ -#define START_PATH_LENGTH 128 -/* Nobody has a maxpath longer than this, as far as I know. And if they - * do, they shouldn't. */ -#define MAX_SANE_PATH_LENGTH 4096 - size_t path_length = START_PATH_LENGTH; - char *path = tor_malloc(path_length); - - errno = 0; - while (getcwd(path, path_length) == NULL) { - if (errno == ERANGE && path_length < MAX_SANE_PATH_LENGTH) { - path_length*=2; - path = tor_realloc(path, path_length); - } else { - tor_free(path); - path = NULL; - break; - } - } - errno = saved_errno; - return path; +#ifdef PATH_MAX +#define MAX_CWD PATH_MAX +#else +#define MAX_CWD 4096 +#endif + + char path_buf[MAX_CWD]; + char *path = getcwd(path_buf, sizeof(path_buf)); + return path ? tor_strdup(path) : NULL; } #endif @@ -2402,11 +2378,13 @@ make_path_absolute(char *fname) tor_asprintf(&absfname, "%s/%s", path, fname); tor_free(path); } else { + /* LCOV_EXCL_START Can't make getcwd fail. */ /* If getcwd failed, the best we can do here is keep using the * relative path. (Perhaps / isn't readable by this UID/GID.) */ log_warn(LD_GENERAL, "Unable to find current working directory: %s", strerror(errno)); absfname = tor_strdup(fname); + /* LCOV_EXCL_STOP */ } } return absfname; @@ -2770,7 +2748,9 @@ MOCK_IMPL(const char *, get_uname, (void)) } #endif #else + /* LCOV_EXCL_START -- can't provoke uname failure */ strlcpy(uname_result, "Unknown platform", sizeof(uname_result)); + /* LCOV_EXCL_STOP */ #endif } uname_result_is_set = 1; @@ -2844,11 +2824,14 @@ compute_num_cpus(void) if (num_cpus == -2) { num_cpus = compute_num_cpus_impl(); tor_assert(num_cpus != -2); - if (num_cpus > MAX_DETECTABLE_CPUS) + if (num_cpus > MAX_DETECTABLE_CPUS) { + /* LCOV_EXCL_START */ log_notice(LD_GENERAL, "Wow! I detected that you have %d CPUs. I " "will not autodetect any more than %d, though. If you " "want to configure more, set NumCPUs in your torrc", num_cpus, MAX_DETECTABLE_CPUS); + /* LCOV_EXCL_STOP */ + } } return num_cpus; } @@ -2873,18 +2856,22 @@ tor_gettimeofday(struct timeval *timeval) /* number of 100-nsec units since Jan 1, 1601 */ GetSystemTimeAsFileTime(&ft.ft_ft); if (ft.ft_64 < EPOCH_BIAS) { + /* LCOV_EXCL_START */ log_err(LD_GENERAL,"System time is before 1970; failing."); exit(1); + /* LCOV_EXCL_STOP */ } ft.ft_64 -= EPOCH_BIAS; timeval->tv_sec = (unsigned) (ft.ft_64 / UNITS_PER_SEC); timeval->tv_usec = (unsigned) ((ft.ft_64 / UNITS_PER_USEC) % USEC_PER_SEC); #elif defined(HAVE_GETTIMEOFDAY) if (gettimeofday(timeval, NULL)) { + /* LCOV_EXCL_START */ log_err(LD_GENERAL,"gettimeofday failed."); /* If gettimeofday dies, we have either given a bad timezone (we didn't), or segfaulted.*/ exit(1); + /* LCOV_EXCL_STOP */ } #elif defined(HAVE_FTIME) struct timeb tb; @@ -2975,11 +2962,12 @@ correct_tm(int islocal, const time_t *timep, struct tm *resultbuf, /* If we get here, then gmtime/localtime failed without getting an extreme * value for *timep */ - + /* LCOV_EXCL_START */ tor_fragile_assert(); r = resultbuf; memset(resultbuf, 0, sizeof(struct tm)); outcome="can't recover"; + /* LCOV_EXCL_STOP */ done: log_warn(LD_BUG, "%s("I64_FORMAT") failed with error %s: %s", islocal?"localtime":"gmtime", @@ -3373,9 +3361,11 @@ get_total_system_memory_impl(void) return result * 1024; err: + /* LCOV_EXCL_START Can't reach this unless proc is broken. */ tor_free(s); close(fd); return 0; + /* LCOV_EXCL_STOP */ #elif defined (_WIN32) /* Windows has MEMORYSTATUSEX; pretty straightforward. */ MEMORYSTATUSEX ms; @@ -3424,6 +3414,7 @@ get_total_system_memory(size_t *mem_out) static size_t mem_cached=0; uint64_t m = get_total_system_memory_impl(); if (0 == m) { + /* LCOV_EXCL_START -- can't make this happen without mocking. */ /* We couldn't find our memory total */ if (0 == mem_cached) { /* We have no cached value either */ @@ -3433,6 +3424,7 @@ get_total_system_memory(size_t *mem_out) *mem_out = mem_cached; return 0; + /* LCOV_EXCL_STOP */ } #if SIZE_MAX != UINT64_MAX diff --git a/src/common/compat.h b/src/common/compat.h index 8cf84580c6..6f102becc2 100644 --- a/src/common/compat.h +++ b/src/common/compat.h @@ -82,6 +82,44 @@ #define CHECK_SCANF(formatIdx, firstArg) #endif +/* What GCC do we have? */ +#ifdef __GNUC__ +#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) +#else +#define GCC_VERSION 0 +#endif + +/* Temporarily enable and disable warnings. */ +#ifdef __GNUC__ +# define PRAGMA_STRINGIFY_(s) #s +# define PRAGMA_JOIN_STRINGIFY_(a,b) PRAGMA_STRINGIFY_(a ## b) +/* Support for macro-generated pragmas (c99) */ +# define PRAGMA_(x) _Pragma (#x) +# ifdef __clang__ +# define PRAGMA_DIAGNOSTIC_(x) PRAGMA_(clang diagnostic x) +# else +# define PRAGMA_DIAGNOSTIC_(x) PRAGMA_(GCC diagnostic x) +# endif +# if defined(__clang__) || GCC_VERSION >= 406 +/* we have push/pop support */ +# define DISABLE_GCC_WARNING(warning) \ + PRAGMA_DIAGNOSTIC_(push) \ + PRAGMA_DIAGNOSTIC_(ignored PRAGMA_JOIN_STRINGIFY_(-W,warning)) +# define ENABLE_GCC_WARNING(warning) \ + PRAGMA_DIAGNOSTIC_(pop) +# else +/* older version of gcc: no push/pop support. */ +# define DISABLE_GCC_WARNING(warning) \ + PRAGMA_DIAGNOSTIC_(ignored PRAGMA_JOIN_STRINGIFY_(-W,warning)) +# define ENABLE_GCC_WARNING(warning) \ + PRAGMA_DIAGNOSTIC_(warning PRAGMA_JOIN_STRINGIFY_(-W,warning)) +# endif +#else /* ifdef __GNUC__ */ +/* not gcc at all */ +# define DISABLE_GCC_WARNING(warning) +# define ENABLE_GCC_WARNING(warning) +#endif + /* inline is __inline on windows. */ #ifdef _WIN32 #define inline __inline @@ -430,7 +468,7 @@ typedef int socklen_t; #ifdef _WIN32 /* XXX Actually, this should arguably be SOCKET; we use intptr_t here so that - * any inadvertant checks for the socket being <= 0 or > 0 will probably + * any inadvertent checks for the socket being <= 0 or > 0 will probably * still work. */ #define tor_socket_t intptr_t #define TOR_SOCKET_T_FORMAT INTPTR_T_FORMAT diff --git a/src/common/compat_libevent.c b/src/common/compat_libevent.c index cc58883750..4469f15ac7 100644 --- a/src/common/compat_libevent.c +++ b/src/common/compat_libevent.c @@ -125,7 +125,7 @@ tor_event_free(struct event *ev) #endif /** Global event base for use by the main thread. */ -struct event_base *the_event_base = NULL; +static struct event_base *the_event_base = NULL; /* This is what passes for version detection on OSX. We set * MACOSX_KQUEUE_IS_BROKEN to true iff we're on a version of OSX before @@ -228,8 +228,10 @@ tor_libevent_initialize(tor_libevent_cfg *torcfg) #endif if (!the_event_base) { + /* LCOV_EXCL_START */ log_err(LD_GENERAL, "Unable to initialize Libevent: cannot continue."); exit(1); + /* LCOV_EXCL_STOP */ } /* Making this a NOTICE for now so we can link bugs to a libevent versions diff --git a/src/common/compat_pthreads.c b/src/common/compat_pthreads.c index 1b24cc3c2a..79f5cec43a 100644 --- a/src/common/compat_pthreads.c +++ b/src/common/compat_pthreads.c @@ -10,8 +10,6 @@ * functions. */ -#define _GNU_SOURCE - #include "orconfig.h" #include <pthread.h> #include <signal.h> @@ -104,11 +102,13 @@ void tor_mutex_init(tor_mutex_t *mutex) { if (PREDICT_UNLIKELY(!threads_initialized)) - tor_threads_init(); + tor_threads_init(); // LCOV_EXCL_LINE const int err = pthread_mutex_init(&mutex->mutex, &attr_recursive); if (PREDICT_UNLIKELY(err)) { + // LCOV_EXCL_START log_err(LD_GENERAL, "Error %d creating a mutex.", err); - tor_fragile_assert(); + tor_assert_unreached(); + // LCOV_EXCL_STOP } } @@ -118,12 +118,14 @@ void tor_mutex_init_nonrecursive(tor_mutex_t *mutex) { int err; - if (PREDICT_UNLIKELY(!threads_initialized)) - tor_threads_init(); + if (!threads_initialized) + tor_threads_init(); // LCOV_EXCL_LINE err = pthread_mutex_init(&mutex->mutex, NULL); if (PREDICT_UNLIKELY(err)) { + // LCOV_EXCL_START log_err(LD_GENERAL, "Error %d creating a mutex.", err); - tor_fragile_assert(); + tor_assert_unreached(); + // LCOV_EXCL_STOP } } @@ -135,8 +137,10 @@ tor_mutex_acquire(tor_mutex_t *m) tor_assert(m); err = pthread_mutex_lock(&m->mutex); if (PREDICT_UNLIKELY(err)) { + // LCOV_EXCL_START log_err(LD_GENERAL, "Error %d locking a mutex.", err); - tor_fragile_assert(); + tor_assert_unreached(); + // LCOV_EXCL_STOP } } /** Release the lock <b>m</b> so another thread can have it. */ @@ -147,8 +151,10 @@ tor_mutex_release(tor_mutex_t *m) tor_assert(m); err = pthread_mutex_unlock(&m->mutex); if (PREDICT_UNLIKELY(err)) { + // LCOV_EXCL_START log_err(LD_GENERAL, "Error %d unlocking a mutex.", err); - tor_fragile_assert(); + tor_assert_unreached(); + // LCOV_EXCL_STOP } } /** Clean up the mutex <b>m</b> so that it no longer uses any system @@ -161,8 +167,10 @@ tor_mutex_uninit(tor_mutex_t *m) tor_assert(m); err = pthread_mutex_destroy(&m->mutex); if (PREDICT_UNLIKELY(err)) { + // LCOV_EXCL_START log_err(LD_GENERAL, "Error %d destroying a mutex.", err); - tor_fragile_assert(); + tor_assert_unreached(); + // LCOV_EXCL_STOP } } /** Return an integer representing this thread. */ @@ -212,8 +220,10 @@ void tor_cond_uninit(tor_cond_t *cond) { if (pthread_cond_destroy(&cond->cond)) { + // LCOV_EXCL_START log_warn(LD_GENERAL,"Error freeing condition: %s", strerror(errno)); return; + // LCOV_EXCL_STOP } } /** Wait until one of the tor_cond_signal functions is called on <b>cond</b>. @@ -234,7 +244,7 @@ tor_cond_wait(tor_cond_t *cond, tor_mutex_t *mutex, const struct timeval *tv) /* EINTR should be impossible according to POSIX, but POSIX, like the * Pirate's Code, is apparently treated "more like what you'd call * guidelines than actual rules." */ - continue; + continue; // LCOV_EXCL_LINE } return r ? -1 : 0; } diff --git a/src/common/compat_threads.c b/src/common/compat_threads.c index 8f9001258a..f4809060d6 100644 --- a/src/common/compat_threads.c +++ b/src/common/compat_threads.c @@ -11,8 +11,6 @@ * modules.) */ -#define _GNU_SOURCE - #include "orconfig.h" #include <stdlib.h> #include "compat.h" @@ -63,8 +61,8 @@ tor_cond_t * tor_cond_new(void) { tor_cond_t *cond = tor_malloc(sizeof(tor_cond_t)); - if (tor_cond_init(cond)<0) - tor_free(cond); + if (BUG(tor_cond_init(cond)<0)) + tor_free(cond); // LCOV_EXCL_LINE return cond; } @@ -242,8 +240,11 @@ alert_sockets_create(alert_sockets_t *socks_out, uint32_t flags) if (socks[0] >= 0) { if (fcntl(socks[0], F_SETFD, FD_CLOEXEC) < 0 || set_socket_nonblocking(socks[0]) < 0) { + // LCOV_EXCL_START -- if eventfd succeeds, fcntl will. + tor_assert_nonfatal_unreached(); close(socks[0]); return -1; + // LCOV_EXCL_STOP } } } @@ -277,9 +278,12 @@ alert_sockets_create(alert_sockets_t *socks_out, uint32_t flags) fcntl(socks[1], F_SETFD, FD_CLOEXEC) < 0 || set_socket_nonblocking(socks[0]) < 0 || set_socket_nonblocking(socks[1]) < 0) { + // LCOV_EXCL_START -- if pipe succeeds, you can fcntl the output + tor_assert_nonfatal_unreached(); close(socks[0]); close(socks[1]); return -1; + // LCOV_EXCL_STOP } socks_out->read_fd = socks[0]; socks_out->write_fd = socks[1]; @@ -294,9 +298,12 @@ alert_sockets_create(alert_sockets_t *socks_out, uint32_t flags) tor_socketpair(AF_UNIX, SOCK_STREAM, 0, socks) == 0) { if (set_socket_nonblocking(socks[0]) < 0 || set_socket_nonblocking(socks[1])) { + // LCOV_EXCL_START -- if socketpair worked, you can make it nonblocking. + tor_assert_nonfatal_unreached(); tor_close_socket(socks[0]); tor_close_socket(socks[1]); return -1; + // LCOV_EXCL_STOP } socks_out->read_fd = socks[0]; socks_out->write_fd = socks[1]; diff --git a/src/common/crypto.c b/src/common/crypto.c index 2b96324d33..1c5b5993c9 100644 --- a/src/common/crypto.c +++ b/src/common/crypto.c @@ -29,18 +29,7 @@ #include "crypto_ed25519.h" #include "crypto_format.h" -#ifdef __GNUC__ -#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -#endif - -#if __GNUC__ && GCC_VERSION >= 402 -#if GCC_VERSION >= 406 -#pragma GCC diagnostic push -#endif -/* Some versions of OpenSSL declare X509_STORE_CTX_set_verify_cb twice. - * Suppress the GCC warning so we can build with -Wredundant-decl. */ -#pragma GCC diagnostic ignored "-Wredundant-decls" -#endif +DISABLE_GCC_WARNING(redundant-decls) #include <openssl/err.h> #include <openssl/rsa.h> @@ -53,6 +42,8 @@ #include <openssl/conf.h> #include <openssl/hmac.h> +ENABLE_GCC_WARNING(redundant-decls) + #if __GNUC__ && GCC_VERSION >= 402 #if GCC_VERSION >= 406 #pragma GCC diagnostic pop @@ -65,7 +56,6 @@ #include <ctype.h> #endif #ifdef HAVE_UNISTD_H -#define _GNU_SOURCE #include <unistd.h> #endif #ifdef HAVE_FCNTL_H @@ -155,7 +145,7 @@ crypto_get_rsa_padding_overhead(int padding) switch (padding) { case RSA_PKCS1_OAEP_PADDING: return PKCS1_OAEP_PADDING_OVERHEAD; - default: tor_assert(0); return -1; + default: tor_assert(0); return -1; // LCOV_EXCL_LINE } } @@ -167,7 +157,7 @@ crypto_get_rsa_padding(int padding) switch (padding) { case PK_PKCS1_OAEP_PADDING: return RSA_PKCS1_OAEP_PADDING; - default: tor_assert(0); return -1; + default: tor_assert(0); return -1; // LCOV_EXCL_LINE } } @@ -192,13 +182,9 @@ crypto_log_errors(int severity, const char *doing) if (!msg) msg = "(null)"; if (!lib) lib = "(null)"; if (!func) func = "(null)"; - if (doing) { - tor_log(severity, LD_CRYPTO, "crypto error while %s: %s (in %s:%s)", + if (BUG(!doing)) doing = "(null)"; + tor_log(severity, LD_CRYPTO, "crypto error while %s: %s (in %s:%s)", doing, msg, lib, func); - } else { - tor_log(severity, LD_CRYPTO, "crypto error: %s (in %s:%s)", - msg, lib, func); - } } } @@ -1015,6 +1001,10 @@ crypto_pk_copy_full(crypto_pk_t *env) new_key = RSAPublicKey_dup(env->key); } if (!new_key) { + /* LCOV_EXCL_START + * + * We can't cause RSA*Key_dup() to fail, so we can't really test this. + */ log_err(LD_CRYPTO, "Unable to duplicate a %s key: openssl failed.", privatekey?"private":"public"); crypto_log_errors(LOG_ERR, @@ -1022,6 +1012,7 @@ crypto_pk_copy_full(crypto_pk_t *env) "Duplicating a public key"); tor_fragile_assert(); return NULL; + /* LCOV_EXCL_STOP */ } return crypto_new_pk_from_rsa_(new_key); @@ -1772,8 +1763,10 @@ crypto_digest_algorithm_get_name(digest_algorithm_t alg) case DIGEST_SHA3_512: return "sha3-512"; default: + // LCOV_EXCL_START tor_fragile_assert(); return "??unknown_digest??"; + // LCOV_EXCL_STOP } } @@ -1797,7 +1790,7 @@ crypto_digest_algorithm_parse_name(const char *name) } /** Given an algorithm, return the digest length in bytes. */ -static inline size_t +size_t crypto_digest_algorithm_get_length(digest_algorithm_t alg) { switch (alg) { @@ -1812,8 +1805,8 @@ crypto_digest_algorithm_get_length(digest_algorithm_t alg) case DIGEST_SHA3_512: return DIGEST512_LEN; default: - tor_assert(0); - return 0; /* Unreachable */ + tor_assert(0); // LCOV_EXCL_LINE + return 0; /* Unreachable */ // LCOV_EXCL_LINE } } @@ -1856,23 +1849,53 @@ crypto_digest_alloc_bytes(digest_algorithm_t alg) case DIGEST_SHA3_512: return END_OF_FIELD(d.sha3); default: - tor_assert(0); - return 0; + tor_assert(0); // LCOV_EXCL_LINE + return 0; // LCOV_EXCL_LINE } #undef END_OF_FIELD #undef STRUCT_FIELD_SIZE } +/** + * Internal function: create and return a new digest object for 'algorithm'. + * Does not typecheck the algorithm. + */ +static crypto_digest_t * +crypto_digest_new_internal(digest_algorithm_t algorithm) +{ + crypto_digest_t *r = tor_malloc(crypto_digest_alloc_bytes(algorithm)); + r->algorithm = algorithm; + + switch (algorithm) + { + case DIGEST_SHA1: + SHA1_Init(&r->d.sha1); + break; + case DIGEST_SHA256: + SHA256_Init(&r->d.sha2); + break; + case DIGEST_SHA512: + SHA512_Init(&r->d.sha512); + break; + case DIGEST_SHA3_256: + keccak_digest_init(&r->d.sha3, 256); + break; + case DIGEST_SHA3_512: + keccak_digest_init(&r->d.sha3, 512); + break; + default: + tor_assert_unreached(); + } + + return r; +} + /** Allocate and return a new digest object to compute SHA1 digests. */ crypto_digest_t * crypto_digest_new(void) { - crypto_digest_t *r; - r = tor_malloc(crypto_digest_alloc_bytes(DIGEST_SHA1)); - SHA1_Init(&r->d.sha1); - r->algorithm = DIGEST_SHA1; - return r; + return crypto_digest_new_internal(DIGEST_SHA1); } /** Allocate and return a new digest object to compute 256-bit digests @@ -1880,15 +1903,8 @@ crypto_digest_new(void) crypto_digest_t * crypto_digest256_new(digest_algorithm_t algorithm) { - crypto_digest_t *r; tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256); - r = tor_malloc(crypto_digest_alloc_bytes(algorithm)); - if (algorithm == DIGEST_SHA256) - SHA256_Init(&r->d.sha2); - else - keccak_digest_init(&r->d.sha3, 256); - r->algorithm = algorithm; - return r; + return crypto_digest_new_internal(algorithm); } /** Allocate and return a new digest object to compute 512-bit digests @@ -1896,15 +1912,8 @@ crypto_digest256_new(digest_algorithm_t algorithm) crypto_digest_t * crypto_digest512_new(digest_algorithm_t algorithm) { - crypto_digest_t *r; tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512); - r = tor_malloc(crypto_digest_alloc_bytes(algorithm)); - if (algorithm == DIGEST_SHA512) - SHA512_Init(&r->d.sha512); - else - keccak_digest_init(&r->d.sha3, 512); - r->algorithm = algorithm; - return r; + return crypto_digest_new_internal(algorithm); } /** Deallocate a digest object. @@ -1947,8 +1956,10 @@ crypto_digest_add_bytes(crypto_digest_t *digest, const char *data, keccak_digest_update(&digest->d.sha3, (const uint8_t *)data, len); break; default: + /* LCOV_EXCL_START */ tor_fragile_assert(); break; + /* LCOV_EXCL_STOP */ } } @@ -1987,13 +1998,15 @@ crypto_digest_get_digest(crypto_digest_t *digest, case DIGEST_SHA512: SHA512_Final(r, &tmpenv.d.sha512); break; +//LCOV_EXCL_START case DIGEST_SHA3_256: /* FALLSTHROUGH */ case DIGEST_SHA3_512: - log_warn(LD_BUG, "Handling unexpected algorithm %d", digest->algorithm); - tor_assert(0); /* This is fatal, because it should never happen. */ default: - tor_assert(0); /* Unreachable. */ + log_warn(LD_BUG, "Handling unexpected algorithm %d", digest->algorithm); + /* This is fatal, because it should never happen. */ + tor_assert_unreached(); break; +//LCOV_EXCL_STOP } memcpy(out, r, out_len); memwipe(r, 0, sizeof(r)); @@ -2052,27 +2065,7 @@ crypto_digest_smartlist_prefix(char *digest_out, size_t len_out, const char *append, digest_algorithm_t alg) { - crypto_digest_t *d = NULL; - switch (alg) { - case DIGEST_SHA1: - d = crypto_digest_new(); - break; - case DIGEST_SHA256: /* FALLSTHROUGH */ - case DIGEST_SHA3_256: - d = crypto_digest256_new(alg); - break; - case DIGEST_SHA512: /* FALLSTHROUGH */ - case DIGEST_SHA3_512: - d = crypto_digest512_new(alg); - break; - default: - log_warn(LD_BUG, "Called with unknown algorithm %d", alg); - /* If fragile_assert is not enabled, wipe output and return - * without running any calculations */ - memwipe(digest_out, 0xff, len_out); - tor_fragile_assert(); - goto free; - } + crypto_digest_t *d = crypto_digest_new_internal(alg); if (prepend) crypto_digest_add_bytes(d, prepend, strlen(prepend)); SMARTLIST_FOREACH(lst, const char *, cp, @@ -2080,8 +2073,6 @@ crypto_digest_smartlist_prefix(char *digest_out, size_t len_out, if (append) crypto_digest_add_bytes(d, append, strlen(append)); crypto_digest_get_digest(d, digest_out, len_out); - - free: crypto_digest_free(d); } @@ -2250,9 +2241,14 @@ crypto_set_tls_dh_prime(void) int r; /* If the space is occupied, free the previous TLS DH prime */ - if (dh_param_p_tls) { + if (BUG(dh_param_p_tls)) { + /* LCOV_EXCL_START + * + * We shouldn't be calling this twice. + */ BN_clear_free(dh_param_p_tls); dh_param_p_tls = NULL; + /* LCOV_EXCL_STOP */ } tls_prime = BN_new(); @@ -2284,8 +2280,8 @@ init_dh_param(void) { BIGNUM *circuit_dh_prime; int r; - if (dh_param_p && dh_param_g) - return; + if (BUG(dh_param_p && dh_param_g)) + return; // LCOV_EXCL_LINE This function isn't supposed to be called twice. circuit_dh_prime = BN_new(); tor_assert(circuit_dh_prime); @@ -2375,10 +2371,13 @@ crypto_dh_new(int dh_type) return res; err: + /* LCOV_EXCL_START + * This error condition is only reached when an allocation fails */ crypto_log_errors(LOG_WARN, "creating DH object"); if (res->dh) DH_free(res->dh); /* frees p and g too */ tor_free(res); return NULL; + /* LCOV_EXCL_STOP */ } /** Return a copy of <b>dh</b>, sharing its internal state. */ @@ -2412,8 +2411,11 @@ crypto_dh_generate_public(crypto_dh_t *dh) again: #endif if (!DH_generate_key(dh->dh)) { + /* LCOV_EXCL_START + * To test this we would need some way to tell openssl to break DH. */ crypto_log_errors(LOG_WARN, "generating DH key"); return -1; + /* LCOV_EXCL_STOP */ } #ifdef OPENSSL_1_1_API /* OpenSSL 1.1.x doesn't appear to let you regenerate a DH key, without @@ -2429,6 +2431,8 @@ crypto_dh_generate_public(crypto_dh_t *dh) } #else if (tor_check_dh_key(LOG_WARN, dh->dh->pub_key)<0) { + /* LCOV_EXCL_START + * If this happens, then openssl's DH implementation is busted. */ log_warn(LD_CRYPTO, "Weird! Our own DH key was invalid. I guess once-in-" "the-universe chances really do happen. Trying again."); /* Free and clear the keys, so OpenSSL will actually try again. */ @@ -2436,6 +2440,7 @@ crypto_dh_generate_public(crypto_dh_t *dh) BN_clear_free(dh->dh->priv_key); dh->dh->pub_key = dh->dh->priv_key = NULL; goto again; + /* LCOV_EXCL_STOP */ } #endif return 0; @@ -2500,8 +2505,8 @@ tor_check_dh_key(int severity, const BIGNUM *bn) tor_assert(bn); x = BN_new(); tor_assert(x); - if (!dh_param_p) - init_dh_param(); + if (BUG(!dh_param_p)) + init_dh_param(); //LCOV_EXCL_LINE we already checked whether we did this. BN_set_word(x, 1); if (BN_cmp(bn,x)<=0) { log_fn(severity, LD_CRYPTO, "DH key must be at least 2."); @@ -2523,8 +2528,6 @@ tor_check_dh_key(int severity, const BIGNUM *bn) return -1; } -#undef MIN -#define MIN(a,b) ((a)<(b)?(a):(b)) /** Given a DH key exchange object, and our peer's value of g^y (as a * <b>pubkey_len</b>-byte value in <b>pubkey</b>) generate * <b>secret_bytes_out</b> bytes of shared key material and write them @@ -2712,6 +2715,11 @@ crypto_seed_weak_rng(tor_weak_rng_t *rng) tor_init_weak_random(rng, seed); } +#ifdef TOR_UNIT_TESTS +int break_strongest_rng_syscall = 0; +int break_strongest_rng_fallback = 0; +#endif + /** Try to get <b>out_len</b> bytes of the strongest entropy we can generate, * via system calls, storing it into <b>out</b>. Return 0 on success, -1 on * failure. A maximum request size of 256 bytes is imposed. @@ -2721,6 +2729,11 @@ crypto_strongest_rand_syscall(uint8_t *out, size_t out_len) { tor_assert(out_len <= MAX_STRONGEST_RAND_SIZE); +#ifdef TOR_UNIT_TESTS + if (break_strongest_rng_syscall) + return -1; +#endif + #if defined(_WIN32) static int provider_set = 0; static HCRYPTPROV provider; @@ -2770,6 +2783,7 @@ crypto_strongest_rand_syscall(uint8_t *out, size_t out_len) } while (ret == -1 && ((errno == EINTR) ||(errno == EAGAIN))); if (PREDICT_UNLIKELY(ret == -1)) { + /* LCOV_EXCL_START we can't actually make the syscall fail in testing. */ tor_assert(errno != EAGAIN); tor_assert(errno != EINTR); @@ -2777,6 +2791,7 @@ crypto_strongest_rand_syscall(uint8_t *out, size_t out_len) log_warn(LD_CRYPTO, "Can't get entropy from getrandom()."); getrandom_works = 0; /* Don't bother trying again. */ return -1; + /* LCOV_EXCL_STOP */ } tor_assert(ret == (long)out_len); @@ -2805,6 +2820,11 @@ crypto_strongest_rand_syscall(uint8_t *out, size_t out_len) static int crypto_strongest_rand_fallback(uint8_t *out, size_t out_len) { +#ifdef TOR_UNIT_TESTS + if (break_strongest_rng_fallback) + return -1; +#endif + #ifdef _WIN32 /* Windows exclusively uses crypto_strongest_rand_syscall(). */ (void)out; @@ -2825,10 +2845,13 @@ crypto_strongest_rand_fallback(uint8_t *out, size_t out_len) n = read_all(fd, (char*)out, out_len, 0); close(fd); if (n != out_len) { + /* LCOV_EXCL_START + * We can't make /dev/foorandom actually fail. */ log_warn(LD_CRYPTO, "Error reading from entropy source (read only %lu bytes).", (unsigned long)n); return -1; + /* LCOV_EXCL_STOP */ } return 0; @@ -2842,7 +2865,7 @@ crypto_strongest_rand_fallback(uint8_t *out, size_t out_len) * storing it into <b>out</b>. Return 0 on success, -1 on failure. A maximum * request size of 256 bytes is imposed. */ -static int +STATIC int crypto_strongest_rand_raw(uint8_t *out, size_t out_len) { static const size_t sanity_min_size = 16; @@ -2876,13 +2899,17 @@ crypto_strongest_rand_raw(uint8_t *out, size_t out_len) return 0; } - /* We tried max_attempts times to fill a buffer >= 128 bits long, + /* LCOV_EXCL_START + * + * We tried max_attempts times to fill a buffer >= 128 bits long, * and each time it returned all '0's. Either the system entropy * source is busted, or the user should go out and buy a ticket to * every lottery on the planet. */ log_warn(LD_CRYPTO, "Strong OS entropy returned all zero buffer."); + return -1; + /* LCOV_EXCL_STOP */ } /** Try to get <b>out_len</b> bytes of the strongest entropy we can generate, @@ -2901,10 +2928,12 @@ crypto_strongest_rand(uint8_t *out, size_t out_len) while (out_len) { crypto_rand((char*) inp, DLEN); if (crypto_strongest_rand_raw(inp+DLEN, DLEN) < 0) { + // LCOV_EXCL_START log_err(LD_CRYPTO, "Failed to load strong entropy when generating an " "important key. Exiting."); /* Die with an assertion so we get a stack trace. */ tor_assert(0); + // LCOV_EXCL_STOP } if (out_len >= DLEN) { SHA512(inp, sizeof(inp), out); @@ -2935,7 +2964,7 @@ crypto_seed_rng(void) * functions. If one succeeds, we'll accept the RNG as seeded. */ rand_poll_ok = RAND_poll(); if (rand_poll_ok == 0) - log_warn(LD_CRYPTO, "RAND_poll() failed."); + log_warn(LD_CRYPTO, "RAND_poll() failed."); // LCOV_EXCL_LINE load_entropy_ok = !crypto_strongest_rand_raw(buf, sizeof(buf)); if (load_entropy_ok) { diff --git a/src/common/crypto.h b/src/common/crypto.h index 682c4e3253..f8fb0daa81 100644 --- a/src/common/crypto.h +++ b/src/common/crypto.h @@ -233,6 +233,7 @@ void crypto_digest_smartlist(char *digest_out, size_t len_out, const struct smartlist_t *lst, const char *append, digest_algorithm_t alg); const char *crypto_digest_algorithm_get_name(digest_algorithm_t alg); +size_t crypto_digest_algorithm_get_length(digest_algorithm_t alg); int crypto_digest_algorithm_parse_name(const char *name); crypto_digest_t *crypto_digest_new(void); crypto_digest_t *crypto_digest256_new(digest_algorithm_t algorithm); @@ -317,6 +318,12 @@ void crypto_add_spaces_to_fp(char *out, size_t outlen, const char *in); #ifdef CRYPTO_PRIVATE STATIC int crypto_force_rand_ssleay(void); +STATIC int crypto_strongest_rand_raw(uint8_t *out, size_t out_len); + +#ifdef TOR_UNIT_TESTS +extern int break_strongest_rng_syscall; +extern int break_strongest_rng_fallback; +#endif #endif #endif diff --git a/src/common/crypto_curve25519.c b/src/common/crypto_curve25519.c index 57c878b79a..58ec923638 100644 --- a/src/common/crypto_curve25519.c +++ b/src/common/crypto_curve25519.c @@ -65,8 +65,10 @@ STATIC int curve25519_basepoint_impl(uint8_t *output, const uint8_t *secret) { int r = 0; - if (PREDICT_UNLIKELY(curve25519_use_ed == -1)) { + if (BUG(curve25519_use_ed == -1)) { + /* LCOV_EXCL_START - Only reached if we forgot to call curve25519_init() */ pick_curve25519_basepoint_impl(); + /* LCOV_EXCL_STOP */ } /* TODO: Someone should benchmark curved25519_scalarmult_basepoint versus @@ -290,10 +292,13 @@ pick_curve25519_basepoint_impl(void) if (curve25519_basepoint_spot_check() == 0) return; - log_warn(LD_CRYPTO, "The ed25519-based curve25519 basepoint " + /* LCOV_EXCL_START + * only reachable if our basepoint implementation broken */ + log_warn(LD_BUG|LD_CRYPTO, "The ed25519-based curve25519 basepoint " "multiplication seems broken; using the curve25519 " "implementation."); curve25519_use_ed = 0; + /* LCOV_EXCL_STOP */ } /** Initialize the curve25519 implementations. This is necessary if you're diff --git a/src/common/crypto_ed25519.c b/src/common/crypto_ed25519.c index ea2d8e3892..84c3eece6d 100644 --- a/src/common/crypto_ed25519.c +++ b/src/common/crypto_ed25519.c @@ -94,8 +94,8 @@ static const ed25519_impl_t *ed25519_impl = NULL; static inline const ed25519_impl_t * get_ed_impl(void) { - if (PREDICT_UNLIKELY(ed25519_impl == NULL)) { - pick_ed25519_impl(); + if (BUG(ed25519_impl == NULL)) { + pick_ed25519_impl(); // LCOV_EXCL_LINE - We always call ed25519_init(). } return ed25519_impl; } @@ -259,11 +259,11 @@ ed25519_checksig_batch(int *okay_out, int *oks; int all_ok; - ms = tor_malloc(sizeof(uint8_t*)*n_checkable); - lens = tor_malloc(sizeof(size_t)*n_checkable); - pks = tor_malloc(sizeof(uint8_t*)*n_checkable); - sigs = tor_malloc(sizeof(uint8_t*)*n_checkable); - oks = okay_out ? okay_out : tor_malloc(sizeof(int)*n_checkable); + ms = tor_calloc(n_checkable, sizeof(uint8_t*)); + lens = tor_calloc(n_checkable, sizeof(size_t)); + pks = tor_calloc(n_checkable, sizeof(uint8_t*)); + sigs = tor_calloc(n_checkable, sizeof(uint8_t*)); + oks = okay_out ? okay_out : tor_calloc(n_checkable, sizeof(int)); for (i = 0; i < n_checkable; ++i) { ms[i] = checkable[i].msg; @@ -433,6 +433,7 @@ ed25519_seckey_read_from_file(ed25519_secret_key_t *seckey_out, errno = EINVAL; } + tor_free(*tag_out); return -1; } @@ -472,6 +473,7 @@ ed25519_pubkey_read_from_file(ed25519_public_key_t *pubkey_out, errno = EINVAL; } + tor_free(*tag_out); return -1; } @@ -594,9 +596,12 @@ pick_ed25519_impl(void) if (ed25519_impl_spot_check() == 0) return; + /* LCOV_EXCL_START + * unreachable unless ed25519_donna is broken */ log_warn(LD_CRYPTO, "The Ed25519-donna implementation seems broken; using " "the ref10 implementation."); ed25519_impl = &impl_ref10; + /* LCOV_EXCL_STOP */ } /* Initialize the Ed25519 implementation. This is neccessary if you're diff --git a/src/common/crypto_pwbox.c b/src/common/crypto_pwbox.c index 819dc0c39d..31e37c007d 100644 --- a/src/common/crypto_pwbox.c +++ b/src/common/crypto_pwbox.c @@ -61,7 +61,7 @@ crypto_pwbox(uint8_t **out, size_t *outlen_out, pwbox_encoded_getarray_skey_header(enc), S2K_MAXLEN, s2k_flags); - if (spec_len < 0 || spec_len > S2K_MAXLEN) + if (BUG(spec_len < 0 || spec_len > S2K_MAXLEN)) goto err; pwbox_encoded_setlen_skey_header(enc, spec_len); enc->header_len = spec_len; @@ -76,10 +76,11 @@ crypto_pwbox(uint8_t **out, size_t *outlen_out, /* Now that all the data is in position, derive some keys, encrypt, and * digest */ - if (secret_to_key_derivekey(keys, sizeof(keys), + const int s2k_rv = secret_to_key_derivekey(keys, sizeof(keys), pwbox_encoded_getarray_skey_header(enc), spec_len, - secret, secret_len) < 0) + secret, secret_len); + if (BUG(s2k_rv < 0)) goto err; cipher = crypto_cipher_new_with_iv((char*)keys, (char*)enc->iv); @@ -87,11 +88,11 @@ crypto_pwbox(uint8_t **out, size_t *outlen_out, crypto_cipher_free(cipher); result_len = pwbox_encoded_encoded_len(enc); - if (result_len < 0) + if (BUG(result_len < 0)) goto err; result = tor_malloc(result_len); enc_len = pwbox_encoded_encode(result, result_len, enc); - if (enc_len < 0) + if (BUG(enc_len < 0)) goto err; tor_assert(enc_len == result_len); @@ -107,9 +108,24 @@ crypto_pwbox(uint8_t **out, size_t *outlen_out, goto out; err: + /* LCOV_EXCL_START + + This error case is often unreachable if we're correctly coded, unless + somebody adds a new error case somewhere, or unless you're building + without scrypto support. + + - make_specifier can't fail, unless S2K_MAX_LEN is too short. + - secret_to_key_derivekey can't really fail unless we're missing + scrypt, or the underlying function fails, or we pass it a bogus + algorithm or parameters. + - pwbox_encoded_encoded_len can't fail unless we're using trunnel + incorrectly. + - pwbox_encoded_encode can't fail unless we're using trunnel wrong, + or it's buggy. + */ tor_free(result); rv = -1; - + /* LCOV_EXCL_STOP */ out: pwbox_encoded_free(enc); memwipe(keys, 0, sizeof(keys)); diff --git a/src/common/crypto_s2k.c b/src/common/crypto_s2k.c index 3bc05f1cf9..5dbd2ad91f 100644 --- a/src/common/crypto_s2k.c +++ b/src/common/crypto_s2k.c @@ -57,7 +57,8 @@ #define SCRYPT_KEY_LEN 32 /** Given an algorithm ID (one of S2K_TYPE_*), return the length of the - * specifier part of it, without the prefix type byte. */ + * specifier part of it, without the prefix type byte. Return -1 if it is not + * a valid algorithm ID. */ static int secret_to_key_spec_len(uint8_t type) { @@ -86,7 +87,8 @@ secret_to_key_key_len(uint8_t type) case S2K_TYPE_SCRYPT: return DIGEST256_LEN; default: - return -1; + tor_fragile_assert(); // LCOV_EXCL_LINE + return -1; // LCOV_EXCL_LINE } } @@ -168,7 +170,7 @@ make_specifier(uint8_t *spec_out, uint8_t type, unsigned flags) spec_out[SCRYPT_SPEC_LEN-1] = (3u << 4) | (1u << 0); break; default: - tor_fragile_assert(); + tor_fragile_assert(); // LCOV_EXCL_LINE - we should have returned above. return S2K_BAD_ALGORITHM; } diff --git a/src/common/di_ops.c b/src/common/di_ops.c index 5dfe828066..4ed49e1164 100644 --- a/src/common/di_ops.c +++ b/src/common/di_ops.c @@ -226,3 +226,49 @@ safe_mem_is_zero(const void *mem, size_t sz) return 1 & ((total - 1) >> 8); } +/** Time-invariant 64-bit greater-than; works on two integers in the range + * (0,INT64_MAX). */ +#if SIZEOF_VOID_P == 8 +#define gt_i64_timei(a,b) ((a) > (b)) +#else +static inline int +gt_i64_timei(uint64_t a, uint64_t b) +{ + int64_t diff = (int64_t) (b - a); + int res = diff >> 63; + return res & 1; +} +#endif + +/** + * Given an array of list of <b>n_entries</b> uint64_t values, whose sum is + * <b>total</b>, find the first i such that the total of all elements 0...i is + * greater than rand_val. + * + * Try to perform this operation in a constant-time way. + */ +int +select_array_member_cumulative_timei(const uint64_t *entries, int n_entries, + uint64_t total, uint64_t rand_val) +{ + int i, i_chosen=-1, n_chosen=0; + uint64_t total_so_far = 0; + + for (i = 0; i < n_entries; ++i) { + total_so_far += entries[i]; + if (gt_i64_timei(total_so_far, rand_val)) { + i_chosen = i; + n_chosen++; + /* Set rand_val to INT64_MAX rather than stopping the loop. This way, + * the time we spend in the loop does not leak which element we chose. */ + rand_val = INT64_MAX; + } + } + tor_assert(total_so_far == total); + tor_assert(n_chosen == 1); + tor_assert(i_chosen >= 0); + tor_assert(i_chosen < n_entries); + + return i_chosen; +} + diff --git a/src/common/di_ops.h b/src/common/di_ops.h index 6e77b5cfd7..0a154302bf 100644 --- a/src/common/di_ops.h +++ b/src/common/di_ops.h @@ -42,6 +42,9 @@ void dimap_add_entry(di_digest256_map_t **map, const uint8_t *key, void *val); void *dimap_search(const di_digest256_map_t *map, const uint8_t *key, void *dflt_val); +int select_array_member_cumulative_timei(const uint64_t *entries, + int n_entries, + uint64_t total, uint64_t rand_val); #endif diff --git a/src/common/handles.h b/src/common/handles.h new file mode 100644 index 0000000000..1ee2322579 --- /dev/null +++ b/src/common/handles.h @@ -0,0 +1,153 @@ +/* Copyright (c) 2016, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file handles.h + * \brief Macros for C weak-handle implementation. + * + * A 'handle' is a pointer to an object that is allowed to go away while + * the handle stays alive. When you dereference the handle, you might get + * the object, or you might get "NULL". + * + * Use this pattern when an object has a single obvious lifespan, so you don't + * want to use reference counting, but when other objects might need to refer + * to the first object without caring about its lifetime. + * + * To enable a type to have handles, add a HANDLE_ENTRY() field in its + * definition, as in: + * + * struct walrus { + * HANDLE_ENTRY(wlr, walrus); + * // ... + * }; + * + * And invoke HANDLE_DECL(wlr, walrus, [static]) to declare the handle + * manipulation functions (typically in a header): + * + * // opaque handle to walrus. + * typedef struct wlr_handle_t wlr_handle_t; + * + * // make a new handle + * struct wlr_handle_t *wlr_handle_new(struct walrus *); + * + * // release a handle + * void wlr_handle_free(wlr_handle_t *); + * + * // return the pointed-to walrus, or NULL. + * struct walrus *wlr_handle_get(wlr_handle_t *). + * + * // call this function when you're about to free the walrus; + * // it invalidates all handles. (IF YOU DON'T, YOU WILL HAVE + * // DANGLING REFERENCES) + * void wlr_handles_clear(struct walrus *); + * + * Finally, use HANDLE_IMPL() to define the above functions in some + * appropriate C file: HANDLE_IMPL(wlr, walrus, [static]) + * + **/ + +#ifndef TOR_HANDLE_H +#define TOR_HANDLE_H + +#include "orconfig.h" +#include "tor_queue.h" +#include "util.h" + +#define HANDLE_ENTRY(name, structname) \ + struct name ## _handle_head_t *handle_head + +#define HANDLE_DECL(name, structname, linkage) \ + typedef struct name ## _handle_t name ## _handle_t; \ + linkage name ## _handle_t *name ## _handle_new(struct structname *object); \ + linkage void name ## _handle_free(name ## _handle_t *); \ + linkage struct structname *name ## _handle_get(name ## _handle_t *); \ + linkage void name ## _handles_clear(struct structname *object); + +/* + * Implementation notes: there are lots of possible implementations here. We + * could keep a linked list of handles, each with a backpointer to the object, + * and set all of their backpointers to NULL when the object is freed. Or we + * could have the clear function invalidate the object, but not actually let + * the object get freed until the all the handles went away. We could even + * have a hash-table mapping unique identifiers to objects, and have each + * handle be a copy of the unique identifier. (We'll want to build that last + * one eventually if we want cross-process handles.) + * + * But instead we're opting for a single independent 'head' that knows how + * many handles there are, and where the object is (or isn't). This makes + * all of our functions O(1), and most as fast as a single pointer access. + * + * The handles themselves are opaque structures holding a pointer to the head. + * We could instead have each foo_handle_t* be identical to foo_handle_head_t + * *, and save some allocations ... but doing so would make handle leaks + * harder to debug. As it stands, every handle leak is a memory leak, and + * existing memory debugging tools should help with those. We can revisit + * this decision if handles are too slow. + */ + +#define HANDLE_IMPL(name, structname, linkage) \ + /* The 'head' object for a handle-accessible type. This object */ \ + /* persists for as long as the object, or any handles, exist. */ \ + typedef struct name ## _handle_head_t { \ + struct structname *object; /* pointed-to object, or NULL */ \ + unsigned int references; /* number of existing handles */ \ + } name ## _handle_head_t; \ + \ + struct name ## _handle_t { \ + struct name ## _handle_head_t *head; /* reference to the 'head'. */ \ + }; \ + \ + linkage struct name ## _handle_t * \ + name ## _handle_new(struct structname *object) \ + { \ + tor_assert(object); \ + name ## _handle_head_t *head = object->handle_head; \ + if (PREDICT_UNLIKELY(head == NULL)) { \ + head = object->handle_head = tor_malloc_zero(sizeof(*head)); \ + head->object = object; \ + } \ + name ## _handle_t *new_ref = tor_malloc_zero(sizeof(*new_ref)); \ + new_ref->head = head; \ + ++head->references; \ + return new_ref; \ + } \ + \ + linkage void \ + name ## _handle_free(struct name ## _handle_t *ref) \ + { \ + if (! ref) return; \ + name ## _handle_head_t *head = ref->head; \ + tor_assert(head); \ + --head->references; \ + tor_free(ref); \ + if (head->object == NULL && head->references == 0) { \ + tor_free(head); \ + return; \ + } \ + } \ + \ + linkage struct structname * \ + name ## _handle_get(struct name ## _handle_t *ref) \ + { \ + tor_assert(ref); \ + name ## _handle_head_t *head = ref->head; \ + tor_assert(head); \ + return head->object; \ + } \ + \ + linkage void \ + name ## _handles_clear(struct structname *object) \ + { \ + tor_assert(object); \ + name ## _handle_head_t *head = object->handle_head; \ + if (! head) \ + return; \ + object->handle_head = NULL; \ + head->object = NULL; \ + if (head->references == 0) { \ + tor_free(head); \ + } \ + } + +#endif /* TOR_HANDLE_H */ + diff --git a/src/common/include.am b/src/common/include.am index 5afb30da6a..222afe0291 100644 --- a/src/common/include.am +++ b/src/common/include.am @@ -1,12 +1,14 @@ noinst_LIBRARIES += \ src/common/libor.a \ + src/common/libor-ctime.a \ src/common/libor-crypto.a \ src/common/libor-event.a if UNITTESTS_ENABLED noinst_LIBRARIES += \ src/common/libor-testing.a \ + src/common/libor-ctime-testing.a \ src/common/libor-crypto-testing.a \ src/common/libor-event-testing.a endif @@ -27,12 +29,14 @@ src_common_libcurve25519_donna_a_CFLAGS= if BUILD_CURVE25519_DONNA src_common_libcurve25519_donna_a_SOURCES=\ src/ext/curve25519_donna/curve25519-donna.c +# See bug 13538 -- this code is known to have signed overflow issues. src_common_libcurve25519_donna_a_CFLAGS+=\ - @F_OMIT_FRAME_POINTER@ + @F_OMIT_FRAME_POINTER@ @CFLAGS_CONSTTIME@ noinst_LIBRARIES+=src/common/libcurve25519_donna.a LIBDONNA=src/common/libcurve25519_donna.a else if BUILD_CURVE25519_DONNA_C64 +src_common_libcurve25519_donna_a_CFLAGS+=@CFLAGS_CONSTTIME@ src_common_libcurve25519_donna_a_SOURCES=\ src/ext/curve25519_donna/curve25519-donna-c64.c noinst_LIBRARIES+=src/common/libcurve25519_donna.a @@ -58,22 +62,37 @@ else readpassphrase_source= endif -LIBOR_A_SOURCES = \ +if ADD_MULODI4 +mulodi4_source=src/ext/mulodi/mulodi4.c +else +mulodi4_source= +endif + +LIBOR_CTIME_A_SRC = \ + $(mulodi4_source) \ + src/ext/csiphash.c \ + src/common/di_ops.c + +src_common_libor_ctime_a_SOURCES = $(LIBOR_CTIME_A_SRC) +src_common_libor_ctime_testing_a_SOURCES = $(LIBOR_CTIME_A_SRC) +src_common_libor_ctime_a_CFLAGS = @CFLAGS_CONSTTIME@ +src_common_libor_ctime_testing_a_CFLAGS = @CFLAGS_CONSTTIME@ $(TEST_CFLAGS) + +LIBOR_A_SRC = \ src/common/address.c \ src/common/backtrace.c \ src/common/compat.c \ src/common/compat_threads.c \ src/common/container.c \ - src/common/di_ops.c \ src/common/log.c \ src/common/memarea.c \ + src/common/pubsub.c \ src/common/util.c \ + src/common/util_bug.c \ src/common/util_format.c \ src/common/util_process.c \ src/common/sandbox.c \ src/common/workqueue.c \ - src/ext/csiphash.c \ - src/ext/trunnel/trunnel.c \ $(libor_extra_source) \ $(threads_impl_source) \ $(readpassphrase_source) @@ -81,7 +100,7 @@ LIBOR_A_SOURCES = \ src/common/src_common_libor_testing_a-log.$(OBJEXT) \ src/common/log.$(OBJEXT): micro-revision.i -LIBOR_CRYPTO_A_SOURCES = \ +LIBOR_CRYPTO_A_SRC = \ src/common/aes.c \ src/common/crypto.c \ src/common/crypto_pwbox.c \ @@ -89,21 +108,22 @@ LIBOR_CRYPTO_A_SOURCES = \ src/common/crypto_format.c \ src/common/torgzip.c \ src/common/tortls.c \ - src/trunnel/pwbox.c \ src/common/crypto_curve25519.c \ src/common/crypto_ed25519.c -LIBOR_EVENT_A_SOURCES = \ +LIBOR_EVENT_A_SRC = \ src/common/compat_libevent.c \ - src/common/procmon.c + src/common/procmon.c \ + src/common/timers.c \ + src/ext/timeouts/timeout.c -src_common_libor_a_SOURCES = $(LIBOR_A_SOURCES) -src_common_libor_crypto_a_SOURCES = $(LIBOR_CRYPTO_A_SOURCES) -src_common_libor_event_a_SOURCES = $(LIBOR_EVENT_A_SOURCES) +src_common_libor_a_SOURCES = $(LIBOR_A_SRC) +src_common_libor_crypto_a_SOURCES = $(LIBOR_CRYPTO_A_SRC) +src_common_libor_event_a_SOURCES = $(LIBOR_EVENT_A_SRC) -src_common_libor_testing_a_SOURCES = $(LIBOR_A_SOURCES) -src_common_libor_crypto_testing_a_SOURCES = $(LIBOR_CRYPTO_A_SOURCES) -src_common_libor_event_testing_a_SOURCES = $(LIBOR_EVENT_A_SOURCES) +src_common_libor_testing_a_SOURCES = $(LIBOR_A_SRC) +src_common_libor_crypto_testing_a_SOURCES = $(LIBOR_CRYPTO_A_SRC) +src_common_libor_event_testing_a_SOURCES = $(LIBOR_EVENT_A_SRC) src_common_libor_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_common_libor_crypto_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) @@ -129,16 +149,20 @@ COMMONHEADERS = \ src/common/crypto_pwbox.h \ src/common/crypto_s2k.h \ src/common/di_ops.h \ + src/common/handles.h \ src/common/memarea.h \ src/common/linux_syscalls.inc \ src/common/procmon.h \ + src/common/pubsub.h \ src/common/sandbox.h \ src/common/testsupport.h \ + src/common/timers.h \ src/common/torgzip.h \ src/common/torint.h \ src/common/torlog.h \ src/common/tortls.h \ src/common/util.h \ + src/common/util_bug.h \ src/common/util_format.h \ src/common/util_process.h \ src/common/workqueue.h diff --git a/src/common/log.c b/src/common/log.c index 6c387c6244..51309aa472 100644 --- a/src/common/log.c +++ b/src/common/log.c @@ -75,7 +75,7 @@ sev_to_string(int severity) case LOG_ERR: return "err"; default: /* Call assert, not tor_assert, since tor_assert * calls log on failure. */ - assert(0); return "UNKNOWN"; + assert(0); return "UNKNOWN"; // LCOV_EXCL_LINE } } @@ -95,7 +95,7 @@ should_log_function_name(log_domain_mask_t domain, int severity) return (domain & (LD_BUG|LD_NOFUNCNAME)) == LD_BUG; default: /* Call assert, not tor_assert, since tor_assert calls log on failure. */ - assert(0); return 0; + assert(0); return 0; // LCOV_EXCL_LINE } } @@ -270,7 +270,7 @@ log_tor_version(logfile_t *lf, int reset) return 0; } -const char bug_suffix[] = " (on Tor " VERSION +static const char bug_suffix[] = " (on Tor " VERSION #ifndef _MSC_VER " " #include "micro-revision.i" diff --git a/src/common/memarea.c b/src/common/memarea.c index 173ed4e1cb..7d16b702e3 100644 --- a/src/common/memarea.c +++ b/src/common/memarea.c @@ -131,7 +131,7 @@ alloc_chunk(size_t sz) /** Release <b>chunk</b> from a memarea. */ static void -chunk_free_unchecked(memarea_chunk_t *chunk) +memarea_chunk_free_unchecked(memarea_chunk_t *chunk) { CHECK_SENTINEL(chunk); tor_free(chunk); @@ -154,7 +154,7 @@ memarea_drop_all(memarea_t *area) memarea_chunk_t *chunk, *next; for (chunk = area->first; chunk; chunk = next) { next = chunk->next_chunk; - chunk_free_unchecked(chunk); + memarea_chunk_free_unchecked(chunk); } area->first = NULL; /*fail fast on */ tor_free(area); @@ -170,7 +170,7 @@ memarea_clear(memarea_t *area) if (area->first->next_chunk) { for (chunk = area->first->next_chunk; chunk; chunk = next) { next = chunk->next_chunk; - chunk_free_unchecked(chunk); + memarea_chunk_free_unchecked(chunk); } area->first->next_chunk = NULL; } diff --git a/src/common/procmon.c b/src/common/procmon.c index 12d53fcd41..4ecee26e8d 100644 --- a/src/common/procmon.c +++ b/src/common/procmon.c @@ -116,11 +116,11 @@ struct tor_process_monitor_t { * periodically check whether the process we have a handle to has * ended. */ HANDLE hproc; - /* XXX023 We can and should have Libevent watch hproc for us, - * if/when some version of Libevent 2.x can be told to do so. */ + /* XXXX We should have Libevent watch hproc for us, + * if/when some version of Libevent can be told to do so. */ #endif - /* XXX023 On Linux, we can and should receive the 22nd + /* XXXX On Linux, we can and should receive the 22nd * (space-delimited) field (‘starttime’) of /proc/$PID/stat from the * owning controller and store it, and poll once in a while to see * whether it has changed -- if so, the kernel has *definitely* @@ -130,7 +130,8 @@ struct tor_process_monitor_t { * systems whose admins have mounted procfs, or the start-time field * of the process-information structure returned by kvmgetprocs() on * any system. The latter is ickier. */ - /* XXX023 On FreeBSD (and possibly other kqueue systems), we can and + + /* XXXX On FreeBSD (and possibly other kqueue systems), we can and * should arrange to receive EVFILT_PROC NOTE_EXIT notifications for * pid, so we don't have to do such a heavyweight poll operation in * order to avoid the PID-reassignment race condition. (We would diff --git a/src/common/pubsub.c b/src/common/pubsub.c new file mode 100644 index 0000000000..b3faf40e00 --- /dev/null +++ b/src/common/pubsub.c @@ -0,0 +1,129 @@ +/* Copyright (c) 2016, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file pubsub.c + * + * \brief DOCDOC + */ + +#include "orconfig.h" +#include "pubsub.h" +#include "container.h" + +/** Helper: insert <b>s</b> into <b>topic's</b> list of subscribers, keeping + * them sorted in priority order. */ +static void +subscriber_insert(pubsub_topic_t *topic, pubsub_subscriber_t *s) +{ + int i; + smartlist_t *sl = topic->subscribers; + for (i = 0; i < smartlist_len(sl); ++i) { + pubsub_subscriber_t *other = smartlist_get(sl, i); + if (s->priority < other->priority) { + break; + } + } + smartlist_insert(sl, i, s); +} + +/** + * Add a new subscriber to <b>topic</b>, where (when an event is triggered), + * we'll notify the function <b>fn</b> by passing it <b>subscriber_data</b>. + * Return a handle to the subscribe which can later be passed to + * pubsub_unsubscribe_(). + * + * Functions are called in priority order, from lowest to highest. + * + * See pubsub.h for <b>subscribe_flags</b>. + */ +const pubsub_subscriber_t * +pubsub_subscribe_(pubsub_topic_t *topic, + pubsub_subscriber_fn_t fn, + void *subscriber_data, + unsigned subscribe_flags, + unsigned priority) +{ + tor_assert(! topic->locked); + if (subscribe_flags & SUBSCRIBE_ATSTART) { + tor_assert(topic->n_events_fired == 0); + } + pubsub_subscriber_t *r = tor_malloc_zero(sizeof(*r)); + r->priority = priority; + r->subscriber_flags = subscribe_flags; + r->fn = fn; + r->subscriber_data = subscriber_data; + if (topic->subscribers == NULL) { + topic->subscribers = smartlist_new(); + } + subscriber_insert(topic, r); + return r; +} + +/** + * Remove the subscriber <b>s</b> from <b>topic</b>. After calling this + * function, <b>s</b> may no longer be used. + */ +int +pubsub_unsubscribe_(pubsub_topic_t *topic, + const pubsub_subscriber_t *s) +{ + tor_assert(! topic->locked); + smartlist_t *sl = topic->subscribers; + if (sl == NULL) + return -1; + int i = smartlist_pos(sl, s); + if (i == -1) + return -1; + pubsub_subscriber_t *tmp = smartlist_get(sl, i); + tor_assert(tmp == s); + smartlist_del_keeporder(sl, i); + tor_free(tmp); + return 0; +} + +/** + * For every subscriber s in <b>topic</b>, invoke notify_fn on s and + * event_data. Return 0 if there were no nonzero return values, and -1 if + * there were any. + */ +int +pubsub_notify_(pubsub_topic_t *topic, pubsub_notify_fn_t notify_fn, + void *event_data, unsigned notify_flags) +{ + tor_assert(! topic->locked); + (void) notify_flags; + smartlist_t *sl = topic->subscribers; + int n_bad = 0; + ++topic->n_events_fired; + if (sl == NULL) + return -1; + topic->locked = 1; + SMARTLIST_FOREACH_BEGIN(sl, pubsub_subscriber_t *, s) { + int r = notify_fn(s, event_data); + if (r != 0) + ++n_bad; + } SMARTLIST_FOREACH_END(s); + topic->locked = 0; + return (n_bad == 0) ? 0 : -1; +} + +/** + * Release all storage held by <b>topic</b>. + */ +void +pubsub_clear_(pubsub_topic_t *topic) +{ + tor_assert(! topic->locked); + + smartlist_t *sl = topic->subscribers; + if (sl == NULL) + return; + SMARTLIST_FOREACH_BEGIN(sl, pubsub_subscriber_t *, s) { + tor_free(s); + } SMARTLIST_FOREACH_END(s); + smartlist_free(sl); + topic->subscribers = NULL; + topic->n_events_fired = 0; +} + diff --git a/src/common/pubsub.h b/src/common/pubsub.h new file mode 100644 index 0000000000..bbb4f02a42 --- /dev/null +++ b/src/common/pubsub.h @@ -0,0 +1,179 @@ +/* Copyright (c) 2016, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file pubsub.h + * \brief Macros to implement publish/subscribe abstractions. + * + * To use these macros, call DECLARE_PUBSUB_TOPIC() with an identifier to use + * as your topic. Below, I'm going to assume you say DECLARE_PUBSUB_TOPIC(T). + * + * Doing this will declare the following types: + * typedef struct T_event_data_t T_event_data_t; // you define this struct + * typedef struct T_subscriber_data_t T_subscriber_data_t; // this one too. + * typedef struct T_subscriber_t T_subscriber_t; // opaque + * typedef int (*T_subscriber_fn_t)(T_event_data_t*, T_subscriber_data_t*); + * + * and it will declare the following functions: + * const T_subscriber_t *T_subscribe(T_subscriber_fn_t, + * T_subscriber_data_t *, + * unsigned flags, + * unsigned priority); + * int T_unsubscribe(const T_subscriber_t *) + * + * Elsewhere you can say DECLARE_NOTIFY_PUBSUB_TOPIC(static, T), which + * declares: + * + * static int T_notify(T_event_data_t *, unsigned notify_flags); + * static void T_clear(void); + * + * And in some C file, you would define these functions with: + * IMPLEMENT_PUBSUB_TOPIC(static, T). + * + * The implementations will be small typesafe wrappers over generic versions + * of the above functions. + * + * To use the typesafe functions, you add any number of subscribers with + * T_subscribe(). Each has an associated function pointer, data pointer, + * and priority. Later, you can invoke T_notify() to declare that the + * event has occurred. Each of the subscribers will be invoked once. + **/ + +#ifndef TOR_PUBSUB_H +#define TOR_PUBSUB_H + +#include "torint.h" + +/** + * Flag for T_subscribe: die with an assertion failure if the event + * have ever been published before. Used when a subscriber must absolutely + * never have missed an event. + */ +#define SUBSCRIBE_ATSTART (1u<<0) + +#define DECLARE_PUBSUB_STRUCT_TYPES(name) \ + /* You define this type. */ \ + typedef struct name ## _event_data_t name ## _event_data_t; \ + /* You define this type. */ \ + typedef struct name ## _subscriber_data_t name ## _subscriber_data_t; + +#define DECLARE_PUBSUB_TOPIC(name) \ + /* This type is opaque. */ \ + typedef struct name ## _subscriber_t name ## _subscriber_t; \ + /* You declare functions matching this type. */ \ + typedef int (*name ## _subscriber_fn_t)( \ + name ## _event_data_t *data, \ + name ## _subscriber_data_t *extra); \ + /* Call this function to subscribe to a topic. */ \ + const name ## _subscriber_t *name ## _subscribe( \ + name##_subscriber_fn_t subscriber, \ + name##_subscriber_data_t *extra_data, \ + unsigned flags, \ + unsigned priority); \ + /* Call this function to unsubscribe from a topic. */ \ + int name ## _unsubscribe(const name##_subscriber_t *s); + +#define DECLARE_NOTIFY_PUBSUB_TOPIC(linkage, name) \ + /* Call this function to notify all subscribers. Flags not yet used. */ \ + linkage int name ## _notify(name ## _event_data_t *data, unsigned flags); \ + /* Call this function to release storage held by the topic. */ \ + linkage void name ## _clear(void); + +/** + * Type used to hold a generic function for a subscriber. + * + * [Yes, it is safe to cast to this, so long as we cast back to the original + * type before calling. From C99: "A pointer to a function of one type may be + * converted to a pointer to a function of another type and back again; the + * result shall compare equal to the original pointer."] +*/ +typedef int (*pubsub_subscriber_fn_t)(void *, void *); + +/** + * Helper type to implement pubsub abstraction. Don't use this directly. + * It represents a subscriber. + */ +typedef struct pubsub_subscriber_t { + /** Function to invoke when the event triggers. */ + pubsub_subscriber_fn_t fn; + /** Data associated with this subscriber. */ + void *subscriber_data; + /** Priority for this subscriber. Low priorities happen first. */ + unsigned priority; + /** Flags set on this subscriber. Not yet used.*/ + unsigned subscriber_flags; +} pubsub_subscriber_t; + +/** + * Helper type to implement pubsub abstraction. Don't use this directly. + * It represents a topic, and keeps a record of subscribers. + */ +typedef struct pubsub_topic_t { + /** List of subscribers to this topic. May be NULL. */ + struct smartlist_t *subscribers; + /** Total number of times that pubsub_notify_() has ever been called on this + * topic. */ + uint64_t n_events_fired; + /** True iff we're running 'notify' on this topic, and shouldn't allow + * any concurrent modifications or events. */ + unsigned locked; +} pubsub_topic_t; + +const pubsub_subscriber_t *pubsub_subscribe_(pubsub_topic_t *topic, + pubsub_subscriber_fn_t fn, + void *subscriber_data, + unsigned subscribe_flags, + unsigned priority); +int pubsub_unsubscribe_(pubsub_topic_t *topic, const pubsub_subscriber_t *sub); +void pubsub_clear_(pubsub_topic_t *topic); +typedef int (*pubsub_notify_fn_t)(pubsub_subscriber_t *subscriber, + void *notify_data); +int pubsub_notify_(pubsub_topic_t *topic, pubsub_notify_fn_t notify_fn, + void *notify_data, unsigned notify_flags); + +#define IMPLEMENT_PUBSUB_TOPIC(notify_linkage, name) \ + static pubsub_topic_t name ## _topic_ = { NULL, 0, 0 }; \ + const name ## _subscriber_t * \ + name ## _subscribe(name##_subscriber_fn_t subscriber, \ + name##_subscriber_data_t *extra_data, \ + unsigned flags, \ + unsigned priority) \ + { \ + const pubsub_subscriber_t *s; \ + s = pubsub_subscribe_(&name##_topic_, \ + (pubsub_subscriber_fn_t)subscriber, \ + extra_data, \ + flags, \ + priority); \ + return (const name##_subscriber_t *)s; \ + } \ + int \ + name ## _unsubscribe(const name##_subscriber_t *subscriber) \ + { \ + return pubsub_unsubscribe_(&name##_topic_, \ + (const pubsub_subscriber_t *)subscriber); \ + } \ + static int \ + name##_call_the_notify_fn_(pubsub_subscriber_t *subscriber, \ + void *notify_data) \ + { \ + name ## _subscriber_fn_t fn; \ + fn = (name ## _subscriber_fn_t) subscriber->fn; \ + return fn(notify_data, subscriber->subscriber_data); \ + } \ + notify_linkage int \ + name ## _notify(name ## _event_data_t *event_data, unsigned flags) \ + { \ + return pubsub_notify_(&name##_topic_, \ + name##_call_the_notify_fn_, \ + event_data, \ + flags); \ + } \ + notify_linkage void \ + name ## _clear(void) \ + { \ + pubsub_clear_(&name##_topic_); \ + } + +#endif /* TOR_PUBSUB_H */ + diff --git a/src/common/sandbox.c b/src/common/sandbox.c index 70c5bbd07c..94b2fc6788 100644 --- a/src/common/sandbox.c +++ b/src/common/sandbox.c @@ -39,8 +39,6 @@ #if defined(USE_LIBSECCOMP) -#define _GNU_SOURCE - #include <sys/mman.h> #include <sys/syscall.h> #include <sys/types.h> @@ -1443,7 +1441,7 @@ static HT_HEAD(getaddrinfo_cache, cached_getaddrinfo_item_t) HT_PROTOTYPE(getaddrinfo_cache, cached_getaddrinfo_item_t, node, cached_getaddrinfo_item_hash, - cached_getaddrinfo_items_eq); + cached_getaddrinfo_items_eq) HT_GENERATE2(getaddrinfo_cache, cached_getaddrinfo_item_t, node, cached_getaddrinfo_item_hash, cached_getaddrinfo_items_eq, diff --git a/src/common/sandbox.h b/src/common/sandbox.h index 2defd8bbd4..c5963e3119 100644 --- a/src/common/sandbox.h +++ b/src/common/sandbox.h @@ -39,12 +39,6 @@ typedef struct sandbox_cfg_elem sandbox_cfg_t; */ #ifdef USE_LIBSECCOMP -#ifndef __USE_GNU -#define __USE_GNU -#endif -#ifndef _GNU_SOURCE -#define _GNU_SOURCE -#endif #include <sys/ucontext.h> #include <seccomp.h> #include <netdb.h> diff --git a/src/common/timers.c b/src/common/timers.c new file mode 100644 index 0000000000..5d8d1feafd --- /dev/null +++ b/src/common/timers.c @@ -0,0 +1,297 @@ +/* Copyright (c) 2016, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file timers.c + * \brief Wrapper around William Ahern's fast hierarchical timer wheel + * implementation, to tie it in with a libevent backend. + * + * Only use these functions from the main thread. + * + * The main advantage of tor_timer_t over using libevent's timers is that + * they're way more efficient if we need to have thousands or millions of + * them. For more information, see + * http://www.25thandclement.com/~william/projects/timeout.c.html + * + * Periodic timers are available in the backend, but I've turned them off. + * We can turn them back on if needed. + */ + +/* Notes: + * + * The use of tor_gettimeofday_cached_monotonic() is kind of ugly. It would + * be neat to fix it. + * + * Having a way to free all timers on shutdown would free people from the + * need to track them. Not sure if that's clever though. + * + * In an ideal world, Libevent would just switch to use this backend, and we + * could throw this file away. But even if Libevent does switch, we'll be + * stuck with legacy libevents for some time. + */ + +#include "orconfig.h" + +#include "compat.h" +#include "compat_libevent.h" +#include "timers.h" +#include "torlog.h" +#include "util.h" + +#ifdef HAVE_EVENT2_EVENT_H +#include <event2/event.h> +#else +#include <event.h> +#endif + +struct timeout_cb { + timer_cb_fn_t cb; + void *arg; +}; + +/* + * These definitions are for timeouts.c and timeouts.h. + */ +#ifdef __GNUC__ +/* We're not exposing any of the functions outside this file. */ +#define TIMEOUT_PUBLIC __attribute__((__unused__)) static +#else +/* We're not exposing any of the functions outside this file. */ +#define TIMEOUT_PUBLIC static +#endif +/* We're not using periodic events. */ +#define TIMEOUT_DISABLE_INTERVALS +/* We always know the global_timeouts object, so we don't need each timeout + * to keep a pointer to it. */ +#define TIMEOUT_DISABLE_RELATIVE_ACCESS +/* We're providing our own struct timeout_cb. */ +#define TIMEOUT_CB_OVERRIDE +/* We're going to support timers that are pretty far out in advance. Making + * this big can be inefficient, but having a significant number of timers + * above TIMEOUT_MAX can also be super-inefficent. Choosing 5 here sets + * timeout_max to 2^30 ticks, or 29 hours with our value for USEC_PER_TICK */ +#define WHEEL_NUM 5 +#include "src/ext/timeouts/timeout.c" + +static struct timeouts *global_timeouts = NULL; +static struct event *global_timer_event = NULL; + +/** We need to choose this value carefully. Because we're using timer wheels, + * it actually costs us to have extra resolution we don't use. So for now, + * I'm going to define our resolution as .1 msec, and hope that's good enough. + * + * Note that two of the most popular libevent backends (epoll without timerfd, + * and windows select), simply can't support sub-millisecond resolution, + * do this is optimistic for a lot of users. + */ +#define USEC_PER_TICK 100 + +/** One million microseconds in a second */ +#define USEC_PER_SEC 1000000 + +/** Check at least once every N seconds. */ +#define MIN_CHECK_SECONDS 3600 + +/** Check at least once every N ticks. */ +#define MIN_CHECK_TICKS \ + (((timeout_t)MIN_CHECK_SECONDS) * (1000000 / USEC_PER_TICK)) + +/** + * Convert the timeval in <b>tv</b> to a timeout_t, and return it. + * + * The output resolution is set by USEC_PER_TICK, and the time corresponding + * to 0 is the same as the time corresponding to 0 from + * tor_gettimeofday_cached_monotonic(). + */ +static timeout_t +tv_to_timeout(const struct timeval *tv) +{ + uint64_t usec = tv->tv_usec; + usec += ((uint64_t)USEC_PER_SEC) * tv->tv_sec; + return usec / USEC_PER_TICK; +} + +/** + * Convert the timeout in <b>t</b> to a timeval in <b>tv_out</b> + */ +static void +timeout_to_tv(timeout_t t, struct timeval *tv_out) +{ + t *= USEC_PER_TICK; + tv_out->tv_usec = (int)(t % USEC_PER_SEC); + tv_out->tv_sec = (time_t)(t / USEC_PER_SEC); +} + +/** + * Update the timer <b>tv</b> to the current time in <b>tv</b>. + */ +static void +timer_advance_to_cur_time(const struct timeval *tv) +{ + timeout_t cur_tick = tv_to_timeout(tv); + if (BUG(cur_tick < timeouts_get_curtime(global_timeouts))) { + cur_tick = timeouts_get_curtime(global_timeouts); // LCOV_EXCL_LINE + } + timeouts_update(global_timeouts, cur_tick); +} + +/** + * Adjust the time at which the libevent timer should fire based on + * the next-expiring time in <b>global_timeouts</b> + */ +static void +libevent_timer_reschedule(void) +{ + struct timeval now; + tor_gettimeofday_cached_monotonic(&now); + timer_advance_to_cur_time(&now); + + timeout_t delay = timeouts_timeout(global_timeouts); + struct timeval d; + if (delay > MIN_CHECK_TICKS) + delay = MIN_CHECK_TICKS; + timeout_to_tv(delay, &d); + event_add(global_timer_event, &d); +} + +/** + * Invoked when the libevent timer has expired: see which tor_timer_t events + * have fired, activate their callbacks, and reschedule the libevent timer. + */ +static void +libevent_timer_callback(evutil_socket_t fd, short what, void *arg) +{ + (void)fd; + (void)what; + (void)arg; + + struct timeval now; + tor_gettimeofday_cache_clear(); + tor_gettimeofday_cached_monotonic(&now); + timer_advance_to_cur_time(&now); + + tor_timer_t *t; + while ((t = timeouts_get(global_timeouts))) { + t->callback.cb(t, t->callback.arg, &now); + } + + tor_gettimeofday_cache_clear(); + libevent_timer_reschedule(); +} + +/** + * Initialize the timers subsystem. Requires that libevent has already been + * initialized. + */ +void +timers_initialize(void) +{ + if (BUG(global_timeouts)) + return; // LCOV_EXCL_LINE + + timeout_error_t err; + global_timeouts = timeouts_open(0, &err); + if (!global_timeouts) { + // LCOV_EXCL_START -- this can only fail on malloc failure. + log_err(LD_BUG, "Unable to open timer backend: %s", strerror(err)); + tor_assert(0); + // LCOV_EXCL_STOP + } + + struct event *timer_event; + timer_event = tor_event_new(tor_libevent_get_base(), + -1, 0, libevent_timer_callback, NULL); + tor_assert(timer_event); + global_timer_event = timer_event; + + libevent_timer_reschedule(); +} + +/** + * Release all storage held in the timers subsystem. Does not fire timers. + */ +void +timers_shutdown(void) +{ + if (global_timer_event) { + tor_event_free(global_timer_event); + global_timer_event = NULL; + } + if (global_timeouts) { + timeouts_close(global_timeouts); + global_timeouts = NULL; + } +} + +/** + * Allocate and return a new timer, with given callback and argument. + */ +tor_timer_t * +timer_new(timer_cb_fn_t cb, void *arg) +{ + tor_timer_t *t = tor_malloc(sizeof(tor_timer_t)); + timeout_init(t, 0); + timer_set_cb(t, cb, arg); + return t; +} + +/** + * Release all storage held by <b>t</b>, and unschedule it if was already + * scheduled. + */ +void +timer_free(tor_timer_t *t) +{ + if (! t) + return; + + timeouts_del(global_timeouts, t); + tor_free(t); +} + +/** + * Change the callback and argument associated with a timer <b>t</b>. + */ +void +timer_set_cb(tor_timer_t *t, timer_cb_fn_t cb, void *arg) +{ + t->callback.cb = cb; + t->callback.arg = arg; +} + +/** + * Schedule the timer t to fire at the current time plus a delay of <b>tv</b>. + * All times are relative to tor_gettimeofday_cached_monotonic. + */ +void +timer_schedule(tor_timer_t *t, const struct timeval *tv) +{ + const timeout_t when = tv_to_timeout(tv); + struct timeval now; + tor_gettimeofday_cached_monotonic(&now); + timer_advance_to_cur_time(&now); + + /* Take the old timeout value. */ + timeout_t to = timeouts_timeout(global_timeouts); + + timeouts_add(global_timeouts, t, when); + + /* Should we update the libevent timer? */ + if (to <= when) { + return; /* we're already going to fire before this timer would trigger. */ + } + libevent_timer_reschedule(); +} + +/** + * Cancel the timer <b>t</b> if it is currently scheduled. (It's okay to call + * this on an unscheduled timer. + */ +void +timer_disable(tor_timer_t *t) +{ + timeouts_del(global_timeouts, t); + /* We don't reschedule the libevent timer here, since it's okay if it fires + * early. */ +} + diff --git a/src/common/timers.h b/src/common/timers.h new file mode 100644 index 0000000000..594cf38a64 --- /dev/null +++ b/src/common/timers.h @@ -0,0 +1,22 @@ +/* Copyright (c) 2016, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_TIMERS_H +#define TOR_TIMERS_H + +#include "orconfig.h" +#include "testsupport.h" + +typedef struct timeout tor_timer_t; +typedef void (*timer_cb_fn_t)(tor_timer_t *, void *, const struct timeval *); +tor_timer_t *timer_new(timer_cb_fn_t cb, void *arg); +void timer_set_cb(tor_timer_t *t, timer_cb_fn_t cb, void *arg); +void timer_schedule(tor_timer_t *t, const struct timeval *delay); +void timer_disable(tor_timer_t *t); +void timer_free(tor_timer_t *t); + +void timers_initialize(void); +void timers_shutdown(void); + +#endif + diff --git a/src/common/torgzip.c b/src/common/torgzip.c index 71e55f8723..331bb5a017 100644 --- a/src/common/torgzip.c +++ b/src/common/torgzip.c @@ -46,34 +46,16 @@ #include <zlib.h> +#if defined ZLIB_VERNUM && ZLIB_VERNUM < 0x1200 +#error "We require zlib version 1.2 or later." +#endif + static size_t tor_zlib_state_size_precalc(int inflate, int windowbits, int memlevel); /** Total number of bytes allocated for zlib state */ static size_t total_zlib_allocation = 0; -/** Set to 1 if zlib is a version that supports gzip; set to 0 if it doesn't; - * set to -1 if we haven't checked yet. */ -static int gzip_is_supported = -1; - -/** Return true iff we support gzip-based compression. Otherwise, we need to - * use zlib. */ -int -is_gzip_supported(void) -{ - if (gzip_is_supported >= 0) - return gzip_is_supported; - - if (!strcmpstart(ZLIB_VERSION, "0.") || - !strcmpstart(ZLIB_VERSION, "1.0") || - !strcmpstart(ZLIB_VERSION, "1.1")) - gzip_is_supported = 0; - else - gzip_is_supported = 1; - - return gzip_is_supported; -} - /** Return a string representation of the version of the currently running * version of zlib. */ const char * @@ -165,12 +147,6 @@ tor_gzip_compress(char **out, size_t *out_len, *out = NULL; - if (method == GZIP_METHOD && !is_gzip_supported()) { - /* Old zlib version don't support gzip in deflateInit2 */ - log_warn(LD_BUG, "Gzip not supported with zlib %s", ZLIB_VERSION); - goto err; - } - stream = tor_malloc_zero(sizeof(struct z_stream_s)); stream->zalloc = Z_NULL; stream->zfree = Z_NULL; @@ -182,9 +158,11 @@ tor_gzip_compress(char **out, size_t *out_len, method_bits(method, HIGH_COMPRESSION), get_memlevel(HIGH_COMPRESSION), Z_DEFAULT_STRATEGY) != Z_OK) { + //LCOV_EXCL_START -- we can only provoke failure by giving junk arguments. log_warn(LD_GENERAL, "Error from deflateInit2: %s", stream->msg?stream->msg:"<no message>"); goto err; + //LCOV_EXCL_STOP } /* Guess 50% compression. */ @@ -237,13 +215,12 @@ tor_gzip_compress(char **out, size_t *out_len, * the newly unsigned field isn't negative." */ tor_assert(stream->total_out >= 0); #endif - if (((size_t)stream->total_out) > out_size + 4097) { - /* If we're wasting more than 4k, don't. */ - *out = tor_realloc(*out, stream->total_out + 1); - } if (deflateEnd(stream)!=Z_OK) { + // LCOV_EXCL_START -- unreachable if we handled the zlib structure right + tor_assert_nonfatal_unreached(); log_warn(LD_BUG, "Error freeing gzip structures"); goto err; + // LCOV_EXCL_STOP } tor_free(stream); @@ -291,12 +268,6 @@ tor_gzip_uncompress(char **out, size_t *out_len, tor_assert(in); tor_assert(in_len < UINT_MAX); - if (method == GZIP_METHOD && !is_gzip_supported()) { - /* Old zlib version don't support gzip in inflateInit2 */ - log_warn(LD_BUG, "Gzip not supported with zlib %s", ZLIB_VERSION); - return -1; - } - *out = NULL; stream = tor_malloc_zero(sizeof(struct z_stream_s)); @@ -308,9 +279,11 @@ tor_gzip_uncompress(char **out, size_t *out_len, if (inflateInit2(stream, method_bits(method, HIGH_COMPRESSION)) != Z_OK) { + // LCOV_EXCL_START -- can only hit this if we give bad inputs. log_warn(LD_GENERAL, "Error from inflateInit2: %s", stream->msg?stream->msg:"<no message>"); goto err; + // LCOV_EXCL_STOP } out_size = in_len * 2; /* guess 50% compression. */ @@ -451,12 +424,6 @@ tor_zlib_new(int compress, compress_method_t method, tor_zlib_state_t *out; int bits, memlevel; - if (method == GZIP_METHOD && !is_gzip_supported()) { - /* Old zlib version don't support gzip in inflateInit2 */ - log_warn(LD_BUG, "Gzip not supported with zlib %s", ZLIB_VERSION); - return NULL; - } - if (! compress) { /* use this setting for decompression, since we might have the * max number of window bits */ @@ -474,10 +441,10 @@ tor_zlib_new(int compress, compress_method_t method, if (deflateInit2(&out->stream, Z_BEST_COMPRESSION, Z_DEFLATED, bits, memlevel, Z_DEFAULT_STRATEGY) != Z_OK) - goto err; + goto err; // LCOV_EXCL_LINE } else { if (inflateInit2(&out->stream, bits) != Z_OK) - goto err; + goto err; // LCOV_EXCL_LINE } out->allocation = tor_zlib_state_size_precalc(!compress, bits, memlevel); diff --git a/src/common/torlog.h b/src/common/torlog.h index 578af7caea..80f37e0e48 100644 --- a/src/common/torlog.h +++ b/src/common/torlog.h @@ -176,7 +176,7 @@ void log_fn_ratelim_(struct ratelim_t *ratelim, int severity, const char *format, ...) CHECK_PRINTF(5,6); -#if defined(__GNUC__) +#if defined(__GNUC__) && __GNUC__ <= 3 /* These are the GCC varidaic macros, so that older versions of GCC don't * break. */ diff --git a/src/common/tortls.c b/src/common/tortls.c index b68f5dfcdf..0395205228 100644 --- a/src/common/tortls.c +++ b/src/common/tortls.c @@ -24,18 +24,11 @@ #include <ws2tcpip.h> #endif -#ifdef __GNUC__ -#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -#endif +#include "compat.h" -#if __GNUC__ && GCC_VERSION >= 402 -#if GCC_VERSION >= 406 -#pragma GCC diagnostic push -#endif /* Some versions of OpenSSL declare SSL_get_selected_srtp_profile twice in * srtp.h. Suppress the GCC warning so we can build with -Wredundant-decl. */ -#pragma GCC diagnostic ignored "-Wredundant-decls" -#endif +DISABLE_GCC_WARNING(redundant-decls) #include <openssl/opensslv.h> #include "crypto.h" @@ -53,13 +46,7 @@ #include <openssl/bn.h> #include <openssl/rsa.h> -#if __GNUC__ && GCC_VERSION >= 402 -#if GCC_VERSION >= 406 -#pragma GCC diagnostic pop -#else -#pragma GCC diagnostic warning "-Wredundant-decls" -#endif -#endif +ENABLE_GCC_WARNING(redundant-decls) #ifdef USE_BUFFEREVENTS #include <event2/bufferevent_ssl.h> @@ -577,7 +564,7 @@ MOCK_IMPL(STATIC X509 *, /** List of ciphers that servers should select from when we actually have * our choice of what cipher to use. */ -const char UNRESTRICTED_SERVER_CIPHER_LIST[] = +static const char UNRESTRICTED_SERVER_CIPHER_LIST[] = /* This list is autogenerated with the gen_server_ciphers.py script; * don't hand-edit it. */ #ifdef TLS1_TXT_ECDHE_RSA_WITH_AES_256_GCM_SHA384 diff --git a/src/common/tortls.h b/src/common/tortls.h index 1a59c67df3..b6ab2ec8f5 100644 --- a/src/common/tortls.h +++ b/src/common/tortls.h @@ -164,8 +164,18 @@ STATIC int tor_tls_context_init_one(tor_tls_context_t **ppcontext, int is_client); STATIC void tls_log_errors(tor_tls_t *tls, int severity, int domain, const char *doing); + +#ifdef TOR_UNIT_TESTS +extern int tor_tls_object_ex_data_index; +extern tor_tls_context_t *server_tls_context; +extern tor_tls_context_t *client_tls_context; +extern uint16_t v2_cipher_list[]; +extern uint64_t total_bytes_written_over_tls; +extern uint64_t total_bytes_written_by_tls; #endif +#endif /* endif TORTLS_PRIVATE */ + const char *tor_tls_err_to_string(int err); void tor_tls_get_state_description(tor_tls_t *tls, char *buf, size_t sz); diff --git a/src/common/util.c b/src/common/util.c index f3effe0957..4b6df81b7d 100644 --- a/src/common/util.c +++ b/src/common/util.c @@ -9,10 +9,6 @@ * process control. **/ -/* This is required on rh7 to make strptime not complain. - */ -#define _GNU_SOURCE - #include "orconfig.h" #ifdef HAVE_FCNTL_H #include <fcntl.h> @@ -105,23 +101,6 @@ #endif /* ===== - * Assertion helper. - * ===== */ -/** Helper for tor_assert: report the assertion failure. */ -void -tor_assertion_failed_(const char *fname, unsigned int line, - const char *func, const char *expr) -{ - char buf[256]; - log_err(LD_BUG, "%s:%u: %s: Assertion %s failed; aborting.", - fname, line, func, expr); - tor_snprintf(buf, sizeof(buf), - "Assertion %s failed in %s at %s:%u", - expr, func, fname, line); - log_backtrace(LOG_ERR, LD_BUG, buf); -} - -/* ===== * Memory management * ===== */ #ifdef USE_DMALLOC @@ -172,11 +151,13 @@ tor_malloc_(size_t size DMALLOC_PARAMS) #endif if (PREDICT_UNLIKELY(result == NULL)) { + /* LCOV_EXCL_START */ log_err(LD_MM,"Out of memory on malloc(). Dying."); /* If these functions die within a worker process, they won't call * spawn_exit, but that's ok, since the parent will run out of memory soon * anyway. */ exit(1); + /* LCOV_EXCL_STOP */ } return result; } @@ -221,6 +202,15 @@ size_mul_check(const size_t x, const size_t y) x <= SIZE_MAX / y); } +#ifdef TOR_UNIT_TESTS +/** Exposed for unit tests only */ +int +size_mul_check__(const size_t x, const size_t y) +{ + return size_mul_check(x,y); +} +#endif + /** Allocate a chunk of <b>nmemb</b>*<b>size</b> bytes of memory, fill * the memory with zero bytes, and return a pointer to the result. * Log and terminate the process on error. (Same as @@ -260,8 +250,10 @@ tor_realloc_(void *ptr, size_t size DMALLOC_PARAMS) #endif if (PREDICT_UNLIKELY(result == NULL)) { + /* LCOV_EXCL_START */ log_err(LD_MM,"Out of memory on realloc(). Dying."); exit(1); + /* LCOV_EXCL_STOP */ } return result; } @@ -296,8 +288,10 @@ tor_strdup_(const char *s DMALLOC_PARAMS) dup = strdup(s); #endif if (PREDICT_UNLIKELY(dup == NULL)) { + /* LCOV_EXCL_START */ log_err(LD_MM,"Out of memory on strdup(). Dying."); exit(1); + /* LCOV_EXCL_STOP */ } return dup; } @@ -359,6 +353,7 @@ tor_free_(void *mem) tor_free(mem); } +DISABLE_GCC_WARNING(aggregate-return) /** Call the platform malloc info function, and dump the results to the log at * level <b>severity</b>. If no such function exists, do nothing. */ void @@ -386,6 +381,7 @@ tor_log_mallinfo(int severity) ); #endif } +ENABLE_GCC_WARNING(aggregate-return) /* ===== * Math @@ -530,21 +526,6 @@ round_uint64_to_next_multiple_of(uint64_t number, uint64_t divisor) return number; } -/** Return the lowest x in [INT64_MIN, INT64_MAX] such that x is at least - * <b>number</b>, and x modulo <b>divisor</b> == 0. If no such x can be - * expressed as an int64_t, return INT64_MAX */ -int64_t -round_int64_to_next_multiple_of(int64_t number, int64_t divisor) -{ - tor_assert(divisor > 0); - if (INT64_MAX - divisor + 1 < number) - return INT64_MAX; - if (number >= 0) - number += divisor - 1; - number -= number % divisor; - return number; -} - /** Transform a random value <b>p</b> from the uniform distribution in * [0.0, 1.0[ into a Laplace distributed value with location parameter * <b>mu</b> and scale parameter <b>b</b>. Truncate the final result @@ -1130,6 +1111,9 @@ tor_digest256_is_zero(const char *digest) /* Were there unexpected unconverted characters? */ \ if (!next && *endptr) \ goto err; \ + /* Illogical (max, min) inputs? */ \ + if (BUG(max < min)) \ + goto err; \ /* Is r within limits? */ \ if (r < min || r > max) \ goto err; \ @@ -1410,7 +1394,7 @@ tv_udiff(const struct timeval *start, const struct timeval *end) long udiff; long secdiff = end->tv_sec - start->tv_sec; - if (labs(secdiff+1) > LONG_MAX/1000000) { + if (labs(secdiff)+1 > LONG_MAX/1000000) { log_warn(LD_GENERAL, "comparing times on microsecond detail too far " "apart: %ld seconds", secdiff); return LONG_MAX; @@ -1428,7 +1412,7 @@ tv_mdiff(const struct timeval *start, const struct timeval *end) long mdiff; long secdiff = end->tv_sec - start->tv_sec; - if (labs(secdiff+1) > LONG_MAX/1000) { + if (labs(secdiff)+1 > LONG_MAX/1000) { log_warn(LD_GENERAL, "comparing times on millisecond detail too far " "apart: %ld seconds", secdiff); return LONG_MAX; @@ -1436,7 +1420,13 @@ tv_mdiff(const struct timeval *start, const struct timeval *end) /* Subtract and round */ mdiff = secdiff*1000L + - ((long)end->tv_usec - (long)start->tv_usec + 500L) / 1000L; + /* We add a million usec here to ensure that the result is positive, + * so that the round-towards-zero behavior of the division will give + * the right result for rounding to the nearest msec. Later we subtract + * 1000 in order to get the correct result. + */ + ((long)end->tv_usec - (long)start->tv_usec + 500L + 1000000L) / 1000L + - 1000; return mdiff; } @@ -1638,11 +1628,16 @@ parse_rfc1123_time(const char *buf, time_t *t) tm.tm_sec = (int)tm_sec; if (tm.tm_year < 1970) { + /* LCOV_EXCL_START + * XXXX I think this is dead code; we already checked for + * invalid_year above. */ + tor_assert_nonfatal_unreached(); char *esc = esc_for_log(buf); log_warn(LD_GENERAL, "Got invalid RFC1123 time %s. (Before 1970)", esc); tor_free(esc); return -1; + /* LCOV_EXCL_STOP */ } tm.tm_year -= 1900; @@ -1726,10 +1721,15 @@ parse_iso_time_(const char *cp, time_t *t, int strict) st_tm.tm_wday = 0; /* Should be ignored. */ if (st_tm.tm_year < 70) { + /* LCOV_EXCL_START + * XXXX I think this is dead code; we already checked for + * year < 1970 above. */ + tor_assert_nonfatal_unreached(); char *esc = esc_for_log(cp); log_warn(LD_GENERAL, "Got invalid ISO time %s. (Before 1970)", esc); tor_free(esc); return -1; + /* LCOV_EXCL_STOP */ } return tor_timegm(&st_tm, t); } @@ -1920,6 +1920,8 @@ rate_limit_log(ratelim_t *lim, time_t now) return tor_strdup(""); } else { char *cp=NULL; + /* XXXX this is not exactly correct: the messages could have occurred + * any time between the old value of lim->allowed and now. */ tor_asprintf(&cp, " [%d similar message(s) suppressed in last %d seconds]", n-1, lim->rate); @@ -2820,9 +2822,11 @@ unescape_string(const char *s, char **result, size_t *size_out) if (size_out) *size_out = out - *result; return cp+1; case '\0': + /* LCOV_EXCL_START -- we caught this in parse_config_from_line. */ tor_fragile_assert(); tor_free(*result); return NULL; + /* LCOV_EXCL_STOP */ case '\\': switch (cp[1]) { @@ -2836,8 +2840,12 @@ unescape_string(const char *s, char **result, size_t *size_out) x1 = hex_decode_digit(cp[2]); x2 = hex_decode_digit(cp[3]); if (x1 == -1 || x2 == -1) { - tor_free(*result); - return NULL; + /* LCOV_EXCL_START */ + /* we caught this above in the initial loop. */ + tor_assert_nonfatal_unreached(); + tor_free(*result); + return NULL; + /* LCOV_EXCL_STOP */ } *out++ = ((x1<<4) + x2); @@ -2863,7 +2871,11 @@ unescape_string(const char *s, char **result, size_t *size_out) cp += 2; break; default: + /* LCOV_EXCL_START */ + /* we caught this above in the initial loop. */ + tor_assert_nonfatal_unreached(); tor_free(*result); return NULL; + /* LCOV_EXCL_STOP */ } break; default: @@ -3079,7 +3091,7 @@ digit_to_num(char d) * success, store the result in <b>out</b>, advance bufp to the next * character, and return 0. On failure, return -1. */ static int -scan_unsigned(const char **bufp, unsigned long *out, int width, int base) +scan_unsigned(const char **bufp, unsigned long *out, int width, unsigned base) { unsigned long result = 0; int scanned_so_far = 0; @@ -3092,7 +3104,7 @@ scan_unsigned(const char **bufp, unsigned long *out, int width, int base) while (**bufp && (hex?TOR_ISXDIGIT(**bufp):TOR_ISDIGIT(**bufp)) && scanned_so_far < width) { - int digit = hex?hex_decode_digit(*(*bufp)++):digit_to_num(*(*bufp)++); + unsigned digit = hex?hex_decode_digit(*(*bufp)++):digit_to_num(*(*bufp)++); // Check for overflow beforehand, without actually causing any overflow // This preserves functionality on compilers that don't wrap overflow // (i.e. that trap or optimise away overflow) @@ -3138,14 +3150,15 @@ scan_signed(const char **bufp, long *out, int width) if (neg && result > 0) { if (result > ((unsigned long)LONG_MAX) + 1) return -1; /* Underflow */ - // Avoid overflow on the cast to signed long when result is LONG_MIN - // by subtracting 1 from the unsigned long positive value, - // then, after it has been cast to signed and negated, - // subtracting the original 1 (the double-subtraction is intentional). - // Otherwise, the cast to signed could cause a temporary long - // to equal LONG_MAX + 1, which is undefined. - // We avoid underflow on the subtraction by treating -0 as positive. - *out = (-(long)(result - 1)) - 1; + else if (result == ((unsigned long)LONG_MAX) + 1) + *out = LONG_MIN; + else { + /* We once had a far more clever no-overflow conversion here, but + * some versions of GCC apparently ran it into the ground. Now + * we just check for LONG_MIN explicitly. + */ + *out = -(long)result; + } } else { if (result > LONG_MAX) return -1; /* Overflow */ @@ -3291,8 +3304,10 @@ tor_vsscanf(const char *buf, const char *pattern, va_list ap) *out = lng; } else { int *out = va_arg(ap, int *); +#if LONG_MAX > INT_MAX if (lng < INT_MIN || lng > INT_MAX) return n_matched; +#endif *out = (int)lng; } ++pattern; @@ -3495,13 +3510,17 @@ start_daemon(void) start_daemon_called = 1; if (pipe(daemon_filedes)) { + /* LCOV_EXCL_START */ log_err(LD_GENERAL,"pipe failed; exiting. Error was %s", strerror(errno)); exit(1); + /* LCOV_EXCL_STOP */ } pid = fork(); if (pid < 0) { + /* LCOV_EXCL_START */ log_err(LD_GENERAL,"fork failed. Exiting."); exit(1); + /* LCOV_EXCL_STOP */ } if (pid) { /* Parent */ int ok; @@ -3563,8 +3582,10 @@ finish_daemon(const char *desired_cwd) nullfd = tor_open_cloexec("/dev/null", O_RDWR, 0); if (nullfd < 0) { + /* LCOV_EXCL_START */ log_err(LD_GENERAL,"/dev/null can't be opened. Exiting."); exit(1); + /* LCOV_EXCL_STOP */ } /* close fds linking to invoking terminal, but * close usual incoming fds, but redirect them somewhere @@ -3573,8 +3594,10 @@ finish_daemon(const char *desired_cwd) if (dup2(nullfd,0) < 0 || dup2(nullfd,1) < 0 || dup2(nullfd,2) < 0) { + /* LCOV_EXCL_START */ log_err(LD_GENERAL,"dup2 failed. Exiting."); exit(1); + /* LCOV_EXCL_STOP */ } if (nullfd > 2) close(nullfd); @@ -3771,7 +3794,7 @@ format_number_sigsafe(unsigned long x, char *buf, int buf_len, /* NOT tor_assert; see above. */ if (cp != buf) { - abort(); + abort(); // LCOV_EXCL_LINE } return len; @@ -4349,7 +4372,7 @@ tor_spawn_background(const char *const filename, const char **argv, _exit(255); /* Never reached, but avoids compiler warning */ - return status; + return status; // LCOV_EXCL_LINE } /* In parent */ @@ -5557,7 +5580,7 @@ clamp_double_to_int64(double number) * representable integer for which this is not the case is INT64_MIN, but * it is covered by the logic below. */ if (isfinite(number) && exp <= 63) { - return number; + return (int64_t)number; } /* Handle infinities and finite numbers with magnitude >= 2^63. */ diff --git a/src/common/util.h b/src/common/util.h index ebcf88b32d..7cb33dc680 100644 --- a/src/common/util.h +++ b/src/common/util.h @@ -22,6 +22,7 @@ /* for the correct alias to struct stat */ #include <sys/stat.h> #endif +#include "util_bug.h" #ifndef O_BINARY #define O_BINARY 0 @@ -33,41 +34,6 @@ #define O_NOFOLLOW 0 #endif -/* Replace assert() with a variant that sends failures to the log before - * calling assert() normally. - */ -#ifdef NDEBUG -/* Nobody should ever want to build with NDEBUG set. 99% of our asserts will - * be outside the critical path anyway, so it's silly to disable bug-checking - * throughout the entire program just because a few asserts are slowing you - * down. Profile, optimize the critical path, and keep debugging on. - * - * And I'm not just saying that because some of our asserts check - * security-critical properties. - */ -#error "Sorry; we don't support building with NDEBUG." -#endif - -/* Sometimes we don't want to use assertions during branch coverage tests; it - * leads to tons of unreached branches which in reality are only assertions we - * didn't hit. */ -#if defined(TOR_UNIT_TESTS) && defined(DISABLE_ASSERTS_IN_UNIT_TESTS) -#define tor_assert(a) STMT_BEGIN \ - (void)(a); \ - STMT_END -#else -/** Like assert(3), but send assertion failures to the log as well as to - * stderr. */ -#define tor_assert(expr) STMT_BEGIN \ - if (PREDICT_UNLIKELY(!(expr))) { \ - tor_assertion_failed_(SHORT_FILE__, __LINE__, __func__, #expr); \ - abort(); \ - } STMT_END -#endif - -void tor_assertion_failed_(const char *fname, unsigned int line, - const char *func, const char *expr); - /* If we're building with dmalloc, we want all of our memory allocation * functions to take an extra file/line pair of arguments. If not, not. * We define DMALLOC_PARAMS to the extra parameters to insert in the @@ -81,11 +47,6 @@ void tor_assertion_failed_(const char *fname, unsigned int line, #define DMALLOC_ARGS #endif -/** Define this if you want Tor to crash when any problem comes up, - * so you can get a coredump and track things down. */ -// #define tor_fragile_assert() tor_assert(0) -#define tor_fragile_assert() - /* Memory management */ void *tor_malloc_(size_t size DMALLOC_PARAMS) ATTR_MALLOC; void *tor_malloc_zero_(size_t size DMALLOC_PARAMS) ATTR_MALLOC; @@ -184,7 +145,6 @@ uint64_t round_to_power_of_2(uint64_t u64); unsigned round_to_next_multiple_of(unsigned number, unsigned divisor); uint32_t round_uint32_to_next_multiple_of(uint32_t number, uint32_t divisor); uint64_t round_uint64_to_next_multiple_of(uint64_t number, uint64_t divisor); -int64_t round_int64_to_next_multiple_of(int64_t number, int64_t divisor); int64_t sample_laplace_distribution(double mu, double b, double p); int64_t add_laplace_noise(int64_t signal, double random, double delta_f, double epsilon); @@ -411,8 +371,6 @@ char *read_file_to_str_until_eof(int fd, size_t max_bytes_to_read, const char *parse_config_line_from_str_verbose(const char *line, char **key_out, char **value_out, const char **err_out); -#define parse_config_line_from_str(line,key_out,value_out) \ - parse_config_line_from_str_verbose((line),(key_out),(value_out),NULL) char *expand_filename(const char *filename); struct smartlist_t *tor_listdir(const char *dirname); int path_is_relative(const char *filename); @@ -575,6 +533,10 @@ STATIC int format_helper_exit_status(unsigned char child_state, #endif +#ifdef TOR_UNIT_TESTS +int size_mul_check__(const size_t x, const size_t y); +#endif + #define ARRAY_LENGTH(x) ((sizeof(x)) / sizeof(x[0])) #endif diff --git a/src/common/util_bug.c b/src/common/util_bug.c new file mode 100644 index 0000000000..e3e1d6df90 --- /dev/null +++ b/src/common/util_bug.c @@ -0,0 +1,53 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2016, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file util_bug.c + **/ + +#include "orconfig.h" +#include "util_bug.h" +#include "torlog.h" +#include "backtrace.h" + +/** Helper for tor_assert: report the assertion failure. */ +void +tor_assertion_failed_(const char *fname, unsigned int line, + const char *func, const char *expr) +{ + char buf[256]; + log_err(LD_BUG, "%s:%u: %s: Assertion %s failed; aborting.", + fname, line, func, expr); + tor_snprintf(buf, sizeof(buf), + "Assertion %s failed in %s at %s:%u", + expr, func, fname, line); + log_backtrace(LOG_ERR, LD_BUG, buf); +} + +/** Helper for tor_assert_nonfatal: report the assertion failure. */ +void +tor_bug_occurred_(const char *fname, unsigned int line, + const char *func, const char *expr, + int once) +{ + char buf[256]; + const char *once_str = once ? + " (Future instances of this warning will be silenced.)": ""; + if (! expr) { + log_warn(LD_BUG, "%s:%u: %s: This line should not have been reached.%s", + fname, line, func, once_str); + tor_snprintf(buf, sizeof(buf), + "Line unexpectedly reached at %s at %s:%u", + func, fname, line); + } else { + log_warn(LD_BUG, "%s:%u: %s: Non-fatal assertion %s failed.%s", + fname, line, func, expr, once_str); + tor_snprintf(buf, sizeof(buf), + "Non-fatal assertion %s failed in %s at %s:%u", + expr, func, fname, line); + } + log_backtrace(LOG_WARN, LD_BUG, buf); +} + diff --git a/src/common/util_bug.h b/src/common/util_bug.h new file mode 100644 index 0000000000..3f77e0a99e --- /dev/null +++ b/src/common/util_bug.h @@ -0,0 +1,150 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2016, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file util_bug.h + **/ + +#ifndef TOR_UTIL_BUG_H +#define TOR_UTIL_BUG_H + +#include "orconfig.h" +#include "compat.h" +#include "testsupport.h" + +/* Replace assert() with a variant that sends failures to the log before + * calling assert() normally. + */ +#ifdef NDEBUG +/* Nobody should ever want to build with NDEBUG set. 99% of our asserts will + * be outside the critical path anyway, so it's silly to disable bug-checking + * throughout the entire program just because a few asserts are slowing you + * down. Profile, optimize the critical path, and keep debugging on. + * + * And I'm not just saying that because some of our asserts check + * security-critical properties. + */ +#error "Sorry; we don't support building with NDEBUG." +#endif + +/* Sometimes we don't want to use assertions during branch coverage tests; it + * leads to tons of unreached branches which in reality are only assertions we + * didn't hit. */ +#if defined(TOR_UNIT_TESTS) && defined(DISABLE_ASSERTS_IN_UNIT_TESTS) +#define tor_assert(a) STMT_BEGIN \ + (void)(a); \ + STMT_END +#else +/** Like assert(3), but send assertion failures to the log as well as to + * stderr. */ +#define tor_assert(expr) STMT_BEGIN \ + if (PREDICT_UNLIKELY(!(expr))) { \ + tor_assertion_failed_(SHORT_FILE__, __LINE__, __func__, #expr); \ + abort(); \ + } STMT_END +#endif + +#define tor_assert_unreached() tor_assert(0) + +/* Non-fatal bug assertions. The "unreached" variants mean "this line should + * never be reached." The "once" variants mean "Don't log a warning more than + * once". + * + * The 'BUG' macro checks a boolean condition and logs an error message if it + * is true. Example usage: + * if (BUG(x == NULL)) + * return -1; + */ + +#ifdef ALL_BUGS_ARE_FATAL +#define tor_assert_nonfatal_unreached() tor_assert(0) +#define tor_assert_nonfatal(cond) tor_assert((cond)) +#define tor_assert_nonfatal_unreached_once() tor_assert(0) +#define tor_assert_nonfatal_once(cond) tor_assert((cond)) +#define BUG(cond) \ + (PREDICT_UNLIKELY(cond) ? \ + (tor_assertion_failed_(SHORT_FILE__,__LINE__,__func__,#cond), abort(), 1) \ + : 0) +#elif defined(TOR_UNIT_TESTS) && defined(DISABLE_ASSERTS_IN_UNIT_TESTS) +#define tor_assert_nonfatal_unreached() STMT_NIL +#define tor_assert_nonfatal(cond) ((void)(cond)) +#define tor_assert_nonfatal_unreached_once() STMT_NIL +#define tor_assert_nonfatal_once(cond) ((void)(cond)) +#define BUG(cond) (PREDICT_UNLIKELY(cond) ? 1 : 0) +#else /* Normal case, !ALL_BUGS_ARE_FATAL, !DISABLE_ASSERTS_IN_UNIT_TESTS */ +#define tor_assert_nonfatal_unreached() STMT_BEGIN \ + tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, NULL, 0); \ + STMT_END +#define tor_assert_nonfatal(cond) STMT_BEGIN \ + if (PREDICT_UNLIKELY(!(cond))) { \ + tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, #cond, 0); \ + } \ + STMT_END +#define tor_assert_nonfatal_unreached_once() STMT_BEGIN \ + static int warning_logged__ = 0; \ + if (!warning_logged__) { \ + warning_logged__ = 1; \ + tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, NULL, 1); \ + } \ + STMT_END +#define tor_assert_nonfatal_once(cond) STMT_BEGIN \ + static int warning_logged__ = 0; \ + if (!warning_logged__ && PREDICT_UNLIKELY(!(cond))) { \ + warning_logged__ = 1; \ + tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, #cond, 1); \ + } \ + STMT_END +#define BUG(cond) \ + (PREDICT_UNLIKELY(cond) ? \ + (tor_bug_occurred_(SHORT_FILE__,__LINE__,__func__,#cond,0), 1) \ + : 0) +#endif + +#ifdef __GNUC__ +#define IF_BUG_ONCE__(cond,var) \ + if (( { \ + static int var = 0; \ + int bool_result = (cond); \ + if (PREDICT_UNLIKELY(bool_result) && !var) { \ + var = 1; \ + tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, #cond, 1); \ + } \ + PREDICT_UNLIKELY(bool_result); } )) +#else +#define IF_BUG_ONCE__(cond,var) \ + static int var = 0; \ + if (PREDICT_UNLIKELY(cond)) ? \ + (var ? 1 : \ + (var=1, \ + tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, #cond, 1), \ + 1)) \ + : 0) +#endif +#define IF_BUG_ONCE_VARNAME_(a) \ + warning_logged_on_ ## a ## __ +#define IF_BUG_ONCE_VARNAME__(a) \ + IF_BUG_ONCE_VARNAME_(a) + +/** This macro behaves as 'if (bug(x))', except that it only logs its + * warning once, no matter how many times it triggers. + */ + +#define IF_BUG_ONCE(cond) \ + IF_BUG_ONCE__((cond), \ + IF_BUG_ONCE_VARNAME__(__LINE__)) + +/** Define this if you want Tor to crash when any problem comes up, + * so you can get a coredump and track things down. */ +// #define tor_fragile_assert() tor_assert_unreached(0) +#define tor_fragile_assert() tor_assert_nonfatal_unreached_once() + +void tor_assertion_failed_(const char *fname, unsigned int line, + const char *func, const char *expr); +void tor_bug_occurred_(const char *fname, unsigned int line, + const char *func, const char *expr, + int once); + +#endif + diff --git a/src/common/util_format.c b/src/common/util_format.c index 8aae9e8771..9009e1a814 100644 --- a/src/common/util_format.c +++ b/src/common/util_format.c @@ -21,33 +21,48 @@ #include <string.h> #include <stdlib.h> -/** Implements base32 encoding as in RFC 4648. Limitation: Requires - * that srclen*8 is a multiple of 5. - */ +/* Return the base32 encoded size in bytes using the source length srclen. + * The NUL terminated byte is added as well since every base32 encoding + * requires enough space for it. */ +size_t +base32_encoded_size(size_t srclen) +{ + size_t enclen; + enclen = CEIL_DIV(srclen*8, 5) + 1; + tor_assert(enclen < INT_MAX && enclen > srclen); + return enclen; +} + +/** Implements base32 encoding as in RFC 4648. */ void base32_encode(char *dest, size_t destlen, const char *src, size_t srclen) { unsigned int i, v, u; - size_t nbits = srclen * 8, bit; + size_t nbits = srclen * 8; + size_t bit; tor_assert(srclen < SIZE_T_CEILING/8); - tor_assert((nbits%5) == 0); /* We need an even multiple of 5 bits. */ - tor_assert((nbits/5)+1 <= destlen); /* We need enough space. */ + /* We need enough space for the encoded data and the extra NUL byte. */ + tor_assert(base32_encoded_size(srclen) <= destlen); tor_assert(destlen < SIZE_T_CEILING); + /* Make sure we leave no uninitialized data in the destination buffer. */ + memset(dest, 0, destlen); + for (i=0,bit=0; bit < nbits; ++i, bit+=5) { /* set v to the 16-bit value starting at src[bits/8], 0-padded. */ v = ((uint8_t)src[bit/8]) << 8; - if (bit+5<nbits) v += (uint8_t)src[(bit/8)+1]; - /* set u to the 5-bit value at the bit'th bit of src. */ + if (bit+5<nbits) + v += (uint8_t)src[(bit/8)+1]; + /* set u to the 5-bit value at the bit'th bit of buf. */ u = (v >> (11-(bit%8))) & 0x1F; dest[i] = BASE32_CHARS[u]; } dest[i] = '\0'; } -/** Implements base32 decoding as in RFC 4648. Limitation: Requires - * that srclen*5 is a multiple of 8. Returns 0 if successful, -1 otherwise. +/** Implements base32 decoding as in RFC 4648. + * Returns 0 if successful, -1 otherwise. */ int base32_decode(char *dest, size_t destlen, const char *src, size_t srclen) @@ -57,13 +72,13 @@ base32_decode(char *dest, size_t destlen, const char *src, size_t srclen) unsigned int i; size_t nbits, j, bit; char *tmp; - nbits = srclen * 5; + nbits = ((srclen * 5) / 8) * 8; tor_assert(srclen < SIZE_T_CEILING / 5); - tor_assert((nbits%8) == 0); /* We need an even multiple of 8 bits. */ tor_assert((nbits/8) <= destlen); /* We need enough space. */ tor_assert(destlen < SIZE_T_CEILING); + /* Make sure we leave no uninitialized data in the destination buffer. */ memset(dest, 0, destlen); /* Convert base32 encoded chars to the 5-bit values that they represent. */ @@ -186,7 +201,8 @@ base64_encode(char *dest, size_t destlen, const char *src, size_t srclen, if (enclen > INT_MAX) return -1; - memset(dest, 0, enclen); + /* Make sure we leave no uninitialized data in the destination buffer. */ + memset(dest, 0, destlen); /* XXX/Yawning: If this ends up being too slow, this can be sped up * by separating the multiline format case and the normal case, and @@ -249,7 +265,7 @@ base64_encode(char *dest, size_t destlen, const char *src, size_t srclen, break; default: /* Something went catastrophically wrong. */ - tor_fragile_assert(); + tor_fragile_assert(); // LCOV_EXCL_LINE return -1; } @@ -387,6 +403,7 @@ base64_decode(char *dest, size_t destlen, const char *src, size_t srclen) if (destlen > SIZE_T_CEILING) return -1; + /* Make sure we leave no uninitialized data in the destination buffer. */ memset(dest, 0, destlen); /* Iterate over all the bytes in src. Each one will add 0 or 6 bits to the @@ -461,6 +478,9 @@ base16_encode(char *dest, size_t destlen, const char *src, size_t srclen) tor_assert(destlen >= srclen*2+1); tor_assert(destlen < SIZE_T_CEILING); + /* Make sure we leave no uninitialized data in the destination buffer. */ + memset(dest, 0, destlen); + cp = dest; end = src+srclen; while (src<end) { @@ -504,20 +524,24 @@ hex_decode_digit(char c) return hex_decode_digit_(c); } -/** Given a hexadecimal string of <b>srclen</b> bytes in <b>src</b>, decode it - * and store the result in the <b>destlen</b>-byte buffer at <b>dest</b>. - * Return 0 on success, -1 on failure. */ +/** Given a hexadecimal string of <b>srclen</b> bytes in <b>src</b>, decode + * it and store the result in the <b>destlen</b>-byte buffer at <b>dest</b>. + * Return the number of bytes decoded on success, -1 on failure. If + * <b>destlen</b> is greater than INT_MAX or less than half of + * <b>srclen</b>, -1 is returned. */ int base16_decode(char *dest, size_t destlen, const char *src, size_t srclen) { const char *end; - + char *dest_orig = dest; int v1,v2; + if ((srclen % 2) != 0) return -1; - if (destlen < srclen/2 || destlen > SIZE_T_CEILING) + if (destlen < srclen/2 || destlen > INT_MAX) return -1; + /* Make sure we leave no uninitialized data in the destination buffer. */ memset(dest, 0, destlen); end = src+srclen; @@ -530,6 +554,9 @@ base16_decode(char *dest, size_t destlen, const char *src, size_t srclen) ++dest; src+=2; } - return 0; + + tor_assert((dest-dest_orig) <= (ptrdiff_t) destlen); + + return (int) (dest-dest_orig); } diff --git a/src/common/util_format.h b/src/common/util_format.h index a748a4f3cf..20ac711d10 100644 --- a/src/common/util_format.h +++ b/src/common/util_format.h @@ -24,6 +24,7 @@ int base64_decode_nopad(uint8_t *dest, size_t destlen, #define BASE32_CHARS "abcdefghijklmnopqrstuvwxyz234567" void base32_encode(char *dest, size_t destlen, const char *src, size_t srclen); int base32_decode(char *dest, size_t destlen, const char *src, size_t srclen); +size_t base32_encoded_size(size_t srclen); int hex_decode_digit(char c); void base16_encode(char *dest, size_t destlen, const char *src, size_t srclen); diff --git a/src/common/util_process.c b/src/common/util_process.c index 848b238318..abda63720c 100644 --- a/src/common/util_process.c +++ b/src/common/util_process.c @@ -61,9 +61,9 @@ process_map_entries_eq_(const waitpid_callback_t *a, static HT_HEAD(process_map, waitpid_callback_t) process_map = HT_INITIALIZER(); HT_PROTOTYPE(process_map, waitpid_callback_t, node, process_map_entry_hash_, - process_map_entries_eq_); + process_map_entries_eq_) HT_GENERATE2(process_map, waitpid_callback_t, node, process_map_entry_hash_, - process_map_entries_eq_, 0.6, tor_reallocarray_, tor_free_); + process_map_entries_eq_, 0.6, tor_reallocarray_, tor_free_) /** * Begin monitoring the child pid <b>pid</b> to see if we get a SIGCHLD for diff --git a/src/common/workqueue.c b/src/common/workqueue.c index 0a38550de0..48c0cca01f 100644 --- a/src/common/workqueue.c +++ b/src/common/workqueue.c @@ -262,9 +262,12 @@ workerthread_new(void *state, threadpool_t *pool, replyqueue_t *replyqueue) thr->in_pool = pool; if (spawn_func(worker_thread_main, thr) < 0) { + //LCOV_EXCL_START + tor_assert_nonfatal_unreached(); log_err(LD_GENERAL, "Can't launch worker thread."); tor_free(thr); return NULL; + //LCOV_EXCL_STOP } return thr; @@ -375,8 +378,8 @@ threadpool_queue_update(threadpool_t *pool, static int threadpool_start_threads(threadpool_t *pool, int n) { - if (n < 0) - return -1; + if (BUG(n < 0)) + return -1; // LCOV_EXCL_LINE if (n > MAX_THREADS) n = MAX_THREADS; @@ -391,9 +394,12 @@ threadpool_start_threads(threadpool_t *pool, int n) workerthread_t *thr = workerthread_new(state, pool, pool->reply_queue); if (!thr) { + //LCOV_EXCL_START + tor_assert_nonfatal_unreached(); pool->free_thread_state_fn(state); tor_mutex_release(&pool->lock); return -1; + //LCOV_EXCL_STOP } thr->index = pool->n_threads; pool->threads[pool->n_threads++] = thr; @@ -429,10 +435,13 @@ threadpool_new(int n_threads, pool->reply_queue = replyqueue; if (threadpool_start_threads(pool, n_threads) < 0) { + //LCOV_EXCL_START + tor_assert_nonfatal_unreached(); tor_cond_uninit(&pool->condition); tor_mutex_uninit(&pool->lock); tor_free(pool); return NULL; + //LCOV_EXCL_STOP } return pool; @@ -456,8 +465,10 @@ replyqueue_new(uint32_t alertsocks_flags) rq = tor_malloc_zero(sizeof(replyqueue_t)); if (alert_sockets_create(&rq->alert, alertsocks_flags) < 0) { + //LCOV_EXCL_START tor_free(rq); return NULL; + //LCOV_EXCL_STOP } tor_mutex_init(&rq->lock); @@ -486,10 +497,12 @@ void replyqueue_process(replyqueue_t *queue) { if (queue->alert.drain_fn(queue->alert.read_fd) < 0) { + //LCOV_EXCL_START static ratelim_t warn_limit = RATELIM_INIT(7200); log_fn_ratelim(&warn_limit, LOG_WARN, LD_GENERAL, "Failure from drain_fd: %s", tor_socket_strerror(tor_socket_errno(queue->alert.read_fd))); + //LCOV_EXCL_STOP } tor_mutex_acquire(&queue->lock); diff --git a/src/config/torrc.minimal.in-staging b/src/config/torrc.minimal.in-staging index 248cb5cf02..d4dfd5f6bb 100644 --- a/src/config/torrc.minimal.in-staging +++ b/src/config/torrc.minimal.in-staging @@ -98,6 +98,8 @@ # OutboundBindAddress 10.0.0.5 ## A handle for your relay, so people don't have to refer to it by key. +## Nicknames must be between 1 and 19 characters inclusive, and must +## contain only the characters [a-zA-Z0-9]. #Nickname ididnteditheconfig ## Define these to limit how much relayed traffic you will allow. Your diff --git a/src/config/torrc.sample.in b/src/config/torrc.sample.in index 248cb5cf02..d4dfd5f6bb 100644 --- a/src/config/torrc.sample.in +++ b/src/config/torrc.sample.in @@ -98,6 +98,8 @@ # OutboundBindAddress 10.0.0.5 ## A handle for your relay, so people don't have to refer to it by key. +## Nicknames must be between 1 and 19 characters inclusive, and must +## contain only the characters [a-zA-Z0-9]. #Nickname ididnteditheconfig ## Define these to limit how much relayed traffic you will allow. Your diff --git a/src/ext/README b/src/ext/README index 7ce1bc3b74..dfe620ed16 100644 --- a/src/ext/README +++ b/src/ext/README @@ -73,3 +73,14 @@ readpassphrase.[ch] Portable readpassphrase implementation from OpenSSH portable, version 6.8p1. + +timeouts/ + + William Ahern's hierarchical timer-wheel implementation. MIT license. + +mulodi/ + + Contains an overflow-checking 64-bit signed integer multiply + from LLVM's compiler_rt. For some reason, this is missing from + 32-bit libclang in many places. Dual licensed MIT-license and + BSD-like license; see mulodi/LICENSE.TXT. diff --git a/src/ext/ed25519/donna/curve25519-donna-64bit.h b/src/ext/ed25519/donna/curve25519-donna-64bit.h index 2941d1bcdc..50c9916768 100644 --- a/src/ext/ed25519/donna/curve25519-donna-64bit.h +++ b/src/ext/ed25519/donna/curve25519-donna-64bit.h @@ -8,9 +8,9 @@ typedef uint64_t bignum25519[5]; -static const uint64_t reduce_mask_40 = ((uint64_t)1 << 40) - 1; +//static const uint64_t reduce_mask_40 = ((uint64_t)1 << 40) - 1; static const uint64_t reduce_mask_51 = ((uint64_t)1 << 51) - 1; -static const uint64_t reduce_mask_56 = ((uint64_t)1 << 56) - 1; +//static const uint64_t reduce_mask_56 = ((uint64_t)1 << 56) - 1; /* out = in */ DONNA_INLINE static void diff --git a/src/ext/ed25519/donna/ed25519-donna-64bit-x86.h b/src/ext/ed25519/donna/ed25519-donna-64bit-x86.h index 30bd472762..f6b5570298 100644 --- a/src/ext/ed25519/donna/ed25519-donna-64bit-x86.h +++ b/src/ext/ed25519/donna/ed25519-donna-64bit-x86.h @@ -2,6 +2,11 @@ #define HAVE_GE25519_SCALARMULT_BASE_CHOOSE_NIELS +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Woverlength-strings" +#endif + DONNA_NOINLINE static void ge25519_scalarmult_base_choose_niels(ge25519_niels *t, const uint8_t table[256][96], uint32_t pos, signed char b) { int64_t breg = (int64_t)b; @@ -347,5 +352,9 @@ ge25519_scalarmult_base_choose_niels(ge25519_niels *t, const uint8_t table[256][ ); } +#ifdef __clang__ +#pragma clang diagnostic pop +#endif + #endif /* defined(ED25519_GCC_64BIT_X86_CHOOSE) */ diff --git a/src/ext/ed25519/donna/ed25519-donna-batchverify.h b/src/ext/ed25519/donna/ed25519-donna-batchverify.h index 43c4923b3e..7c64cce787 100644 --- a/src/ext/ed25519/donna/ed25519-donna-batchverify.h +++ b/src/ext/ed25519/donna/ed25519-donna-batchverify.h @@ -188,7 +188,7 @@ ge25519_multi_scalarmult_vartime(ge25519 *r, batch_heap *heap, size_t count) { } /* not actually used for anything other than testing */ -unsigned char batch_point_buffer[3][32]; +static unsigned char batch_point_buffer[3][32]; static int ge25519_is_neutral_vartime(const ge25519 *p) { diff --git a/src/ext/ed25519/donna/ed25519-donna.h b/src/ext/ed25519/donna/ed25519-donna.h index 64561d3288..299c8d90fd 100644 --- a/src/ext/ed25519/donna/ed25519-donna.h +++ b/src/ext/ed25519/donna/ed25519-donna.h @@ -10,6 +10,16 @@ #include "ed25519-donna-portable.h" +#include "orconfig.h" + +#ifdef HAVE_CFLAG_WOVERLENGTH_STRINGS +/* Some of the ASM here is very long strings. */ +#ifdef __clang__ +#pragma clang diagnostic ignored "-Woverlength-strings" +#else +#pragma GCC diagnostic ignored "-Woverlength-strings" +#endif +#endif #if defined(ED25519_SSE2) #else diff --git a/src/ext/ed25519/donna/ed25519_tor.c b/src/ext/ed25519/donna/ed25519_tor.c index 52b259dfe1..07f6a0f23a 100644 --- a/src/ext/ed25519/donna/ed25519_tor.c +++ b/src/ext/ed25519/donna/ed25519_tor.c @@ -44,7 +44,8 @@ typedef unsigned char ed25519_signature[64]; typedef unsigned char ed25519_public_key[32]; typedef unsigned char ed25519_secret_key[32]; -static void gettweak(unsigned char *out, const unsigned char *param); +static void ed25519_donna_gettweak(unsigned char *out, + const unsigned char *param); static int ED25519_FN(ed25519_sign_open) (const unsigned char *m, size_t mlen, const ed25519_public_key pk, const ed25519_signature RS); @@ -242,7 +243,7 @@ ed25519_donna_sign(unsigned char *sig, const unsigned char *m, size_t mlen, } static void -gettweak(unsigned char *out, const unsigned char *param) +ed25519_donna_gettweak(unsigned char *out, const unsigned char *param) { static const char str[] = "Derive temporary signing key"; ed25519_hash_context ctx; @@ -266,7 +267,7 @@ ed25519_donna_blind_secret_key(unsigned char *out, const unsigned char *inp, ed25519_hash_context ctx; bignum256modm ALIGN(16) sk, t; - gettweak(tweak, param); + ed25519_donna_gettweak(tweak, param); expand256_modm(t, tweak, 32); expand256_modm(sk, inp, 32); @@ -297,7 +298,7 @@ ed25519_donna_blind_public_key(unsigned char *out, const unsigned char *inp, ge25519 ALIGN(16) A, Aprime; bignum256modm ALIGN(16) t; - gettweak(tweak, param); + ed25519_donna_gettweak(tweak, param); expand256_modm(t, tweak, 32); /* No "ge25519_unpack", negate the public key. */ diff --git a/src/ext/ed25519/ref10/blinding.c b/src/ext/ed25519/ref10/blinding.c index 4d9a9cbbe7..ee3e8666fa 100644 --- a/src/ext/ed25519/ref10/blinding.c +++ b/src/ext/ed25519/ref10/blinding.c @@ -10,7 +10,7 @@ #include "crypto.h" static void -gettweak(unsigned char *out, const unsigned char *param) +ed25519_ref10_gettweak(unsigned char *out, const unsigned char *param) { const char str[] = "Derive temporary signing key"; crypto_hash_sha512_2(out, (const unsigned char*)str, strlen(str), param, 32); @@ -26,7 +26,7 @@ int ed25519_ref10_blind_secret_key(unsigned char *out, const char str[] = "Derive temporary signing key hash input"; unsigned char tweak[64]; unsigned char zero[32]; - gettweak(tweak, param); + ed25519_ref10_gettweak(tweak, param); memset(zero, 0, 32); sc_muladd(out, inp, tweak, zero); @@ -50,7 +50,7 @@ int ed25519_ref10_blind_public_key(unsigned char *out, ge_p3 A; ge_p2 Aprime; - gettweak(tweak, param); + ed25519_ref10_gettweak(tweak, param); memset(zero, 0, sizeof(zero)); /* Not the greatest implementation of all of this. I wish I had diff --git a/src/ext/eventdns.c b/src/ext/eventdns.c index fc5657cbb4..f5b7723b54 100644 --- a/src/ext/eventdns.c +++ b/src/ext/eventdns.c @@ -50,9 +50,6 @@ #endif #endif -/* #define _POSIX_C_SOURCE 200507 */ -#define _GNU_SOURCE - #ifdef DNS_USE_CPU_CLOCK_FOR_ID #ifdef DNS_USE_OPENSSL_FOR_ID #error Multiple id options selected @@ -2004,8 +2001,7 @@ evdns_request_timeout_callback(int fd, short events, void *arg) { } else { /* retransmit it */ /* Stop waiting for the timeout. No need to do this in - * request_finished; that one already deletes the timeout event. - * XXXX023 port this change to libevent. */ + * request_finished; that one already deletes the timeout event. */ del_timeout_event(req); evdns_request_transmit(req); } diff --git a/src/ext/ht.h b/src/ext/ht.h index 28d1fe49d5..1b6cbe6632 100644 --- a/src/ext/ht.h +++ b/src/ext/ht.h @@ -203,6 +203,7 @@ ht_string_hash(const char *s) name##_HT_GROW(head, head->hth_n_entries+1); \ HT_SET_HASH_(elm, field, hashfn); \ p = name##_HT_FIND_P_(head, elm); \ + HT_ASSERT_(p != NULL); /* this holds because we called HT_GROW */ \ r = *p; \ *p = elm; \ if (r && (r!=elm)) { \ @@ -470,6 +471,7 @@ ht_string_hash(const char *s) name##_HT_GROW(var##_head_, var##_head_->hth_n_entries+1); \ HT_SET_HASH_((elm), field, hashfn); \ var = name##_HT_FIND_P_(var##_head_, (elm)); \ + HT_ASSERT_(var); /* Holds because we called HT_GROW */ \ if (*var) { \ y; \ } else { \ diff --git a/src/ext/include.am b/src/ext/include.am index bf678f2c9d..6cfdbcc447 100644 --- a/src/ext/include.am +++ b/src/ext/include.am @@ -12,11 +12,16 @@ EXTHEADERS = \ src/ext/strlcpy.c \ src/ext/tinytest_macros.h \ src/ext/tor_queue.h \ - src/ext/siphash.h + src/ext/siphash.h \ + src/ext/timeouts/timeout.h \ + src/ext/timeouts/timeout-debug.h \ + src/ext/timeouts/timeout-bitops.c \ + src/ext/timeouts/timeout.c noinst_HEADERS+= $(EXTHEADERS) -src_ext_ed25519_ref10_libed25519_ref10_a_CFLAGS= +src_ext_ed25519_ref10_libed25519_ref10_a_CFLAGS=\ + @CFLAGS_CONSTTIME@ src_ext_ed25519_ref10_libed25519_ref10_a_SOURCES= \ src/ext/ed25519/ref10/fe_0.c \ @@ -93,7 +98,8 @@ noinst_HEADERS += $(ED25519_REF10_HDRS) LIBED25519_REF10=src/ext/ed25519/ref10/libed25519_ref10.a noinst_LIBRARIES += $(LIBED25519_REF10) -src_ext_ed25519_donna_libed25519_donna_a_CFLAGS= \ +src_ext_ed25519_donna_libed25519_donna_a_CFLAGS=\ + @CFLAGS_CONSTTIME@ \ -DED25519_CUSTOMRANDOM \ -DED25519_SUFFIX=_donna @@ -135,7 +141,8 @@ noinst_HEADERS += $(ED25519_DONNA_HDRS) LIBED25519_DONNA=src/ext/ed25519/donna/libed25519_donna.a noinst_LIBRARIES += $(LIBED25519_DONNA) -src_ext_keccak_tiny_libkeccak_tiny_a_CFLAGS= +src_ext_keccak_tiny_libkeccak_tiny_a_CFLAGS=\ + @CFLAGS_CONSTTIME@ src_ext_keccak_tiny_libkeccak_tiny_a_SOURCES= \ src/ext/keccak-tiny/keccak-tiny-unrolled.c @@ -148,3 +155,21 @@ noinst_HEADERS += $(LIBKECCAK_TINY_HDRS) LIBKECCAK_TINY=src/ext/keccak-tiny/libkeccak-tiny.a noinst_LIBRARIES += $(LIBKECCAK_TINY) +EXTRA_DIST += \ + src/ext/timeouts/bench/bench-add.lua \ + src/ext/timeouts/bench/bench-aux.lua \ + src/ext/timeouts/bench/bench.c \ + src/ext/timeouts/bench/bench-del.lua \ + src/ext/timeouts/bench/bench-expire.lua \ + src/ext/timeouts/bench/bench.h \ + src/ext/timeouts/bench/bench-heap.c \ + src/ext/timeouts/bench/bench-llrb.c \ + src/ext/timeouts/bench/bench.plt \ + src/ext/timeouts/bench/bench-wheel.c \ + src/ext/timeouts/bench/Rules.mk \ + src/ext/timeouts/lua/Rules.mk \ + src/ext/timeouts/lua/timeout-lua.c \ + src/ext/timeouts/Makefile \ + src/ext/timeouts/Rules.shrc \ + src/ext/timeouts/test-timeout.c + diff --git a/src/ext/mulodi/LICENSE.TXT b/src/ext/mulodi/LICENSE.TXT new file mode 100644 index 0000000000..a17dc12b27 --- /dev/null +++ b/src/ext/mulodi/LICENSE.TXT @@ -0,0 +1,91 @@ +============================================================================== +compiler_rt License +============================================================================== + +The compiler_rt library is dual licensed under both the University of Illinois +"BSD-Like" license and the MIT license. As a user of this code you may choose +to use it under either license. As a contributor, you agree to allow your code +to be used under both. + +Full text of the relevant licenses is included below. + +============================================================================== + +University of Illinois/NCSA +Open Source License + +Copyright (c) 2009-2016 by the contributors listed in CREDITS.TXT + +All rights reserved. + +Developed by: + + LLVM Team + + University of Illinois at Urbana-Champaign + + http://llvm.org + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal with +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimers. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimers in the + documentation and/or other materials provided with the distribution. + + * Neither the names of the LLVM Team, University of Illinois at + Urbana-Champaign, nor the names of its contributors may be used to + endorse or promote products derived from this Software without specific + prior written permission. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE +SOFTWARE. + +============================================================================== + +Copyright (c) 2009-2015 by the contributors listed in CREDITS.TXT + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +============================================================================== +Copyrights and Licenses for Third Party Software Distributed with LLVM: +============================================================================== +The LLVM software contains code written by third parties. Such software will +have its own individual LICENSE.TXT file in the directory in which it appears. +This file will describe the copyrights, license, and restrictions which apply +to that code. + +The disclaimer of warranty in the University of Illinois Open Source License +applies to all code in the LLVM Distribution, and nothing in any of the +other licenses gives permission to use the names of the LLVM Team or the +University of Illinois to endorse or promote products derived from this +Software. + diff --git a/src/ext/mulodi/mulodi4.c b/src/ext/mulodi/mulodi4.c new file mode 100644 index 0000000000..bfa5e01295 --- /dev/null +++ b/src/ext/mulodi/mulodi4.c @@ -0,0 +1,66 @@ +/*===-- mulodi4.c - Implement __mulodi4 -----------------------------------=== + * + * The LLVM Compiler Infrastructure + * + * This file is dual licensed under the MIT and the University of Illinois Open + * Source Licenses. See LICENSE.TXT for details. + * + * ===----------------------------------------------------------------------=== + * + * This file implements __mulodi4 for the compiler_rt library. + * + * ===----------------------------------------------------------------------=== + */ + +#if 0 +#include "int_lib.h" +#else +#define COMPILER_RT_ABI +#define di_int int64_t +#include "torint.h" + +di_int __mulodi4(di_int a, di_int b, int* overflow); +#endif + +/* Returns: a * b */ + +/* Effects: sets *overflow to 1 if a * b overflows */ + +COMPILER_RT_ABI di_int +__mulodi4(di_int a, di_int b, int* overflow) +{ + const int N = (int)(sizeof(di_int) * CHAR_BIT); + const di_int MIN = (di_int)1 << (N-1); + const di_int MAX = ~MIN; + *overflow = 0; + di_int result = a * b; + if (a == MIN) + { + if (b != 0 && b != 1) + *overflow = 1; + return result; + } + if (b == MIN) + { + if (a != 0 && a != 1) + *overflow = 1; + return result; + } + di_int sa = a >> (N - 1); + di_int abs_a = (a ^ sa) - sa; + di_int sb = b >> (N - 1); + di_int abs_b = (b ^ sb) - sb; + if (abs_a < 2 || abs_b < 2) + return result; + if (sa == sb) + { + if (abs_a > MAX / abs_b) + *overflow = 1; + } + else + { + if (abs_a > MIN / -abs_b) + *overflow = 1; + } + return result; +} diff --git a/src/ext/timeouts/Makefile b/src/ext/timeouts/Makefile new file mode 100644 index 0000000000..554ebb9ddd --- /dev/null +++ b/src/ext/timeouts/Makefile @@ -0,0 +1,68 @@ +# NOTE: GNU Make 3.81 won't export MAKEFLAGS if .POSIX is specified, but +# Solaris make won't export MAKEFLAGS unless .POSIX is specified. +$(firstword ignore).POSIX: + +.DEFAULT_GOAL = all + +.SUFFIXES: + +all: + +# +# USER-MODIFIABLE MACROS +# +top_srcdir = . +top_builddir = . + +CFLAGS = -O2 -march=native -g -Wall -Wextra -Wno-unused-parameter -Wno-unused-function +SOFLAGS = $$(auto_soflags) +LIBS = $$(auto_libs) + +ALL_CPPFLAGS = -I$(top_srcdir) -DWHEEL_BIT=$(WHEEL_BIT) -DWHEEL_NUM=$(WHEEL_NUM) $(CPPFLAGS) +ALL_CFLAGS = $(CFLAGS) +ALL_SOFLAGS = $(SOFLAGS) +ALL_LDFLAGS = $(LDFLAGS) +ALL_LIBS = $(LIBS) + +LUA_API = 5.3 +LUA = lua +LUA51_CPPFLAGS = $(LUA_CPPFLAGS) +LUA52_CPPFLAGS = $(LUA_CPPFLAGS) +LUA53_CPPFLAGS = $(LUA_CPPFLAGS) + +WHEEL_BIT = 6 +WHEEL_NUM = 4 + +RM = rm -f + +# END MACROS + +SHRC = \ + top_srcdir="$(top_srcdir)"; \ + top_builddir="$(top_builddir)"; \ + . "$${top_srcdir}/Rules.shrc" + +LUA_APIS = 5.1 5.2 5.3 + +include $(top_srcdir)/lua/Rules.mk +include $(top_srcdir)/bench/Rules.mk + +all: test-timeout + +timeout.o: $(top_srcdir)/timeout.c +test-timeout.o: $(top_srcdir)/test-timeout.c + +timeout.o test-timeout.o: + @$(SHRC); echo_cmd $(CC) $(ALL_CFLAGS) -c -o $@ $${top_srcdir}/$(@F:%.o=%.c) $(ALL_CPPFLAGS) + +test-timeout: timeout.o test-timeout.o + @$(SHRC); echo_cmd $(CC) $(ALL_CPPFLAGS) $(ALL_CFLAGS) -o $@ timeout.o test-timeout.o + +.PHONY: clean clean~ + +clean: + $(RM) $(top_builddir)/test-timeout $(top_builddir)/*.o + $(RM) -r $(top_builddir)/*.dSYM + +clean~: + find $(top_builddir) $(top_srcdir) -name "*~" -exec $(RM) -- {} "+" diff --git a/src/ext/timeouts/Rules.shrc b/src/ext/timeouts/Rules.shrc new file mode 100644 index 0000000000..ece75d42d4 --- /dev/null +++ b/src/ext/timeouts/Rules.shrc @@ -0,0 +1,40 @@ +# convert to absolute paths +top_srcdir="$(cd "${top_srcdir}" && pwd -L)" +top_builddir="$(cd "${top_builddir}" && pwd -L)" + +# Paths for Lua modules (benchmarks and installed modules) +export LUA_CPATH="${top_builddir}/lua/5.1/?.so;${top_builddir}/bench/?.so;;" +export LUA_PATH="${top_srcdir}/lua/?.lua;${top_srcdir}/bench/?.lua;;" +export LUA_CPATH_5_2="${top_builddir}/lua/5.2/?.so;${top_builddir}/bench/?.so;;" +export LUA_PATH_5_2="${top_srcdir}/lua/?.lua;${top_srcdir}/bench/?.lua;;" +export LUA_CPATH_5_3="${top_builddir}/lua/5.3/?.so;${top_builddir}/bench/?.so;;" +export LUA_PATH_5_3="${top_srcdir}/lua/?.lua;${top_srcdir}/bench/?.lua;;" + +# preserve stdout so we can print commands to terminal +exec 9>&1; +echo_cmd() { + printf "%s\n" "$*" >&9; + "$@"; +} + +auto_soflags() { + case "$(uname -s)" in + Darwin) + printf -- "-bundle -undefined dynamic_lookup" + ;; + *) + printf -- "-fPIC -shared" + ;; + esac +} + +auto_libs() { + case "$(uname -s)" in + Linux) + printf -- "-lrt" + ;; + *) + ;; + esac +} + diff --git a/src/ext/timeouts/bench/Rules.mk b/src/ext/timeouts/bench/Rules.mk new file mode 100644 index 0000000000..3ee72f3eff --- /dev/null +++ b/src/ext/timeouts/bench/Rules.mk @@ -0,0 +1,49 @@ +BENCH_MODS = bench.so $(BENCH_ALGOS:%=bench-%.so) +BENCH_ALGOS = wheel heap llrb +BENCH_OPS = add del expire + +$(top_builddir)/bench/bench.so: $(top_srcdir)/bench/bench.c +$(top_builddir)/bench/bench-wheel.so: $(top_srcdir)/bench/bench-wheel.c +$(top_builddir)/bench/bench-heap.so: $(top_srcdir)/bench/bench-heap.c +$(top_builddir)/bench/bench-llrb.so: $(top_srcdir)/bench/bench-llrb.c + +$(BENCH_MODS:%=$(top_builddir)/bench/%): $(top_srcdir)/timeout.h $(top_srcdir)/timeout.c $(top_srcdir)/bench/bench.h + mkdir -p $(@D) + @$(SHRC); echo_cmd $(CC) -o $@ $(top_srcdir)/bench/$(@F:%.so=%.c) $(ALL_CPPFLAGS) $(ALL_CFLAGS) $(ALL_SOFLAGS) $(ALL_LDFLAGS) $(ALL_LIBS) + +$(BENCH_OPS:%=$(top_builddir)/bench/wheel-%.dat): $(top_builddir)/bench/bench-wheel.so $(top_builddir)/bench/bench.so $(top_srcdir)/bench/bench-aux.lua +$(BENCH_OPS:%=$(top_builddir)/bench/heap-%.dat): $(top_builddir)/bench/bench-heap.so $(top_builddir)/bench/bench.so $(top_srcdir)/bench/bench-aux.lua +$(BENCH_OPS:%=$(top_builddir)/bench/llrb-%.dat): $(top_builddir)/bench/bench-llrb.so $(top_builddir)/bench/bench.so $(top_srcdir)/bench/bench-aux.lua + +$(BENCH_ALGOS:%=$(top_builddir)/bench/%-add.dat): $(top_srcdir)/bench/bench-add.lua + @$(SHRC); echo_cmd cd $(@D) && echo_cmd $(LUA) $${top_srcdir}/bench/bench-add.lua $${top_builddir}/bench/bench-$(@F:%-add.dat=%).so > $(@F).tmp + mv $@.tmp $@ + +$(BENCH_ALGOS:%=$(top_builddir)/bench/%-del.dat): $(top_srcdir)/bench/bench-del.lua + @$(SHRC); echo_cmd cd $(@D) && echo_cmd $(LUA) $${top_srcdir}/bench/bench-del.lua $${top_builddir}/bench/bench-$(@F:%-del.dat=%).so > $(@F).tmp + mv $@.tmp $@ + +$(BENCH_ALGOS:%=$(top_builddir)/bench/%-expire.dat): $(top_srcdir)/bench/bench-expire.lua + @$(SHRC); echo_cmd cd $(@D) && echo_cmd $(LUA) $${top_srcdir}/bench/bench-expire.lua $${top_builddir}/bench/bench-$(@F:%-expire.dat=%).so > $(@F).tmp + mv $@.tmp $@ + +$(top_builddir)/bench/bench.eps: \ + $(BENCH_OPS:%=$(top_builddir)/bench/wheel-%.dat) \ + $(BENCH_OPS:%=$(top_builddir)/bench/heap-%.dat) +# $(BENCH_OPS:%=$(top_builddir)/bench/llrb-%.dat) + +$(top_builddir)/bench/bench.eps: $(top_srcdir)/bench/bench.plt + @$(SHRC); echo_cmd cd $(@D) && echo_cmd gnuplot $${top_srcdir}/bench/bench.plt > $(@F).tmp + mv $@.tmp $@ + +$(top_builddir)/bench/bench.pdf: $(top_builddir)/bench/bench.eps + @$(SHRC); echo_cmd ps2pdf $${top_builddir}/bench/bench.eps $@ + +bench-mods: $(BENCH_MODS:%=$(top_builddir)/bench/%) + +bench-all: $(top_builddir)/bench/bench.pdf + +bench-clean: + $(RM) -r $(top_builddir)/bench/*.so $(top_builddir)/bench/*.dSYM + $(RM) $(top_builddir)/bench/*.dat $(top_builddir)/bench/*.tmp + $(RM) $(top_builddir)/bench/bench.{eps,pdf} diff --git a/src/ext/timeouts/bench/bench-add.lua b/src/ext/timeouts/bench/bench-add.lua new file mode 100755 index 0000000000..64a921d3de --- /dev/null +++ b/src/ext/timeouts/bench/bench-add.lua @@ -0,0 +1,30 @@ +#!/usr/bin/env lua + +local bench = require"bench" +local aux = require"bench-aux" + +local lib = ... or aux.optenv("BENCH_L", "bench-wheel.so") +local limit = tonumber(aux.optenv("BENCH_N", 1000000)) +local step = tonumber(aux.optenv("BENCH_S", limit / 100)) +local exp_step = tonumber(aux.optenv("BENCH_E", 1.0)) +local verbose = aux.toboolean(os.getenv("BENCH_V", false)) + +local B = bench.new(lib, count, nil, verbose) +local fill_count, fill_last = B:fill(limit) + +for i=0,limit,step do + local exp_elapsed, fill_elapsed, fill_rate + + -- expire all timeouts + --exp_elapsed = aux.time(B.expire, B, fill_count, fill_last * exp_step) + exp_elapsed = aux.time(B.del, B, 0, fill_count) + assert(B:empty()) + + -- add i timeouts + fill_elapsed, fill_count, fill_last = aux.time(B.fill, B, i) + assert(fill_count == i) + fill_rate = fill_elapsed > 0 and (fill_count / fill_elapsed) or 0 + + local fmt = verbose and "%d\t%f\t(%d/s)\t(exp:%f)" or "%d\t%f" + aux.say(fmt, i, fill_elapsed, fill_rate, exp_elapsed) +end diff --git a/src/ext/timeouts/bench/bench-aux.lua b/src/ext/timeouts/bench/bench-aux.lua new file mode 100644 index 0000000000..6321247421 --- /dev/null +++ b/src/ext/timeouts/bench/bench-aux.lua @@ -0,0 +1,30 @@ +local bench = require"bench" +local clock = bench.clock + +local aux = {} + +local function time_return(begun, ...) + local duration = clock() - begun + return duration, ... +end + +function aux.time(f, ...) + local begun = clock() + return time_return(begun, f(...)) +end + +function aux.say(...) + print(string.format(...)) +end + +function aux.toboolean(s) + return tostring(s):match("^[1TtYy]") and true or false +end + +function aux.optenv(k, def) + local s = os.getenv(k) + + return (s and #s > 0 and s) or def +end + +return aux diff --git a/src/ext/timeouts/bench/bench-del.lua b/src/ext/timeouts/bench/bench-del.lua new file mode 100755 index 0000000000..4306745f21 --- /dev/null +++ b/src/ext/timeouts/bench/bench-del.lua @@ -0,0 +1,25 @@ +#!/usr/bin/env lua + +local bench = require"bench" +local aux = require"bench-aux" + +local lib = ... or aux.optenv("BENCH_L", "bench-wheel.so") +local limit = tonumber(aux.optenv("BENCH_N", 1000000)) +local step = tonumber(aux.optenv("BENCH_S", limit / 100)) +local verbose = aux.toboolean(os.getenv("BENCH_V", false)) + +local B = bench.new(lib, count) + +for i=0,limit,step do + -- add i timeouts + local fill_elapsed, fill_count = aux.time(B.fill, B, i, 60 * 1000000) + assert(i == fill_count) + + --- delete i timeouts + local del_elapsed = aux.time(B.del, B, 0, fill_count) + assert(B:empty()) + local del_rate = i > 0 and i / del_elapsed or 0 + + local fmt = verbose and "%d\t%f\t(%d/s)\t(fill:%f)" or "%d\t%f" + aux.say(fmt, i, del_elapsed, del_rate, fill_elapsed) +end diff --git a/src/ext/timeouts/bench/bench-expire.lua b/src/ext/timeouts/bench/bench-expire.lua new file mode 100755 index 0000000000..3e6374ed52 --- /dev/null +++ b/src/ext/timeouts/bench/bench-expire.lua @@ -0,0 +1,29 @@ +#!/usr/bin/env lua + +local bench = require"bench" +local aux = require"bench-aux" + +local lib = ... or aux.optenv("BENCH_L", "bench-wheel.so") +local limit = tonumber(aux.optenv("BENCH_N", 1000000)) +local step = tonumber(aux.optenv("BENCH_S", limit / 100)) +-- expire 1/1000 * #timeouts per clock update +local exp_step = tonumber(aux.optenv("BENCH_E", 0.0001)) +local verbose = aux.toboolean(os.getenv("BENCH_V", false)) + +local B = require"bench".new(lib, count) + +for i=0,limit,step do + -- add i timeouts + local fill_elapsed, fill_count, fill_last = aux.time(B.fill, B, i) + + -- expire timeouts by iteratively updating clock. exp_step is the + -- approximate number of timeouts (as a fraction of the total number + -- of timeouts) that will expire per update. + local exp_elapsed, exp_count = aux.time(B.expire, B, fill_count, math.floor(fill_last * exp_step)) + assert(exp_count == i) + assert(B:empty()) + local exp_rate = i > 0 and i / exp_elapsed or 0 + + local fmt = verbose and "%d\t%f\t(%d/s)\t(fill:%f)" or "%d\t%f" + aux.say(fmt, i, exp_elapsed, exp_rate, fill_elapsed) +end diff --git a/src/ext/timeouts/bench/bench-heap.c b/src/ext/timeouts/bench/bench-heap.c new file mode 100644 index 0000000000..f1166a4d7e --- /dev/null +++ b/src/ext/timeouts/bench/bench-heap.c @@ -0,0 +1,236 @@ +/* + * Copyright (c) 2006 Maxim Yegorushkin <maxim.yegorushkin@gmail.com> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef _MIN_HEAP_H_ +#define _MIN_HEAP_H_ + +#include <stdlib.h> +#include <err.h> +#include "timeout.h" +#include "bench.h" + +#define min_heap_idx interval + +typedef timeout_t min_heap_idx_t; + +typedef struct min_heap +{ + struct timeout** p; + unsigned n, a; + timeout_t curtime; +} min_heap_t; + +static inline void min_heap_ctor(min_heap_t* s); +static inline void min_heap_dtor(min_heap_t* s); +static inline void min_heap_elem_init(struct timeout* e); +static inline int min_heap_elem_greater(struct timeout *a, struct timeout *b); +static inline int min_heap_empty(min_heap_t* s); +static inline unsigned min_heap_size(min_heap_t* s); +static inline struct timeout* min_heap_top(min_heap_t* s); +static inline int min_heap_reserve(min_heap_t* s, unsigned n); +static inline int min_heap_push(min_heap_t* s, struct timeout* e); +static inline struct timeout* min_heap_pop(min_heap_t* s); +static inline int min_heap_erase(min_heap_t* s, struct timeout* e); +static inline void min_heap_shift_up_(min_heap_t* s, unsigned hole_index, struct timeout* e); +static inline void min_heap_shift_down_(min_heap_t* s, unsigned hole_index, struct timeout* e); + +int min_heap_elem_greater(struct timeout *a, struct timeout *b) +{ + return a->expires > b->expires; +} + +void min_heap_ctor(min_heap_t* s) { s->p = 0; s->n = 0; s->a = 0; } +void min_heap_dtor(min_heap_t* s) { if(s->p) free(s->p); } +void min_heap_elem_init(struct timeout* e) { e->min_heap_idx = -1; } +int min_heap_empty(min_heap_t* s) { return 0u == s->n; } +unsigned min_heap_size(min_heap_t* s) { return s->n; } +struct timeout* min_heap_top(min_heap_t* s) { return s->n ? *s->p : 0; } + +int min_heap_push(min_heap_t* s, struct timeout* e) +{ + if(min_heap_reserve(s, s->n + 1)) + return -1; + min_heap_shift_up_(s, s->n++, e); + return 0; +} + +struct timeout* min_heap_pop(min_heap_t* s) +{ + if(s->n) + { + struct timeout* e = *s->p; + min_heap_shift_down_(s, 0u, s->p[--s->n]); + e->min_heap_idx = -1; + return e; + } + return 0; +} + +int min_heap_erase(min_heap_t* s, struct timeout* e) +{ + if(((min_heap_idx_t)-1) != e->min_heap_idx) + { + struct timeout *last = s->p[--s->n]; + unsigned parent = (e->min_heap_idx - 1) / 2; + /* we replace e with the last element in the heap. We might need to + shift it upward if it is less than its parent, or downward if it is + greater than one or both its children. Since the children are known + to be less than the parent, it can't need to shift both up and + down. */ + if (e->min_heap_idx > 0 && min_heap_elem_greater(s->p[parent], last)) + min_heap_shift_up_(s, e->min_heap_idx, last); + else + min_heap_shift_down_(s, e->min_heap_idx, last); + e->min_heap_idx = -1; + return 0; + } + return -1; +} + +int min_heap_reserve(min_heap_t* s, unsigned n) +{ + if(s->a < n) + { + struct timeout** p; + unsigned a = s->a ? s->a * 2 : 8; + if(a < n) + a = n; + if(!(p = (struct timeout**)realloc(s->p, a * sizeof *p))) + return -1; + s->p = p; + s->a = a; + } + return 0; +} + +void min_heap_shift_up_(min_heap_t* s, unsigned hole_index, struct timeout* e) +{ + unsigned parent = (hole_index - 1) / 2; + while(hole_index && min_heap_elem_greater(s->p[parent], e)) + { + (s->p[hole_index] = s->p[parent])->min_heap_idx = hole_index; + hole_index = parent; + parent = (hole_index - 1) / 2; + } + (s->p[hole_index] = e)->min_heap_idx = hole_index; +} + +void min_heap_shift_down_(min_heap_t* s, unsigned hole_index, struct timeout* e) +{ + unsigned min_child = 2 * (hole_index + 1); + while(min_child <= s->n) + { + min_child -= min_child == s->n || min_heap_elem_greater(s->p[min_child], s->p[min_child - 1]); + if(!(min_heap_elem_greater(e, s->p[min_child]))) + break; + (s->p[hole_index] = s->p[min_child])->min_heap_idx = hole_index; + hole_index = min_child; + min_child = 2 * (hole_index + 1); + } + min_heap_shift_up_(s, hole_index, e); +} + +#endif /* _MIN_HEAP_H_ */ + + +static void *init(struct timeout *timeout, size_t count, int verbose) { + min_heap_t *H; + size_t i; + + H = calloc(1, sizeof *H); + + min_heap_ctor(H); + if (0 != min_heap_reserve(H, count)) + err(1, "realloc"); + + for (i = 0; i < count; i++) { + min_heap_elem_init(&timeout[i]); + } + + return H; +} /* init() */ + + +static void add(void *ctx, struct timeout *to, timeout_t expires) { + min_heap_t *H = ctx; + min_heap_erase(H, to); + to->expires = H->curtime + expires; + if (0 != min_heap_push(H, to)) + err(1, "realloc"); +} /* add() */ + + +static void del(void *ctx, struct timeout *to) { + min_heap_erase(ctx, to); +} /* del() */ + + +static struct timeout *get(void *ctx) { + min_heap_t *H = ctx; + struct timeout *to; + + if ((to = min_heap_top(H)) && to->expires <= H->curtime) + return min_heap_pop(H); + + return NULL; +} /* get() */ + + +static void update(void *ctx, timeout_t ts) { + min_heap_t *H = ctx; + H->curtime = ts; +} /* update() */ + + +static void check(void *ctx) { + return; +} /* check() */ + + +static int empty(void *ctx) { + min_heap_t *H = ctx; + + return (NULL == min_heap_top(H)); +} /* empty() */ + + +static void destroy(void *H) { + free(H); + return; +} /* destroy() */ + + +const struct benchops benchops = { + .init = &init, + .add = &add, + .del = &del, + .get = &get, + .update = &update, + .check = &check, + .empty = &empty, + .destroy = &destroy, +}; + diff --git a/src/ext/timeouts/bench/bench-llrb.c b/src/ext/timeouts/bench/bench-llrb.c new file mode 100644 index 0000000000..bdb02f0704 --- /dev/null +++ b/src/ext/timeouts/bench/bench-llrb.c @@ -0,0 +1,425 @@ +/* ========================================================================== + * llrb.h - Iterative Left-leaning Red-Black Tree. + * -------------------------------------------------------------------------- + * Copyright (c) 2011, 2013 William Ahern <william@25thandClement.com> + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to permit + * persons to whom the Software is furnished to do so, subject to the + * following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * -------------------------------------------------------------------------- + * CREDITS: + * o Algorithm courtesy of Robert Sedgewick, "Left-leaning Red-Black + * Trees" (September 2008); and Robert Sedgewick and Kevin Wayne, + * Algorithms (4th ed. 2011). + * + * Sedgewick touts the simplicity of the recursive implementation, + * but at least for the 2-3 tree variant the iterative approach is + * almost line-for-line identical. The magic of C pointers helps; + * it'd be uglier with Java. + * + * A couple of missing NULL checks were added to Sedgewick's deletion + * example, and insert was optimized to short-circuit rotations when + * walking up the tree. + * + * o Code implemented in the fashion of Niels Provos' excellent *BSD + * sys/tree.h pre-processor library. + * + * Regarding relative performance, I've refrained from sharing my own + * benchmarks. Differences in run-time speed were too correlated to + * compiler options and other external factors. + * + * Provos' delete implementation doesn't need to start at the root of + * the tree. However, RB_REMOVE must be passed the actual node to be + * removed. LLRB_REMOVE merely requires a key, much like + * RB_FIND/LLRB_FIND. + * ========================================================================== + */ +#ifndef LLRB_H +#define LLRB_H + +#define LLRB_VENDOR "william@25thandClement.com" +#define LLRB_VERSION 0x20130925 + +#ifndef LLRB_STATIC +#ifdef __GNUC__ +#define LLRB_STATIC __attribute__((__unused__)) static +#else +#define LLRB_STATIC static +#endif +#endif + +#define LLRB_HEAD(name, type) \ +struct name { struct type *rbh_root; } + +#define LLRB_INITIALIZER(root) { 0 } + +#define LLRB_INIT(root) do { (root)->rbh_root = 0; } while (0) + +#define LLRB_BLACK 0 +#define LLRB_RED 1 + +#define LLRB_ENTRY(type) \ +struct { struct type *rbe_left, *rbe_right, *rbe_parent; _Bool rbe_color; } + +#define LLRB_LEFT(elm, field) (elm)->field.rbe_left +#define LLRB_RIGHT(elm, field) (elm)->field.rbe_right +#define LLRB_PARENT(elm, field) (elm)->field.rbe_parent +#define LLRB_EDGE(head, elm, field) (((elm) == LLRB_ROOT(head))? &LLRB_ROOT(head) : ((elm) == LLRB_LEFT(LLRB_PARENT((elm), field), field))? &LLRB_LEFT(LLRB_PARENT((elm), field), field) : &LLRB_RIGHT(LLRB_PARENT((elm), field), field)) +#define LLRB_COLOR(elm, field) (elm)->field.rbe_color +#define LLRB_ROOT(head) (head)->rbh_root +#define LLRB_EMPTY(head) ((head)->rbh_root == 0) +#define LLRB_ISRED(elm, field) ((elm) && LLRB_COLOR((elm), field) == LLRB_RED) + +#define LLRB_PROTOTYPE(name, type, field, cmp) \ + LLRB_PROTOTYPE_INTERNAL(name, type, field, cmp,) +#define LLRB_PROTOTYPE_STATIC(name, type, field, cmp) \ + LLRB_PROTOTYPE_INTERNAL(name, type, field, cmp, LLRB_STATIC) +#define LLRB_PROTOTYPE_INTERNAL(name, type, field, cmp, attr) \ +attr struct type *name##_LLRB_INSERT(struct name *, struct type *); \ +attr struct type *name##_LLRB_DELETE(struct name *, struct type *); \ +attr struct type *name##_LLRB_FIND(struct name *, struct type *); \ +attr struct type *name##_LLRB_MIN(struct type *); \ +attr struct type *name##_LLRB_MAX(struct type *); \ +attr struct type *name##_LLRB_NEXT(struct type *); + +#define LLRB_GENERATE(name, type, field, cmp) \ + LLRB_GENERATE_INTERNAL(name, type, field, cmp,) +#define LLRB_GENERATE_STATIC(name, type, field, cmp) \ + LLRB_GENERATE_INTERNAL(name, type, field, cmp, LLRB_STATIC) +#define LLRB_GENERATE_INTERNAL(name, type, field, cmp, attr) \ +static inline void name##_LLRB_ROTL(struct type **pivot) { \ + struct type *a = *pivot; \ + struct type *b = LLRB_RIGHT(a, field); \ + if ((LLRB_RIGHT(a, field) = LLRB_LEFT(b, field))) \ + LLRB_PARENT(LLRB_RIGHT(a, field), field) = a; \ + LLRB_LEFT(b, field) = a; \ + LLRB_COLOR(b, field) = LLRB_COLOR(a, field); \ + LLRB_COLOR(a, field) = LLRB_RED; \ + LLRB_PARENT(b, field) = LLRB_PARENT(a, field); \ + LLRB_PARENT(a, field) = b; \ + *pivot = b; \ +} \ +static inline void name##_LLRB_ROTR(struct type **pivot) { \ + struct type *b = *pivot; \ + struct type *a = LLRB_LEFT(b, field); \ + if ((LLRB_LEFT(b, field) = LLRB_RIGHT(a, field))) \ + LLRB_PARENT(LLRB_LEFT(b, field), field) = b; \ + LLRB_RIGHT(a, field) = b; \ + LLRB_COLOR(a, field) = LLRB_COLOR(b, field); \ + LLRB_COLOR(b, field) = LLRB_RED; \ + LLRB_PARENT(a, field) = LLRB_PARENT(b, field); \ + LLRB_PARENT(b, field) = a; \ + *pivot = a; \ +} \ +static inline void name##_LLRB_FLIP(struct type *root) { \ + LLRB_COLOR(root, field) = !LLRB_COLOR(root, field); \ + LLRB_COLOR(LLRB_LEFT(root, field), field) = !LLRB_COLOR(LLRB_LEFT(root, field), field); \ + LLRB_COLOR(LLRB_RIGHT(root, field), field) = !LLRB_COLOR(LLRB_RIGHT(root, field), field); \ +} \ +static inline void name##_LLRB_FIXUP(struct type **root) { \ + if (LLRB_ISRED(LLRB_RIGHT(*root, field), field) && !LLRB_ISRED(LLRB_LEFT(*root, field), field)) \ + name##_LLRB_ROTL(root); \ + if (LLRB_ISRED(LLRB_LEFT(*root, field), field) && LLRB_ISRED(LLRB_LEFT(LLRB_LEFT(*root, field), field), field)) \ + name##_LLRB_ROTR(root); \ + if (LLRB_ISRED(LLRB_LEFT(*root, field), field) && LLRB_ISRED(LLRB_RIGHT(*root, field), field)) \ + name##_LLRB_FLIP(*root); \ +} \ +attr struct type *name##_LLRB_INSERT(struct name *head, struct type *elm) { \ + struct type **root = &LLRB_ROOT(head); \ + struct type *parent = 0; \ + while (*root) { \ + int comp = (cmp)((elm), (*root)); \ + parent = *root; \ + if (comp < 0) \ + root = &LLRB_LEFT(*root, field); \ + else if (comp > 0) \ + root = &LLRB_RIGHT(*root, field); \ + else \ + return *root; \ + } \ + LLRB_LEFT((elm), field) = 0; \ + LLRB_RIGHT((elm), field) = 0; \ + LLRB_COLOR((elm), field) = LLRB_RED; \ + LLRB_PARENT((elm), field) = parent; \ + *root = (elm); \ + while (parent && (LLRB_ISRED(LLRB_LEFT(parent, field), field) || LLRB_ISRED(LLRB_RIGHT(parent, field), field))) { \ + root = LLRB_EDGE(head, parent, field); \ + parent = LLRB_PARENT(parent, field); \ + name##_LLRB_FIXUP(root); \ + } \ + LLRB_COLOR(LLRB_ROOT(head), field) = LLRB_BLACK; \ + return 0; \ +} \ +static inline void name##_LLRB_MOVL(struct type **pivot) { \ + name##_LLRB_FLIP(*pivot); \ + if (LLRB_ISRED(LLRB_LEFT(LLRB_RIGHT(*pivot, field), field), field)) { \ + name##_LLRB_ROTR(&LLRB_RIGHT(*pivot, field)); \ + name##_LLRB_ROTL(pivot); \ + name##_LLRB_FLIP(*pivot); \ + } \ +} \ +static inline void name##_LLRB_MOVR(struct type **pivot) { \ + name##_LLRB_FLIP(*pivot); \ + if (LLRB_ISRED(LLRB_LEFT(LLRB_LEFT(*pivot, field), field), field)) { \ + name##_LLRB_ROTR(pivot); \ + name##_LLRB_FLIP(*pivot); \ + } \ +} \ +static inline struct type *name##_DELETEMIN(struct name *head, struct type **root) { \ + struct type **pivot = root, *deleted, *parent; \ + while (LLRB_LEFT(*pivot, field)) { \ + if (!LLRB_ISRED(LLRB_LEFT(*pivot, field), field) && !LLRB_ISRED(LLRB_LEFT(LLRB_LEFT(*pivot, field), field), field)) \ + name##_LLRB_MOVL(pivot); \ + pivot = &LLRB_LEFT(*pivot, field); \ + } \ + deleted = *pivot; \ + parent = LLRB_PARENT(*pivot, field); \ + *pivot = 0; \ + while (root != pivot) { \ + pivot = LLRB_EDGE(head, parent, field); \ + parent = LLRB_PARENT(parent, field); \ + name##_LLRB_FIXUP(pivot); \ + } \ + return deleted; \ +} \ +attr struct type *name##_LLRB_DELETE(struct name *head, struct type *elm) { \ + struct type **root = &LLRB_ROOT(head), *parent = 0, *deleted = 0; \ + int comp; \ + while (*root) { \ + parent = LLRB_PARENT(*root, field); \ + comp = (cmp)(elm, *root); \ + if (comp < 0) { \ + if (LLRB_LEFT(*root, field) && !LLRB_ISRED(LLRB_LEFT(*root, field), field) && !LLRB_ISRED(LLRB_LEFT(LLRB_LEFT(*root, field), field), field)) \ + name##_LLRB_MOVL(root); \ + root = &LLRB_LEFT(*root, field); \ + } else { \ + if (LLRB_ISRED(LLRB_LEFT(*root, field), field)) { \ + name##_LLRB_ROTR(root); \ + comp = (cmp)(elm, *root); \ + } \ + if (!comp && !LLRB_RIGHT(*root, field)) { \ + deleted = *root; \ + *root = 0; \ + break; \ + } \ + if (LLRB_RIGHT(*root, field) && !LLRB_ISRED(LLRB_RIGHT(*root, field), field) && !LLRB_ISRED(LLRB_LEFT(LLRB_RIGHT(*root, field), field), field)) { \ + name##_LLRB_MOVR(root); \ + comp = (cmp)(elm, *root); \ + } \ + if (!comp) { \ + struct type *orphan = name##_DELETEMIN(head, &LLRB_RIGHT(*root, field)); \ + LLRB_COLOR(orphan, field) = LLRB_COLOR(*root, field); \ + LLRB_PARENT(orphan, field) = LLRB_PARENT(*root, field); \ + if ((LLRB_RIGHT(orphan, field) = LLRB_RIGHT(*root, field))) \ + LLRB_PARENT(LLRB_RIGHT(orphan, field), field) = orphan; \ + if ((LLRB_LEFT(orphan, field) = LLRB_LEFT(*root, field))) \ + LLRB_PARENT(LLRB_LEFT(orphan, field), field) = orphan; \ + deleted = *root; \ + *root = orphan; \ + parent = *root; \ + break; \ + } else \ + root = &LLRB_RIGHT(*root, field); \ + } \ + } \ + while (parent) { \ + root = LLRB_EDGE(head, parent, field); \ + parent = LLRB_PARENT(parent, field); \ + name##_LLRB_FIXUP(root); \ + } \ + if (LLRB_ROOT(head)) \ + LLRB_COLOR(LLRB_ROOT(head), field) = LLRB_BLACK; \ + return deleted; \ +} \ +attr struct type *name##_LLRB_FIND(struct name *head, struct type *key) { \ + struct type *elm = LLRB_ROOT(head); \ + while (elm) { \ + int comp = (cmp)(key, elm); \ + if (comp < 0) \ + elm = LLRB_LEFT(elm, field); \ + else if (comp > 0) \ + elm = LLRB_RIGHT(elm, field); \ + else \ + return elm; \ + } \ + return 0; \ +} \ +attr struct type *name##_LLRB_MIN(struct type *elm) { \ + while (elm && LLRB_LEFT(elm, field)) \ + elm = LLRB_LEFT(elm, field); \ + return elm; \ +} \ +attr struct type *name##_LLRB_MAX(struct type *elm) { \ + while (elm && LLRB_RIGHT(elm, field)) \ + elm = LLRB_RIGHT(elm, field); \ + return elm; \ +} \ +attr struct type *name##_LLRB_NEXT(struct type *elm) { \ + if (LLRB_RIGHT(elm, field)) { \ + return name##_LLRB_MIN(LLRB_RIGHT(elm, field)); \ + } else if (LLRB_PARENT(elm, field)) { \ + if (elm == LLRB_LEFT(LLRB_PARENT(elm, field), field)) \ + return LLRB_PARENT(elm, field); \ + while (LLRB_PARENT(elm, field) && elm == LLRB_RIGHT(LLRB_PARENT(elm, field), field)) \ + elm = LLRB_PARENT(elm, field); \ + return LLRB_PARENT(elm, field); \ + } else return 0; \ +} + +#define LLRB_INSERT(name, head, elm) name##_LLRB_INSERT((head), (elm)) +#define LLRB_DELETE(name, head, elm) name##_LLRB_DELETE((head), (elm)) +#define LLRB_REMOVE(name, head, elm) name##_LLRB_DELETE((head), (elm)) +#define LLRB_FIND(name, head, elm) name##_LLRB_FIND((head), (elm)) +#define LLRB_MIN(name, head) name##_LLRB_MIN(LLRB_ROOT((head))) +#define LLRB_MAX(name, head) name##_LLRB_MAX(LLRB_ROOT((head))) +#define LLRB_NEXT(name, head, elm) name##_LLRB_NEXT((elm)) + +#define LLRB_FOREACH(elm, name, head) \ +for ((elm) = LLRB_MIN(name, head); (elm); (elm) = name##_LLRB_NEXT((elm))) + +#endif /* LLRB_H */ + + +#include <stdlib.h> + +#include "timeout.h" +#include "bench.h" + + +struct rbtimeout { + timeout_t expires; + + int pending; + + LLRB_ENTRY(rbtimeout) rbe; +}; + +struct rbtimeouts { + timeout_t curtime; + LLRB_HEAD(tree, rbtimeout) tree; +}; + + +static int timeoutcmp(struct rbtimeout *a, struct rbtimeout *b) { + if (a->expires < b->expires) { + return -1; + } else if (a->expires > b->expires) { + return 1; + } else if (a < b) { + return -1; + } else if (a > b) { + return 1; + } else { + return 0; + } +} /* timeoutcmp() */ + +LLRB_GENERATE_STATIC(tree, rbtimeout, rbe, timeoutcmp) + +static void *init(struct timeout *timeout, size_t count, int verbose) { + struct rbtimeouts *T; + size_t i; + + T = malloc(sizeof *T); + T->curtime = 0; + LLRB_INIT(&T->tree); + + for (i = 0; i < count; i++) { + struct rbtimeout *to = (void *)&timeout[i]; + to->expires = 0; + to->pending = 0; + } + + return T; +} /* init() */ + + +static void add(void *ctx, struct timeout *_to, timeout_t expires) { + struct rbtimeouts *T = ctx; + struct rbtimeout *to = (void *)_to; + + if (to->pending) + LLRB_REMOVE(tree, &T->tree, to); + + to->expires = T->curtime + expires; + LLRB_INSERT(tree, &T->tree, to); + to->pending = 1; +} /* add() */ + + +static void del(void *ctx, struct timeout *_to) { + struct rbtimeouts *T = ctx; + struct rbtimeout *to = (void *)_to; + + LLRB_REMOVE(tree, &T->tree, to); + to->pending = 0; + to->expires = 0; +} /* del() */ + + +static struct timeout *get(void *ctx) { + struct rbtimeouts *T = ctx; + struct rbtimeout *to; + + if ((to = LLRB_MIN(tree, &T->tree)) && to->expires <= T->curtime) { + LLRB_REMOVE(tree, &T->tree, to); + to->pending = 0; + to->expires = 0; + + return (void *)to; + } + + return NULL; +} /* get() */ + + +static void update(void *ctx, timeout_t ts) { + struct rbtimeouts *T = ctx; + T->curtime = ts; +} /* update() */ + + +static void check(void *ctx) { + return; +} /* check() */ + + +static int empty(void *ctx) { + struct rbtimeouts *T = ctx; + + return LLRB_EMPTY(&T->tree); +} /* empty() */ + + +static void destroy(void *ctx) { + free(ctx); + return; +} /* destroy() */ + + +const struct benchops benchops = { + .init = &init, + .add = &add, + .del = &del, + .get = &get, + .update = &update, + .check = &check, + .empty = &empty, + .destroy = &destroy, +}; + diff --git a/src/ext/timeouts/bench/bench-wheel.c b/src/ext/timeouts/bench/bench-wheel.c new file mode 100644 index 0000000000..0cba1af83e --- /dev/null +++ b/src/ext/timeouts/bench/bench-wheel.c @@ -0,0 +1,81 @@ +#include <stdlib.h> + +#define TIMEOUT_PUBLIC static + +#include "timeout.h" +#include "timeout.c" +#include "bench.h" + + +static void *init(struct timeout *timeout, size_t count, int verbose) { + struct timeouts *T; + size_t i; + int error; + + T = timeouts_open(TIMEOUT_mHZ, &error); + + for (i = 0; i < count; i++) { + timeout_init(&timeout[i], 0); + } + +#if TIMEOUT_DEBUG - 0 + timeout_debug = verbose; +#endif + + return T; +} /* init() */ + + +static void add(void *T, struct timeout *to, timeout_t expires) { + timeouts_add(T, to, expires); +} /* add() */ + + +static void del(void *T, struct timeout *to) { + timeouts_del(T, to); +} /* del() */ + + +static struct timeout *get(void *T) { + return timeouts_get(T); +} /* get() */ + + +static void update(void *T, timeout_t ts) { + timeouts_update(T, ts); +} /* update() */ + + +static void (check)(void *T) { + if (!timeouts_check(T, stderr)) + _Exit(1); +} /* check() */ + + +static int empty(void *T) { + return !(timeouts_pending(T) || timeouts_expired(T)); +} /* empty() */ + + +static struct timeout *next(void *T, struct timeouts_it *it) { + return timeouts_next(T, it); +} /* next() */ + + +static void destroy(void *T) { + timeouts_close(T); +} /* destroy() */ + + +const struct benchops benchops = { + .init = &init, + .add = &add, + .del = &del, + .get = &get, + .update = &update, + .check = &check, + .empty = &empty, + .next = &next, + .destroy = &destroy +}; + diff --git a/src/ext/timeouts/bench/bench.c b/src/ext/timeouts/bench/bench.c new file mode 100644 index 0000000000..0d4cee44a0 --- /dev/null +++ b/src/ext/timeouts/bench/bench.c @@ -0,0 +1,293 @@ +#include <stdlib.h> +#include <string.h> +#include <time.h> +#include <errno.h> +#include <unistd.h> +#include <dlfcn.h> + +#if __APPLE__ +#include <mach/mach_time.h> +#endif + +#include <lua.h> +#include <lualib.h> +#include <lauxlib.h> + +#include "timeout.h" +#include "bench.h" + +#if LUA_VERSION_NUM < 502 +static int lua_absindex(lua_State *L, int idx) { + return (idx > 0 || idx <= LUA_REGISTRYINDEX)? idx : lua_gettop(L) + idx + 1; +} /* lua_absindex() */ + +static void luaL_setfuncs(lua_State *L, const luaL_Reg *l, int nup) { + int i, t = lua_absindex(L, -1 - nup); + + for (; l->name; l++) { + for (i = 0; i < nup; i++) + lua_pushvalue(L, -nup); + lua_pushcclosure(L, l->func, nup); + lua_setfield(L, t, l->name); + } + + lua_pop(L, nup); +} /* luaL_setfuncs() */ + +#define luaL_newlibtable(L, l) \ + lua_createtable(L, 0, (sizeof (l) / sizeof *(l)) - 1) + +#define luaL_newlib(L, l) \ + (luaL_newlibtable((L), (l)), luaL_setfuncs((L), (l), 0)) +#endif + +#ifndef MAX +#define MAX(a, b) (((a) > (b))? (a) : (b)) +#endif + + +struct bench { + const char *path; + void *solib; + size_t count; + timeout_t timeout_max; + int verbose; + + void *state; + struct timeout *timeout; + struct benchops ops; + timeout_t curtime; +}; /* struct bench */ + + +#if __APPLE__ +static mach_timebase_info_data_t timebase; +#endif + + +static int long long monotime(void) { +#if __APPLE__ + unsigned long long abt; + + abt = mach_absolute_time(); + abt = abt * timebase.numer / timebase.denom; + + return abt / 1000LL; +#else + struct timespec ts; + + clock_gettime(CLOCK_MONOTONIC, &ts); + + return (ts.tv_sec * 1000000L) + (ts.tv_nsec / 1000L); +#endif +} /* monotime() */ + + +static int bench_clock(lua_State *L) { + lua_pushnumber(L, (double)monotime() / 1000000L); + + return 1; +} /* bench_clock() */ + + +static int bench_new(lua_State *L) { + const char *path = luaL_checkstring(L, 1); + size_t count = luaL_optinteger(L, 2, 1000000); + timeout_t timeout_max = luaL_optinteger(L, 3, 300 * 1000000L); + int verbose = (lua_isnone(L, 4))? 0 : lua_toboolean(L, 4); + struct bench *B; + struct benchops *ops; + + B = lua_newuserdata(L, sizeof *B); + memset(B, 0, sizeof *B); + + luaL_getmetatable(L, "BENCH*"); + lua_setmetatable(L, -2); + + B->count = count; + B->timeout_max = timeout_max; + B->verbose = verbose; + + if (!(B->timeout = calloc(count, sizeof *B->timeout))) + return luaL_error(L, "%s", strerror(errno)); + + if (!(B->solib = dlopen(path, RTLD_NOW|RTLD_LOCAL))) + return luaL_error(L, "%s: %s", path, dlerror()); + + if (!(ops = dlsym(B->solib, "benchops"))) + return luaL_error(L, "%s: %s", path, dlerror()); + + B->ops = *ops; + B->state = B->ops.init(B->timeout, B->count, B->verbose); + + return 1; +} /* bench_new() */ + + +static int bench_add(lua_State *L) { + struct bench *B = lua_touserdata(L, 1); + unsigned i; + timeout_t t; + + i = (lua_isnoneornil(L, 2))? random() % B->count : (unsigned)luaL_checkinteger(L, 2); + t = (lua_isnoneornil(L, 3))? random() % B->timeout_max : (unsigned)luaL_checkinteger(L, 3); + + B->ops.add(B->state, &B->timeout[i], t); + + return 0; +} /* bench_add() */ + + +static int bench_del(lua_State *L) { + struct bench *B = lua_touserdata(L, 1); + size_t i = luaL_optinteger(L, 2, random() % B->count); + size_t j = luaL_optinteger(L, 3, i); + + while (i <= j && i < B->count) { + B->ops.del(B->state, &B->timeout[i]); + ++i; + } + + return 0; +} /* bench_del() */ + + +static int bench_fill(lua_State *L) { + struct bench *B = lua_touserdata(L, 1); + size_t count = luaL_optinteger(L, 2, B->count); + long timeout_inc = luaL_optinteger(L, 3, -1), timeout_max = 0, timeout; + size_t i; + + if (timeout_inc < 0) { + for (i = 0; i < count; i++) { + timeout = random() % B->timeout_max; + B->ops.add(B->state, &B->timeout[i], timeout); + timeout_max = MAX(timeout, timeout_max); + } + } else { + for (i = 0; i < count; i++) { + timeout = timeout_inc + i; + B->ops.add(B->state, &B->timeout[i], timeout_inc + i); + timeout_max = MAX(timeout, timeout_max); + } + } + + lua_pushinteger(L, (lua_Integer)count); + lua_pushinteger(L, (lua_Integer)timeout_max); + + return 2; +} /* bench_fill() */ + + +static int bench_expire(lua_State *L) { + struct bench *B = lua_touserdata(L, 1); + unsigned count = luaL_optinteger(L, 2, B->count); + unsigned step = luaL_optinteger(L, 3, 300000); + size_t i = 0; + + while (i < count && !B->ops.empty(B->state)) { + B->curtime += step; + B->ops.update(B->state, B->curtime); + + while (B->ops.get(B->state)) + i++; + } + + lua_pushinteger(L, (lua_Integer)i); + + return 1; +} /* bench_expire() */ + + +static int bench_empty(lua_State *L) { + struct bench *B = lua_touserdata(L, 1); + + lua_pushboolean(L, B->ops.empty(B->state)); + + return 1; +} /* bench_empty() */ + + +static int bench__next(lua_State *L) { + struct bench *B = lua_touserdata(L, lua_upvalueindex(1)); + struct timeouts_it *it = lua_touserdata(L, lua_upvalueindex(2)); + struct timeout *to; + + if (!B->ops.next || !(to = B->ops.next(B->state, it))) + return 0; + + lua_pushinteger(L, luaL_optinteger(L, 2, 0) + 1); + + lua_newtable(L); + lua_pushinteger(L, to->expires); + lua_setfield(L, -2, "expires"); + + return 2; +} /* bench__next() */ + +static int bench__pairs(lua_State *L) { + struct timeouts_it *it; + + lua_settop(L, 1); + + it = lua_newuserdata(L, sizeof *it); + TIMEOUTS_IT_INIT(it, TIMEOUTS_ALL); + + lua_pushcclosure(L, &bench__next, 2); + lua_pushvalue(L, 1); + lua_pushinteger(L, 0); + + return 3; +} /* bench__pairs() */ + + +static int bench__gc(lua_State *L) { + struct bench *B = lua_touserdata(L, 1); + + if (B->state) { + B->ops.destroy(B->state); + B->state = NULL; + } + + return 0; +} /* bench__gc() */ + + +static const luaL_Reg bench_methods[] = { + { "add", &bench_add }, + { "del", &bench_del }, + { "fill", &bench_fill }, + { "expire", &bench_expire }, + { "empty", &bench_empty }, + { "close", &bench__gc }, + { NULL, NULL } +}; + +static const luaL_Reg bench_metatable[] = { + { "__pairs", &bench__pairs }, + { "__gc", &bench__gc }, + { NULL, NULL } +}; + +static const luaL_Reg bench_globals[] = { + { "new", &bench_new }, + { "clock", &bench_clock }, + { NULL, NULL } +}; + +int luaopen_bench(lua_State *L) { +#if __APPLE__ + mach_timebase_info(&timebase); +#endif + + if (luaL_newmetatable(L, "BENCH*")) { + luaL_setfuncs(L, bench_metatable, 0); + luaL_newlib(L, bench_methods); + lua_setfield(L, -2, "__index"); + } + + luaL_newlib(L, bench_globals); + + return 1; +} /* luaopen_bench() */ + diff --git a/src/ext/timeouts/bench/bench.h b/src/ext/timeouts/bench/bench.h new file mode 100644 index 0000000000..bc1f7cf177 --- /dev/null +++ b/src/ext/timeouts/bench/bench.h @@ -0,0 +1,11 @@ +struct benchops { + void *(*init)(struct timeout *, size_t, int); + void (*add)(void *, struct timeout *, timeout_t); + void (*del)(void *, struct timeout *); + struct timeout *(*get)(void *); + void (*update)(void *, timeout_t); + void (*check)(void *); + int (*empty)(void *); + struct timeout *(*next)(void *, struct timeouts_it *); + void (*destroy)(void *); +}; /* struct benchops() */ diff --git a/src/ext/timeouts/bench/bench.plt b/src/ext/timeouts/bench/bench.plt new file mode 100644 index 0000000000..6e143c65e1 --- /dev/null +++ b/src/ext/timeouts/bench/bench.plt @@ -0,0 +1,19 @@ +set terminal postscript color + +set key top left +set xlabel "Number of timeouts" +set ylabel "Time\n(microseconds)" +#set logscale x + +set title "Time spent installing timeouts" font ",20" +plot 'heap-add.dat' using 1:($2*1000000) title "min-heap" with lines ls 1 lw 3 lc "red", \ + 'wheel-add.dat' using 1:($2*1000000) title "hierarchical wheel" with lines ls 1 lw 3 lc "forest-green" + +set title "Time spent deleting timeouts" font ",20" +plot 'heap-del.dat' using 1:($2*1000000) title "min-heap" with lines ls 1 lw 3 lc "red", \ + 'wheel-del.dat' using 1:($2*1000000) title "hierarchical wheel" with lines ls 1 lw 3 lc "forest-green" + +set title "Time spent expiring timeouts\n(by iteratively updating clock ~1000 times)" font ",20" +plot 'heap-expire.dat' using 1:($2*1000000) title "min-heap" with lines ls 1 lw 3 lc "red", \ + 'wheel-expire.dat' using 1:($2*1000000) title "hierarchical wheel" with lines ls 1 lw 3 lc "forest-green" + diff --git a/src/ext/timeouts/lua/Rules.mk b/src/ext/timeouts/lua/Rules.mk new file mode 100644 index 0000000000..0f06fce30b --- /dev/null +++ b/src/ext/timeouts/lua/Rules.mk @@ -0,0 +1,20 @@ +$(LUA_APIS:%=$(top_builddir)/lua/%/timeout.so): $(top_srcdir)/lua/timeout-lua.c $(top_srcdir)/timeout.h $(top_srcdir)/timeout.c + mkdir -p $(@D) + @$(SHRC); echo_cmd $(CC) -o $@ $(top_srcdir)/lua/timeout-lua.c -I$(top_srcdir) -DWHEEL_BIT=$(WHEEL_BIT) -DWHEEL_NUM=$(WHEEL_NUM) $(LUA53_CPPFLAGS) $(ALL_CPPFLAGS) $(ALL_CFLAGS) $(ALL_SOFLAGS) $(ALL_LDFLAGS) $(ALL_LIBS) + +$(top_builddir)/lua/5.1/timeouts.so: $(top_builddir)/lua/5.1/timeout.so +$(top_builddir)/lua/5.2/timeouts.so: $(top_builddir)/lua/5.2/timeout.so +$(top_builddir)/lua/5.3/timeouts.so: $(top_builddir)/lua/5.3/timeout.so + +$(LUA_APIS:%=$(top_builddir)/lua/%/timeouts.so): + cd $(@D) && ln -fs timeout.so timeouts.so + +lua-5.1: $(top_builddir)/lua/5.1/timeout.so $(top_builddir)/lua/5.1/timeouts.so +lua-5.2: $(top_builddir)/lua/5.2/timeout.so $(top_builddir)/lua/5.2/timeouts.so +lua-5.3: $(top_builddir)/lua/5.3/timeout.so $(top_builddir)/lua/5.3/timeouts.so + +lua-clean: + $(RM) -r $(top_builddir)/lua/5.? + +clean: lua-clean + diff --git a/src/ext/timeouts/lua/timeout-lua.c b/src/ext/timeouts/lua/timeout-lua.c new file mode 100644 index 0000000000..4d4e54cba6 --- /dev/null +++ b/src/ext/timeouts/lua/timeout-lua.c @@ -0,0 +1,396 @@ +#include <assert.h> +#include <string.h> + +#include <lua.h> +#include <lualib.h> +#include <lauxlib.h> + +#if LUA_VERSION_NUM != 503 +#error only Lua 5.3 supported +#endif + +#define TIMEOUT_PUBLIC static +#include "timeout.h" +#include "timeout.c" + +#define TIMEOUT_METANAME "struct timeout" +#define TIMEOUTS_METANAME "struct timeouts*" + +static struct timeout * +to_checkudata(lua_State *L, int index) +{ + return luaL_checkudata(L, index, TIMEOUT_METANAME); +} + +static struct timeouts * +tos_checkudata(lua_State *L, int index) +{ + return *(struct timeouts **)luaL_checkudata(L, index, TIMEOUTS_METANAME); +} + +static void +tos_bind(lua_State *L, int tos_index, int to_index) +{ + lua_getuservalue(L, tos_index); + lua_pushlightuserdata(L, to_checkudata(L, to_index)); + lua_pushvalue(L, to_index); + lua_rawset(L, -3); + lua_pop(L, 1); +} + +static void +tos_unbind(lua_State *L, int tos_index, int to_index) +{ + lua_getuservalue(L, tos_index); + lua_pushlightuserdata(L, to_checkudata(L, to_index)); + lua_pushnil(L); + lua_rawset(L, -3); + lua_pop(L, 1); +} + +static int +to__index(lua_State *L) +{ + struct timeout *to = to_checkudata(L, 1); + + if (lua_type(L, 2 == LUA_TSTRING)) { + const char *key = lua_tostring(L, 2); + + if (!strcmp(key, "flags")) { + lua_pushinteger(L, to->flags); + + return 1; + } else if (!strcmp(key, "expires")) { + lua_pushinteger(L, to->expires); + + return 1; + } + } + + if (LUA_TNIL != lua_getuservalue(L, 1)) { + lua_pushvalue(L, 2); + if (LUA_TNIL != lua_rawget(L, -2)) + return 1; + } + + lua_pushvalue(L, 2); + if (LUA_TNIL != lua_rawget(L, lua_upvalueindex(1))) + return 1; + + return 0; +} + +static int +to__newindex(lua_State *L) +{ + if (LUA_TNIL == lua_getuservalue(L, 1)) { + lua_newtable(L); + lua_pushvalue(L, -1); + lua_setuservalue(L, 1); + } + + lua_pushvalue(L, 2); + lua_pushvalue(L, 3); + lua_rawset(L, -3); + + return 0; +} + +static int +to__gc(lua_State *L) +{ + struct timeout *to = to_checkudata(L, 1); + + /* + * NB: On script exit it's possible for a timeout to still be + * associated with a timeouts object, particularly when the timeouts + * object was created first. + */ + timeout_del(to); + + return 0; +} + +static int +to_new(lua_State *L) +{ + int flags = luaL_optinteger(L, 1, 0); + struct timeout *to; + + to = lua_newuserdata(L, sizeof *to); + timeout_init(to, flags); + luaL_setmetatable(L, TIMEOUT_METANAME); + + return 1; +} + +static const luaL_Reg to_methods[] = { + { NULL, NULL }, +}; + +static const luaL_Reg to_metatable[] = { + { "__index", &to__index }, + { "__newindex", &to__newindex }, + { "__gc", &to__gc }, + { NULL, NULL }, +}; + +static const luaL_Reg to_globals[] = { + { "new", &to_new }, + { NULL, NULL }, +}; + +static void +to_newmetatable(lua_State *L) +{ + if (luaL_newmetatable(L, TIMEOUT_METANAME)) { + /* + * fill metamethod table, capturing the methods table as an + * upvalue for use by __index metamethod + */ + luaL_newlib(L, to_methods); + luaL_setfuncs(L, to_metatable, 1); + } +} + +int +luaopen_timeout(lua_State *L) +{ + to_newmetatable(L); + + luaL_newlib(L, to_globals); + lua_pushinteger(L, TIMEOUT_INT); + lua_setfield(L, -2, "INT"); + lua_pushinteger(L, TIMEOUT_ABS); + lua_setfield(L, -2, "ABS"); + + return 1; +} + +static int +tos_update(lua_State *L) +{ + struct timeouts *T = tos_checkudata(L, 1); + lua_Number n = luaL_checknumber(L, 2); + + timeouts_update(T, timeouts_f2i(T, n)); + + lua_pushvalue(L, 1); + + return 1; +} + +static int +tos_step(lua_State *L) +{ + struct timeouts *T = tos_checkudata(L, 1); + lua_Number n = luaL_checknumber(L, 2); + + timeouts_step(T, timeouts_f2i(T, n)); + + lua_pushvalue(L, 1); + + return 1; +} + +static int +tos_timeout(lua_State *L) +{ + struct timeouts *T = tos_checkudata(L, 1); + + lua_pushnumber(L, timeouts_i2f(T, timeouts_timeout(T))); + + return 1; +} + +static int +tos_add(lua_State *L) +{ + struct timeouts *T = tos_checkudata(L, 1); + struct timeout *to = to_checkudata(L, 2); + lua_Number timeout = luaL_checknumber(L, 3); + + tos_bind(L, 1, 2); + timeouts_addf(T, to, timeout); + + return lua_pushvalue(L, 1), 1; +} + +static int +tos_del(lua_State *L) +{ + struct timeouts *T = tos_checkudata(L, 1); + struct timeout *to = to_checkudata(L, 2); + + timeouts_del(T, to); + tos_unbind(L, 1, 2); + + return lua_pushvalue(L, 1), 1; +} + +static int +tos_get(lua_State *L) +{ + struct timeouts *T = tos_checkudata(L, 1); + struct timeout *to; + + if (!(to = timeouts_get(T))) + return 0; + + lua_getuservalue(L, 1); + lua_rawgetp(L, -1, to); + + if (!timeout_pending(to)) + tos_unbind(L, 1, lua_absindex(L, -1)); + + return 1; +} + +static int +tos_pending(lua_State *L) +{ + struct timeouts *T = tos_checkudata(L, 1); + + lua_pushboolean(L, timeouts_pending(T)); + + return 1; +} + +static int +tos_expired(lua_State *L) +{ + struct timeouts *T = tos_checkudata(L, 1); + + lua_pushboolean(L, timeouts_expired(T)); + + return 1; +} + +static int +tos_check(lua_State *L) +{ + struct timeouts *T = tos_checkudata(L, 1); + + lua_pushboolean(L, timeouts_check(T, NULL)); + + return 1; +} + +static int +tos__next(lua_State *L) +{ + struct timeouts *T = tos_checkudata(L, lua_upvalueindex(1)); + struct timeouts_it *it = lua_touserdata(L, lua_upvalueindex(2)); + struct timeout *to; + + if (!(to = timeouts_next(T, it))) + return 0; + + lua_getuservalue(L, lua_upvalueindex(1)); + lua_rawgetp(L, -1, to); + + return 1; +} + +static int +tos_timeouts(lua_State *L) +{ + int flags = luaL_checkinteger(L, 2); + struct timeouts_it *it; + + tos_checkudata(L, 1); + lua_pushvalue(L, 1); + it = lua_newuserdata(L, sizeof *it); + TIMEOUTS_IT_INIT(it, flags); + lua_pushcclosure(L, &tos__next, 2); + + return 1; +} + +static int +tos__gc(lua_State *L) +{ + struct timeouts **tos = luaL_checkudata(L, 1, TIMEOUTS_METANAME); + struct timeout *to; + + TIMEOUTS_FOREACH(to, *tos, TIMEOUTS_ALL) { + timeouts_del(*tos, to); + } + + timeouts_close(*tos); + *tos = NULL; + + return 0; +} + +static int +tos_new(lua_State *L) +{ + timeout_t hz = luaL_optinteger(L, 1, 0); + struct timeouts **T; + int error; + + T = lua_newuserdata(L, sizeof *T); + luaL_setmetatable(L, TIMEOUTS_METANAME); + + lua_newtable(L); + lua_setuservalue(L, -2); + + if (!(*T = timeouts_open(hz, &error))) + return luaL_error(L, "%s", strerror(error)); + + return 1; +} + +static const luaL_Reg tos_methods[] = { + { "update", &tos_update }, + { "step", &tos_step }, + { "timeout", &tos_timeout }, + { "add", &tos_add }, + { "del", &tos_del }, + { "get", &tos_get }, + { "pending", &tos_pending }, + { "expired", &tos_expired }, + { "check", &tos_check }, + { "timeouts", &tos_timeouts }, + { NULL, NULL }, +}; + +static const luaL_Reg tos_metatable[] = { + { "__gc", &tos__gc }, + { NULL, NULL }, +}; + +static const luaL_Reg tos_globals[] = { + { "new", &tos_new }, + { NULL, NULL }, +}; + +static void +tos_newmetatable(lua_State *L) +{ + if (luaL_newmetatable(L, TIMEOUTS_METANAME)) { + luaL_setfuncs(L, tos_metatable, 0); + luaL_newlib(L, tos_methods); + lua_setfield(L, -2, "__index"); + } +} + +int +luaopen_timeouts(lua_State *L) +{ + to_newmetatable(L); + tos_newmetatable(L); + + luaL_newlib(L, tos_globals); + lua_pushinteger(L, TIMEOUTS_PENDING); + lua_setfield(L, -2, "PENDING"); + lua_pushinteger(L, TIMEOUTS_EXPIRED); + lua_setfield(L, -2, "EXPIRED"); + lua_pushinteger(L, TIMEOUTS_ALL); + lua_setfield(L, -2, "ALL"); + lua_pushinteger(L, TIMEOUTS_CLEAR); + lua_setfield(L, -2, "CLEAR"); + + return 1; +} diff --git a/src/ext/timeouts/test-timeout.c b/src/ext/timeouts/test-timeout.c new file mode 100644 index 0000000000..8077129376 --- /dev/null +++ b/src/ext/timeouts/test-timeout.c @@ -0,0 +1,530 @@ +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <assert.h> +#include <limits.h> + +#include "timeout.h" + +#define THE_END_OF_TIME ((timeout_t)-1) + +static int check_misc(void) { + if (TIMEOUT_VERSION != timeout_version()) + return 1; + if (TIMEOUT_V_REL != timeout_v_rel()) + return 1; + if (TIMEOUT_V_API != timeout_v_api()) + return 1; + if (TIMEOUT_V_ABI != timeout_v_abi()) + return 1; + if (strcmp(timeout_vendor(), TIMEOUT_VENDOR)) + return 1; + return 0; +} + +static int check_open_close(timeout_t hz_set, timeout_t hz_expect) { + int err=0; + struct timeouts *tos = timeouts_open(hz_set, &err); + if (!tos) + return 1; + if (err) + return 1; + if (hz_expect != timeouts_hz(tos)) + return 1; + timeouts_close(tos); + return 0; +} + +/* Not very random */ +static timeout_t random_to(timeout_t min, timeout_t max) +{ + if (max <= min) + return min; + /* Not actually all that random, but should exercise the code. */ + timeout_t rand64 = random() * (timeout_t)INT_MAX + random(); + return min + (rand64 % (max-min)); +} + +/* configuration for check_randomized */ +struct rand_cfg { + /* When creating timeouts, smallest possible delay */ + timeout_t min_timeout; + /* When creating timeouts, largest possible delay */ + timeout_t max_timeout; + /* First time to start the clock at. */ + timeout_t start_at; + /* Do not advance the clock past this time. */ + timeout_t end_at; + /* Number of timeouts to create and monitor. */ + int n_timeouts; + /* Advance the clock by no more than this each step. */ + timeout_t max_step; + /* Use relative timers and stepping */ + int relative; + /* Every time the clock ticks, try removing this many timeouts at + * random. */ + int try_removing; + /* When we're done, advance the clock to the end of time. */ + int finalize; +}; + +static int check_randomized(const struct rand_cfg *cfg) +{ +#define FAIL() do { \ + printf("Failure on line %d\n", __LINE__); \ + goto done; \ + } while (0) + + int i, err; + int rv = 1; + struct timeout *t = calloc(cfg->n_timeouts, sizeof(struct timeout)); + timeout_t *timeouts = calloc(cfg->n_timeouts, sizeof(timeout_t)); + uint8_t *fired = calloc(cfg->n_timeouts, sizeof(uint8_t)); + uint8_t *found = calloc(cfg->n_timeouts, sizeof(uint8_t)); + uint8_t *deleted = calloc(cfg->n_timeouts, sizeof(uint8_t)); + struct timeouts *tos = timeouts_open(0, &err); + timeout_t now = cfg->start_at; + int n_added_pending = 0, cnt_added_pending = 0; + int n_added_expired = 0, cnt_added_expired = 0; + struct timeouts_it it_p, it_e, it_all; + int p_done = 0, e_done = 0, all_done = 0; + struct timeout *to = NULL; + const int rel = cfg->relative; + + if (!t || !timeouts || !tos || !fired || !found || !deleted) + FAIL(); + timeouts_update(tos, cfg->start_at); + + for (i = 0; i < cfg->n_timeouts; ++i) { + if (&t[i] != timeout_init(&t[i], rel ? 0 : TIMEOUT_ABS)) + FAIL(); + if (timeout_pending(&t[i])) + FAIL(); + if (timeout_expired(&t[i])) + FAIL(); + + timeouts[i] = random_to(cfg->min_timeout, cfg->max_timeout); + + timeouts_add(tos, &t[i], timeouts[i] - (rel ? now : 0)); + if (timeouts[i] <= cfg->start_at) { + if (timeout_pending(&t[i])) + FAIL(); + if (! timeout_expired(&t[i])) + FAIL(); + ++n_added_expired; + } else { + if (! timeout_pending(&t[i])) + FAIL(); + if (timeout_expired(&t[i])) + FAIL(); + ++n_added_pending; + } + } + + if (!!n_added_pending != timeouts_pending(tos)) + FAIL(); + if (!!n_added_expired != timeouts_expired(tos)) + FAIL(); + + /* Test foreach, interleaving a few iterators. */ + TIMEOUTS_IT_INIT(&it_p, TIMEOUTS_PENDING); + TIMEOUTS_IT_INIT(&it_e, TIMEOUTS_EXPIRED); + TIMEOUTS_IT_INIT(&it_all, TIMEOUTS_ALL); + while (! (p_done && e_done && all_done)) { + if (!p_done) { + to = timeouts_next(tos, &it_p); + if (to) { + i = to - &t[0]; + ++found[i]; + ++cnt_added_pending; + } else { + p_done = 1; + } + } + if (!e_done) { + to = timeouts_next(tos, &it_e); + if (to) { + i = to - &t[0]; + ++found[i]; + ++cnt_added_expired; + } else { + e_done = 1; + } + } + if (!all_done) { + to = timeouts_next(tos, &it_all); + if (to) { + i = to - &t[0]; + ++found[i]; + } else { + all_done = 1; + } + } + } + + for (i = 0; i < cfg->n_timeouts; ++i) { + if (found[i] != 2) + FAIL(); + } + if (cnt_added_expired != n_added_expired) + FAIL(); + if (cnt_added_pending != n_added_pending) + FAIL(); + + while (NULL != (to = timeouts_get(tos))) { + i = to - &t[0]; + assert(&t[i] == to); + if (timeouts[i] > cfg->start_at) + FAIL(); /* shouldn't have happened yet */ + + --n_added_expired; /* drop expired timeouts. */ + ++fired[i]; + } + + if (n_added_expired != 0) + FAIL(); + + while (now < cfg->end_at) { + int n_fired_this_time = 0; + timeout_t first_at = timeouts_timeout(tos) + now; + + timeout_t oldtime = now; + timeout_t step = random_to(1, cfg->max_step); + int another; + now += step; + if (rel) + timeouts_step(tos, step); + else + timeouts_update(tos, now); + + for (i = 0; i < cfg->try_removing; ++i) { + int idx = random() % cfg->n_timeouts; + if (! fired[idx]) { + timeout_del(&t[idx]); + ++deleted[idx]; + } + } + + another = (timeouts_timeout(tos) == 0); + + while (NULL != (to = timeouts_get(tos))) { + if (! another) + FAIL(); /* Thought we saw the last one! */ + i = to - &t[0]; + assert(&t[i] == to); + if (timeouts[i] > now) + FAIL(); /* shouldn't have happened yet */ + if (timeouts[i] <= oldtime) + FAIL(); /* should have happened already */ + if (timeouts[i] < first_at) + FAIL(); /* first_at should've been earlier */ + fired[i]++; + n_fired_this_time++; + another = (timeouts_timeout(tos) == 0); + } + if (n_fired_this_time && first_at > now) + FAIL(); /* first_at should've been earlier */ + if (another) + FAIL(); /* Huh? We think there are more? */ + if (!timeouts_check(tos, stderr)) + FAIL(); + } + + for (i = 0; i < cfg->n_timeouts; ++i) { + if (fired[i] > 1) + FAIL(); /* Nothing fired twice. */ + if (timeouts[i] <= now) { + if (!(fired[i] || deleted[i])) + FAIL(); + } else { + if (fired[i]) + FAIL(); + } + if (fired[i] && deleted[i]) + FAIL(); + if (cfg->finalize > 1) { + if (!fired[i]) + timeout_del(&t[i]); + } + } + + /* Now nothing more should fire between now and the end of time. */ + if (cfg->finalize) { + timeouts_update(tos, THE_END_OF_TIME); + if (cfg->finalize > 1) { + if (timeouts_get(tos)) + FAIL(); + TIMEOUTS_FOREACH(to, tos, TIMEOUTS_ALL) + FAIL(); + } + } + rv = 0; + + done: + if (tos) timeouts_close(tos); + if (t) free(t); + if (timeouts) free(timeouts); + if (fired) free(fired); + if (found) free(found); + if (deleted) free(deleted); + return rv; +} + +struct intervals_cfg { + const timeout_t *timeouts; + int n_timeouts; + timeout_t start_at; + timeout_t end_at; + timeout_t skip; +}; + +int +check_intervals(struct intervals_cfg *cfg) +{ + int i, err; + int rv = 1; + struct timeout *to; + struct timeout *t = calloc(cfg->n_timeouts, sizeof(struct timeout)); + unsigned *fired = calloc(cfg->n_timeouts, sizeof(unsigned)); + struct timeouts *tos = timeouts_open(0, &err); + + timeout_t now = cfg->start_at; + if (!t || !tos || !fired) + FAIL(); + + timeouts_update(tos, now); + + for (i = 0; i < cfg->n_timeouts; ++i) { + if (&t[i] != timeout_init(&t[i], TIMEOUT_INT)) + FAIL(); + if (timeout_pending(&t[i])) + FAIL(); + if (timeout_expired(&t[i])) + FAIL(); + + timeouts_add(tos, &t[i], cfg->timeouts[i]); + if (! timeout_pending(&t[i])) + FAIL(); + if (timeout_expired(&t[i])) + FAIL(); + } + + while (now < cfg->end_at) { + timeout_t delay = timeouts_timeout(tos); + if (cfg->skip && delay < cfg->skip) + delay = cfg->skip; + timeouts_step(tos, delay); + now += delay; + + while (NULL != (to = timeouts_get(tos))) { + i = to - &t[0]; + assert(&t[i] == to); + fired[i]++; + if (0 != (to->expires - cfg->start_at) % cfg->timeouts[i]) + FAIL(); + if (to->expires <= now) + FAIL(); + if (to->expires > now + cfg->timeouts[i]) + FAIL(); + } + if (!timeouts_check(tos, stderr)) + FAIL(); + } + + timeout_t duration = now - cfg->start_at; + for (i = 0; i < cfg->n_timeouts; ++i) { + if (cfg->skip) { + if (fired[i] > duration / cfg->timeouts[i]) + FAIL(); + } else { + if (fired[i] != duration / cfg->timeouts[i]) + FAIL(); + } + if (!timeout_pending(&t[i])) + FAIL(); + } + + rv = 0; + done: + if (t) free(t); + if (fired) free(fired); + if (tos) free(tos); + return rv; +} + +int +main(int argc, char **argv) +{ + int j; + int n_failed = 0; +#define DO(fn) do { \ + printf("."); fflush(stdout); \ + if (fn) { \ + ++n_failed; \ + printf("%s failed\n", #fn); \ + } \ + } while (0) + +#define DO_N(n, fn) do { \ + for (j = 0; j < (n); ++j) { \ + DO(fn); \ + } \ + } while (0) + + DO(check_misc()); + DO(check_open_close(1000, 1000)); + DO(check_open_close(0, TIMEOUT_mHZ)); + + struct rand_cfg cfg1 = { + .min_timeout = 1, + .max_timeout = 100, + .start_at = 5, + .end_at = 1000, + .n_timeouts = 1000, + .max_step = 10, + .relative = 0, + .try_removing = 0, + .finalize = 2, + }; + DO_N(300,check_randomized(&cfg1)); + + struct rand_cfg cfg2 = { + .min_timeout = 20, + .max_timeout = 1000, + .start_at = 10, + .end_at = 100, + .n_timeouts = 1000, + .max_step = 5, + .relative = 1, + .try_removing = 0, + .finalize = 2, + }; + DO_N(300,check_randomized(&cfg2)); + + struct rand_cfg cfg2b = { + .min_timeout = 20, + .max_timeout = 1000, + .start_at = 10, + .end_at = 100, + .n_timeouts = 1000, + .max_step = 5, + .relative = 1, + .try_removing = 0, + .finalize = 1, + }; + DO_N(300,check_randomized(&cfg2b)); + + struct rand_cfg cfg2c = { + .min_timeout = 20, + .max_timeout = 1000, + .start_at = 10, + .end_at = 100, + .n_timeouts = 1000, + .max_step = 5, + .relative = 1, + .try_removing = 0, + .finalize = 0, + }; + DO_N(300,check_randomized(&cfg2c)); + + struct rand_cfg cfg3 = { + .min_timeout = 2000, + .max_timeout = ((uint64_t)1) << 50, + .start_at = 100, + .end_at = ((uint64_t)1) << 49, + .n_timeouts = 1000, + .max_step = 1<<31, + .relative = 0, + .try_removing = 0, + .finalize = 2, + }; + DO_N(10,check_randomized(&cfg3)); + + struct rand_cfg cfg3b = { + .min_timeout = ((uint64_t)1) << 50, + .max_timeout = ((uint64_t)1) << 52, + .start_at = 100, + .end_at = ((uint64_t)1) << 53, + .n_timeouts = 1000, + .max_step = ((uint64_t)1)<<48, + .relative = 0, + .try_removing = 0, + .finalize = 2, + }; + DO_N(10,check_randomized(&cfg3b)); + + struct rand_cfg cfg4 = { + .min_timeout = 2000, + .max_timeout = ((uint64_t)1) << 30, + .start_at = 100, + .end_at = ((uint64_t)1) << 26, + .n_timeouts = 10000, + .max_step = 1<<16, + .relative = 0, + .try_removing = 3, + .finalize = 2, + }; + DO_N(10,check_randomized(&cfg4)); + + const timeout_t primes[] = { + 2,3,5,7,11,13,17,19,23,29,31,37,41,43,47,53, + 59,61,67,71,73,79,83,89,97 + }; + const timeout_t factors_of_1337[] = { + 1, 7, 191, 1337 + }; + const timeout_t multiples_of_five[] = { + 5, 10, 15, 20, 25, 30, 35, 40, 45, 50 + }; + + struct intervals_cfg icfg1 = { + .timeouts = primes, + .n_timeouts = sizeof(primes)/sizeof(timeout_t), + .start_at = 50, + .end_at = 5322, + .skip = 0, + }; + DO(check_intervals(&icfg1)); + + struct intervals_cfg icfg2 = { + .timeouts = factors_of_1337, + .n_timeouts = sizeof(factors_of_1337)/sizeof(timeout_t), + .start_at = 50, + .end_at = 50000, + .skip = 0, + }; + DO(check_intervals(&icfg2)); + + struct intervals_cfg icfg3 = { + .timeouts = multiples_of_five, + .n_timeouts = sizeof(multiples_of_five)/sizeof(timeout_t), + .start_at = 49, + .end_at = 5333, + .skip = 0, + }; + DO(check_intervals(&icfg3)); + + struct intervals_cfg icfg4 = { + .timeouts = primes, + .n_timeouts = sizeof(primes)/sizeof(timeout_t), + .start_at = 50, + .end_at = 5322, + .skip = 16, + }; + DO(check_intervals(&icfg4)); + + if (n_failed) { + puts("\nFAIL"); + } else { + puts("\nOK"); + } + return !!n_failed; +} + +/* TODO: + + * Solve PR#3. + + * Investigate whether any untaken branches are possible. + + */ diff --git a/src/ext/timeouts/timeout-bitops.c b/src/ext/timeouts/timeout-bitops.c new file mode 100644 index 0000000000..a018f33b95 --- /dev/null +++ b/src/ext/timeouts/timeout-bitops.c @@ -0,0 +1,254 @@ +#include <stdint.h> +#include <limits.h> +#ifdef _MSC_VER +#include <intrin.h> /* _BitScanForward, _BitScanReverse */ +#endif + +/* First define ctz and clz functions; these are compiler-dependent if + * you want them to be fast. */ +#if defined(__GNUC__) && !defined(TIMEOUT_DISABLE_GNUC_BITOPS) + +#ifndef LONG_BIT +#define LONG_BIT (SIZEOF_LONG*CHAR_BIT) +#endif + +/* On GCC and clang and some others, we can use __builtin functions. They + * are not defined for n==0, but timeout.s never calls them with n==0. */ + +#define ctz64(n) __builtin_ctzll(n) +#define clz64(n) __builtin_clzll(n) +#if LONG_BIT == 32 +#define ctz32(n) __builtin_ctzl(n) +#define clz32(n) __builtin_clzl(n) +#else +#define ctz32(n) __builtin_ctz(n) +#define clz32(n) __builtin_clz(n) +#endif + +#elif defined(_MSC_VER) && !defined(TIMEOUT_DISABLE_MSVC_BITOPS) + +/* On MSVC, we have these handy functions. We can ignore their return + * values, since we will never supply val == 0. */ + +static __inline int ctz32(unsigned long val) +{ + DWORD zeros = 0; + _BitScanForward(&zeros, val); + return zeros; +} +static __inline int clz32(unsigned long val) +{ + DWORD zeros = 0; + _BitScanReverse(&zeros, val); + return zeros; +} +#ifdef _WIN64 +/* According to the documentation, these only exist on Win64. */ +static __inline int ctz64(uint64_t val) +{ + DWORD zeros = 0; + _BitScanForward64(&zeros, val); + return zeros; +} +static __inline int clz64(uint64_t val) +{ + DWORD zeros = 0; + _BitScanReverse64(&zeros, val); + return zeros; +} +#else +static __inline int ctz64(uint64_t val) +{ + uint32_t lo = (uint32_t) val; + uint32_t hi = (uint32_t) (val >> 32); + return lo ? ctz32(lo) : 32 + ctz32(hi); +} +static __inline int clz64(uint64_t val) +{ + uint32_t lo = (uint32_t) val; + uint32_t hi = (uint32_t) (val >> 32); + return hi ? clz32(hi) : 32 + clz32(lo); +} +#endif + +/* End of MSVC case. */ + +#else + +/* TODO: There are more clever ways to do this in the generic case. */ + + +#define process_(one, cz_bits, bits) \ + if (x < ( one << (cz_bits - bits))) { rv += bits; x <<= bits; } + +#define process64(bits) process_((UINT64_C(1)), 64, (bits)) +static inline int clz64(uint64_t x) +{ + int rv = 0; + + process64(32); + process64(16); + process64(8); + process64(4); + process64(2); + process64(1); + return rv; +} +#define process32(bits) process_((UINT32_C(1)), 32, (bits)) +static inline int clz32(uint32_t x) +{ + int rv = 0; + + process32(16); + process32(8); + process32(4); + process32(2); + process32(1); + return rv; +} + +#undef process_ +#undef process32 +#undef process64 +#define process_(one, bits) \ + if ((x & ((one << (bits))-1)) == 0) { rv += bits; x >>= bits; } + +#define process64(bits) process_((UINT64_C(1)), bits) +static inline int ctz64(uint64_t x) +{ + int rv = 0; + + process64(32); + process64(16); + process64(8); + process64(4); + process64(2); + process64(1); + return rv; +} + +#define process32(bits) process_((UINT32_C(1)), bits) +static inline int ctz32(uint32_t x) +{ + int rv = 0; + + process32(16); + process32(8); + process32(4); + process32(2); + process32(1); + return rv; +} + +#undef process32 +#undef process64 +#undef process_ + +/* End of generic case */ + +#endif /* End of defining ctz */ + +#ifdef TEST_BITOPS +#include <stdio.h> +#include <stdlib.h> + +static uint64_t testcases[] = { + 13371337 * 10, + 100, + 385789752, + 82574, + (((uint64_t)1)<<63) + (((uint64_t)1)<<31) + 10101 +}; + +static int +naive_clz(int bits, uint64_t v) +{ + int r = 0; + uint64_t bit = ((uint64_t)1) << (bits-1); + while (bit && 0 == (v & bit)) { + r++; + bit >>= 1; + } + /* printf("clz(%d,%lx) -> %d\n", bits, v, r); */ + return r; +} + +static int +naive_ctz(int bits, uint64_t v) +{ + int r = 0; + uint64_t bit = 1; + while (bit && 0 == (v & bit)) { + r++; + bit <<= 1; + if (r == bits) + break; + } + /* printf("ctz(%d,%lx) -> %d\n", bits, v, r); */ + return r; +} + +static int +check(uint64_t vv) +{ + uint32_t v32 = (uint32_t) vv; + + if (vv == 0) + return 1; /* c[tl]z64(0) is undefined. */ + + if (ctz64(vv) != naive_ctz(64, vv)) { + printf("mismatch with ctz64: %d\n", ctz64(vv)); + exit(1); + return 0; + } + if (clz64(vv) != naive_clz(64, vv)) { + printf("mismatch with clz64: %d\n", clz64(vv)); + exit(1); + return 0; + } + + if (v32 == 0) + return 1; /* c[lt]z(0) is undefined. */ + + if (ctz32(v32) != naive_ctz(32, v32)) { + printf("mismatch with ctz32: %d\n", ctz32(v32)); + exit(1); + return 0; + } + if (clz32(v32) != naive_clz(32, v32)) { + printf("mismatch with clz32: %d\n", clz32(v32)); + exit(1); + return 0; + } + return 1; +} + +int +main(int c, char **v) +{ + unsigned int i; + const unsigned int n = sizeof(testcases)/sizeof(testcases[0]); + int result = 0; + + for (i = 0; i <= 63; ++i) { + uint64_t x = 1 << i; + if (!check(x)) + result = 1; + --x; + if (!check(x)) + result = 1; + } + + for (i = 0; i < n; ++i) { + if (! check(testcases[i])) + result = 1; + } + if (result) { + puts("FAIL"); + } else { + puts("OK"); + } + return result; +} +#endif + diff --git a/src/ext/timeouts/timeout-debug.h b/src/ext/timeouts/timeout-debug.h new file mode 100644 index 0000000000..fc727a6b42 --- /dev/null +++ b/src/ext/timeouts/timeout-debug.h @@ -0,0 +1,77 @@ +/* + * D E B U G R O U T I N E S + * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#if TIMEOUT_DEBUG - 0 +#include <stdlib.h> +#include <stdio.h> + +#undef TIMEOUT_DEBUG +#define TIMEOUT_DEBUG 1 +#define DEBUG_LEVEL timeout_debug + +static int timeout_debug; + +#define SAYit_(lvl, fmt, ...) do { \ + if (DEBUG_LEVEL >= (lvl)) \ + fprintf(stderr, fmt "%s", __FILE__, __LINE__, __func__, __VA_ARGS__); \ +} while (0) + +#define SAYit(lvl, ...) SAYit_((lvl), "%s:%d:%s: " __VA_ARGS__, "\n") + +#define PANIC(...) do { \ + SAYit(0, __VA_ARGS__); \ + _Exit(EXIT_FAILURE); \ +} while (0) +#else +#undef TIMEOUT_DEBUG +#define TIMEOUT_DEBUG 0 +#define DEBUG_LEVEL 0 + +#define SAYit(...) (void)0 +#endif + +#define SAY(...) SAYit(1, __VA_ARGS__) +#define HAI SAY("HAI") + + +static inline char *fmt_(char *buf, uint64_t ts, int wheel_bit, int wheel_num) { + char *p = buf; + int wheel, n, i; + + for (wheel = wheel_num - 2; wheel >= 0; wheel--) { + n = ((1 << wheel_bit) - 1) & (ts >> (wheel * WHEEL_BIT)); + + for (i = wheel_bit - 1; i >= 0; i--) { + *p++ = '0' + !!(n & (1 << i)); + } + + if (wheel != 0) + *p++ = ':'; + } + + *p = 0; + + return buf; +} /* fmt_() */ + +#define fmt(ts) fmt_(((char[((1 << WHEEL_BIT) * WHEEL_NUM) + WHEEL_NUM + 1]){ 0 }), (ts), WHEEL_BIT, WHEEL_NUM) + + +static inline char *bin64_(char *buf, uint64_t n, int wheel_bit) { + char *p = buf; + int i; + + for (i = 0; i < (1 << wheel_bit); i++) { + *p++ = "01"[0x1 & (n >> (((1 << wheel_bit) - 1) - i))]; + } + + *p = 0; + + return buf; +} /* bin64_() */ + +#define bin64(ts) bin64_(((char[((1 << WHEEL_BIT) * WHEEL_NUM) + 1]){ 0 }), (ts), WHEEL_BIT) + + diff --git a/src/ext/timeouts/timeout.c b/src/ext/timeouts/timeout.c new file mode 100644 index 0000000000..bd463a700d --- /dev/null +++ b/src/ext/timeouts/timeout.c @@ -0,0 +1,754 @@ +/* ========================================================================== + * timeout.c - Tickless hierarchical timing wheel. + * -------------------------------------------------------------------------- + * Copyright (c) 2013, 2014 William Ahern + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to permit + * persons to whom the Software is furnished to do so, subject to the + * following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * ========================================================================== + */ +#ifdef HAVE_CONFIG_H +#include "orconfig.h" +#endif +#include <limits.h> /* CHAR_BIT */ + +#include <stddef.h> /* NULL */ +#include <stdlib.h> /* malloc(3) free(3) */ +#include <stdio.h> /* FILE fprintf(3) */ + +#include <inttypes.h> /* UINT64_C uint64_t */ + +#include <string.h> /* memset(3) */ + +#include <errno.h> /* errno */ + +#include "tor_queue.h" /* TAILQ(3) */ + +#include "timeout.h" + +#ifndef TIMEOUT_DEBUG +#define TIMEOUT_DEBUG 0 +#endif + +#if TIMEOUT_DEBUG - 0 +#include "timeout-debug.h" +#endif + +#ifdef TIMEOUT_DISABLE_RELATIVE_ACCESS +#define TO_SET_TIMEOUTS(to, T) ((void)0) +#else +#define TO_SET_TIMEOUTS(to, T) ((to)->timeouts = (T)) +#endif + +/* + * A N C I L L A R Y R O U T I N E S + * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#define abstime_t timeout_t /* for documentation purposes */ +#define reltime_t timeout_t /* "" */ + +#if !defined countof +#define countof(a) (sizeof (a) / sizeof *(a)) +#endif + +#if !defined endof +#define endof(a) (&(a)[countof(a)]) +#endif + +#if !defined MIN +#define MIN(a, b) (((a) < (b))? (a) : (b)) +#endif + +#if !defined MAX +#define MAX(a, b) (((a) > (b))? (a) : (b)) +#endif + +#if !defined TOR_TAILQ_CONCAT +#define TOR_TAILQ_CONCAT(head1, head2, field) do { \ + if (!TOR_TAILQ_EMPTY(head2)) { \ + *(head1)->tqh_last = (head2)->tqh_first; \ + (head2)->tqh_first->field.tqe_prev = (head1)->tqh_last; \ + (head1)->tqh_last = (head2)->tqh_last; \ + TOR_TAILQ_INIT((head2)); \ + } \ +} while (0) +#endif + +#if !defined TOR_TAILQ_FOREACH_SAFE +#define TOR_TAILQ_FOREACH_SAFE(var, head, field, tvar) \ + for ((var) = TOR_TAILQ_FIRST(head); \ + (var) && ((tvar) = TOR_TAILQ_NEXT(var, field), 1); \ + (var) = (tvar)) +#endif + + +/* + * B I T M A N I P U L A T I O N R O U T I N E S + * + * The macros and routines below implement wheel parameterization. The + * inputs are: + * + * WHEEL_BIT - The number of value bits mapped in each wheel. The + * lowest-order WHEEL_BIT bits index the lowest-order (highest + * resolution) wheel, the next group of WHEEL_BIT bits the + * higher wheel, etc. + * + * WHEEL_NUM - The number of wheels. WHEEL_BIT * WHEEL_NUM = the number of + * value bits used by all the wheels. For the default of 6 and + * 4, only the low 24 bits are processed. Any timeout value + * larger than this will cycle through again. + * + * The implementation uses bit fields to remember which slot in each wheel + * is populated, and to generate masks of expiring slots according to the + * current update interval (i.e. the "tickless" aspect). The slots to + * process in a wheel are (populated-set & interval-mask). + * + * WHEEL_BIT cannot be larger than 6 bits because 2^6 -> 64 is the largest + * number of slots which can be tracked in a uint64_t integer bit field. + * WHEEL_BIT cannot be smaller than 3 bits because of our rotr and rotl + * routines, which only operate on all the value bits in an integer, and + * there's no integer smaller than uint8_t. + * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#if !defined WHEEL_BIT +#define WHEEL_BIT 6 +#endif + +#if !defined WHEEL_NUM +#define WHEEL_NUM 4 +#endif + +#define WHEEL_LEN (1U << WHEEL_BIT) +#define WHEEL_MAX (WHEEL_LEN - 1) +#define WHEEL_MASK (WHEEL_LEN - 1) +#define TIMEOUT_MAX ((TIMEOUT_C(1) << (WHEEL_BIT * WHEEL_NUM)) - 1) + +#include "timeout-bitops.c" + +#if WHEEL_BIT == 6 +#define ctz(n) ctz64(n) +#define clz(n) clz64(n) +#define fls(n) ((int)(64 - clz64(n))) +#else +#define ctz(n) ctz32(n) +#define clz(n) clz32(n) +#define fls(n) ((int)(32 - clz32(n))) +#endif + +#if WHEEL_BIT == 6 +#define WHEEL_C(n) UINT64_C(n) +#define WHEEL_PRIu PRIu64 +#define WHEEL_PRIx PRIx64 + +typedef uint64_t wheel_t; + +#elif WHEEL_BIT == 5 + +#define WHEEL_C(n) UINT32_C(n) +#define WHEEL_PRIu PRIu32 +#define WHEEL_PRIx PRIx32 + +typedef uint32_t wheel_t; + +#elif WHEEL_BIT == 4 + +#define WHEEL_C(n) UINT16_C(n) +#define WHEEL_PRIu PRIu16 +#define WHEEL_PRIx PRIx16 + +typedef uint16_t wheel_t; + +#elif WHEEL_BIT == 3 + +#define WHEEL_C(n) UINT8_C(n) +#define WHEEL_PRIu PRIu8 +#define WHEEL_PRIx PRIx8 + +typedef uint8_t wheel_t; + +#else +#error invalid WHEEL_BIT value +#endif + + +static inline wheel_t rotl(const wheel_t v, int c) { + if (!(c &= (sizeof v * CHAR_BIT - 1))) + return v; + + return (v << c) | (v >> (sizeof v * CHAR_BIT - c)); +} /* rotl() */ + + +static inline wheel_t rotr(const wheel_t v, int c) { + if (!(c &= (sizeof v * CHAR_BIT - 1))) + return v; + + return (v >> c) | (v << (sizeof v * CHAR_BIT - c)); +} /* rotr() */ + + +/* + * T I M E R R O U T I N E S + * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +TOR_TAILQ_HEAD(timeout_list, timeout); + +struct timeouts { + struct timeout_list wheel[WHEEL_NUM][WHEEL_LEN], expired; + + wheel_t pending[WHEEL_NUM]; + + timeout_t curtime; + timeout_t hertz; +}; /* struct timeouts */ + + +static struct timeouts *timeouts_init(struct timeouts *T, timeout_t hz) { + unsigned i, j; + + for (i = 0; i < countof(T->wheel); i++) { + for (j = 0; j < countof(T->wheel[i]); j++) { + TOR_TAILQ_INIT(&T->wheel[i][j]); + } + } + + TOR_TAILQ_INIT(&T->expired); + + for (i = 0; i < countof(T->pending); i++) { + T->pending[i] = 0; + } + + T->curtime = 0; + T->hertz = (hz)? hz : TIMEOUT_mHZ; + + return T; +} /* timeouts_init() */ + + +TIMEOUT_PUBLIC struct timeouts *timeouts_open(timeout_t hz, int *error) { + struct timeouts *T; + + if ((T = malloc(sizeof *T))) + return timeouts_init(T, hz); + + *error = errno; + + return NULL; +} /* timeouts_open() */ + + +static void timeouts_reset(struct timeouts *T) { + struct timeout_list reset; + struct timeout *to; + unsigned i, j; + + TOR_TAILQ_INIT(&reset); + + for (i = 0; i < countof(T->wheel); i++) { + for (j = 0; j < countof(T->wheel[i]); j++) { + TOR_TAILQ_CONCAT(&reset, &T->wheel[i][j], tqe); + } + } + + TOR_TAILQ_CONCAT(&reset, &T->expired, tqe); + + TOR_TAILQ_FOREACH(to, &reset, tqe) { + to->pending = NULL; + TO_SET_TIMEOUTS(to, NULL); + } +} /* timeouts_reset() */ + + +TIMEOUT_PUBLIC void timeouts_close(struct timeouts *T) { + /* + * NOTE: Delete installed timeouts so timeout_pending() and + * timeout_expired() worked as expected. + */ + timeouts_reset(T); + + free(T); +} /* timeouts_close() */ + + +TIMEOUT_PUBLIC timeout_t timeouts_hz(struct timeouts *T) { + return T->hertz; +} /* timeouts_hz() */ + + +TIMEOUT_PUBLIC void timeouts_del(struct timeouts *T, struct timeout *to) { + if (to->pending) { + TOR_TAILQ_REMOVE(to->pending, to, tqe); + + if (to->pending != &T->expired && TOR_TAILQ_EMPTY(to->pending)) { + ptrdiff_t index = to->pending - &T->wheel[0][0]; + int wheel = (int) (index / WHEEL_LEN); + int slot = index % WHEEL_LEN; + + T->pending[wheel] &= ~(WHEEL_C(1) << slot); + } + + to->pending = NULL; + TO_SET_TIMEOUTS(to, NULL); + } +} /* timeouts_del() */ + + +static inline reltime_t timeout_rem(struct timeouts *T, struct timeout *to) { + return to->expires - T->curtime; +} /* timeout_rem() */ + + +static inline int timeout_wheel(timeout_t timeout) { + /* must be called with timeout != 0, so fls input is nonzero */ + return (fls(MIN(timeout, TIMEOUT_MAX)) - 1) / WHEEL_BIT; +} /* timeout_wheel() */ + + +static inline int timeout_slot(int wheel, timeout_t expires) { + return WHEEL_MASK & ((expires >> (wheel * WHEEL_BIT)) - !!wheel); +} /* timeout_slot() */ + + +static void timeouts_sched(struct timeouts *T, struct timeout *to, timeout_t expires) { + timeout_t rem; + int wheel, slot; + + timeouts_del(T, to); + + to->expires = expires; + + TO_SET_TIMEOUTS(to, T); + + if (expires > T->curtime) { + rem = timeout_rem(T, to); + + /* rem is nonzero since: + * rem == timeout_rem(T,to), + * == to->expires - T->curtime + * and above we have expires > T->curtime. + */ + wheel = timeout_wheel(rem); + slot = timeout_slot(wheel, to->expires); + + to->pending = &T->wheel[wheel][slot]; + TOR_TAILQ_INSERT_TAIL(to->pending, to, tqe); + + T->pending[wheel] |= WHEEL_C(1) << slot; + } else { + to->pending = &T->expired; + TOR_TAILQ_INSERT_TAIL(to->pending, to, tqe); + } +} /* timeouts_sched() */ + + +#ifndef TIMEOUT_DISABLE_INTERVALS +static void timeouts_readd(struct timeouts *T, struct timeout *to) { + to->expires += to->interval; + + if (to->expires <= T->curtime) { + /* If we've missed the next firing of this timeout, reschedule + * it to occur at the next multiple of its interval after + * the last time that it fired. + */ + timeout_t n = T->curtime - to->expires; + timeout_t r = n % to->interval; + to->expires = T->curtime + (to->interval - r); + } + + timeouts_sched(T, to, to->expires); +} /* timeouts_readd() */ +#endif + + +TIMEOUT_PUBLIC void timeouts_add(struct timeouts *T, struct timeout *to, timeout_t timeout) { +#ifndef TIMEOUT_DISABLE_INTERVALS + if (to->flags & TIMEOUT_INT) + to->interval = MAX(1, timeout); +#endif + + if (to->flags & TIMEOUT_ABS) + timeouts_sched(T, to, timeout); + else + timeouts_sched(T, to, T->curtime + timeout); +} /* timeouts_add() */ + + +TIMEOUT_PUBLIC void timeouts_update(struct timeouts *T, abstime_t curtime) { + timeout_t elapsed = curtime - T->curtime; + struct timeout_list todo; + int wheel; + + TOR_TAILQ_INIT(&todo); + + /* + * There's no avoiding looping over every wheel. It's best to keep + * WHEEL_NUM smallish. + */ + for (wheel = 0; wheel < WHEEL_NUM; wheel++) { + wheel_t pending; + + /* + * Calculate the slots expiring in this wheel + * + * If the elapsed time is greater than the maximum period of + * the wheel, mark every position as expiring. + * + * Otherwise, to determine the expired slots fill in all the + * bits between the last slot processed and the current + * slot, inclusive of the last slot. We'll bitwise-AND this + * with our pending set below. + * + * If a wheel rolls over, force a tick of the next higher + * wheel. + */ + if ((elapsed >> (wheel * WHEEL_BIT)) > WHEEL_MAX) { + pending = (wheel_t)~WHEEL_C(0); + } else { + wheel_t _elapsed = WHEEL_MASK & (elapsed >> (wheel * WHEEL_BIT)); + int oslot, nslot; + + /* + * TODO: It's likely that at least one of the + * following three bit fill operations is redundant + * or can be replaced with a simpler operation. + */ + oslot = WHEEL_MASK & (T->curtime >> (wheel * WHEEL_BIT)); + pending = rotl(((UINT64_C(1) << _elapsed) - 1), oslot); + + nslot = WHEEL_MASK & (curtime >> (wheel * WHEEL_BIT)); + pending |= rotr(rotl(((WHEEL_C(1) << _elapsed) - 1), nslot), (int)_elapsed); + pending |= WHEEL_C(1) << nslot; + } + + while (pending & T->pending[wheel]) { + /* ctz input cannot be zero: loop condition. */ + int slot = ctz(pending & T->pending[wheel]); + TOR_TAILQ_CONCAT(&todo, &T->wheel[wheel][slot], tqe); + T->pending[wheel] &= ~(UINT64_C(1) << slot); + } + + if (!(0x1 & pending)) + break; /* break if we didn't wrap around end of wheel */ + + /* if we're continuing, the next wheel must tick at least once */ + elapsed = MAX(elapsed, (WHEEL_LEN << (wheel * WHEEL_BIT))); + } + + T->curtime = curtime; + + while (!TOR_TAILQ_EMPTY(&todo)) { + struct timeout *to = TOR_TAILQ_FIRST(&todo); + + TOR_TAILQ_REMOVE(&todo, to, tqe); + to->pending = NULL; + + timeouts_sched(T, to, to->expires); + } + + return; +} /* timeouts_update() */ + +TIMEOUT_PUBLIC timeout_t timeouts_get_curtime(struct timeouts *T) { + return T->curtime; +} /* timeouts_get_curtime() */ + +TIMEOUT_PUBLIC void timeouts_step(struct timeouts *T, reltime_t elapsed) { + timeouts_update(T, T->curtime + elapsed); +} /* timeouts_step() */ + + +TIMEOUT_PUBLIC bool timeouts_pending(struct timeouts *T) { + wheel_t pending = 0; + int wheel; + + for (wheel = 0; wheel < WHEEL_NUM; wheel++) { + pending |= T->pending[wheel]; + } + + return !!pending; +} /* timeouts_pending() */ + + +TIMEOUT_PUBLIC bool timeouts_expired(struct timeouts *T) { + return !TOR_TAILQ_EMPTY(&T->expired); +} /* timeouts_expired() */ + + +/* + * Calculate the interval before needing to process any timeouts pending on + * any wheel. + * + * (This is separated from the public API routine so we can evaluate our + * wheel invariant assertions irrespective of the expired queue.) + * + * This might return a timeout value sooner than any installed timeout if + * only higher-order wheels have timeouts pending. We can only know when to + * process a wheel, not precisely when a timeout is scheduled. Our timeout + * accuracy could be off by 2^(N*M)-1 units where N is the wheel number and + * M is WHEEL_BIT. Only timeouts which have fallen through to wheel 0 can be + * known exactly. + * + * We should never return a timeout larger than the lowest actual timeout. + */ +static timeout_t timeouts_int(struct timeouts *T) { + timeout_t timeout = ~TIMEOUT_C(0), _timeout; + timeout_t relmask; + int wheel, slot; + + relmask = 0; + + for (wheel = 0; wheel < WHEEL_NUM; wheel++) { + if (T->pending[wheel]) { + slot = WHEEL_MASK & (T->curtime >> (wheel * WHEEL_BIT)); + + /* ctz input cannot be zero: T->pending[wheel] is + * nonzero, so rotr() is nonzero. */ + _timeout = (ctz(rotr(T->pending[wheel], slot)) + !!wheel) << (wheel * WHEEL_BIT); + /* +1 to higher order wheels as those timeouts are one rotation in the future (otherwise they'd be on a lower wheel or expired) */ + + _timeout -= relmask & T->curtime; + /* reduce by how much lower wheels have progressed */ + + timeout = MIN(_timeout, timeout); + } + + relmask <<= WHEEL_BIT; + relmask |= WHEEL_MASK; + } + + return timeout; +} /* timeouts_int() */ + + +/* + * Calculate the interval our caller can wait before needing to process + * events. + */ +TIMEOUT_PUBLIC timeout_t timeouts_timeout(struct timeouts *T) { + if (!TOR_TAILQ_EMPTY(&T->expired)) + return 0; + + return timeouts_int(T); +} /* timeouts_timeout() */ + + +TIMEOUT_PUBLIC struct timeout *timeouts_get(struct timeouts *T) { + if (!TOR_TAILQ_EMPTY(&T->expired)) { + struct timeout *to = TOR_TAILQ_FIRST(&T->expired); + + TOR_TAILQ_REMOVE(&T->expired, to, tqe); + to->pending = NULL; + TO_SET_TIMEOUTS(to, NULL); + +#ifndef TIMEOUT_DISABLE_INTERVALS + if ((to->flags & TIMEOUT_INT) && to->interval > 0) + timeouts_readd(T, to); +#endif + + return to; + } else { + return 0; + } +} /* timeouts_get() */ + + +/* + * Use dumb looping to locate the earliest timeout pending on the wheel so + * our invariant assertions can check the result of our optimized code. + */ +static struct timeout *timeouts_min(struct timeouts *T) { + struct timeout *to, *min = NULL; + unsigned i, j; + + for (i = 0; i < countof(T->wheel); i++) { + for (j = 0; j < countof(T->wheel[i]); j++) { + TOR_TAILQ_FOREACH(to, &T->wheel[i][j], tqe) { + if (!min || to->expires < min->expires) + min = to; + } + } + } + + return min; +} /* timeouts_min() */ + + +/* + * Check some basic algorithm invariants. If these invariants fail then + * something is definitely broken. + */ +#define report(...) do { \ + if ((fp)) \ + fprintf(fp, __VA_ARGS__); \ +} while (0) + +#define check(expr, ...) do { \ + if (!(expr)) { \ + report(__VA_ARGS__); \ + return 0; \ + } \ +} while (0) + +TIMEOUT_PUBLIC bool timeouts_check(struct timeouts *T, FILE *fp) { + timeout_t timeout; + struct timeout *to; + + if ((to = timeouts_min(T))) { + check(to->expires > T->curtime, "missed timeout (expires:%" TIMEOUT_PRIu " <= curtime:%" TIMEOUT_PRIu ")\n", to->expires, T->curtime); + + timeout = timeouts_int(T); + check(timeout <= to->expires - T->curtime, "wrong soft timeout (soft:%" TIMEOUT_PRIu " > hard:%" TIMEOUT_PRIu ") (expires:%" TIMEOUT_PRIu "; curtime:%" TIMEOUT_PRIu ")\n", timeout, (to->expires - T->curtime), to->expires, T->curtime); + + timeout = timeouts_timeout(T); + check(timeout <= to->expires - T->curtime, "wrong soft timeout (soft:%" TIMEOUT_PRIu " > hard:%" TIMEOUT_PRIu ") (expires:%" TIMEOUT_PRIu "; curtime:%" TIMEOUT_PRIu ")\n", timeout, (to->expires - T->curtime), to->expires, T->curtime); + } else { + timeout = timeouts_timeout(T); + + if (!TOR_TAILQ_EMPTY(&T->expired)) + check(timeout == 0, "wrong soft timeout (soft:%" TIMEOUT_PRIu " != hard:%" TIMEOUT_PRIu ")\n", timeout, TIMEOUT_C(0)); + else + check(timeout == ~TIMEOUT_C(0), "wrong soft timeout (soft:%" TIMEOUT_PRIu " != hard:%" TIMEOUT_PRIu ")\n", timeout, ~TIMEOUT_C(0)); + } + + return 1; +} /* timeouts_check() */ + + +#define ENTER \ + do { \ + static const int pc0 = __LINE__; \ + switch (pc0 + it->pc) { \ + case __LINE__: (void)0 + +#define SAVE_AND_DO(do_statement) \ + do { \ + it->pc = __LINE__ - pc0; \ + do_statement; \ + case __LINE__: (void)0; \ + } while (0) + +#define YIELD(rv) \ + SAVE_AND_DO(return (rv)) + +#define LEAVE \ + SAVE_AND_DO(break); \ + } \ + } while (0) + +TIMEOUT_PUBLIC struct timeout *timeouts_next(struct timeouts *T, struct timeouts_it *it) { + struct timeout *to; + + ENTER; + + if (it->flags & TIMEOUTS_EXPIRED) { + if (it->flags & TIMEOUTS_CLEAR) { + while ((to = timeouts_get(T))) { + YIELD(to); + } + } else { + TOR_TAILQ_FOREACH_SAFE(to, &T->expired, tqe, it->to) { + YIELD(to); + } + } + } + + if (it->flags & TIMEOUTS_PENDING) { + for (it->i = 0; it->i < countof(T->wheel); it->i++) { + for (it->j = 0; it->j < countof(T->wheel[it->i]); it->j++) { + TOR_TAILQ_FOREACH_SAFE(to, &T->wheel[it->i][it->j], tqe, it->to) { + YIELD(to); + } + } + } + } + + LEAVE; + + return NULL; +} /* timeouts_next */ + +#undef LEAVE +#undef YIELD +#undef SAVE_AND_DO +#undef ENTER + + +/* + * T I M E O U T R O U T I N E S + * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +TIMEOUT_PUBLIC struct timeout *timeout_init(struct timeout *to, int flags) { + memset(to, 0, sizeof *to); + + to->flags = flags; + + return to; +} /* timeout_init() */ + + +#ifndef TIMEOUT_DISABLE_RELATIVE_ACCESS +TIMEOUT_PUBLIC bool timeout_pending(struct timeout *to) { + return to->pending && to->pending != &to->timeouts->expired; +} /* timeout_pending() */ + + +TIMEOUT_PUBLIC bool timeout_expired(struct timeout *to) { + return to->pending && to->pending == &to->timeouts->expired; +} /* timeout_expired() */ + + +TIMEOUT_PUBLIC void timeout_del(struct timeout *to) { + timeouts_del(to->timeouts, to); +} /* timeout_del() */ +#endif + + +/* + * V E R S I O N I N T E R F A C E S + * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +TIMEOUT_PUBLIC int timeout_version(void) { + return TIMEOUT_VERSION; +} /* timeout_version() */ + + +TIMEOUT_PUBLIC const char *timeout_vendor(void) { + return TIMEOUT_VENDOR; +} /* timeout_version() */ + + +TIMEOUT_PUBLIC int timeout_v_rel(void) { + return TIMEOUT_V_REL; +} /* timeout_version() */ + + +TIMEOUT_PUBLIC int timeout_v_abi(void) { + return TIMEOUT_V_ABI; +} /* timeout_version() */ + + +TIMEOUT_PUBLIC int timeout_v_api(void) { + return TIMEOUT_V_API; +} /* timeout_version() */ + diff --git a/src/ext/timeouts/timeout.h b/src/ext/timeouts/timeout.h new file mode 100644 index 0000000000..b35874e153 --- /dev/null +++ b/src/ext/timeouts/timeout.h @@ -0,0 +1,256 @@ +/* ========================================================================== + * timeout.h - Tickless hierarchical timing wheel. + * -------------------------------------------------------------------------- + * Copyright (c) 2013, 2014 William Ahern + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to permit + * persons to whom the Software is furnished to do so, subject to the + * following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * ========================================================================== + */ +#ifndef TIMEOUT_H +#define TIMEOUT_H + +#include <stdbool.h> /* bool */ +#include <stdio.h> /* FILE */ + +#include <inttypes.h> /* PRIu64 PRIx64 PRIX64 uint64_t */ + +#include "tor_queue.h" /* TAILQ(3) */ + + +/* + * V E R S I O N I N T E R F A C E S + * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#if !defined TIMEOUT_PUBLIC +#define TIMEOUT_PUBLIC +#endif + +#define TIMEOUT_VERSION TIMEOUT_V_REL +#define TIMEOUT_VENDOR "william@25thandClement.com" + +#define TIMEOUT_V_REL 0x20160226 +#define TIMEOUT_V_ABI 0x20160224 +#define TIMEOUT_V_API 0x20160226 + +TIMEOUT_PUBLIC int timeout_version(void); + +TIMEOUT_PUBLIC const char *timeout_vendor(void); + +TIMEOUT_PUBLIC int timeout_v_rel(void); + +TIMEOUT_PUBLIC int timeout_v_abi(void); + +TIMEOUT_PUBLIC int timeout_v_api(void); + + +/* + * I N T E G E R T Y P E I N T E R F A C E S + * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#define TIMEOUT_C(n) UINT64_C(n) +#define TIMEOUT_PRIu PRIu64 +#define TIMEOUT_PRIx PRIx64 +#define TIMEOUT_PRIX PRIX64 + +#define TIMEOUT_mHZ TIMEOUT_C(1000) +#define TIMEOUT_uHZ TIMEOUT_C(1000000) +#define TIMEOUT_nHZ TIMEOUT_C(1000000000) + +typedef uint64_t timeout_t; + +#define timeout_error_t int /* for documentation purposes */ + + +/* + * C A L L B A C K I N T E R F A C E + * + * Callback function parameters unspecified to make embedding into existing + * applications easier. + * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#ifndef TIMEOUT_CB_OVERRIDE +struct timeout_cb { + void (*fn)(void); + void *arg; +}; /* struct timeout_cb */ +#endif + +/* + * T I M E O U T I N T E R F A C E S + * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#ifndef TIMEOUT_DISABLE_INTERVALS +#define TIMEOUT_INT 0x01 /* interval (repeating) timeout */ +#endif +#define TIMEOUT_ABS 0x02 /* treat timeout values as absolute */ + +#define TIMEOUT_INITIALIZER(flags) { (flags) } + +#define timeout_setcb(to, fn, arg) do { \ + (to)->callback.fn = (fn); \ + (to)->callback.arg = (arg); \ +} while (0) + +struct timeout { + int flags; + + timeout_t expires; + /* absolute expiration time */ + + struct timeout_list *pending; + /* timeout list if pending on wheel or expiry queue */ + + TOR_TAILQ_ENTRY(timeout) tqe; + /* entry member for struct timeout_list lists */ + +#ifndef TIMEOUT_DISABLE_CALLBACKS + struct timeout_cb callback; + /* optional callback information */ +#endif + +#ifndef TIMEOUT_DISABLE_INTERVALS + timeout_t interval; + /* timeout interval if periodic */ +#endif + +#ifndef TIMEOUT_DISABLE_RELATIVE_ACCESS + struct timeouts *timeouts; + /* timeouts collection if member of */ +#endif +}; /* struct timeout */ + + +TIMEOUT_PUBLIC struct timeout *timeout_init(struct timeout *, int); +/* initialize timeout structure (same as TIMEOUT_INITIALIZER) */ + +#ifndef TIMEOUT_DISABLE_RELATIVE_ACCESS +TIMEOUT_PUBLIC bool timeout_pending(struct timeout *); +/* true if on timing wheel, false otherwise */ + +TIMEOUT_PUBLIC bool timeout_expired(struct timeout *); +/* true if on expired queue, false otherwise */ + +TIMEOUT_PUBLIC void timeout_del(struct timeout *); +/* remove timeout from any timing wheel (okay if not member of any) */ +#endif + +/* + * T I M I N G W H E E L I N T E R F A C E S + * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +struct timeouts; + +TIMEOUT_PUBLIC struct timeouts *timeouts_open(timeout_t, timeout_error_t *); +/* open a new timing wheel, setting optional HZ (for float conversions) */ + +TIMEOUT_PUBLIC void timeouts_close(struct timeouts *); +/* destroy timing wheel */ + +TIMEOUT_PUBLIC timeout_t timeouts_hz(struct timeouts *); +/* return HZ setting (for float conversions) */ + +TIMEOUT_PUBLIC void timeouts_update(struct timeouts *, timeout_t); +/* update timing wheel with current absolute time */ + +TIMEOUT_PUBLIC void timeouts_step(struct timeouts *, timeout_t); +/* step timing wheel by relative time */ + +TIMEOUT_PUBLIC timeout_t timeouts_get_curtime(struct timeouts *); +/* Return the current tick. */ + +TIMEOUT_PUBLIC timeout_t timeouts_timeout(struct timeouts *); +/* return interval to next required update */ + +TIMEOUT_PUBLIC void timeouts_add(struct timeouts *, struct timeout *, timeout_t); +/* add timeout to timing wheel */ + +TIMEOUT_PUBLIC void timeouts_del(struct timeouts *, struct timeout *); +/* remove timeout from any timing wheel or expired queue (okay if on neither) */ + +TIMEOUT_PUBLIC struct timeout *timeouts_get(struct timeouts *); +/* return any expired timeout (caller should loop until NULL-return) */ + +TIMEOUT_PUBLIC bool timeouts_pending(struct timeouts *); +/* return true if any timeouts pending on timing wheel */ + +TIMEOUT_PUBLIC bool timeouts_expired(struct timeouts *); +/* return true if any timeouts on expired queue */ + +TIMEOUT_PUBLIC bool timeouts_check(struct timeouts *, FILE *); +/* return true if invariants hold. describes failures to optional file handle. */ + +#define TIMEOUTS_PENDING 0x10 +#define TIMEOUTS_EXPIRED 0x20 +#define TIMEOUTS_ALL (TIMEOUTS_PENDING|TIMEOUTS_EXPIRED) +#define TIMEOUTS_CLEAR 0x40 + +#define TIMEOUTS_IT_INITIALIZER(flags) { (flags), 0, 0, 0, 0 } + +#define TIMEOUTS_IT_INIT(cur, _flags) do { \ + (cur)->flags = (_flags); \ + (cur)->pc = 0; \ +} while (0) + +struct timeouts_it { + int flags; + unsigned pc, i, j; + struct timeout *to; +}; /* struct timeouts_it */ + +TIMEOUT_PUBLIC struct timeout *timeouts_next(struct timeouts *, struct timeouts_it *); +/* return next timeout in pending wheel or expired queue. caller can delete + * the returned timeout, but should not otherwise manipulate the timing + * wheel. in particular, caller SHOULD NOT delete any other timeout as that + * could invalidate cursor state and trigger a use-after-free. + */ + +#define TIMEOUTS_FOREACH(var, T, flags) \ + struct timeouts_it _it = TIMEOUTS_IT_INITIALIZER((flags)); \ + while (((var) = timeouts_next((T), &_it))) + + +/* + * B O N U S W H E E L I N T E R F A C E S + * + * I usually use floating point timeouts in all my code, but it's cleaner to + * separate it to keep the core algorithmic code simple. + * + * Using macros instead of static inline routines where <math.h> routines + * might be used to keep -lm linking optional. + * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include <math.h> /* ceil(3) */ + +#define timeouts_f2i(T, f) \ + ((timeout_t)ceil((f) * timeouts_hz((T)))) /* prefer late expiration over early */ + +#define timeouts_i2f(T, i) \ + ((double)(i) / timeouts_hz((T))) + +#define timeouts_addf(T, to, timeout) \ + timeouts_add((T), (to), timeouts_f2i((T), (timeout))) + +#endif /* TIMEOUT_H */ diff --git a/src/ext/tinytest.c b/src/ext/tinytest.c index f6baeeb9a5..3fb1b39c71 100644 --- a/src/ext/tinytest.c +++ b/src/ext/tinytest.c @@ -69,15 +69,16 @@ static int n_skipped = 0; /**< Number of tests that have been skipped. */ static int opt_forked = 0; /**< True iff we're called from inside a win32 fork*/ static int opt_nofork = 0; /**< Suppress calls to fork() for debugging. */ static int opt_verbosity = 1; /**< -==quiet,0==terse,1==normal,2==verbose */ -const char *verbosity_flag = ""; +static const char *verbosity_flag = ""; -const struct testlist_alias_t *cfg_aliases=NULL; +static const struct testlist_alias_t *cfg_aliases=NULL; enum outcome { SKIP=2, OK=1, FAIL=0 }; static enum outcome cur_test_outcome = 0; -const char *cur_test_prefix = NULL; /**< prefix of the current test group */ +/** prefix of the current test group */ +static const char *cur_test_prefix = NULL; /** Name of the current test, if we haven't logged is yet. Used for --quiet */ -const char *cur_test_name = NULL; +static const char *cur_test_name = NULL; #ifdef _WIN32 /* Copy of argv[0] for win32. */ diff --git a/src/or/buffers.c b/src/or/buffers.c index f93cc48f33..8b9a53c699 100644 --- a/src/or/buffers.c +++ b/src/or/buffers.c @@ -107,7 +107,7 @@ chunk_repack(chunk_t *chunk) /** Keep track of total size of allocated chunks for consistency asserts */ static size_t total_bytes_allocated_in_chunks = 0; static void -chunk_free_unchecked(chunk_t *chunk) +buf_chunk_free_unchecked(chunk_t *chunk) { if (!chunk) return; @@ -228,7 +228,7 @@ buf_pullup(buf_t *buf, size_t bytes) dest->next = src->next; if (buf->tail == src) buf->tail = dest; - chunk_free_unchecked(src); + buf_chunk_free_unchecked(src); } else { memcpy(CHUNK_WRITE_PTR(dest), src->data, n); dest->datalen += n; @@ -274,7 +274,7 @@ buf_remove_from_front(buf_t *buf, size_t n) buf->head = victim->next; if (buf->tail == victim) buf->tail = NULL; - chunk_free_unchecked(victim); + buf_chunk_free_unchecked(victim); } } check(); @@ -314,7 +314,7 @@ buf_clear(buf_t *buf) buf->datalen = 0; for (chunk = buf->head; chunk; chunk = next) { next = chunk->next; - chunk_free_unchecked(chunk); + buf_chunk_free_unchecked(chunk); } buf->head = buf->tail = NULL; } @@ -509,12 +509,12 @@ read_to_chunk_tls(buf_t *buf, chunk_t *chunk, tor_tls_t *tls, * (because of EOF), set *<b>reached_eof</b> to 1 and return 0. Return -1 on * error; else return the number of bytes read. */ -/* XXXX024 indicate "read blocked" somehow? */ +/* XXXX indicate "read blocked" somehow? */ int read_to_buf(tor_socket_t s, size_t at_most, buf_t *buf, int *reached_eof, int *socket_error) { - /* XXXX024 It's stupid to overload the return values for these functions: + /* XXXX It's stupid to overload the return values for these functions: * "error status" and "number of bytes read" are not mutually exclusive. */ int r = 0; @@ -687,7 +687,7 @@ flush_chunk_tls(tor_tls_t *tls, buf_t *buf, chunk_t *chunk, int flush_buf(tor_socket_t s, buf_t *buf, size_t sz, size_t *buf_flushlen) { - /* XXXX024 It's stupid to overload the return values for these functions: + /* XXXX It's stupid to overload the return values for these functions: * "error status" and "number of bytes flushed" are not mutually exclusive. */ int r; diff --git a/src/or/channel.c b/src/or/channel.c index 5f69a0864b..87fa721089 100644 --- a/src/or/channel.c +++ b/src/or/channel.c @@ -122,7 +122,7 @@ STATIC uint64_t estimated_total_queue_size = 0; * If more than one channel exists, follow the next_with_same_id pointer * as a linked list. */ -HT_HEAD(channel_idmap, channel_idmap_entry_s) channel_identity_map = +static HT_HEAD(channel_idmap, channel_idmap_entry_s) channel_identity_map = HT_INITIALIZER(); typedef struct channel_idmap_entry_s { @@ -145,9 +145,9 @@ channel_idmap_eq(const channel_idmap_entry_t *a, } HT_PROTOTYPE(channel_idmap, channel_idmap_entry_s, node, channel_idmap_hash, - channel_idmap_eq); + channel_idmap_eq) HT_GENERATE2(channel_idmap, channel_idmap_entry_s, node, channel_idmap_hash, - channel_idmap_eq, 0.5, tor_reallocarray_, tor_free_); + channel_idmap_eq, 0.5, tor_reallocarray_, tor_free_) static cell_queue_entry_t * cell_queue_entry_dup(cell_queue_entry_t *q); #if 0 @@ -3510,7 +3510,7 @@ channel_dump_statistics, (channel_t *chan, int severity)) have_remote_addr = channel_get_addr_if_possible(chan, &remote_addr); if (have_remote_addr) { char *actual = tor_strdup(channel_get_actual_remote_descr(chan)); - remote_addr_str = tor_dup_addr(&remote_addr); + remote_addr_str = tor_addr_to_str_dup(&remote_addr); tor_log(severity, LD_GENERAL, " * Channel " U64_FORMAT " says its remote address" " is %s, and gives a canonical description of \"%s\" and an " @@ -4524,8 +4524,8 @@ channel_update_xmit_queue_size(channel_t *chan) /* Next, adjust by the overhead factor, if any is available */ if (chan->get_overhead_estimate) { overhead = chan->get_overhead_estimate(chan); - if (overhead >= 1.0f) { - queued *= overhead; + if (overhead >= 1.0) { + queued = (uint64_t)(queued * overhead); } else { /* Ignore silly overhead factors */ log_notice(LD_CHANNEL, "Ignoring silly overhead factor %f", overhead); diff --git a/src/or/channel.h b/src/or/channel.h index 129c0c2013..78e1b71014 100644 --- a/src/or/channel.h +++ b/src/or/channel.h @@ -18,7 +18,7 @@ typedef void (*channel_cell_handler_fn_ptr)(channel_t *, cell_t *); typedef void (*channel_var_cell_handler_fn_ptr)(channel_t *, var_cell_t *); struct cell_queue_entry_s; -TOR_SIMPLEQ_HEAD(chan_cell_queue, cell_queue_entry_s) incoming_queue; +TOR_SIMPLEQ_HEAD(chan_cell_queue, cell_queue_entry_s); typedef struct chan_cell_queue chan_cell_queue_t; /** @@ -469,6 +469,10 @@ void channel_notify_flushed(channel_t *chan); /* Handle stuff we need to do on open like notifying circuits */ void channel_do_open_actions(channel_t *chan); +#ifdef TOR_UNIT_TESTS +extern uint64_t estimated_total_queue_size; +#endif + #endif /* Helper functions to perform operations on channels */ diff --git a/src/or/channeltls.c b/src/or/channeltls.c index c65af5d040..2bb88dd505 100644 --- a/src/or/channeltls.c +++ b/src/or/channeltls.c @@ -22,6 +22,7 @@ #include "channeltls.h" #include "circuitmux.h" #include "circuitmux_ewma.h" +#include "command.h" #include "config.h" #include "connection.h" #include "connection_or.h" @@ -51,7 +52,7 @@ uint64_t stats_n_authenticate_cells_processed = 0; uint64_t stats_n_authorize_cells_processed = 0; /** Active listener, if any */ -channel_listener_t *channel_tls_listener = NULL; +static channel_listener_t *channel_tls_listener = NULL; /* channel_tls_t method declarations */ @@ -445,7 +446,7 @@ channel_tls_free_method(channel_t *chan) static double channel_tls_get_overhead_estimate_method(channel_t *chan) { - double overhead = 1.0f; + double overhead = 1.0; channel_tls_t *tlschan = BASE_CHAN_TO_TLS(chan); tor_assert(tlschan); @@ -462,7 +463,8 @@ channel_tls_get_overhead_estimate_method(channel_t *chan) * Never estimate more than 2.0; otherwise we get silly large estimates * at the very start of a new TLS connection. */ - if (overhead > 2.0f) overhead = 2.0f; + if (overhead > 2.0) + overhead = 2.0; } log_debug(LD_CHANNEL, @@ -554,7 +556,7 @@ channel_tls_get_remote_descr_method(channel_t *chan, int flags) break; case GRD_FLAG_ORIGINAL: /* Actual address with port */ - addr_str = tor_dup_addr(&(tlschan->conn->real_addr)); + addr_str = tor_addr_to_str_dup(&(tlschan->conn->real_addr)); tor_snprintf(buf, MAX_DESCR_LEN + 1, "%s:%u", addr_str, conn->port); tor_free(addr_str); @@ -567,7 +569,7 @@ channel_tls_get_remote_descr_method(channel_t *chan, int flags) break; case GRD_FLAG_ORIGINAL|GRD_FLAG_ADDR_ONLY: /* Actual address, no port */ - addr_str = tor_dup_addr(&(tlschan->conn->real_addr)); + addr_str = tor_addr_to_str_dup(&(tlschan->conn->real_addr)); strlcpy(buf, addr_str, sizeof(buf)); tor_free(addr_str); answer = buf; @@ -797,6 +799,7 @@ static int channel_tls_write_packed_cell_method(channel_t *chan, packed_cell_t *packed_cell) { + tor_assert(chan); channel_tls_t *tlschan = BASE_CHAN_TO_TLS(chan); size_t cell_network_size = get_cell_network_size(chan->wide_circ_ids); int written = 0; diff --git a/src/or/channeltls.h b/src/or/channeltls.h index a4d9c7a095..8b5863a461 100644 --- a/src/or/channeltls.h +++ b/src/or/channeltls.h @@ -52,6 +52,14 @@ void channel_tls_update_marks(or_connection_t *conn); /* Cleanup at shutdown */ void channel_tls_free_all(void); +extern uint64_t stats_n_authorize_cells_processed; +extern uint64_t stats_n_authenticate_cells_processed; +extern uint64_t stats_n_versions_cells_processed; +extern uint64_t stats_n_netinfo_cells_processed; +extern uint64_t stats_n_vpadding_cells_processed; +extern uint64_t stats_n_certs_cells_processed; +extern uint64_t stats_n_auth_challenge_cells_processed; + #ifdef CHANNELTLS_PRIVATE STATIC void channel_tls_process_certs_cell(var_cell_t *cell, channel_tls_t *tlschan); diff --git a/src/or/circpathbias.c b/src/or/circpathbias.c index 552947eba2..9f93e737f7 100644 --- a/src/or/circpathbias.c +++ b/src/or/circpathbias.c @@ -85,7 +85,6 @@ pathbias_get_notice_rate(const or_options_t *options) DFLT_PATH_BIAS_NOTICE_PCT, 0, 100)/100.0; } -/* XXXX024 I'd like to have this be static again, but entrynodes.c needs it. */ /** The circuit success rate below which we issue a warn */ static double pathbias_get_warn_rate(const or_options_t *options) @@ -98,7 +97,7 @@ pathbias_get_warn_rate(const or_options_t *options) DFLT_PATH_BIAS_WARN_PCT, 0, 100)/100.0; } -/* XXXX024 I'd like to have this be static again, but entrynodes.c needs it. */ +/* XXXX I'd like to have this be static again, but entrynodes.c needs it. */ /** * The extreme rate is the rate at which we would drop the guard, * if pb_dropguard is also set. Otherwise we just warn. @@ -114,7 +113,7 @@ pathbias_get_extreme_rate(const or_options_t *options) DFLT_PATH_BIAS_EXTREME_PCT, 0, 100)/100.0; } -/* XXXX024 I'd like to have this be static again, but entrynodes.c needs it. */ +/* XXXX I'd like to have this be static again, but entrynodes.c needs it. */ /** * If 1, we actually disable use of guards that fall below * the extreme_pct. diff --git a/src/or/circuitbuild.c b/src/or/circuitbuild.c index 820724adea..13cc16670c 100644 --- a/src/or/circuitbuild.c +++ b/src/or/circuitbuild.c @@ -47,10 +47,6 @@ #include "routerset.h" #include "crypto.h" -#ifndef MIN -#define MIN(a,b) ((a)<(b)?(a):(b)) -#endif - static channel_t * channel_connect_for_circuit(const tor_addr_t *addr, uint16_t port, const char *id_digest); @@ -809,6 +805,7 @@ circuit_pick_create_handshake(uint8_t *cell_type_out, uint16_t *handshake_type_out, const extend_info_t *ei) { + /* XXXX029 Remove support for deciding to use TAP. */ if (!tor_mem_is_zero((const char*)ei->curve25519_onion_key.public_key, CURVE25519_PUBKEY_LEN) && circuits_can_use_ntor()) { @@ -835,9 +832,8 @@ circuit_pick_extend_handshake(uint8_t *cell_type_out, { uint8_t t; circuit_pick_create_handshake(&t, handshake_type_out, ei); - /* XXXX024 The check for whether the node has a curve25519 key is a bad - * proxy for whether it can do extend2 cells; once a version that - * handles extend2 cells is out, remove it. */ + + /* XXXX029 Remove support for deciding to use TAP. */ if (node_prev && *handshake_type_out != ONION_HANDSHAKE_TYPE_TAP && (node_has_curve25519_onion_key(node_prev) || @@ -888,14 +884,12 @@ circuit_send_next_onion_skin(origin_circuit_t *circ) */ circuit_pick_create_handshake(&cc.cell_type, &cc.handshake_type, circ->cpath->extend_info); - note_request("cell: create", 1); } else { /* We are not an OR, and we're building the first hop of a circuit to a * new OR: we can be speedy and use CREATE_FAST to save an RSA operation * and a DH operation. */ cc.cell_type = CELL_CREATE_FAST; cc.handshake_type = ONION_HANDSHAKE_TYPE_FAST; - note_request("cell: create fast", 1); } len = onion_skin_create(cc.handshake_type, @@ -1028,7 +1022,6 @@ circuit_send_next_onion_skin(origin_circuit_t *circ) ec.create_cell.handshake_len = len; log_info(LD_CIRC,"Sending extend relay cell."); - note_request("cell: extend", 1); { uint8_t command = 0; uint16_t payload_len=0; @@ -2146,7 +2139,6 @@ choose_good_middle_server(uint8_t purpose, * If <b>state</b> is NULL, we're choosing a router to serve as an entry * guard, not for any particular circuit. */ -/* XXXX024 I'd like to have this be static again, but entrynodes.c needs it. */ const node_t * choose_good_entry_server(uint8_t purpose, cpath_build_state_t *state) { @@ -2179,7 +2171,7 @@ choose_good_entry_server(uint8_t purpose, cpath_build_state_t *state) * This is an incomplete fix, but is no worse than the previous behaviour, * and only applies to minimal, testing tor networks * (so it's no less secure) */ - /*XXXX025 use the using_as_guard flag to accomplish this.*/ + /*XXXX++ use the using_as_guard flag to accomplish this.*/ if (options->UseEntryGuards && (!options->TestingTorNetwork || smartlist_len(nodelist_get_list()) > smartlist_len(get_entry_guards()) diff --git a/src/or/circuitlist.c b/src/or/circuitlist.c index 1efb7ef4d0..d2ba7d4781 100644 --- a/src/or/circuitlist.c +++ b/src/or/circuitlist.c @@ -109,7 +109,7 @@ HT_GENERATE2(chan_circid_map, chan_circid_circuit_map_t, node, * used to improve performance when many cells arrive in a row from the * same circuit. */ -chan_circid_circuit_map_t *_last_circid_chan_ent = NULL; +static chan_circid_circuit_map_t *_last_circid_chan_ent = NULL; /** Implementation helper for circuit_set_{p,n}_circid_channel: A circuit ID * and/or channel for circ has just changed from <b>old_chan, old_id</b> diff --git a/src/or/circuitmux.c b/src/or/circuitmux.c index cc1c4cd401..038904e68a 100644 --- a/src/or/circuitmux.c +++ b/src/or/circuitmux.c @@ -362,7 +362,7 @@ HT_HEAD(chanid_circid_muxinfo_map, chanid_circid_muxinfo_t); /* Emit a bunch of hash table stuff */ HT_PROTOTYPE(chanid_circid_muxinfo_map, chanid_circid_muxinfo_t, node, - chanid_circid_entry_hash, chanid_circid_entries_eq); + chanid_circid_entry_hash, chanid_circid_entries_eq) HT_GENERATE2(chanid_circid_muxinfo_map, chanid_circid_muxinfo_t, node, chanid_circid_entry_hash, chanid_circid_entries_eq, 0.6, tor_reallocarray_, tor_free_) diff --git a/src/or/circuitmux_ewma.h b/src/or/circuitmux_ewma.h index 58aac1e196..a7b8961ac6 100644 --- a/src/or/circuitmux_ewma.h +++ b/src/or/circuitmux_ewma.h @@ -12,13 +12,8 @@ #include "or.h" #include "circuitmux.h" -/* Everything but circuitmux_ewma.c should see this extern */ -#ifndef TOR_CIRCUITMUX_EWMA_C_ - extern circuitmux_policy_t ewma_policy; -#endif /* !(TOR_CIRCUITMUX_EWMA_C_) */ - /* Externally visible EWMA functions */ int cell_ewma_enabled(void); unsigned int cell_ewma_get_tick(void); diff --git a/src/or/circuituse.c b/src/or/circuituse.c index 2c724dee05..f344703331 100644 --- a/src/or/circuituse.c +++ b/src/or/circuituse.c @@ -203,7 +203,7 @@ circuit_is_better(const origin_circuit_t *oa, const origin_circuit_t *ob, timercmp(&a->timestamp_began, &b->timestamp_began, OP_GT)) return 1; if (ob->build_state->is_internal) - /* XXX023 what the heck is this internal thing doing here. I + /* XXXX++ what the heck is this internal thing doing here. I * think we can get rid of it. circuit_is_acceptable() already * makes sure that is_internal is exactly what we need it to * be. -RD */ @@ -222,7 +222,7 @@ circuit_is_better(const origin_circuit_t *oa, const origin_circuit_t *ob, break; } - /* XXXX023 Maybe this check should get a higher priority to avoid + /* XXXX Maybe this check should get a higher priority to avoid * using up circuits too rapidly. */ a_bits = connection_edge_update_circuit_isolation(conn, @@ -1067,7 +1067,7 @@ circuit_predict_and_launch_new(void) if (rep_hist_get_predicted_internal(now, &hidserv_needs_uptime, &hidserv_needs_capacity) && ((num_uptime_internal<2 && hidserv_needs_uptime) || - num_internal<2) + num_internal<3) && router_have_consensus_path() != CONSENSUS_PATH_UNKNOWN) { if (hidserv_needs_uptime) flags |= CIRCLAUNCH_NEED_UPTIME; @@ -1936,8 +1936,8 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn, return -1; } } else { - /* XXXX024 Duplicates checks in connection_ap_handshake_attach_circuit: - * refactor into a single function? */ + /* XXXX Duplicates checks in connection_ap_handshake_attach_circuit: + * refactor into a single function. */ const node_t *node = node_get_by_nickname(conn->chosen_exit_name, 1); int opt = conn->chosen_exit_optional; if (node && !connection_ap_can_use_exit(conn, node)) { @@ -2028,7 +2028,8 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn, char *hexdigest = conn->chosen_exit_name+1; tor_addr_t addr; if (strlen(hexdigest) < HEX_DIGEST_LEN || - base16_decode(digest,DIGEST_LEN,hexdigest,HEX_DIGEST_LEN)<0) { + base16_decode(digest,DIGEST_LEN, + hexdigest,HEX_DIGEST_LEN) != DIGEST_LEN) { log_info(LD_DIR, "Broken exit digest on tunnel conn. Closing."); return -1; } @@ -2146,10 +2147,11 @@ optimistic_data_enabled(void) { const or_options_t *options = get_options(); if (options->OptimisticData < 0) { - /* XXX023 consider having auto default to 1 rather than 0 before - * the 0.2.3 branch goes stable. See bug 3617. -RD */ + /* Note: this default was 0 before #18815 was merged. We can't take the + * parameter out of the consensus until versions before that are all + * obsolete. */ const int32_t enabled = - networkstatus_get_param(NULL, "UseOptimisticData", 0, 0, 1); + networkstatus_get_param(NULL, "UseOptimisticData", /*default*/ 1, 0, 1); return (int)enabled; } return options->OptimisticData; @@ -2415,7 +2417,7 @@ connection_ap_handshake_attach_circuit(entry_connection_t *conn) /* find the circuit that we should use, if there is one. */ retval = circuit_get_open_circ_or_launch( conn, CIRCUIT_PURPOSE_C_GENERAL, &circ); - if (retval < 1) // XXX023 if we totally fail, this still returns 0 -RD + if (retval < 1) // XXXX++ if we totally fail, this still returns 0 -RD return retval; log_debug(LD_APP|LD_CIRC, @@ -2590,7 +2592,7 @@ mark_circuit_unusable_for_new_conns(origin_circuit_t *circ) const or_options_t *options = get_options(); tor_assert(circ); - /* XXXX025 This is a kludge; we're only keeping it around in case there's + /* XXXX This is a kludge; we're only keeping it around in case there's * something that doesn't check unusable_for_new_conns, and to avoid * deeper refactoring of our expiration logic. */ if (! circ->base_.timestamp_dirty) diff --git a/src/or/config.c b/src/or/config.c index 0850013d33..cdd4f10e92 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -65,9 +65,6 @@ #include <systemd/sd-daemon.h> #endif -/* From main.c */ -extern int quiet_level; - /* Prefix used to indicate a Unix socket in a FooPort configuration. */ static const char unix_socket_prefix[] = "unix:"; @@ -99,7 +96,7 @@ static config_abbrev_t option_abbrevs_[] = { { "BandwidthRateBytes", "BandwidthRate", 0, 0}, { "BandwidthBurstBytes", "BandwidthBurst", 0, 0}, { "DirFetchPostPeriod", "StatusFetchPeriod", 0, 0}, - { "DirServer", "DirAuthority", 0, 0}, /* XXXX024 later, make this warn? */ + { "DirServer", "DirAuthority", 0, 0}, /* XXXX later, make this warn? */ { "MaxConn", "ConnLimit", 0, 1}, { "MaxMemInCellQueues", "MaxMemInQueues", 0, 0}, { "ORBindAddress", "ORListenAddress", 0, 0}, @@ -2004,11 +2001,6 @@ static const struct { { "--list-fingerprint", TAKES_NO_ARGUMENT }, { "--keygen", TAKES_NO_ARGUMENT }, { "--newpass", TAKES_NO_ARGUMENT }, -#if 0 -/* XXXX028: This is not working yet in 0.2.7, so disabling with the - * minimal code modification. */ - { "--master-key", ARGUMENT_NECESSARY }, -#endif { "--no-passphrase", TAKES_NO_ARGUMENT }, { "--passphrase-fd", ARGUMENT_NECESSARY }, { "--verify-config", TAKES_NO_ARGUMENT }, @@ -2489,7 +2481,6 @@ is_local_addr, (const tor_addr_t *addr)) if (get_options()->EnforceDistinctSubnets == 0) return 0; if (tor_addr_family(addr) == AF_INET) { - /*XXXX023 IP6 what corresponds to an /24? */ uint32_t ip = tor_addr_to_ipv4h(addr); /* It's possible that this next check will hit before the first time @@ -2683,7 +2674,7 @@ options_validate_cb(void *old_options, void *options, void *default_options, #define REJECT(arg) \ STMT_BEGIN *msg = tor_strdup(arg); return -1; STMT_END -#ifdef __GNUC__ +#if defined(__GNUC__) && __GNUC__ <= 3 #define COMPLAIN(args...) \ STMT_BEGIN log_warn(LD_CONFIG, args); STMT_END #else @@ -2796,7 +2787,8 @@ options_validate(or_options_t *old_options, or_options_t *options, } else { if (!is_legal_nickname(options->Nickname)) { tor_asprintf(msg, - "Nickname '%s' is wrong length or contains illegal characters.", + "Nickname '%s', nicknames must be between 1 and 19 characters " + "inclusive, and must contain only the characters [a-zA-Z0-9].", options->Nickname); return -1; } @@ -5030,7 +5022,7 @@ config_register_addressmaps(const or_options_t *options) /** As addressmap_register(), but detect the wildcarded status of "from" and * "to", and do not steal a reference to <b>to</b>. */ -/* XXXX024 move to connection_edge.c */ +/* XXXX move to connection_edge.c */ int addressmap_register_auto(const char *from, const char *to, time_t expires, @@ -5338,7 +5330,7 @@ parse_bridge_line(const char *line) goto err; } if (base16_decode(bridge_line->digest, DIGEST_LEN, - fingerprint, HEX_DIGEST_LEN)<0) { + fingerprint, HEX_DIGEST_LEN) != DIGEST_LEN) { log_warn(LD_CONFIG, "Unable to decode Bridge key digest."); goto err; } @@ -5781,7 +5773,7 @@ parse_dir_authority_line(const char *line, dirinfo_type_t required_type, } else if (!strcmpstart(flag, "weight=")) { int ok; const char *wstring = flag + strlen("weight="); - weight = tor_parse_double(wstring, 0, UINT64_MAX, &ok, NULL); + weight = tor_parse_double(wstring, 0, (double)UINT64_MAX, &ok, NULL); if (!ok) { log_warn(LD_CONFIG, "Invalid weight '%s' on DirAuthority line.",flag); weight=1.0; @@ -5789,7 +5781,8 @@ parse_dir_authority_line(const char *line, dirinfo_type_t required_type, } else if (!strcasecmpstart(flag, "v3ident=")) { char *idstr = flag + strlen("v3ident="); if (strlen(idstr) != HEX_DIGEST_LEN || - base16_decode(v3_digest, DIGEST_LEN, idstr, HEX_DIGEST_LEN)<0) { + base16_decode(v3_digest, DIGEST_LEN, + idstr, HEX_DIGEST_LEN) != DIGEST_LEN) { log_warn(LD_CONFIG, "Bad v3 identity digest '%s' on DirAuthority line", flag); } else { @@ -5838,7 +5831,8 @@ parse_dir_authority_line(const char *line, dirinfo_type_t required_type, fingerprint, (int)strlen(fingerprint)); goto err; } - if (base16_decode(digest, DIGEST_LEN, fingerprint, HEX_DIGEST_LEN)<0) { + if (base16_decode(digest, DIGEST_LEN, + fingerprint, HEX_DIGEST_LEN) != DIGEST_LEN) { log_warn(LD_CONFIG, "Unable to decode DirAuthority key digest."); goto err; } @@ -5906,8 +5900,8 @@ parse_dir_fallback_line(const char *line, orport = (int)tor_parse_long(cp+strlen("orport="), 10, 1, 65535, &ok, NULL); } else if (!strcmpstart(cp, "id=")) { - ok = !base16_decode(id, DIGEST_LEN, - cp+strlen("id="), strlen(cp)-strlen("id=")); + ok = base16_decode(id, DIGEST_LEN, cp+strlen("id="), + strlen(cp)-strlen("id=")) == DIGEST_LEN; } else if (!strcasecmpstart(cp, "ipv6=")) { if (ipv6_addrport_ptr) { log_warn(LD_CONFIG, "Redundant ipv6 addr/port on FallbackDir line"); @@ -5925,7 +5919,7 @@ parse_dir_fallback_line(const char *line, } else if (!strcmpstart(cp, "weight=")) { int ok; const char *wstring = cp + strlen("weight="); - weight = tor_parse_double(wstring, 0, UINT64_MAX, &ok, NULL); + weight = tor_parse_double(wstring, 0, (double)UINT64_MAX, &ok, NULL); if (!ok) { log_warn(LD_CONFIG, "Invalid weight '%s' on FallbackDir line.", cp); weight=1.0; @@ -7571,7 +7565,7 @@ static void config_maybe_load_geoip_files_(const or_options_t *options, const or_options_t *old_options) { - /* XXXX024 Reload GeoIPFile on SIGHUP. -NM */ + /* XXXX Reload GeoIPFile on SIGHUP. -NM */ if (options->GeoIPFile && ((!old_options || !opt_streq(old_options->GeoIPFile, diff --git a/src/or/config.h b/src/or/config.h index 02121cf95c..e08ad81304 100644 --- a/src/or/config.h +++ b/src/or/config.h @@ -115,7 +115,7 @@ int config_parse_commandline(int argc, char **argv, int ignore_errors, config_line_t **cmdline_result); void config_register_addressmaps(const or_options_t *options); -/* XXXX024 move to connection_edge.h */ +/* XXXX move to connection_edge.h */ int addressmap_register_auto(const char *from, const char *to, time_t expires, addressmap_entry_source_t addrmap_source, diff --git a/src/or/confparse.c b/src/or/confparse.c index 4f446d07c3..3532b39d93 100644 --- a/src/or/confparse.c +++ b/src/or/confparse.c @@ -1238,7 +1238,7 @@ config_parse_units(const char *val, struct unit_table_t *u, int *ok) v = tor_parse_uint64(val, 10, 0, UINT64_MAX, ok, &cp); if (!*ok || (cp && *cp == '.')) { - d = tor_parse_double(val, 0, UINT64_MAX, ok, &cp); + d = tor_parse_double(val, 0, (double)UINT64_MAX, ok, &cp); if (!*ok) goto done; use_float = 1; @@ -1255,7 +1255,7 @@ config_parse_units(const char *val, struct unit_table_t *u, int *ok) for ( ;u->unit;++u) { if (!strcasecmp(u->unit, cp)) { if (use_float) - v = u->multiplier * d; + v = (uint64_t)(u->multiplier * d); else v *= u->multiplier; *ok = 1; diff --git a/src/or/connection.c b/src/or/connection.c index 4fbbaf1abd..9eef063f18 100644 --- a/src/or/connection.c +++ b/src/or/connection.c @@ -98,7 +98,7 @@ static int get_proxy_type(void); /** The last addresses that our network interface seemed to have been * binding to. We use this as one way to detect when our IP changes. * - * XXX024 We should really use the entire list of interfaces here. + * XXXX+ We should really use the entire list of interfaces here. **/ static tor_addr_t *last_interface_ipv4 = NULL; /* DOCDOC last_interface_ipv6 */ @@ -665,9 +665,7 @@ connection_free,(connection_t *conn)) return; tor_assert(!connection_is_on_closeable_list(conn)); tor_assert(!connection_in_array(conn)); - if (conn->linked_conn) { - log_err(LD_BUG, "Called with conn->linked_conn still set."); - tor_fragile_assert(); + if (BUG(conn->linked_conn)) { conn->linked_conn->linked_conn = NULL; if (! conn->linked_conn->marked_for_close && conn->linked_conn->reading_from_linked_conn) @@ -1564,7 +1562,7 @@ connection_handle_listener_read(connection_t *conn, int new_type) /* remember the remote address */ tor_addr_copy(&newconn->addr, &addr); newconn->port = port; - newconn->address = tor_dup_addr(&addr); + newconn->address = tor_addr_to_str_dup(&addr); if (new_type == CONN_TYPE_AP && conn->socket_family != AF_UNIX) { log_info(LD_NET, "New SOCKS connection opened from %s.", @@ -2242,7 +2240,7 @@ connection_send_socks5_connect(connection_t *conn) } else { /* AF_INET6 */ buf[3] = 4; reqsize += 16; - memcpy(buf + 4, tor_addr_to_in6(&conn->addr), 16); + memcpy(buf + 4, tor_addr_to_in6_addr8(&conn->addr), 16); memcpy(buf + 20, &port, 2); } @@ -2538,7 +2536,7 @@ retry_listener_ports(smartlist_t *old_conns, real_port, listensockaddr, sizeof(struct sockaddr_storage)); - address = tor_dup_addr(&port->addr); + address = tor_addr_to_str_dup(&port->addr); } if (listensockaddr) { @@ -2699,8 +2697,6 @@ connection_is_rate_limited(connection_t *conn) #ifdef USE_BUFFEREVENTS static struct bufferevent_rate_limit_group *global_rate_limit = NULL; #else -extern int global_read_bucket, global_write_bucket; -extern int global_relayed_read_bucket, global_relayed_write_bucket; /** Did either global write bucket run dry last second? If so, * we are likely to run dry again this second, so be stingy with the @@ -2934,7 +2930,7 @@ static void record_num_bytes_transferred(connection_t *conn, time_t now, size_t num_read, size_t num_written) { - /* XXX024 check if this is necessary */ + /* XXXX check if this is necessary */ if (num_written >= INT_MAX || num_read >= INT_MAX) { log_err(LD_BUG, "Value out of range. num_read=%lu, num_written=%lu, " "connection type=%s, state=%s", @@ -3644,7 +3640,7 @@ connection_read_to_buf(connection_t *conn, ssize_t *max_to_read, * take us over our read allotment, but really we shouldn't be * believing that SSL bytes are the same as TCP bytes anyway. */ int r2 = read_to_buf_tls(or_conn->tls, pending, conn->inbuf); - if (r2<0) { + if (BUG(r2<0)) { log_warn(LD_BUG, "apparently, reading pending bytes can fail."); return -1; } @@ -3761,7 +3757,7 @@ evbuffer_inbuf_callback(struct evbuffer *buf, connection_consider_empty_read_buckets(conn); if (conn->type == CONN_TYPE_AP) { edge_connection_t *edge_conn = TO_EDGE_CONN(conn); - /*XXXX024 check for overflow*/ + /*XXXX++ check for overflow*/ edge_conn->n_read += (int)info->n_added; } } @@ -3782,7 +3778,7 @@ evbuffer_outbuf_callback(struct evbuffer *buf, connection_consider_empty_write_buckets(conn); if (conn->type == CONN_TYPE_AP) { edge_connection_t *edge_conn = TO_EDGE_CONN(conn); - /*XXXX024 check for overflow*/ + /*XXXX++ check for overflow*/ edge_conn->n_written += (int)info->n_deleted; } } @@ -4141,7 +4137,7 @@ connection_handle_write_impl(connection_t *conn, int force) or_conn->bytes_xmitted += result; or_conn->bytes_xmitted_by_tls += n_written; /* So we notice bytes were written even on error */ - /* XXXX024 This cast is safe since we can never write INT_MAX bytes in a + /* XXXX This cast is safe since we can never write INT_MAX bytes in a * single set of TLS operations. But it looks kinda ugly. If we refactor * the *_buf_tls functions, we should make them return ssize_t or size_t * or something. */ diff --git a/src/or/connection_edge.c b/src/or/connection_edge.c index 754e9762ea..799baa2acc 100644 --- a/src/or/connection_edge.c +++ b/src/or/connection_edge.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2001 Matej Pfajfar. + /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. * Copyright (c) 2007-2016, The Tor Project, Inc. */ @@ -919,7 +919,7 @@ connection_ap_warn_and_unmark_if_pending_circ(entry_connection_t *entry_conn, /** Tell any AP streams that are waiting for a one-hop tunnel to * <b>failed_digest</b> that they are going to fail. */ -/* XXX024 We should get rid of this function, and instead attach +/* XXXX We should get rid of this function, and instead attach * one-hop streams to circ->p_streams so they get marked in * circuit_mark_for_close like normal p_streams. */ void @@ -1442,7 +1442,7 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn, connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL); return -1; } - /* XXXX024-1090 Should we also allow foo.bar.exit if ExitNodes is set and + /* XXXX-1090 Should we also allow foo.bar.exit if ExitNodes is set and Bar is not listed in it? I say yes, but our revised manpage branch implies no. */ } @@ -1691,7 +1691,7 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn, rend_service_authorization_t *client_auth = rend_client_lookup_service_authorization(socks->address); - const char *cookie = NULL; + const uint8_t *cookie = NULL; rend_auth_type_t auth_type = REND_NO_AUTH; if (client_auth) { log_info(LD_REND, "Using previously configured client authorization " @@ -1703,7 +1703,8 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn, /* Fill in the rend_data field so we can start doing a connection to * a hidden service. */ rend_data_t *rend_data = ENTRY_TO_EDGE_CONN(conn)->rend_data = - rend_data_client_create(socks->address, NULL, cookie, auth_type); + rend_data_client_create(socks->address, NULL, (char *) cookie, + auth_type); if (rend_data == NULL) { return -1; } @@ -2290,7 +2291,7 @@ connection_ap_handshake_send_begin(entry_connection_t *ap_conn) edge_conn->stream_id = get_unique_stream_id_by_circ(circ); if (edge_conn->stream_id==0) { - /* XXXX024 Instead of closing this stream, we should make it get + /* XXXX+ Instead of closing this stream, we should make it get * retried on another circuit. */ connection_mark_unattached_ap(ap_conn, END_STREAM_REASON_INTERNAL); @@ -2382,7 +2383,7 @@ connection_ap_handshake_send_resolve(entry_connection_t *ap_conn) edge_conn->stream_id = get_unique_stream_id_by_circ(circ); if (edge_conn->stream_id==0) { - /* XXXX024 Instead of closing this stream, we should make it get + /* XXXX+ Instead of closing this stream, we should make it get * retried on another circuit. */ connection_mark_unattached_ap(ap_conn, END_STREAM_REASON_INTERNAL); @@ -2433,7 +2434,7 @@ connection_ap_handshake_send_resolve(entry_connection_t *ap_conn) if (!base_conn->address) { /* This might be unnecessary. XXXX */ - base_conn->address = tor_dup_addr(&base_conn->addr); + base_conn->address = tor_addr_to_str_dup(&base_conn->addr); } base_conn->state = AP_CONN_STATE_RESOLVE_WAIT; log_info(LD_APP,"Address sent for resolve, ap socket "TOR_SOCKET_T_FORMAT diff --git a/src/or/connection_or.c b/src/or/connection_or.c index 9730e1a952..c69a2ad377 100644 --- a/src/or/connection_or.c +++ b/src/or/connection_or.c @@ -584,7 +584,7 @@ connection_or_process_inbuf(or_connection_t *conn) * check would otherwise just let data accumulate. It serves no purpose * in 0.2.3. * - * XXX024 Remove this check once we verify that the above paragraph is + * XXXX Remove this check once we verify that the above paragraph is * 100% true. */ if (buf_datalen(conn->base_.inbuf) > MAX_OR_INBUF_WHEN_NONOPEN) { log_fn(LOG_PROTOCOL_WARN, LD_NET, "Accumulated too much data (%d bytes) " @@ -935,7 +935,7 @@ connection_or_init_conn_from_address(or_connection_t *conn, } conn->nickname = tor_strdup(node_get_nickname(r)); tor_free(conn->base_.address); - conn->base_.address = tor_dup_addr(&node_ap.addr); + conn->base_.address = tor_addr_to_str_dup(&node_ap.addr); } else { conn->nickname = tor_malloc(HEX_DIGEST_LEN+2); conn->nickname[0] = '$'; @@ -943,7 +943,7 @@ connection_or_init_conn_from_address(or_connection_t *conn, conn->identity_digest, DIGEST_LEN); tor_free(conn->base_.address); - conn->base_.address = tor_dup_addr(addr); + conn->base_.address = tor_addr_to_str_dup(addr); } /* @@ -1282,11 +1282,9 @@ connection_or_connect, (const tor_addr_t *_addr, uint16_t port, switch (connection_connect(TO_CONN(conn), conn->base_.address, &addr, port, &socket_error)) { case -1: - /* If the connection failed immediately, and we're using - * a proxy, our proxy is down. Don't blame the Tor server. */ - if (conn->base_.proxy_state == PROXY_INFANT) - entry_guard_register_connect_status(conn->identity_digest, - 0, 1, time(NULL)); + /* We failed to establish a connection probably because of a local + * error. No need to blame the guard in this case. Notify the networking + * system of this failure. */ connection_or_connect_failed(conn, errno_to_orconn_end_reason(socket_error), tor_socket_strerror(socket_error)); diff --git a/src/or/control.c b/src/or/control.c index e2ad8cc6dc..6e5dcf62e6 100644 --- a/src/or/control.c +++ b/src/or/control.c @@ -1211,7 +1211,8 @@ decode_hashed_passwords(config_line_t *passwords) const char *hashed = cl->value; if (!strcmpstart(hashed, "16:")) { - if (base16_decode(decoded, sizeof(decoded), hashed+3, strlen(hashed+3))<0 + if (base16_decode(decoded, sizeof(decoded), hashed+3, strlen(hashed+3)) + != S2K_RFC2440_SPECIFIER_LEN + DIGEST_LEN || strlen(hashed+3) != (S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN)*2) { goto err; } @@ -1262,7 +1263,8 @@ handle_control_authenticate(control_connection_t *conn, uint32_t len, tor_assert(i>0); password_len = i/2; password = tor_malloc(password_len + 1); - if (base16_decode(password, password_len+1, body, i)<0) { + if (base16_decode(password, password_len+1, body, i) + != (int) password_len) { connection_write_str_to_buf( "551 Invalid hexadecimal encoding. Maybe you tried a plain text " "password? If so, the standard requires that you put it in " @@ -1724,8 +1726,6 @@ getinfo_helper_misc(control_connection_t *conn, const char *question, } else if (!strcmp(question, "limits/max-mem-in-queues")) { tor_asprintf(answer, U64_FORMAT, U64_PRINTF_ARG(get_options()->MaxMemInQueues)); - } else if (!strcmp(question, "dir-usage")) { - *answer = directory_dump_request_log(); } else if (!strcmp(question, "fingerprint")) { crypto_pk_t *server_key; if (!server_mode(get_options())) { @@ -1865,7 +1865,7 @@ getinfo_helper_dir(control_connection_t *control_conn, *answer = tor_strndup(body, ri->cache_info.signed_descriptor_len); } } else if (!strcmpstart(question, "desc/name/")) { - /* XXX023 Setting 'warn_if_unnamed' here is a bit silly -- the + /* XXX Setting 'warn_if_unnamed' here is a bit silly -- the * warning goes to the user, not to the controller. */ node = node_get_by_nickname(question+strlen("desc/name/"), 1); if (node) @@ -1951,7 +1951,7 @@ getinfo_helper_dir(control_connection_t *control_conn, *answer = tor_strndup(md->body, md->bodylen); } } else if (!strcmpstart(question, "md/name/")) { - /* XXX023 Setting 'warn_if_unnamed' here is a bit silly -- the + /* XXX Setting 'warn_if_unnamed' here is a bit silly -- the * warning goes to the user, not to the controller. */ const node_t *node = node_get_by_nickname(question+strlen("md/name/"), 1); /* XXXX duplicated code */ @@ -2028,7 +2028,8 @@ getinfo_helper_dir(control_connection_t *control_conn, if (strlen(question) == HEX_DIGEST_LEN) { char d[DIGEST_LEN]; signed_descriptor_t *sd = NULL; - if (base16_decode(d, sizeof(d), question, strlen(question))==0) { + if (base16_decode(d, sizeof(d), question, strlen(question)) + != sizeof(d)) { /* XXXX this test should move into extrainfo_get_by_descriptor_digest, * but I don't want to risk affecting other parts of the code, * especially since the rules for using our own extrainfo (including @@ -2561,7 +2562,6 @@ static const getinfo_item_t getinfo_items[] = { "Username under which the tor process is running."), ITEM("process/descriptor-limit", misc, "File descriptor limit."), ITEM("limits/max-mem-in-queues", misc, "Actual limit on memory in queues"), - ITEM("dir-usage", misc, "Breakdown of bytes transferred over DirPort."), PREFIX("desc-annotations/id/", dir, "Router annotations by hexdigest."), PREFIX("dir/server/", dir,"Router descriptors as retrieved from a DirPort."), PREFIX("dir/status/", dir, @@ -3445,7 +3445,8 @@ handle_control_authchallenge(control_connection_t *conn, uint32_t len, client_nonce = tor_malloc_zero(client_nonce_len); if (base16_decode(client_nonce, client_nonce_len, - cp, client_nonce_encoded_len) < 0) { + cp, client_nonce_encoded_len) + != (int) client_nonce_len) { connection_write_str_to_buf("513 Invalid base16 client nonce\r\n", conn); connection_mark_for_close(TO_CONN(conn)); @@ -3791,14 +3792,18 @@ handle_control_add_onion(control_connection_t *conn, * the other arguments are malformed. */ smartlist_t *port_cfgs = smartlist_new(); + smartlist_t *auth_clients = NULL; + smartlist_t *auth_created_clients = NULL; int discard_pk = 0; int detach = 0; int max_streams = 0; int max_streams_close_circuit = 0; + rend_auth_type_t auth_type = REND_NO_AUTH; for (size_t i = 1; i < arg_len; i++) { static const char *port_prefix = "Port="; static const char *flags_prefix = "Flags="; static const char *max_s_prefix = "MaxStreams="; + static const char *auth_prefix = "ClientAuth="; const char *arg = smartlist_get(args, i); if (!strcasecmpstart(arg, port_prefix)) { @@ -3829,10 +3834,12 @@ handle_control_add_onion(control_connection_t *conn, * connection. * * 'MaxStreamsCloseCircuit' - Close the circuit if MaxStreams is * exceeded. + * * 'BasicAuth' - Client authorization using the 'basic' method. */ static const char *discard_flag = "DiscardPK"; static const char *detach_flag = "Detach"; static const char *max_s_close_flag = "MaxStreamsCloseCircuit"; + static const char *basicauth_flag = "BasicAuth"; smartlist_t *flags = smartlist_new(); int bad = 0; @@ -3851,6 +3858,8 @@ handle_control_add_onion(control_connection_t *conn, detach = 1; } else if (!strcasecmp(flag, max_s_close_flag)) { max_streams_close_circuit = 1; + } else if (!strcasecmp(flag, basicauth_flag)) { + auth_type = REND_BASIC_AUTH; } else { connection_printf_to_buf(conn, "512 Invalid 'Flags' argument: %s\r\n", @@ -3863,6 +3872,42 @@ handle_control_add_onion(control_connection_t *conn, smartlist_free(flags); if (bad) goto out; + } else if (!strcasecmpstart(arg, auth_prefix)) { + char *err_msg = NULL; + int created = 0; + rend_authorized_client_t *client = + add_onion_helper_clientauth(arg + strlen(auth_prefix), + &created, &err_msg); + if (!client) { + if (err_msg) { + connection_write_str_to_buf(err_msg, conn); + tor_free(err_msg); + } + goto out; + } + + if (auth_clients != NULL) { + int bad = 0; + SMARTLIST_FOREACH_BEGIN(auth_clients, rend_authorized_client_t *, ac) { + if (strcmp(ac->client_name, client->client_name) == 0) { + bad = 1; + break; + } + } SMARTLIST_FOREACH_END(ac); + if (bad) { + connection_printf_to_buf(conn, + "512 Duplicate name in ClientAuth\r\n"); + rend_authorized_client_free(client); + goto out; + } + } else { + auth_clients = smartlist_new(); + auth_created_clients = smartlist_new(); + } + smartlist_add(auth_clients, client); + if (created) { + smartlist_add(auth_created_clients, client); + } } else { connection_printf_to_buf(conn, "513 Invalid argument\r\n"); goto out; @@ -3871,6 +3916,18 @@ handle_control_add_onion(control_connection_t *conn, if (smartlist_len(port_cfgs) == 0) { connection_printf_to_buf(conn, "512 Missing 'Port' argument\r\n"); goto out; + } else if (auth_type == REND_NO_AUTH && auth_clients != NULL) { + connection_printf_to_buf(conn, "512 No auth type specified\r\n"); + goto out; + } else if (auth_type != REND_NO_AUTH && auth_clients == NULL) { + connection_printf_to_buf(conn, "512 No auth clients specified\r\n"); + goto out; + } else if ((auth_type == REND_BASIC_AUTH && + smartlist_len(auth_clients) > 512) || + (auth_type == REND_STEALTH_AUTH && + smartlist_len(auth_clients) > 16)) { + connection_printf_to_buf(conn, "512 Too many auth clients\r\n"); + goto out; } /* Parse the "keytype:keyblob" argument. */ @@ -3891,35 +3948,21 @@ handle_control_add_onion(control_connection_t *conn, } tor_assert(!err_msg); - /* Create the HS, using private key pk, and port config port_cfg. + /* Create the HS, using private key pk, client authentication auth_type, + * the list of auth_clients, and port config port_cfg. * rend_service_add_ephemeral() will take ownership of pk and port_cfg, * regardless of success/failure. */ char *service_id = NULL; int ret = rend_service_add_ephemeral(pk, port_cfgs, max_streams, max_streams_close_circuit, + auth_type, auth_clients, &service_id); port_cfgs = NULL; /* port_cfgs is now owned by the rendservice code. */ + auth_clients = NULL; /* so is auth_clients */ switch (ret) { case RSAE_OKAY: { - char *buf = NULL; - tor_assert(service_id); - if (key_new_alg) { - tor_assert(key_new_blob); - tor_asprintf(&buf, - "250-ServiceID=%s\r\n" - "250-PrivateKey=%s:%s\r\n" - "250 OK\r\n", - service_id, - key_new_alg, - key_new_blob); - } else { - tor_asprintf(&buf, - "250-ServiceID=%s\r\n" - "250 OK\r\n", - service_id); - } if (detach) { if (!detached_onion_services) detached_onion_services = smartlist_new(); @@ -3930,9 +3973,26 @@ handle_control_add_onion(control_connection_t *conn, smartlist_add(conn->ephemeral_onion_services, service_id); } - connection_write_str_to_buf(buf, conn); - memwipe(buf, 0, strlen(buf)); - tor_free(buf); + tor_assert(service_id); + connection_printf_to_buf(conn, "250-ServiceID=%s\r\n", service_id); + if (key_new_alg) { + tor_assert(key_new_blob); + connection_printf_to_buf(conn, "250-PrivateKey=%s:%s\r\n", + key_new_alg, key_new_blob); + } + if (auth_created_clients) { + SMARTLIST_FOREACH(auth_created_clients, rend_authorized_client_t *, ac, { + char *encoded = rend_auth_encode_cookie(ac->descriptor_cookie, + auth_type); + tor_assert(encoded); + connection_printf_to_buf(conn, "250-ClientAuth=%s:%s\r\n", + ac->client_name, encoded); + memwipe(encoded, 0, strlen(encoded)); + tor_free(encoded); + }); + } + + connection_printf_to_buf(conn, "250 OK\r\n"); break; } case RSAE_BADPRIVKEY: @@ -3944,6 +4004,9 @@ handle_control_add_onion(control_connection_t *conn, case RSAE_BADVIRTPORT: connection_printf_to_buf(conn, "512 Invalid VIRTPORT/TARGET\r\n"); break; + case RSAE_BADAUTH: + connection_printf_to_buf(conn, "512 Invalid client authorization\r\n"); + break; case RSAE_INTERNAL: /* FALLSTHROUGH */ default: connection_printf_to_buf(conn, "551 Failed to add Onion Service\r\n"); @@ -3960,6 +4023,16 @@ handle_control_add_onion(control_connection_t *conn, smartlist_free(port_cfgs); } + if (auth_clients) { + SMARTLIST_FOREACH(auth_clients, rend_authorized_client_t *, ac, + rend_authorized_client_free(ac)); + smartlist_free(auth_clients); + } + if (auth_created_clients) { + // Do not free entries; they are the same as auth_clients + smartlist_free(auth_created_clients); + } + SMARTLIST_FOREACH(args, char *, cp, { memwipe(cp, 0, strlen(cp)); tor_free(cp); @@ -4068,6 +4141,65 @@ add_onion_helper_keyarg(const char *arg, int discard_pk, return pk; } +/** Helper function to handle parsing a ClientAuth argument to the + * ADD_ONION command. Return a new rend_authorized_client_t, or NULL + * and an optional control protocol error message on failure. The + * caller is responsible for freeing the returned auth_client and err_msg. + * + * If 'created' is specified, it will be set to 1 when a new cookie has + * been generated. + */ +STATIC rend_authorized_client_t * +add_onion_helper_clientauth(const char *arg, int *created, char **err_msg) +{ + int ok = 0; + + tor_assert(arg); + tor_assert(created); + tor_assert(err_msg); + *err_msg = NULL; + + smartlist_t *auth_args = smartlist_new(); + rend_authorized_client_t *client = + tor_malloc_zero(sizeof(rend_authorized_client_t)); + smartlist_split_string(auth_args, arg, ":", 0, 0); + if (smartlist_len(auth_args) < 1 || smartlist_len(auth_args) > 2) { + *err_msg = tor_strdup("512 Invalid ClientAuth syntax\r\n"); + goto err; + } + client->client_name = tor_strdup(smartlist_get(auth_args, 0)); + if (smartlist_len(auth_args) == 2) { + char *decode_err_msg = NULL; + if (rend_auth_decode_cookie(smartlist_get(auth_args, 1), + client->descriptor_cookie, + NULL, &decode_err_msg) < 0) { + tor_assert(decode_err_msg); + tor_asprintf(err_msg, "512 %s\r\n", decode_err_msg); + tor_free(decode_err_msg); + goto err; + } + *created = 0; + } else { + crypto_rand((char *) client->descriptor_cookie, REND_DESC_COOKIE_LEN); + *created = 1; + } + + if (!rend_valid_client_name(client->client_name)) { + *err_msg = tor_strdup("512 Invalid name in ClientAuth\r\n"); + goto err; + } + + ok = 1; + err: + SMARTLIST_FOREACH(auth_args, char *, arg, tor_free(arg)); + smartlist_free(auth_args); + if (!ok) { + rend_authorized_client_free(client); + client = NULL; + } + return client; +} + /** Called when we get a DEL_ONION command; parse the body, and remove * the existing ephemeral Onion Service. */ static int diff --git a/src/or/control.h b/src/or/control.h index 008bfb1c3b..b3902e64bd 100644 --- a/src/or/control.h +++ b/src/or/control.h @@ -259,6 +259,8 @@ STATIC crypto_pk_t *add_onion_helper_keyarg(const char *arg, int discard_pk, const char **key_new_alg_out, char **key_new_blob_out, char **err_msg_out); +STATIC rend_authorized_client_t * +add_onion_helper_clientauth(const char *arg, int *created, char **err_msg_out); #endif #endif diff --git a/src/or/dircollate.c b/src/or/dircollate.c index 3f9d78f02d..756011b934 100644 --- a/src/or/dircollate.c +++ b/src/or/dircollate.c @@ -67,9 +67,9 @@ ddmap_entry_set_digests(ddmap_entry_t *ent, } HT_PROTOTYPE(double_digest_map, ddmap_entry_s, node, ddmap_entry_hash, - ddmap_entry_eq); + ddmap_entry_eq) HT_GENERATE2(double_digest_map, ddmap_entry_s, node, ddmap_entry_hash, - ddmap_entry_eq, 0.6, tor_reallocarray, tor_free_); + ddmap_entry_eq, 0.6, tor_reallocarray, tor_free_) /** Helper: add a single vote_routerstatus_t <b>vrs</b> to the collator * <b>dc</b>, indexing it by its RSA key digest, and by the 2-tuple of diff --git a/src/or/directory.c b/src/or/directory.c index 89b08223d2..1fe4d6804d 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -80,7 +80,6 @@ static void dir_routerdesc_download_failed(smartlist_t *failed, int was_descriptor_digests); static void dir_microdesc_download_failed(smartlist_t *failed, int status_code); -static void note_client_request(int purpose, int compressed, size_t bytes); static int client_likes_consensus(networkstatus_t *v, const char *want_url); static void directory_initiate_command_rend( @@ -495,8 +494,9 @@ MOCK_IMPL(void, directory_get_from_dirserver, ( * sort of dir fetch we'll be doing, so it won't return a bridge * that can't answer our question. */ - /* XXX024 Not all bridges handle conditional consensus downloading, - * so, for now, never assume the server supports that. -PP */ + /* XXX+++++ Not all bridges handle conditional consensus downloading, + * so, for now, never assume the server supports that. -PP + * Is that assumption still so in 2016? -NM */ const node_t *node = choose_random_dirguard(type); if (node && node->ri) { /* every bridge has a routerinfo. */ @@ -727,6 +727,10 @@ directory_initiate_command_routerstatus_rend(const routerstatus_t *status, node = node_get_by_id(status->identity_digest); + /* XXX The below check is wrong: !node means it's not in the consensus, + * but we haven't checked if we have a descriptor for it -- and also, + * we only care about the descriptor if it's a begindir-style anonymized + * connection. */ if (!node && anonymized_connection) { log_info(LD_DIR, "Not sending anonymized request to directory '%s'; we " "don't have its router descriptor.", @@ -744,7 +748,7 @@ directory_initiate_command_routerstatus_rend(const routerstatus_t *status, return; } - /* At this point, if we are a clients making a direct connection to a + /* At this point, if we are a client making a direct connection to a * directory server, we have selected a server that has at least one address * allowed by ClientUseIPv4/6 and Reachable{"",OR,Dir}Addresses. This * selection uses the preference in ClientPreferIPv6{OR,Dir}Port, if @@ -869,7 +873,7 @@ connection_dir_retry_bridges(smartlist_t *descs) char digest[DIGEST_LEN]; SMARTLIST_FOREACH(descs, const char *, cp, { - if (base16_decode(digest, DIGEST_LEN, cp, strlen(cp))<0) { + if (base16_decode(digest, DIGEST_LEN, cp, strlen(cp)) != DIGEST_LEN) { log_warn(LD_BUG, "Malformed fingerprint in list: %s", escaped(cp)); continue; @@ -1178,7 +1182,7 @@ directory_initiate_command_rend(const tor_addr_port_t *or_addr_port, /* set up conn so it's got all the data we need to remember */ tor_addr_copy(&conn->base_.addr, &addr); conn->base_.port = port; - conn->base_.address = tor_dup_addr(&addr); + conn->base_.address = tor_addr_to_str_dup(&addr); memcpy(conn->identity_digest, digest, DIGEST_LEN); conn->base_.purpose = dir_purpose; @@ -1839,7 +1843,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) char *body; char *headers; char *reason = NULL; - size_t body_len = 0, orig_len = 0; + size_t body_len = 0; int status_code; time_t date_header = 0; long apparent_skew; @@ -1849,7 +1853,6 @@ connection_dir_client_reached_eof(dir_connection_t *conn) int allow_partial = (conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC || conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO || conn->base_.purpose == DIR_PURPOSE_FETCH_MICRODESC); - int was_compressed = 0; time_t now = time(NULL); int src_code; @@ -1868,7 +1871,6 @@ connection_dir_client_reached_eof(dir_connection_t *conn) return -1; /* case 1, fall through */ } - orig_len = body_len; if (parse_http_response(headers, &status_code, &date_header, &compression, &reason) < 0) { @@ -1986,7 +1988,6 @@ connection_dir_client_reached_eof(dir_connection_t *conn) tor_free(body); body = new_body; body_len = new_len; - was_compressed = 1; } } @@ -2006,7 +2007,8 @@ connection_dir_client_reached_eof(dir_connection_t *conn) } log_info(LD_DIR,"Received consensus directory (size %d) from server " "'%s:%d'", (int)body_len, conn->base_.address, conn->base_.port); - if ((r=networkstatus_set_current_consensus(body, flavname, 0))<0) { + if ((r=networkstatus_set_current_consensus(body, flavname, 0, + conn->identity_digest))<0) { log_fn(r<-1?LOG_WARN:LOG_INFO, LD_DIR, "Unable to load %s consensus directory downloaded from " "server '%s:%d'. I'll try again soon.", @@ -2053,7 +2055,8 @@ connection_dir_client_reached_eof(dir_connection_t *conn) } if (src_code != -1) { - if (trusted_dirs_load_certs_from_string(body, src_code, 1)<0) { + if (trusted_dirs_load_certs_from_string(body, src_code, 1, + conn->identity_digest)<0) { log_warn(LD_DIR, "Unable to parse fetched certificates"); /* if we fetched more than one and only some failed, the successful * ones got flushed to disk so it's safe to call this on them */ @@ -2249,7 +2252,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) ds->nickname); /* XXXX use this information; be sure to upload next one * sooner. -NM */ - /* XXXX023 On further thought, the task above implies that we're + /* XXXX++ On further thought, the task above implies that we're * basing our regenerate-descriptor time on when we uploaded the * last descriptor, not on the published time of the last * descriptor. If those are different, that's a bad thing to @@ -2450,7 +2453,6 @@ connection_dir_client_reached_eof(dir_connection_t *conn) break; } } - note_client_request(conn->base_.purpose, was_compressed, orig_len); tor_free(body); tor_free(headers); tor_free(reason); return 0; } @@ -2651,129 +2653,6 @@ write_http_response_header(dir_connection_t *conn, ssize_t length, cache_lifetime); } -#if defined(INSTRUMENT_DOWNLOADS) || defined(RUNNING_DOXYGEN) -/* DOCDOC */ -typedef struct request_t { - uint64_t bytes; /**< How many bytes have we transferred? */ - uint64_t count; /**< How many requests have we made? */ -} request_t; - -/** Map used to keep track of how much data we've up/downloaded in what kind - * of request. Maps from request type to pointer to request_t. */ -static strmap_t *request_map = NULL; - -/** Record that a client request of <b>purpose</b> was made, and that - * <b>bytes</b> bytes of possibly <b>compressed</b> data were sent/received. - * Used to keep track of how much we've up/downloaded in what kind of - * request. */ -static void -note_client_request(int purpose, int compressed, size_t bytes) -{ - char *key; - const char *kind = NULL; - switch (purpose) { - case DIR_PURPOSE_FETCH_CONSENSUS: kind = "dl/consensus"; break; - case DIR_PURPOSE_FETCH_CERTIFICATE: kind = "dl/cert"; break; - case DIR_PURPOSE_FETCH_STATUS_VOTE: kind = "dl/vote"; break; - case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES: kind = "dl/detached_sig"; - break; - case DIR_PURPOSE_FETCH_SERVERDESC: kind = "dl/server"; break; - case DIR_PURPOSE_FETCH_EXTRAINFO: kind = "dl/extra"; break; - case DIR_PURPOSE_UPLOAD_DIR: kind = "dl/ul-dir"; break; - case DIR_PURPOSE_UPLOAD_VOTE: kind = "dl/ul-vote"; break; - case DIR_PURPOSE_UPLOAD_SIGNATURES: kind = "dl/ul-sig"; break; - case DIR_PURPOSE_FETCH_RENDDESC_V2: kind = "dl/rend2"; break; - case DIR_PURPOSE_UPLOAD_RENDDESC_V2: kind = "dl/ul-rend2"; break; - } - if (kind) { - tor_asprintf(&key, "%s%s", kind, compressed?".z":""); - } else { - tor_asprintf(&key, "unknown purpose (%d)%s", - purpose, compressed?".z":""); - } - note_request(key, bytes); - tor_free(key); -} - -/** Helper: initialize the request map to instrument downloads. */ -static void -ensure_request_map_initialized(void) -{ - if (!request_map) - request_map = strmap_new(); -} - -/** Called when we just transmitted or received <b>bytes</b> worth of data - * because of a request of type <b>key</b> (an arbitrary identifier): adds - * <b>bytes</b> to the total associated with key. */ -void -note_request(const char *key, size_t bytes) -{ - request_t *r; - ensure_request_map_initialized(); - - r = strmap_get(request_map, key); - if (!r) { - r = tor_malloc_zero(sizeof(request_t)); - strmap_set(request_map, key, r); - } - r->bytes += bytes; - r->count++; -} - -/** Return a newly allocated string holding a summary of bytes used per - * request type. */ -char * -directory_dump_request_log(void) -{ - smartlist_t *lines; - char *result; - strmap_iter_t *iter; - - ensure_request_map_initialized(); - - lines = smartlist_new(); - - for (iter = strmap_iter_init(request_map); - !strmap_iter_done(iter); - iter = strmap_iter_next(request_map, iter)) { - const char *key; - void *val; - request_t *r; - strmap_iter_get(iter, &key, &val); - r = val; - smartlist_add_asprintf(lines, "%s "U64_FORMAT" "U64_FORMAT"\n", - key, U64_PRINTF_ARG(r->bytes), U64_PRINTF_ARG(r->count)); - } - smartlist_sort_strings(lines); - result = smartlist_join_strings(lines, "", 0, NULL); - SMARTLIST_FOREACH(lines, char *, cp, tor_free(cp)); - smartlist_free(lines); - return result; -} -#else -static void -note_client_request(int purpose, int compressed, size_t bytes) -{ - (void)purpose; - (void)compressed; - (void)bytes; -} - -void -note_request(const char *key, size_t bytes) -{ - (void)key; - (void)bytes; -} - -char * -directory_dump_request_log(void) -{ - return tor_strdup("Not supported."); -} -#endif - /** Decide whether a client would accept the consensus we have. * * Clients can say they only want a consensus if it's signed by more @@ -2803,7 +2682,8 @@ client_likes_consensus(networkstatus_t *v, const char *want_url) if (want_len > DIGEST_LEN) want_len = DIGEST_LEN; - if (base16_decode(want_digest, DIGEST_LEN, d, want_len*2) < 0) { + if (base16_decode(want_digest, DIGEST_LEN, d, want_len*2) + != (int) want_len) { log_fn(LOG_PROTOCOL_WARN, LD_DIR, "Failed to decode requested authority digest %s.", escaped(d)); continue; @@ -2845,18 +2725,81 @@ choose_compression_level(ssize_t n_bytes) } } +/** Information passed to handle a GET request. */ +typedef struct get_handler_args_t { + /** True if the client asked for compressed data. */ + int compressed; + /** If nonzero, the time included an if-modified-since header with this + * value. */ + time_t if_modified_since; + /** String containing the requested URL or resource. */ + const char *url; + /** String containing the HTTP headers */ + const char *headers; +} get_handler_args_t; + +/** Entry for handling an HTTP GET request. + * + * This entry matches a request if "string" is equal to the requested + * resource, or if "is_prefix" is true and "string" is a prefix of the + * requested resource. + * + * The 'handler' function is called to handle the request. It receives + * an arguments structure, and must return 0 on success or -1 if we should + * close the connection. + **/ +typedef struct url_table_ent_s { + const char *string; + int is_prefix; + int (*handler)(dir_connection_t *conn, const get_handler_args_t *args); +} url_table_ent_t; + +static int handle_get_frontpage(dir_connection_t *conn, + const get_handler_args_t *args); +static int handle_get_current_consensus(dir_connection_t *conn, + const get_handler_args_t *args); +static int handle_get_status_vote(dir_connection_t *conn, + const get_handler_args_t *args); +static int handle_get_microdesc(dir_connection_t *conn, + const get_handler_args_t *args); +static int handle_get_descriptor(dir_connection_t *conn, + const get_handler_args_t *args); +static int handle_get_keys(dir_connection_t *conn, + const get_handler_args_t *args); +static int handle_get_rendezvous2(dir_connection_t *conn, + const get_handler_args_t *args); +static int handle_get_robots(dir_connection_t *conn, + const get_handler_args_t *args); +static int handle_get_networkstatus_bridges(dir_connection_t *conn, + const get_handler_args_t *args); + +/** Table for handling GET requests. */ +static const url_table_ent_t url_table[] = { + { "/tor/", 0, handle_get_frontpage }, + { "/tor/status-vote/current/consensus", 1, handle_get_current_consensus }, + { "/tor/status-vote/current/", 1, handle_get_status_vote }, + { "/tor/status-vote/next/", 1, handle_get_status_vote }, + { "/tor/micro/d/", 1, handle_get_microdesc }, + { "/tor/server/", 1, handle_get_descriptor }, + { "/tor/extra/", 1, handle_get_descriptor }, + { "/tor/keys/", 1, handle_get_keys }, + { "/tor/rendezvous2/", 1, handle_get_rendezvous2 }, + { "/tor/robots.txt", 0, handle_get_robots }, + { "/tor/networkstatus-bridges", 0, handle_get_networkstatus_bridges }, + { NULL, 0, NULL }, +}; + /** Helper function: called when a dirserver gets a complete HTTP GET * request. Look for a request for a directory or for a rendezvous * service descriptor. On finding one, write a response into - * conn-\>outbuf. If the request is unrecognized, send a 400. - * Always return 0. */ + * conn-\>outbuf. If the request is unrecognized, send a 404. + * Return 0 if we handled this successfully, or -1 if we need to close + * the connection. */ STATIC int directory_handle_command_get(dir_connection_t *conn, const char *headers, const char *req_body, size_t req_body_len) { - size_t dlen; char *url, *url_mem, *header; - const or_options_t *options = get_options(); time_t if_modified_since = 0; int compressed; size_t url_len; @@ -2896,29 +2839,73 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, url_len -= 2; } - if (!strcmp(url,"/tor/")) { - const char *frontpage = get_dirportfrontpage(); - - if (frontpage) { - dlen = strlen(frontpage); - /* Let's return a disclaimer page (users shouldn't use V1 anymore, - and caches don't fetch '/', so this is safe). */ - - /* [We don't check for write_bucket_low here, since we want to serve - * this page no matter what.] */ - note_request(url, dlen); - write_http_response_header_impl(conn, dlen, "text/html", "identity", - NULL, DIRPORTFRONTPAGE_CACHE_LIFETIME); - connection_write_to_buf(frontpage, dlen, TO_CONN(conn)); + get_handler_args_t args; + args.url = url; + args.headers = headers; + args.if_modified_since = if_modified_since; + args.compressed = compressed; + + int i, result = -1; + for (i = 0; url_table[i].string; ++i) { + int match; + if (url_table[i].is_prefix) { + match = !strcmpstart(url, url_table[i].string); + } else { + match = !strcmp(url, url_table[i].string); + } + if (match) { + result = url_table[i].handler(conn, &args); goto done; } - /* if no disclaimer file, fall through and continue */ } - if (!strcmpstart(url, "/tor/status-vote/current/consensus")) { + /* we didn't recognize the url */ + write_http_status_line(conn, 404, "Not found"); + result = 0; + + done: + tor_free(url_mem); + return result; +} + +/** Helper function for GET / or GET /tor/ + */ +static int +handle_get_frontpage(dir_connection_t *conn, const get_handler_args_t *args) +{ + (void) args; /* unused */ + const char *frontpage = get_dirportfrontpage(); + + if (frontpage) { + size_t dlen; + dlen = strlen(frontpage); + /* Let's return a disclaimer page (users shouldn't use V1 anymore, + and caches don't fetch '/', so this is safe). */ + + /* [We don't check for write_bucket_low here, since we want to serve + * this page no matter what.] */ + write_http_response_header_impl(conn, dlen, "text/html", "identity", + NULL, DIRPORTFRONTPAGE_CACHE_LIFETIME); + connection_write_to_buf(frontpage, dlen, TO_CONN(conn)); + } else { + write_http_status_line(conn, 404, "Not found"); + } + return 0; +} + +/** Helper function for GET /tor/status-vote/current/consensus + */ +static int +handle_get_current_consensus(dir_connection_t *conn, + const get_handler_args_t *args) +{ + const char *url = args->url; + const int compressed = args->compressed; + const time_t if_modified_since = args->if_modified_since; + + { /* v3 network status fetch. */ smartlist_t *dir_fps = smartlist_new(); - const char *request_type = NULL; long lifetime = NETWORKSTATUS_CACHE_LIFETIME; if (1) { @@ -2967,7 +2954,6 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, tor_free(flavor); smartlist_add(dir_fps, fp); } - request_type = compressed?"v3.z":"v3"; lifetime = (v && v->fresh_until > now) ? v->fresh_until - now : 0; } @@ -2992,7 +2978,7 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, goto done; } - dlen = dirserv_estimate_data_size(dir_fps, 0, compressed); + size_t dlen = dirserv_estimate_data_size(dir_fps, 0, compressed); if (global_write_bucket_low(TO_CONN(conn), dlen, 2)) { log_debug(LD_DIRSERV, "Client asked for network status lists, but we've been " @@ -3022,8 +3008,6 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, } } - // note_request(request_type,dlen); - (void) request_type; write_http_response_header(conn, -1, compressed, smartlist_len(dir_fps) == 1 ? lifetime : 0); conn->fingerprint_stack = dir_fps; @@ -3036,17 +3020,24 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, goto done; } - if (!strcmpstart(url,"/tor/status-vote/current/") || - !strcmpstart(url,"/tor/status-vote/next/")) { - /* XXXX If-modified-since is only implemented for the current - * consensus: that's probably fine, since it's the only vote document - * people fetch much. */ + done: + return 0; +} + +/** Helper function for GET /tor/status-vote/{current,next}/... + */ +static int +handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args) +{ + const char *url = args->url; + const int compressed = args->compressed; + { int current; ssize_t body_len = 0; ssize_t estimated_len = 0; smartlist_t *items = smartlist_new(); smartlist_t *dir_items = smartlist_new(); - int lifetime = 60; /* XXXX023 should actually use vote intervals. */ + int lifetime = 60; /* XXXX?? should actually use vote intervals. */ url += strlen("/tor/status-vote/"); current = !strcmpstart(url, "current/"); url = strchr(url, '/'); @@ -3136,8 +3127,18 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, smartlist_free(dir_items); goto done; } + done: + return 0; +} - if (!strcmpstart(url, "/tor/micro/d/")) { +/** Helper function for GET /tor/micro/d/... + */ +static int +handle_get_microdesc(dir_connection_t *conn, const get_handler_args_t *args) +{ + const char *url = args->url; + const int compressed = args->compressed; + { smartlist_t *fps = smartlist_new(); dir_split_resource_into_fingerprints(url+strlen("/tor/micro/d/"), @@ -3150,7 +3151,7 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, smartlist_free(fps); goto done; } - dlen = dirserv_estimate_microdesc_size(fps, compressed); + size_t dlen = dirserv_estimate_microdesc_size(fps, compressed); if (global_write_bucket_low(TO_CONN(conn), dlen, 2)) { log_info(LD_DIRSERV, "Client asked for server descriptors, but we've been " @@ -3173,12 +3174,24 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, goto done; } + done: + return 0; +} + +/** Helper function for GET /tor/{server,extra}/... + */ +static int +handle_get_descriptor(dir_connection_t *conn, const get_handler_args_t *args) +{ + const char *url = args->url; + const int compressed = args->compressed; + const or_options_t *options = get_options(); if (!strcmpstart(url,"/tor/server/") || (!options->BridgeAuthoritativeDir && !options->BridgeRelay && !strcmpstart(url,"/tor/extra/"))) { + size_t dlen; int res; const char *msg; - const char *request_type = NULL; int cache_lifetime = 0; int is_extra = !strcmpstart(url,"/tor/extra/"); url += is_extra ? strlen("/tor/extra/") : strlen("/tor/server/"); @@ -3189,24 +3202,16 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, is_extra); if (!strcmpstart(url, "fp/")) { - request_type = compressed?"/tor/server/fp.z":"/tor/server/fp"; if (smartlist_len(conn->fingerprint_stack) == 1) cache_lifetime = ROUTERDESC_CACHE_LIFETIME; } else if (!strcmpstart(url, "authority")) { - request_type = compressed?"/tor/server/authority.z": - "/tor/server/authority"; cache_lifetime = ROUTERDESC_CACHE_LIFETIME; } else if (!strcmpstart(url, "all")) { - request_type = compressed?"/tor/server/all.z":"/tor/server/all"; cache_lifetime = FULL_DIR_CACHE_LIFETIME; } else if (!strcmpstart(url, "d/")) { - request_type = compressed?"/tor/server/d.z":"/tor/server/d"; if (smartlist_len(conn->fingerprint_stack) == 1) cache_lifetime = ROUTERDESC_BY_DIGEST_CACHE_LIFETIME; - } else { - request_type = "/tor/server/?"; } - (void) request_type; /* usable for note_request. */ if (!strcmpstart(url, "d/")) conn->dir_spool_src = is_extra ? DIR_SPOOL_EXTRA_BY_DIGEST : DIR_SPOOL_SERVER_BY_DIGEST; @@ -3242,8 +3247,19 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, } goto done; } + done: + return 0; +} - if (!strcmpstart(url,"/tor/keys/")) { +/** Helper function for GET /tor/keys/... + */ +static int +handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args) +{ + const char *url = args->url; + const int compressed = args->compressed; + const time_t if_modified_since = args->if_modified_since; + { smartlist_t *certs = smartlist_new(); ssize_t len = -1; if (!strcmp(url, "/tor/keys/all")) { @@ -3328,9 +3344,17 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, smartlist_free(certs); goto done; } + done: + return 0; +} - if (connection_dir_is_encrypted(conn) && - !strcmpstart(url,"/tor/rendezvous2/")) { +/** Helper function for GET /tor/rendezvous2/ + */ +static int +handle_get_rendezvous2(dir_connection_t *conn, const get_handler_args_t *args) +{ + const char *url = args->url; + if (connection_dir_is_encrypted(conn)) { /* Handle v2 rendezvous descriptor fetch request. */ const char *descp; const char *query = url + strlen("/tor/rendezvous2/"); @@ -3353,16 +3377,30 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, write_http_status_line(conn, 400, "Bad request"); } goto done; + } else { + /* Not encrypted! */ + write_http_status_line(conn, 404, "Not found"); } + done: + return 0; +} +/** Helper function for GET /tor/networkstatus-bridges + */ +static int +handle_get_networkstatus_bridges(dir_connection_t *conn, + const get_handler_args_t *args) +{ + const char *headers = args->headers; + + const or_options_t *options = get_options(); if (options->BridgeAuthoritativeDir && options->BridgePassword_AuthDigest_ && - connection_dir_is_encrypted(conn) && - !strcmp(url,"/tor/networkstatus-bridges")) { + connection_dir_is_encrypted(conn)) { char *status; char digest[DIGEST256_LEN]; - header = http_get_header(headers, "Authorization: Basic "); + char *header = http_get_header(headers, "Authorization: Basic "); if (header) crypto_digest256(digest, header, strlen(header), DIGEST_SHA256); @@ -3378,75 +3416,27 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, /* all happy now. send an answer. */ status = networkstatus_getinfo_by_purpose("bridge", time(NULL)); - dlen = strlen(status); + size_t dlen = strlen(status); write_http_response_header(conn, dlen, 0, 0); connection_write_to_buf(status, dlen, TO_CONN(conn)); tor_free(status); goto done; } + done: + return 0; +} - if (!strcmpstart(url,"/tor/bytes.txt")) { - char *bytes = directory_dump_request_log(); - size_t len = strlen(bytes); - write_http_response_header(conn, len, 0, 0); - connection_write_to_buf(bytes, len, TO_CONN(conn)); - tor_free(bytes); - goto done; - } - - if (!strcmp(url,"/tor/robots.txt")) { /* /robots.txt will have been - rewritten to /tor/robots.txt */ - char robots[] = "User-agent: *\r\nDisallow: /\r\n"; +/** Helper function for GET robots.txt or /tor/robots.txt */ +static int +handle_get_robots(dir_connection_t *conn, const get_handler_args_t *args) +{ + (void)args; + { + const char robots[] = "User-agent: *\r\nDisallow: /\r\n"; size_t len = strlen(robots); write_http_response_header(conn, len, 0, ROBOTS_CACHE_LIFETIME); connection_write_to_buf(robots, len, TO_CONN(conn)); - goto done; } - -#if defined(EXPORTMALLINFO) && defined(HAVE_MALLOC_H) && defined(HAVE_MALLINFO) -#define ADD_MALLINFO_LINE(x) do { \ - smartlist_add_asprintf(lines, "%s %d\n", #x, mi.x); \ - }while(0); - - if (!strcmp(url,"/tor/mallinfo.txt") && - (tor_addr_eq_ipv4h(&conn->base_.addr, 0x7f000001ul))) { - char *result; - size_t len; - struct mallinfo mi; - smartlist_t *lines; - - memset(&mi, 0, sizeof(mi)); - mi = mallinfo(); - lines = smartlist_new(); - - ADD_MALLINFO_LINE(arena) - ADD_MALLINFO_LINE(ordblks) - ADD_MALLINFO_LINE(smblks) - ADD_MALLINFO_LINE(hblks) - ADD_MALLINFO_LINE(hblkhd) - ADD_MALLINFO_LINE(usmblks) - ADD_MALLINFO_LINE(fsmblks) - ADD_MALLINFO_LINE(uordblks) - ADD_MALLINFO_LINE(fordblks) - ADD_MALLINFO_LINE(keepcost) - - result = smartlist_join_strings(lines, "", 0, NULL); - SMARTLIST_FOREACH(lines, char *, cp, tor_free(cp)); - smartlist_free(lines); - - len = strlen(result); - write_http_response_header(conn, len, 0, 0); - connection_write_to_buf(result, len, TO_CONN(conn)); - tor_free(result); - goto done; - } -#endif - - /* we didn't recognize the url */ - write_http_status_line(conn, 404, "Not found"); - - done: - tor_free(url_mem); return 0; } @@ -3773,17 +3763,83 @@ find_dl_schedule(download_status_t *dls, const or_options_t *options) return NULL; } -/* Find the current delay for dls based on schedule. - * Set dls->next_attempt_at based on now, and return the delay. +/** Decide which minimum and maximum delay step we want to use based on + * descriptor type in <b>dls</b> and <b>options</b>. + * Helper function for download_status_schedule_get_delay(). */ +STATIC void +find_dl_min_and_max_delay(download_status_t *dls, const or_options_t *options, + int *min, int *max) +{ + tor_assert(dls); + tor_assert(options); + tor_assert(min); + tor_assert(max); + + /* + * For now, just use the existing schedule config stuff and pick the + * first/last entries off to get min/max delay for backoff purposes + */ + const smartlist_t *schedule = find_dl_schedule(dls, options); + tor_assert(schedule != NULL && smartlist_len(schedule) >= 2); + *min = *((int *)(smartlist_get(schedule, 0))); + *max = *((int *)((smartlist_get(schedule, smartlist_len(schedule) - 1)))); +} + +/** Advance one delay step. The algorithm is to use the previous delay to + * compute an increment, we construct a value uniformly at random between + * delay and MAX(delay*2,delay+1). We then clamp that value to be no larger + * than max_delay, and return it. + * + * Requires that delay is less than INT_MAX, and delay is in [0,max_delay]. + */ +STATIC int +next_random_exponential_delay(int delay, int max_delay) +{ + /* Check preconditions */ + if (BUG(delay > max_delay)) + delay = max_delay; + if (BUG(delay == INT_MAX)) + delay -= 1; /* prevent overflow */ + if (BUG(delay < 0)) + delay = 0; + + /* How much are we willing to add to the delay? */ + int max_increment; + + if (delay) + max_increment = delay; /* no more than double. */ + else + max_increment = 1; /* we're always willing to slow down a little. */ + + /* the + 1 here is so that we include the end of the interval */ + int increment = crypto_rand_int(max_increment+1); + + if (increment < max_delay - delay) + return delay + increment; + else + return max_delay; +} + +/** Find the current delay for dls based on schedule or min_delay/ + * max_delay if we're using exponential backoff. If dls->backoff is + * DL_SCHED_RANDOM_EXPONENTIAL, we must have 0 <= min_delay <= max_delay <= + * INT_MAX, but schedule may be set to NULL; otherwise schedule is required. + * This function sets dls->next_attempt_at based on now, and returns the delay. * Helper for download_status_increment_failure and * download_status_increment_attempt. */ STATIC int download_status_schedule_get_delay(download_status_t *dls, const smartlist_t *schedule, + int min_delay, int max_delay, time_t now) { tor_assert(dls); - tor_assert(schedule); + /* We don't need a schedule if we're using random exponential backoff */ + tor_assert(dls->backoff == DL_SCHED_RANDOM_EXPONENTIAL || + schedule != NULL); + /* If we're using random exponential backoff, we do need min/max delay */ + tor_assert(dls->backoff != DL_SCHED_RANDOM_EXPONENTIAL || + (min_delay >= 0 && max_delay >= min_delay)); int delay = INT_MAX; uint8_t dls_schedule_position = (dls->increment_on @@ -3791,12 +3847,42 @@ download_status_schedule_get_delay(download_status_t *dls, ? dls->n_download_attempts : dls->n_download_failures); - if (dls_schedule_position < smartlist_len(schedule)) - delay = *(int *)smartlist_get(schedule, dls_schedule_position); - else if (dls_schedule_position == IMPOSSIBLE_TO_DOWNLOAD) - delay = INT_MAX; - else - delay = *(int *)smartlist_get(schedule, smartlist_len(schedule) - 1); + if (dls->backoff == DL_SCHED_DETERMINISTIC) { + if (dls_schedule_position < smartlist_len(schedule)) + delay = *(int *)smartlist_get(schedule, dls_schedule_position); + else if (dls_schedule_position == IMPOSSIBLE_TO_DOWNLOAD) + delay = INT_MAX; + else + delay = *(int *)smartlist_get(schedule, smartlist_len(schedule) - 1); + } else if (dls->backoff == DL_SCHED_RANDOM_EXPONENTIAL) { + /* Check if we missed a reset somehow */ + if (dls->last_backoff_position > dls_schedule_position) { + dls->last_backoff_position = 0; + dls->last_delay_used = 0; + } + + if (dls_schedule_position > 0) { + delay = dls->last_delay_used; + + while (dls->last_backoff_position < dls_schedule_position) { + /* Do one increment step */ + delay = next_random_exponential_delay(delay, max_delay); + /* Update our position */ + ++(dls->last_backoff_position); + } + } else { + /* If we're just starting out, use the minimum delay */ + delay = min_delay; + } + + /* Clamp it within min/max if we have them */ + if (min_delay >= 0 && delay < min_delay) delay = min_delay; + if (max_delay != INT_MAX && delay > max_delay) delay = max_delay; + + /* Store it for next time */ + dls->last_backoff_position = dls_schedule_position; + dls->last_delay_used = delay; + } /* A negative delay makes no sense. Knowing that delay is * non-negative allows us to safely do the wrapping check below. */ @@ -3857,6 +3943,8 @@ download_status_increment_failure(download_status_t *dls, int status_code, const char *item, int server, time_t now) { int increment = -1; + int min_delay = 0, max_delay = INT_MAX; + tor_assert(dls); /* only count the failure if it's permanent, or we're a server */ @@ -3877,7 +3965,9 @@ download_status_increment_failure(download_status_t *dls, int status_code, /* only return a failure retry time if this schedule increments on failures */ const smartlist_t *schedule = find_dl_schedule(dls, get_options()); - increment = download_status_schedule_get_delay(dls, schedule, now); + find_dl_min_and_max_delay(dls, get_options(), &min_delay, &max_delay); + increment = download_status_schedule_get_delay(dls, schedule, + min_delay, max_delay, now); } download_status_log_helper(item, !dls->increment_on, "failed", @@ -3906,6 +3996,8 @@ download_status_increment_attempt(download_status_t *dls, const char *item, time_t now) { int delay = -1; + int min_delay = 0, max_delay = INT_MAX; + tor_assert(dls); if (dls->increment_on == DL_SCHED_INCREMENT_FAILURE) { @@ -3920,7 +4012,9 @@ download_status_increment_attempt(download_status_t *dls, const char *item, ++dls->n_download_attempts; const smartlist_t *schedule = find_dl_schedule(dls, get_options()); - delay = download_status_schedule_get_delay(dls, schedule, now); + find_dl_min_and_max_delay(dls, get_options(), &min_delay, &max_delay); + delay = download_status_schedule_get_delay(dls, schedule, + min_delay, max_delay, now); download_status_log_helper(item, dls->increment_on, "attempted", "on failure", dls->n_download_attempts, @@ -3952,6 +4046,8 @@ download_status_reset(download_status_t *dls) dls->n_download_failures = 0; dls->n_download_attempts = 0; dls->next_attempt_at = time(NULL) + *(int *)smartlist_get(schedule, 0); + dls->last_backoff_position = 0; + dls->last_delay_used = 0; /* Don't reset dls->want_authority or dls->increment_on */ } @@ -4001,7 +4097,7 @@ dir_routerdesc_download_failed(smartlist_t *failed, int status_code, } SMARTLIST_FOREACH_BEGIN(failed, const char *, cp) { download_status_t *dls = NULL; - if (base16_decode(digest, DIGEST_LEN, cp, strlen(cp)) < 0) { + if (base16_decode(digest, DIGEST_LEN, cp, strlen(cp)) != DIGEST_LEN) { log_warn(LD_BUG, "Malformed fingerprint in list: %s", escaped(cp)); continue; } @@ -4098,9 +4194,10 @@ dir_split_resource_into_fingerprint_pairs(const char *res, "Skipping digest pair %s with missing dash.", escaped(cp)); } else { fp_pair_t pair; - if (base16_decode(pair.first, DIGEST_LEN, cp, HEX_DIGEST_LEN)<0 || - base16_decode(pair.second, - DIGEST_LEN, cp+HEX_DIGEST_LEN+1, HEX_DIGEST_LEN)<0) { + if (base16_decode(pair.first, DIGEST_LEN, + cp, HEX_DIGEST_LEN) != DIGEST_LEN || + base16_decode(pair.second,DIGEST_LEN, + cp+HEX_DIGEST_LEN+1, HEX_DIGEST_LEN) != DIGEST_LEN) { log_info(LD_DIR, "Skipping non-decodable digest pair %s", escaped(cp)); } else { smartlist_add(pairs_result, tor_memdup(&pair, sizeof(pair))); @@ -4178,8 +4275,9 @@ dir_split_resource_into_fingerprints(const char *resource, } d = tor_malloc_zero(digest_len); if (decode_hex ? - (base16_decode(d, digest_len, cp, hex_digest_len)<0) : - (base64_decode(d, digest_len, cp, base64_digest_len)<0)) { + (base16_decode(d, digest_len, cp, hex_digest_len) != digest_len) : + (base64_decode(d, digest_len, cp, base64_digest_len) + != digest_len)) { log_info(LD_DIR, "Skipping non-decodable digest %s", escaped(cp)); smartlist_del_keeporder(fp_tmp, i--); goto again; diff --git a/src/or/directory.h b/src/or/directory.h index 7646cac03f..afa3bcc611 100644 --- a/src/or/directory.h +++ b/src/or/directory.h @@ -146,6 +146,7 @@ STATIC int directory_handle_command_get(dir_connection_t *conn, size_t req_body_len); STATIC int download_status_schedule_get_delay(download_status_t *dls, const smartlist_t *schedule, + int min_delay, int max_delay, time_t now); STATIC char* authdir_type_to_string(dirinfo_type_t auth); @@ -154,6 +155,11 @@ STATIC int should_use_directory_guards(const or_options_t *options); STATIC zlib_compression_level_t choose_compression_level(ssize_t n_bytes); STATIC const smartlist_t *find_dl_schedule(download_status_t *dls, const or_options_t *options); +STATIC void find_dl_min_and_max_delay(download_status_t *dls, + const or_options_t *options, + int *min, int *max); +STATIC int next_random_exponential_delay(int delay, int max_delay); + #endif #endif diff --git a/src/or/dirserv.c b/src/or/dirserv.c index dafaed8bf2..e616373b61 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -19,6 +19,7 @@ #include "dirvote.h" #include "hibernate.h" #include "keypin.h" +#include "main.h" #include "microdesc.h" #include "networkstatus.h" #include "nodelist.h" @@ -44,10 +45,6 @@ * directory authorities. */ #define MAX_UNTRUSTED_NETWORKSTATUSES 16 -extern time_t time_of_process_start; /* from main.c */ - -extern long stats_n_seconds_working; /* from main.c */ - /** Total number of routers with measured bandwidth; this is set by * dirserv_count_measured_bws() before the loop in * dirserv_generate_networkstatus_vote_obj() and checked by @@ -125,7 +122,8 @@ add_fingerprint_to_dir(const char *fp, authdir_config_t *list, fingerprint = tor_strdup(fp); tor_strstrip(fingerprint, " "); - if (base16_decode(d, DIGEST_LEN, fingerprint, strlen(fingerprint))) { + if (base16_decode(d, DIGEST_LEN, + fingerprint, strlen(fingerprint)) != DIGEST_LEN) { log_warn(LD_DIRSERV, "Couldn't decode fingerprint \"%s\"", escaped(fp)); tor_free(fingerprint); @@ -202,7 +200,7 @@ dirserv_load_fingerprint_file(void) tor_strstrip(fingerprint, " "); /* remove spaces */ if (strlen(fingerprint) != HEX_DIGEST_LEN || base16_decode(digest_tmp, sizeof(digest_tmp), - fingerprint, HEX_DIGEST_LEN) < 0) { + fingerprint, HEX_DIGEST_LEN) != sizeof(digest_tmp)) { log_notice(LD_CONFIG, "Invalid fingerprint (nickname '%s', " "fingerprint %s). Skipping.", @@ -349,7 +347,7 @@ dirserv_get_status_impl(const char *id_digest, const char *nickname, if (result & FP_REJECT) { if (msg) - *msg = "Fingerprint is marked rejected"; + *msg = "Fingerprint is marked rejected -- please contact us?"; return FP_REJECT; } else if (result & FP_INVALID) { if (msg) @@ -367,7 +365,7 @@ dirserv_get_status_impl(const char *id_digest, const char *nickname, log_fn(severity, LD_DIRSERV, "Rejecting '%s' because of address '%s'", nickname, fmt_addr32(addr)); if (msg) - *msg = "Authdir is rejecting routers in this range."; + *msg = "Suspicious relay address range -- please contact us?"; return FP_REJECT; } if (!authdir_policy_valid_address(addr, or_port)) { @@ -823,7 +821,7 @@ running_long_enough_to_decide_unreachable(void) void dirserv_set_router_is_running(routerinfo_t *router, time_t now) { - /*XXXX024 This function is a mess. Separate out the part that calculates + /*XXXX This function is a mess. Separate out the part that calculates whether it's reachable and the part that tells rephist that the router was unreachable. */ @@ -985,94 +983,6 @@ router_is_active(const routerinfo_t *ri, const node_t *node, time_t now) return 1; } -/** Generate a new v1 directory and write it into a newly allocated string. - * Point *<b>dir_out</b> to the allocated string. Sign the - * directory with <b>private_key</b>. Return 0 on success, -1 on - * failure. If <b>complete</b> is set, give us all the descriptors; - * otherwise leave out non-running and non-valid ones. - */ -int -dirserv_dump_directory_to_string(char **dir_out, - crypto_pk_t *private_key) -{ - /* XXXX 024 Get rid of this function if we can confirm that nobody's - * fetching these any longer */ - char *cp; - char *identity_pkey; /* Identity key, DER64-encoded. */ - char *recommended_versions; - char digest[DIGEST_LEN]; - char published[ISO_TIME_LEN+1]; - char *buf = NULL; - size_t buf_len; - size_t identity_pkey_len; - time_t now = time(NULL); - - tor_assert(dir_out); - *dir_out = NULL; - - if (crypto_pk_write_public_key_to_string(private_key,&identity_pkey, - &identity_pkey_len)<0) { - log_warn(LD_BUG,"write identity_pkey to string failed!"); - return -1; - } - - recommended_versions = - format_versions_list(get_options()->RecommendedVersions); - - format_iso_time(published, now); - - buf_len = 2048+strlen(recommended_versions); - - buf = tor_malloc(buf_len); - /* We'll be comparing against buf_len throughout the rest of the - function, though strictly speaking we shouldn't be able to exceed - it. This is C, after all, so we may as well check for buffer - overruns.*/ - - tor_snprintf(buf, buf_len, - "signed-directory\n" - "published %s\n" - "recommended-software %s\n" - "router-status %s\n" - "dir-signing-key\n%s\n", - published, recommended_versions, "", - identity_pkey); - - tor_free(recommended_versions); - tor_free(identity_pkey); - - cp = buf + strlen(buf); - *cp = '\0'; - - /* These multiple strlcat calls are inefficient, but dwarfed by the RSA - signature. */ - if (strlcat(buf, "directory-signature ", buf_len) >= buf_len) - goto truncated; - if (strlcat(buf, get_options()->Nickname, buf_len) >= buf_len) - goto truncated; - if (strlcat(buf, "\n", buf_len) >= buf_len) - goto truncated; - - if (router_get_dir_hash(buf,digest)) { - log_warn(LD_BUG,"couldn't compute digest"); - tor_free(buf); - return -1; - } - note_crypto_pk_op(SIGN_DIR); - if (router_append_dirobj_signature(buf,buf_len,digest,DIGEST_LEN, - private_key)<0) { - tor_free(buf); - return -1; - } - - *dir_out = buf; - return 0; - truncated: - log_warn(LD_BUG,"tried to exceed string length."); - tor_free(buf); - return -1; -} - /********************************************************************/ /* A set of functions to answer questions about how we'd like to behave @@ -1329,7 +1239,7 @@ dirserv_thinks_router_is_unreliable(time_t now, { if (need_uptime) { if (!enough_mtbf_info) { - /* XXX024 Once most authorities are on v3, we should change the rule from + /* XXXX We should change the rule from * "use uptime if we don't have mtbf data" to "don't advertise Stable on * v3 if we don't have enough mtbf data." Or maybe not, since if we ever * hit a point where we need to reset a lot of authorities at once, @@ -2206,7 +2116,7 @@ set_routerstatus_from_routerinfo(routerstatus_t *rs, rs->is_valid = node->is_valid; - if (node->is_fast && + if (node->is_fast && node->is_stable && ((options->AuthDirGuardBWGuarantee && routerbw_kb >= options->AuthDirGuardBWGuarantee/1000) || routerbw_kb >= MIN(guard_bandwidth_including_exits_kb, @@ -2365,7 +2275,8 @@ guardfraction_file_parse_guard_line(const char *guard_line, inputs_tmp = smartlist_get(sl, 0); if (strlen(inputs_tmp) != HEX_DIGEST_LEN || - base16_decode(guard_id, DIGEST_LEN, inputs_tmp, HEX_DIGEST_LEN)) { + base16_decode(guard_id, DIGEST_LEN, + inputs_tmp, HEX_DIGEST_LEN) != DIGEST_LEN) { tor_asprintf(err_msg, "bad digest '%s'", inputs_tmp); goto done; } @@ -2669,7 +2580,8 @@ measured_bw_line_parse(measured_bw_line_t *out, const char *orig_line) cp+=strlen("node_id=$"); if (strlen(cp) != HEX_DIGEST_LEN || - base16_decode(out->node_id, DIGEST_LEN, cp, HEX_DIGEST_LEN)) { + base16_decode(out->node_id, DIGEST_LEN, + cp, HEX_DIGEST_LEN) != DIGEST_LEN) { log_warn(LD_DIRSERV, "Invalid node_id in bandwidth file line: %s", escaped(orig_line)); tor_free(line); @@ -3340,7 +3252,7 @@ lookup_cached_dir_by_fp(const char *fp) d = strmap_get(cached_consensuses, "ns"); } else if (memchr(fp, '\0', DIGEST_LEN) && cached_consensuses && (d = strmap_get(cached_consensuses, fp))) { - /* this here interface is a nasty hack XXXX024 */; + /* this here interface is a nasty hack XXXX */; } return d; } diff --git a/src/or/dirserv.h b/src/or/dirserv.h index 9a9725ad6f..3c914e9311 100644 --- a/src/or/dirserv.h +++ b/src/or/dirserv.h @@ -47,8 +47,6 @@ enum was_router_added_t dirserv_add_descriptor(routerinfo_t *ri, void dirserv_set_router_is_running(routerinfo_t *router, time_t now); int list_server_status_v1(smartlist_t *routers, char **router_status_out, int for_controller); -int dirserv_dump_directory_to_string(char **dir_out, - crypto_pk_t *private_key); char *dirserv_get_flag_thresholds_line(void); void dirserv_compute_bridge_flag_thresholds(void); diff --git a/src/or/dirvote.c b/src/or/dirvote.c index 62f85877fe..ad0696eddb 100644 --- a/src/or/dirvote.c +++ b/src/or/dirvote.c @@ -106,6 +106,7 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key, SMARTLIST_FOREACH(v3_ns->package_lines, const char *, p, if (validate_recommended_package_line(p)) smartlist_add_asprintf(tmp, "package %s\n", p)); + smartlist_sort_strings(tmp); packages = smartlist_join_strings(tmp, "", 0, NULL); SMARTLIST_FOREACH(tmp, char *, cp, tor_free(cp)); smartlist_free(tmp); @@ -2916,7 +2917,8 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out) /* Hey, it's a new cert! */ trusted_dirs_load_certs_from_string( vote->cert->cache_info.signed_descriptor_body, - TRUSTED_DIRS_CERTS_SRC_FROM_VOTE, 1 /*flush*/); + TRUSTED_DIRS_CERTS_SRC_FROM_VOTE, 1 /*flush*/, + NULL); if (!authority_cert_get_by_digests(vote->cert->cache_info.identity_digest, vote->cert->signing_key_digest)) { log_warn(LD_BUG, "We added a cert, but still couldn't find it."); @@ -3019,6 +3021,30 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out) return any_failed ? NULL : pending_vote; } +/* Write the votes in <b>pending_vote_list</b> to disk. */ +static void +write_v3_votes_to_disk(const smartlist_t *pending_vote_list) +{ + smartlist_t *votestrings = smartlist_new(); + char *votefile = NULL; + + SMARTLIST_FOREACH(pending_vote_list, pending_vote_t *, v, + { + sized_chunk_t *c = tor_malloc(sizeof(sized_chunk_t)); + c->bytes = v->vote_body->dir; + c->len = v->vote_body->dir_len; + smartlist_add(votestrings, c); /* collect strings to write to disk */ + }); + + votefile = get_datadir_fname("v3-status-votes"); + write_chunks_to_file(votefile, votestrings, 0, 0); + log_debug(LD_DIR, "Wrote votes to disk (%s)!", votefile); + + tor_free(votefile); + SMARTLIST_FOREACH(votestrings, sized_chunk_t *, c, tor_free(c)); + smartlist_free(votestrings); +} + /** Try to compute a v3 networkstatus consensus from the currently pending * votes. Return 0 on success, -1 on failure. Store the consensus in * pending_consensus: it won't be ready to be published until we have @@ -3028,8 +3054,8 @@ dirvote_compute_consensuses(void) { /* Have we got enough votes to try? */ int n_votes, n_voters, n_vote_running = 0; - smartlist_t *votes = NULL, *votestrings = NULL; - char *consensus_body = NULL, *signatures = NULL, *votefile; + smartlist_t *votes = NULL; + char *consensus_body = NULL, *signatures = NULL; networkstatus_t *consensus = NULL; authority_cert_t *my_cert; pending_consensus_t pending[N_CONSENSUS_FLAVORS]; @@ -3040,6 +3066,17 @@ dirvote_compute_consensuses(void) if (!pending_vote_list) pending_vote_list = smartlist_new(); + /* Write votes to disk */ + write_v3_votes_to_disk(pending_vote_list); + + /* Setup votes smartlist */ + votes = smartlist_new(); + SMARTLIST_FOREACH(pending_vote_list, pending_vote_t *, v, + { + smartlist_add(votes, v->vote); /* collect votes to compute consensus */ + }); + + /* See if consensus managed to achieve majority */ n_voters = get_n_authorities(V3_DIRINFO); n_votes = smartlist_len(pending_vote_list); if (n_votes <= n_voters/2) { @@ -3066,24 +3103,6 @@ dirvote_compute_consensuses(void) goto err; } - votes = smartlist_new(); - votestrings = smartlist_new(); - SMARTLIST_FOREACH(pending_vote_list, pending_vote_t *, v, - { - sized_chunk_t *c = tor_malloc(sizeof(sized_chunk_t)); - c->bytes = v->vote_body->dir; - c->len = v->vote_body->dir_len; - smartlist_add(votestrings, c); /* collect strings to write to disk */ - - smartlist_add(votes, v->vote); /* collect votes to compute consensus */ - }); - - votefile = get_datadir_fname("v3-status-votes"); - write_chunks_to_file(votefile, votestrings, 0, 0); - tor_free(votefile); - SMARTLIST_FOREACH(votestrings, sized_chunk_t *, c, tor_free(c)); - smartlist_free(votestrings); - { char legacy_dbuf[DIGEST_LEN]; crypto_pk_t *legacy_sign=NULL; @@ -3373,7 +3392,7 @@ dirvote_publish_consensus(void) continue; } - if (networkstatus_set_current_consensus(pending->body, name, 0)) + if (networkstatus_set_current_consensus(pending->body, name, 0, NULL)) log_warn(LD_DIR, "Error publishing %s consensus", name); else log_notice(LD_DIR, "Published %s consensus", name); @@ -3515,7 +3534,7 @@ dirvote_create_microdescriptor(const routerinfo_t *ri, int consensus_method) if (consensus_method >= MIN_METHOD_FOR_P6_LINES && ri->ipv6_exit_policy) { - /* XXXX024 This doesn't match proposal 208, which says these should + /* XXXX+++ This doesn't match proposal 208, which says these should * be taken unchanged from the routerinfo. That's bogosity, IMO: * the proposal should have said to do this instead.*/ char *p6 = write_short_policy(ri->ipv6_exit_policy); diff --git a/src/or/dnsserv.c b/src/or/dnsserv.c index 74f17ce78c..edca50f6f9 100644 --- a/src/or/dnsserv.c +++ b/src/or/dnsserv.c @@ -130,7 +130,7 @@ evdns_server_callback(struct evdns_server_request *req, void *data_) tor_addr_copy(&TO_CONN(conn)->addr, &tor_addr); TO_CONN(conn)->port = port; - TO_CONN(conn)->address = tor_dup_addr(&tor_addr); + TO_CONN(conn)->address = tor_addr_to_str_dup(&tor_addr); if (q->type == EVDNS_TYPE_A || q->type == EVDNS_TYPE_AAAA || q->type == EVDNS_QTYPE_ALL) { @@ -205,7 +205,7 @@ dnsserv_launch_request(const char *name, int reverse, tor_addr_copy(&TO_CONN(conn)->addr, &control_conn->base_.addr); #ifdef AF_UNIX /* - * The control connection can be AF_UNIX and if so tor_dup_addr will + * The control connection can be AF_UNIX and if so tor_addr_to_str_dup will * unhelpfully say "<unknown address type>"; say "(Tor_internal)" * instead. */ @@ -214,11 +214,11 @@ dnsserv_launch_request(const char *name, int reverse, TO_CONN(conn)->address = tor_strdup("(Tor_internal)"); } else { TO_CONN(conn)->port = control_conn->base_.port; - TO_CONN(conn)->address = tor_dup_addr(&control_conn->base_.addr); + TO_CONN(conn)->address = tor_addr_to_str_dup(&control_conn->base_.addr); } #else TO_CONN(conn)->port = control_conn->base_.port; - TO_CONN(conn)->address = tor_dup_addr(&control_conn->base_.addr); + TO_CONN(conn)->address = tor_addr_to_str_dup(&control_conn->base_.addr); #endif if (reverse) diff --git a/src/or/entrynodes.c b/src/or/entrynodes.c index 310a948b35..68241af987 100644 --- a/src/or/entrynodes.c +++ b/src/or/entrynodes.c @@ -76,6 +76,14 @@ static const node_t *choose_random_entry_impl(cpath_build_state_t *state, int *n_options_out); static int num_bridges_usable(void); +/* Default number of entry guards in the case where the NumEntryGuards + * consensus parameter is not set */ +#define DEFAULT_N_GUARDS 1 +/* Minimum and maximum number of entry guards (in case the NumEntryGuards + * consensus parameter is set). */ +#define MIN_N_GUARDS 1 +#define MAX_N_GUARDS 10 + /** Return the list of entry guards, creating it if necessary. */ const smartlist_t * get_entry_guards(void) @@ -488,7 +496,8 @@ decide_num_guards(const or_options_t *options, int for_directory) return options->NumEntryGuards; /* Use the value from the consensus, or 3 if no guidance. */ - return networkstatus_get_param(NULL, "NumEntryGuards", 3, 1, 10); + return networkstatus_get_param(NULL, "NumEntryGuards", DEFAULT_N_GUARDS, + MIN_N_GUARDS, MAX_N_GUARDS); } /** If the use of entry guards is configured, choose more entry guards @@ -722,8 +731,9 @@ entry_guards_compute_status(const or_options_t *options, time_t now) * * If <b>mark_relay_status</b>, also call router_set_status() on this * relay. - * - * XXX024 change succeeded and mark_relay_status into 'int flags'. + */ +/* XXX We could change succeeded and mark_relay_status into 'int flags'. + * Too many boolean arguments is a recipe for confusion. */ int entry_guard_register_connect_status(const char *digest, int succeeded, @@ -1243,7 +1253,7 @@ entry_guards_parse_state(or_state_t *state, int set, char **msg) } else { strlcpy(node->nickname, smartlist_get(args,0), MAX_NICKNAME_LEN+1); if (base16_decode(node->identity, DIGEST_LEN, smartlist_get(args,1), - strlen(smartlist_get(args,1)))<0) { + strlen(smartlist_get(args,1))) != DIGEST_LEN) { *msg = tor_strdup("Unable to parse entry nodes: " "Bad hex digest for EntryGuard"); } @@ -1299,8 +1309,9 @@ entry_guards_parse_state(or_state_t *state, int set, char **msg) log_warn(LD_BUG, "EntryGuardAddedBy line is not long enough."); continue; } - if (base16_decode(d, sizeof(d), line->value, HEX_DIGEST_LEN)<0 || - line->value[HEX_DIGEST_LEN] != ' ') { + if (base16_decode(d, sizeof(d), + line->value, HEX_DIGEST_LEN) != sizeof(d) || + line->value[HEX_DIGEST_LEN] != ' ') { log_warn(LD_BUG, "EntryGuardAddedBy line %s does not begin with " "hex digest", escaped(line->value)); continue; @@ -1466,7 +1477,7 @@ entry_guards_parse_state(or_state_t *state, int set, char **msg) } entry_guards = new_entry_guards; entry_guards_dirty = 0; - /* XXX024 hand new_entry_guards to this func, and move it up a + /* XXX hand new_entry_guards to this func, and move it up a * few lines, so we don't have to re-dirty it */ if (remove_obsolete_entry_guards(now)) entry_guards_dirty = 1; @@ -2022,6 +2033,7 @@ bridge_add_from_config(bridge_line_t *bridge_line) if (bridge_line->transport_name) b->transport_name = bridge_line->transport_name; b->fetch_status.schedule = DL_SCHED_BRIDGE; + b->fetch_status.backoff = DL_SCHED_RANDOM_EXPONENTIAL; b->socks_args = bridge_line->socks_args; if (!bridge_list) bridge_list = smartlist_new(); diff --git a/src/or/ext_orport.c b/src/or/ext_orport.c index aa1b3e26fe..8ba3c6afa3 100644 --- a/src/or/ext_orport.c +++ b/src/or/ext_orport.c @@ -461,8 +461,8 @@ connection_ext_or_handle_cmd_useraddr(connection_t *conn, return -1; { /* do some logging */ - char *old_address = tor_dup_addr(&conn->addr); - char *new_address = tor_dup_addr(&addr); + char *old_address = tor_addr_to_str_dup(&conn->addr); + char *new_address = tor_addr_to_str_dup(&addr); log_debug(LD_NET, "Received USERADDR." "We rewrite our address from '%s:%u' to '%s:%u'.", @@ -478,7 +478,7 @@ connection_ext_or_handle_cmd_useraddr(connection_t *conn, if (conn->address) { tor_free(conn->address); } - conn->address = tor_dup_addr(&addr); + conn->address = tor_addr_to_str_dup(&addr); return 0; } diff --git a/src/or/geoip.c b/src/or/geoip.c index b563db0418..874052495e 100644 --- a/src/or/geoip.c +++ b/src/or/geoip.c @@ -80,9 +80,9 @@ geoip_add_entry(const tor_addr_t *low, const tor_addr_t *high, intptr_t idx; void *idxplus1_; - if (tor_addr_family(low) != tor_addr_family(high)) + IF_BUG_ONCE(tor_addr_family(low) != tor_addr_family(high)) return; - if (tor_addr_compare(high, low, CMP_EXACT) < 0) + IF_BUG_ONCE(tor_addr_compare(high, low, CMP_EXACT) < 0) return; idxplus1_ = strmap_get_lc(country_idxplus1_by_lc_code, country); @@ -110,8 +110,8 @@ geoip_add_entry(const tor_addr_t *low, const tor_addr_t *high, smartlist_add(geoip_ipv4_entries, ent); } else if (tor_addr_family(low) == AF_INET6) { geoip_ipv6_entry_t *ent = tor_malloc_zero(sizeof(geoip_ipv6_entry_t)); - ent->ip_low = *tor_addr_to_in6(low); - ent->ip_high = *tor_addr_to_in6(high); + ent->ip_low = *tor_addr_to_in6_assert(low); + ent->ip_high = *tor_addr_to_in6_assert(high); ent->country = idx; smartlist_add(geoip_ipv6_entries, ent); } @@ -504,7 +504,7 @@ clientmap_entries_eq(const clientmap_entry_t *a, const clientmap_entry_t *b) } HT_PROTOTYPE(clientmap, clientmap_entry_t, node, clientmap_entry_hash, - clientmap_entries_eq); + clientmap_entries_eq) HT_GENERATE2(clientmap, clientmap_entry_t, node, clientmap_entry_hash, clientmap_entries_eq, 0.6, tor_reallocarray_, tor_free_) @@ -718,7 +718,7 @@ dirreq_map_ent_hash(const dirreq_map_entry_t *entry) } HT_PROTOTYPE(dirreqmap, dirreq_map_entry_t, node, dirreq_map_ent_hash, - dirreq_map_ent_eq); + dirreq_map_ent_eq) HT_GENERATE2(dirreqmap, dirreq_map_entry_t, node, dirreq_map_ent_hash, dirreq_map_ent_eq, 0.6, tor_reallocarray_, tor_free_) diff --git a/src/or/hibernate.c b/src/or/hibernate.c index 9408925d96..209aae01cf 100644 --- a/src/or/hibernate.c +++ b/src/or/hibernate.c @@ -28,13 +28,12 @@ hibernating, phase 2: #include "config.h" #include "connection.h" #include "connection_edge.h" +#include "control.h" #include "hibernate.h" #include "main.h" #include "router.h" #include "statefile.h" -extern long stats_n_seconds_working; /* published uptime */ - /** Are we currently awake, asleep, running out of bandwidth, or shutting * down? */ static hibernate_state_t hibernate_state = HIBERNATE_STATE_INITIAL; @@ -111,11 +110,34 @@ static int cfg_start_day = 0, cfg_start_min = 0; /** @} */ +static const char *hibernate_state_to_string(hibernate_state_t state); static void reset_accounting(time_t now); static int read_bandwidth_usage(void); static time_t start_of_accounting_period_after(time_t now); static time_t start_of_accounting_period_containing(time_t now); static void accounting_set_wakeup_time(void); +static void on_hibernate_state_change(hibernate_state_t prev_state); + +/** + * Return the human-readable name for the hibernation state <b>state</b> + */ +static const char * +hibernate_state_to_string(hibernate_state_t state) +{ + static char buf[64]; + switch (state) { + case HIBERNATE_STATE_EXITING: return "EXITING"; + case HIBERNATE_STATE_LOWBANDWIDTH: return "SOFT"; + case HIBERNATE_STATE_DORMANT: return "HARD"; + case HIBERNATE_STATE_INITIAL: + case HIBERNATE_STATE_LIVE: + return "AWAKE"; + default: + log_warn(LD_BUG, "unknown hibernate state %d", state); + tor_snprintf(buf, sizeof(buf), "unknown [%d]", state); + return buf; + } +} /* ************ * Functions for bandwidth accounting. @@ -935,6 +957,7 @@ consider_hibernation(time_t now) { int accounting_enabled = get_options()->AccountingMax != 0; char buf[ISO_TIME_LEN+1]; + hibernate_state_t prev_state = hibernate_state; /* If we're in 'exiting' mode, then we just shut down after the interval * elapses. */ @@ -990,6 +1013,10 @@ consider_hibernation(time_t now) hibernate_end_time_elapsed(now); } } + + /* Dispatch a controller event if the hibernation state changed. */ + if (hibernate_state != prev_state) + on_hibernate_state_change(prev_state); } /** Helper function: called when we get a GETINFO request for an @@ -1007,12 +1034,8 @@ getinfo_helper_accounting(control_connection_t *conn, if (!strcmp(question, "accounting/enabled")) { *answer = tor_strdup(accounting_is_enabled(get_options()) ? "1" : "0"); } else if (!strcmp(question, "accounting/hibernating")) { - if (hibernate_state == HIBERNATE_STATE_DORMANT) - *answer = tor_strdup("hard"); - else if (hibernate_state == HIBERNATE_STATE_LOWBANDWIDTH) - *answer = tor_strdup("soft"); - else - *answer = tor_strdup("awake"); + *answer = tor_strdup(hibernate_state_to_string(hibernate_state)); + tor_strlower(*answer); } else if (!strcmp(question, "accounting/bytes")) { tor_asprintf(answer, U64_FORMAT" "U64_FORMAT, U64_PRINTF_ARG(n_bytes_read_in_interval), @@ -1062,6 +1085,20 @@ getinfo_helper_accounting(control_connection_t *conn, return 0; } +/** + * Helper function: called when the hibernation state changes, and sends a + * SERVER_STATUS event to notify interested controllers of the accounting + * state change. + */ +static void +on_hibernate_state_change(hibernate_state_t prev_state) +{ + (void)prev_state; /* Should we do something with this? */ + control_event_server_status(LOG_NOTICE, + "HIBERNATION_STATUS STATUS=%s", + hibernate_state_to_string(hibernate_state)); +} + #ifdef TOR_UNIT_TESTS /** * Manually change the hibernation state. Private; used only by the unit diff --git a/src/or/include.am b/src/or/include.am index 712ae18406..19f1a7fe0a 100644 --- a/src/or/include.am +++ b/src/or/include.am @@ -109,7 +109,7 @@ src_or_libtor_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) src_or_tor_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ @TOR_LDFLAGS_libevent@ -src_or_tor_LDADD = src/or/libtor.a src/common/libor.a \ +src_or_tor_LDADD = src/or/libtor.a src/common/libor.a src/common/libor-ctime.a \ src/common/libor-crypto.a $(LIBKECCAK_TINY) $(LIBDONNA) \ src/common/libor-event.a src/trunnel/libor-trunnel.a \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ \ @@ -121,6 +121,7 @@ src_or_tor_cov_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_or_tor_cov_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) src_or_tor_cov_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ @TOR_LDFLAGS_libevent@ src_or_tor_cov_LDADD = src/or/libtor-testing.a src/common/libor-testing.a \ + src/common/libor-ctime-testing.a \ src/common/libor-crypto-testing.a $(LIBKECCAK_TINY) $(LIBDONNA) \ src/common/libor-event-testing.a src/trunnel/libor-trunnel-testing.a \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ \ diff --git a/src/or/keypin.c b/src/or/keypin.c index 1f82eccf86..749bc6121c 100644 --- a/src/or/keypin.c +++ b/src/or/keypin.c @@ -93,14 +93,14 @@ return (unsigned) siphash24g(a->ed25519_key, sizeof(a->ed25519_key)); } HT_PROTOTYPE(rsamap, keypin_ent_st, rsamap_node, keypin_ent_hash_rsa, - keypin_ents_eq_rsa); + keypin_ents_eq_rsa) HT_GENERATE2(rsamap, keypin_ent_st, rsamap_node, keypin_ent_hash_rsa, - keypin_ents_eq_rsa, 0.6, tor_reallocarray, tor_free_); + keypin_ents_eq_rsa, 0.6, tor_reallocarray, tor_free_) HT_PROTOTYPE(edmap, keypin_ent_st, edmap_node, keypin_ent_hash_ed, - keypin_ents_eq_ed); + keypin_ents_eq_ed) HT_GENERATE2(edmap, keypin_ent_st, edmap_node, keypin_ent_hash_ed, - keypin_ents_eq_ed, 0.6, tor_reallocarray, tor_free_); + keypin_ents_eq_ed, 0.6, tor_reallocarray, tor_free_) /** * Check whether we already have an entry in the key pinning table for a diff --git a/src/or/main.c b/src/or/main.c index f585f0be62..4de2e70a1d 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -962,7 +962,7 @@ conn_close_if_marked(int i) connection_stop_writing(conn); } if (connection_is_reading(conn)) { - /* XXXX024 We should make this code unreachable; if a connection is + /* XXXX+ We should make this code unreachable; if a connection is * marked for close and flushing, there is no point in reading to it * at all. Further, checking at this point is a bit of a hack: it * would make much more sense to react in @@ -1632,8 +1632,8 @@ rotate_x509_certificate_callback(time_t now, const or_options_t *options) * TLS context. */ log_info(LD_GENERAL,"Rotating tls context."); if (router_initialize_tls_context() < 0) { - log_warn(LD_BUG, "Error reinitializing TLS context"); - tor_assert(0); + log_err(LD_BUG, "Error reinitializing TLS context"); + tor_assert_unreached(); } /* We also make sure to rotate the TLS connections themselves if they've @@ -2558,9 +2558,7 @@ run_main_loop_once(void) return -1; #endif } else { - if (ERRNO_IS_EINPROGRESS(e)) - log_warn(LD_BUG, - "libevent call returned EINPROGRESS? Please report."); + tor_assert_nonfatal_once(! ERRNO_IS_EINPROGRESS(e)); log_debug(LD_NET,"libevent call interrupted."); /* You can't trust the results of this poll(). Go back to the * top of the big for loop. */ @@ -2691,9 +2689,6 @@ get_uptime,(void)) return stats_n_seconds_working; } -extern uint64_t rephist_total_alloc; -extern uint32_t rephist_total_num; - /** * Write current memory usage information to the log. */ diff --git a/src/or/main.h b/src/or/main.h index ad865b8124..31a22de424 100644 --- a/src/or/main.h +++ b/src/or/main.h @@ -75,6 +75,14 @@ int tor_main(int argc, char *argv[]); int do_main_loop(void); int tor_init(int argc, char **argv); +extern time_t time_of_process_start; +extern long stats_n_seconds_working; +extern int quiet_level; +extern int global_read_bucket; +extern int global_write_bucket; +extern int global_relayed_read_bucket; +extern int global_relayed_write_bucket; + #ifdef MAIN_PRIVATE STATIC void init_connection_lists(void); STATIC void close_closeable_connections(void); diff --git a/src/or/microdesc.c b/src/or/microdesc.c index 5b5c29a6d2..130259a29f 100644 --- a/src/or/microdesc.c +++ b/src/or/microdesc.c @@ -69,7 +69,7 @@ microdesc_eq_(microdesc_t *a, microdesc_t *b) } HT_PROTOTYPE(microdesc_map, microdesc_t, node, - microdesc_hash_, microdesc_eq_); + microdesc_hash_, microdesc_eq_) HT_GENERATE2(microdesc_map, microdesc_t, node, microdesc_hash_, microdesc_eq_, 0.6, tor_reallocarray_, tor_free_) @@ -925,7 +925,7 @@ we_use_microdescriptors_for_circuits(const or_options_t *options) return 0; /* Otherwise, we decide that we'll use microdescriptors iff we are * not a server, and we're not autofetching everything. */ - /* XXX023 what does not being a server have to do with it? also there's + /* XXXX++ what does not being a server have to do with it? also there's * a partitioning issue here where bridges differ from clients. */ ret = !server_mode(options) && !options->FetchUselessDescriptors; } diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c index 51fc01108f..3471288969 100644 --- a/src/or/networkstatus.c +++ b/src/or/networkstatus.c @@ -86,9 +86,9 @@ static time_t time_to_download_next_consensus[N_CONSENSUS_FLAVORS]; static download_status_t consensus_dl_status[N_CONSENSUS_FLAVORS] = { { 0, 0, 0, DL_SCHED_CONSENSUS, DL_WANT_ANY_DIRSERVER, - DL_SCHED_INCREMENT_FAILURE }, + DL_SCHED_INCREMENT_FAILURE, DL_SCHED_RANDOM_EXPONENTIAL, 0, 0 }, { 0, 0, 0, DL_SCHED_CONSENSUS, DL_WANT_ANY_DIRSERVER, - DL_SCHED_INCREMENT_FAILURE }, + DL_SCHED_INCREMENT_FAILURE, DL_SCHED_RANDOM_EXPONENTIAL, 0, 0 }, }; #define N_CONSENSUS_BOOTSTRAP_SCHEDULES 2 @@ -105,10 +105,10 @@ static download_status_t consensus_bootstrap_dl_status[N_CONSENSUS_BOOTSTRAP_SCHEDULES] = { { 0, 0, 0, DL_SCHED_CONSENSUS, DL_WANT_AUTHORITY, - DL_SCHED_INCREMENT_ATTEMPT }, + DL_SCHED_INCREMENT_ATTEMPT, DL_SCHED_RANDOM_EXPONENTIAL, 0, 0 }, /* During bootstrap, DL_WANT_ANY_DIRSERVER means "use fallbacks". */ { 0, 0, 0, DL_SCHED_CONSENSUS, DL_WANT_ANY_DIRSERVER, - DL_SCHED_INCREMENT_ATTEMPT }, + DL_SCHED_INCREMENT_ATTEMPT, DL_SCHED_RANDOM_EXPONENTIAL, 0, 0 }, }; /** True iff we have logged a warning about this OR's version being older than @@ -173,7 +173,7 @@ router_reload_consensus_networkstatus(void) } s = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL); if (s) { - if (networkstatus_set_current_consensus(s, flavor, flags) < -1) { + if (networkstatus_set_current_consensus(s, flavor, flags, NULL) < -1) { log_warn(LD_FS, "Couldn't load consensus %s networkstatus from \"%s\"", flavor, filename); } @@ -191,7 +191,8 @@ router_reload_consensus_networkstatus(void) s = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL); if (s) { if (networkstatus_set_current_consensus(s, flavor, - flags|NSSET_WAS_WAITING_FOR_CERTS)) { + flags|NSSET_WAS_WAITING_FOR_CERTS, + NULL)) { log_info(LD_FS, "Couldn't load consensus %s networkstatus from \"%s\"", flavor, filename); } @@ -1160,13 +1161,13 @@ update_certificate_downloads(time_t now) for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) { if (consensus_waiting_for_certs[i].consensus) authority_certs_fetch_missing(consensus_waiting_for_certs[i].consensus, - now); + now, NULL); } if (current_ns_consensus) - authority_certs_fetch_missing(current_ns_consensus, now); + authority_certs_fetch_missing(current_ns_consensus, now, NULL); if (current_md_consensus) - authority_certs_fetch_missing(current_md_consensus, now); + authority_certs_fetch_missing(current_md_consensus, now, NULL); } /** Return 1 if we have a consensus but we don't have enough certificates @@ -1460,6 +1461,10 @@ networkstatus_set_current_consensus_from_ns(networkstatus_t *c, * If flags & NSSET_ACCEPT_OBSOLETE, then we should be willing to take this * consensus, even if it comes from many days in the past. * + * If source_dir is non-NULL, it's the identity digest for a directory that + * we've just successfully retrieved a consensus or certificates from, so try + * it first to fetch any missing certificates. + * * Return 0 on success, <0 on failure. On failure, caller should increment * the failure count as appropriate. * @@ -1469,7 +1474,8 @@ networkstatus_set_current_consensus_from_ns(networkstatus_t *c, int networkstatus_set_current_consensus(const char *consensus, const char *flavor, - unsigned flags) + unsigned flags, + const char *source_dir) { networkstatus_t *c=NULL; int r, result = -1; @@ -1591,7 +1597,7 @@ networkstatus_set_current_consensus(const char *consensus, write_str_to_file(unverified_fname, consensus, 0); } if (dl_certs) - authority_certs_fetch_missing(c, now); + authority_certs_fetch_missing(c, now, source_dir); /* This case is not a success or a failure until we get the certs * or fail to get the certs. */ result = 0; @@ -1629,7 +1635,7 @@ networkstatus_set_current_consensus(const char *consensus, /* Are we missing any certificates at all? */ if (r != 1 && dl_certs) - authority_certs_fetch_missing(c, now); + authority_certs_fetch_missing(c, now, source_dir); if (flav == usable_consensus_flavor()) { notify_control_networkstatus_changed(current_consensus, c); @@ -1703,7 +1709,7 @@ networkstatus_set_current_consensus(const char *consensus, channel_set_cmux_policy_everywhere(NULL); } - /* XXXX024 this call might be unnecessary here: can changing the + /* XXXX this call might be unnecessary here: can changing the * current consensus really alter our view of any OR's rate limits? */ connection_or_update_token_buckets(get_connection_array(), options); @@ -1752,9 +1758,14 @@ networkstatus_set_current_consensus(const char *consensus, } /** Called when we have gotten more certificates: see whether we can - * now verify a pending consensus. */ + * now verify a pending consensus. + * + * If source_dir is non-NULL, it's the identity digest for a directory that + * we've just successfully retrieved certificates from, so try it first to + * fetch any missing certificates. + */ void -networkstatus_note_certs_arrived(void) +networkstatus_note_certs_arrived(const char *source_dir) { int i; for (i=0; i<N_CONSENSUS_FLAVORS; ++i) { @@ -1766,7 +1777,8 @@ networkstatus_note_certs_arrived(void) if (!networkstatus_set_current_consensus( waiting_body, networkstatus_get_flavor_name(i), - NSSET_WAS_WAITING_FOR_CERTS)) { + NSSET_WAS_WAITING_FOR_CERTS, + source_dir)) { tor_free(waiting_body); } } @@ -2204,7 +2216,7 @@ getinfo_helper_networkstatus(control_connection_t *conn, if (*q == '$') ++q; - if (base16_decode(d, DIGEST_LEN, q, strlen(q))) { + if (base16_decode(d, DIGEST_LEN, q, strlen(q)) != DIGEST_LEN) { *errmsg = "Data not decodeable as hex"; return -1; } diff --git a/src/or/networkstatus.h b/src/or/networkstatus.h index ac93e5de91..aee6641c6e 100644 --- a/src/or/networkstatus.h +++ b/src/or/networkstatus.h @@ -84,8 +84,9 @@ int networkstatus_consensus_is_already_downloading(const char *resource); #define NSSET_REQUIRE_FLAVOR 16 int networkstatus_set_current_consensus(const char *consensus, const char *flavor, - unsigned flags); -void networkstatus_note_certs_arrived(void); + unsigned flags, + const char *source_dir); +void networkstatus_note_certs_arrived(const char *source_dir); void routers_update_all_from_networkstatus(time_t now, int dir_version); void routers_update_status_from_consensus_networkstatus(smartlist_t *routers, int reset_failures); diff --git a/src/or/nodelist.c b/src/or/nodelist.c index 89b5355c8d..a49bf03f61 100644 --- a/src/or/nodelist.c +++ b/src/or/nodelist.c @@ -77,7 +77,7 @@ node_id_eq(const node_t *node1, const node_t *node2) return tor_memeq(node1->identity, node2->identity, DIGEST_LEN); } -HT_PROTOTYPE(nodelist_map, node_t, ht_ent, node_id_hash, node_id_eq); +HT_PROTOTYPE(nodelist_map, node_t, ht_ent, node_id_hash, node_id_eq) HT_GENERATE2(nodelist_map, node_t, ht_ent, node_id_hash, node_id_eq, 0.6, tor_reallocarray_, tor_free_) diff --git a/src/or/onion.c b/src/or/onion.c index d6ef3673dd..7c7f97fc42 100644 --- a/src/or/onion.c +++ b/src/or/onion.c @@ -38,9 +38,9 @@ typedef struct onion_queue_t { /** Array of queues of circuits waiting for CPU workers. An element is NULL * if that queue is empty.*/ -TOR_TAILQ_HEAD(onion_queue_head_t, onion_queue_t) - ol_list[MAX_ONION_HANDSHAKE_TYPE+1] = { - TOR_TAILQ_HEAD_INITIALIZER(ol_list[0]), /* tap */ +static TOR_TAILQ_HEAD(onion_queue_head_t, onion_queue_t) + ol_list[MAX_ONION_HANDSHAKE_TYPE+1] = +{ TOR_TAILQ_HEAD_INITIALIZER(ol_list[0]), /* tap */ TOR_TAILQ_HEAD_INITIALIZER(ol_list[1]), /* fast */ TOR_TAILQ_HEAD_INITIALIZER(ol_list[2]), /* ntor */ }; @@ -51,7 +51,7 @@ static int ol_entries[MAX_ONION_HANDSHAKE_TYPE+1]; static int num_ntors_per_tap(void); static void onion_queue_entry_remove(onion_queue_t *victim); -/* XXXX024 Check lengths vs MAX_ONIONSKIN_{CHALLENGE,REPLY}_LEN. +/* XXXX Check lengths vs MAX_ONIONSKIN_{CHALLENGE,REPLY}_LEN. * * (By which I think I meant, "make sure that no * X_ONIONSKIN_CHALLENGE/REPLY_LEN is greater than @@ -527,7 +527,7 @@ onion_skin_server_handshake(int type, * <b>rend_authenticator_out</b> to the "KH" field that can be used to * establish introduction points at this hop, and return 0. On failure, * return -1, and set *msg_out to an error message if this is worth - * complaining to the usre about. */ + * complaining to the user about. */ int onion_skin_client_handshake(int type, const onion_handshake_state_t *handshake_state, diff --git a/src/or/onion_ntor.c b/src/or/onion_ntor.c index 9f97a4cfbe..33afc27895 100644 --- a/src/or/onion_ntor.c +++ b/src/or/onion_ntor.c @@ -47,7 +47,7 @@ typedef struct tweakset_t { } tweakset_t; /** The tweaks to be used with our handshake. */ -const tweakset_t proto1_tweaks = { +static const tweakset_t proto1_tweaks = { #define PROTOID "ntor-curve25519-sha256-1" #define PROTOID_LEN 24 PROTOID ":mac", diff --git a/src/or/or.h b/src/or/or.h index 2252f38161..ea38022f43 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -14,14 +14,6 @@ #include "orconfig.h" -#if defined(__clang_analyzer__) || defined(__COVERITY__) -/* If we're building for a static analysis, turn on all the off-by-default - * features. */ -#ifndef INSTRUMENT_DOWNLOADS -#define INSTRUMENT_DOWNLOADS 1 -#endif -#endif - #ifdef HAVE_UNISTD_H #include <unistd.h> #endif @@ -784,7 +776,7 @@ typedef enum rend_auth_type_t { /** Client-side configuration of authorization for a hidden service. */ typedef struct rend_service_authorization_t { - char descriptor_cookie[REND_DESC_COOKIE_LEN]; + uint8_t descriptor_cookie[REND_DESC_COOKIE_LEN]; char onion_address[REND_SERVICE_ADDRESS_LEN+1]; rend_auth_type_t auth_type; } rend_service_authorization_t; @@ -1294,21 +1286,26 @@ typedef struct connection_t { time_t timestamp_created; /**< When was this connection_t created? */ - /* XXXX_IP6 make this IPv6-capable */ int socket_family; /**< Address family of this connection's socket. Usually - * AF_INET, but it can also be AF_UNIX, or in the future - * AF_INET6 */ - tor_addr_t addr; /**< IP of the other side of the connection; used to - * identify routers, along with port. */ - uint16_t port; /**< If non-zero, port on the other end - * of the connection. */ + * AF_INET, but it can also be AF_UNIX, or AF_INET6 */ + tor_addr_t addr; /**< IP that socket "s" is directly connected to; + * may be the IP address for a proxy or pluggable transport, + * see "address" for the address of the final destination. + */ + uint16_t port; /**< If non-zero, port that socket "s" is directly connected + * to; may be the port for a proxy or pluggable transport, + * see "address" for the port at the final destination. */ uint16_t marked_for_close; /**< Should we close this conn on the next * iteration of the main loop? (If true, holds * the line number where this connection was * marked.) */ const char *marked_for_close_file; /**< For debugging: in which file were * we marked for close? */ - char *address; /**< FQDN (or IP) of the other end. + char *address; /**< FQDN (or IP) and port of the final destination for this + * connection; this is always the remote address, it is + * passed to a proxy or pluggable transport if one in use. + * See "addr" and "port" for the address that socket "s" is + * directly connected to. * strdup into this, because free_connection() frees it. */ /** Another connection that's connected to this one in lieu of a socket. */ struct connection_t *linked_conn; @@ -1990,6 +1987,15 @@ typedef enum { #define download_schedule_increment_bitfield_t \ ENUM_BF(download_schedule_increment_t) +/** Enumeration: do we want to use the random exponential backoff + * mechanism? */ +typedef enum { + DL_SCHED_DETERMINISTIC = 0, + DL_SCHED_RANDOM_EXPONENTIAL = 1, +} download_schedule_backoff_t; +#define download_schedule_backoff_bitfield_t \ + ENUM_BF(download_schedule_backoff_t) + /** Information about our plans for retrying downloads for a downloadable * directory object. * Each type of downloadable directory object has a corresponding retry @@ -2036,6 +2042,15 @@ typedef struct download_status_t { download_schedule_increment_bitfield_t increment_on : 1; /**< does this * schedule increment on each attempt, * or after each failure? */ + download_schedule_backoff_bitfield_t backoff : 1; /**< do we use the + * deterministic schedule, or random + * exponential backoffs? */ + uint8_t last_backoff_position; /**< number of attempts/failures, depending + * on increment_on, when we last recalculated + * the delay. Only updated if backoff + * == 1. */ + int last_delay_used; /**< last delay used for random exponential backoff; + * only updated if backoff == 1 */ } download_status_t; /** If n_download_failures is this high, the download can never happen. */ @@ -2958,17 +2973,17 @@ typedef struct circuit_t { /** When the circuit was first used, or 0 if the circuit is clean. * - * XXXX023 Note that some code will artifically adjust this value backward + * XXXX Note that some code will artifically adjust this value backward * in time in order to indicate that a circuit shouldn't be used for new * streams, but that it can stay alive as long as it has streams on it. * That's a kludge we should fix. * - * XXX023 The CBT code uses this field to record when HS-related + * XXX The CBT code uses this field to record when HS-related * circuits entered certain states. This usage probably won't * interfere with this field's primary purpose, but we should * document it more thoroughly to make sure of that. * - * XXX027 The SocksPort option KeepaliveIsolateSOCKSAuth will artificially + * XXX The SocksPort option KeepaliveIsolateSOCKSAuth will artificially * adjust this value forward each time a suitable stream is attached to an * already constructed circuit, potentially keeping the circuit alive * indefinitely. @@ -5032,7 +5047,7 @@ typedef enum { /** Hidden-service side configuration of client authorization. */ typedef struct rend_authorized_client_t { char *client_name; - char descriptor_cookie[REND_DESC_COOKIE_LEN]; + uint8_t descriptor_cookie[REND_DESC_COOKIE_LEN]; crypto_pk_t *client_key; } rend_authorized_client_t; @@ -5060,12 +5075,12 @@ typedef struct rend_encoded_v2_service_descriptor_t { * INTRO_POINT_LIFETIME_INTRODUCTIONS INTRODUCE2 cells, it may expire * sooner.) * - * XXX023 Should this be configurable? */ + * XXX Should this be configurable? */ #define INTRO_POINT_LIFETIME_MIN_SECONDS (18*60*60) /** The maximum number of seconds that an introduction point will last * before expiring due to old age. * - * XXX023 Should this be configurable? */ + * XXX Should this be configurable? */ #define INTRO_POINT_LIFETIME_MAX_SECONDS (24*60*60) /** The maximum number of circuit creation retry we do to an intro point diff --git a/src/or/policies.c b/src/or/policies.c index f9718b6a95..2703d7edef 100644 --- a/src/or/policies.c +++ b/src/or/policies.c @@ -103,7 +103,7 @@ policy_expand_private(smartlist_t **policy) if (tor_addr_parse_mask_ports(private_nets[i], 0, &newpolicy.addr, &newpolicy.maskbits, &port_min, &port_max)<0) { - tor_assert(0); + tor_assert_unreached(); } smartlist_add(tmp, addr_policy_get_canonical_entry(&newpolicy)); } diff --git a/src/or/rendcache.c b/src/or/rendcache.c index f8206cd53b..e61a96b677 100644 --- a/src/or/rendcache.c +++ b/src/or/rendcache.c @@ -956,25 +956,25 @@ rend_cache_store_v2_desc_as_client(const char *desc, * avoid an evil HSDir serving old descriptor. We validate if the * timestamp is greater than and not equal because it's a rounded down * timestamp to the hour so if the descriptor changed in the same hour, - * the rend cache failure will tells us if we have a new descriptor. */ + * the rend cache failure will tell us if we have a new descriptor. */ if (e && e->parsed->timestamp > parsed->timestamp) { log_info(LD_REND, "We already have a new enough service descriptor for " "service ID %s with the same desc ID and version.", safe_str_client(service_id)); goto okay; } - /* Lookup our failure cache for intro point that might be unsuable. */ + /* Lookup our failure cache for intro point that might be unusable. */ validate_intro_point_failure(parsed, service_id); - /* It's now possible that our intro point list is empty, this means that + /* It's now possible that our intro point list is empty, which means that * this descriptor is useless to us because intro points have all failed * somehow before. Discard the descriptor. */ if (smartlist_len(parsed->intro_nodes) == 0) { - log_info(LD_REND, "Service descriptor with service ID %s, every " - "intro points are unusable. Discarding it.", + log_info(LD_REND, "Service descriptor with service ID %s has no " + "usable intro points. Discarding it.", safe_str_client(service_id)); goto err; } - /* Now either purge the current one and replace it's content or create a + /* Now either purge the current one and replace its content or create a * new one and add it to the rend cache. */ if (!e) { e = tor_malloc_zero(sizeof(rend_cache_entry_t)); diff --git a/src/or/rendcache.h b/src/or/rendcache.h index 0e8b918753..270b614c38 100644 --- a/src/or/rendcache.h +++ b/src/or/rendcache.h @@ -102,6 +102,13 @@ STATIC void validate_intro_point_failure(const rend_service_descriptor_t *desc, const char *service_id); STATIC void rend_cache_failure_entry_free_(void *entry); + +#ifdef TOR_UNIT_TESTS +extern strmap_t *rend_cache; +extern strmap_t *rend_cache_failure; +extern digestmap_t *rend_cache_v2_dir; +extern size_t rend_cache_total_allocation; +#endif #endif #endif /* TOR_RENDCACHE_H */ diff --git a/src/or/rendclient.c b/src/or/rendclient.c index 609c45c71d..64d367354b 100644 --- a/src/or/rendclient.c +++ b/src/or/rendclient.c @@ -510,7 +510,7 @@ lookup_last_hid_serv_request(routerstatus_t *hs_dir, tor_snprintf(hsdir_desc_comb_id, sizeof(hsdir_desc_comb_id), "%s%s", hsdir_id_base32, desc_id_base32); - /* XXX023 tor_assert(strlen(hsdir_desc_comb_id) == + /* XXX++?? tor_assert(strlen(hsdir_desc_comb_id) == LAST_HID_SERV_REQUEST_KEY_LEN); */ if (set) { time_t *oldptr; @@ -572,7 +572,7 @@ purge_hid_serv_from_last_hid_serv_requests(const char *desc_id) const char *key; void *val; strmap_iter_get(iter, &key, &val); - /* XXX023 tor_assert(strlen(key) == LAST_HID_SERV_REQUEST_KEY_LEN); */ + /* XXX++?? tor_assert(strlen(key) == LAST_HID_SERV_REQUEST_KEY_LEN); */ if (tor_memeq(key + LAST_HID_SERV_REQUEST_KEY_LEN - REND_DESC_ID_V2_LEN_BASE32, desc_id_base32, @@ -895,12 +895,6 @@ rend_client_refetch_v2_renddesc(rend_data_t *rend_query) rend_cache_entry_t *e = NULL; tor_assert(rend_query); - /* Are we configured to fetch descriptors? */ - if (!get_options()->FetchHidServDescriptors) { - log_warn(LD_REND, "We received an onion address for a v2 rendezvous " - "service descriptor, but are not fetching service descriptors."); - return; - } /* Before fetching, check if we already have a usable descriptor here. */ if (rend_cache_lookup_entry(rend_query->onion_address, -1, &e) == 0 && rend_client_any_intro_points_usable(e)) { @@ -908,6 +902,12 @@ rend_client_refetch_v2_renddesc(rend_data_t *rend_query) "already have a usable descriptor here. Not fetching."); return; } + /* Are we configured to fetch descriptors? */ + if (!get_options()->FetchHidServDescriptors) { + log_warn(LD_REND, "We received an onion address for a v2 rendezvous " + "service descriptor, but are not fetching service descriptors."); + return; + } log_debug(LD_REND, "Fetching v2 rendezvous descriptor for service %s", safe_str_client(rend_query->onion_address)); @@ -1099,7 +1099,7 @@ rend_client_rendezvous_acked(origin_circuit_t *circ, const uint8_t *request, * service and never reply to the client's rend requests */ pathbias_mark_use_success(circ); - /* XXXX This is a pretty brute-force approach. It'd be better to + /* XXXX++ This is a pretty brute-force approach. It'd be better to * attach only the connections that are waiting on this circuit, rather * than trying to attach them all. See comments bug 743. */ /* If we already have the introduction circuit built, make sure we send @@ -1466,12 +1466,10 @@ rend_parse_service_authorization(const or_options_t *options, strmap_t *parsed = strmap_new(); smartlist_t *sl = smartlist_new(); rend_service_authorization_t *auth = NULL; - char descriptor_cookie_tmp[REND_DESC_COOKIE_LEN+2]; - char descriptor_cookie_base64ext[REND_DESC_COOKIE_LEN_BASE64+2+1]; + char *err_msg = NULL; for (line = options->HidServAuth; line; line = line->next) { char *onion_address, *descriptor_cookie; - int auth_type_val = 0; auth = NULL; SMARTLIST_FOREACH(sl, char *, c, tor_free(c);); smartlist_clear(sl); @@ -1500,31 +1498,13 @@ rend_parse_service_authorization(const or_options_t *options, } /* Parse descriptor cookie. */ descriptor_cookie = smartlist_get(sl, 1); - if (strlen(descriptor_cookie) != REND_DESC_COOKIE_LEN_BASE64) { - log_warn(LD_CONFIG, "Authorization cookie has wrong length: '%s'", - descriptor_cookie); - goto err; - } - /* Add trailing zero bytes (AA) to make base64-decoding happy. */ - tor_snprintf(descriptor_cookie_base64ext, - REND_DESC_COOKIE_LEN_BASE64+2+1, - "%sAA", descriptor_cookie); - if (base64_decode(descriptor_cookie_tmp, sizeof(descriptor_cookie_tmp), - descriptor_cookie_base64ext, - strlen(descriptor_cookie_base64ext)) < 0) { - log_warn(LD_CONFIG, "Decoding authorization cookie failed: '%s'", - descriptor_cookie); - goto err; - } - auth_type_val = (((uint8_t)descriptor_cookie_tmp[16]) >> 4) + 1; - if (auth_type_val < 1 || auth_type_val > 2) { - log_warn(LD_CONFIG, "Authorization cookie has unknown authorization " - "type encoded."); + if (rend_auth_decode_cookie(descriptor_cookie, auth->descriptor_cookie, + &auth->auth_type, &err_msg) < 0) { + tor_assert(err_msg); + log_warn(LD_CONFIG, "%s", err_msg); + tor_free(err_msg); goto err; } - auth->auth_type = auth_type_val == 1 ? REND_BASIC_AUTH : REND_STEALTH_AUTH; - memcpy(auth->descriptor_cookie, descriptor_cookie_tmp, - REND_DESC_COOKIE_LEN); if (strmap_get(parsed, auth->onion_address)) { log_warn(LD_CONFIG, "Duplicate authorization for the same hidden " "service."); @@ -1547,8 +1527,6 @@ rend_parse_service_authorization(const or_options_t *options, } else { strmap_free(parsed, rend_service_authorization_strmap_item_free); } - memwipe(descriptor_cookie_tmp, 0, sizeof(descriptor_cookie_tmp)); - memwipe(descriptor_cookie_base64ext, 0, sizeof(descriptor_cookie_base64ext)); return res; } diff --git a/src/or/rendcommon.c b/src/or/rendcommon.c index 438fbc4d9a..01b0766cf0 100644 --- a/src/or/rendcommon.c +++ b/src/or/rendcommon.c @@ -211,7 +211,7 @@ rend_encode_v2_intro_points(char **encoded, rend_service_descriptor_t *desc) goto done; } /* Assemble everything for this introduction point. */ - address = tor_dup_addr(&info->addr); + address = tor_addr_to_str_dup(&info->addr); res = tor_snprintf(unenc + unenc_written, unenc_len - unenc_written, "introduction-point %s\n" "ip-address %s\n" @@ -720,6 +720,22 @@ rend_valid_descriptor_id(const char *query) return 0; } +/** Return true iff <b>client_name</b> is a syntactically valid name + * for rendezvous client authentication. */ +int +rend_valid_client_name(const char *client_name) +{ + size_t len = strlen(client_name); + if (len < 1 || len > REND_CLIENTNAME_MAX_LEN) { + return 0; + } + if (strspn(client_name, REND_LEGAL_CLIENTNAME_CHARACTERS) != len) { + return 0; + } + + return 1; +} + /** Called when we get a rendezvous-related relay cell on circuit * <b>circ</b>. Dispatch on rendezvous relay command. */ void @@ -941,3 +957,113 @@ hid_serv_get_responsible_directories(smartlist_t *responsible_dirs, return smartlist_len(responsible_dirs) ? 0 : -1; } +/* Length of the 'extended' auth cookie used to encode auth type before + * base64 encoding. */ +#define REND_DESC_COOKIE_LEN_EXT (REND_DESC_COOKIE_LEN + 1) +/* Length of the zero-padded auth cookie when base64 encoded. These two + * padding bytes always (A=) are stripped off of the returned cookie. */ +#define REND_DESC_COOKIE_LEN_EXT_BASE64 (REND_DESC_COOKIE_LEN_BASE64 + 2) + +/** Encode a client authorization descriptor cookie. + * The result of this function is suitable for use in the HidServAuth + * option. The trailing padding characters are removed, and the + * auth type is encoded into the cookie. + * + * Returns a new base64-encoded cookie. This function cannot fail. + * The caller is responsible for freeing the returned value. + */ +char * +rend_auth_encode_cookie(const uint8_t *cookie_in, rend_auth_type_t auth_type) +{ + uint8_t extended_cookie[REND_DESC_COOKIE_LEN_EXT]; + char *cookie_out = tor_malloc_zero(REND_DESC_COOKIE_LEN_EXT_BASE64 + 1); + int re; + + tor_assert(cookie_in); + + memcpy(extended_cookie, cookie_in, REND_DESC_COOKIE_LEN); + extended_cookie[REND_DESC_COOKIE_LEN] = ((int)auth_type - 1) << 4; + re = base64_encode(cookie_out, REND_DESC_COOKIE_LEN_EXT_BASE64 + 1, + (const char *) extended_cookie, REND_DESC_COOKIE_LEN_EXT, + 0); + tor_assert(re == REND_DESC_COOKIE_LEN_EXT_BASE64); + + /* Remove the trailing 'A='. Auth type is encoded in the high bits + * of the last byte, so the last base64 character will always be zero + * (A). This is subtly different behavior from base64_encode_nopad. */ + cookie_out[REND_DESC_COOKIE_LEN_BASE64] = '\0'; + memwipe(extended_cookie, 0, sizeof(extended_cookie)); + return cookie_out; +} + +/** Decode a base64-encoded client authorization descriptor cookie. + * The descriptor_cookie can be truncated to REND_DESC_COOKIE_LEN_BASE64 + * characters (as given to clients), or may include the two padding + * characters (as stored by the service). + * + * The result is stored in REND_DESC_COOKIE_LEN bytes of cookie_out. + * The rend_auth_type_t decoded from the cookie is stored in the + * optional auth_type_out parameter. + * + * Return 0 on success, or -1 on error. The caller is responsible for + * freeing the returned err_msg. + */ +int +rend_auth_decode_cookie(const char *cookie_in, uint8_t *cookie_out, + rend_auth_type_t *auth_type_out, char **err_msg_out) +{ + uint8_t descriptor_cookie_decoded[REND_DESC_COOKIE_LEN_EXT + 1] = { 0 }; + char descriptor_cookie_base64ext[REND_DESC_COOKIE_LEN_EXT_BASE64 + 1]; + const char *descriptor_cookie = cookie_in; + char *err_msg = NULL; + int auth_type_val = 0; + int res = -1; + int decoded_len; + + size_t len = strlen(descriptor_cookie); + if (len == REND_DESC_COOKIE_LEN_BASE64) { + /* Add a trailing zero byte to make base64-decoding happy. */ + tor_snprintf(descriptor_cookie_base64ext, + sizeof(descriptor_cookie_base64ext), + "%sA=", descriptor_cookie); + descriptor_cookie = descriptor_cookie_base64ext; + } else if (len != REND_DESC_COOKIE_LEN_EXT_BASE64) { + tor_asprintf(&err_msg, "Authorization cookie has wrong length: %s", + escaped(cookie_in)); + goto err; + } + + decoded_len = base64_decode((char *) descriptor_cookie_decoded, + sizeof(descriptor_cookie_decoded), + descriptor_cookie, + REND_DESC_COOKIE_LEN_EXT_BASE64); + if (decoded_len != REND_DESC_COOKIE_LEN && + decoded_len != REND_DESC_COOKIE_LEN_EXT) { + tor_asprintf(&err_msg, "Authorization cookie has invalid characters: %s", + escaped(cookie_in)); + goto err; + } + + if (auth_type_out) { + auth_type_val = (descriptor_cookie_decoded[REND_DESC_COOKIE_LEN] >> 4) + 1; + if (auth_type_val < 1 || auth_type_val > 2) { + tor_asprintf(&err_msg, "Authorization cookie type is unknown: %s", + escaped(cookie_in)); + goto err; + } + *auth_type_out = auth_type_val == 1 ? REND_BASIC_AUTH : REND_STEALTH_AUTH; + } + + memcpy(cookie_out, descriptor_cookie_decoded, REND_DESC_COOKIE_LEN); + res = 0; + err: + if (err_msg_out) { + *err_msg_out = err_msg; + } else { + tor_free(err_msg); + } + memwipe(descriptor_cookie_decoded, 0, sizeof(descriptor_cookie_decoded)); + memwipe(descriptor_cookie_base64ext, 0, sizeof(descriptor_cookie_base64ext)); + return res; +} + diff --git a/src/or/rendcommon.h b/src/or/rendcommon.h index d67552e405..88cf512f4a 100644 --- a/src/or/rendcommon.h +++ b/src/or/rendcommon.h @@ -45,6 +45,7 @@ void rend_intro_point_free(rend_intro_point_t *intro); int rend_valid_service_id(const char *query); int rend_valid_descriptor_id(const char *query); +int rend_valid_client_name(const char *client_name); int rend_encode_v2_descriptors(smartlist_t *descs_out, rend_service_descriptor_t *desc, time_t now, uint8_t period, rend_auth_type_t auth_type, @@ -68,5 +69,13 @@ rend_data_t *rend_data_service_create(const char *onion_address, const char *pk_digest, const uint8_t *cookie, rend_auth_type_t auth_type); + +char *rend_auth_encode_cookie(const uint8_t *cookie_in, + rend_auth_type_t auth_type); +int rend_auth_decode_cookie(const char *cookie_in, + uint8_t *cookie_out, + rend_auth_type_t *auth_type_out, + char **err_msg_out); + #endif diff --git a/src/or/rendmid.c b/src/or/rendmid.c index a33ad92966..ca0ad7b0d4 100644 --- a/src/or/rendmid.c +++ b/src/or/rendmid.c @@ -309,7 +309,7 @@ rend_mid_rendezvous(or_circuit_t *circ, const uint8_t *request, goto err; } - if (request_len != REND_COOKIE_LEN+DH_KEY_LEN+DIGEST_LEN) { + if (request_len < REND_COOKIE_LEN) { log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "Rejecting RENDEZVOUS1 cell with bad length (%d) on circuit %u.", (int)request_len, (unsigned)circ->p_circ_id); diff --git a/src/or/rendservice.c b/src/or/rendservice.c index b81a01c568..c50de83f7e 100644 --- a/src/or/rendservice.c +++ b/src/or/rendservice.c @@ -183,14 +183,15 @@ num_rend_services(void) } /** Helper: free storage held by a single service authorized client entry. */ -static void +void rend_authorized_client_free(rend_authorized_client_t *client) { if (!client) return; if (client->client_key) crypto_pk_free(client->client_key); - memwipe(client->client_name, 0, strlen(client->client_name)); + if (client->client_name) + memwipe(client->client_name, 0, strlen(client->client_name)); tor_free(client->client_name); memwipe(client->descriptor_cookie, 0, sizeof(client->descriptor_cookie)); tor_free(client); @@ -671,27 +672,17 @@ rend_config_services(const or_options_t *options, int validate_only) SMARTLIST_FOREACH_BEGIN(clients, const char *, client_name) { rend_authorized_client_t *client; - size_t len = strlen(client_name); - if (len < 1 || len > REND_CLIENTNAME_MAX_LEN) { + if (!rend_valid_client_name(client_name)) { log_warn(LD_CONFIG, "HiddenServiceAuthorizeClient contains an " - "illegal client name: '%s'. Length must be " - "between 1 and %d characters.", + "illegal client name: '%s'. Names must be " + "between 1 and %d characters and contain " + "only [A-Za-z0-9+_-].", client_name, REND_CLIENTNAME_MAX_LEN); SMARTLIST_FOREACH(clients, char *, cp, tor_free(cp)); smartlist_free(clients); rend_service_free(service); return -1; } - if (strspn(client_name, REND_LEGAL_CLIENTNAME_CHARACTERS) != len) { - log_warn(LD_CONFIG, "HiddenServiceAuthorizeClient contains an " - "illegal client name: '%s'. Valid " - "characters are [A-Za-z0-9+_-].", - client_name); - SMARTLIST_FOREACH(clients, char *, cp, tor_free(cp)); - smartlist_free(clients); - rend_service_free(service); - return -1; - } client = tor_malloc_zero(sizeof(rend_authorized_client_t)); client->client_name = tor_strdup(client_name); smartlist_add(service->clients, client); @@ -827,14 +818,17 @@ rend_config_services(const or_options_t *options, int validate_only) return 0; } -/** Add the ephemeral service <b>pk</b>/<b>ports</b> if possible, with +/** Add the ephemeral service <b>pk</b>/<b>ports</b> if possible, using + * client authorization <b>auth_type</b> and an optional list of + * rend_authorized_client_t in <b>auth_clients</b>, with * <b>max_streams_per_circuit</b> streams allowed per rendezvous circuit, * and circuit closure on max streams being exceeded set by * <b>max_streams_close_circuit</b>. * - * Regardless of sucess/failure, callers should not touch pk/ports after - * calling this routine, and may assume that correct cleanup has been done - * on failure. + * Ownership of pk, ports, and auth_clients is passed to this routine. + * Regardless of success/failure, callers should not touch these values + * after calling this routine, and may assume that correct cleanup has + * been done on failure. * * Return an appropriate rend_service_add_ephemeral_status_t. */ @@ -843,6 +837,8 @@ rend_service_add_ephemeral(crypto_pk_t *pk, smartlist_t *ports, int max_streams_per_circuit, int max_streams_close_circuit, + rend_auth_type_t auth_type, + smartlist_t *auth_clients, char **service_id_out) { *service_id_out = NULL; @@ -852,7 +848,8 @@ rend_service_add_ephemeral(crypto_pk_t *pk, rend_service_t *s = tor_malloc_zero(sizeof(rend_service_t)); s->directory = NULL; /* This indicates the service is ephemeral. */ s->private_key = pk; - s->auth_type = REND_NO_AUTH; + s->auth_type = auth_type; + s->clients = auth_clients; s->ports = ports; s->intro_period_started = time(NULL); s->n_intro_points_wanted = NUM_INTRO_POINTS_DEFAULT; @@ -868,6 +865,12 @@ rend_service_add_ephemeral(crypto_pk_t *pk, rend_service_free(s); return RSAE_BADVIRTPORT; } + if (s->auth_type != REND_NO_AUTH && + (!s->clients || smartlist_len(s->clients) == 0)) { + log_warn(LD_CONFIG, "At least one authorized client must be specified."); + rend_service_free(s); + return RSAE_BADAUTH; + } /* Enforcing pk/id uniqueness should be done by rend_service_load_keys(), but * it's not, see #14828. @@ -923,7 +926,6 @@ rend_service_del_ephemeral(const char *service_id) */ SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) { if (!circ->marked_for_close && - circ->state == CIRCUIT_STATE_OPEN && (circ->purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO || circ->purpose == CIRCUIT_PURPOSE_S_INTRO)) { origin_circuit_t *oc = TO_ORIGIN_CIRCUIT(circ); @@ -1156,7 +1158,6 @@ rend_service_load_auth_keys(rend_service_t *s, const char *hfname) strmap_t *parsed_clients = strmap_new(); FILE *cfile, *hfile; open_file_t *open_cfile = NULL, *open_hfile = NULL; - char extended_desc_cookie[REND_DESC_COOKIE_LEN+1]; char desc_cook_out[3*REND_DESC_COOKIE_LEN_BASE64+1]; char service_id[16+1]; char buf[1500]; @@ -1208,10 +1209,12 @@ rend_service_load_auth_keys(rend_service_t *s, const char *hfname) memcpy(client->descriptor_cookie, parsed->descriptor_cookie, REND_DESC_COOKIE_LEN); } else { - crypto_rand(client->descriptor_cookie, REND_DESC_COOKIE_LEN); + crypto_rand((char *) client->descriptor_cookie, REND_DESC_COOKIE_LEN); } + /* For compatibility with older tor clients, this does not + * truncate the padding characters, unlike rend_auth_encode_cookie. */ if (base64_encode(desc_cook_out, 3*REND_DESC_COOKIE_LEN_BASE64+1, - client->descriptor_cookie, + (char *) client->descriptor_cookie, REND_DESC_COOKIE_LEN, 0) < 0) { log_warn(LD_BUG, "Could not base64-encode descriptor cookie."); goto err; @@ -1272,6 +1275,8 @@ rend_service_load_auth_keys(rend_service_t *s, const char *hfname) log_warn(LD_BUG, "Could not write client entry."); goto err; } + } else { + strlcpy(service_id, s->service_id, sizeof(service_id)); } if (fputs(buf, cfile) < 0) { @@ -1280,27 +1285,18 @@ rend_service_load_auth_keys(rend_service_t *s, const char *hfname) goto err; } - /* Add line to hostname file. */ - if (s->auth_type == REND_BASIC_AUTH) { - /* Remove == signs (newline has been removed above). */ - desc_cook_out[strlen(desc_cook_out)-2] = '\0'; - tor_snprintf(buf, sizeof(buf),"%s.onion %s # client: %s\n", - s->service_id, desc_cook_out, client->client_name); - } else { - memcpy(extended_desc_cookie, client->descriptor_cookie, - REND_DESC_COOKIE_LEN); - extended_desc_cookie[REND_DESC_COOKIE_LEN] = - ((int)s->auth_type - 1) << 4; - if (base64_encode(desc_cook_out, 3*REND_DESC_COOKIE_LEN_BASE64+1, - extended_desc_cookie, - REND_DESC_COOKIE_LEN+1, 0) < 0) { - log_warn(LD_BUG, "Could not base64-encode descriptor cookie."); - goto err; - } - desc_cook_out[strlen(desc_cook_out)-2] = '\0'; /* Remove A=. */ - tor_snprintf(buf, sizeof(buf),"%s.onion %s # client: %s\n", - service_id, desc_cook_out, client->client_name); + /* Add line to hostname file. This is not the same encoding as in + * client_keys. */ + char *encoded_cookie = rend_auth_encode_cookie(client->descriptor_cookie, + s->auth_type); + if (!encoded_cookie) { + log_warn(LD_BUG, "Could not base64-encode descriptor cookie."); + goto err; } + tor_snprintf(buf, sizeof(buf), "%s.onion %s # client: %s\n", + service_id, encoded_cookie, client->client_name); + memwipe(encoded_cookie, 0, strlen(encoded_cookie)); + tor_free(encoded_cookie); if (fputs(buf, hfile)<0) { log_warn(LD_FS, "Could not append host entry to file: %s", @@ -1332,7 +1328,6 @@ rend_service_load_auth_keys(rend_service_t *s, const char *hfname) memwipe(buf, 0, sizeof(buf)); memwipe(desc_cook_out, 0, sizeof(desc_cook_out)); memwipe(service_id, 0, sizeof(service_id)); - memwipe(extended_desc_cookie, 0, sizeof(extended_desc_cookie)); return r; } diff --git a/src/or/rendservice.h b/src/or/rendservice.h index 101b37e18d..4966cb0302 100644 --- a/src/or/rendservice.h +++ b/src/or/rendservice.h @@ -106,8 +106,11 @@ rend_service_port_config_t *rend_service_parse_port_config(const char *string, char **err_msg_out); void rend_service_port_config_free(rend_service_port_config_t *p); +void rend_authorized_client_free(rend_authorized_client_t *client); + /** Return value from rend_service_add_ephemeral. */ typedef enum { + RSAE_BADAUTH = -5, /**< Invalid auth_type/auth_clients */ RSAE_BADVIRTPORT = -4, /**< Invalid VIRTPORT/TARGET(s) */ RSAE_ADDREXISTS = -3, /**< Onion address collision */ RSAE_BADPRIVKEY = -2, /**< Invalid public key */ @@ -118,6 +121,8 @@ rend_service_add_ephemeral_status_t rend_service_add_ephemeral(crypto_pk_t *pk, smartlist_t *ports, int max_streams_per_circuit, int max_streams_close_circuit, + rend_auth_type_t auth_type, + smartlist_t *auth_clients, char **service_id_out); int rend_service_del_ephemeral(const char *service_id); diff --git a/src/or/rephist.c b/src/or/rephist.c index 04ed7aef0f..8992571c52 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -604,7 +604,7 @@ rep_hist_get_weighted_time_known(const char *id, time_t when) int rep_hist_have_measured_enough_stability(void) { - /* XXXX023 This doesn't do so well when we change our opinion + /* XXXX++ This doesn't do so well when we change our opinion * as to whether we're tracking router stability. */ return started_tracking_stability < time(NULL) - 4*60*60; } @@ -1074,7 +1074,8 @@ rep_hist_load_mtbf_data(time_t now) if (mtbf_idx > i) i = mtbf_idx; } - if (base16_decode(digest, DIGEST_LEN, hexbuf, HEX_DIGEST_LEN) < 0) { + if (base16_decode(digest, DIGEST_LEN, + hexbuf, HEX_DIGEST_LEN) != DIGEST_LEN) { log_warn(LD_HIST, "Couldn't hex string %s", escaped(hexbuf)); continue; } @@ -2738,7 +2739,7 @@ bidi_map_ent_hash(const bidi_map_entry_t *entry) } HT_PROTOTYPE(bidimap, bidi_map_entry_t, node, bidi_map_ent_hash, - bidi_map_ent_eq); + bidi_map_ent_eq) HT_GENERATE2(bidimap, bidi_map_entry_t, node, bidi_map_ent_hash, bidi_map_ent_eq, 0.6, tor_reallocarray_, tor_free_) @@ -2933,7 +2934,7 @@ static time_t start_of_hs_stats_interval; * information needed. */ typedef struct hs_stats_t { /** How many relay cells have we seen as rendezvous points? */ - int64_t rp_relay_cells_seen; + uint64_t rp_relay_cells_seen; /** Set of unique public key digests we've seen this stat period * (could also be implemented as sorted smartlist). */ @@ -3074,16 +3075,20 @@ rep_hist_format_hs_stats(time_t now) int64_t obfuscated_cells_seen; int64_t obfuscated_onions_seen; - obfuscated_cells_seen = round_int64_to_next_multiple_of( - hs_stats->rp_relay_cells_seen, - REND_CELLS_BIN_SIZE); - obfuscated_cells_seen = add_laplace_noise(obfuscated_cells_seen, + uint64_t rounded_cells_seen + = round_uint64_to_next_multiple_of(hs_stats->rp_relay_cells_seen, + REND_CELLS_BIN_SIZE); + rounded_cells_seen = MIN(rounded_cells_seen, INT64_MAX); + obfuscated_cells_seen = add_laplace_noise((int64_t)rounded_cells_seen, crypto_rand_double(), REND_CELLS_DELTA_F, REND_CELLS_EPSILON); - obfuscated_onions_seen = round_int64_to_next_multiple_of(digestmap_size( - hs_stats->onions_seen_this_period), - ONIONS_SEEN_BIN_SIZE); - obfuscated_onions_seen = add_laplace_noise(obfuscated_onions_seen, + + uint64_t rounded_onions_seen = + round_uint64_to_next_multiple_of((size_t)digestmap_size( + hs_stats->onions_seen_this_period), + ONIONS_SEEN_BIN_SIZE); + rounded_onions_seen = MIN(rounded_onions_seen, INT64_MAX); + obfuscated_onions_seen = add_laplace_noise((int64_t)rounded_onions_seen, crypto_rand_double(), ONIONS_SEEN_DELTA_F, ONIONS_SEEN_EPSILON); @@ -3217,7 +3222,7 @@ rep_hist_free_all(void) rep_hist_desc_stats_term(); total_descriptor_downloads = 0; - tor_assert(rephist_total_alloc == 0); - tor_assert(rephist_total_num == 0); + tor_assert_nonfatal(rephist_total_alloc == 0); + tor_assert_nonfatal_once(rephist_total_num == 0); } diff --git a/src/or/rephist.h b/src/or/rephist.h index 145da97d02..ff4810a56d 100644 --- a/src/or/rephist.h +++ b/src/or/rephist.h @@ -112,5 +112,12 @@ void rep_hist_note_negotiated_link_proto(unsigned link_proto, int started_here); void rep_hist_log_link_protocol_counts(void); +extern uint64_t rephist_total_alloc; +extern uint32_t rephist_total_num; +#ifdef TOR_UNIT_TESTS +extern int onion_handshakes_requested[MAX_ONION_HANDSHAKE_TYPE+1]; +extern int onion_handshakes_assigned[MAX_ONION_HANDSHAKE_TYPE+1]; +#endif + #endif diff --git a/src/or/router.c b/src/or/router.c index 01316c1bc2..a671591ad7 100644 --- a/src/or/router.c +++ b/src/or/router.c @@ -40,8 +40,6 @@ * and uploading server descriptors, retrying OR connections. **/ -extern long stats_n_seconds_working; - /************************************************************/ /***** @@ -1054,7 +1052,8 @@ init_keys(void) log_info(LD_DIR, "adding my own v3 cert"); if (trusted_dirs_load_certs_from_string( cert->cache_info.signed_descriptor_body, - TRUSTED_DIRS_CERTS_SRC_SELF, 0)<0) { + TRUSTED_DIRS_CERTS_SRC_SELF, 0, + NULL)<0) { log_warn(LD_DIR, "Unable to parse my own v3 cert! Failing."); return -1; } @@ -1537,7 +1536,7 @@ MOCK_IMPL(int, server_mode,(const or_options_t *options)) { if (options->ClientOnly) return 0; - /* XXXX024 I believe we can kill off ORListenAddress here.*/ + /* XXXX I believe we can kill off ORListenAddress here.*/ return (options->ORPort_set || options->ORListenAddress); } diff --git a/src/or/routerlist.c b/src/or/routerlist.c index 49d76807b7..5925d97ad5 100644 --- a/src/or/routerlist.c +++ b/src/or/routerlist.c @@ -67,7 +67,7 @@ typedef struct cert_list_t cert_list_t; /* static function prototypes */ static int compute_weighted_bandwidths(const smartlist_t *sl, bandwidth_weight_rule_t rule, - u64_dbl_t **bandwidths_out); + double **bandwidths_out); static const routerstatus_t *router_pick_trusteddirserver_impl( const smartlist_t *sourcelist, dirinfo_type_t auth, int flags, int *n_busy_out); @@ -159,6 +159,9 @@ download_status_cert_init(download_status_t *dlstatus) dlstatus->schedule = DL_SCHED_CONSENSUS; dlstatus->want_authority = DL_WANT_ANY_DIRSERVER; dlstatus->increment_on = DL_SCHED_INCREMENT_FAILURE; + dlstatus->backoff = DL_SCHED_RANDOM_EXPONENTIAL; + dlstatus->last_backoff_position = 0; + dlstatus->last_delay_used = 0; /* Use the new schedule to set next_attempt_at */ download_status_reset(dlstatus); @@ -287,7 +290,7 @@ trusted_dirs_reload_certs(void) return 0; r = trusted_dirs_load_certs_from_string( contents, - TRUSTED_DIRS_CERTS_SRC_FROM_STORE, 1); + TRUSTED_DIRS_CERTS_SRC_FROM_STORE, 1, NULL); tor_free(contents); return r; } @@ -317,16 +320,20 @@ already_have_cert(authority_cert_t *cert) * or TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_SK_DIGEST. If <b>flush</b> is true, we * need to flush any changed certificates to disk now. Return 0 on success, * -1 if any certs fail to parse. + * + * If source_dir is non-NULL, it's the identity digest for a directory that + * we've just successfully retrieved certificates from, so try it first to + * fetch any missing certificates. */ - int trusted_dirs_load_certs_from_string(const char *contents, int source, - int flush) + int flush, const char *source_dir) { dir_server_t *ds; const char *s, *eos; int failure_code = 0; int from_store = (source == TRUSTED_DIRS_CERTS_SRC_FROM_STORE); + int added_trusted_cert = 0; for (s = contents; *s; s = eos) { authority_cert_t *cert = authority_cert_parse_from_string(s, &eos); @@ -386,6 +393,7 @@ trusted_dirs_load_certs_from_string(const char *contents, int source, } if (ds) { + added_trusted_cert = 1; log_info(LD_DIR, "Adding %s certificate for directory authority %s with " "signing key %s", from_store ? "cached" : "downloaded", ds->nickname, hex_str(cert->signing_key_digest,DIGEST_LEN)); @@ -430,8 +438,15 @@ trusted_dirs_load_certs_from_string(const char *contents, int source, trusted_dirs_flush_certs_to_disk(); /* call this even if failure_code is <0, since some certs might have - * succeeded. */ - networkstatus_note_certs_arrived(); + * succeeded, but only pass source_dir if there were no failures, + * and at least one more authority certificate was added to the store. + * This avoids retrying a directory that's serving bad or entirely duplicate + * certificates. */ + if (failure_code == 0 && added_trusted_cert) { + networkstatus_note_certs_arrived(source_dir); + } else { + networkstatus_note_certs_arrived(NULL); + } return failure_code; } @@ -718,9 +733,14 @@ authority_cert_dl_looks_uncertain(const char *id_digest) * <b>status</b>. Additionally, try to have a non-expired certificate for * every V3 authority in trusted_dir_servers. Don't fetch certificates we * already have. + * + * If dir_hint is non-NULL, it's the identity digest for a directory that + * we've just successfully retrieved a consensus or certificates from, so try + * it first to fetch any missing certificates. **/ void -authority_certs_fetch_missing(networkstatus_t *status, time_t now) +authority_certs_fetch_missing(networkstatus_t *status, time_t now, + const char *dir_hint) { /* * The pending_id digestmap tracks pending certificate downloads by @@ -884,6 +904,37 @@ authority_certs_fetch_missing(networkstatus_t *status, time_t now) } SMARTLIST_FOREACH_END(voter); } + /* Look up the routerstatus for the dir_hint */ + const routerstatus_t *rs = NULL; + + /* If we still need certificates, try the directory that just successfully + * served us a consensus or certificates. + * As soon as the directory fails to provide additional certificates, we try + * another, randomly selected directory. This avoids continual retries. + * (We only ever have one outstanding request per certificate.) + * + * Bridge clients won't find their bridges using this hint, so they will + * fall back to using directory_get_from_dirserver, which selects a bridge. + */ + if (dir_hint) { + /* First try the consensus routerstatus, then the fallback + * routerstatus */ + rs = router_get_consensus_status_by_id(dir_hint); + if (!rs) { + /* This will also find authorities */ + const dir_server_t *ds = router_get_fallback_dirserver_by_digest( + dir_hint); + if (ds) { + rs = &ds->fake_status; + } + } + + if (!rs) { + log_warn(LD_BUG, "Directory %s delivered a consensus, but a " + "routerstatus could not be found for it.", dir_hint); + } + } + /* Do downloads by identity digest */ if (smartlist_len(missing_id_digests) > 0) { int need_plus = 0; @@ -913,11 +964,25 @@ authority_certs_fetch_missing(networkstatus_t *status, time_t now) if (smartlist_len(fps) > 1) { resource = smartlist_join_strings(fps, "", 0, NULL); - /* We want certs from mirrors, because they will almost always succeed. - */ - directory_get_from_dirserver(DIR_PURPOSE_FETCH_CERTIFICATE, 0, - resource, PDS_RETRY_IF_NO_SERVERS, - DL_WANT_ANY_DIRSERVER); + + /* If we've just downloaded a consensus from a directory, re-use that + * directory */ + if (rs) { + /* Certificate fetches are one-hop, unless AllDirActionsPrivate is 1 */ + int get_via_tor = get_options()->AllDirActionsPrivate; + const dir_indirection_t indirection = get_via_tor ? DIRIND_ANONYMOUS + : DIRIND_ONEHOP; + directory_initiate_command_routerstatus(rs, + DIR_PURPOSE_FETCH_CERTIFICATE, + 0, indirection, resource, NULL, + 0, 0); + } else { + /* Otherwise, we want certs from a random fallback or directory + * mirror, because they will almost always succeed. */ + directory_get_from_dirserver(DIR_PURPOSE_FETCH_CERTIFICATE, 0, + resource, PDS_RETRY_IF_NO_SERVERS, + DL_WANT_ANY_DIRSERVER); + } tor_free(resource); } /* else we didn't add any: they were all pending */ @@ -960,11 +1025,25 @@ authority_certs_fetch_missing(networkstatus_t *status, time_t now) if (smartlist_len(fp_pairs) > 1) { resource = smartlist_join_strings(fp_pairs, "", 0, NULL); - /* We want certs from mirrors, because they will almost always succeed. - */ - directory_get_from_dirserver(DIR_PURPOSE_FETCH_CERTIFICATE, 0, - resource, PDS_RETRY_IF_NO_SERVERS, - DL_WANT_ANY_DIRSERVER); + + /* If we've just downloaded a consensus from a directory, re-use that + * directory */ + if (rs) { + /* Certificate fetches are one-hop, unless AllDirActionsPrivate is 1 */ + int get_via_tor = get_options()->AllDirActionsPrivate; + const dir_indirection_t indirection = get_via_tor ? DIRIND_ANONYMOUS + : DIRIND_ONEHOP; + directory_initiate_command_routerstatus(rs, + DIR_PURPOSE_FETCH_CERTIFICATE, + 0, indirection, resource, NULL, + 0, 0); + } else { + /* Otherwise, we want certs from a random fallback or directory + * mirror, because they will almost always succeed. */ + directory_get_from_dirserver(DIR_PURPOSE_FETCH_CERTIFICATE, 0, + resource, PDS_RETRY_IF_NO_SERVERS, + DL_WANT_ANY_DIRSERVER); + } tor_free(resource); } /* else they were all pending */ @@ -1817,20 +1896,23 @@ dirserver_choose_by_weight(const smartlist_t *servers, double authority_weight) { int n = smartlist_len(servers); int i; - u64_dbl_t *weights; + double *weights_dbl; + uint64_t *weights_u64; const dir_server_t *ds; - weights = tor_calloc(n, sizeof(u64_dbl_t)); + weights_dbl = tor_calloc(n, sizeof(double)); + weights_u64 = tor_calloc(n, sizeof(uint64_t)); for (i = 0; i < n; ++i) { ds = smartlist_get(servers, i); - weights[i].dbl = ds->weight; + weights_dbl[i] = ds->weight; if (ds->is_authority) - weights[i].dbl *= authority_weight; + weights_dbl[i] *= authority_weight; } - scale_array_elements_to_u64(weights, n, NULL); - i = choose_array_element_by_weight(weights, n); - tor_free(weights); + scale_array_elements_to_u64(weights_u64, weights_dbl, n, NULL); + i = choose_array_element_by_weight(weights_u64, n); + tor_free(weights_dbl); + tor_free(weights_u64); return (i < 0) ? NULL : smartlist_get(servers, i); } @@ -2092,59 +2174,43 @@ router_get_advertised_bandwidth_capped(const routerinfo_t *router) * much of the range of uint64_t. If <b>total_out</b> is provided, set it to * the sum of all elements in the array _before_ scaling. */ STATIC void -scale_array_elements_to_u64(u64_dbl_t *entries, int n_entries, +scale_array_elements_to_u64(uint64_t *entries_out, const double *entries_in, + int n_entries, uint64_t *total_out) { double total = 0.0; double scale_factor = 0.0; int i; - /* big, but far away from overflowing an int64_t */ -#define SCALE_TO_U64_MAX ((int64_t) (INT64_MAX / 4)) for (i = 0; i < n_entries; ++i) - total += entries[i].dbl; + total += entries_in[i]; - if (total > 0.0) - scale_factor = SCALE_TO_U64_MAX / total; + if (total > 0.0) { + scale_factor = ((double)INT64_MAX) / total; + scale_factor /= 4.0; /* make sure we're very far away from overflowing */ + } for (i = 0; i < n_entries; ++i) - entries[i].u64 = tor_llround(entries[i].dbl * scale_factor); + entries_out[i] = tor_llround(entries_in[i] * scale_factor); if (total_out) *total_out = (uint64_t) total; - -#undef SCALE_TO_U64_MAX } -/** Time-invariant 64-bit greater-than; works on two integers in the range - * (0,INT64_MAX). */ -#if SIZEOF_VOID_P == 8 -#define gt_i64_timei(a,b) ((a) > (b)) -#else -static inline int -gt_i64_timei(uint64_t a, uint64_t b) -{ - int64_t diff = (int64_t) (b - a); - int res = diff >> 63; - return res & 1; -} -#endif - /** Pick a random element of <b>n_entries</b>-element array <b>entries</b>, * choosing each element with a probability proportional to its (uint64_t) * value, and return the index of that element. If all elements are 0, choose * an index at random. Return -1 on error. */ STATIC int -choose_array_element_by_weight(const u64_dbl_t *entries, int n_entries) +choose_array_element_by_weight(const uint64_t *entries, int n_entries) { - int i, i_chosen=-1, n_chosen=0; - uint64_t total_so_far = 0; + int i; uint64_t rand_val; uint64_t total = 0; for (i = 0; i < n_entries; ++i) - total += entries[i].u64; + total += entries[i]; if (n_entries < 1) return -1; @@ -2156,22 +2222,8 @@ choose_array_element_by_weight(const u64_dbl_t *entries, int n_entries) rand_val = crypto_rand_uint64(total); - for (i = 0; i < n_entries; ++i) { - total_so_far += entries[i].u64; - if (gt_i64_timei(total_so_far, rand_val)) { - i_chosen = i; - n_chosen++; - /* Set rand_val to INT64_MAX rather than stopping the loop. This way, - * the time we spend in the loop does not leak which element we chose. */ - rand_val = INT64_MAX; - } - } - tor_assert(total_so_far == total); - tor_assert(n_chosen == 1); - tor_assert(i_chosen >= 0); - tor_assert(i_chosen < n_entries); - - return i_chosen; + return select_array_member_cumulative_timei( + entries, n_entries, total, rand_val); } /** When weighting bridges, enforce these values as lower and upper @@ -2223,17 +2275,21 @@ static const node_t * smartlist_choose_node_by_bandwidth_weights(const smartlist_t *sl, bandwidth_weight_rule_t rule) { - u64_dbl_t *bandwidths=NULL; + double *bandwidths_dbl=NULL; + uint64_t *bandwidths_u64=NULL; - if (compute_weighted_bandwidths(sl, rule, &bandwidths) < 0) + if (compute_weighted_bandwidths(sl, rule, &bandwidths_dbl) < 0) return NULL; - scale_array_elements_to_u64(bandwidths, smartlist_len(sl), NULL); + bandwidths_u64 = tor_calloc(smartlist_len(sl), sizeof(uint64_t)); + scale_array_elements_to_u64(bandwidths_u64, bandwidths_dbl, + smartlist_len(sl), NULL); { - int idx = choose_array_element_by_weight(bandwidths, + int idx = choose_array_element_by_weight(bandwidths_u64, smartlist_len(sl)); - tor_free(bandwidths); + tor_free(bandwidths_dbl); + tor_free(bandwidths_u64); return idx < 0 ? NULL : smartlist_get(sl, idx); } } @@ -2246,14 +2302,14 @@ smartlist_choose_node_by_bandwidth_weights(const smartlist_t *sl, static int compute_weighted_bandwidths(const smartlist_t *sl, bandwidth_weight_rule_t rule, - u64_dbl_t **bandwidths_out) + double **bandwidths_out) { int64_t weight_scale; double Wg = -1, Wm = -1, We = -1, Wd = -1; double Wgb = -1, Wmb = -1, Web = -1, Wdb = -1; uint64_t weighted_bw = 0; guardfraction_bandwidth_t guardfraction_bw; - u64_dbl_t *bandwidths; + double *bandwidths; /* Can't choose exit and guard at same time */ tor_assert(rule == NO_WEIGHTING || @@ -2335,7 +2391,7 @@ compute_weighted_bandwidths(const smartlist_t *sl, Web /= weight_scale; Wdb /= weight_scale; - bandwidths = tor_calloc(smartlist_len(sl), sizeof(u64_dbl_t)); + bandwidths = tor_calloc(smartlist_len(sl), sizeof(double)); // Cycle through smartlist and total the bandwidth. static int warned_missing_bw = 0; @@ -2422,7 +2478,7 @@ compute_weighted_bandwidths(const smartlist_t *sl, final_weight = weight*this_bw; } - bandwidths[node_sl_idx].dbl = final_weight + 0.5; + bandwidths[node_sl_idx] = final_weight + 0.5; } SMARTLIST_FOREACH_END(node); log_debug(LD_CIRC, "Generated weighted bandwidths for rule %s based " @@ -2443,7 +2499,7 @@ double frac_nodes_with_descriptors(const smartlist_t *sl, bandwidth_weight_rule_t rule) { - u64_dbl_t *bandwidths = NULL; + double *bandwidths = NULL; double total, present; if (smartlist_len(sl) == 0) @@ -2460,7 +2516,7 @@ frac_nodes_with_descriptors(const smartlist_t *sl, total = present = 0.0; SMARTLIST_FOREACH_BEGIN(sl, const node_t *, node) { - const double bw = bandwidths[node_sl_idx].dbl; + const double bw = bandwidths[node_sl_idx]; total += bw; if (node_has_descriptor(node)) present += bw; @@ -2634,7 +2690,8 @@ hex_digest_nickname_decode(const char *hexdigest, return -1; } - if (base16_decode(digest_out, DIGEST_LEN, hexdigest, HEX_DIGEST_LEN)<0) + if (base16_decode(digest_out, DIGEST_LEN, + hexdigest, HEX_DIGEST_LEN) != DIGEST_LEN) return -1; return 0; } @@ -2719,7 +2776,7 @@ hexdigest_to_digest(const char *hexdigest, char *digest) if (hexdigest[0]=='$') ++hexdigest; if (strlen(hexdigest) < HEX_DIGEST_LEN || - base16_decode(digest,DIGEST_LEN,hexdigest,HEX_DIGEST_LEN) < 0) + base16_decode(digest,DIGEST_LEN,hexdigest,HEX_DIGEST_LEN) != DIGEST_LEN) return -1; return 0; } @@ -3704,7 +3761,7 @@ router_add_extrainfo_to_routerlist(extrainfo_t *ei, const char **msg, was_router_added_t inserted; (void)from_fetch; if (msg) *msg = NULL; - /*XXXX023 Do something with msg */ + /*XXXX Do something with msg */ inserted = extrainfo_insert(router_get_routerlist(), ei, !from_cache); @@ -4307,7 +4364,7 @@ dir_server_new(int is_authority, return NULL; if (!hostname) - hostname_ = tor_dup_addr(addr); + hostname_ = tor_addr_to_str_dup(addr); else hostname_ = tor_strdup(hostname); @@ -4917,7 +4974,7 @@ update_consensus_router_descriptor_downloads(time_t now, int is_vote, /** How often should we launch a server/authority request to be sure of getting * a guess for our IP? */ -/*XXXX024 this info should come from netinfo cells or something, or we should +/*XXXX+ this info should come from netinfo cells or something, or we should * do this only when we aren't seeing incoming data. see bug 652. */ #define DUMMY_DOWNLOAD_INTERVAL (20*60) @@ -4928,7 +4985,7 @@ launch_dummy_descriptor_download_as_needed(time_t now, const or_options_t *options) { static time_t last_dummy_download = 0; - /* XXXX024 we could be smarter here; see notes on bug 652. */ + /* XXXX+ we could be smarter here; see notes on bug 652. */ /* If we're a server that doesn't have a configured address, we rely on * directory fetches to learn when our address changes. So if we haven't * tried to get any routerdescs in a long time, try a dummy fetch now. */ diff --git a/src/or/routerlist.h b/src/or/routerlist.h index 67cc253c5a..be242d6e87 100644 --- a/src/or/routerlist.h +++ b/src/or/routerlist.h @@ -29,7 +29,7 @@ int trusted_dirs_reload_certs(void); #define TRUSTED_DIRS_CERTS_SRC_FROM_VOTE 4 int trusted_dirs_load_certs_from_string(const char *contents, int source, - int flush); + int flush, const char *source_dir); void trusted_dirs_flush_certs_to_disk(void); authority_cert_t *authority_cert_get_newest_by_id(const char *id_digest); authority_cert_t *authority_cert_get_by_sk_digest(const char *sk_digest); @@ -38,7 +38,8 @@ authority_cert_t *authority_cert_get_by_digests(const char *id_digest, void authority_cert_get_all(smartlist_t *certs_out); void authority_cert_dl_failed(const char *id_digest, const char *signing_key_digest, int status); -void authority_certs_fetch_missing(networkstatus_t *status, time_t now); +void authority_certs_fetch_missing(networkstatus_t *status, time_t now, + const char *dir_hint); int router_reload_router_list(void); int authority_cert_dl_looks_uncertain(const char *id_digest); const smartlist_t *router_get_trusted_dir_servers(void); @@ -217,17 +218,11 @@ int hex_digest_nickname_matches(const char *hexdigest, const char *nickname, int is_named); #ifdef ROUTERLIST_PRIVATE -/** Helper type for choosing routers by bandwidth: contains a union of - * double and uint64_t. Before we call scale_array_elements_to_u64, it holds - * a double; after, it holds a uint64_t. */ -typedef union u64_dbl_t { - uint64_t u64; - double dbl; -} u64_dbl_t; - -STATIC int choose_array_element_by_weight(const u64_dbl_t *entries, +STATIC int choose_array_element_by_weight(const uint64_t *entries, int n_entries); -STATIC void scale_array_elements_to_u64(u64_dbl_t *entries, int n_entries, +STATIC void scale_array_elements_to_u64(uint64_t *entries_out, + const double *entries_in, + int n_entries, uint64_t *total_out); STATIC const routerstatus_t *router_pick_directory_server_impl( dirinfo_type_t auth, int flags, diff --git a/src/or/routerparse.c b/src/or/routerparse.c index 91025c1568..6550fa6715 100644 --- a/src/or/routerparse.c +++ b/src/or/routerparse.c @@ -1513,7 +1513,8 @@ router_parse_entry_from_string(const char *s, const char *end, char d[DIGEST_LEN]; tor_assert(tok->n_args == 1); tor_strstrip(tok->args[0], " "); - if (base16_decode(d, DIGEST_LEN, tok->args[0], strlen(tok->args[0]))) { + if (base16_decode(d, DIGEST_LEN, + tok->args[0], strlen(tok->args[0])) != DIGEST_LEN) { log_warn(LD_DIR, "Couldn't decode router fingerprint %s", escaped(tok->args[0])); goto err; @@ -1594,8 +1595,10 @@ router_parse_entry_from_string(const char *s, const char *end, if ((tok = find_opt_by_keyword(tokens, K_EXTRA_INFO_DIGEST))) { tor_assert(tok->n_args >= 1); if (strlen(tok->args[0]) == HEX_DIGEST_LEN) { - base16_decode(router->cache_info.extra_info_digest, - DIGEST_LEN, tok->args[0], HEX_DIGEST_LEN); + if (base16_decode(router->cache_info.extra_info_digest, DIGEST_LEN, + tok->args[0], HEX_DIGEST_LEN) != DIGEST_LEN) { + log_warn(LD_DIR,"Invalid extra info digest"); + } } else { log_warn(LD_DIR, "Invalid extra info digest %s", escaped(tok->args[0])); } @@ -1738,7 +1741,7 @@ extrainfo_parse_entry_from_string(const char *s, const char *end, strlcpy(extrainfo->nickname, tok->args[0], sizeof(extrainfo->nickname)); if (strlen(tok->args[1]) != HEX_DIGEST_LEN || base16_decode(extrainfo->cache_info.identity_digest, DIGEST_LEN, - tok->args[1], HEX_DIGEST_LEN)) { + tok->args[1], HEX_DIGEST_LEN) != DIGEST_LEN) { log_warn(LD_DIR,"Invalid fingerprint %s on \"extra-info\"", escaped(tok->args[1])); goto err; @@ -1960,7 +1963,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) tok = find_by_keyword(tokens, K_FINGERPRINT); tor_assert(tok->n_args); if (base16_decode(fp_declared, DIGEST_LEN, tok->args[0], - strlen(tok->args[0]))) { + strlen(tok->args[0])) != DIGEST_LEN) { log_warn(LD_DIR, "Couldn't decode key certificate fingerprint %s", escaped(tok->args[0])); goto err; @@ -1981,7 +1984,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) struct in_addr in; char *address = NULL; tor_assert(tok->n_args); - /* XXX024 use some tor_addr parse function below instead. -RD */ + /* XXX++ use some tor_addr parse function below instead. -RD */ if (tor_addr_port_split(LOG_WARN, tok->args[0], &address, &cert->dir_port) < 0 || tor_inet_aton(address, &in) == 0) { @@ -3097,7 +3100,8 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, voter->nickname = tor_strdup(tok->args[0]); if (strlen(tok->args[1]) != HEX_DIGEST_LEN || base16_decode(voter->identity_digest, sizeof(voter->identity_digest), - tok->args[1], HEX_DIGEST_LEN) < 0) { + tok->args[1], HEX_DIGEST_LEN) + != sizeof(voter->identity_digest)) { log_warn(LD_DIR, "Error decoding identity digest %s in " "network-status document.", escaped(tok->args[1])); goto err; @@ -3146,7 +3150,8 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, } if (strlen(tok->args[0]) != HEX_DIGEST_LEN || base16_decode(voter->vote_digest, sizeof(voter->vote_digest), - tok->args[0], HEX_DIGEST_LEN) < 0) { + tok->args[0], HEX_DIGEST_LEN) + != sizeof(voter->vote_digest)) { log_warn(LD_DIR, "Error decoding vote digest %s in " "network-status consensus.", escaped(tok->args[0])); goto err; @@ -3171,7 +3176,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, if (strlen(tok->args[0]) == HEX_DIGEST_LEN) { networkstatus_voter_info_t *voter = smartlist_get(ns->voters, 0); if (base16_decode(voter->legacy_id_digest, DIGEST_LEN, - tok->args[0], HEX_DIGEST_LEN)<0) + tok->args[0], HEX_DIGEST_LEN) != DIGEST_LEN) bad = 1; else bad = 0; @@ -3203,8 +3208,11 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, if ((rs = routerstatus_parse_entry_from_string(rs_area, &s, rs_tokens, NULL, NULL, ns->consensus_method, - flav))) + flav))) { + /* Use exponential-backoff scheduling when downloading microdescs */ + rs->dl_status.backoff = DL_SCHED_RANDOM_EXPONENTIAL; smartlist_add(ns->routerstatus_list, rs); + } } } for (i = 1; i < smartlist_len(ns->routerstatus_list); ++i) { @@ -3330,7 +3338,8 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, if (strlen(id_hexdigest) != HEX_DIGEST_LEN || base16_decode(declared_identity, sizeof(declared_identity), - id_hexdigest, HEX_DIGEST_LEN) < 0) { + id_hexdigest, HEX_DIGEST_LEN) + != sizeof(declared_identity)) { log_warn(LD_DIR, "Error decoding declared identity %s in " "network-status document.", escaped(id_hexdigest)); goto err; @@ -3345,7 +3354,8 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, sig->alg = alg; if (strlen(sk_hexdigest) != HEX_DIGEST_LEN || base16_decode(sig->signing_key_digest, sizeof(sig->signing_key_digest), - sk_hexdigest, HEX_DIGEST_LEN) < 0) { + sk_hexdigest, HEX_DIGEST_LEN) + != sizeof(sig->signing_key_digest)) { log_warn(LD_DIR, "Error decoding declared signing key digest %s in " "network-status document.", escaped(sk_hexdigest)); tor_free(sig); @@ -3508,7 +3518,7 @@ networkstatus_parse_detached_signatures(const char *s, const char *eos) digest_algorithm_t alg; const char *flavor; const char *hexdigest; - size_t expected_length; + size_t expected_length, digest_length; tok = _tok; @@ -3531,8 +3541,8 @@ networkstatus_parse_detached_signatures(const char *s, const char *eos) continue; } - expected_length = - (alg == DIGEST_SHA1) ? HEX_DIGEST_LEN : HEX_DIGEST256_LEN; + digest_length = crypto_digest_algorithm_get_length(alg); + expected_length = digest_length * 2; /* hex encoding */ if (strlen(hexdigest) != expected_length) { log_warn(LD_DIR, "Wrong length on consensus-digest in detached " @@ -3541,13 +3551,13 @@ networkstatus_parse_detached_signatures(const char *s, const char *eos) } digests = detached_get_digests(sigs, flavor); tor_assert(digests); - if (!tor_mem_is_zero(digests->d[alg], DIGEST256_LEN)) { + if (!tor_mem_is_zero(digests->d[alg], digest_length)) { log_warn(LD_DIR, "Multiple digests for %s with %s on detached " "signatures document", flavor, algname); continue; } - if (base16_decode(digests->d[alg], DIGEST256_LEN, - hexdigest, strlen(hexdigest)) < 0) { + if (base16_decode(digests->d[alg], digest_length, + hexdigest, strlen(hexdigest)) != (int) digest_length) { log_warn(LD_DIR, "Bad encoding on consensus-digest in detached " "networkstatus signatures"); goto err; @@ -3620,14 +3630,14 @@ networkstatus_parse_detached_signatures(const char *s, const char *eos) if (strlen(id_hexdigest) != HEX_DIGEST_LEN || base16_decode(id_digest, sizeof(id_digest), - id_hexdigest, HEX_DIGEST_LEN) < 0) { + id_hexdigest, HEX_DIGEST_LEN) != sizeof(id_digest)) { log_warn(LD_DIR, "Error decoding declared identity %s in " "network-status vote.", escaped(id_hexdigest)); goto err; } if (strlen(sk_hexdigest) != HEX_DIGEST_LEN || base16_decode(sk_digest, sizeof(sk_digest), - sk_hexdigest, HEX_DIGEST_LEN) < 0) { + sk_hexdigest, HEX_DIGEST_LEN) != sizeof(sk_digest)) { log_warn(LD_DIR, "Error decoding declared signing key digest %s in " "network-status vote.", escaped(sk_hexdigest)); goto err; @@ -4829,7 +4839,7 @@ tor_version_parse(const char *s, tor_version_t *out) memwipe(digest, 0, sizeof(digest)); if ( hexlen == 0 || (hexlen % 2) == 1) return -1; - if (base16_decode(digest, hexlen/2, cp, hexlen)) + if (base16_decode(digest, hexlen/2, cp, hexlen) != hexlen/2) return -1; memcpy(out->git_tag, digest, hexlen/2); out->git_tag_len = hexlen/2; @@ -4974,7 +4984,7 @@ rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out, eos = eos + 1; /* Check length. */ if (eos-desc > REND_DESC_MAX_SIZE) { - /* XXX023 If we are parsing this descriptor as a server, this + /* XXXX+ If we are parsing this descriptor as a server, this * should be a protocol warning. */ log_warn(LD_REND, "Descriptor length is %d which exceeds " "maximum rendezvous descriptor size of %d bytes.", @@ -5374,6 +5384,7 @@ rend_parse_client_keys(strmap_t *parsed_clients, const char *ckstr) directory_token_t *tok; const char *current_entry = NULL; memarea_t *area = NULL; + char *err_msg = NULL; if (!ckstr || strlen(ckstr) == 0) return -1; tokens = smartlist_new(); @@ -5383,8 +5394,6 @@ rend_parse_client_keys(strmap_t *parsed_clients, const char *ckstr) current_entry = eat_whitespace(ckstr); while (!strcmpstart(current_entry, "client-name ")) { rend_authorized_client_t *parsed_entry; - size_t len; - char descriptor_cookie_tmp[REND_DESC_COOKIE_LEN+2]; /* Determine end of string. */ const char *eos = strstr(current_entry, "\nclient-name "); if (!eos) @@ -5413,12 +5422,10 @@ rend_parse_client_keys(strmap_t *parsed_clients, const char *ckstr) tor_assert(tok == smartlist_get(tokens, 0)); tor_assert(tok->n_args == 1); - len = strlen(tok->args[0]); - if (len < 1 || len > 19 || - strspn(tok->args[0], REND_LEGAL_CLIENTNAME_CHARACTERS) != len) { + if (!rend_valid_client_name(tok->args[0])) { log_warn(LD_CONFIG, "Illegal client name: %s. (Length must be " - "between 1 and 19, and valid characters are " - "[A-Za-z0-9+-_].)", tok->args[0]); + "between 1 and %d, and valid characters are " + "[A-Za-z0-9+-_].)", tok->args[0], REND_CLIENTNAME_MAX_LEN); goto err; } /* Check if client name is duplicate. */ @@ -5440,23 +5447,13 @@ rend_parse_client_keys(strmap_t *parsed_clients, const char *ckstr) /* Parse descriptor cookie. */ tok = find_by_keyword(tokens, C_DESCRIPTOR_COOKIE); tor_assert(tok->n_args == 1); - if (strlen(tok->args[0]) != REND_DESC_COOKIE_LEN_BASE64 + 2) { - log_warn(LD_REND, "Descriptor cookie has illegal length: %s", - escaped(tok->args[0])); - goto err; - } - /* The size of descriptor_cookie_tmp needs to be REND_DESC_COOKIE_LEN+2, - * because a base64 encoding of length 24 does not fit into 16 bytes in all - * cases. */ - if (base64_decode(descriptor_cookie_tmp, sizeof(descriptor_cookie_tmp), - tok->args[0], strlen(tok->args[0])) - != REND_DESC_COOKIE_LEN) { - log_warn(LD_REND, "Descriptor cookie contains illegal characters: " - "%s", escaped(tok->args[0])); + if (rend_auth_decode_cookie(tok->args[0], parsed_entry->descriptor_cookie, + NULL, &err_msg) < 0) { + tor_assert(err_msg); + log_warn(LD_REND, "%s", err_msg); + tor_free(err_msg); goto err; } - memcpy(parsed_entry->descriptor_cookie, descriptor_cookie_tmp, - REND_DESC_COOKIE_LEN); } result = strmap_size(parsed_clients); goto done; diff --git a/src/or/scheduler.h b/src/or/scheduler.h index 94a44a0aa3..3dcfd2faca 100644 --- a/src/or/scheduler.h +++ b/src/or/scheduler.h @@ -44,6 +44,13 @@ MOCK_DECL(STATIC int, scheduler_compare_channels, (const void *c1_v, const void *c2_v)); STATIC uint64_t scheduler_get_queue_heuristic(void); STATIC void scheduler_update_queue_heuristic(time_t now); + +#ifdef TOR_UNIT_TESTS +extern smartlist_t *channels_pending; +extern struct event *run_sched_ev; +extern uint64_t queue_heuristic; +extern time_t queue_heuristic_timestamp; +#endif #endif #endif /* !defined(TOR_SCHEDULER_H) */ diff --git a/src/or/tor_main.c b/src/or/tor_main.c index ac32eef559..21fbe3efb5 100644 --- a/src/or/tor_main.c +++ b/src/or/tor_main.c @@ -3,6 +3,8 @@ * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ +extern const char tor_git_revision[]; + /** String describing which Tor Git repository version the source was * built from. This string is generated by a bit of shell kludging in * src/or/include.am, and is usually right. diff --git a/src/or/transports.c b/src/or/transports.c index 1b8b1e678c..92539b1693 100644 --- a/src/or/transports.c +++ b/src/or/transports.c @@ -1363,7 +1363,7 @@ create_managed_proxy_environment(const managed_proxy_t *mp) } } - /* XXX024 Remove the '=' here once versions of obfsproxy which + /* XXXX Remove the '=' here once versions of obfsproxy which * assert that this env var exists are sufficiently dead. * * (If we remove this line entirely, some joker will stick this diff --git a/src/test/bench.c b/src/test/bench.c index 5aefda5ff2..5595988f31 100644 --- a/src/test/bench.c +++ b/src/test/bench.c @@ -3,6 +3,7 @@ * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ +extern const char tor_git_revision[]; /* Ordinarily defined in tor_main.c; this bit is just here to provide one * since we're not linking to tor_main.c */ const char tor_git_revision[] = ""; diff --git a/src/test/example_extrainfo.inc b/src/test/example_extrainfo.inc index e096afd6c4..0bf2341ef5 100644 --- a/src/test/example_extrainfo.inc +++ b/src/test/example_extrainfo.inc @@ -133,7 +133,7 @@ static const char EX_EI_BAD_NICKNAME_KEY[] = "/UBWNSyXCFDMqnddb/LZ8+VgttmxfYkpeRzSSmDijN3RbOvYJhhBAgMBAAE=\n" "-----END RSA PUBLIC KEY-----\n"; -const char EX_EI_BAD_TOKENS[] = +static const char EX_EI_BAD_TOKENS[] = "extra-info bob 6F314FB01A31162BD5E473D4977AC570DC5B86BB\n" "published 2014-10-05 20:07:00\n" "published 2014-10-05 20:07:00\n" @@ -145,8 +145,9 @@ const char EX_EI_BAD_TOKENS[] = "-----END SIGNATURE-----\n" ; -const char EX_EI_BAD_TOKENS_FP[] = "6F314FB01A31162BD5E473D4977AC570DC5B86BB"; -const char EX_EI_BAD_TOKENS_KEY[] = +static const char EX_EI_BAD_TOKENS_FP[] = + "6F314FB01A31162BD5E473D4977AC570DC5B86BB"; +static const char EX_EI_BAD_TOKENS_KEY[] = "-----BEGIN RSA PUBLIC KEY-----\n" "MIGJAoGBAL7Z8tz45Tb4tnEFS2sAyjubBV/giSfZdmXRkDV8Jo4xqWqhWFJn7+zN\n" "AXBWBThGeVH2WXrpz5seNJXgZJPxMTMsrnSCGcRXZw0Npti2MkLuQ6+prZa+OPwE\n" @@ -210,7 +211,8 @@ static const char EX_EI_GOOD_ED_EI[] = "\n" "\n" ; -const char EX_EI_GOOD_ED_EI_FP[] = "A692FE045C32B5E3A54B52882EF678A9DAC46A73"; +static const char EX_EI_GOOD_ED_EI_FP[] = + "A692FE045C32B5E3A54B52882EF678A9DAC46A73"; static const char EX_EI_GOOD_ED_EI_KEY[] = "-----BEGIN RSA PUBLIC KEY-----\n" "MIGJAoGBAM3jdYwjwGxDWYj/vyFkQT7RgeCNIn89Ei6D2+L/fdtFnqrMXOreFFHL\n" @@ -237,7 +239,8 @@ static const char EX_EI_ED_MISSING_SIG[] = "\n" "\n" ; -const char EX_EI_ED_MISSING_SIG_FP[] = "2A7521497B91A8437021515308A47491164EDBA1"; +static const char EX_EI_ED_MISSING_SIG_FP[] = + "2A7521497B91A8437021515308A47491164EDBA1"; static const char EX_EI_ED_MISSING_SIG_KEY[] = "-----BEGIN RSA PUBLIC KEY-----\n" "MIGJAoGBAOOB8ccxbtk2dB5FuKFhGndDcO6STNjB6KiG0b9X2QwKrOZMfmXSigto\n" @@ -260,7 +263,8 @@ static const char EX_EI_ED_MISSING_CERT[] = "\n" "\n" ; -const char EX_EI_ED_MISSING_CERT_FP[] = "E88E43E86015345A323D93D825C33E4AD1028F65"; +static const char EX_EI_ED_MISSING_CERT_FP[] = + "E88E43E86015345A323D93D825C33E4AD1028F65"; static const char EX_EI_ED_MISSING_CERT_KEY[] = "-----BEGIN RSA PUBLIC KEY-----\n" "MIGJAoGBALjA/geb0TR9rp/UPvLhABQpB0XUDYuZAnLkrv+i7AAV7FemTDveEGnc\n" @@ -284,7 +288,8 @@ static const char EX_EI_ED_BAD_CERT1[] = "-----END SIGNATURE-----\n" "\n" ; -const char EX_EI_ED_BAD_CERT1_FP[] = "F78D8A655607D32281D02144817A4F1D26AE520F"; +static const char EX_EI_ED_BAD_CERT1_FP[] = + "F78D8A655607D32281D02144817A4F1D26AE520F"; static const char EX_EI_ED_BAD_CERT1_KEY[] = "-----BEGIN RSA PUBLIC KEY-----\n" "MIGJAoGBAMlR46JhxsCmWYtmIB/JjTV2TUYIhJLmHy+X7FfkK3ZVQvvl9/3GSXFL\n" @@ -309,7 +314,8 @@ static const char EX_EI_ED_BAD_CERT2[] = "cVrtU6RVmzldSbyir8V/Z4S/Cm67gYAgjM5gfoFUqDs=\n" "-----END SIGNATURE-----\n" ; -const char EX_EI_ED_BAD_CERT2_FP[] = "7C2B42E783C4E0EB0CC3BDB37385D16737BACFBD"; +static const char EX_EI_ED_BAD_CERT2_FP[] = + "7C2B42E783C4E0EB0CC3BDB37385D16737BACFBD"; static const char EX_EI_ED_BAD_CERT2_KEY[] = "-----BEGIN RSA PUBLIC KEY-----\n" "MIGJAoGBALAM1F/0XJEsbxIQqb3+ObX/yGVnq9of8Q9sLsmxffD6hwVpCqnV3lTg\n" @@ -335,7 +341,8 @@ static const char EX_EI_ED_BAD_SIG1[] = "-----END SIGNATURE-----\n" "\n" ; -const char EX_EI_ED_BAD_SIG1_FP[] = "5AC3A538FEEFC6F9FCC5FA0CE64704396C30D62A"; +static const char EX_EI_ED_BAD_SIG1_FP[] = + "5AC3A538FEEFC6F9FCC5FA0CE64704396C30D62A"; static const char EX_EI_ED_BAD_SIG1_KEY[] = "-----BEGIN RSA PUBLIC KEY-----\n" "MIGJAoGBAMvb6SuoIkPfBkJgQuo5aQDepAs1kEETZ9VXotMlhB0JJikrqBrAAz+7\n" @@ -361,7 +368,8 @@ static const char EX_EI_ED_BAD_SIG2[] = "-----END SIGNATURE-----\n" "\n" ; -const char EX_EI_ED_BAD_SIG2_FP[] = "7F1D4DD477E340C6D6B389FAC26EDC746113082F"; +static const char EX_EI_ED_BAD_SIG2_FP[] = + "7F1D4DD477E340C6D6B389FAC26EDC746113082F"; static const char EX_EI_ED_BAD_SIG2_KEY[] = "-----BEGIN RSA PUBLIC KEY-----\n" "MIGJAoGBALzOyfCEUZnvCyhlyMctPkdXg/XRE3Cr6QgyzdKf5kQbUiu2n0FgSHOX\n" @@ -388,7 +396,8 @@ static const char EX_EI_ED_MISPLACED_CERT[] = "-----END SIGNATURE-----\n" "\n" ; -const char EX_EI_ED_MISPLACED_CERT_FP[] = "3B788BD0CE348BC5CED48313307C78175EB6D0F3"; +static const char EX_EI_ED_MISPLACED_CERT_FP[] = + "3B788BD0CE348BC5CED48313307C78175EB6D0F3"; static const char EX_EI_ED_MISPLACED_CERT_KEY[] = "-----BEGIN RSA PUBLIC KEY-----\n" "MIGJAoGBALTwNqhTprg1oC6bEbDqwIYBoER6prqUXQFbwbFDn+ekXhZj8vltgGwp\n" @@ -414,7 +423,8 @@ static const char EX_EI_ED_MISPLACED_SIG[] = "-----END SIGNATURE-----\n" "\n" ; -const char EX_EI_ED_MISPLACED_SIG_FP[] = "384E40A5DEED4AB1D8A74F1FCBDB18B7C24A8284"; +static const char EX_EI_ED_MISPLACED_SIG_FP[] = + "384E40A5DEED4AB1D8A74F1FCBDB18B7C24A8284"; static const char EX_EI_ED_MISPLACED_SIG_KEY[] = "-----BEGIN RSA PUBLIC KEY-----\n" "MIGJAoGBAK0HgOCG/6433VCrwz/vhk3cKmyOfenCp0GZ4DIUwPWt4DeyP4nTbN6T\n" diff --git a/src/test/include.am b/src/test/include.am index 7d80fdf152..5a91c74cde 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -9,6 +9,12 @@ TESTS_ENVIRONMENT = \ export TESTING_TOR_BINARY="$(TESTING_TOR_BINARY)"; TESTSCRIPTS = src/test/test_zero_length_keys.sh \ + src/test/test_workqueue_cancel.sh \ + src/test/test_workqueue_efd.sh \ + src/test/test_workqueue_efd2.sh \ + src/test/test_workqueue_pipe.sh \ + src/test/test_workqueue_pipe2.sh \ + src/test/test_workqueue_socketpair.sh \ src/test/test_switch_id.sh if USEPYTHON @@ -16,7 +22,9 @@ TESTSCRIPTS += src/test/test_ntor.sh src/test/test_bt.sh endif TESTS += src/test/test src/test/test-slow src/test/test-memwipe \ - src/test/test_workqueue src/test/test_keygen.sh \ + src/test/test_workqueue \ + src/test/test_keygen.sh \ + src/test/test-timers \ $(TESTSCRIPTS) # These flavors are run using automake's test-driver and test-network.sh @@ -40,7 +48,8 @@ noinst_PROGRAMS+= \ src/test/test-memwipe \ src/test/test-child \ src/test/test_workqueue \ - src/test/test-switch-id + src/test/test-switch-id \ + src/test/test-timers endif src_test_AM_CPPFLAGS = -DSHARE_DATADIR="\"$(datadir)\"" \ @@ -86,6 +95,7 @@ src_test_test_SOURCES = \ src/test/test_guardfraction.c \ src/test/test_extorport.c \ src/test/test_hs.c \ + src/test/test_handles.c \ src/test/test_introduce.c \ src/test/test_keypin.c \ src/test/test_link_handshake.c \ @@ -97,6 +107,7 @@ src_test_test_SOURCES = \ src/test/test_policy.c \ src/test/test_procmon.c \ src/test/test_pt.c \ + src/test/test_pubsub.c \ src/test/test_relay.c \ src/test/test_relaycell.c \ src/test/test_rendcache.c \ @@ -127,6 +138,8 @@ src_test_test_slow_SOURCES = \ src_test_test_memwipe_SOURCES = \ src/test/test-memwipe.c +src_test_test_timers_SOURCES = \ + src/test/test-timers.c src_test_test_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) @@ -147,6 +160,7 @@ src_test_test_switch_id_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) src_test_test_switch_id_LDFLAGS = @TOR_LDFLAGS_zlib@ src_test_test_switch_id_LDADD = \ src/common/libor-testing.a \ + src/common/libor-ctime-testing.a \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ src_test_test_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ \ @@ -156,6 +170,7 @@ src_test_test_LDADD = src/or/libtor-testing.a \ $(LIBKECCAK_TINY) \ $(LIBDONNA) \ src/common/libor-testing.a \ + src/common/libor-ctime-testing.a \ src/common/libor-event-testing.a \ src/trunnel/libor-trunnel-testing.a \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \ @@ -168,13 +183,17 @@ src_test_test_slow_LDADD = $(src_test_test_LDADD) src_test_test_slow_LDFLAGS = $(src_test_test_LDFLAGS) src_test_test_memwipe_CPPFLAGS = $(src_test_test_CPPFLAGS) -src_test_test_memwipe_CFLAGS = $(src_test_test_CFLAGS) +# Don't use bugtrap cflags here: memwipe tests require memory violations. +src_test_test_memwipe_CFLAGS = $(TEST_CFLAGS) src_test_test_memwipe_LDADD = $(src_test_test_LDADD) -src_test_test_memwipe_LDFLAGS = $(src_test_test_LDFLAGS) +# The LDFLAGS need to include the bugtrap cflags, or else we won't link +# successfully with the libraries built with them. +src_test_test_memwipe_LDFLAGS = $(src_test_test_LDFLAGS) @CFLAGS_BUGTRAP@ src_test_bench_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ \ @TOR_LDFLAGS_libevent@ src_test_bench_LDADD = src/or/libtor.a src/common/libor.a \ + src/common/libor-ctime.a \ src/common/libor-crypto.a $(LIBKECCAK_TINY) $(LIBDONNA) \ src/common/libor-event.a src/trunnel/libor-trunnel.a \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \ @@ -185,11 +204,23 @@ src_test_test_workqueue_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ \ @TOR_LDFLAGS_libevent@ src_test_test_workqueue_LDADD = src/or/libtor-testing.a \ src/common/libor-testing.a \ + src/common/libor-ctime-testing.a \ src/common/libor-crypto-testing.a $(LIBKECCAK_TINY) $(LIBDONNA) \ src/common/libor-event-testing.a \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ +src_test_test_timers_CPPFLAGS = $(src_test_test_CPPFLAGS) +src_test_test_timers_CFLAGS = $(src_test_test_CFLAGS) +src_test_test_timers_LDADD = \ + src/common/libor-testing.a \ + src/common/libor-ctime-testing.a \ + src/common/libor-event-testing.a \ + src/common/libor-crypto-testing.a $(LIBKECCAK_TINY) $(LIBDONNA) \ + @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \ + @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ +src_test_test_timers_LDFLAGS = $(src_test_test_LDFLAGS) + noinst_HEADERS+= \ src/test/fakechans.h \ src/test/log_test_helpers.h \ @@ -208,6 +239,7 @@ noinst_PROGRAMS+= src/test/test-ntor-cl src_test_test_ntor_cl_SOURCES = src/test/test_ntor_cl.c src_test_test_ntor_cl_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ src_test_test_ntor_cl_LDADD = src/or/libtor.a src/common/libor.a \ + src/common/libor-ctime.a \ src/common/libor-crypto.a $(LIBKECCAK_TINY) $(LIBDONNA) \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ @@ -217,6 +249,7 @@ src_test_test_ntor_cl_AM_CPPFLAGS = \ noinst_PROGRAMS += src/test/test-bt-cl src_test_test_bt_cl_SOURCES = src/test/test_bt_cl.c src_test_test_bt_cl_LDADD = src/common/libor-testing.a \ + src/common/libor-ctime-testing.a \ @TOR_LIB_MATH@ \ @TOR_LIB_WS32@ @TOR_LIB_GDI@ src_test_test_bt_cl_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) @@ -228,7 +261,14 @@ EXTRA_DIST += \ src/test/slownacl_curve25519.py \ src/test/zero_length_keys.sh \ src/test/test_keygen.sh \ - src/test/test_zero_length_keys.sh \ - src/test/test_ntor.sh src/test/test_bt.sh \ + src/test/test_zero_length_keys.sh \ + src/test/test_ntor.sh src/test/test_bt.sh \ src/test/test-network.sh \ - src/test/test_switch_id.sh + src/test/test_switch_id.sh \ + src/test/test_workqueue_cancel.sh \ + src/test/test_workqueue_efd.sh \ + src/test/test_workqueue_efd2.sh \ + src/test/test_workqueue_pipe.sh \ + src/test/test_workqueue_pipe2.sh \ + src/test/test_workqueue_socketpair.sh + diff --git a/src/test/test-memwipe.c b/src/test/test-memwipe.c index 5d4fcec664..c28d5054a2 100644 --- a/src/test/test-memwipe.c +++ b/src/test/test-memwipe.c @@ -6,9 +6,6 @@ #include "crypto.h" #include "compat.h" -#undef MIN -#define MIN(a,b) ( ((a)<(b)) ? (a) : (b) ) - static unsigned fill_a_buffer_memset(void) __attribute__((noinline)); static unsigned fill_a_buffer_memwipe(void) __attribute__((noinline)); static unsigned fill_a_buffer_nothing(void) __attribute__((noinline)); @@ -17,6 +14,7 @@ static unsigned fill_heap_buffer_memwipe(void) __attribute__((noinline)); static unsigned fill_heap_buffer_nothing(void) __attribute__((noinline)); static unsigned check_a_buffer(void) __attribute__((noinline)); +extern const char *s; /* Make the linkage global */ const char *s = NULL; #define BUF_LEN 2048 diff --git a/src/test/test-timers.c b/src/test/test-timers.c new file mode 100644 index 0000000000..0196ec1fef --- /dev/null +++ b/src/test/test-timers.c @@ -0,0 +1,134 @@ +/* Copyright 2016, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" + +#include <math.h> +#include <stdio.h> +#include <string.h> + +#ifdef HAVE_EVENT2_EVENT_H +#include <event2/event.h> +#else +#include <event.h> +#endif + +#include "compat.h" +#include "compat_libevent.h" +#include "crypto.h" +#include "timers.h" +#include "util.h" + +#define N_TIMERS 1000 +#define MAX_DURATION 30 +#define N_DISABLE 5 + +static struct timeval fire_at[N_TIMERS] = { {0,0} }; +static int is_disabled[N_TIMERS] = {0}; +static int fired[N_TIMERS] = {0}; +static struct timeval difference[N_TIMERS] = { {0,0} }; +static tor_timer_t *timers[N_TIMERS] = {NULL}; + +static int n_active_timers = 0; +static int n_fired = 0; + +static void +timer_cb(tor_timer_t *t, void *arg, const struct timeval *now) +{ + tor_timer_t **t_ptr = arg; + tor_assert(*t_ptr == t); + int idx = (int) (t_ptr - timers); + ++fired[idx]; + timersub(now, &fire_at[idx], &difference[idx]); + ++n_fired; + // printf("%d / %d\n",n_fired, N_TIMERS); + if (n_fired == n_active_timers) { + event_base_loopbreak(tor_libevent_get_base()); + } +} + +int +main(int argc, char **argv) +{ + (void)argc; + (void)argv; + tor_libevent_cfg cfg; + memset(&cfg, 0, sizeof(cfg)); + tor_libevent_initialize(&cfg); + timers_initialize(); + + int i; + int ret; + struct timeval now; + tor_gettimeofday(&now); + for (i = 0; i < N_TIMERS; ++i) { + struct timeval delay; + delay.tv_sec = crypto_rand_int_range(0,MAX_DURATION); + delay.tv_usec = crypto_rand_int_range(0,1000000); + timeradd(&now, &delay, &fire_at[i]); + timers[i] = timer_new(timer_cb, &timers[i]); + timer_schedule(timers[i], &delay); + ++n_active_timers; + } + + /* Disable some; we'll make sure they don't trigger. */ + for (i = 0; i < N_DISABLE; ++i) { + int idx = crypto_rand_int_range(0, N_TIMERS); + if (is_disabled[idx]) + continue; + is_disabled[idx] = 1; + timer_disable(timers[idx]); + --n_active_timers; + } + + event_base_loop(tor_libevent_get_base(), 0); + + int64_t total_difference = 0; + uint64_t total_square_difference = 0; + tor_assert(n_fired == n_active_timers); + for (i = 0; i < N_TIMERS; ++i) { + if (is_disabled[i]) { + tor_assert(fired[i] == 0); + continue; + } + tor_assert(fired[i] == 1); + int64_t diff = difference[i].tv_usec + difference[i].tv_sec * 1000000; + total_difference += diff; + total_square_difference += diff*diff; + } + const int64_t mean_diff = total_difference / n_active_timers; + printf("mean difference: "U64_FORMAT" usec\n", + U64_PRINTF_ARG(mean_diff)); + + const double mean_sq = ((double)total_square_difference)/ n_active_timers; + const double sq_mean = mean_diff * mean_diff; + const double stddev = sqrt(mean_sq - sq_mean); + printf("standard deviation: %lf usec\n", stddev); + +#define MAX_DIFF_USEC (500*1000) +#define MAX_STDDEV_USEC (500*1000) +#define ODD_DIFF_USEC (2000) +#define ODD_STDDEV_USEC (2000) + + if (mean_diff < 0 || mean_diff > MAX_DIFF_USEC || stddev > MAX_STDDEV_USEC) { + printf("Either your system is under ridiculous load, or the " + "timer backend is broken.\n"); + ret = 1; + } else if (mean_diff > ODD_DIFF_USEC || stddev > ODD_STDDEV_USEC) { + printf("Either your system is a bit slow or the " + "timer backend is odd.\n"); + ret = 0; + } else { + printf("Looks good enough.\n"); + ret = 0; + } + + timer_free(NULL); + + for (i = 0; i < N_TIMERS; ++i) { + timer_free(timers[i]); + } + timers_shutdown(); + return ret; +} + diff --git a/src/test/test.c b/src/test/test.c index ed167a3e67..c0faec3027 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -1124,60 +1124,6 @@ static struct testcase_t test_array[] = { END_OF_TESTCASES }; -extern struct testcase_t accounting_tests[]; -extern struct testcase_t addr_tests[]; -extern struct testcase_t address_tests[]; -extern struct testcase_t buffer_tests[]; -extern struct testcase_t cell_format_tests[]; -extern struct testcase_t cell_queue_tests[]; -extern struct testcase_t channel_tests[]; -extern struct testcase_t channeltls_tests[]; -extern struct testcase_t checkdir_tests[]; -extern struct testcase_t circuitlist_tests[]; -extern struct testcase_t circuitmux_tests[]; -extern struct testcase_t compat_libevent_tests[]; -extern struct testcase_t config_tests[]; -extern struct testcase_t connection_tests[]; -extern struct testcase_t container_tests[]; -extern struct testcase_t controller_tests[]; -extern struct testcase_t controller_event_tests[]; -extern struct testcase_t crypto_tests[]; -extern struct testcase_t dir_tests[]; -extern struct testcase_t dir_handle_get_tests[]; -extern struct testcase_t entryconn_tests[]; -extern struct testcase_t entrynodes_tests[]; -extern struct testcase_t guardfraction_tests[]; -extern struct testcase_t extorport_tests[]; -extern struct testcase_t hs_tests[]; -extern struct testcase_t introduce_tests[]; -extern struct testcase_t keypin_tests[]; -extern struct testcase_t link_handshake_tests[]; -extern struct testcase_t logging_tests[]; -extern struct testcase_t microdesc_tests[]; -extern struct testcase_t nodelist_tests[]; -extern struct testcase_t oom_tests[]; -extern struct testcase_t options_tests[]; -extern struct testcase_t policy_tests[]; -extern struct testcase_t procmon_tests[]; -extern struct testcase_t pt_tests[]; -extern struct testcase_t relay_tests[]; -extern struct testcase_t relaycell_tests[]; -extern struct testcase_t rend_cache_tests[]; -extern struct testcase_t replaycache_tests[]; -extern struct testcase_t router_tests[]; -extern struct testcase_t routerkeys_tests[]; -extern struct testcase_t routerlist_tests[]; -extern struct testcase_t routerset_tests[]; -extern struct testcase_t scheduler_tests[]; -extern struct testcase_t socks_tests[]; -extern struct testcase_t status_tests[]; -extern struct testcase_t thread_tests[]; -extern struct testcase_t tortls_tests[]; -extern struct testcase_t util_tests[]; -extern struct testcase_t util_format_tests[]; -extern struct testcase_t util_process_tests[]; -extern struct testcase_t dns_tests[]; - struct testgroup_t testgroups[] = { { "", test_array }, { "accounting/", accounting_tests }, @@ -1230,7 +1176,9 @@ struct testgroup_t testgroups[] = { { "util/format/", util_format_tests }, { "util/logging/", logging_tests }, { "util/process/", util_process_tests }, + { "util/pubsub/", pubsub_tests }, { "util/thread/", thread_tests }, + { "util/handle/", handle_tests }, { "dns/", dns_tests }, END_OF_GROUPS }; diff --git a/src/test/test.h b/src/test/test.h index e618ce1224..747b61d669 100644 --- a/src/test/test.h +++ b/src/test/test.h @@ -73,7 +73,7 @@ {print_ = (I64_PRINTF_TYPE) value_;}, {}, TT_EXIT_TEST_FUNCTION) const char *get_fname(const char *name); -crypto_pk_t *pk_generate(int idx); +struct crypto_pk_t *pk_generate(int idx); #define US2_CONCAT_2__(a, b) a ## __ ## b #define US_CONCAT_2__(a, b) a ## _ ## b @@ -163,11 +163,86 @@ crypto_pk_t *pk_generate(int idx); #define CALLED(mock_name) US_CONCAT_2_(NS(mock_name), called) #define NS_DECL(retval, mock_fn, args) \ + extern int CALLED(mock_fn); \ static retval NS(mock_fn) args; int CALLED(mock_fn) = 0 #define NS_MOCK(name) MOCK(name, NS(name)) #define NS_UNMOCK(name) UNMOCK(name) extern const struct testcase_setup_t passthrough_setup; +extern struct testcase_t accounting_tests[]; +extern struct testcase_t addr_tests[]; +extern struct testcase_t address_tests[]; +extern struct testcase_t buffer_tests[]; +extern struct testcase_t cell_format_tests[]; +extern struct testcase_t cell_queue_tests[]; +extern struct testcase_t channel_tests[]; +extern struct testcase_t channeltls_tests[]; +extern struct testcase_t checkdir_tests[]; +extern struct testcase_t circuitlist_tests[]; +extern struct testcase_t circuitmux_tests[]; +extern struct testcase_t compat_libevent_tests[]; +extern struct testcase_t config_tests[]; +extern struct testcase_t connection_tests[]; +extern struct testcase_t container_tests[]; +extern struct testcase_t controller_tests[]; +extern struct testcase_t controller_event_tests[]; +extern struct testcase_t crypto_tests[]; +extern struct testcase_t dir_tests[]; +extern struct testcase_t dir_handle_get_tests[]; +extern struct testcase_t entryconn_tests[]; +extern struct testcase_t entrynodes_tests[]; +extern struct testcase_t guardfraction_tests[]; +extern struct testcase_t extorport_tests[]; +extern struct testcase_t hs_tests[]; +extern struct testcase_t introduce_tests[]; +extern struct testcase_t keypin_tests[]; +extern struct testcase_t link_handshake_tests[]; +extern struct testcase_t logging_tests[]; +extern struct testcase_t microdesc_tests[]; +extern struct testcase_t nodelist_tests[]; +extern struct testcase_t oom_tests[]; +extern struct testcase_t options_tests[]; +extern struct testcase_t policy_tests[]; +extern struct testcase_t procmon_tests[]; +extern struct testcase_t pubsub_tests[]; +extern struct testcase_t pt_tests[]; +extern struct testcase_t relay_tests[]; +extern struct testcase_t relaycell_tests[]; +extern struct testcase_t rend_cache_tests[]; +extern struct testcase_t replaycache_tests[]; +extern struct testcase_t router_tests[]; +extern struct testcase_t routerkeys_tests[]; +extern struct testcase_t routerlist_tests[]; +extern struct testcase_t routerset_tests[]; +extern struct testcase_t scheduler_tests[]; +extern struct testcase_t socks_tests[]; +extern struct testcase_t status_tests[]; +extern struct testcase_t thread_tests[]; +extern struct testcase_t tortls_tests[]; +extern struct testcase_t util_tests[]; +extern struct testcase_t util_format_tests[]; +extern struct testcase_t util_process_tests[]; +extern struct testcase_t dns_tests[]; +extern struct testcase_t handle_tests[]; + +extern struct testcase_t slow_crypto_tests[]; +extern struct testcase_t slow_util_tests[]; + +extern struct testgroup_t testgroups[]; + +extern const char AUTHORITY_CERT_1[]; +extern const char AUTHORITY_SIGNKEY_1[]; +extern const char AUTHORITY_SIGNKEY_A_DIGEST[]; +extern const char AUTHORITY_SIGNKEY_A_DIGEST256[]; +extern const char AUTHORITY_CERT_2[]; +extern const char AUTHORITY_SIGNKEY_2[]; +extern const char AUTHORITY_SIGNKEY_B_DIGEST[]; +extern const char AUTHORITY_SIGNKEY_B_DIGEST256[]; +extern const char AUTHORITY_CERT_3[]; +extern const char AUTHORITY_SIGNKEY_3[]; +extern const char AUTHORITY_SIGNKEY_C_DIGEST[]; +extern const char AUTHORITY_SIGNKEY_C_DIGEST256[]; + #endif diff --git a/src/test/test_bt.sh b/src/test/test_bt.sh index 033acac955..312905a4e2 100755 --- a/src/test/test_bt.sh +++ b/src/test/test_bt.sh @@ -3,8 +3,11 @@ exitcode=0 +export ASAN_OPTIONS="handle_segv=0:allow_user_segv_handler=1" "${builddir:-.}/src/test/test-bt-cl" backtraces || exit $? -"${builddir:-.}/src/test/test-bt-cl" assert | "${PYTHON:-python}" "${abs_top_srcdir:-.}/src/test/bt_test.py" || exitcode="$?" -"${builddir:-.}/src/test/test-bt-cl" crash | "${PYTHON:-python}" "${abs_top_srcdir:-.}/src/test/bt_test.py" || exitcode="$?" +"${builddir:-.}/src/test/test-bt-cl" assert 2>&1 | "${PYTHON:-python}" "${abs_top_srcdir:-.}/src/test/bt_test.py" || exitcode="$?" +"${builddir:-.}/src/test/test-bt-cl" crash 2>&1 | "${PYTHON:-python}" "${abs_top_srcdir:-.}/src/test/bt_test.py" || exitcode="$?" + +"${builddir:-.}/src/test/test-bt-cl" none || exitcode="$?" exit ${exitcode} diff --git a/src/test/test_bt_cl.c b/src/test/test_bt_cl.c index 2f5e50fbf5..95b4f48f11 100644 --- a/src/test/test_bt_cl.c +++ b/src/test/test_bt_cl.c @@ -28,6 +28,9 @@ int a_tangled_web(int x) NOINLINE; int we_weave(int x) NOINLINE; static void abort_handler(int s) NORETURN; +#ifdef HAVE_CFLAG_WNULL_DEREFERENCE +DISABLE_GCC_WARNING(null-dereference) +#endif int crash(int x) { @@ -47,6 +50,9 @@ crash(int x) crashtype *= x; return crashtype; } +#ifdef HAVE_CFLAG_WNULL_DEREFERENCE +ENABLE_GCC_WARNING(null-dereference) +#endif int oh_what(int x) diff --git a/src/test/test_buffers.c b/src/test/test_buffers.c index e5e56edf75..95584d54a8 100644 --- a/src/test/test_buffers.c +++ b/src/test/test_buffers.c @@ -695,9 +695,9 @@ test_buffers_zlib_fin_at_chunk_end(void *arg) tor_free(msg); } -const uint8_t *tls_read_ptr; -int n_remaining; -int next_reply_val[16]; +static const uint8_t *tls_read_ptr; +static int n_remaining; +static int next_reply_val[16]; static int mock_tls_read(tor_tls_t *tls, char *cp, size_t len) diff --git a/src/test/test_channel.c b/src/test/test_channel.c index 846e419fea..a9e0634d9e 100644 --- a/src/test/test_channel.c +++ b/src/test/test_channel.c @@ -20,9 +20,6 @@ #include "test.h" #include "fakechans.h" -/* This comes from channel.c */ -extern uint64_t estimated_total_queue_size; - static int test_chan_accept_cells = 0; static int test_chan_fixed_cells_recved = 0; static cell_t * test_chan_last_seen_fixed_cell_ptr = NULL; @@ -33,7 +30,7 @@ static int test_destroy_not_pending_calls = 0; static int test_doesnt_want_writes_count = 0; static int test_dumpstats_calls = 0; static int test_has_waiting_cells_count = 0; -static double test_overhead_estimate = 1.0f; +static double test_overhead_estimate = 1.0; static int test_releases_count = 0; static circuitmux_t *test_target_cmux = NULL; static unsigned int test_cmux_cells = 0; @@ -792,7 +789,7 @@ test_channel_incoming(void *arg) /* Accept cells to lower layer */ test_chan_accept_cells = 1; /* Use default overhead factor */ - test_overhead_estimate = 1.0f; + test_overhead_estimate = 1.0; ch = new_fake_channel(); tt_assert(ch); @@ -881,7 +878,7 @@ test_channel_lifecycle(void *arg) /* Accept cells to lower layer */ test_chan_accept_cells = 1; /* Use default overhead factor */ - test_overhead_estimate = 1.0f; + test_overhead_estimate = 1.0; ch1 = new_fake_channel(); tt_assert(ch1); @@ -989,7 +986,7 @@ test_channel_lifecycle_2(void *arg) /* Accept cells to lower layer */ test_chan_accept_cells = 1; /* Use default overhead factor */ - test_overhead_estimate = 1.0f; + test_overhead_estimate = 1.0; ch = new_fake_channel(); tt_assert(ch); @@ -1136,7 +1133,7 @@ test_channel_multi(void *arg) /* Accept cells to lower layer */ test_chan_accept_cells = 1; /* Use default overhead factor */ - test_overhead_estimate = 1.0f; + test_overhead_estimate = 1.0; ch1 = new_fake_channel(); tt_assert(ch1); @@ -1444,7 +1441,7 @@ test_channel_queue_incoming(void *arg) /* Accept cells to lower layer */ test_chan_accept_cells = 1; /* Use default overhead factor */ - test_overhead_estimate = 1.0f; + test_overhead_estimate = 1.0; ch = new_fake_channel(); tt_assert(ch); @@ -1584,16 +1581,16 @@ test_channel_queue_size(void *arg) /* One cell, times an overhead factor of 1.0 */ tt_u64_op(ch->bytes_queued_for_xmit, ==, 512); /* Try a different overhead factor */ - test_overhead_estimate = 0.5f; + test_overhead_estimate = 0.5; /* This one should be ignored since it's below 1.0 */ channel_update_xmit_queue_size(ch); tt_u64_op(ch->bytes_queued_for_xmit, ==, 512); /* Now try a larger one */ - test_overhead_estimate = 2.0f; + test_overhead_estimate = 2.0; channel_update_xmit_queue_size(ch); tt_u64_op(ch->bytes_queued_for_xmit, ==, 1024); /* Go back to 1.0 */ - test_overhead_estimate = 1.0f; + test_overhead_estimate = 1.0; channel_update_xmit_queue_size(ch); tt_u64_op(ch->bytes_queued_for_xmit, ==, 512); /* Check the global estimate too */ diff --git a/src/test/test_channeltls.c b/src/test/test_channeltls.c index 04ae9a6da7..394612f155 100644 --- a/src/test/test_channeltls.c +++ b/src/test/test_channeltls.c @@ -185,7 +185,7 @@ test_channeltls_overhead_estimate(void *arg) const char test_digest[DIGEST_LEN] = { 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14 }; - float r; + double r; channel_tls_t *tlschan = NULL; (void)arg; @@ -206,31 +206,31 @@ test_channeltls_overhead_estimate(void *arg) ch = channel_tls_connect(&test_addr, 567, test_digest); tt_assert(ch != NULL); - /* First case: silly low ratios should get clamped to 1.0f */ + /* First case: silly low ratios should get clamped to 1.0 */ tlschan = BASE_CHAN_TO_TLS(ch); tt_assert(tlschan != NULL); tlschan->conn->bytes_xmitted = 128; tlschan->conn->bytes_xmitted_by_tls = 64; r = ch->get_overhead_estimate(ch); - tt_assert(fabsf(r - 1.0f) < 1E-12); + tt_assert(fabs(r - 1.0) < 1E-12); tlschan->conn->bytes_xmitted_by_tls = 127; r = ch->get_overhead_estimate(ch); - tt_assert(fabsf(r - 1.0f) < 1E-12); + tt_assert(fabs(r - 1.0) < 1E-12); /* Now middle of the range */ tlschan->conn->bytes_xmitted_by_tls = 192; r = ch->get_overhead_estimate(ch); - tt_assert(fabsf(r - 1.5f) < 1E-12); + tt_assert(fabs(r - 1.5) < 1E-12); - /* Now above the 2.0f clamp */ + /* Now above the 2.0 clamp */ tlschan->conn->bytes_xmitted_by_tls = 257; r = ch->get_overhead_estimate(ch); - tt_assert(fabsf(r - 2.0f) < 1E-12); + tt_assert(fabs(r - 2.0) < 1E-12); tlschan->conn->bytes_xmitted_by_tls = 512; r = ch->get_overhead_estimate(ch); - tt_assert(fabsf(r - 2.0f) < 1E-12); + tt_assert(fabs(r - 2.0) < 1E-12); done: if (ch) { diff --git a/src/test/test_controller.c b/src/test/test_controller.c index 7f9db4312f..b276e06787 100644 --- a/src/test/test_controller.c +++ b/src/test/test_controller.c @@ -154,10 +154,61 @@ test_rend_service_parse_port_config(void *arg) tor_free(err_msg); } +static void +test_add_onion_helper_clientauth(void *arg) +{ + rend_authorized_client_t *client = NULL; + char *err_msg = NULL; + int created = 0; + + (void)arg; + + /* Test "ClientName" only. */ + client = add_onion_helper_clientauth("alice", &created, &err_msg); + tt_assert(client); + tt_assert(created); + tt_assert(!err_msg); + rend_authorized_client_free(client); + + /* Test "ClientName:Blob" */ + client = add_onion_helper_clientauth("alice:475hGBHPlq7Mc0cRZitK/B", + &created, &err_msg); + tt_assert(client); + tt_assert(!created); + tt_assert(!err_msg); + rend_authorized_client_free(client); + + /* Test invalid client names */ + client = add_onion_helper_clientauth("no*asterisks*allowed", &created, + &err_msg); + tt_assert(!client); + tt_assert(err_msg); + tor_free(err_msg); + + /* Test invalid auth cookie */ + client = add_onion_helper_clientauth("alice:12345", &created, &err_msg); + tt_assert(!client); + tt_assert(err_msg); + tor_free(err_msg); + + /* Test invalid syntax */ + client = add_onion_helper_clientauth(":475hGBHPlq7Mc0cRZitK/B", &created, + &err_msg); + tt_assert(!client); + tt_assert(err_msg); + tor_free(err_msg); + + done: + rend_authorized_client_free(client); + tor_free(err_msg); +} + struct testcase_t controller_tests[] = { { "add_onion_helper_keyarg", test_add_onion_helper_keyarg, 0, NULL, NULL }, { "rend_service_parse_port_config", test_rend_service_parse_port_config, 0, NULL, NULL }, + { "add_onion_helper_clientauth", test_add_onion_helper_clientauth, 0, NULL, + NULL }, END_OF_TESTCASES }; diff --git a/src/test/test_crypto.c b/src/test/test_crypto.c index 6a95e92733..ba2fb86246 100644 --- a/src/test/test_crypto.c +++ b/src/test/test_crypto.c @@ -18,15 +18,12 @@ #include <openssl/evp.h> #include <openssl/rand.h> -extern const char AUTHORITY_SIGNKEY_3[]; -extern const char AUTHORITY_SIGNKEY_A_DIGEST[]; -extern const char AUTHORITY_SIGNKEY_A_DIGEST256[]; - /** Run unit tests for Diffie-Hellman functionality. */ static void test_crypto_dh(void *arg) { crypto_dh_t *dh1 = crypto_dh_new(DH_TYPE_CIRCUIT); + crypto_dh_t *dh1_dup = NULL; crypto_dh_t *dh2 = crypto_dh_new(DH_TYPE_CIRCUIT); char p1[DH_BYTES]; char p2[DH_BYTES]; @@ -41,6 +38,9 @@ test_crypto_dh(void *arg) memset(p1, 0, DH_BYTES); memset(p2, 0, DH_BYTES); tt_mem_op(p1,OP_EQ, p2, DH_BYTES); + + tt_int_op(-1, OP_EQ, crypto_dh_get_public(dh1, p1, 6)); /* too short */ + tt_assert(! crypto_dh_get_public(dh1, p1, DH_BYTES)); tt_mem_op(p1,OP_NE, p2, DH_BYTES); tt_assert(! crypto_dh_get_public(dh2, p2, DH_BYTES)); @@ -54,15 +54,119 @@ test_crypto_dh(void *arg) tt_int_op(s1len,OP_EQ, s2len); tt_mem_op(s1,OP_EQ, s2, s1len); + /* test dh_dup; make sure it works the same. */ + dh1_dup = crypto_dh_dup(dh1); + s1len = crypto_dh_compute_secret(LOG_WARN, dh1_dup, p2, DH_BYTES, s1, 50); + tt_mem_op(s1,OP_EQ, s2, s1len); + { - /* XXXX Now fabricate some bad values and make sure they get caught, - * Check 0, 1, N-1, >= N, etc. - */ + /* Now fabricate some bad values and make sure they get caught. */ + + /* 1 and 0 should both fail. */ + s1len = crypto_dh_compute_secret(LOG_WARN, dh1, "\x01", 1, s1, 50); + tt_int_op(-1, OP_EQ, s1len); + + s1len = crypto_dh_compute_secret(LOG_WARN, dh1, "\x00", 1, s1, 50); + tt_int_op(-1, OP_EQ, s1len); + + memset(p1, 0, DH_BYTES); /* 0 with padding. */ + s1len = crypto_dh_compute_secret(LOG_WARN, dh1, p1, DH_BYTES, s1, 50); + tt_int_op(-1, OP_EQ, s1len); + + p1[DH_BYTES-1] = 1; /* 1 with padding*/ + s1len = crypto_dh_compute_secret(LOG_WARN, dh1, p1, DH_BYTES, s1, 50); + tt_int_op(-1, OP_EQ, s1len); + + /* 2 is okay, though weird. */ + s1len = crypto_dh_compute_secret(LOG_WARN, dh1, "\x02", 1, s1, 50); + tt_int_op(50, OP_EQ, s1len); + + const char P[] = + "FFFFFFFFFFFFFFFFC90FDAA22168C234C4C6628B80DC1CD129024E08" + "8A67CC74020BBEA63B139B22514A08798E3404DDEF9519B3CD3A431B" + "302B0A6DF25F14374FE1356D6D51C245E485B576625E7EC6F44C42E9" + "A637ED6B0BFF5CB6F406B7EDEE386BFB5A899FA5AE9F24117C4B1FE6" + "49286651ECE65381FFFFFFFFFFFFFFFF"; + + /* p-1, p, and so on are not okay. */ + base16_decode(p1, sizeof(p1), P, strlen(P)); + + s1len = crypto_dh_compute_secret(LOG_WARN, dh1, p1, DH_BYTES, s1, 50); + tt_int_op(-1, OP_EQ, s1len); + + p1[DH_BYTES-1] = 0xFE; /* p-1 */ + s1len = crypto_dh_compute_secret(LOG_WARN, dh1, p1, DH_BYTES, s1, 50); + tt_int_op(-1, OP_EQ, s1len); + + p1[DH_BYTES-1] = 0xFD; /* p-2 works fine */ + s1len = crypto_dh_compute_secret(LOG_WARN, dh1, p1, DH_BYTES, s1, 50); + tt_int_op(50, OP_EQ, s1len); + + const char P_plus_one[] = + "FFFFFFFFFFFFFFFFC90FDAA22168C234C4C6628B80DC1CD129024E08" + "8A67CC74020BBEA63B139B22514A08798E3404DDEF9519B3CD3A431B" + "302B0A6DF25F14374FE1356D6D51C245E485B576625E7EC6F44C42E9" + "A637ED6B0BFF5CB6F406B7EDEE386BFB5A899FA5AE9F24117C4B1FE6" + "49286651ECE653820000000000000000"; + + base16_decode(p1, sizeof(p1), P_plus_one, strlen(P_plus_one)); + + s1len = crypto_dh_compute_secret(LOG_WARN, dh1, p1, DH_BYTES, s1, 50); + tt_int_op(-1, OP_EQ, s1len); + + p1[DH_BYTES-1] = 0x01; /* p+2 */ + s1len = crypto_dh_compute_secret(LOG_WARN, dh1, p1, DH_BYTES, s1, 50); + tt_int_op(-1, OP_EQ, s1len); + + p1[DH_BYTES-1] = 0xff; /* p+256 */ + s1len = crypto_dh_compute_secret(LOG_WARN, dh1, p1, DH_BYTES, s1, 50); + tt_int_op(-1, OP_EQ, s1len); + + memset(p1, 0xff, DH_BYTES), /* 2^1024-1 */ + s1len = crypto_dh_compute_secret(LOG_WARN, dh1, p1, DH_BYTES, s1, 50); + tt_int_op(-1, OP_EQ, s1len); + } + + { + /* provoke an error in the openssl DH_compute_key function; make sure we + * survive. */ + tt_assert(! crypto_dh_get_public(dh1, p1, DH_BYTES)); + + crypto_dh_free(dh2); + dh2= crypto_dh_new(DH_TYPE_CIRCUIT); /* no private key set */ + s1len = crypto_dh_compute_secret(LOG_WARN, dh2, + p1, DH_BYTES, + s1, 50); + tt_int_op(s1len, OP_EQ, -1); } done: crypto_dh_free(dh1); crypto_dh_free(dh2); + crypto_dh_free(dh1_dup); +} + +static void +test_crypto_openssl_version(void *arg) +{ + (void)arg; + const char *version = crypto_openssl_get_version_str(); + const char *h_version = crypto_openssl_get_header_version_str(); + tt_assert(version); + tt_assert(h_version); + tt_assert(!strcmpstart(version, h_version)); /* "-fips" suffix, etc */ + tt_assert(!strstr(version, "OpenSSL")); + int a=-1,b=-1,c=-1; + if (!strcmpstart(version, "LibreSSL") || !strcmpstart(version, "BoringSSL")) + return; + int r = tor_sscanf(version, "%d.%d.%d", &a,&b,&c); + tt_int_op(r, OP_EQ, 3); + tt_int_op(a, OP_GE, 0); + tt_int_op(b, OP_GE, 0); + tt_int_op(c, OP_GE, 0); + + done: + ; } /** Run unit tests for our random number generation function and its wrappers. @@ -73,6 +177,7 @@ test_crypto_rng(void *arg) int i, j, allok; char data1[100], data2[100]; double d; + char *h=NULL; /* Try out RNG. */ (void)arg; @@ -104,9 +209,16 @@ test_crypto_rng(void *arg) allok = 0; tor_free(host); } + + /* Make sure crypto_random_hostname clips its inputs properly. */ + h = crypto_random_hostname(20000, 9000, "www.", ".onion"); + tt_assert(! strcmpstart(h,"www.")); + tt_assert(! strcmpend(h,".onion")); + tt_int_op(63+4+6, OP_EQ, strlen(h)); + tt_assert(allok); done: - ; + tor_free(h); } static void @@ -125,14 +237,100 @@ test_crypto_rng_range(void *arg) if (x == 8) got_largest = 1; } - /* These fail with probability 1/10^603. */ tt_assert(got_smallest); tt_assert(got_largest); + + got_smallest = got_largest = 0; + const uint64_t ten_billion = 10 * ((uint64_t)1000000000000); + for (i = 0; i < 1000; ++i) { + uint64_t x = crypto_rand_uint64_range(ten_billion, ten_billion+10); + tt_u64_op(x, OP_GE, ten_billion); + tt_u64_op(x, OP_LT, ten_billion+10); + if (x == ten_billion) + got_smallest = 1; + if (x == ten_billion+9) + got_largest = 1; + } + + tt_assert(got_smallest); + tt_assert(got_largest); + + const time_t now = time(NULL); + for (i = 0; i < 2000; ++i) { + time_t x = crypto_rand_time_range(now, now+60); + tt_i64_op(x, OP_GE, now); + tt_i64_op(x, OP_LT, now+60); + if (x == now) + got_smallest = 1; + if (x == now+59) + got_largest = 1; + } + + tt_assert(got_smallest); + tt_assert(got_largest); done: ; } +static void +test_crypto_rng_strongest(void *arg) +{ + const char *how = arg; + int broken = 0; + + if (how == NULL) { + ; + } else if (!strcmp(how, "nosyscall")) { + break_strongest_rng_syscall = 1; + } else if (!strcmp(how, "nofallback")) { + break_strongest_rng_fallback = 1; + } else if (!strcmp(how, "broken")) { + broken = break_strongest_rng_syscall = break_strongest_rng_fallback = 1; + } + +#define N 128 + uint8_t combine_and[N]; + uint8_t combine_or[N]; + int i, j; + + memset(combine_and, 0xff, N); + memset(combine_or, 0, N); + + for (i = 0; i < 100; ++i) { /* 2^-100 chances just don't happen. */ + uint8_t output[N]; + memset(output, 0, N); + if (how == NULL) { + /* this one can't fail. */ + crypto_strongest_rand(output, sizeof(output)); + } else { + int r = crypto_strongest_rand_raw(output, sizeof(output)); + if (r == -1) { + if (broken) { + goto done; /* we're fine. */ + } + /* This function is allowed to break, but only if it always breaks. */ + tt_int_op(i, OP_EQ, 0); + tt_skip(); + } else { + tt_assert(! broken); + } + } + for (j = 0; j < N; ++j) { + combine_and[j] &= output[j]; + combine_or[j] |= output[j]; + } + } + + for (j = 0; j < N; ++j) { + tt_int_op(combine_and[j], OP_EQ, 0); + tt_int_op(combine_or[j], OP_EQ, 0xff); + } + done: + ; +#undef N +} + /* Test for rectifying openssl RAND engine. */ static void test_crypto_rng_engine(void *arg) @@ -312,6 +510,43 @@ test_crypto_aes(void *arg) tor_free(data3); } +static void +test_crypto_aes_ctr_testvec(void *arg) +{ + (void)arg; + char *mem_op_hex_tmp=NULL; + + /* from NIST SP800-38a, section F.5 */ + const char key16[] = "2b7e151628aed2a6abf7158809cf4f3c"; + const char ctr16[] = "f0f1f2f3f4f5f6f7f8f9fafbfcfdfeff"; + const char plaintext16[] = + "6bc1bee22e409f96e93d7e117393172a" + "ae2d8a571e03ac9c9eb76fac45af8e51" + "30c81c46a35ce411e5fbc1191a0a52ef" + "f69f2445df4f9b17ad2b417be66c3710"; + const char ciphertext16[] = + "874d6191b620e3261bef6864990db6ce" + "9806f66b7970fdff8617187bb9fffdff" + "5ae4df3edbd5d35e5b4f09020db03eab" + "1e031dda2fbe03d1792170a0f3009cee"; + + char key[16]; + char iv[16]; + char plaintext[16*4]; + base16_decode(key, sizeof(key), key16, strlen(key16)); + base16_decode(iv, sizeof(iv), ctr16, strlen(ctr16)); + base16_decode(plaintext, sizeof(plaintext), + plaintext16, strlen(plaintext16)); + + crypto_cipher_t *c = crypto_cipher_new_with_iv(key, iv); + crypto_cipher_crypt_inplace(c, plaintext, sizeof(plaintext)); + test_memeq_hex(plaintext, ciphertext16); + + done: + tor_free(mem_op_hex_tmp); + crypto_cipher_free(c); +} + /** Run unit tests for our SHA-1 functionality */ static void test_crypto_sha(void *arg) @@ -1084,6 +1319,29 @@ test_crypto_pk_base64(void *arg) tor_free(encoded); } +#ifdef HAVE_TRUNCATE +#define do_truncate truncate +#else +static int +do_truncate(const char *fname, size_t len) +{ + struct stat st; + char *bytes; + + bytes = read_file_to_str(fname, RFTS_BIN, &st); + if (!bytes) + return -1; + /* This cast isn't so great, but it should be safe given the actual files + * and lengths we're using. */ + if (st.st_size < (off_t)len) + len = MIN(len, (size_t)st.st_size); + + int r = write_bytes_to_file(fname, bytes, len, 1); + tor_free(bytes); + return r; +} +#endif + /** Sanity check for crypto pk digests */ static void test_crypto_digests(void *arg) @@ -1114,6 +1372,33 @@ test_crypto_digests(void *arg) crypto_pk_free(k); } +static void +test_crypto_digest_names(void *arg) +{ + static const struct { + int a; const char *n; + } names[] = { + { DIGEST_SHA1, "sha1" }, + { DIGEST_SHA256, "sha256" }, + { DIGEST_SHA512, "sha512" }, + { DIGEST_SHA3_256, "sha3-256" }, + { DIGEST_SHA3_512, "sha3-512" }, + { -1, NULL } + }; + (void)arg; + + int i; + for (i = 0; names[i].n; ++i) { + tt_str_op(names[i].n, OP_EQ,crypto_digest_algorithm_get_name(names[i].a)); + tt_int_op(names[i].a, + OP_EQ,crypto_digest_algorithm_parse_name(names[i].n)); + } + tt_int_op(-1, OP_EQ, + crypto_digest_algorithm_parse_name("TimeCubeHash-4444")); + done: + ; +} + #ifndef OPENSSL_1_1_API #define EVP_ENCODE_CTX_new() tor_malloc_zero(sizeof(EVP_ENCODE_CTX)) #define EVP_ENCODE_CTX_free(ctx) tor_free(ctx) @@ -1236,7 +1521,7 @@ test_crypto_formats(void *arg) strlcpy(data1, "f0d678affc000100", 1024); i = base16_decode(data2, 8, data1, 16); - tt_int_op(i,OP_EQ, 0); + tt_int_op(i,OP_EQ, 8); tt_mem_op(data2,OP_EQ, "\xf0\xd6\x78\xaf\xfc\x00\x01\x00",8); /* now try some failing base16 decodes */ @@ -1507,13 +1792,98 @@ test_crypto_hkdf_sha256(void *arg) "b206fa34e5bc78d063fc291501beec53b36e5a0e434561200c" "5f8bd13e0f88b3459600b4dc21d69363e2895321c06184879d" "94b18f078411be70b767c7fc40679a9440a0c95ea83a23efbf"); - done: tor_free(mem_op_hex_tmp); #undef EXPAND } static void +test_crypto_hkdf_sha256_testvecs(void *arg) +{ + (void) arg; + /* Test vectors from RFC5869, sections A.1 through A.3 */ + const struct { + const char *ikm16, *salt16, *info16; + int L; + const char *okm16; + } vecs[] = { + { /* from A.1 */ + "0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b", + "000102030405060708090a0b0c", + "f0f1f2f3f4f5f6f7f8f9", + 42, + "3cb25f25faacd57a90434f64d0362f2a2d2d0a90cf1a5a4c5db02d56ecc4c5bf" + "34007208d5b887185865" + }, + { /* from A.2 */ + "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f" + "202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f" + "404142434445464748494a4b4c4d4e4f", + "606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f" + "808182838485868788898a8b8c8d8e8f909192939495969798999a9b9c9d9e9f" + "a0a1a2a3a4a5a6a7a8a9aaabacadaeaf", + "b0b1b2b3b4b5b6b7b8b9babbbcbdbebfc0c1c2c3c4c5c6c7c8c9cacbcccdcecf" + "d0d1d2d3d4d5d6d7d8d9dadbdcdddedfe0e1e2e3e4e5e6e7e8e9eaebecedeeef" + "f0f1f2f3f4f5f6f7f8f9fafbfcfdfeff", + 82, + "b11e398dc80327a1c8e7f78c596a49344f012eda2d4efad8a050cc4c19afa97c" + "59045a99cac7827271cb41c65e590e09da3275600c2f09b8367793a9aca3db71" + "cc30c58179ec3e87c14c01d5c1f3434f1d87" + }, + { /* from A.3 */ + "0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b", + "", + "", + 42, + "8da4e775a563c18f715f802a063c5a31b8a11f5c5ee1879ec3454e5f3c738d2d" + "9d201395faa4b61a96c8", + }, + { NULL, NULL, NULL, -1, NULL } + }; + + int i; + char *ikm = NULL; + char *salt = NULL; + char *info = NULL; + char *okm = NULL; + char *mem_op_hex_tmp = NULL; + + for (i = 0; vecs[i].ikm16; ++i) { + size_t ikm_len = strlen(vecs[i].ikm16)/2; + size_t salt_len = strlen(vecs[i].salt16)/2; + size_t info_len = strlen(vecs[i].info16)/2; + size_t okm_len = vecs[i].L; + + ikm = tor_malloc(ikm_len); + salt = tor_malloc(salt_len); + info = tor_malloc(info_len); + okm = tor_malloc(okm_len); + + base16_decode(ikm, ikm_len, vecs[i].ikm16, strlen(vecs[i].ikm16)); + base16_decode(salt, salt_len, vecs[i].salt16, strlen(vecs[i].salt16)); + base16_decode(info, info_len, vecs[i].info16, strlen(vecs[i].info16)); + + int r = crypto_expand_key_material_rfc5869_sha256( + (const uint8_t*)ikm, ikm_len, + (const uint8_t*)salt, salt_len, + (const uint8_t*)info, info_len, + (uint8_t*)okm, okm_len); + tt_int_op(r, OP_EQ, 0); + test_memeq_hex(okm, vecs[i].okm16); + tor_free(ikm); + tor_free(salt); + tor_free(info); + tor_free(okm); + } + done: + tor_free(ikm); + tor_free(salt); + tor_free(info); + tor_free(okm); + tor_free(mem_op_hex_tmp); +} + +static void test_crypto_curve25519_impl(void *arg) { /* adapted from curve25519_donna, which adapted it from test-curve25519 @@ -1605,6 +1975,47 @@ test_crypto_curve25519_basepoint(void *arg) } static void +test_crypto_curve25519_testvec(void *arg) +{ + (void)arg; + char *mem_op_hex_tmp = NULL; + + /* From RFC 7748, section 6.1 */ + /* Alice's private key, a: */ + const char a16[] = + "77076d0a7318a57d3c16c17251b26645df4c2f87ebc0992ab177fba51db92c2a"; + /* Alice's public key, X25519(a, 9): */ + const char a_pub16[] = + "8520f0098930a754748b7ddcb43ef75a0dbf3a0d26381af4eba4a98eaa9b4e6a"; + /* Bob's private key, b: */ + const char b16[] = + "5dab087e624a8a4b79e17f8b83800ee66f3bb1292618b6fd1c2f8b27ff88e0eb"; + /* Bob's public key, X25519(b, 9): */ + const char b_pub16[] = + "de9edb7d7b7dc1b4d35b61c2ece435373f8343c85b78674dadfc7e146f882b4f"; + /* Their shared secret, K: */ + const char k16[] = + "4a5d9d5ba4ce2de1728e3bf480350f25e07e21c947d19e3376f09b3c1e161742"; + + uint8_t a[32], b[32], a_pub[32], b_pub[32], k1[32], k2[32]; + base16_decode((char*)a, sizeof(a), a16, strlen(a16)); + base16_decode((char*)b, sizeof(b), b16, strlen(b16)); + curve25519_basepoint_impl(a_pub, a); + curve25519_basepoint_impl(b_pub, b); + curve25519_impl(k1, a, b_pub); + curve25519_impl(k2, b, a_pub); + + test_memeq_hex(a, a16); + test_memeq_hex(b, b16); + test_memeq_hex(a_pub, a_pub16); + test_memeq_hex(b_pub, b_pub16); + test_memeq_hex(k1, k16); + test_memeq_hex(k2, k16); + done: + tor_free(mem_op_hex_tmp); +} + +static void test_crypto_curve25519_wrappers(void *arg) { curve25519_public_key_t pubkey1, pubkey2; @@ -1896,7 +2307,67 @@ test_crypto_ed25519_test_vectors(void *arg) "1fbc1e08682f2cc0c92efe8f4985dec61dcbd54d4b94a22547d24451271c8b00", "0a688e79be24f866286d4646b5d81c" }, - + /* These come from draft-irtf-cfrg-eddsa-05 section 7.1 */ + { + "9d61b19deffd5a60ba844af492ec2cc44449c5697b326919703bac031cae7f60", + "d75a980182b10ab7d54bfed3c964073a0ee172f3daa62325af021a68f707511a", + "e5564300c360ac729086e2cc806e828a84877f1eb8e5d974d873e06522490155" + "5fb8821590a33bacc61e39701cf9b46bd25bf5f0595bbe24655141438e7a100b", + "" + }, + { + "4ccd089b28ff96da9db6c346ec114e0f5b8a319f35aba624da8cf6ed4fb8a6fb", + "3d4017c3e843895a92b70aa74d1b7ebc9c982ccf2ec4968cc0cd55f12af4660c", + "92a009a9f0d4cab8720e820b5f642540a2b27b5416503f8fb3762223ebdb69da" + "085ac1e43e15996e458f3613d0f11d8c387b2eaeb4302aeeb00d291612bb0c00", + "72" + }, + { + "f5e5767cf153319517630f226876b86c8160cc583bc013744c6bf255f5cc0ee5", + "278117fc144c72340f67d0f2316e8386ceffbf2b2428c9c51fef7c597f1d426e", + "0aab4c900501b3e24d7cdf4663326a3a87df5e4843b2cbdb67cbf6e460fec350" + "aa5371b1508f9f4528ecea23c436d94b5e8fcd4f681e30a6ac00a9704a188a03", + "08b8b2b733424243760fe426a4b54908632110a66c2f6591eabd3345e3e4eb98" + "fa6e264bf09efe12ee50f8f54e9f77b1e355f6c50544e23fb1433ddf73be84d8" + "79de7c0046dc4996d9e773f4bc9efe5738829adb26c81b37c93a1b270b20329d" + "658675fc6ea534e0810a4432826bf58c941efb65d57a338bbd2e26640f89ffbc" + "1a858efcb8550ee3a5e1998bd177e93a7363c344fe6b199ee5d02e82d522c4fe" + "ba15452f80288a821a579116ec6dad2b3b310da903401aa62100ab5d1a36553e" + "06203b33890cc9b832f79ef80560ccb9a39ce767967ed628c6ad573cb116dbef" + "efd75499da96bd68a8a97b928a8bbc103b6621fcde2beca1231d206be6cd9ec7" + "aff6f6c94fcd7204ed3455c68c83f4a41da4af2b74ef5c53f1d8ac70bdcb7ed1" + "85ce81bd84359d44254d95629e9855a94a7c1958d1f8ada5d0532ed8a5aa3fb2" + "d17ba70eb6248e594e1a2297acbbb39d502f1a8c6eb6f1ce22b3de1a1f40cc24" + "554119a831a9aad6079cad88425de6bde1a9187ebb6092cf67bf2b13fd65f270" + "88d78b7e883c8759d2c4f5c65adb7553878ad575f9fad878e80a0c9ba63bcbcc" + "2732e69485bbc9c90bfbd62481d9089beccf80cfe2df16a2cf65bd92dd597b07" + "07e0917af48bbb75fed413d238f5555a7a569d80c3414a8d0859dc65a46128ba" + "b27af87a71314f318c782b23ebfe808b82b0ce26401d2e22f04d83d1255dc51a" + "ddd3b75a2b1ae0784504df543af8969be3ea7082ff7fc9888c144da2af58429e" + "c96031dbcad3dad9af0dcbaaaf268cb8fcffead94f3c7ca495e056a9b47acdb7" + "51fb73e666c6c655ade8297297d07ad1ba5e43f1bca32301651339e22904cc8c" + "42f58c30c04aafdb038dda0847dd988dcda6f3bfd15c4b4c4525004aa06eeff8" + "ca61783aacec57fb3d1f92b0fe2fd1a85f6724517b65e614ad6808d6f6ee34df" + "f7310fdc82aebfd904b01e1dc54b2927094b2db68d6f903b68401adebf5a7e08" + "d78ff4ef5d63653a65040cf9bfd4aca7984a74d37145986780fc0b16ac451649" + "de6188a7dbdf191f64b5fc5e2ab47b57f7f7276cd419c17a3ca8e1b939ae49e4" + "88acba6b965610b5480109c8b17b80e1b7b750dfc7598d5d5011fd2dcc5600a3" + "2ef5b52a1ecc820e308aa342721aac0943bf6686b64b2579376504ccc493d97e" + "6aed3fb0f9cd71a43dd497f01f17c0e2cb3797aa2a2f256656168e6c496afc5f" + "b93246f6b1116398a346f1a641f3b041e989f7914f90cc2c7fff357876e506b5" + "0d334ba77c225bc307ba537152f3f1610e4eafe595f6d9d90d11faa933a15ef1" + "369546868a7f3a45a96768d40fd9d03412c091c6315cf4fde7cb68606937380d" + "b2eaaa707b4c4185c32eddcdd306705e4dc1ffc872eeee475a64dfac86aba41c" + "0618983f8741c5ef68d3a101e8a3b8cac60c905c15fc910840b94c00a0b9d0" + }, + { + "833fe62409237b9d62ec77587520911e9a759cec1d19755b7da901b96dca3d42", + "ec172b93ad5e563bf4932c70e1245034c35467ef2efd4d64ebf819683467e2bf", + "dc2a4459e7369633a52b1bf277839a00201009a3efbf3ecb69bea2186c26b589" + "09351fc9ac90b3ecfdfbc7c66431e0303dca179c138ac17ad9bef1177331a704", + "ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a" + "2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f" + }, { NULL, NULL, NULL, NULL} }; @@ -2066,8 +2537,9 @@ test_crypto_ed25519_testvectors(void *arg) #define DECODE(p,s) base16_decode((char*)(p),sizeof(p),(s),strlen(s)) #define EQ(a,h) test_memeq_hex((const char*)(a), (h)) - tt_int_op(0, OP_EQ, DECODE(sk, ED25519_SECRET_KEYS[i])); - tt_int_op(0, OP_EQ, DECODE(blinding_param, ED25519_BLINDING_PARAMS[i])); + tt_int_op(sizeof(sk), OP_EQ, DECODE(sk, ED25519_SECRET_KEYS[i])); + tt_int_op(sizeof(blinding_param), OP_EQ, DECODE(blinding_param, + ED25519_BLINDING_PARAMS[i])); tt_int_op(0, OP_EQ, ed25519_secret_key_from_seed(&esk, sk)); EQ(esk.seckey, ED25519_EXPANDED_SECRET_KEYS[i]); @@ -2183,6 +2655,54 @@ test_crypto_ed25519_fuzz_donna(void *arg) } static void +test_crypto_ed25519_storage(void *arg) +{ + (void)arg; + ed25519_keypair_t *keypair = NULL; + ed25519_public_key_t pub; + ed25519_secret_key_t sec; + char *fname_1 = tor_strdup(get_fname("ed_seckey_1")); + char *fname_2 = tor_strdup(get_fname("ed_pubkey_2")); + char *contents = NULL; + char *tag = NULL; + + keypair = tor_malloc_zero(sizeof(ed25519_keypair_t)); + tt_int_op(0,OP_EQ,ed25519_keypair_generate(keypair, 0)); + tt_int_op(0,OP_EQ, + ed25519_seckey_write_to_file(&keypair->seckey, fname_1, "foo")); + tt_int_op(0,OP_EQ, + ed25519_pubkey_write_to_file(&keypair->pubkey, fname_2, "bar")); + + tt_int_op(-1, OP_EQ, ed25519_pubkey_read_from_file(&pub, &tag, fname_1)); + tt_ptr_op(tag, OP_EQ, NULL); + tt_int_op(-1, OP_EQ, ed25519_seckey_read_from_file(&sec, &tag, fname_2)); + tt_ptr_op(tag, OP_EQ, NULL); + + tt_int_op(0, OP_EQ, ed25519_pubkey_read_from_file(&pub, &tag, fname_2)); + tt_str_op(tag, OP_EQ, "bar"); + tor_free(tag); + tt_int_op(0, OP_EQ, ed25519_seckey_read_from_file(&sec, &tag, fname_1)); + tt_str_op(tag, OP_EQ, "foo"); + tor_free(tag); + + /* whitebox test: truncated keys. */ + tt_int_op(0, ==, do_truncate(fname_1, 40)); + tt_int_op(0, ==, do_truncate(fname_2, 40)); + tt_int_op(-1, OP_EQ, ed25519_pubkey_read_from_file(&pub, &tag, fname_2)); + tt_ptr_op(tag, OP_EQ, NULL); + tor_free(tag); + tt_int_op(-1, OP_EQ, ed25519_seckey_read_from_file(&sec, &tag, fname_1)); + tt_ptr_op(tag, OP_EQ, NULL); + + done: + tor_free(fname_1); + tor_free(fname_2); + tor_free(contents); + tor_free(tag); + ed25519_keypair_free(keypair); +} + +static void test_crypto_siphash(void *arg) { /* From the reference implementation, taking @@ -2398,13 +2918,23 @@ struct testcase_t crypto_tests[] = { CRYPTO_LEGACY(rng), { "rng_range", test_crypto_rng_range, 0, NULL, NULL }, { "rng_engine", test_crypto_rng_engine, TT_FORK, NULL, NULL }, + { "rng_strongest", test_crypto_rng_strongest, TT_FORK, NULL, NULL }, + { "rng_strongest_nosyscall", test_crypto_rng_strongest, TT_FORK, + &passthrough_setup, (void*)"nosyscall" }, + { "rng_strongest_nofallback", test_crypto_rng_strongest, TT_FORK, + &passthrough_setup, (void*)"nofallback" }, + { "rng_strongest_broken", test_crypto_rng_strongest, TT_FORK, + &passthrough_setup, (void*)"broken" }, + { "openssl_version", test_crypto_openssl_version, TT_FORK, NULL, NULL }, { "aes_AES", test_crypto_aes, TT_FORK, &passthrough_setup, (void*)"aes" }, { "aes_EVP", test_crypto_aes, TT_FORK, &passthrough_setup, (void*)"evp" }, + { "aes_ctr_testvec", test_crypto_aes_ctr_testvec, 0, NULL, NULL }, CRYPTO_LEGACY(sha), CRYPTO_LEGACY(pk), { "pk_fingerprints", test_crypto_pk_fingerprints, TT_FORK, NULL, NULL }, { "pk_base64", test_crypto_pk_base64, TT_FORK, NULL, NULL }, CRYPTO_LEGACY(digests), + { "digest_names", test_crypto_digest_names, 0, NULL, NULL }, { "sha3", test_crypto_sha3, TT_FORK, NULL, NULL}, { "sha3_xof", test_crypto_sha3_xof, TT_FORK, NULL, NULL}, CRYPTO_LEGACY(dh), @@ -2415,8 +2945,10 @@ struct testcase_t crypto_tests[] = { CRYPTO_LEGACY(base32_decode), { "kdf_TAP", test_crypto_kdf_TAP, 0, NULL, NULL }, { "hkdf_sha256", test_crypto_hkdf_sha256, 0, NULL, NULL }, + { "hkdf_sha256_testvecs", test_crypto_hkdf_sha256_testvecs, 0, NULL, NULL }, { "curve25519_impl", test_crypto_curve25519_impl, 0, NULL, NULL }, { "curve25519_impl_hibit", test_crypto_curve25519_impl, 0, NULL, (void*)"y"}, + { "curve25516_testvec", test_crypto_curve25519_testvec, 0, NULL, NULL }, { "curve25519_basepoint", test_crypto_curve25519_basepoint, TT_FORK, NULL, NULL }, { "curve25519_wrappers", test_crypto_curve25519_wrappers, 0, NULL, NULL }, @@ -2429,6 +2961,7 @@ struct testcase_t crypto_tests[] = { ED25519_TEST(blinding, 0), ED25519_TEST(testvectors, 0), ED25519_TEST(fuzz_donna, TT_FORK), + { "ed25519_storage", test_crypto_ed25519_storage, 0, NULL, NULL }, { "siphash", test_crypto_siphash, 0, NULL, NULL }, { "failure_modes", test_crypto_failure_modes, TT_FORK, NULL, NULL }, END_OF_TESTCASES diff --git a/src/test/test_data.c b/src/test/test_data.c index 32de54bc84..788489a097 100644 --- a/src/test/test_data.c +++ b/src/test/test_data.c @@ -3,6 +3,8 @@ * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ +#include "test.h" + /* Our unit test expect that the AUTHORITY_CERT_* public keys will sort * in this order. */ #define AUTHORITY_CERT_A AUTHORITY_CERT_3 diff --git a/src/test/test_dir.c b/src/test/test_dir.c index 26b0e72a9a..b8dcab39d5 100644 --- a/src/test/test_dir.c +++ b/src/test/test_dir.c @@ -192,7 +192,7 @@ test_dir_formats(void *arg) tt_assert(!crypto_pk_write_public_key_to_string(pk2 , &pk2_str, &pk2_str_len)); - /* XXXX025 router_dump_to_string should really take this from ri.*/ + /* XXXX+++ router_dump_to_string should really take this from ri.*/ options->ContactInfo = tor_strdup("Magri White " "<magri@elsewhere.example.com>"); /* Skip reachability checks for DirPort and tunnelled-dir-server */ @@ -580,7 +580,7 @@ test_dir_extrainfo_parsing(void *arg) crypto_pk_t *pk = ri->identity_pkey = crypto_pk_new(); \ tt_assert(! crypto_pk_read_public_key_from_string(pk, \ name##_KEY, strlen(name##_KEY))); \ - tt_int_op(0,OP_EQ,base16_decode(d, 20, name##_FP, strlen(name##_FP))); \ + tt_int_op(20,OP_EQ,base16_decode(d, 20, name##_FP, strlen(name##_FP))); \ digestmap_set((digestmap_t*)map, d, ri); \ ri = NULL; \ } while (0) @@ -2196,56 +2196,57 @@ test_dir_scale_bw(void *testdata) 1.0/7, 12.0, 24.0 }; - u64_dbl_t vals[8]; + double vals_dbl[8]; + uint64_t vals_u64[8]; uint64_t total; int i; (void) testdata; for (i=0; i<8; ++i) - vals[i].dbl = v[i]; + vals_dbl[i] = v[i]; - scale_array_elements_to_u64(vals, 8, &total); + scale_array_elements_to_u64(vals_u64, vals_dbl, 8, &total); tt_int_op((int)total, OP_EQ, 48); total = 0; for (i=0; i<8; ++i) { - total += vals[i].u64; + total += vals_u64[i]; } tt_assert(total >= (U64_LITERAL(1)<<60)); tt_assert(total <= (U64_LITERAL(1)<<62)); for (i=0; i<8; ++i) { /* vals[2].u64 is the scaled value of 1.0 */ - double ratio = ((double)vals[i].u64) / vals[2].u64; + double ratio = ((double)vals_u64[i]) / vals_u64[2]; tt_double_op(fabs(ratio - v[i]), OP_LT, .00001); } /* test handling of no entries */ total = 1; - scale_array_elements_to_u64(vals, 0, &total); + scale_array_elements_to_u64(vals_u64, vals_dbl, 0, &total); tt_assert(total == 0); /* make sure we don't read the array when we have no entries * may require compiler flags to catch NULL dereferences */ total = 1; - scale_array_elements_to_u64(NULL, 0, &total); + scale_array_elements_to_u64(NULL, NULL, 0, &total); tt_assert(total == 0); - scale_array_elements_to_u64(NULL, 0, NULL); + scale_array_elements_to_u64(NULL, NULL, 0, NULL); /* test handling of zero totals */ total = 1; - vals[0].dbl = 0.0; - scale_array_elements_to_u64(vals, 1, &total); + vals_dbl[0] = 0.0; + scale_array_elements_to_u64(vals_u64, vals_dbl, 1, &total); tt_assert(total == 0); - tt_assert(vals[0].u64 == 0); + tt_assert(vals_u64[0] == 0); - vals[0].dbl = 0.0; - vals[1].dbl = 0.0; - scale_array_elements_to_u64(vals, 2, NULL); - tt_assert(vals[0].u64 == 0); - tt_assert(vals[1].u64 == 0); + vals_dbl[0] = 0.0; + vals_dbl[1] = 0.0; + scale_array_elements_to_u64(vals_u64, vals_dbl, 2, NULL); + tt_assert(vals_u64[0] == 0); + tt_assert(vals_u64[1] == 0); done: ; @@ -2256,7 +2257,7 @@ test_dir_random_weighted(void *testdata) { int histogram[10]; uint64_t vals[10] = {3,1,2,4,6,0,7,5,8,9}, total=0; - u64_dbl_t inp[10]; + uint64_t inp_u64[10]; int i, choice; const int n = 50000; double max_sq_error; @@ -2266,12 +2267,12 @@ test_dir_random_weighted(void *testdata) * in a scrambled order to make sure we don't depend on order. */ memset(histogram,0,sizeof(histogram)); for (i=0; i<10; ++i) { - inp[i].u64 = vals[i]; + inp_u64[i] = vals[i]; total += vals[i]; } tt_u64_op(total, OP_EQ, 45); for (i=0; i<n; ++i) { - choice = choose_array_element_by_weight(inp, 10); + choice = choose_array_element_by_weight(inp_u64, 10); tt_int_op(choice, OP_GE, 0); tt_int_op(choice, OP_LT, 10); histogram[choice]++; @@ -2298,16 +2299,16 @@ test_dir_random_weighted(void *testdata) /* Now try a singleton; do we choose it? */ for (i = 0; i < 100; ++i) { - choice = choose_array_element_by_weight(inp, 1); + choice = choose_array_element_by_weight(inp_u64, 1); tt_int_op(choice, OP_EQ, 0); } /* Now try an array of zeros. We should choose randomly. */ memset(histogram,0,sizeof(histogram)); for (i = 0; i < 5; ++i) - inp[i].u64 = 0; + inp_u64[i] = 0; for (i = 0; i < n; ++i) { - choice = choose_array_element_by_weight(inp, 5); + choice = choose_array_element_by_weight(inp_u64, 5); tt_int_op(choice, OP_GE, 0); tt_int_op(choice, OP_LT, 5); histogram[choice]++; @@ -3332,13 +3333,16 @@ test_dir_download_status_schedule(void *arg) (void)arg; download_status_t dls_failure = { 0, 0, 0, DL_SCHED_GENERIC, DL_WANT_AUTHORITY, - DL_SCHED_INCREMENT_FAILURE }; + DL_SCHED_INCREMENT_FAILURE, + DL_SCHED_DETERMINISTIC, 0, 0 }; download_status_t dls_attempt = { 0, 0, 0, DL_SCHED_CONSENSUS, DL_WANT_ANY_DIRSERVER, - DL_SCHED_INCREMENT_ATTEMPT}; + DL_SCHED_INCREMENT_ATTEMPT, + DL_SCHED_DETERMINISTIC, 0, 0 }; download_status_t dls_bridge = { 0, 0, 0, DL_SCHED_BRIDGE, DL_WANT_AUTHORITY, - DL_SCHED_INCREMENT_FAILURE}; + DL_SCHED_INCREMENT_FAILURE, + DL_SCHED_DETERMINISTIC, 0, 0 }; int increment = -1; int expected_increment = -1; time_t current_time = time(NULL); @@ -3354,6 +3358,7 @@ test_dir_download_status_schedule(void *arg) delay1 = 1000; increment = download_status_schedule_get_delay(&dls_failure, schedule, + 0, INT_MAX, TIME_MIN); expected_increment = delay1; tt_assert(increment == expected_increment); @@ -3362,6 +3367,7 @@ test_dir_download_status_schedule(void *arg) delay1 = INT_MAX; increment = download_status_schedule_get_delay(&dls_failure, schedule, + 0, INT_MAX, -1); expected_increment = delay1; tt_assert(increment == expected_increment); @@ -3370,6 +3376,7 @@ test_dir_download_status_schedule(void *arg) delay1 = 0; increment = download_status_schedule_get_delay(&dls_attempt, schedule, + 0, INT_MAX, 0); expected_increment = delay1; tt_assert(increment == expected_increment); @@ -3378,6 +3385,7 @@ test_dir_download_status_schedule(void *arg) delay1 = 1000; increment = download_status_schedule_get_delay(&dls_attempt, schedule, + 0, INT_MAX, 1); expected_increment = delay1; tt_assert(increment == expected_increment); @@ -3386,6 +3394,7 @@ test_dir_download_status_schedule(void *arg) delay1 = INT_MAX; increment = download_status_schedule_get_delay(&dls_bridge, schedule, + 0, INT_MAX, current_time); expected_increment = delay1; tt_assert(increment == expected_increment); @@ -3394,6 +3403,7 @@ test_dir_download_status_schedule(void *arg) delay1 = 1; increment = download_status_schedule_get_delay(&dls_bridge, schedule, + 0, INT_MAX, TIME_MAX); expected_increment = delay1; tt_assert(increment == expected_increment); @@ -3406,6 +3416,7 @@ test_dir_download_status_schedule(void *arg) delay2 = 100; increment = download_status_schedule_get_delay(&dls_attempt, schedule, + 0, INT_MAX, current_time); expected_increment = delay2; tt_assert(increment == expected_increment); @@ -3414,6 +3425,7 @@ test_dir_download_status_schedule(void *arg) delay2 = 1; increment = download_status_schedule_get_delay(&dls_bridge, schedule, + 0, INT_MAX, current_time); expected_increment = delay2; tt_assert(increment == expected_increment); @@ -3426,6 +3438,7 @@ test_dir_download_status_schedule(void *arg) delay2 = 5; increment = download_status_schedule_get_delay(&dls_attempt, schedule, + 0, INT_MAX, current_time); expected_increment = delay2; tt_assert(increment == expected_increment); @@ -3434,6 +3447,7 @@ test_dir_download_status_schedule(void *arg) delay2 = 17; increment = download_status_schedule_get_delay(&dls_bridge, schedule, + 0, INT_MAX, current_time); expected_increment = delay2; tt_assert(increment == expected_increment); @@ -3446,6 +3460,7 @@ test_dir_download_status_schedule(void *arg) delay2 = 35; increment = download_status_schedule_get_delay(&dls_attempt, schedule, + 0, INT_MAX, current_time); expected_increment = INT_MAX; tt_assert(increment == expected_increment); @@ -3454,6 +3469,7 @@ test_dir_download_status_schedule(void *arg) delay2 = 99; increment = download_status_schedule_get_delay(&dls_bridge, schedule, + 0, INT_MAX, current_time); expected_increment = INT_MAX; tt_assert(increment == expected_increment); @@ -3465,15 +3481,58 @@ test_dir_download_status_schedule(void *arg) } static void +test_dir_download_status_random_backoff(void *arg) +{ + download_status_t dls_random = + { 0, 0, 0, DL_SCHED_GENERIC, DL_WANT_AUTHORITY, + DL_SCHED_INCREMENT_FAILURE, DL_SCHED_RANDOM_EXPONENTIAL, 0, 0 }; + int increment = -1; + int old_increment; + time_t current_time = time(NULL); + const int min_delay = 0; + const int max_delay = 1000000; + + (void)arg; + + /* Check the random backoff cases */ + old_increment = 0; + do { + increment = download_status_schedule_get_delay(&dls_random, + NULL, + min_delay, max_delay, + current_time); + /* Test */ + tt_int_op(increment, OP_GE, min_delay); + tt_int_op(increment, OP_LE, max_delay); + tt_int_op(increment, OP_GE, old_increment); + /* We at most double, and maybe add one */ + tt_int_op(increment, OP_LE, 2 * old_increment + 1); + + /* Advance */ + current_time += increment; + ++(dls_random.n_download_attempts); + ++(dls_random.n_download_failures); + + /* Try another maybe */ + old_increment = increment; + } while (increment < max_delay); + + done: + return; +} + +static void test_dir_download_status_increment(void *arg) { (void)arg; download_status_t dls_failure = { 0, 0, 0, DL_SCHED_GENERIC, DL_WANT_AUTHORITY, - DL_SCHED_INCREMENT_FAILURE }; + DL_SCHED_INCREMENT_FAILURE, + DL_SCHED_DETERMINISTIC, 0, 0 }; download_status_t dls_attempt = { 0, 0, 0, DL_SCHED_BRIDGE, DL_WANT_ANY_DIRSERVER, - DL_SCHED_INCREMENT_ATTEMPT}; + DL_SCHED_INCREMENT_ATTEMPT, + DL_SCHED_DETERMINISTIC, 0, 0 }; int delay0 = -1; int delay1 = -1; int delay2 = -1; @@ -4242,6 +4301,7 @@ struct testcase_t dir_tests[] = { DIR(fetch_type, 0), DIR(packages, 0), DIR(download_status_schedule, 0), + DIR(download_status_random_backoff, 0), DIR(download_status_increment, 0), DIR(authdir_type_to_string, 0), DIR(conn_purpose_to_string, 0), diff --git a/src/test/test_dir_common.c b/src/test/test_dir_common.c index 0b446c2dfd..2448d307b2 100644 --- a/src/test/test_dir_common.c +++ b/src/test/test_dir_common.c @@ -21,13 +21,6 @@ networkstatus_t * dir_common_add_rs_and_parse(networkstatus_t *vote, crypto_pk_t *sign_skey, int *n_vrs, time_t now, int clear_rl); -extern const char AUTHORITY_CERT_1[]; -extern const char AUTHORITY_SIGNKEY_1[]; -extern const char AUTHORITY_CERT_2[]; -extern const char AUTHORITY_SIGNKEY_2[]; -extern const char AUTHORITY_CERT_3[]; -extern const char AUTHORITY_SIGNKEY_3[]; - /** Initialize and set auth certs and keys * Returns 0 on success, -1 on failure. Clean up handled by caller. */ diff --git a/src/test/test_dir_handle_get.c b/src/test/test_dir_handle_get.c index 05657ca452..927fa8b61d 100644 --- a/src/test/test_dir_handle_get.c +++ b/src/test/test_dir_handle_get.c @@ -38,7 +38,15 @@ #include <dirent.h> #endif +#ifdef HAVE_CFLAG_WOVERLENGTH_STRINGS +DISABLE_GCC_WARNING(overlength-strings) +/* We allow huge string constants in the unit tests, but not in the code + * at large. */ +#endif #include "vote_descriptors.inc" +#ifdef HAVE_CFLAG_WOVERLENGTH_STRINGS +ENABLE_GCC_WARNING(overlength-strings) +#endif #define NS_MODULE dir_handle_get @@ -224,51 +232,6 @@ test_dir_handle_get_robots_txt(void *data) tor_free(body); } -static void -test_dir_handle_get_bytes_txt(void *data) -{ - dir_connection_t *conn = NULL; - char *header = NULL; - char *body = NULL; - size_t body_used = 0, body_len = 0; - char buff[30]; - char *exp_body = NULL; - (void) data; - - exp_body = directory_dump_request_log(); - body_len = strlen(exp_body); - - MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock); - - conn = dir_connection_new(tor_addr_family(&MOCK_TOR_ADDR)); - - tt_int_op(directory_handle_command_get(conn, GET("/tor/bytes.txt"), NULL, 0), - OP_EQ, 0); - fetch_from_buf_http(TO_CONN(conn)->outbuf, &header, MAX_HEADERS_SIZE, - &body, &body_used, body_len+1, 0); - - tt_assert(header); - tt_assert(body); - - tt_ptr_op(strstr(header, "HTTP/1.0 200 OK\r\n"), OP_EQ, header); - tt_assert(strstr(header, "Content-Type: text/plain\r\n")); - tt_assert(strstr(header, "Content-Encoding: identity\r\n")); - tt_assert(strstr(header, "Pragma: no-cache\r\n")); - - tor_snprintf(buff, sizeof(buff), "Content-Length: %ld\r\n", (long) body_len); - tt_assert(strstr(header, buff)); - - tt_int_op(body_used, OP_EQ, strlen(body)); - tt_str_op(body, OP_EQ, exp_body); - - done: - UNMOCK(connection_write_to_buf_impl_); - connection_free_(TO_CONN(conn)); - tor_free(header); - tor_free(body); - tor_free(exp_body); -} - #define RENDEZVOUS2_GET(descid) GET("/tor/rendezvous2/" descid) static void test_dir_handle_get_rendezvous2_not_found_if_not_encrypted(void *data) @@ -438,7 +401,7 @@ test_dir_handle_get_rendezvous2_on_encrypted_conn_success(void *data) TO_CONN(conn)->linked = 1; tt_assert(connection_dir_is_encrypted(conn)); - sprintf(req, RENDEZVOUS2_GET("%s"), desc_id_base32); + tor_snprintf(req, sizeof(req), RENDEZVOUS2_GET("%s"), desc_id_base32); tt_int_op(directory_handle_command_get(conn, req, NULL, 0), OP_EQ, 0); @@ -453,7 +416,7 @@ test_dir_handle_get_rendezvous2_on_encrypted_conn_success(void *data) tt_assert(strstr(header, "Content-Type: text/plain\r\n")); tt_assert(strstr(header, "Content-Encoding: identity\r\n")); tt_assert(strstr(header, "Pragma: no-cache\r\n")); - sprintf(buff, "Content-Length: %ld\r\n", (long) body_len); + tor_snprintf(buff, sizeof(buff), "Content-Length: %ld\r\n", (long) body_len); tt_assert(strstr(header, buff)); tt_int_op(body_used, OP_EQ, strlen(body)); @@ -565,7 +528,7 @@ test_dir_handle_get_micro_d(void *data) /* Make the request */ conn = dir_connection_new(tor_addr_family(&MOCK_TOR_ADDR)); - sprintf(path, MICRODESC_GET("%s"), digest_base64); + tor_snprintf(path, sizeof(path), MICRODESC_GET("%s"), digest_base64); tt_int_op(directory_handle_command_get(conn, path, NULL, 0), OP_EQ, 0); fetch_from_buf_http(TO_CONN(conn)->outbuf, &header, MAX_HEADERS_SIZE, @@ -635,7 +598,7 @@ test_dir_handle_get_micro_d_server_busy(void *data) /* Make the request */ conn = dir_connection_new(tor_addr_family(&MOCK_TOR_ADDR)); - sprintf(path, MICRODESC_GET("%s"), digest_base64); + tor_snprintf(path, sizeof(path), MICRODESC_GET("%s"), digest_base64); tt_int_op(directory_handle_command_get(conn, path, NULL, 0), OP_EQ, 0); fetch_from_buf_http(TO_CONN(conn)->outbuf, &header, MAX_HEADERS_SIZE, @@ -997,7 +960,8 @@ test_dir_handle_get_server_descriptors_fp(void* data) DIGEST_LEN); char req[155]; - sprintf(req, SERVER_DESC_GET("fp/%s+" HEX1 "+" HEX2), hex_digest); + tor_snprintf(req, sizeof(req), SERVER_DESC_GET("fp/%s+" HEX1 "+" HEX2), + hex_digest); tt_int_op(directory_handle_command_get(conn, req, NULL, 0), OP_EQ, 0); //TODO: Is this a BUG? @@ -1056,8 +1020,9 @@ test_dir_handle_get_server_descriptors_d(void* data) conn = dir_connection_new(tor_addr_family(&MOCK_TOR_ADDR)); - char req_header[155]; - sprintf(req_header, SERVER_DESC_GET("d/%s+" HEX1 "+" HEX2), hex_digest); + char req_header[155]; /* XXX Why 155? What kind of number is that?? */ + tor_snprintf(req_header, sizeof(req_header), + SERVER_DESC_GET("d/%s+" HEX1 "+" HEX2), hex_digest); tt_int_op(directory_handle_command_get(conn, req_header, NULL, 0), OP_EQ, 0); //TODO: Is this a BUG? @@ -1125,8 +1090,9 @@ test_dir_handle_get_server_descriptors_busy(void* data) #define HEX1 "Fe0daff89127389bc67558691231234551193EEE" #define HEX2 "Deadbeef99999991111119999911111111f00ba4" - char req_header[155]; - sprintf(req_header, SERVER_DESC_GET("d/%s+" HEX1 "+" HEX2), hex_digest); + char req_header[155]; /* XXX 155? Why 155? */ + tor_snprintf(req_header, sizeof(req_header), + SERVER_DESC_GET("d/%s+" HEX1 "+" HEX2), hex_digest); tt_int_op(directory_handle_command_get(conn, req_header, NULL, 0), OP_EQ, 0); fetch_from_buf_http(TO_CONN(conn)->outbuf, &header, MAX_HEADERS_SIZE, @@ -1204,8 +1170,6 @@ test_dir_handle_get_server_keys_all_not_found(void* data) #define TEST_CERTIFICATE AUTHORITY_CERT_3 #define TEST_SIGNING_KEY AUTHORITY_SIGNKEY_A_DIGEST -extern const char AUTHORITY_CERT_3[]; -extern const char AUTHORITY_SIGNKEY_A_DIGEST[]; static const char TEST_CERT_IDENT_KEY[] = "D867ACF56A9D229B35C25F0090BC9867E906BE69"; @@ -1237,7 +1201,7 @@ test_dir_handle_get_server_keys_all(void* data) base16_decode(ds->v3_identity_digest, DIGEST_LEN, TEST_CERT_IDENT_KEY, HEX_DIGEST_LEN); tt_int_op(0, OP_EQ, trusted_dirs_load_certs_from_string(TEST_CERTIFICATE, - TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST, 1)); + TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST, 1, NULL)); conn = dir_connection_new(tor_addr_family(&MOCK_TOR_ADDR)); @@ -1396,11 +1360,12 @@ test_dir_handle_get_server_keys_fp(void* data) TEST_CERT_IDENT_KEY, HEX_DIGEST_LEN); tt_int_op(0, OP_EQ, trusted_dirs_load_certs_from_string(TEST_CERTIFICATE, - TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST, 1)); + TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST, 1, NULL)); conn = dir_connection_new(tor_addr_family(&MOCK_TOR_ADDR)); char req[71]; - sprintf(req, GET("/tor/keys/fp/%s"), TEST_CERT_IDENT_KEY); + tor_snprintf(req, sizeof(req), + GET("/tor/keys/fp/%s"), TEST_CERT_IDENT_KEY); tt_int_op(directory_handle_command_get(conn, req, NULL, 0), OP_EQ, 0); fetch_from_buf_http(TO_CONN(conn)->outbuf, &header, MAX_HEADERS_SIZE, @@ -1468,11 +1433,12 @@ test_dir_handle_get_server_keys_sk(void* data) routerlist_free_all(); tt_int_op(0, OP_EQ, trusted_dirs_load_certs_from_string(TEST_CERTIFICATE, - TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST, 1)); + TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST, 1, NULL)); conn = dir_connection_new(tor_addr_family(&MOCK_TOR_ADDR)); char req[71]; - sprintf(req, GET("/tor/keys/sk/%s"), TEST_SIGNING_KEY); + tor_snprintf(req, sizeof(req), + GET("/tor/keys/sk/%s"), TEST_SIGNING_KEY); tt_int_op(directory_handle_command_get(conn, req, NULL, 0), OP_EQ, 0); fetch_from_buf_http(TO_CONN(conn)->outbuf, &header, MAX_HEADERS_SIZE, @@ -1550,13 +1516,14 @@ test_dir_handle_get_server_keys_fpsk(void* data) dir_server_add(ds); tt_int_op(0, OP_EQ, trusted_dirs_load_certs_from_string(TEST_CERTIFICATE, - TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST, 1)); + TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST, 1, NULL)); conn = dir_connection_new(tor_addr_family(&MOCK_TOR_ADDR)); char req[115]; - sprintf(req, GET("/tor/keys/fp-sk/%s-%s"), - TEST_CERT_IDENT_KEY, TEST_SIGNING_KEY); + tor_snprintf(req, sizeof(req), + GET("/tor/keys/fp-sk/%s-%s"), + TEST_CERT_IDENT_KEY, TEST_SIGNING_KEY); tt_int_op(directory_handle_command_get(conn, req, NULL, 0), OP_EQ, 0); @@ -1606,7 +1573,7 @@ test_dir_handle_get_server_keys_busy(void* data) dir_server_add(ds); tt_int_op(0, OP_EQ, trusted_dirs_load_certs_from_string(TEST_CERTIFICATE, - TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST, 1)); + TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST, 1, NULL)); MOCK(get_options, mock_get_options); MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock); @@ -1617,7 +1584,7 @@ test_dir_handle_get_server_keys_busy(void* data) conn = dir_connection_new(tor_addr_family(&MOCK_TOR_ADDR)); char req[71]; - sprintf(req, GET("/tor/keys/fp/%s"), TEST_CERT_IDENT_KEY); + tor_snprintf(req, sizeof(req), GET("/tor/keys/fp/%s"), TEST_CERT_IDENT_KEY); tt_int_op(directory_handle_command_get(conn, req, NULL, 0), OP_EQ, 0); fetch_from_buf_http(TO_CONN(conn)->outbuf, &header, MAX_HEADERS_SIZE, @@ -2344,7 +2311,7 @@ test_dir_handle_get_status_vote_next_authority(void* data) base16_decode(ds->v3_identity_digest, DIGEST_LEN, TEST_CERT_IDENT_KEY, HEX_DIGEST_LEN); tt_int_op(0, OP_EQ, trusted_dirs_load_certs_from_string(TEST_CERTIFICATE, - TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST, 1)); + TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST, 1, NULL)); init_mock_options(); mock_options->AuthoritativeDir = 1; @@ -2423,7 +2390,7 @@ test_dir_handle_get_status_vote_current_authority(void* data) TEST_CERT_IDENT_KEY, HEX_DIGEST_LEN); tt_int_op(0, OP_EQ, trusted_dirs_load_certs_from_string(TEST_CERTIFICATE, - TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST, 1)); + TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST, 1, NULL)); init_mock_options(); mock_options->AuthoritativeDir = 1; @@ -2484,7 +2451,6 @@ struct testcase_t dir_handle_get_tests[] = { DIR_HANDLE_CMD(v1_command_not_found, 0), DIR_HANDLE_CMD(v1_command, 0), DIR_HANDLE_CMD(robots_txt, 0), - DIR_HANDLE_CMD(bytes_txt, 0), DIR_HANDLE_CMD(rendezvous2_not_found_if_not_encrypted, 0), DIR_HANDLE_CMD(rendezvous2_not_found, 0), DIR_HANDLE_CMD(rendezvous2_on_encrypted_conn_with_invalid_desc_id, 0), diff --git a/src/test/test_guardfraction.c b/src/test/test_guardfraction.c index 300590a3d9..130aff11aa 100644 --- a/src/test/test_guardfraction.c +++ b/src/test/test_guardfraction.c @@ -40,7 +40,7 @@ gen_vote_routerstatus_for_tests(const char *digest_in_hex, int is_guard) tt_int_op(strlen(digest_in_hex), ==, HEX_DIGEST_LEN); retval = base16_decode(digest_tmp, sizeof(digest_tmp), digest_in_hex, HEX_DIGEST_LEN); - tt_int_op(retval, ==, 0); + tt_int_op(retval, ==, sizeof(digest_tmp)); memcpy(rs->identity_digest, digest_tmp, DIGEST_LEN); } diff --git a/src/test/test_handles.c b/src/test/test_handles.c new file mode 100644 index 0000000000..536a478689 --- /dev/null +++ b/src/test/test_handles.c @@ -0,0 +1,95 @@ +/* Copyright (c) 2016, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#include "test.h" + +#include "util.h" +#include "handles.h" + +typedef struct demo_t { + HANDLE_ENTRY(demo, demo_t); + int val; +} demo_t; + +HANDLE_DECL(demo, demo_t, static) +HANDLE_IMPL(demo, demo_t, static) + +static demo_t * +demo_new(int val) +{ + demo_t *d = tor_malloc_zero(sizeof(demo_t)); + d->val = val; + return d; +} + +static void +demo_free(demo_t *d) +{ + if (d == NULL) + return; + demo_handles_clear(d); + tor_free(d); +} + +static void +test_handle_basic(void *arg) +{ + (void) arg; + demo_t *d1 = NULL, *d2 = NULL; + demo_handle_t *wr1 = NULL, *wr2 = NULL, *wr3 = NULL, *wr4 = NULL; + + d1 = demo_new(9000); + d2 = demo_new(9009); + + wr1 = demo_handle_new(d1); + wr2 = demo_handle_new(d1); + wr3 = demo_handle_new(d1); + wr4 = demo_handle_new(d2); + + tt_assert(wr1); + tt_assert(wr2); + tt_assert(wr3); + tt_assert(wr4); + + tt_ptr_op(demo_handle_get(wr1), OP_EQ, d1); + tt_ptr_op(demo_handle_get(wr2), OP_EQ, d1); + tt_ptr_op(demo_handle_get(wr3), OP_EQ, d1); + tt_ptr_op(demo_handle_get(wr4), OP_EQ, d2); + + demo_handle_free(wr1); + wr1 = NULL; + tt_ptr_op(demo_handle_get(wr2), OP_EQ, d1); + tt_ptr_op(demo_handle_get(wr3), OP_EQ, d1); + tt_ptr_op(demo_handle_get(wr4), OP_EQ, d2); + + demo_free(d1); + d1 = NULL; + tt_ptr_op(demo_handle_get(wr2), OP_EQ, NULL); + tt_ptr_op(demo_handle_get(wr3), OP_EQ, NULL); + tt_ptr_op(demo_handle_get(wr4), OP_EQ, d2); + + demo_handle_free(wr2); + wr2 = NULL; + tt_ptr_op(demo_handle_get(wr3), OP_EQ, NULL); + tt_ptr_op(demo_handle_get(wr4), OP_EQ, d2); + + demo_handle_free(wr3); + wr3 = NULL; + done: + demo_handle_free(wr1); + demo_handle_free(wr2); + demo_handle_free(wr3); + demo_handle_free(wr4); + demo_free(d1); + demo_free(d2); +} + +#define HANDLE_TEST(name, flags) \ + { #name, test_handle_ ##name, (flags), NULL, NULL } + +struct testcase_t handle_tests[] = { + HANDLE_TEST(basic, 0), + END_OF_TESTCASES +}; + diff --git a/src/test/test_helpers.c b/src/test/test_helpers.c index c6daaf220a..ae9fc7a243 100644 --- a/src/test/test_helpers.c +++ b/src/test/test_helpers.c @@ -16,7 +16,15 @@ #include "test.h" #include "test_helpers.h" +#ifdef HAVE_CFLAG_WOVERLENGTH_STRINGS +DISABLE_GCC_WARNING(overlength-strings) +/* We allow huge string constants in the unit tests, but not in the code + * at large. */ +#endif #include "test_descriptors.inc" +#ifdef HAVE_CFLAG_WOVERLENGTH_STRINGS +ENABLE_GCC_WARNING(overlength-strings) +#endif /* Return a statically allocated string representing yesterday's date * in ISO format. We use it so that state file items are not found to diff --git a/src/test/test_hs.c b/src/test/test_hs.c index 49939a53cf..1daa1552e9 100644 --- a/src/test/test_hs.c +++ b/src/test/test_hs.c @@ -435,6 +435,67 @@ test_hs_rend_data(void *arg) rend_data_free(client_dup); } +/* Test encoding and decoding service authorization cookies */ +static void +test_hs_auth_cookies(void *arg) +{ +#define TEST_COOKIE_RAW ((const uint8_t *) "abcdefghijklmnop") +#define TEST_COOKIE_ENCODED "YWJjZGVmZ2hpamtsbW5vcA" +#define TEST_COOKIE_ENCODED_STEALTH "YWJjZGVmZ2hpamtsbW5vcB" +#define TEST_COOKIE_ENCODED_INVALID "YWJjZGVmZ2hpamtsbW5vcD" + + char *encoded_cookie; + uint8_t raw_cookie[REND_DESC_COOKIE_LEN]; + rend_auth_type_t auth_type; + char *err_msg; + int re; + + (void)arg; + + /* Test that encoding gives the expected result */ + encoded_cookie = rend_auth_encode_cookie(TEST_COOKIE_RAW, REND_BASIC_AUTH); + tt_str_op(encoded_cookie, OP_EQ, TEST_COOKIE_ENCODED); + tor_free(encoded_cookie); + + encoded_cookie = rend_auth_encode_cookie(TEST_COOKIE_RAW, REND_STEALTH_AUTH); + tt_str_op(encoded_cookie, OP_EQ, TEST_COOKIE_ENCODED_STEALTH); + tor_free(encoded_cookie); + + /* Decoding should give the original value */ + re = rend_auth_decode_cookie(TEST_COOKIE_ENCODED, raw_cookie, &auth_type, + &err_msg); + tt_assert(!re); + tt_assert(!err_msg); + tt_mem_op(raw_cookie, OP_EQ, TEST_COOKIE_RAW, REND_DESC_COOKIE_LEN); + tt_int_op(auth_type, OP_EQ, REND_BASIC_AUTH); + memset(raw_cookie, 0, sizeof(raw_cookie)); + + re = rend_auth_decode_cookie(TEST_COOKIE_ENCODED_STEALTH, raw_cookie, + &auth_type, &err_msg); + tt_assert(!re); + tt_assert(!err_msg); + tt_mem_op(raw_cookie, OP_EQ, TEST_COOKIE_RAW, REND_DESC_COOKIE_LEN); + tt_int_op(auth_type, OP_EQ, REND_STEALTH_AUTH); + memset(raw_cookie, 0, sizeof(raw_cookie)); + + /* Decoding with padding characters should also work */ + re = rend_auth_decode_cookie(TEST_COOKIE_ENCODED "==", raw_cookie, NULL, + &err_msg); + tt_assert(!re); + tt_assert(!err_msg); + tt_mem_op(raw_cookie, OP_EQ, TEST_COOKIE_RAW, REND_DESC_COOKIE_LEN); + + /* Decoding with an unknown type should fail */ + re = rend_auth_decode_cookie(TEST_COOKIE_ENCODED_INVALID, raw_cookie, + &auth_type, &err_msg); + tt_int_op(re, OP_LT, 0); + tt_assert(err_msg); + tor_free(err_msg); + + done: + return; +} + struct testcase_t hs_tests[] = { { "hs_rend_data", test_hs_rend_data, TT_FORK, NULL, NULL }, @@ -445,6 +506,8 @@ struct testcase_t hs_tests[] = { { "pick_bad_tor2web_rendezvous_node", test_pick_bad_tor2web_rendezvous_node, TT_FORK, NULL, NULL }, + { "hs_auth_cookies", test_hs_auth_cookies, TT_FORK, + NULL, NULL }, END_OF_TESTCASES }; diff --git a/src/test/test_introduce.c b/src/test/test_introduce.c index 9c7a86da66..810b03c93d 100644 --- a/src/test/test_introduce.c +++ b/src/test/test_introduce.c @@ -9,8 +9,6 @@ #define RENDSERVICE_PRIVATE #include "rendservice.h" -extern const char AUTHORITY_SIGNKEY_1[]; - static uint8_t v0_test_plaintext[] = /* 20 bytes of rendezvous point nickname */ { 0x4e, 0x69, 0x63, 0x6b, 0x6e, 0x61, 0x6d, 0x65, diff --git a/src/test/test_link_handshake.c b/src/test/test_link_handshake.c index e8856c60de..4038783459 100644 --- a/src/test/test_link_handshake.c +++ b/src/test/test_link_handshake.c @@ -16,7 +16,7 @@ #include "test.h" -var_cell_t *mock_got_var_cell = NULL; +static var_cell_t *mock_got_var_cell = NULL; static void mock_write_var_cell(const var_cell_t *vc, or_connection_t *conn) diff --git a/src/test/test_logging.c b/src/test/test_logging.c index eb294fe6f8..15471e46d0 100644 --- a/src/test/test_logging.c +++ b/src/test/test_logging.c @@ -127,9 +127,47 @@ test_sigsafe_err(void *arg) smartlist_free(lines); } +static void +test_ratelim(void *arg) +{ + (void) arg; + ratelim_t ten_min = RATELIM_INIT(10*60); + + const time_t start = 1466091600; + time_t now = start; + /* Initially, we're ready. */ + + char *msg = NULL; + + msg = rate_limit_log(&ten_min, now); + tt_assert(msg != NULL); + tt_str_op(msg, OP_EQ, ""); /* nothing was suppressed. */ + + tt_int_op(ten_min.last_allowed, OP_EQ, now); + tor_free(msg); + + int i; + for (i = 0; i < 9; ++i) { + now += 60; /* one minute has passed. */ + msg = rate_limit_log(&ten_min, now); + tt_assert(msg == NULL); + tt_int_op(ten_min.last_allowed, OP_EQ, start); + tt_int_op(ten_min.n_calls_since_last_time, OP_EQ, i + 1); + } + + now += 240; /* Okay, we can be done. */ + msg = rate_limit_log(&ten_min, now); + tt_assert(msg != NULL); + tt_str_op(msg, OP_EQ, + " [9 similar message(s) suppressed in last 600 seconds]"); + done: + tor_free(msg); +} + struct testcase_t logging_tests[] = { { "sigsafe_err_fds", test_get_sigsafe_err_fds, TT_FORK, NULL, NULL }, { "sigsafe_err", test_sigsafe_err, TT_FORK, NULL, NULL }, + { "ratelim", test_ratelim, 0, NULL, NULL }, END_OF_TESTCASES }; diff --git a/src/test/test_microdesc.c b/src/test/test_microdesc.c index dbd1e5ac48..2afbdde88a 100644 --- a/src/test/test_microdesc.c +++ b/src/test/test_microdesc.c @@ -14,30 +14,11 @@ #include "test.h" -#ifdef __GNUC__ -#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -#endif - -#if __GNUC__ && GCC_VERSION >= 402 -#if GCC_VERSION >= 406 -#pragma GCC diagnostic push -#endif -/* Some versions of OpenSSL declare X509_STORE_CTX_set_verify_cb twice. - * Suppress the GCC warning so we can build with -Wredundant-decl. */ -#pragma GCC diagnostic ignored "-Wredundant-decls" -#endif - +DISABLE_GCC_WARNING(redundant-decls) #include <openssl/rsa.h> #include <openssl/bn.h> #include <openssl/pem.h> - -#if __GNUC__ && GCC_VERSION >= 402 -#if GCC_VERSION >= 406 -#pragma GCC diagnostic pop -#else -#pragma GCC diagnostic warning "-Wredundant-decls" -#endif -#endif +ENABLE_GCC_WARNING(redundant-decls) #ifdef _WIN32 /* For mkdir() */ @@ -511,6 +492,11 @@ test_md_generate(void *arg) routerinfo_free(ri); } +#ifdef HAVE_CFLAG_WOVERLENGTH_STRINGS +DISABLE_GCC_WARNING(overlength-strings) +/* We allow huge string constants in the unit tests, but not in the code + * at large. */ +#endif /* Taken at random from my ~/.tor/cached-microdescs file and then * hand-munged */ static const char MD_PARSE_TEST_DATA[] = @@ -666,6 +652,9 @@ static const char MD_PARSE_TEST_DATA[] = "id rsa1024 2A8wYpHxnkKJ92orocvIQBzeHlE\n" "p6 allow 80\n" ; +#ifdef HAVE_CFLAG_WOVERLENGTH_STRINGS +ENABLE_GCC_WARNING(overlength-strings) +#endif /** More tests for parsing different kinds of microdescriptors, and getting * invalid digests trackd from them. */ @@ -794,7 +783,8 @@ test_md_reject_cache(void *arg) mc = get_microdesc_cache(); #define ADD(hex) \ do { \ - tt_int_op(0,OP_EQ,base16_decode(buf,sizeof(buf),hex,strlen(hex))); \ + tt_int_op(sizeof(buf),OP_EQ,base16_decode(buf,sizeof(buf), \ + hex,strlen(hex)));\ smartlist_add(wanted, tor_memdup(buf, DIGEST256_LEN)); \ } while (0) diff --git a/src/test/test_ntor_cl.c b/src/test/test_ntor_cl.c index 6df123162e..a560e5fc5e 100644 --- a/src/test/test_ntor_cl.c +++ b/src/test/test_ntor_cl.c @@ -21,7 +21,7 @@ } STMT_END #define BASE16(idx, var, n) STMT_BEGIN { \ const char *s = argv[(idx)]; \ - if (base16_decode((char*)var, n, s, strlen(s)) < 0 ) { \ + if (base16_decode((char*)var, n, s, strlen(s)) < (int)n ) { \ fprintf(stderr, "couldn't decode argument %d (%s)\n",idx,s); \ return 1; \ } \ @@ -153,7 +153,10 @@ main(int argc, char **argv) if (argc < 2) { fprintf(stderr, "I need arguments. Read source for more info.\n"); return 1; - } else if (!strcmp(argv[1], "client1")) { + } + + curve25519_init(); + if (!strcmp(argv[1], "client1")) { return client1(argc, argv); } else if (!strcmp(argv[1], "server1")) { return server1(argc, argv); diff --git a/src/test/test_options.c b/src/test/test_options.c index 4f24757a85..8d1d6f901e 100644 --- a/src/test/test_options.c +++ b/src/test/test_options.c @@ -12,7 +12,7 @@ #define ROUTERSET_PRIVATE #include "routerset.h" - +#include "main.h" #include "log_test_helpers.h" #include "sandbox.h" @@ -513,8 +513,9 @@ test_options_validate__nickname(void *ignored) ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg); tt_int_op(ret, OP_EQ, -1); tt_str_op(msg, OP_EQ, - "Nickname 'ThisNickNameIsABitTooLong' is wrong length or" - " contains illegal characters."); + "Nickname 'ThisNickNameIsABitTooLong', nicknames must be between " + "1 and 19 characters inclusive, and must contain only the " + "characters [a-zA-Z0-9]."); tor_free(msg); free_options_test_data(tdata); @@ -571,8 +572,6 @@ test_options_validate__contactinfo(void *ignored) tor_free(msg); } -extern int quiet_level; - static void test_options_validate__logs(void *ignored) { diff --git a/src/test/test_policy.c b/src/test/test_policy.c index a939ebf54f..913a2f303a 100644 --- a/src/test/test_policy.c +++ b/src/test/test_policy.c @@ -778,8 +778,8 @@ test_policies_reject_port_address(void *arg) UNMOCK(get_configured_ports); } -smartlist_t *mock_ipv4_addrs = NULL; -smartlist_t *mock_ipv6_addrs = NULL; +static smartlist_t *mock_ipv4_addrs = NULL; +static smartlist_t *mock_ipv6_addrs = NULL; /* mock get_interface_address6_list, returning a deep copy of the template * address list ipv4_interface_address_list or ipv6_interface_address_list */ diff --git a/src/test/test_pubsub.c b/src/test/test_pubsub.c new file mode 100644 index 0000000000..547d6c6b32 --- /dev/null +++ b/src/test/test_pubsub.c @@ -0,0 +1,85 @@ +/* Copyright (c) 2016, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file test_pubsub.c + * \brief Unit tests for publish-subscribe abstraction. + **/ + +#include "or.h" +#include "test.h" +#include "pubsub.h" + +DECLARE_PUBSUB_STRUCT_TYPES(foobar) +DECLARE_PUBSUB_TOPIC(foobar) +DECLARE_NOTIFY_PUBSUB_TOPIC(static, foobar) +IMPLEMENT_PUBSUB_TOPIC(static, foobar) + +struct foobar_event_data_t { + unsigned u; + const char *s; +}; + +struct foobar_subscriber_data_t { + const char *name; + long l; +}; + +static int +foobar_sub1(foobar_event_data_t *ev, foobar_subscriber_data_t *mine) +{ + ev->u += 10; + mine->l += 100; + return 0; +} + +static int +foobar_sub2(foobar_event_data_t *ev, foobar_subscriber_data_t *mine) +{ + ev->u += 5; + mine->l += 50; + return 0; +} + +static void +test_pubsub_basic(void *arg) +{ + (void)arg; + foobar_subscriber_data_t subdata1 = { "hi", 0 }; + foobar_subscriber_data_t subdata2 = { "wow", 0 }; + const foobar_subscriber_t *sub1; + const foobar_subscriber_t *sub2; + foobar_event_data_t ed = { 0, "x" }; + foobar_event_data_t ed2 = { 0, "y" }; + sub1 = foobar_subscribe(foobar_sub1, &subdata1, SUBSCRIBE_ATSTART, 100); + tt_assert(sub1); + + foobar_notify(&ed, 0); + tt_int_op(subdata1.l, OP_EQ, 100); + tt_int_op(subdata2.l, OP_EQ, 0); + tt_int_op(ed.u, OP_EQ, 10); + + sub2 = foobar_subscribe(foobar_sub2, &subdata2, 0, 5); + tt_assert(sub2); + + foobar_notify(&ed2, 0); + tt_int_op(subdata1.l, OP_EQ, 200); + tt_int_op(subdata2.l, OP_EQ, 50); + tt_int_op(ed2.u, OP_EQ, 15); + + foobar_unsubscribe(sub1); + + foobar_notify(&ed, 0); + tt_int_op(subdata1.l, OP_EQ, 200); + tt_int_op(subdata2.l, OP_EQ, 100); + tt_int_op(ed.u, OP_EQ, 15); + + done: + foobar_clear(); +} + +struct testcase_t pubsub_tests[] = { + { "pubsub_basic", test_pubsub_basic, TT_FORK, NULL, NULL }, + END_OF_TESTCASES +}; + diff --git a/src/test/test_relaycell.c b/src/test/test_relaycell.c index 1cd9ff064b..fb6748965a 100644 --- a/src/test/test_relaycell.c +++ b/src/test/test_relaycell.c @@ -95,7 +95,7 @@ test_relaycell_resolved(void *arg) tt_int_op(srm_ncalls, OP_EQ, 1); \ tt_ptr_op(srm_conn, OP_EQ, entryconn); \ tt_int_op(srm_atype, OP_EQ, (atype)); \ - if (answer) { \ + if ((answer) != NULL) { \ tt_int_op(srm_alen, OP_EQ, sizeof(answer)-1); \ tt_int_op(srm_alen, OP_LT, 512); \ tt_int_op(srm_answer_is_set, OP_EQ, 1); \ diff --git a/src/test/test_rendcache.c b/src/test/test_rendcache.c index d1b52649b2..c8279fcced 100644 --- a/src/test/test_rendcache.c +++ b/src/test/test_rendcache.c @@ -17,13 +17,8 @@ static const int RECENT_TIME = -10; static const int TIME_IN_THE_PAST = -(REND_CACHE_MAX_AGE + \ - REND_CACHE_MAX_SKEW + 10); -static const int TIME_IN_THE_FUTURE = REND_CACHE_MAX_SKEW + 10; - -extern strmap_t *rend_cache; -extern digestmap_t *rend_cache_v2_dir; -extern strmap_t *rend_cache_failure; -extern size_t rend_cache_total_allocation; + REND_CACHE_MAX_SKEW + 60); +static const int TIME_IN_THE_FUTURE = REND_CACHE_MAX_SKEW + 60; static rend_data_t * mock_rend_data(const char *onion_address) diff --git a/src/test/test_routerlist.c b/src/test/test_routerlist.c index 2cffa6e801..34b70ac8a8 100644 --- a/src/test/test_routerlist.c +++ b/src/test/test_routerlist.c @@ -24,13 +24,6 @@ #include "test.h" #include "test_dir_common.h" -extern const char AUTHORITY_CERT_1[]; -extern const char AUTHORITY_SIGNKEY_1[]; -extern const char AUTHORITY_CERT_2[]; -extern const char AUTHORITY_SIGNKEY_2[]; -extern const char AUTHORITY_CERT_3[]; -extern const char AUTHORITY_SIGNKEY_3[]; - void construct_consensus(char **consensus_text_md); /* 4 digests + 3 sep + pre + post + NULL */ @@ -423,7 +416,7 @@ test_router_pick_directory_server_impl(void *arg) networkstatus_vote_free(con_md); } -connection_t *mocked_connection = NULL; +static connection_t *mocked_connection = NULL; /* Mock connection_get_by_type_addr_port_purpose by returning * mocked_connection. */ diff --git a/src/test/test_routerset.c b/src/test/test_routerset.c index 74b39c0486..1b526d430b 100644 --- a/src/test/test_routerset.c +++ b/src/test/test_routerset.c @@ -432,7 +432,7 @@ NS(test_main)(void *arg) NS_DECL(addr_policy_t *, router_parse_addr_policy_item_from_string, (const char *s, int assume_action, int *malformed_list)); -addr_policy_t *NS(mock_addr_policy); +static addr_policy_t *NS(mock_addr_policy); static void NS(test_main)(void *arg) @@ -480,7 +480,7 @@ NS(router_parse_addr_policy_item_from_string)(const char *s, NS_DECL(addr_policy_t *, router_parse_addr_policy_item_from_string, (const char *s, int assume_action, int *bogus)); -addr_policy_t *NS(mock_addr_policy); +static addr_policy_t *NS(mock_addr_policy); static void NS(test_main)(void *arg) @@ -527,7 +527,7 @@ NS(router_parse_addr_policy_item_from_string)(const char *s, int assume_action, NS_DECL(addr_policy_t *, router_parse_addr_policy_item_from_string, (const char *s, int assume_action, int *bad)); -addr_policy_t *NS(mock_addr_policy); +static addr_policy_t *NS(mock_addr_policy); static void NS(test_main)(void *arg) @@ -1477,7 +1477,7 @@ NS(test_main)(void *arg) * routerset or routerinfo. */ -node_t NS(mock_node); +static node_t NS(mock_node); static void NS(test_main)(void *arg) @@ -1504,7 +1504,7 @@ NS(test_main)(void *arg) * routerset and no routerinfo. */ -node_t NS(mock_node); +static node_t NS(mock_node); static void NS(test_main)(void *arg) @@ -1603,7 +1603,7 @@ NS(test_main)(void *arg) NS_DECL(const node_t *, node_get_by_nickname, (const char *nickname, int warn_if_unused)); -const char *NS(mock_nickname); +static const char *NS(mock_nickname); static void NS(test_main)(void *arg) @@ -1652,8 +1652,8 @@ NS(node_get_by_nickname)(const char *nickname, int warn_if_unused) NS_DECL(const node_t *, node_get_by_nickname, (const char *nickname, int warn_if_unused)); -const char *NS(mock_nickname); -node_t NS(mock_node); +static const char *NS(mock_nickname); +static node_t NS(mock_node); static void NS(test_main)(void *arg) @@ -1702,8 +1702,8 @@ NS(node_get_by_nickname)(const char *nickname, int warn_if_unused) NS_DECL(const node_t *, node_get_by_nickname, (const char *nickname, int warn_if_unused)); -char *NS(mock_nickname); -node_t NS(mock_node); +static char *NS(mock_nickname); +static node_t NS(mock_node); static void NS(test_main)(void *arg) @@ -1754,7 +1754,7 @@ NS(node_get_by_nickname)(const char *nickname, int warn_if_unused) NS_DECL(smartlist_t *, nodelist_get_list, (void)); -smartlist_t *NS(mock_smartlist); +static smartlist_t *NS(mock_smartlist); static void NS(test_main)(void *arg) @@ -1800,8 +1800,8 @@ NS(nodelist_get_list)(void) NS_DECL(smartlist_t *, nodelist_get_list, (void)); -smartlist_t *NS(mock_smartlist); -node_t NS(mock_node); +static smartlist_t *NS(mock_smartlist); +static node_t NS(mock_node); static void NS(test_main)(void *arg) diff --git a/src/test/test_scheduler.c b/src/test/test_scheduler.c index 6e9889b48b..15fbb2d186 100644 --- a/src/test/test_scheduler.c +++ b/src/test/test_scheduler.c @@ -24,12 +24,6 @@ #include "test.h" #include "fakechans.h" -/* Statics in scheduler.c exposed to the test suite */ -extern smartlist_t *channels_pending; -extern struct event *run_sched_ev; -extern uint64_t queue_heuristic; -extern time_t queue_heuristic_timestamp; - /* Event base for scheduelr tests */ static struct event_base *mock_event_base = NULL; diff --git a/src/test/test_slow.c b/src/test/test_slow.c index c1d2e81914..7c9f0b1cc2 100644 --- a/src/test/test_slow.c +++ b/src/test/test_slow.c @@ -18,9 +18,6 @@ #include "or.h" #include "test.h" -extern struct testcase_t slow_crypto_tests[]; -extern struct testcase_t slow_util_tests[]; - struct testgroup_t testgroups[] = { { "slow/crypto/", slow_crypto_tests }, { "slow/util/", slow_util_tests }, diff --git a/src/test/test_socks.c b/src/test/test_socks.c index 6da09fd653..62ff12fe15 100644 --- a/src/test/test_socks.c +++ b/src/test/test_socks.c @@ -34,7 +34,7 @@ socks_test_cleanup(const struct testcase_t *testcase, void *ptr) return 1; } -const struct testcase_setup_t socks_setup = { +static const struct testcase_setup_t socks_setup = { socks_test_setup, socks_test_cleanup }; diff --git a/src/test/test_status.c b/src/test/test_status.c index 84a0f6c024..b4438aabe9 100644 --- a/src/test/test_status.c +++ b/src/test/test_status.c @@ -310,8 +310,6 @@ NS_DECL(void, logv, (int severity, log_domain_mask_t domain, NS_DECL(int, server_mode, (const or_options_t *options)); static routerinfo_t *mock_routerinfo; -extern int onion_handshakes_requested[MAX_ONION_HANDSHAKE_TYPE+1]; -extern int onion_handshakes_assigned[MAX_ONION_HANDSHAKE_TYPE+1]; static void NS(test_main)(void *arg) diff --git a/src/test/test_switch_id.sh b/src/test/test_switch_id.sh index 1b4e0998b5..79c44f2eb1 100755 --- a/src/test/test_switch_id.sh +++ b/src/test/test_switch_id.sh @@ -10,6 +10,10 @@ if test "`id -u nobody`" = ""; then exit 1 fi +if test "$OVERRIDE_GCDA_PERMISSIONS_HACK" = "yes"; then + find src -type f -name '*gcda' -print0 | xargs -0 chmod 0666 +fi + "${builddir:-.}/src/test/test-switch-id" nobody setuid || exit 1 "${builddir:-.}/src/test/test-switch-id" nobody root-bind-low || exit 1 "${builddir:-.}/src/test/test-switch-id" nobody setuid-strict || exit 1 @@ -19,6 +23,9 @@ fi "${builddir:-.}/src/test/test-switch-id" nobody have-caps || exit 1 "${builddir:-.}/src/test/test-switch-id" nobody setuid-keepcaps || exit 1 +if test "$OVERRIDE_GCDA_PERMISSIONS_HACK" = "yes"; then + find src -type f -name '*gcda' -print0 | xargs -0 chmod 0644 +fi echo "All okay" diff --git a/src/test/test_tortls.c b/src/test/test_tortls.c index b9b74a1e96..3a048fb1f0 100644 --- a/src/test/test_tortls.c +++ b/src/test/test_tortls.c @@ -8,19 +8,13 @@ #ifdef _WIN32 #include <winsock2.h> #endif +#include <math.h> -#ifdef __GNUC__ -#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -#endif +#include "compat.h" -#if __GNUC__ && GCC_VERSION >= 402 -#if GCC_VERSION >= 406 -#pragma GCC diagnostic push -#endif /* Some versions of OpenSSL declare SSL_get_selected_srtp_profile twice in * srtp.h. Suppress the GCC warning so we can build with -Wredundant-decl. */ -#pragma GCC diagnostic ignored "-Wredundant-decls" -#endif +DISABLE_GCC_WARNING(redundant-decls) #include <openssl/opensslv.h> @@ -33,13 +27,7 @@ #include <openssl/evp.h> #include <openssl/bn.h> -#if __GNUC__ && GCC_VERSION >= 402 -#if GCC_VERSION >= 406 -#pragma GCC diagnostic pop -#else -#pragma GCC diagnostic warning "-Wredundant-decls" -#endif -#endif +ENABLE_GCC_WARNING(redundant-decls) #include "or.h" #include "torlog.h" @@ -50,9 +38,6 @@ #include "log_test_helpers.h" #define NS_MODULE tortls -extern tor_tls_context_t *server_tls_context; -extern tor_tls_context_t *client_tls_context; - #if OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,1,0) \ && !defined(LIBRESSL_VERSION_NUMBER) #define OPENSSL_OPAQUE @@ -277,8 +262,6 @@ test_tortls_get_state_description(void *ignored) tor_free(tls); } -extern int tor_tls_object_ex_data_index; - static void test_tortls_get_by_ssl(void *ignored) { @@ -791,8 +774,6 @@ get_cipher_by_id(uint16_t id) return NULL; } -extern uint16_t v2_cipher_list[]; - static void test_tortls_classify_client_ciphers(void *ignored) { @@ -1185,9 +1166,6 @@ test_tortls_get_forced_write_size(void *ignored) tor_free(tls); } -extern uint64_t total_bytes_written_over_tls; -extern uint64_t total_bytes_written_by_tls; - static void test_tortls_get_write_overhead_ratio(void *ignored) { @@ -1196,17 +1174,17 @@ test_tortls_get_write_overhead_ratio(void *ignored) total_bytes_written_over_tls = 0; ret = tls_get_write_overhead_ratio(); - tt_int_op(ret, OP_EQ, 1.0); + tt_double_op(fabs(ret - 1.0), OP_LT, 1E-12); total_bytes_written_by_tls = 10; total_bytes_written_over_tls = 1; ret = tls_get_write_overhead_ratio(); - tt_int_op(ret, OP_EQ, 10.0); + tt_double_op(fabs(ret - 10.0), OP_LT, 1E-12); total_bytes_written_by_tls = 10; total_bytes_written_over_tls = 2; ret = tls_get_write_overhead_ratio(); - tt_int_op(ret, OP_EQ, 5.0); + tt_double_op(fabs(ret - 5.0), OP_LT, 1E-12); done: (void)0; diff --git a/src/test/test_util.c b/src/test/test_util.c index d534cc0b52..56c71e871d 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -30,6 +30,9 @@ #include <ctype.h> #include <float.h> +#define INFINITY_DBL ((double)INFINITY) +#define NAN_DBL ((double)NAN) + /* XXXX this is a minimal wrapper to make the unit tests compile with the * changed tor_timegm interface. */ static time_t @@ -258,7 +261,7 @@ test_util_time(void *arg) int i; struct timeval tv; - /* Test tv_udiff */ + /* Test tv_udiff and tv_mdiff */ (void)arg; start.tv_sec = 5; @@ -268,22 +271,36 @@ test_util_time(void *arg) end.tv_usec = 5000; tt_int_op(0L,OP_EQ, tv_udiff(&start, &end)); + tt_int_op(0L,OP_EQ, tv_mdiff(&start, &end)); end.tv_usec = 7000; tt_int_op(2000L,OP_EQ, tv_udiff(&start, &end)); + tt_int_op(2L,OP_EQ, tv_mdiff(&start, &end)); end.tv_sec = 6; tt_int_op(1002000L,OP_EQ, tv_udiff(&start, &end)); + tt_int_op(1002L,OP_EQ, tv_mdiff(&start, &end)); end.tv_usec = 0; tt_int_op(995000L,OP_EQ, tv_udiff(&start, &end)); + tt_int_op(995L,OP_EQ, tv_mdiff(&start, &end)); end.tv_sec = 4; tt_int_op(-1005000L,OP_EQ, tv_udiff(&start, &end)); + tt_int_op(-1005L,OP_EQ, tv_mdiff(&start, &end)); + +#ifdef _WIN32 + /* Would you believe that tv_sec is a long on windows? Of course you would.*/ + end.tv_sec = LONG_MAX; +#else + end.tv_sec = TIME_MAX; +#endif + tt_int_op(LONG_MAX, OP_EQ, tv_udiff(&start, &end)); + tt_int_op(LONG_MAX, OP_EQ, tv_mdiff(&start, &end)); /* Test tor_timegm & tor_gmtime_r */ @@ -622,9 +639,16 @@ test_util_time(void *arg) parse_rfc1123_time("Wed, 30 Ene 2011 23:59:59 GMT", &t_res)); tt_int_op(-1,OP_EQ, parse_rfc1123_time("Wed, 30 Mar 2011 23:59:59 GM", &t_res)); + tt_int_op(-1,OP_EQ, + parse_rfc1123_time("Wed, 30 Mar 1900 23:59:59 GMT", &t_res)); + /* Leap year. */ tt_int_op(-1,OP_EQ, parse_rfc1123_time("Wed, 29 Feb 2011 16:00:00 GMT", &t_res)); + tt_int_op(0,OP_EQ, + parse_rfc1123_time("Wed, 29 Feb 2012 16:00:00 GMT", &t_res)); + + /* Leap second plus one */ tt_int_op(-1,OP_EQ, parse_rfc1123_time("Wed, 30 Mar 2011 23:59:61 GMT", &t_res)); @@ -865,106 +889,106 @@ test_util_config_line(void *arg) , sizeof(buf)); str = buf; - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "k"); tt_str_op(v,OP_EQ, "v"); tor_free(k); tor_free(v); tt_assert(!strcmpstart(str, "key value with")); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "key"); tt_str_op(v,OP_EQ, "value with spaces"); tor_free(k); tor_free(v); tt_assert(!strcmpstart(str, "keykey")); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "keykey"); tt_str_op(v,OP_EQ, "val"); tor_free(k); tor_free(v); tt_assert(!strcmpstart(str, "k2\n")); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "k2"); tt_str_op(v,OP_EQ, ""); tor_free(k); tor_free(v); tt_assert(!strcmpstart(str, "k3 \n")); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "k3"); tt_str_op(v,OP_EQ, ""); tor_free(k); tor_free(v); tt_assert(!strcmpstart(str, "#comment")); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "k4"); tt_str_op(v,OP_EQ, ""); tor_free(k); tor_free(v); tt_assert(!strcmpstart(str, "k5#abc")); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "k5"); tt_str_op(v,OP_EQ, ""); tor_free(k); tor_free(v); tt_assert(!strcmpstart(str, "k6")); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "k6"); tt_str_op(v,OP_EQ, "val"); tor_free(k); tor_free(v); tt_assert(!strcmpstart(str, "kseven")); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "kseven"); tt_str_op(v,OP_EQ, "a quoted \'string"); tor_free(k); tor_free(v); tt_assert(!strcmpstart(str, "k8 ")); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "k8"); tt_str_op(v,OP_EQ, "a quoted\n\"str\\ing\t\x01\x01\x01\""); tor_free(k); tor_free(v); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "k9"); tt_str_op(v,OP_EQ, "a line that spans two lines."); tor_free(k); tor_free(v); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "k10"); tt_str_op(v,OP_EQ, "more than one continuation"); tor_free(k); tor_free(v); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "k11"); tt_str_op(v,OP_EQ, "continuation at the start"); tor_free(k); tor_free(v); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "k12"); tt_str_op(v,OP_EQ, "line with a embedded"); tor_free(k); tor_free(v); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "k13"); tt_str_op(v,OP_EQ, "continuation at the very start"); tor_free(k); tor_free(v); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "k14"); tt_str_op(v,OP_EQ, "a line that has a comment and" ); tor_free(k); tor_free(v); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "k15"); tt_str_op(v,OP_EQ, "this should be the next new line"); tor_free(k); tor_free(v); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "k16"); tt_str_op(v,OP_EQ, "a line that has a comment and" ); tor_free(k); tor_free(v); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "k17"); tt_str_op(v,OP_EQ, "this should be the next new line"); tor_free(k); tor_free(v); @@ -999,32 +1023,36 @@ test_util_config_line_quotes(void *arg) , sizeof(buf4)); str = buf1; - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "kTrailingSpace"); tt_str_op(v,OP_EQ, "quoted value"); tor_free(k); tor_free(v); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_ptr_op(str,OP_EQ, NULL); tor_free(k); tor_free(v); str = buf2; - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_ptr_op(str,OP_EQ, NULL); tor_free(k); tor_free(v); str = buf3; - str = parse_config_line_from_str(str, &k, &v); + const char *err = NULL; + str = parse_config_line_from_str_verbose(str, &k, &v, &err); tt_ptr_op(str,OP_EQ, NULL); tor_free(k); tor_free(v); + tt_str_op(err, OP_EQ, "Invalid escape sequence in quoted string"); str = buf4; - str = parse_config_line_from_str(str, &k, &v); + err = NULL; + str = parse_config_line_from_str_verbose(str, &k, &v, &err); tt_ptr_op(str,OP_EQ, NULL); tor_free(k); tor_free(v); + tt_str_op(err, OP_EQ, "Invalid escape sequence in quoted string"); done: tor_free(k); @@ -1046,12 +1074,12 @@ test_util_config_line_comment_character(void *arg) , sizeof(buf)); str = buf; - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "k1"); tt_str_op(v,OP_EQ, "# in quotes"); tor_free(k); tor_free(v); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "k2"); tt_str_op(v,OP_EQ, "some value"); tor_free(k); tor_free(v); @@ -1059,7 +1087,7 @@ test_util_config_line_comment_character(void *arg) tt_str_op(str,OP_EQ, "k3 /home/user/myTorNetwork#2\n"); #if 0 - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); test_streq(k, "k3"); test_streq(v, "/home/user/myTorNetwork#2"); tor_free(k); tor_free(v); @@ -1116,57 +1144,57 @@ test_util_config_line_escaped_content(void *arg) str = buf1; - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "HexadecimalLower"); tt_str_op(v,OP_EQ, "*"); tor_free(k); tor_free(v); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "HexadecimalUpper"); tt_str_op(v,OP_EQ, "*"); tor_free(k); tor_free(v); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "HexadecimalUpperX"); tt_str_op(v,OP_EQ, "*"); tor_free(k); tor_free(v); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "Octal"); tt_str_op(v,OP_EQ, "*"); tor_free(k); tor_free(v); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "Newline"); tt_str_op(v,OP_EQ, "\n"); tor_free(k); tor_free(v); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "Tab"); tt_str_op(v,OP_EQ, "\t"); tor_free(k); tor_free(v); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "CarriageReturn"); tt_str_op(v,OP_EQ, "\r"); tor_free(k); tor_free(v); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "DoubleQuote"); tt_str_op(v,OP_EQ, "\""); tor_free(k); tor_free(v); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "SimpleQuote"); tt_str_op(v,OP_EQ, "'"); tor_free(k); tor_free(v); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "Backslash"); tt_str_op(v,OP_EQ, "\\"); tor_free(k); tor_free(v); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "Mix"); tt_str_op(v,OP_EQ, "This is a \"star\":\t'*'\nAnd second line"); tor_free(k); tor_free(v); @@ -1174,35 +1202,80 @@ test_util_config_line_escaped_content(void *arg) str = buf2; - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_ptr_op(str,OP_EQ, NULL); tor_free(k); tor_free(v); str = buf3; - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_ptr_op(str,OP_EQ, NULL); tor_free(k); tor_free(v); str = buf4; - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_ptr_op(str,OP_EQ, NULL); tor_free(k); tor_free(v); #if 0 str = buf5; - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_ptr_op(str, OP_EQ, NULL); tor_free(k); tor_free(v); #endif str = buf6; - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); + tt_ptr_op(str,OP_EQ, NULL); + tor_free(k); tor_free(v); + + /* more things to try. */ + /* Bad hex: */ + strlcpy(buf1, "Foo \"\\x9g\"\n", sizeof(buf1)); + strlcpy(buf2, "Foo \"\\xg0\"\n", sizeof(buf2)); + strlcpy(buf3, "Foo \"\\xf\"\n", sizeof(buf3)); + /* bad escape */ + strlcpy(buf4, "Foo \"\\q\"\n", sizeof(buf4)); + /* missing endquote */ + strlcpy(buf5, "Foo \"hello\n", sizeof(buf5)); + /* extra stuff */ + strlcpy(buf6, "Foo \"hello\" world\n", sizeof(buf6)); + + str=buf1; + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); + tt_ptr_op(str,OP_EQ, NULL); + tor_free(k); tor_free(v); + + str=buf2; + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); + tt_ptr_op(str,OP_EQ, NULL); + tor_free(k); tor_free(v); + + str=buf3; + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); + tt_ptr_op(str,OP_EQ, NULL); + tor_free(k); tor_free(v); + + str=buf4; + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); + tt_ptr_op(str,OP_EQ, NULL); + tor_free(k); tor_free(v); + + str=buf5; + + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); + tt_ptr_op(str,OP_EQ, NULL); + tor_free(k); tor_free(v); + + str=buf6; + const char *err = NULL; + str = parse_config_line_from_str_verbose(str, &k, &v, &err); tt_ptr_op(str,OP_EQ, NULL); tor_free(k); tor_free(v); + tt_str_op(err,OP_EQ, "Excess data after quoted string"); done: tor_free(k); @@ -1428,6 +1501,11 @@ test_util_strmisc(void *arg) tt_int_op(-50L,OP_EQ, tor_parse_long("-50 plus garbage",10,-100,100,&i,&cp)); tt_int_op(1,OP_EQ, i); tt_str_op(cp,OP_EQ, " plus garbage"); + /* Illogical min max */ + tt_int_op(0L,OP_EQ, tor_parse_long("10",10,50,4,&i,NULL)); + tt_int_op(0,OP_EQ, i); + tt_int_op(0L,OP_EQ, tor_parse_long("-50",10,100,-100,&i,NULL)); + tt_int_op(0,OP_EQ, i); /* Out of bounds */ tt_int_op(0L,OP_EQ, tor_parse_long("10",10,50,100,&i,NULL)); tt_int_op(0,OP_EQ, i); @@ -1472,24 +1550,24 @@ test_util_strmisc(void *arg) { /* Test parse_double */ - double d = tor_parse_double("10", 0, UINT64_MAX,&i,NULL); + double d = tor_parse_double("10", 0, (double)UINT64_MAX,&i,NULL); tt_int_op(1,OP_EQ, i); tt_assert(DBL_TO_U64(d) == 10); - d = tor_parse_double("0", 0, UINT64_MAX,&i,NULL); + d = tor_parse_double("0", 0, (double)UINT64_MAX,&i,NULL); tt_int_op(1,OP_EQ, i); tt_assert(DBL_TO_U64(d) == 0); - d = tor_parse_double(" ", 0, UINT64_MAX,&i,NULL); + d = tor_parse_double(" ", 0, (double)UINT64_MAX,&i,NULL); tt_int_op(0,OP_EQ, i); - d = tor_parse_double(".0a", 0, UINT64_MAX,&i,NULL); + d = tor_parse_double(".0a", 0, (double)UINT64_MAX,&i,NULL); tt_int_op(0,OP_EQ, i); - d = tor_parse_double(".0a", 0, UINT64_MAX,&i,&cp); + d = tor_parse_double(".0a", 0, (double)UINT64_MAX,&i,&cp); tt_int_op(1,OP_EQ, i); - d = tor_parse_double("-.0", 0, UINT64_MAX,&i,NULL); + d = tor_parse_double("-.0", 0, (double)UINT64_MAX,&i,NULL); tt_int_op(1,OP_EQ, i); tt_assert(DBL_TO_U64(d) == 0); d = tor_parse_double("-10", -100.0, 100.0,&i,NULL); tt_int_op(1,OP_EQ, i); - tt_int_op(-10.0,OP_EQ, d); + tt_double_op(fabs(d - -10.0),OP_LT, 1E-12); } { @@ -1583,6 +1661,17 @@ test_util_strmisc(void *arg) tt_str_op("\"z\\001abc\\277d\"",OP_EQ, escaped("z\001abc\277d")); tt_str_op("\"z\\336\\255 ;foo\"",OP_EQ, escaped("z\xde\xad\x20;foo")); + /* Other cases of esc_for_log{,_len} */ + cp_tmp = esc_for_log(NULL); + tt_str_op(cp_tmp, OP_EQ, "(null)"); + tor_free(cp_tmp); + cp_tmp = esc_for_log_len("abcdefg", 3); + tt_str_op(cp_tmp, OP_EQ, "\"abc\""); + tor_free(cp_tmp); + cp_tmp = esc_for_log_len("abcdefg", 100); + tt_str_op(cp_tmp, OP_EQ, "\"abcdefg\""); + tor_free(cp_tmp); + /* Test strndup and memdup */ { const char *s = "abcdefghijklmnopqrstuvwxyz"; @@ -1737,22 +1826,21 @@ test_util_gzip(void *arg) (void)arg; buf1 = tor_strdup("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ"); tt_assert(detect_compression_method(buf1, strlen(buf1)) == UNKNOWN_METHOD); - if (is_gzip_supported()) { - tt_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1, - GZIP_METHOD)); - tt_assert(buf2); - tt_assert(len1 < strlen(buf1)); - tt_assert(detect_compression_method(buf2, len1) == GZIP_METHOD); - - tt_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1, - GZIP_METHOD, 1, LOG_INFO)); - tt_assert(buf3); - tt_int_op(strlen(buf1) + 1,OP_EQ, len2); - tt_str_op(buf1,OP_EQ, buf3); - - tor_free(buf2); - tor_free(buf3); - } + + tt_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1, + GZIP_METHOD)); + tt_assert(buf2); + tt_assert(len1 < strlen(buf1)); + tt_assert(detect_compression_method(buf2, len1) == GZIP_METHOD); + + tt_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1, + GZIP_METHOD, 1, LOG_INFO)); + tt_assert(buf3); + tt_int_op(strlen(buf1) + 1,OP_EQ, len2); + tt_str_op(buf1,OP_EQ, buf3); + + tor_free(buf2); + tor_free(buf3); tt_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1, ZLIB_METHOD)); @@ -1836,6 +1924,52 @@ test_util_gzip(void *arg) tor_free(buf1); } +static void +test_util_gzip_compression_bomb(void *arg) +{ + /* A 'compression bomb' is a very small object that uncompresses to a huge + * one. Most compression formats support them, but they can be a DOS vector. + * In Tor we try not to generate them, and we don't accept them. + */ + (void) arg; + size_t one_million = 1<<20; + char *one_mb = tor_malloc_zero(one_million); + char *result = NULL; + size_t result_len = 0; + tor_zlib_state_t *state = NULL; + + /* Make sure we can't produce a compression bomb */ + tt_int_op(-1, OP_EQ, tor_gzip_compress(&result, &result_len, + one_mb, one_million, + ZLIB_METHOD)); + + /* Here's a compression bomb that we made manually. */ + const char compression_bomb[1039] = + { 0x78, 0xDA, 0xED, 0xC1, 0x31, 0x01, 0x00, 0x00, 0x00, 0xC2, + 0xA0, 0xF5, 0x4F, 0x6D, 0x08, 0x5F, 0xA0 /* .... */ }; + tt_int_op(-1, OP_EQ, tor_gzip_uncompress(&result, &result_len, + compression_bomb, 1039, + ZLIB_METHOD, 0, LOG_WARN)); + + /* Now try streaming that. */ + state = tor_zlib_new(0, ZLIB_METHOD, HIGH_COMPRESSION); + tor_zlib_output_t r; + const char *inp = compression_bomb; + size_t inlen = 1039; + do { + char *outp = one_mb; + size_t outleft = 4096; /* small on purpose */ + r = tor_zlib_process(state, &outp, &outleft, &inp, &inlen, 0); + tt_int_op(inlen, OP_NE, 0); + } while (r == TOR_ZLIB_BUF_FULL); + + tt_int_op(r, OP_EQ, TOR_ZLIB_ERR); + + done: + tor_free(one_mb); + tor_zlib_free(state); +} + /** Run unit tests for mmap() wrapper functionality. */ static void test_util_mmap(void *arg) @@ -2842,19 +2976,40 @@ test_util_memarea(void *arg) p1 = memarea_alloc(area, 1); tt_ptr_op(p1,OP_EQ, p1_orig); memarea_clear(area); + size_t total = 0, initial_allocation, allocation2, dummy; + memarea_get_stats(area, &initial_allocation, &dummy); /* Check for running over an area's size. */ - for (i = 0; i < 512; ++i) { - p1 = memarea_alloc(area, crypto_rand_int(5)+1); + for (i = 0; i < 4096; ++i) { + size_t n = crypto_rand_int(6); + p1 = memarea_alloc(area, n); + total += n; tt_assert(memarea_owns_ptr(area, p1)); } memarea_assert_ok(area); + memarea_get_stats(area, &allocation2, &dummy); /* Make sure we can allocate a too-big object. */ p1 = memarea_alloc_zero(area, 9000); p2 = memarea_alloc_zero(area, 16); + total += 9000; + total += 16; tt_assert(memarea_owns_ptr(area, p1)); tt_assert(memarea_owns_ptr(area, p2)); + /* Now test stats... */ + size_t allocated = 0, used = 0; + memarea_get_stats(area, &allocated, &used); + tt_int_op(used, OP_LE, allocated); + tt_int_op(used, OP_GE, total); /* not EQ, because of alignment and headers*/ + tt_int_op(allocated, OP_GT, allocation2); + + tt_int_op(allocation2, OP_GT, initial_allocation); + + memarea_clear(area); + memarea_get_stats(area, &allocated, &used); + tt_int_op(used, OP_LT, 128); /* Not 0, because of header */ + tt_int_op(allocated, OP_EQ, initial_allocation); + done: memarea_drop_all(area); tor_free(malloced_ptr); @@ -3244,6 +3399,21 @@ test_util_ftruncate(void *ptr) tor_free(buf); } +static void +test_util_num_cpus(void *arg) +{ + (void)arg; + int num = compute_num_cpus(); + if (num < 0) + tt_skip(); + + tt_int_op(num, OP_GE, 1); + tt_int_op(num, OP_LE, 16); + + done: + ; +} + #ifdef _WIN32 static void test_util_load_win_lib(void *ptr) @@ -4223,21 +4393,6 @@ test_util_round_to_next_multiple_of(void *arg) tt_u64_op(round_uint64_to_next_multiple_of(UINT64_MAX,2), ==, UINT64_MAX); - tt_i64_op(round_int64_to_next_multiple_of(0,1), ==, 0); - tt_i64_op(round_int64_to_next_multiple_of(0,7), ==, 0); - - tt_i64_op(round_int64_to_next_multiple_of(99,1), ==, 99); - tt_i64_op(round_int64_to_next_multiple_of(99,7), ==, 105); - tt_i64_op(round_int64_to_next_multiple_of(99,9), ==, 99); - - tt_i64_op(round_int64_to_next_multiple_of(-99,1), ==, -99); - tt_i64_op(round_int64_to_next_multiple_of(-99,7), ==, -98); - tt_i64_op(round_int64_to_next_multiple_of(-99,9), ==, -99); - - tt_i64_op(round_int64_to_next_multiple_of(INT64_MIN,2), ==, INT64_MIN); - tt_i64_op(round_int64_to_next_multiple_of(INT64_MAX,2), ==, - INT64_MAX); - tt_int_op(round_uint32_to_next_multiple_of(0,1), ==, 0); tt_int_op(round_uint32_to_next_multiple_of(0,7), ==, 0); @@ -4407,7 +4562,7 @@ test_util_clamp_double_to_int64(void *arg) { (void)arg; - tt_i64_op(INT64_MIN, ==, clamp_double_to_int64(-INFINITY)); + tt_i64_op(INT64_MIN, ==, clamp_double_to_int64(-INFINITY_DBL)); tt_i64_op(INT64_MIN, ==, clamp_double_to_int64(-1.0 * pow(2.0, 64.0) - 1.0)); tt_i64_op(INT64_MIN, ==, @@ -4420,7 +4575,7 @@ test_util_clamp_double_to_int64(void *arg) tt_i64_op(0, ==, clamp_double_to_int64(-0.9)); tt_i64_op(0, ==, clamp_double_to_int64(-0.1)); tt_i64_op(0, ==, clamp_double_to_int64(0.0)); - tt_i64_op(0, ==, clamp_double_to_int64(NAN)); + tt_i64_op(0, ==, clamp_double_to_int64(NAN_DBL)); tt_i64_op(0, ==, clamp_double_to_int64(0.1)); tt_i64_op(0, ==, clamp_double_to_int64(0.9)); tt_i64_op(1, ==, clamp_double_to_int64(1.0)); @@ -4432,7 +4587,7 @@ test_util_clamp_double_to_int64(void *arg) clamp_double_to_int64(pow(2.0, 63.0))); tt_i64_op(INT64_MAX, ==, clamp_double_to_int64(pow(2.0, 64.0))); - tt_i64_op(INT64_MAX, ==, clamp_double_to_int64(INFINITY)); + tt_i64_op(INT64_MAX, ==, clamp_double_to_int64(INFINITY_DBL)); done: ; @@ -4780,12 +4935,71 @@ test_util_pwdb(void *arg) dir = get_user_homedir(name); tt_assert(dir != NULL); + /* Try failing cases. First find a user that doesn't exist by name */ + char rand[4]; + char badname[9]; + int i, found=0; + for (i = 0; i < 100; ++i) { + crypto_rand(rand, sizeof(rand)); + base16_encode(badname, sizeof(badname), rand, sizeof(rand)); + if (tor_getpwnam(badname) == NULL) { + found = 1; + break; + } + } + tt_assert(found); + tor_free(dir); + dir = get_user_homedir(badname); + tt_assert(dir == NULL); + + /* Now try to find a user that doesn't exist by ID. */ + found = 0; + for (i = 0; i < 1000; ++i) { + uid_t u; + crypto_rand((char*)&u, sizeof(u)); + if (tor_getpwuid(u) == NULL) { + found = 1; + break; + } + } + tt_assert(found); + done: tor_free(name); tor_free(dir); } #endif +static void +test_util_calloc_check(void *arg) +{ + (void) arg; + /* Easy cases that are good. */ + tt_assert(size_mul_check__(0,0)); + tt_assert(size_mul_check__(0,100)); + tt_assert(size_mul_check__(100,0)); + tt_assert(size_mul_check__(100,100)); + + /* Harder cases that are still good. */ + tt_assert(size_mul_check__(SIZE_MAX, 1)); + tt_assert(size_mul_check__(1, SIZE_MAX)); + tt_assert(size_mul_check__(SIZE_MAX / 10, 9)); + tt_assert(size_mul_check__(11, SIZE_MAX / 12)); + const size_t sqrt_size_max_p1 = ((size_t)1) << (sizeof(size_t) * 4); + tt_assert(size_mul_check__(sqrt_size_max_p1, sqrt_size_max_p1 - 1)); + + /* Cases that overflow */ + tt_assert(! size_mul_check__(SIZE_MAX, 2)); + tt_assert(! size_mul_check__(2, SIZE_MAX)); + tt_assert(! size_mul_check__(SIZE_MAX / 10, 11)); + tt_assert(! size_mul_check__(11, SIZE_MAX / 10)); + tt_assert(! size_mul_check__(SIZE_MAX / 8, 9)); + tt_assert(! size_mul_check__(sqrt_size_max_p1, sqrt_size_max_p1)); + + done: + ; +} + #define UTIL_LEGACY(name) \ { #name, test_util_ ## name , 0, NULL, NULL } @@ -4815,11 +5029,12 @@ struct testcase_t util_tests[] = { UTIL_LEGACY(strmisc), UTIL_LEGACY(pow2), UTIL_LEGACY(gzip), + UTIL_LEGACY(gzip_compression_bomb), UTIL_LEGACY(datadir), UTIL_LEGACY(memarea), UTIL_LEGACY(control_formats), UTIL_LEGACY(mmap), - UTIL_LEGACY(sscanf), + UTIL_TEST(sscanf, TT_FORK), UTIL_LEGACY(format_time_interval), UTIL_LEGACY(path_is_relative), UTIL_LEGACY(strtok), @@ -4834,6 +5049,7 @@ struct testcase_t util_tests[] = { UTIL_TEST(listdir, 0), UTIL_TEST(parent_dir, 0), UTIL_TEST(ftruncate, 0), + UTIL_TEST(num_cpus, 0), UTIL_TEST_WIN_ONLY(load_win_lib, 0), UTIL_TEST_NO_WIN(exit_status, 0), UTIL_TEST_NO_WIN(fgets_eagain, 0), @@ -4871,6 +5087,7 @@ struct testcase_t util_tests[] = { UTIL_TEST(get_avail_disk_space, 0), UTIL_TEST(touch_file, 0), UTIL_TEST_NO_WIN(pwdb, TT_FORK), + UTIL_TEST(calloc_check, 0), END_OF_TESTCASES }; diff --git a/src/test/test_util_format.c b/src/test/test_util_format.c index a25054cd0a..f3dcd1184c 100644 --- a/src/test/test_util_format.c +++ b/src/test/test_util_format.c @@ -263,14 +263,14 @@ test_util_format_base16_decode(void *ignored) res = base16_decode(dst, 1, src, 10); tt_int_op(res, OP_EQ, -1); - res = base16_decode(dst, SIZE_T_CEILING+2, src, 10); + res = base16_decode(dst, ((size_t)INT_MAX)+1, src, 10); tt_int_op(res, OP_EQ, -1); res = base16_decode(dst, 1000, "", 0); tt_int_op(res, OP_EQ, 0); res = base16_decode(dst, 1000, "aabc", 4); - tt_int_op(res, OP_EQ, 0); + tt_int_op(res, OP_EQ, 2); tt_mem_op(dst, OP_EQ, "\xaa\xbc", 2); res = base16_decode(dst, 1000, "aabcd", 6); @@ -280,7 +280,7 @@ test_util_format_base16_decode(void *ignored) tt_int_op(res, OP_EQ, -1); res = base16_decode(real_dst, 10, real_src, 14); - tt_int_op(res, OP_EQ, 0); + tt_int_op(res, OP_EQ, 7); tt_mem_op(real_dst, OP_EQ, expected, 7); done: @@ -289,6 +289,95 @@ test_util_format_base16_decode(void *ignored) tor_free(real_dst); } +static void +test_util_format_base32_encode(void *arg) +{ + (void) arg; + size_t real_dstlen = 32; + char *dst = tor_malloc_zero(real_dstlen); + + /* Basic use case that doesn't require a source length correction. */ + { + /* Length of 10 bytes. */ + const char *src = "blahbleh12"; + size_t srclen = strlen(src); + /* Expected result encoded base32. This was created using python as + * such (and same goes for all test case.): + * + * b = bytes("blahbleh12", 'utf-8') + * base64.b32encode(b) + * (result in lower case) + */ + const char *expected = "mjwgc2dcnrswqmjs"; + + base32_encode(dst, base32_encoded_size(srclen), src, srclen); + tt_mem_op(expected, OP_EQ, dst, strlen(expected)); + /* Encode but to a larger size destination. */ + memset(dst, 0, real_dstlen); + base32_encode(dst, real_dstlen, src, srclen); + tt_mem_op(expected, OP_EQ, dst, strlen(expected)); + } + + /* Non multiple of 5 for the source buffer length. */ + { + /* Length of 8 bytes. */ + const char *expected = "mjwgc2dcnrswq"; + const char *src = "blahbleh"; + size_t srclen = strlen(src); + + memset(dst, 0, real_dstlen); + base32_encode(dst, base32_encoded_size(srclen), src, srclen); + tt_mem_op(expected, OP_EQ, dst, strlen(expected)); + } + + done: + tor_free(dst); +} + +static void +test_util_format_base32_decode(void *arg) +{ + (void) arg; + int ret; + size_t real_dstlen = 32; + char *dst = tor_malloc_zero(real_dstlen); + + /* Basic use case. */ + { + /* Length of 10 bytes. */ + const char *expected = "blahbleh12"; + /* Expected result encoded base32. */ + const char *src = "mjwgc2dcnrswqmjs"; + + ret = base32_decode(dst, strlen(expected), src, strlen(src)); + tt_int_op(ret, ==, 0); + tt_str_op(expected, OP_EQ, dst); + } + + /* Non multiple of 5 for the source buffer length. */ + { + /* Length of 8 bytes. */ + const char *expected = "blahbleh"; + const char *src = "mjwgc2dcnrswq"; + + ret = base32_decode(dst, strlen(expected), src, strlen(src)); + tt_int_op(ret, ==, 0); + tt_mem_op(expected, OP_EQ, dst, strlen(expected)); + } + + /* Invalid values. */ + { + /* Invalid character '#'. */ + ret = base32_decode(dst, real_dstlen, "#abcde", 6); + tt_int_op(ret, ==, -1); + /* Make sure the destination buffer has been zeroed even on error. */ + tt_int_op(tor_mem_is_zero(dst, real_dstlen), ==, 1); + } + + done: + tor_free(dst); +} + struct testcase_t util_format_tests[] = { { "unaligned_accessors", test_util_format_unaligned_accessors, 0, NULL, NULL }, @@ -297,6 +386,10 @@ struct testcase_t util_format_tests[] = { NULL, NULL }, { "base64_decode", test_util_format_base64_decode, 0, NULL, NULL }, { "base16_decode", test_util_format_base16_decode, 0, NULL, NULL }, + { "base32_encode", test_util_format_base32_encode, 0, + NULL, NULL }, + { "base32_decode", test_util_format_base32_decode, 0, + NULL, NULL }, END_OF_TESTCASES }; diff --git a/src/test/test_workqueue.c b/src/test/test_workqueue.c index cbcf596b22..75d68a7ce0 100644 --- a/src/test/test_workqueue.c +++ b/src/test/test_workqueue.c @@ -400,6 +400,9 @@ main(int argc, char **argv) } rq = replyqueue_new(as_flags); + if (as_flags && rq == NULL) + return 77; // 77 means "skipped". + tor_assert(rq); tp = threadpool_new(opt_n_threads, rq, new_state, free_state, NULL); diff --git a/src/test/test_workqueue_cancel.sh b/src/test/test_workqueue_cancel.sh new file mode 100755 index 0000000000..f7c663171e --- /dev/null +++ b/src/test/test_workqueue_cancel.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +${builddir:-.}/src/test/test_workqueue -C 1 + diff --git a/src/test/test_workqueue_efd.sh b/src/test/test_workqueue_efd.sh new file mode 100755 index 0000000000..4d89396819 --- /dev/null +++ b/src/test/test_workqueue_efd.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +${builddir:-.}/src/test/test_workqueue \ + --no-eventfd2 --no-pipe2 --no-pipe --no-socketpair diff --git a/src/test/test_workqueue_efd2.sh b/src/test/test_workqueue_efd2.sh new file mode 100755 index 0000000000..7cfff45ff3 --- /dev/null +++ b/src/test/test_workqueue_efd2.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +${builddir:-.}/src/test/test_workqueue \ + --no-eventfd --no-pipe2 --no-pipe --no-socketpair diff --git a/src/test/test_workqueue_pipe.sh b/src/test/test_workqueue_pipe.sh new file mode 100755 index 0000000000..afcef87853 --- /dev/null +++ b/src/test/test_workqueue_pipe.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +${builddir:-.}/src/test/test_workqueue \ + --no-eventfd2 --no-eventfd --no-pipe2 --no-socketpair diff --git a/src/test/test_workqueue_pipe2.sh b/src/test/test_workqueue_pipe2.sh new file mode 100755 index 0000000000..a20a1427e0 --- /dev/null +++ b/src/test/test_workqueue_pipe2.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +${builddir:-.}/src/test/test_workqueue \ + --no-eventfd2 --no-eventfd --no-pipe --no-socketpair diff --git a/src/test/test_workqueue_socketpair.sh b/src/test/test_workqueue_socketpair.sh new file mode 100755 index 0000000000..76af79746d --- /dev/null +++ b/src/test/test_workqueue_socketpair.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +${builddir:-.}/src/test/test_workqueue \ + --no-eventfd2 --no-eventfd --no-pipe2 --no-pipe diff --git a/src/test/testing_common.c b/src/test/testing_common.c index 39c3d02ab1..ea9366305c 100644 --- a/src/test/testing_common.c +++ b/src/test/testing_common.c @@ -3,6 +3,8 @@ * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ +extern const char tor_git_revision[]; + /* Ordinarily defined in tor_main.c; this bit is just here to provide one * since we're not linking to tor_main.c */ const char tor_git_revision[] = ""; @@ -215,8 +217,6 @@ const struct testcase_setup_t passthrough_setup = { passthrough_test_setup, passthrough_test_cleanup }; -extern struct testgroup_t testgroups[]; - /** Main entry point for unit test code: parse the command line, and run * some unit tests. */ int diff --git a/src/test/vote_descriptors.inc b/src/test/vote_descriptors.inc index c5ce21f744..895dc6c65c 100644 --- a/src/test/vote_descriptors.inc +++ b/src/test/vote_descriptors.inc @@ -1,4 +1,4 @@ -const char* VOTE_BODY_V3 = +static const char* VOTE_BODY_V3 = "network-status-version 3\n" "vote-status vote\n" "consensus-methods 13 14 15 16 17 18 19 20 21\n" diff --git a/src/tools/include.am b/src/tools/include.am index 38ed57546f..d0185b5887 100644 --- a/src/tools/include.am +++ b/src/tools/include.am @@ -7,23 +7,27 @@ endif src_tools_tor_resolve_SOURCES = src/tools/tor-resolve.c src_tools_tor_resolve_LDFLAGS = -src_tools_tor_resolve_LDADD = src/common/libor.a @TOR_LIB_MATH@ @TOR_LIB_WS32@ +src_tools_tor_resolve_LDADD = src/common/libor.a \ + src/common/libor-ctime.a \ + @TOR_LIB_MATH@ @TOR_LIB_WS32@ if COVERAGE_ENABLED src_tools_tor_cov_resolve_SOURCES = src/tools/tor-resolve.c src_tools_tor_cov_resolve_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_tools_tor_cov_resolve_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) src_tools_tor_cov_resolve_LDADD = src/common/libor-testing.a \ - @TOR_LIB_MATH@ @TOR_LIB_WS32@ + src/common/libor-ctime-testing.a \ + @TOR_LIB_MATH@ @TOR_LIB_WS32@ endif src_tools_tor_gencert_SOURCES = src/tools/tor-gencert.c src_tools_tor_gencert_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ src_tools_tor_gencert_LDADD = src/common/libor.a src/common/libor-crypto.a \ + src/common/libor-ctime.a \ $(LIBKECCAK_TINY) \ $(LIBDONNA) \ - @TOR_LIB_MATH@ @TOR_ZLIB_LIBS@ @TOR_OPENSSL_LIBS@ \ - @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ + @TOR_LIB_MATH@ @TOR_ZLIB_LIBS@ @TOR_OPENSSL_LIBS@ \ + @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ if COVERAGE_ENABLED src_tools_tor_cov_gencert_SOURCES = src/tools/tor-gencert.c @@ -32,18 +36,21 @@ src_tools_tor_cov_gencert_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) src_tools_tor_cov_gencert_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ src_tools_tor_cov_gencert_LDADD = src/common/libor-testing.a \ src/common/libor-crypto-testing.a \ + src/common/libor-ctime-testing.a \ $(LIBKECCAK_TINY) \ $(LIBDONNA) \ - @TOR_LIB_MATH@ @TOR_ZLIB_LIBS@ @TOR_OPENSSL_LIBS@ \ - @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ + @TOR_LIB_MATH@ @TOR_ZLIB_LIBS@ @TOR_OPENSSL_LIBS@ \ + @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ endif src_tools_tor_checkkey_SOURCES = src/tools/tor-checkkey.c src_tools_tor_checkkey_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ -src_tools_tor_checkkey_LDADD = src/common/libor.a src/common/libor-crypto.a \ +src_tools_tor_checkkey_LDADD = src/common/libor.a \ + src/common/libor-ctime.a \ + src/common/libor-crypto.a \ $(LIBKECCAK_TINY) \ $(LIBDONNA) \ - @TOR_LIB_MATH@ @TOR_ZLIB_LIBS@ @TOR_OPENSSL_LIBS@ \ - @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ + @TOR_LIB_MATH@ @TOR_ZLIB_LIBS@ @TOR_OPENSSL_LIBS@ \ + @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ EXTRA_DIST += src/tools/tor-fw-helper/README diff --git a/src/tools/tor-gencert.c b/src/tools/tor-gencert.c index 5f2cd3a92d..4ddfbc9657 100644 --- a/src/tools/tor-gencert.c +++ b/src/tools/tor-gencert.c @@ -13,19 +13,12 @@ #include <unistd.h> #endif -#ifdef __GNUC__ -#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -#endif +#include "compat.h" -#if __GNUC__ && GCC_VERSION >= 402 -#if GCC_VERSION >= 406 -#pragma GCC diagnostic push -#endif /* Some versions of OpenSSL declare X509_STORE_CTX_set_verify_cb twice in * x509.h and x509_vfy.h. Suppress the GCC warning so we can build with * -Wredundant-decl. */ -#pragma GCC diagnostic ignored "-Wredundant-decls" -#endif +DISABLE_GCC_WARNING(redundant-decls) #include <openssl/evp.h> #include <openssl/pem.h> @@ -34,13 +27,7 @@ #include <openssl/obj_mac.h> #include <openssl/err.h> -#if __GNUC__ && GCC_VERSION >= 402 -#if GCC_VERSION >= 406 -#pragma GCC diagnostic pop -#else -#pragma GCC diagnostic warning "-Wredundant-decls" -#endif -#endif +ENABLE_GCC_WARNING(redundant-decls) #include <errno.h> #if 0 @@ -61,21 +48,21 @@ #define DEFAULT_LIFETIME 12 /* These globals are set via command line options. */ -char *identity_key_file = NULL; -char *signing_key_file = NULL; -char *certificate_file = NULL; -int reuse_signing_key = 0; -int verbose = 0; -int make_new_id = 0; -int months_lifetime = DEFAULT_LIFETIME; -int passphrase_fd = -1; -char *address = NULL; - -char *passphrase = NULL; -size_t passphrase_len = 0; - -EVP_PKEY *identity_key = NULL; -EVP_PKEY *signing_key = NULL; +static char *identity_key_file = NULL; +static char *signing_key_file = NULL; +static char *certificate_file = NULL; +static int reuse_signing_key = 0; +static int verbose = 0; +static int make_new_id = 0; +static int months_lifetime = DEFAULT_LIFETIME; +static int passphrase_fd = -1; +static char *address = NULL; + +static char *passphrase = NULL; +static size_t passphrase_len = 0; + +static EVP_PKEY *identity_key = NULL; +static EVP_PKEY *signing_key = NULL; /** Write a usage message for tor-gencert to stderr. */ static void diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h index 610ef2f611..9469245ba7 100644 --- a/src/win32/orconfig.h +++ b/src/win32/orconfig.h @@ -229,7 +229,7 @@ #define USING_TWOS_COMPLEMENT /* Version number of package */ -#define VERSION "0.2.8.4-rc-dev" +#define VERSION "0.2.9.0-alpha-dev" |