diff options
218 files changed, 21110 insertions, 10880 deletions
diff --git a/.gitignore b/.gitignore index 46917c311f..44e6bd0c47 100644 --- a/.gitignore +++ b/.gitignore @@ -80,6 +80,7 @@ # /debian/ /debian/files +/debian/micro-revision.i /debian/patched /debian/tor /debian/tor.postinst.debhelper @@ -108,8 +109,10 @@ # /src/common/ /src/common/Makefile /src/common/Makefile.in +/src/common/common_sha1.i /src/common/libor.a /src/common/libor-crypto.a +/src/common/libor-event.a # /src/config/ /src/config/Makefile @@ -121,9 +124,16 @@ # /src/or/ /src/or/Makefile /src/or/Makefile.in +/src/or/or_sha1.i /src/or/micro-revision.* /src/or/tor -/src/or/test +/src/or/libtor.a + +# /src/test +/src/test/Makefile +/src/test/Makefile.in +/src/test/test + # /src/tools/ /src/tools/tor-checkkey @@ -1,3 +1,11 @@ +Changes in version 0.2.2.8-alpha - 2010-??-?? + o Minor bugfixes: + - Ignore OutboundBindAddress when connecting to localhost. + Connections to localhost need to come _from_ localhost, or else + local servers (like DNS and outgoing HTTP/SOCKS proxies) will often + refuse to listen. + + Changes in version 0.2.1.23 - 2010-0?-?? o Major bugfixes (performance): - We were selecting our guards uniformly at random, and then weighting @@ -16,6 +24,127 @@ Changes in version 0.2.1.23 - 2010-0?-?? over the behavior introduced in 0.1.2.17. +Changes in version 0.2.2.7-alpha - 2010-01-19 + o Directory authority changes: + - Rotate keys (both v3 identity and relay identity) for moria1 + and gabelmoo. + + o Major features (performance): + - We were selecting our guards uniformly at random, and then weighting + which of our guards we'd use uniformly at random. This imbalance + meant that Tor clients were severely limited on throughput (and + probably latency too) by the first hop in their circuit. Now we + select guards weighted by currently advertised bandwidth. We also + automatically discard guards picked using the old algorithm. Fixes + bug 1217; bugfix on 0.2.1.3-alpha. Found by Mike Perry. + - When choosing which cells to relay first, relays can now favor + circuits that have been quiet recently, to provide lower latency + for low-volume circuits. By default, relays enable or disable this + feature based on a setting in the consensus. You can override + this default by using the new "CircuitPriorityHalflife" config + option. Design and code by Ian Goldberg, Can Tang, and Chris + Alexander. + - Add separate per-conn write limiting to go with the per-conn read + limiting. We added a global write limit in Tor 0.1.2.5-alpha, + but never per-conn write limits. + - New consensus params "bwconnrate" and "bwconnburst" to let us + rate-limit client connections as they enter the network. It's + controlled in the consensus so we can turn it on and off for + experiments. It's starting out off. Based on proposal 163. + + o Major features (relay selection options): + - Switch to a StrictNodes config option, rather than the previous + "StrictEntryNodes" / "StrictExitNodes" separation that was missing a + "StrictExcludeNodes" option. + - If EntryNodes, ExitNodes, ExcludeNodes, or ExcludeExitNodes + change during a config reload, mark and discard all our origin + circuits. This fix should address edge cases where we change the + config options and but then choose a circuit that we created before + the change. + - If EntryNodes or ExitNodes are set, be more willing to use an + unsuitable (e.g. slow or unstable) circuit. The user asked for it, + they get it. + - Make EntryNodes config option much more aggressive even when + StrictNodes is not set. Before it would prepend your requested + entrynodes to your list of guard nodes, but feel free to use others + after that. Now it chooses only from your EntryNodes if any of + those are available, and only falls back to others if a) they're + all down and b) StrictNodes is not set. + - Now we refresh your entry guards from EntryNodes at each consensus + fetch -- rather than just at startup and then they slowly rot as + the network changes. + + o Major bugfixes: + - Stop bridge directory authorities from answering dbg-stability.txt + directory queries, which would let people fetch a list of all + bridge identities they track. Bugfix on 0.2.1.6-alpha. + + o Minor features: + - Log a notice when we get a new control connection. Now it's easier + for security-conscious users to recognize when a local application + is knocking on their controller door. Suggested by bug 1196. + - New config option "CircuitStreamTimeout" to override our internal + timeout schedule for how many seconds until we detach a stream from + a circuit and try a new circuit. If your network is particularly + slow, you might want to set this to a number like 60. + - New controller command "getinfo config-text". It returns the + contents that Tor would write if you send it a SAVECONF command, + so the controller can write the file to disk itself. + - New options for SafeLogging to allow scrubbing only log messages + generated while acting as a relay. + - Ship the bridges spec file in the tarball too. + - Avoid a mad rush at the beginning of each month when each client + rotates half of its guards. Instead we spread the rotation out + throughout the month, but we still avoid leaving a precise timestamp + in the state file about when we first picked the guard. Improves + over the behavior introduced in 0.1.2.17. + + o Minor bugfixes (compiling): + - Fix compilation on OS X 10.3, which has a stub mlockall() but + hides it. Bugfix on 0.2.2.6-alpha. + - Fix compilation on Solaris by removing support for the + DisableAllSwap config option. Solaris doesn't have an rlimit for + mlockall, so we cannot use it safely. Fixes bug 1198; bugfix on + 0.2.2.6-alpha. + + o Minor bugfixes (crashes): + - Do not segfault when writing buffer stats when we haven't observed + a single circuit to report about. Found by Fabian Lanze. Bugfix on + 0.2.2.1-alpha. + - If we're in the pathological case where there's no exit bandwidth + but there is non-exit bandwidth, or no guard bandwidth but there + is non-guard bandwidth, don't crash during path selection. Bugfix + on 0.2.0.3-alpha. + - Fix an impossible-to-actually-trigger buffer overflow in relay + descriptor generation. Bugfix on 0.1.0.15. + + o Minor bugfixes (privacy): + - Fix an instance where a Tor directory mirror might accidentally + log the IP address of a misbehaving Tor client. Bugfix on + 0.1.0.1-rc. + - Don't list Windows capabilities in relay descriptors. We never made + use of them, and maybe it's a bad idea to publish them. Bugfix + on 0.1.1.8-alpha. + + o Minor bugfixes (other): + - Resolve an edge case in path weighting that could make us misweight + our relay selection. Fixes bug 1203; bugfix on 0.0.8rc1. + - Fix statistics on client numbers by country as seen by bridges that + were broken in 0.2.2.1-alpha. Also switch to reporting full 24-hour + intervals instead of variable 12-to-48-hour intervals. + - After we free an internal connection structure, overwrite it + with a different memory value than we use for overwriting a freed + internal circuit structure. Should help with debugging. Suggested + by bug 1055. + - Update our OpenSSL 0.9.8l fix so that it works with OpenSSL 0.9.8m + too. + + o Removed features: + - Remove the HSAuthorityRecordStats option that version 0 hidden + service authorities could have used to track statistics of overall + hidden service usage. + + Changes in version 0.2.1.22 - 2010-01-19 Tor 0.2.1.22 fixes a critical privacy problem in bridge directory authorities -- it would tell you its whole history of bridge descriptors @@ -59,6 +188,69 @@ Changes in version 0.2.1.21 - 2009-12-21 trigger assert. Fixes bug 1173. +Changes in version 0.2.2.6-alpha - 2009-11-19 + Tor 0.2.2.6-alpha lays the groundwork for many upcoming features: + support for the new lower-footprint "microdescriptor" directory design, + future-proofing our consensus format against new hash functions or + other changes, and an Android port. It also makes Tor compatible with + the upcoming OpenSSL 0.9.8l release, and fixes a variety of bugs. + + o Major features: + - Directory authorities can now create, vote on, and serve multiple + parallel formats of directory data as part of their voting process. + Partially implements Proposal 162: "Publish the consensus in + multiple flavors". + - Directory authorities can now agree on and publish small summaries + of router information that clients can use in place of regular + server descriptors. This transition will eventually allow clients + to use far less bandwidth for downloading information about the + network. Begins the implementation of Proposal 158: "Clients + download consensus + microdescriptors". + - The directory voting system is now extensible to use multiple hash + algorithms for signatures and resource selection. Newer formats + are signed with SHA256, with a possibility for moving to a better + hash algorithm in the future. + - New DisableAllSwap option. If set to 1, Tor will attempt to lock all + current and future memory pages via mlockall(). On supported + platforms (modern Linux and probably BSD but not Windows or OS X), + this should effectively disable any and all attempts to page out + memory. This option requires that you start your Tor as root -- + if you use DisableAllSwap, please consider using the User option + to properly reduce the privileges of your Tor. + - Numerous changes, bugfixes, and workarounds from Nathan Freitas + to help Tor build correctly for Android phones. + + o Major bugfixes: + - Work around a security feature in OpenSSL 0.9.8l that prevents our + handshake from working unless we explicitly tell OpenSSL that we + are using SSL renegotiation safely. We are, but OpenSSL 0.9.8l + won't work unless we say we are. + + o Minor bugfixes: + - Fix a crash bug when trying to initialize the evdns module in + Libevent 2. Bugfix on 0.2.1.16-rc. + - Stop logging at severity 'warn' when some other Tor client tries + to establish a circuit with us using weak DH keys. It's a protocol + violation, but that doesn't mean ordinary users need to hear about + it. Fixes the bug part of bug 1114. Bugfix on 0.1.0.13. + - Do not refuse to learn about authority certs and v2 networkstatus + documents that are older than the latest consensus. This bug might + have degraded client bootstrapping. Bugfix on 0.2.0.10-alpha. + Spotted and fixed by xmux. + - Fix numerous small code-flaws found by Coverity Scan Rung 3. + - If all authorities restart at once right before a consensus vote, + nobody will vote about "Running", and clients will get a consensus + with no usable relays. Instead, authorities refuse to build a + consensus if this happens. Bugfix on 0.2.0.10-alpha; fixes bug 1066. + - If your relay can't keep up with the number of incoming create + cells, it would log one warning per failure into your logs. Limit + warnings to 1 per minute. Bugfix on 0.0.2pre10; fixes bug 1042. + - Bridges now use "reject *:*" as their default exit policy. Bugfix + on 0.2.0.3-alpha; fixes bug 1113. + - Fix a memory leak on directory authorities during voting that was + introduced in 0.2.2.1-alpha. Found via valgrind. + + Changes in version 0.2.1.20 - 2009-10-15 Tor 0.2.1.20 fixes a crash bug when you're accessing many hidden services at once, prepares for more performance improvements, and @@ -134,6 +326,262 @@ Changes in version 0.2.1.20 - 2009-10-15 getinfo and status events until we have a better design for them. +Changes in version 0.2.2.5-alpha - 2009-10-11 + Tor 0.2.2.5-alpha fixes a few compile problems in 0.2.2.4-alpha. + + o Major bugfixes: + - Make the tarball compile again. Oops. Bugfix on 0.2.2.4-alpha. + + o New directory authorities: + - Move dizum to an alternate IP address. + + +Changes in version 0.2.2.4-alpha - 2009-10-10 + Tor 0.2.2.4-alpha fixes more crash bugs in 0.2.2.2-alpha. It also + introduces a new unit test framework, shifts directry authority + addresses around to reduce the impact from recent blocking events, + and fixes a few smaller bugs. + + o Major bugfixes: + - Fix several more asserts in the circuit_build_times code, for + example one that causes Tor to fail to start once we have + accumulated 5000 build times in the state file. Bugfixes on + 0.2.2.2-alpha; fixes bug 1108. + + o New directory authorities: + - Move moria1 and Tonga to alternate IP addresses. + + o Minor features: + - Log SSL state transitions at debug level during handshake, and + include SSL states in error messages. This may help debug future + SSL handshake issues. + - Add a new "Handshake" log domain for activities that happen + during the TLS handshake. + - Revert to the "June 3 2009" ip-to-country file. The September one + seems to have removed most US IP addresses. + - Directory authorities now reject Tor relays with versions less than + 0.1.2.14. This step cuts out four relays from the current network, + none of which are very big. + + o Minor bugfixes: + - Fix a couple of smaller issues with gathering statistics. Bugfixes + on 0.2.2.1-alpha. + - Fix two memory leaks in the error case of + circuit_build_times_parse_state(). Bugfix on 0.2.2.2-alpha. + - Don't count one-hop circuits when we're estimating how long it + takes circuits to build on average. Otherwise we'll set our circuit + build timeout lower than we should. Bugfix on 0.2.2.2-alpha. + - Directory authorities no longer change their opinion of, or vote on, + whether a router is Running, unless they have themselves been + online long enough to have some idea. Bugfix on 0.2.0.6-alpha. + Fixes bug 1023. + + o Code simplifications and refactoring: + - Revise our unit tests to use the "tinytest" framework, so we + can run tests in their own processes, have smarter setup/teardown + code, and so on. The unit test code has moved to its own + subdirectory, and has been split into multiple modules. + + +Changes in version 0.2.2.3-alpha - 2009-09-23 + Tor 0.2.2.3-alpha fixes a few crash bugs in 0.2.2.2-alpha. + + o Major bugfixes: + - Fix an overzealous assert in our new circuit build timeout code. + Bugfix on 0.2.2.2-alpha; fixes bug 1103. + + o Minor bugfixes: + - If the networkstatus consensus tells us that we should use a + negative circuit package window, ignore it. Otherwise we'll + believe it and then trigger an assert. Bugfix on 0.2.2.2-alpha. + + +Changes in version 0.2.2.2-alpha - 2009-09-21 + Tor 0.2.2.2-alpha introduces our latest performance improvement for + clients: Tor tracks the average time it takes to build a circuit, and + avoids using circuits that take too long to build. For fast connections, + this feature can cut your expected latency in half. For slow or flaky + connections, it could ruin your Tor experience. Let us know if it does! + + o Major features: + - Tor now tracks how long it takes to build client-side circuits + over time, and adapts its timeout to local network performance. + Since a circuit that takes a long time to build will also provide + bad performance, we get significant latency improvements by + discarding the slowest 20% of circuits. Specifically, Tor creates + circuits more aggressively than usual until it has enough data + points for a good timeout estimate. Implements proposal 151. + We are especially looking for reports (good and bad) from users with + both EDGE and broadband connections that can move from broadband + to EDGE and find out if the build-time data in the .tor/state gets + reset without loss of Tor usability. You should also see a notice + log message telling you that Tor has reset its timeout. + - Directory authorities can now vote on arbitary integer values as + part of the consensus process. This is designed to help set + network-wide parameters. Implements proposal 167. + - Tor now reads the "circwindow" parameter out of the consensus, + and uses that value for its circuit package window rather than the + default of 1000 cells. Begins the implementation of proposal 168. + + o Major bugfixes: + - Fix a remotely triggerable memory leak when a consensus document + contains more than one signature from the same voter. Bugfix on + 0.2.0.3-alpha. + + o Minor bugfixes: + - Fix an extremely rare infinite recursion bug that could occur if + we tried to log a message after shutting down the log subsystem. + Found by Matt Edman. Bugfix on 0.2.0.16-alpha. + - Fix parsing for memory or time units given without a space between + the number and the unit. Bugfix on 0.2.2.1-alpha; fixes bug 1076. + - A networkstatus vote must contain exactly one signature. Spec + conformance issue. Bugfix on 0.2.0.3-alpha. + - Fix an obscure bug where hidden services on 64-bit big-endian + systems might mis-read the timestamp in v3 introduce cells, and + refuse to connect back to the client. Discovered by "rotor". + Bugfix on 0.2.1.6-alpha. + - We were triggering a CLOCK_SKEW controller status event whenever + we connect via the v2 connection protocol to any relay that has + a wrong clock. Instead, we should only inform the controller when + it's a trusted authority that claims our clock is wrong. Bugfix + on 0.2.0.20-rc; starts to fix bug 1074. Reported by SwissTorExit. + - We were telling the controller about CHECKING_REACHABILITY and + REACHABILITY_FAILED status events whenever we launch a testing + circuit or notice that one has failed. Instead, only tell the + controller when we want to inform the user of overall success or + overall failure. Bugfix on 0.1.2.6-alpha. Fixes bug 1075. Reported + by SwissTorExit. + - Don't warn when we're using a circuit that ends with a node + excluded in ExcludeExitNodes, but the circuit is not used to access + the outside world. This should help fix bug 1090, but more problems + remain. Bugfix on 0.2.1.6-alpha. + - Work around a small memory leak in some versions of OpenSSL that + stopped the memory used by the hostname TLS extension from being + freed. + - Make our 'torify' script more portable; if we have only one of + 'torsocks' or 'tsocks' installed, don't complain to the user; + and explain our warning about tsocks better. + + o Minor features: + - Add a "getinfo status/accepted-server-descriptor" controller + command, which is the recommended way for controllers to learn + whether our server descriptor has been successfully received by at + least on directory authority. Un-recommend good-server-descriptor + getinfo and status events until we have a better design for them. + - Update to the "September 4 2009" ip-to-country file. + + +Changes in version 0.2.2.1-alpha - 2009-08-26 + Tor 0.2.2.1-alpha disables ".exit" address notation by default, allows + Tor clients to bootstrap on networks where only port 80 is reachable, + makes it more straightforward to support hardware crypto accelerators, + and starts the groundwork for gathering stats safely at relays. + + o Security fixes: + - Start the process of disabling ".exit" address notation, since it + can be used for a variety of esoteric application-level attacks + on users. To reenable it, set "AllowDotExit 1" in your torrc. Fix + on 0.0.9rc5. + + o New directory authorities: + - Set up urras (run by Jacob Appelbaum) as the seventh v3 directory + authority. + + o Major features: + - New AccelName and AccelDir options add support for dynamic OpenSSL + hardware crypto acceleration engines. + - Tor now supports tunneling all of its outgoing connections over + a SOCKS proxy, using the SOCKS4Proxy and/or SOCKS5Proxy + configuration options. Code by Christopher Davis. + + o Major bugfixes: + - Send circuit or stream sendme cells when our window has decreased + by 100 cells, not when it has decreased by 101 cells. Bug uncovered + by Karsten when testing the "reduce circuit window" performance + patch. Bugfix on the 54th commit on Tor -- from July 2002, + before the release of Tor 0.0.0. This is the new winner of the + oldest-bug prize. + + o New options for gathering stats safely: + - Directories that set "DirReqStatistics 1" write statistics on + directory request to disk every 24 hours. As compared to the + --enable-geoip-stats flag in 0.2.1.x, there are a few improvements: + 1) stats are written to disk exactly every 24 hours; 2) estimated + shares of v2 and v3 requests are determined as mean values, not at + the end of a measurement period; 3) unresolved requests are listed + with country code '??'; 4) directories also measure download times. + - Exit nodes that set "ExitPortStatistics 1" write statistics on the + number of exit streams and transferred bytes per port to disk every + 24 hours. + - Relays that set "CellStatistics 1" write statistics on how long + cells spend in their circuit queues to disk every 24 hours. + - Entry nodes that set "EntryStatistics 1" write statistics on the + rough number and origins of connecting clients to disk every 24 + hours. + - Relays that write any of the above statistics to disk and set + "ExtraInfoStatistics 1" include the past 24 hours of statistics in + their extra-info documents. + + o Minor features: + - New --digests command-line switch to output the digests of the + source files Tor was built with. + - The "torify" script now uses torsocks where available. + - The memarea code now uses a sentinel value at the end of each area + to make sure nothing writes beyond the end of an area. This might + help debug some conceivable causes of bug 930. + - Time and memory units in the configuration file can now be set to + fractional units. For example, "2.5 GB" is now a valid value for + AccountingMax. + - Certain Tor clients (such as those behind check.torproject.org) may + want to fetch the consensus in an extra early manner. To enable this + a user may now set FetchDirInfoExtraEarly to 1. This also depends on + setting FetchDirInfoEarly to 1. Previous behavior will stay the same + as only certain clients who must have this information sooner should + set this option. + - Instead of adding the svn revision to the Tor version string, report + the git commit (when we're building from a git checkout). + + o Minor bugfixes: + - If any the v3 certs we download are unparseable, we should actually + notice the failure so we don't retry indefinitely. Bugfix on + 0.2.0.x; reported by "rotator". + - If the cached cert file is unparseable, warn but don't exit. + - Fix possible segmentation fault on directory authorities. Bugfix on + 0.2.1.14-rc. + - When Tor fails to parse a descriptor of any kind, dump it to disk. + Might help diagnosing bug 1051. + + o Deprecated and removed features: + - The controller no longer accepts the old obsolete "addr-mappings/" + or "unregistered-servers-" GETINFO values. + - Hidden services no longer publish version 0 descriptors, and clients + do not request or use version 0 descriptors. However, the old hidden + service authorities still accept and serve version 0 descriptors + when contacted by older hidden services/clients. + - The EXTENDED_EVENTS and VERBOSE_NAMES controller features are now + always on; using them is necessary for correct forward-compatible + controllers. + - Remove support for .noconnect style addresses. Nobody was using + them, and they provided another avenue for detecting Tor users + via application-level web tricks. + + o Packaging changes: + - Upgrade Vidalia from 0.1.15 to 0.2.3 in the Windows and OS X + installer bundles. See + https://trac.vidalia-project.net/browser/vidalia/tags/vidalia-0.2.3/CHANGELOG + for details of what's new in Vidalia 0.2.3. + - Windows Vidalia Bundle: update Privoxy from 3.0.6 to 3.0.14-beta. + - OS X Vidalia Bundle: move to Polipo 1.0.4 with Tor specific + configuration file, rather than the old Privoxy. + - OS X Vidalia Bundle: Vidalia, Tor, and Polipo are compiled as + x86-only for better compatibility with OS X 10.6, aka Snow Leopard. + - OS X Tor Expert Bundle: Tor is compiled as x86-only for + better compatibility with OS X 10.6, aka Snow Leopard. + - OS X Vidalia Bundle: The multi-package installer is now replaced + by a simple drag and drop to the /Applications folder. This change + occurred with the upgrade to Vidalia 0.2.3. + + Changes in version 0.2.1.19 - 2009-07-28 Tor 0.2.1.19 fixes a major bug with accessing and providing hidden services on Tor 0.2.1.3-alpha through 0.2.1.18. diff --git a/Doxyfile.in b/Doxyfile.in index b4d21c334d..24355f5f04 100644 --- a/Doxyfile.in +++ b/Doxyfile.in @@ -1,4 +1,3 @@ -# $Id$ # Doxyfile 1.5.1 # This file describes the settings to be used by the documentation system diff --git a/Makefile.am b/Makefile.am index 39ea570dd7..850ab71bb5 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,4 +1,3 @@ -# $Id$ # Copyright (c) 2001-2004, Roger Dingledine # Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson # Copyright (c) 2007-2009, The Tor Project, Inc. @@ -59,15 +58,17 @@ dist: check doxygen: doxygen && cd doc/doxygen/latex && make -test: - ./src/or/test +test: all + ./src/test/test -# Avoid strlcpy.c, strlcat.c, tree.h +# Avoid strlcpy.c, strlcat.c, aes.c, OpenBSD_malloc_Linux.c, sha256.c, +# eventdns.[hc], tinytest.[ch] check-spaces: ./contrib/checkSpace.pl -C \ src/common/*.h \ src/common/[^asO]*.c src/common/address.c \ - src/or/[^et]*.[ch] src/or/t*.c src/or/eventdns_tor.h + src/or/[^et]*.[ch] src/or/t*.c src/or/eventdns_tor.h \ + src/test/test*.[ch] check-docs: ./contrib/checkOptionDocs.pl diff --git a/acinclude.m4 b/acinclude.m4 index 76e992572c..3db25aa59a 100644 --- a/acinclude.m4 +++ b/acinclude.m4 @@ -1,4 +1,3 @@ -dnl $Id$ dnl Helper macros for Tor configure.in dnl Copyright (c) 2001-2004, Roger Dingledine dnl Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson diff --git a/configure.in b/configure.in index 92dc45fd68..80cdfc417a 100644 --- a/configure.in +++ b/configure.in @@ -1,11 +1,10 @@ -dnl $Id$ dnl Copyright (c) 2001-2004, Roger Dingledine dnl Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson dnl Copyright (c) 2007-2008, The Tor Project, Inc. dnl See LICENSE for licensing information AC_INIT -AM_INIT_AUTOMAKE(tor, 0.2.1.22) +AM_INIT_AUTOMAKE(tor, 0.2.2.7-alpha) AM_CONFIG_HEADER(orconfig.h) AC_CANONICAL_HOST @@ -90,13 +89,6 @@ case $host in ;; esac -AC_ARG_ENABLE(geoip-stats, - AS_HELP_STRING(--enable-geoip-stats, enable code for directories to collect per-country statistics)) - -if test "$enable_geoip_stats" = "yes"; then - AC_DEFINE(ENABLE_GEOIP_STATS, 1, [Defined if we try to collect per-country statistics]) -fi - AC_ARG_ENABLE(gcc-warnings, AS_HELP_STRING(--enable-gcc-warnings, enable verbose warnings)) AC_ARG_ENABLE(gcc-warnings-advisory, @@ -114,6 +106,12 @@ AC_PROG_CPP AC_PROG_MAKE_SET AC_PROG_RANLIB +dnl autoconf 2.59 appears not to support AC_PROG_SED +AC_CHECK_PROG([SED],[sed],[sed],[/bin/false]) + +AC_PATH_PROG([SHA1SUM], [sha1sum], none) +AC_PATH_PROG([OPENSSL], [openssl], none) + TORUSER=_tor AC_ARG_WITH(tor-user, [ --with-tor-user=NAME Specify username for tor daemon ], @@ -276,15 +274,20 @@ save_CPPFLAGS="$CPPFLAGS" LIBS="-levent $TOR_LIB_WS32 $LIBS" LDFLAGS="$TOR_LDFLAGS_libevent $LDFLAGS" CPPFLAGS="$TOR_CPPFLAGS_libevent $CPPFLAGS" -AC_CHECK_FUNCS(event_get_version event_get_method event_set_log_callback) +AC_CHECK_FUNCS(event_get_version event_get_version_number event_get_method event_set_log_callback evdns_set_outgoing_bind_address event_base_loopexit) AC_CHECK_MEMBERS([struct event.min_heap_idx], , , [#include <event.h> ]) +AC_CHECK_HEADERS(event2/event.h event2/dns.h) + LIBS="$save_LIBS" LDFLAGS="$save_LDFLAGS" CPPFLAGS="$save_CPPFLAGS" + +AM_CONDITIONAL(USE_EXTERNAL_EVDNS, test x$ac_cv_header_event2_dns_h = xyes) + if test "$enable_static_libevent" = "yes"; then if test "$tor_cv_library_libevent_dir" = "(system)"; then AC_MSG_ERROR("You must specify an explicit --with-libevent-dir=x option when using --enable-static-libevent") @@ -296,6 +299,7 @@ else fi AC_SUBST(TOR_LIBEVENT_LIBS) + dnl ------------------------------------------------------ dnl Where do you live, openssl? And how do we call you? @@ -656,6 +660,16 @@ if test x$tcmalloc = xyes ; then LDFLAGS="-ltcmalloc $LDFLAGS" fi +# By default, we're going to assume we don't have mlockall() +# bionic and other platforms have various broken mlockall subsystems. +# Some systems don't have a working mlockall, some aren't linkable, +# and some have it but don't declare it. +AC_CHECK_FUNCS(mlockall) +AC_CHECK_DECLS([mlockall], , , [ +#ifdef HAVE_SYS_MMAN_H +#include <sys/mman.h> +#endif]) + # Allow user to specify an alternate syslog facility AC_ARG_WITH(syslog-facility, [ --with-syslog-facility=LOG syslog facility to use (default=LOG_DAEMON)], @@ -858,7 +872,7 @@ fi CPPFLAGS="$CPPFLAGS $TOR_CPPFLAGS_libevent $TOR_CPPFLAGS_openssl $TOR_CPPFLAGS_zlib" -AC_CONFIG_FILES([Makefile tor.spec Doxyfile contrib/tor.sh contrib/torctl contrib/torify contrib/tor.logrotate contrib/Makefile contrib/osx/Makefile contrib/osx/TorBundleDesc.plist contrib/osx/TorBundleInfo.plist contrib/osx/TorDesc.plist contrib/osx/TorInfo.plist contrib/osx/TorStartupDesc.plist src/config/torrc.sample doc/tor.1 src/Makefile doc/Makefile doc/design-paper/Makefile doc/spec/Makefile src/config/Makefile src/common/Makefile src/or/Makefile src/win32/Makefile src/tools/Makefile contrib/suse/Makefile contrib/suse/tor.sh]) +AC_CONFIG_FILES([Makefile tor.spec Doxyfile contrib/tor.sh contrib/torctl contrib/torify contrib/tor.logrotate contrib/Makefile contrib/osx/Makefile contrib/osx/TorBundleDesc.plist contrib/osx/TorBundleInfo.plist contrib/osx/TorDesc.plist contrib/osx/TorInfo.plist contrib/osx/TorStartupDesc.plist src/config/torrc.sample doc/tor.1 src/Makefile doc/Makefile doc/design-paper/Makefile doc/spec/Makefile src/config/Makefile src/common/Makefile src/or/Makefile src/test/Makefile src/win32/Makefile src/tools/Makefile contrib/suse/Makefile contrib/suse/tor.sh]) AC_OUTPUT if test -x /usr/bin/perl && test -x ./contrib/updateVersions.pl ; then diff --git a/contrib/auto-naming/README b/contrib/auto-naming/README index 77e6af6483..e2f9ff8c2a 100644 --- a/contrib/auto-naming/README +++ b/contrib/auto-naming/README @@ -1,65 +1,6 @@ -=== AUTONAMING FOR TOR === - Tor directory authorities may maintain a binding of server identities -(their long term identity key) and nicknames. In their status documents -they may for each router they know tell if this is indeed the owner of -that nickname or not. - -This toolset allows automatic maintaining of a binding list of nicknames -to identity keys, implementing Tor proposal 123[1]. - -The rules are simple: - - A router claiming to be Bob is named (i.e. added to the binding list) - if there currently does not exist a different binding for that - nickname, the router has been around for a bit (2 weeks), no other - router has used that nickname in a while (1 month). - - A binding is removed if the server that owns it has not been seen - in a long time (6 months). - - -=== REQUIREMENTS === - - * ruby, and its postgres DBI interface (Debian packages: ruby, ruby1.8, libdbi-ruby1.8, libdbd-pg-ruby1.8) - * postgres (tested with >= 8.1) - * cron - -=== SETUP === - - * copy this tree some place, like into a 'auto-naming' directory in your Tor's - data directory - * create a database and a user, modifying db-config.rb accordingly - * initialize the database by executing the sql statements in create-db.sql - * setup a cronjob that feeds the current consensus to the process-consensus - script regularly. - * once the database is sufficiently populated, maybe a month or so after the - previous step, setup a cronjob to regularly build the binding list using - the build-approved-routers script. You probably want to append a manually - managed list of rejections to that file and give it to tor as its - "approved-routers" file. - The Sample-Makefile and Sample-crontab demonstrate the method used at tor26. - - -1. https://tor-svn.freehaven.net/svn/tor/trunk/doc/spec/proposals/123-autonaming.txt - - - - -Copyright (c) 2007 Peter Palfrader - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: +(their long term identity key) and nicknames. -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. +The auto-naming scripts have been moved to svn in +projects/tor-naming/auto-naming/trunk/ -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/contrib/auto-naming/Sample-Makefile b/contrib/auto-naming/Sample-Makefile deleted file mode 100644 index e3e0351df8..0000000000 --- a/contrib/auto-naming/Sample-Makefile +++ /dev/null @@ -1,20 +0,0 @@ - -all: ../approved-routers - -update: - wget -q -O - http://tor.noreply.org/tor/status-vote/current/consensus | \ - ./process-consensus - -.PHONY: approved-routers-auto -approved-routers-auto: - ./build-approved-routers > "$@" - -.INTERMEDIATE: approved-routers -approved-routers: approved-routers-auto /etc/tor/approved-routers - cat $^ > "$@" - -../approved-routers: approved-routers - if ! diff -q "$<" "$@"; then \ - mv "$<" "$@" &&\ - (! [ -e /var/run/tor/tor.pid ] || kill -HUP `cat /var/run/tor/tor.pid`) ; \ - fi diff --git a/contrib/auto-naming/Sample-crontab b/contrib/auto-naming/Sample-crontab deleted file mode 100644 index b50c07bb81..0000000000 --- a/contrib/auto-naming/Sample-crontab +++ /dev/null @@ -1,3 +0,0 @@ -MAILTO=admin -# cronjob for tor naming -23 * * * * make -s -C auto-naming update && make -s -C auto-naming diff --git a/contrib/auto-naming/build-approved-routers b/contrib/auto-naming/build-approved-routers deleted file mode 100755 index 805321f208..0000000000 --- a/contrib/auto-naming/build-approved-routers +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/ruby - -# build-approved-routers - create a name-binding list for use at a Tor -# directory authority -# -# Copyright (c) 2007 Peter Palfrader -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -require "yaml" - -require 'db' -require 'db-config' - -verbose = ARGV.first == "-v" - -db = Db.new($CONFIG['database']['dbhost'], $CONFIG['database']['dbname'], $CONFIG['database']['user'], $CONFIG['database']['password']) - -db.transaction_begin -named = db.query2(" - SELECT fingerprint, router_id, nickname_id, nick, first_seen, last_seen - FROM router NATURAL JOIN router_claims_nickname NATURAL JOIN nickname - WHERE named") -while (n=named.next) do - puts "# (r##{n['router_id']},n##{n['nickname_id']}); first_seen: #{n['first_seen']}, last_seen: #{n['last_seen']}" - fpr = n['fingerprint'].split(/(....)/).delete_if{|x| x=="" }.join(' ') - puts "#{n['nick']} #{fpr}" -end -db.transaction_commit diff --git a/contrib/auto-naming/create-db.sql b/contrib/auto-naming/create-db.sql deleted file mode 100644 index 86e3e63911..0000000000 --- a/contrib/auto-naming/create-db.sql +++ /dev/null @@ -1,50 +0,0 @@ - -CREATE TABLE router ( - router_id SERIAL PRIMARY KEY, - fingerprint CHAR(40) NOT NULL, - UNIQUE(fingerprint) -); --- already created implicitly due to unique contraint --- CREATE INDEX router_fingerprint ON router(fingerprint); - -CREATE TABLE nickname ( - nickname_id SERIAL PRIMARY KEY, - nick VARCHAR(30) NOT NULL, - UNIQUE(nick) -); --- already created implicitly due to unique contraint --- CREATE INDEX nickname_nick ON nickname(nick); - -CREATE TABLE router_claims_nickname ( - router_id INTEGER NOT NULL REFERENCES router(router_id) ON DELETE CASCADE, - nickname_id INTEGER NOT NULL REFERENCES nickname(nickname_id) ON DELETE CASCADE, - first_seen TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, - last_seen TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, - named BOOLEAN NOT NULL DEFAULT 'false', - UNIQUE(router_id, nickname_id) -); -CREATE INDEX router_claims_nickname_router_id ON router_claims_nickname(router_id); -CREATE INDEX router_claims_nickname_nickname_id ON router_claims_nickname(nickname_id); -CREATE INDEX router_claims_nickname_first_seen ON router_claims_nickname(first_seen); -CREATE INDEX router_claims_nickname_last_seen ON router_claims_nickname(last_seen); - - --- Copyright (c) 2007 Peter Palfrader --- --- Permission is hereby granted, free of charge, to any person obtaining a copy --- of this software and associated documentation files (the "Software"), to deal --- in the Software without restriction, including without limitation the rights --- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell --- copies of the Software, and to permit persons to whom the Software is --- furnished to do so, subject to the following conditions: --- --- The above copyright notice and this permission notice shall be included in --- all copies or substantial portions of the Software. --- --- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR --- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, --- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE --- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER --- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, --- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE --- SOFTWARE. diff --git a/contrib/auto-naming/db-config.rb b/contrib/auto-naming/db-config.rb deleted file mode 100644 index b1508c1b70..0000000000 --- a/contrib/auto-naming/db-config.rb +++ /dev/null @@ -1,8 +0,0 @@ -$CONFIG = {} unless $CONFIG -$CONFIG['database'] = {} unless $CONFIG['database'] - -# if you use postgres' "ident sameuser" auth set dbhost to '' -$CONFIG['database']['dbhost'] = 'localhost'; -$CONFIG['database']['dbname'] = 'tornaming'; -$CONFIG['database']['user'] = 'tornaming'; -$CONFIG['database']['password'] = 'x'; diff --git a/contrib/auto-naming/db.rb b/contrib/auto-naming/db.rb deleted file mode 100644 index 822a26bad7..0000000000 --- a/contrib/auto-naming/db.rb +++ /dev/null @@ -1,165 +0,0 @@ -#!/usr/bin/ruby - -# Copyright (c) 2006, 2007 Peter Palfrader -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -require "dbi" - -class WeaselDbQueryHandle - def initialize(sth) - @sth = sth - end - - def next() - row = @sth.fetch_hash - if row - return row - else - @sth.finish - return nil - end - end -end - -class Db - def initialize(host, database, user, password) - @dbh = DBI.connect("dbi:Pg:#{database}:#{host}", user, password); - @dbh['AutoCommit'] = false - @transaction = false - @pre_initial_transaction=true - end - - def do(query,*args) - @dbh.do(query,*args) - end - def transaction_begin() - @dbh.do("BEGIN") unless @pre_initial_transaction - @transaction = true - @pre_initial_transaction=false - end - def transaction_commit() - @dbh.do("COMMIT") - @transaction = false - end - def transaction_rollback() - @dbh.do("ROLLBACK") - end - def get_primarykey_name(table); - #return 'ref'; - return table+'_id'; - end - - def update(table, values, keys) - cols = [] - vals = [] - values.each_pair{ |k,v| - cols << "#{k}=?" - vals << v - } - - wheres = [] - keys.each_pair{ |k,v| - wheres << "#{k}=?" - vals << v - } - - throw "update value set empty" unless cols.size > 0 - throw "where clause empty" unless wheres.size > 0 - - query = "UPDATE #{table} SET #{cols.join(',')} WHERE #{wheres.join(' AND ')}" - transaction_begin unless transaction_before=@transaction - r = @dbh.do(query, *vals) - transaction_commit unless transaction_before - return r - end - - def update_row(table, values) - pk_name = get_primarykey_name(table); - throw "Ref not defined" unless values[pk_name] - return update(table, values.clone.delete_if{|k,v| k == pk_name}, { pk_name => values[pk_name] }); - end - def insert(table, values) - cols = values.keys - vals = values.values - qmarks = values.values.collect{ '?' } - - query = "INSERT INTO #{table} (#{cols.join(',')}) VALUES (#{qmarks.join(',')})" - transaction_begin unless transaction_before=@transaction - @dbh.do(query, *vals) - transaction_commit unless transaction_before - end - - def insert_row(table, values) - pk_name = get_primarykey_name(table); - if values[pk_name] - insert(table, values) - else - transaction_begin unless transaction_before=@transaction - row = query_row("SELECT nextval(pg_get_serial_sequence('#{table}', '#{pk_name}')) AS newref"); - throw "No newref?" unless row['newref'] - values[pk_name] = row['newref'] - insert(table, values); - transaction_commit unless transaction_before - end - end - def delete_row(table, ref) - pk_name = get_primarykey_name(table); - query = "DELETE FROM #{table} WHERE #{pk_name}=?" - transaction_begin unless transaction_before=@transaction - @dbh.do(query, ref) - transaction_commit unless transaction_before - end - def query(query, *params) - sth = @dbh.execute(query, *params) - while row = sth.fetch_hash - yield row - end - sth.finish - end - # nil if no results - # hash if one match - # throw otherwise - def query_row(query, *params) - sth = @dbh.execute(query, *params) - - row = sth.fetch_hash - if row == nil - sth.finish - return nil - elsif sth.fetch_hash != nil - sth.finish - throw "More than one result when querying for #{query}" - else - sth.finish - return row - end - end - def query_all(query, *params) - sth = @dbh.execute(query, *params) - - rows = sth.fetch_all - return nil if rows.size == 0 - return rows - end - def query2(query, *params) - sth = @dbh.execute(query, *params) - return WeaselDbQueryHandle.new(sth) - end -end diff --git a/contrib/auto-naming/process-consensus b/contrib/auto-naming/process-consensus deleted file mode 100755 index dc9d207e43..0000000000 --- a/contrib/auto-naming/process-consensus +++ /dev/null @@ -1,119 +0,0 @@ -#!/usr/bin/ruby - -# process-consensus - read a current consensus document, inserting the -# information into a database then calling -# update-named-status.rb to update the name-binding -# flags -# -# Copyright (c) 2007 Peter Palfrader -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -require "yaml" - -require 'db' -require 'db-config' -require 'update-named-status' - -$db = Db.new($CONFIG['database']['dbhost'], $CONFIG['database']['dbname'], $CONFIG['database']['user'], $CONFIG['database']['password']) - -$router_cache = {} -$nickname_cache = {} - -def parse_consensus consensus - ts = nil - routers = [] - consensus.each do |line| - (key, value) = line.split(' ',2) - case key - when "valid-after", "published": ts = DateTime.parse(value) - when "r": - (nick, fpr, _) = value.split(' ', 3) - nick.downcase! - next if nick == 'unnamed' - routers << { - 'nick' => nick, - 'fingerprint' => (fpr+'=').unpack('m').first.unpack('H*').first - } - end - end - throw "Did not find a timestamp" unless ts - throw "Did not find any routers" unless routers.size > 0 - return ts, routers -end - -def insert_routers_into_db(router, table, field, value) - pk = table+'_id' - row = $db.query_row("SELECT #{pk} FROM #{table} WHERE #{field}=?", value) - if row - return row[pk] - else - r = { field => value } - $db.insert_row( table, r ) - return r[pk] - end -end - -def handle_one_consensus(c) - puts "parsing..." if $verbose - timestamp, routers = parse_consensus c - puts "storing..." if $verbose - - routers.each do |router| - fpr = router['fingerprint'] - nick = router['nick'] - $router_cache[fpr] = router_id = ($router_cache[fpr] or insert_routers_into_db(router, 'router', 'fingerprint', router['fingerprint'])) - $nickname_cache[nick] = nickname_id = ($nickname_cache[nick] or insert_routers_into_db(router, 'nickname', 'nick', router['nick'])) - - row = $db.update( - 'router_claims_nickname', - { 'last_seen' => timestamp.to_s }, - { 'router_id' => router_id, 'nickname_id' => nickname_id} ) - case row - when 0: - $db.insert('router_claims_nickname', - { - 'first_seen' => timestamp.to_s, - 'last_seen' => timestamp.to_s, - 'router_id' => router_id, 'nickname_id' => nickname_id} ) - when 1: - else - throw "Update of router_claims_nickname returned unexpected number of affected rows(#{row})" - end - end -end - -$db.transaction_begin -if ARGV.first == '-v' - $verbose = true - ARGV.shift -end - -if ARGV.size == 0 - handle_one_consensus STDIN.readlines - do_update $verbose -else - ARGV.each do |filename| - puts filename if $verbose - handle_one_consensus File.new(filename).readlines - puts "updating..." if $verbose - do_update $verbose - end -end -$db.transaction_commit diff --git a/contrib/auto-naming/update-named-status.rb b/contrib/auto-naming/update-named-status.rb deleted file mode 100755 index b58b24d58f..0000000000 --- a/contrib/auto-naming/update-named-status.rb +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/ruby - -# update-named-status.rb - update the named status of routers in our database -# -# Copyright (c) 2007 Peter Palfrader -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -require "yaml" - -require 'db' -require 'db-config' - -def do_update(verbose) - now = $db.query_row("SELECT max(last_seen) AS max FROM router_claims_nickname")['max'] - unless now - STDERR.puts "Could not find the latest last_seen timestamp. Is the database empty still?" - return - end - now = "TIMESTAMP '" + now.to_s + "'" - - denamed = $db.do(" - UPDATE router_claims_nickname - SET named=false - WHERE named - AND last_seen < #{now} - INTERVAL '6 months'") - puts "de-named: #{denamed}" if verbose - - named = $db.do(" - UPDATE router_claims_nickname - SET named=true - WHERE NOT named - AND first_seen < #{now} - INTERVAL '2 weeks' - AND last_seen > #{now} - INTERVAL '2 days' - AND NOT EXISTS (SELECT * - FROM router_claims_nickname AS innertable - WHERE named - AND router_claims_nickname.nickname_id=innertable.nickname_id) "+ # if that nickname is already named, we lose. - " AND NOT EXISTS (SELECT * - FROM router_claims_nickname AS innertable - WHERE router_claims_nickname.nickname_id=innertable.nickname_id - AND router_claims_nickname.router_id <> innertable.router_id - AND last_seen > #{now} - INTERVAL '1 month') ") # if nobody else wanted that nickname in the last month we are set - puts "named: #{named}" if verbose -end - -if __FILE__ == $0 - $db = Db.new($CONFIG['database']['dbhost'], $CONFIG['database']['dbname'], $CONFIG['database']['user'], $CONFIG['database']['password']) - verbose = ARGV.first == "-v" - - $db.transaction_begin - do_update verbose - $db.transaction_commit -end diff --git a/contrib/checkOptionDocs.pl b/contrib/checkOptionDocs.pl index ca3fba55e3..c2e8757362 100755 --- a/contrib/checkOptionDocs.pl +++ b/contrib/checkOptionDocs.pl @@ -1,5 +1,4 @@ #!/usr/bin/perl -w -# $Id use strict; my %options = (); diff --git a/contrib/checkSpace.pl b/contrib/checkSpace.pl index 37f079c52b..db061a0828 100755 --- a/contrib/checkSpace.pl +++ b/contrib/checkSpace.pl @@ -42,9 +42,8 @@ for $fn (@ARGV) { $lastnil = 0; } ## Terminals are still 80 columns wide in my world. I refuse to - ## accept double-line lines. Except, of course, svn Id tags - ## can make us go long. - if (/^.{80}/ && !/\$Id: /) { + ## accept double-line lines. + if (/^.{80}/) { print " Wide:$fn:$.\n"; } ### Juju to skip over comments and strings, since the tests diff --git a/contrib/cross.sh b/contrib/cross.sh index e660be780d..a6085a400f 100755 --- a/contrib/cross.sh +++ b/contrib/cross.sh @@ -1,5 +1,4 @@ #!/bin/bash -# $Id$ # Copyright 2006 Michael Mohr with modifications by Roger Dingledine # See LICENSE for licensing information. @@ -186,7 +185,7 @@ if [ ! -z $STRIP ] then ${HOST_TRIPLET}strip \ src/or/tor \ - src/or/test \ + src/test/test \ src/tools/tor-resolve fi diff --git a/contrib/id_to_fp.c b/contrib/id_to_fp.c index 73395e16c1..55b025dfaf 100644 --- a/contrib/id_to_fp.c +++ b/contrib/id_to_fp.c @@ -1,5 +1,4 @@ /* Copyright 2006 Nick Mathewson; see LICENSE for licensing information */ -/* $Id$ */ /* id_to_fp.c : Helper for directory authority ops. When somebody sends us * a private key, this utility converts the private key into a fingerprint diff --git a/contrib/nagios-check-tor-authority-cert b/contrib/nagios-check-tor-authority-cert index 0e2c1d06c4..46dc7284b7 100755 --- a/contrib/nagios-check-tor-authority-cert +++ b/contrib/nagios-check-tor-authority-cert @@ -8,8 +8,6 @@ # Usage: nagios-check-tor-authority-cert <authority identity fingerprint> # e.g.: nagios-check-tor-authority-cert A9AC67E64B200BBF2FA26DF194AC0469E2A948C6 -# $Id$ - # Copyright (c) 2008 Peter Palfrader <peter@palfrader.org> # # Permission is hereby granted, free of charge, to any person obtaining diff --git a/contrib/osx/Tor b/contrib/osx/Tor index 0660fd7c8d..bcddc0c42b 100755 --- a/contrib/osx/Tor +++ b/contrib/osx/Tor @@ -25,9 +25,9 @@ if [ -x /usr/bin/sw_vers ]; then # the OS version OSVER=`/usr/bin/sw_vers | grep ProductVersion | cut -f2 | cut -d"." -f1,2` case "$OSVER" in - "10.6") ARCH="universal";; - "10.5") ARCH="universal";; - "10.4") ARCH="universal";; + "10.6") ARCH="i386";; + "10.5") ARCH="i386";; + "10.4") ARCH="i386";; "10.3") ARCH="ppc";; "10.2") ARCH="ppc";; "10.1") ARCH="ppc";; @@ -37,7 +37,7 @@ else ARCH="unknown" fi -if [ $ARCH != "universal" ]; then +if [ $ARCH != "i386" ]; then export EVENT_NOKQUEUE=1 fi diff --git a/contrib/osx/package.sh b/contrib/osx/package.sh index 040c7cd4c9..488bd27c1b 100644 --- a/contrib/osx/package.sh +++ b/contrib/osx/package.sh @@ -1,5 +1,4 @@ #!/bin/sh -# $Id$ # Copyright 2004-2005 Nick Mathewson. # Copyright 2005-2007 Andrew Lewman # Copyright 2008 The Tor Project, Inc. @@ -35,9 +34,9 @@ if [ -x /usr/bin/sw_vers ]; then # the OS version OSVER=`/usr/bin/sw_vers | grep ProductVersion | cut -f2 | cut -d"." -f1,2` case "$OSVER" in - "10.6") ARCH="universal";; - "10.5") ARCH="universal";; - "10.4") ARCH="universal";; + "10.6") ARCH="i386";; + "10.5") ARCH="i386";; + "10.4") ARCH="i386";; "10.3") ARCH="ppc";; "10.2") ARCH="ppc";; "10.1") ARCH="ppc";; diff --git a/contrib/polipo/package.sh b/contrib/polipo/package.sh index 83f74212be..4ec72c81d8 100644 --- a/contrib/polipo/package.sh +++ b/contrib/polipo/package.sh @@ -1,5 +1,4 @@ #!/bin/sh -# $Id: package.sh 8992 2006-12-23 03:12:09Z phobos $ # Copyright 2004-2005 Nick Mathewson & Andrew Lewman. # Copyright 2005-2008 Andrew Lewman # This is licensed under the Modified BSD License. diff --git a/contrib/rc.subr b/contrib/rc.subr index 117ae71d47..d757e89528 100644 --- a/contrib/rc.subr +++ b/contrib/rc.subr @@ -1,5 +1,4 @@ #!/bin/sh -# $Id$ # $FreeBSD: ports/security/tor-devel/files/tor.in,v 1.1 2006/02/17 22:21:25 mnag Exp $ # # (rc.subr written by Peter Thoenen for Net/FreeBSD) diff --git a/contrib/tor-mingw.nsi.in b/contrib/tor-mingw.nsi.in index a6124c8568..83bcf50b88 100644 --- a/contrib/tor-mingw.nsi.in +++ b/contrib/tor-mingw.nsi.in @@ -8,8 +8,7 @@ !include "LogicLib.nsh" !include "FileFunc.nsh" !insertmacro GetParameters - -!define VERSION "0.2.1.22" +!define VERSION "0.2.2.7-alpha" !define INSTALLER "tor-${VERSION}-win32.exe" !define WEBSITE "https://www.torproject.org/" !define LICENSE "LICENSE" diff --git a/contrib/tor-resolve.py b/contrib/tor-resolve.py index 919bc876cc..47ae1a0c38 100755 --- a/contrib/tor-resolve.py +++ b/contrib/tor-resolve.py @@ -1,5 +1,4 @@ #!/usr/bin/python -#$Id$ import socket import struct diff --git a/contrib/torify.1 b/contrib/torify.1 index b08d468451..9ae4e40d9d 100644 --- a/contrib/torify.1 +++ b/contrib/torify.1 @@ -1,22 +1,27 @@ .TH torify 1 "" Jan-2009 "" -.\" manual page by Peter Palfrader +.\" manual page by Peter Palfrader and Jacob Appelbaum .SH NAME .LP -torify \- wrapper for tsocks and tor +torify \- wrapper for torsocks or tsocks and tor .SH SYNOPSIS \fBtorify\fP\ \fIapplication\fP\ [\fIapplication's\ arguments\fP] .SH DESCRIPTION -\fBtorify\fR is a simple wrapper that calls tsocks with a tor specific +\fBtorify\fR is a simple wrapper that attempts to find the best underlying Tor +wrapper available on a system. It calls torsocks or tsocks with a tor specific configuration file. +torsocks is an improved wrapper that explictly rejects UDP, safely resolves DNS +lookups and properly socksifies your TCP connections. + tsocks itself is a wrapper between the tsocks library and the application that you would like to run socksified. -Please note that since tsocks uses LD_PRELOAD, torify cannot be applied +Please note that since both method use LD_PRELOAD, torify cannot be applied to suid binaries. +.SH WARNING You should also be aware that the way tsocks currently works only TCP connections are socksified. Be aware that this will in most circumstances not include hostname lookups which would still be routed through your @@ -25,8 +30,13 @@ normal system resolver to your usual resolving nameservers. The The Tor FAQ at https://wiki.torproject.org/noreply/TheOnionRouter/TorFAQ might have further information on this subject. +When used with torsocks, torify should not leak DNS requests or UDP data. + +Both will leak ICMP data. + .SH SEE ALSO .BR tor (1), .BR tor-resolve (1), +.BR torsocks (1), .BR tsocks (1), .BR tsocks.conf (5). diff --git a/contrib/torify.in b/contrib/torify.in index 05645fd07c..d430da8ce7 100755 --- a/contrib/torify.in +++ b/contrib/torify.in @@ -3,43 +3,69 @@ # Wrapper script for use of the tsocks(8) transparent socksification library # See the tsocks(1) and torify(1) manpages. -# Copyright (c) 2004, 2006 Peter Palfrader +# Copyright (c) 2004, 2006, 2009 Peter Palfrader # Modified by Jacob Appelbaum <jacob@appelbaum.net> April 16th 2006 # May be distributed under the same terms as Tor itself +# taken from Debian's Developer's Reference, 6.4 +pathfind() { + OLDIFS="$IFS" + IFS=: + for p in $PATH; do + if [ -x "$p/$*" ]; then + IFS="$OLDIFS" + return 0 + fi + done + IFS="$OLDIFS" + return 1 +} -# Define and ensure we have tsocks -# XXX: what if we don't have which? -TSOCKS="`which tsocks`" -if [ ! -x "$TSOCKS" ] -then - echo "$0: Can't find tsocks in PATH. Perhaps you haven't installed it?" >&2 - exit 1 +# Check for any argument list +if [ "$#" = 0 ]; then + echo "Usage: $0 [-hv] <command> [<options>...]" >&2 + exit 1 fi -# Check for any argument list -if [ "$#" = 0 ] -then - echo "Usage: $0 <command> [<options>...]" >&2 - exit 1 +if [ "$#" = 1 ] && ( [ "$1" = "-h" ] || [ "$1" = "--help" ] ); then + echo "Usage: $0 [-hv] <command> [<options>...]" + exit 0 fi -if [ "$#" = 1 ] && ( [ "$1" = "-h" ] || [ "$1" = "--help" ] ) -then - echo "Usage: $0 <command> [<options>...]" - exit 0 + +if [ "$1" = "-v" ] || [ "$1" = "--verbose" ]; then + verbose=1 + shift 1 +else + verbose=0 fi -# Define our tsocks config file -TSOCKS_CONF_FILE="@CONFDIR@/tor-tsocks.conf" -export TSOCKS_CONF_FILE +if pathfind torsocks; then + ! [ "$verbose" -ge 1 ] || echo "Using torsocks as socksifier." >&2 -# Check that we've got a tsocks config file -if [ -r "$TSOCKS_CONF_FILE" ] -then - exec tsocks "$@" - echo "$0: Failed to exec tsocks $@" >&2 + exec torsocks "$@" + echo "$0: Failed to exec torsocks $@" >&2 exit 1 + +elif pathfind tsocks; then + ! [ "$verbose" -ge 1 ] || echo "Using tsocks as socksifier." >&2 + + # Define our tsocks config file + TSOCKS_CONF_FILE="/etc/tor/tor-tsocks.conf" + export TSOCKS_CONF_FILE + + # Check that we've got a tsocks config file + if [ -r "$TSOCKS_CONF_FILE" ] + then + echo "WARNING: tsocks is known to leak DNS and UDP data. If you had torsocks we would use that." >&2 + exec tsocks "$@" + echo "$0: Failed to exec tsocks $@" >&2 + exit 1 + else + echo "$0: Missing tsocks configuration file \"$TSOCKS_CONF_FILE\"." >&2 + exit 1 + fi + else - echo "$0: Missing tsocks configuration file \"$TSOCKS_CONF_FILE\"." >&2 + echo "$0: Can't find either tsocks or torsocks in your PATH. Perhaps you haven't installed either?" >&2 exit 1 fi diff --git a/debian/changelog b/debian/changelog index 90afa018bb..11b2dc3815 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,8 +1,74 @@ -tor (0.2.1.20-1) unstable; urgency=low +tor (0.2.2.6-alpha-1) experimental; urgency=low * New upstream version. + - Drop debian/patches/0a58567c-work-with-reneg-ssl.dpatch + (part of upstream). - -- Peter Palfrader <weasel@debian.org> Fri, 13 Nov 2009 19:02:47 +0100 + -- Peter Palfrader <weasel@debian.org> Mon, 23 Nov 2009 18:52:04 +0100 + +tor (0.2.2.5-alpha-1) experimental; urgency=low + + * New upstream version. + * Pick 0a58567ce3418f410cf1dd0143dd3e56b4a4bd1f from master git tree: + - work with libssl that has renegotiation disabled by default. + (debian/patches/0a58567c-work-with-reneg-ssl.dpatch) + * Therefore build-depend on libssl-dev >= 0.9.8k-6. If we build against + earlier versions we will not work once libssl gets upgraded to a version + that disabled renegotiations. + * Change order of recommends from privoxy | polipo to polipo | privoxy. + * Allegedly echo -e is a bashism. Remove it from debian/rules, we don't + need it anyways (closes: #478631). + * Change the dependency on tsocks to torsocks | tsocks (see: #554717). + + -- Peter Palfrader <weasel@debian.org> Sun, 15 Nov 2009 11:04:02 +0100 + +tor (0.2.2.4-alpha-1) experimental; urgency=low + + * New upstream version. + * The testsuite moved from src/or/test to src/test/test, + but let's call it using "make check" now. + * Upstream failed to ship src/test/test.h. Ship it in debian/ and + manually copy it in place during configure and clean up in clean. + Let's not use the patch system as this will most likely be rectified + by next release. + + -- Peter Palfrader <weasel@debian.org> Sun, 11 Oct 2009 10:38:55 +0200 + +tor (0.2.2.3-alpha-1) experimental; urgency=low + + * New upstream version. + + -- Peter Palfrader <weasel@debian.org> Wed, 23 Sep 2009 10:27:40 +0200 + +tor (0.2.2.2-alpha-1) experimental; urgency=low + + * New upstream version. + * The files src/common/common_sha1.i src/or/or_sha1.i get changed + during the build - they contain the checksums of the individual + files that end up in the binary. Of couse changes only end up + in the debian diff.gz after building a second time in the same + directory. So, remove those files in clean to get both a cleaner + diff.gz and idempotent builds. + * If we have a debian/micro-revision.i, replace the one in src/or + with our copy so that this will be the revision that ends up in + the binary. This is an informational only version string, but + it'd be kinda nice if it was (more) accurate nonetheless. + . + Of course this won't help if people manually patch around but + it's still preferable to claiming we are exactly upstream's source. + . + If we are building directly out of a git tree, update + debian/micro-revision.i in the clean target. + + -- Peter Palfrader <weasel@debian.org> Mon, 21 Sep 2009 14:51:20 +0200 + +tor (0.2.2.1-alpha-1) experimental; urgency=low + + * New upstream version. + * Forward port patches/03_tor_manpage_in_section_8.dpatch. + * Forward port patches/06_add_compile_time_defaults.dpatch. + + -- Peter Palfrader <weasel@debian.org> Thu, 03 Sep 2009 15:10:26 +0200 tor (0.2.1.19-1) unstable; urgency=low diff --git a/debian/control b/debian/control index 79ba422307..a2d7332a5c 100644 --- a/debian/control +++ b/debian/control @@ -2,15 +2,15 @@ Source: tor Section: comm Priority: optional Maintainer: Peter Palfrader <weasel@debian.org> -Build-Depends: debhelper (>= 5), libssl-dev, dpatch, zlib1g-dev, libevent-dev (>= 1.1), texlive-base-bin, texlive-latex-base, texlive-fonts-recommended, transfig, ghostscript, binutils (>= 2.14.90.0.7) +Build-Depends: debhelper (>= 5), libssl-dev (>= 0.9.8k-6), dpatch, zlib1g-dev, libevent-dev (>= 1.1), texlive-base-bin, texlive-latex-base, texlive-fonts-recommended, transfig, ghostscript, binutils (>= 2.14.90.0.7) Standards-Version: 3.8.1 Homepage: https://www.torproject.org/ Package: tor Architecture: any -Depends: ${shlibs:Depends}, adduser, tsocks +Depends: ${shlibs:Depends}, adduser, torsocks | tsocks Conflicts: libssl0.9.8 (<< 0.9.8g-9) -Recommends: privoxy | polipo (>= 1), socat, logrotate, tor-geoipdb +Recommends: polipo (>= 1) | privoxy, socat, logrotate, tor-geoipdb Suggests: mixmaster, mixminion, anon-proxy Description: anonymizing overlay network for TCP Tor is a connection-based low-latency anonymous communication system which diff --git a/debian/patches/03_tor_manpage_in_section_8.dpatch b/debian/patches/03_tor_manpage_in_section_8.dpatch index 28bbf957c0..ece1fe67b2 100755 --- a/debian/patches/03_tor_manpage_in_section_8.dpatch +++ b/debian/patches/03_tor_manpage_in_section_8.dpatch @@ -26,20 +26,21 @@ exit 0 diff -urNad tor-0.1.1.5/contrib/torify.1 /tmp/dpep.fOA3Mm/tor-0.1.1.5/contrib/torify.1 --- tor-0.1.1.5/contrib/torify.1 +++ /tmp/dpep.fOA3Mm/tor-0.1.1.5/contrib/torify.1 -@@ -18,6 +18,6 @@ +@@ -35,7 +35,7 @@ to suid binaries. .SH SEE ALSO -.BR tor (1), +.BR tor (8), .BR tor-resolve (1), + .BR torsocks (1), .BR tsocks (1), diff -urNad tor-0.1.1.5/doc/tor.1.in /tmp/dpep.fOA3Mm/tor-0.1.1.5/doc/tor.1.in --- tor-0.1.1.5/doc/tor.1.in +++ /tmp/dpep.fOA3Mm/tor-0.1.1.5/doc/tor.1.in @@ -1,4 +1,4 @@ --.TH TOR 1 "January 2009" "TOR" -+.TH TOR 8 "January 2009" "TOR" +-.TH TOR 1 "August 2009" "TOR" ++.TH TOR 8 "August 2009" "TOR" .SH NAME tor \- The second-generation onion router .SH SYNOPSIS diff --git a/debian/patches/06_add_compile_time_defaults.dpatch b/debian/patches/06_add_compile_time_defaults.dpatch index a2472d1db8..e64d4618af 100755 --- a/debian/patches/06_add_compile_time_defaults.dpatch +++ b/debian/patches/06_add_compile_time_defaults.dpatch @@ -23,9 +23,9 @@ esac exit 0 @DPATCH@ -diff -urNad tor-trunk~/src/or/config.c tor-trunk/src/or/config.c ---- tor-trunk~/src/or/config.c 2009-01-18 01:47:33.000000000 +0100 -+++ tor-trunk/src/or/config.c 2009-02-05 00:25:17.614844812 +0100 +diff -urNad tor~/src/or/config.c tor/src/or/config.c +--- tor~/src/or/config.c 2009-09-03 15:05:41.000000000 +0200 ++++ tor/src/or/config.c 2009-09-03 15:09:37.662104166 +0200 @@ -12,6 +12,7 @@ #define CONFIG_PRIVATE @@ -34,16 +34,17 @@ diff -urNad tor-trunk~/src/or/config.c tor-trunk/src/or/config.c #ifdef MS_WINDOWS #include <shlobj.h> #endif -@@ -711,6 +712,8 @@ - #if defined(HAVE_EVENT_GET_VERSION) && defined(HAVE_EVENT_GET_METHOD) - static void check_libevent_version(const char *m, int server); - #endif +@@ -717,6 +718,9 @@ + static void init_libevent(void); + static int opt_streq(const char *s1, const char *s2); + +static int debian_running_as_debiantor(); +static int debian_config_fix_defaults(); - ++ /** Magic value for or_options_t. */ #define OR_OPTIONS_MAGIC 9090909 -@@ -3917,6 +3920,9 @@ + +@@ -4086,6 +4090,9 @@ char *command_arg = NULL; char *errmsg=NULL; @@ -53,7 +54,7 @@ diff -urNad tor-trunk~/src/or/config.c tor-trunk/src/or/config.c if (argv) { /* first time we're called. save command line args */ backup_argv = argv; backup_argc = argc; -@@ -5307,3 +5313,62 @@ +@@ -5304,3 +5311,62 @@ return 0; } diff --git a/debian/rules b/debian/rules index f5aaa95545..28d7359521 100755 --- a/debian/rules +++ b/debian/rules @@ -66,6 +66,11 @@ endif configure: patch-stamp config.status: configure + # clean up test.h stuff. XXX - expected to no longer be needed after tor-0.2.2.4-alpha + rm -f src/test/test.h.orig + ! [ -e src/test/test.h ] || mv src/test/test.h src/test/test.h.orig + cp debian/src-test-test.h src/test/test.h + @if [ "$(LOCALHOST_IP)" != "127.0.0.1" ]; then echo; echo; echo; echo; echo; echo "######################################################################"; echo "WARNING: This system does not think localhost is 127.0.0.1. Will ignore result of testsuite. Please fix your system/chroot."; echo "######################################################################"; echo; echo; echo; echo; echo "Note: 'getent hosts localhost' should return '127.0.0.1 localhost'"; echo; fi dh_testdir CFLAGS="$(CFLAGS)" ./configure \ @@ -82,6 +87,7 @@ build: build-stamp build-stamp: config.status dh_testdir + ! [ debian/micro-revision.i ] || cp debian/micro-revision.i src/or/micro-revision.i $(MAKE) @echo @@ -90,14 +96,14 @@ build-stamp: config.status @if [ "$(RUN_TEST)" != "no" ]; then \ if [ "$(LOCALHOST_IP)" != "127.0.0.1" ]; then \ echo; echo; echo "######################################################################"; echo "WARNING: This system does not think localhost is 127.0.0.1. Will ignore result of testsuite. Please fix your system/chroot."; echo "######################################################################"; echo; echo; \ - echo "src/or/test || true"; \ - src/or/test || true; \ + echo "make check || true"; \ + make check || true; \ else \ - echo "src/or/test"; \ - src/or/test; \ + echo "make check"; \ + make check; \ fi; \ else \ - echo -e "\n\nSkipping unittests\n\n"; \ + echo; echo; echo "Skipping unittests"; echo; \ fi @echo @@ -129,9 +135,20 @@ clean: unpatch dh_testdir dh_testroot rm -f build-stamp + rm -f src/common/common_sha1.i src/or/or_sha1.i + rm -f src/or/micro-revision.i [ ! -f Makefile ] || $(MAKE) distclean + # clean up test.h stuff. XXX - expected to no longer be needed after tor-0.2.2.4-alpha + rm -f src/test/test.h + ! [ -e src/test/test.h.orig ] || mv src/test/test.h.orig src/test/test.h + + # Normally the .deb wouldn't ship with a ../.git + if [ -d .git ] && which git >/dev/null; then \ + echo "\"`git rev-parse --short=16 HEAD`\"" > "debian/micro-revision.i" ; \ + fi + dh_clean install: build diff --git a/debian/src-test-test.h b/debian/src-test-test.h new file mode 100644 index 0000000000..ed0eb316ad --- /dev/null +++ b/debian/src-test-test.h @@ -0,0 +1,75 @@ +/* Copyright (c) 2001-2003, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2009, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef _TOR_TEST_H +#define _TOR_TEST_H + +/** + * \file test.h + * \brief Macros and functions used by unit tests. + */ + +#include "compat.h" +#include "tinytest.h" +#define TT_EXIT_TEST_FUNCTION STMT_BEGIN goto done; STMT_END +#include "tinytest_macros.h" + +#ifdef __GNUC__ +#define PRETTY_FUNCTION __PRETTY_FUNCTION__ +#else +#define PRETTY_FUNCTION "" +#endif + +#define test_fail_msg(msg) TT_DIE((msg)) + +#define test_fail() test_fail_msg("Assertion failed.") + +#define test_assert(expr) tt_assert(expr) + +#define test_eq(expr1, expr2) tt_int_op((expr1), ==, (expr2)) +#define test_eq_ptr(expr1, expr2) tt_ptr_op((expr1), ==, (expr2)) +#define test_neq(expr1, expr2) tt_int_op((expr1), !=, (expr2)) +#define test_neq_ptr(expr1, expr2) tt_ptr_op((expr1), !=, (expr2)) +#define test_streq(expr1, expr2) tt_str_op((expr1), ==, (expr2)) +#define test_strneq(expr1, expr2) tt_str_op((expr1), !=, (expr2)) +#define test_streq(expr1, expr2) tt_str_op((expr1), ==, (expr2)) + +#define test_mem_op(expr1, op, expr2, len) \ + tt_assert_test_fmt_type(expr1,expr2,#expr1" "#op" "#expr2, \ + const char *, \ + (memcmp(_val1, _val2, len) op 0), \ + char *, "%s", \ + { size_t printlen = (len)*2+1; \ + _print = tor_malloc(printlen); \ + base16_encode(_print, printlen, _value, \ + (len)); }, \ + { tor_free(_print); } \ + ); + +#define test_memeq(expr1, expr2, len) test_mem_op((expr1), ==, (expr2), len) +#define test_memneq(expr1, expr2, len) test_mem_op((expr1), !=, (expr2), len) + +/* As test_mem_op, but decodes 'hex' before comparing. There must be a + * local char* variable called mem_op_hex_tmp for this to work. */ +#define test_mem_op_hex(expr1, op, hex) \ + STMT_BEGIN \ + size_t length = strlen(hex); \ + tor_free(mem_op_hex_tmp); \ + mem_op_hex_tmp = tor_malloc(length/2); \ + tor_assert((length&1)==0); \ + base16_decode(mem_op_hex_tmp, length/2, hex, length); \ + test_mem_op(expr1, op, mem_op_hex_tmp, length/2); \ + STMT_END + +#define test_memeq_hex(expr1, hex) test_mem_op_hex(expr1, ==, hex) + +const char *get_fname(const char *name); +crypto_pk_env_t *pk_generate(int idx); + +void legacy_test_helper(void *data); +extern const struct testcase_setup_t legacy_setup; + +#endif + diff --git a/doc/FAQ b/doc/FAQ deleted file mode 100644 index 3a7db3ae66..0000000000 --- a/doc/FAQ +++ /dev/null @@ -1,4 +0,0 @@ -This file is obsolete. Check out the online FAQ at the wiki -for more accurate and complete questions and answers: -http://wiki.noreply.org/wiki/TheOnionRouter/TorFAQ - diff --git a/doc/HACKING b/doc/HACKING index 50b5d80d18..6e6f020628 100644 --- a/doc/HACKING +++ b/doc/HACKING @@ -3,25 +3,30 @@ 0.0 The buildbot. - http://tor-buildbot.freehaven.net:8010/ - - - Down because nickm isn't running services at home any more. ioerror says - he will resurrect it. + https://buildbot.vidalia-project.net/one_line_per_build 0.1. Useful command-lines that are non-trivial to reproduce but can help with tracking bugs or leaks. +0.1.1. Dmalloc + dmalloc -l ~/dmalloc.log (run the commands it tells you) ./configure --with-dmalloc +0.2.2. Valgrind + valgrind --leak-check=yes --error-limit=no --show-reachable=yes src/or/tor +(Note that if you get a zillion openssl warnings, you will also need to + pass --undef-value-errors=no to valgrind, or rebuild your openssl + with -DPURIFY.) + 0.2. Running gcov for unit test coverage make clean make CFLAGS='-g -fprofile-arcs -ftest-coverage' - ./src/or/test + ./src/test/test cd src/common; gcov *.[ch] cd ../or; gcov *.[ch] @@ -145,7 +150,7 @@ valgrind --leak-check=yes --error-limit=no --show-reachable=yes src/or/tor 1.4. Log conventions - http://wiki.noreply.org/noreply/TheOnionRouter/TorFAQ#LogLevels + https://wiki.torproject.org/noreply/TheOnionRouter/TorFAQ#LogLevels No error or warning messages should be expected during normal OR or OP operation. @@ -4,8 +4,8 @@ We've split out our TODO into three files: TODO.02x is the list of items we're planning to get done in the next stable release. -TODO.external is the list of external constraints and deliverables that -we all need to keep in mind. +TODO.external lives in svn under /projects/todo/. It's the list of +external constraints and deliverables that we all need to keep in mind. TODO.future is the list of other items we plan to get to in later releases. diff --git a/doc/TODO.021 b/doc/TODO.021 index 881ba5ee4b..37c5b9845b 100644 --- a/doc/TODO.021 +++ b/doc/TODO.021 @@ -1,4 +1,3 @@ -$Id$ Legend: SPEC!! - Not specified SPEC - Spec not finalized diff --git a/doc/TODO.022 b/doc/TODO.022 index 3eeae006cb..f4fe2ebb2a 100644 --- a/doc/TODO.022 +++ b/doc/TODO.022 @@ -8,14 +8,17 @@ NOTE 2: It's easy to list stuff like this with no time estimates and 0.2.2, figure out how long the stuff we want will take, and triage accordingly, or vice versa. -- Design +- Design only - Begin design work for UDP transition; identify areas where we need to make changes or instrument stuff early. + [multiple weeks, ongoing. Need to do a draft early.] - Performance, mostly protocol-neutral. - Work with Libevent 2.0's bufferevent interface - Identify any performance stuff we need to push back into libevent to make it as fast as we want. + - Get a decent rate-limiting feature into Libevent + - Get openssl support into Libevent. - Revise how we do bandwidth limiting and round-robining between circuits on a connection. @@ -30,21 +33,76 @@ NOTE 2: It's easy to list stuff like this with no time estimates and - Figure out good ways to instrument Tor internals so we can tell how well our bandwidth and flow-control stuff is actually working. + - What ports eat the bandwidth? + - How full do queues get? + - How much latency do queues get? -- Features + - Rate limit at clients: + - Give clients an upper bound on how much they're willing to use + the network if they're not relaying? + - ... or group client circuits by IP at the server and rate-limit + like that. + + - Use if-modified-since to download consensuses + + +- Other features - Proposals to implement: - - 146: reflect long-term stability + - 146: reflect long-term stability in consensuses - 147: Stop using v2 directories to generate v3 votes. + - Start pinging as soon as we learn about a relay, not on a + 22-minute cycle. Prioritize new and volatile relays for + testing. - Proposals to improve and implement - 158: microdescriptors + o Revise proposal + - Implement + o 160: list bandwidth in consensus + o Finish proposal + o and actually set it reasonably + o and actually use it. - Proposals to improve and implement if not broken - - IPv6 support. (Parts of 117, but figure out how to handle DNS + D IPv6 support. (Parts of 117, but figure out how to handle DNS requests.) - 140: Directory diffs + - Need a decent simple C diff implementation. + - Need a decent simple C ed patch implementation. - 149: learn info from netinfo cells. - - 134: handle authority fragmentation (Needs more analysis) + o Start discussion + - Revise proposal based on discussion. + X 134: handle authority fragmentation (Needs more analysis) + - 165: Easy migration for voting authority sets + - 163: Detect client-status better + o Write proposal + - Possibly implement, depending on discussion. + - 164: Have authorities report relay and voting status better: make it + easy to answer, "Why is my server not listed/not Guard/not + Running/etc" + o Write proposal + - Possibly implement, depending on discussion + - 162: Have consensuses come in multiple "flavours". + o Write proposal + - Possibly implement, depending on discussion. + + - Needs a proposal, or at least some design + - Weaken the requirements for being a Guard, based on K's + measurements. +K - Finish measurements +K? - Write proposal + - Adaptive timeouts for giving up on circuits and streams. +M - Revise proposal 151 + - Downweight guards more sensibly: be more forgiving about using + Guard nodes as non-first-hop. + - Write proposal. + - Lagged weight updates in consensuses: don't just move abruptly. +M? - Write proposal + d Don't kill a circuit on the first failed extend. + +- Installers + - Switch to MSI on win32 + - Use Thandy, perhaps? - Deprecations - Make .exit safe, or make it off-by-default. diff --git a/doc/TODO.external b/doc/TODO.external index c02d6aca54..2e7e536efc 100644 --- a/doc/TODO.external +++ b/doc/TODO.external @@ -1,196 +1,4 @@ -$Id$ -Legend: -SPEC!! - Not specified -SPEC - Spec not finalized -N - nick claims -R - arma claims -P - phobos claims -S - Steven claims -E - Matt claims -M - Mike claims -J - Jeff claims -I - ioerror claims -W - weasel claims -K - Karsten claims -C - coderman claims - - Not done - * Top priority - . Partially done - o Done - d Deferrable - D Deferred - X Abandoned -======================================================================= - -External constraints: - -For June/July: -NR - Work more on Paul's NRL research problem. - -For March 22: -I * Email auto-responder - * teach gettor how to ask for (and attach) split files. - -K . Metrics. - . With Mike's help, use Torflow to start doing monthly rudimentary - performance evaluations: - . Circuit throughput and latency - - Measure via Broadband and dialup - . Publish a report addressing key long-term metrics questions: - . What metrics should we present? - . What data are available for these metrics? - . What data are missing, and can collect them safely? Can we - publish them safely? - . What systems are available to present this data? - -E . Vidalia improvements - o Vidalia displays by-country user summary for bridge operators -? - write a help page for vidalia, "what is this" - -For mid August: - -Section 0, items that didn't make it into the original roadmap: - -0.1, installers and packaging -C . i18n for the msi bundle files -P . more consistent TBB builds -IC- get a buildbot up again. Have Linux and BSD build machines. - (Windows would be nice but realistically will come later.) -E - Get Tor to work properly on the iPhone. - -3.1, performance work. [Section numbers in here are from performance.pdf] - - High-priority items from performance.pdf -RS - 1.2, new circuit window sizes. make the default package window lower. -R+ - 2.1, squeeze loud circuits - - Evaluate the code to see what stats we can keep about circuit use. - - Write proposals for various meddling. Look at the research papers - that Juliusz pointed us to. Ask our systems friends. Plan to put - a lot of the parameters in the consensus, so we can tune it with - short turnaround times. -E+ - 2.5, Change Vidalia's default exit policy to not click "other - protocols". Or choose not to. Think this through first. -R+ - 2.6, Tell users not to file-share. - - Put statement on the Tor front page - - Put statement on the download pages too - - And the FAQ - - 3.1.2, Tor weather -I - Implement time-to-notification (immediate, a day, a week) -I - Get a relay operator mailing list going, with a plan and supporting - scripts and so on. -R - Link to them from the Tor relay page -R - and the torrc.sample? -SM - 4.1, balance traffic better - - Steven and Mike should decide if we should do Steven's plan - (rejigger the bandwidth numbers at the authorities based on - Steven's algorithm), or Mike's plan (relay scanning to identify - the unbalanced relays and fix them on the fly), or both. - - Figure out how to actually modify bandwidths in the consensus. We - may need to change the consensus voting algorithm to decide what - bandwidth to advertise based on something other than median: - if 7 authorities provide bandwidths, and 2 are doing scanning, - then the 5 that aren't scanning will outvote any changes. Should - all 7 scan? Should only some vote? Extra points if it doesn't - change all the numbers every new consensus, so consensus diffing - is still practical. -? - 4.5, Older entry guards are overloaded - - Pick a conservative timeout like a month, and implement. -M - 5.2, better timeouts for giving up on circuits/streams - - clients gather data about circuit timeouts, and then abandon - circuits that take more than a std dev above that. - -4.1, IOCP / libevent / windows / tor -N - get it working for nick -N - put out a release so other people can start testing it. -N - both the libevent buffer abstraction, and the - tor-uses-libevent-buffer-abstraction. Unless we think that's - unreachable for this milestone? - -4.2.1, risks from becoming a relay -S - Have a clear plan for how users who become relays will be safe, - and be confident that we can build this plan. - - evaluate all the various attacks that are made possible by relaying. - specifically, see "relaying-traffic attacks" in 6.6. - - identify and evaluate ways to make them not a big deal - - setting a low RelayBandwidth - - Nick Hopper's FC08 paper suggesting that we should do a modified - round-robin so we leak less about other circuits - - instructing clients to disable pings in their firewall, etc - - pick the promising ones, improve them so they're even better, and - spec them out so we know how to build them and how much effort is - involved in building them. - -4.5, clients download less directory info -N * deploy proposal 158. -N - decide whether to do proposal 140. if so, construct an implementation - plan for how we'll do it. if not, explain why not. - -5.1, Normalize TLS fingerprint -N o write a draft list of possible attacks for this section, with - estimates about difficulty of attack, difficulty of solution, etc -N - revisit the list and revise our plans as needed -NR- put up a blog post about the two contradictory conclusions: we can - discuss the theory of arms races, and our quandry, without revealing - any specific vulnerabilities. (or decide not to put up a blog post, - and explain why not.) - -5.5, email autoresponder -I . maintenance and keeping it running - -5.7.2, metrics - -XXX. - -6.2, Vidalia work -E - add breakpad support or similar for windows debugging -E o let vidalia change languages without needing a restart -E - Implement the status warning event interface started for the - phase one deliverables. -E - Work with Steve Tyree on building a Vidalia plugin API to enable - building Herdict and TBB plugins. - -6.3, Node scanning -M - Steps toward automation - - Set up email list for results - - Map failure types to potential BadExit lines -M - Improve the ability of SoaT to mimic various real web browsers - - randomizing user agents and locale strings - - caching, XMLHTTPRequest, form posting, content sniffing - - Investigate ideas like running Chrome/xulrunner in parallel -M - Other protocols - - SSH, IMAPS, POPS, SMTPS -M - Add ability to geolocalize exit selection based on scanner location - - Use this to rescan dynamic urls filtered by the URL filter - -6.4, Torbutton development -M - Resolve extension conflicts and other high priority bugs -M - Fix or hack around ugly firefox bugs, especially Timezone issue. - Definitely leaning towards "hack around" unless we see some - level of love from Mozilla. -M - Vidalia New Nym Integration - - Implement for Torbutton to pick up on Vidalia's NEWNYM and clear - cookies based on FoeBud's source - - Do this in such a way that we could adapt polipo to purge cache - if we were so inclined -M - Write up a summary of our options for dealing with the google - you-must-solve-a-captcha-to-search problem, and pick one as our - favorite option. - -6.6, Evaluate new anonymity attacks -S - relaying-traffic attacks - - original murdoch-danezis attack - - nick hopper's latency measurement attack - - columbia bandwidth measurement attack - - christian grothoff's long-circuit attack -S - client attacks - - website fingerprinting - -7.1, Tor VM Research, analysis, and prototyping -C . Get a working package out, meaning other people are testing it. - -7.2, Tor Browser Bundle -I - Port to one of OS X or Linux, and start the port to the other. -I . Make it the recommended Tor download on Windows -I - Make sure it's easy to un-brand TBB in case Firefox asks us to -I - Evaluate CCC's Freedom Stick +[This file moved to svn in /projects/todo/. More people can edit +it more easily there. -RD] diff --git a/doc/TODO.future b/doc/TODO.future index 64169ecfec..4220799574 100644 --- a/doc/TODO.future +++ b/doc/TODO.future @@ -1,4 +1,3 @@ -$Id$ Legend: SPEC!! - Not specified SPEC - Spec not finalized @@ -273,7 +272,7 @@ Future versions: the RPM and other startup scripts should too? - add a "default.action" file to the tor/vidalia bundle so we can fix the https thing in the default configuration: - http://wiki.noreply.org/noreply/TheOnionRouter/TorFAQ#PrivoxyWeirdSSLPort + https://wiki.torproject.org/noreply/TheOnionRouter/TorFAQ#PrivoxyWeirdSSLPort ======================================================================= diff --git a/doc/design-paper/latex8.bst b/doc/design-paper/latex8.bst index 2dd3249633..bae8e209ee 100644 --- a/doc/design-paper/latex8.bst +++ b/doc/design-paper/latex8.bst @@ -1,8 +1,6 @@ % --------------------------------------------------------------- % -% $Id$ -% % by Paolo.Ienne@di.epfl.ch % diff --git a/doc/design-paper/usenix.sty b/doc/design-paper/usenix.sty index 4442f11574..575c854e77 100644 --- a/doc/design-paper/usenix.sty +++ b/doc/design-paper/usenix.sty @@ -5,8 +5,6 @@ % \usepackage{usenix-2e} % and put {\rm ....} around the author names. % -% $Id$ -% % The following definitions are modifications of standard article.sty % definitions, arranged to do a better job of matching the USENIX % guidelines. diff --git a/doc/spec/Makefile.am b/doc/spec/Makefile.am index 208901d9db..e2fef42e81 100644 --- a/doc/spec/Makefile.am +++ b/doc/spec/Makefile.am @@ -1,5 +1,5 @@ EXTRA_DIST = tor-spec.txt rend-spec.txt control-spec.txt \ dir-spec.txt socks-extensions.txt path-spec.txt \ - version-spec.txt address-spec.txt + version-spec.txt address-spec.txt bridges-spec.txt diff --git a/doc/spec/address-spec.txt b/doc/spec/address-spec.txt index 2a84d857e6..2e1aff2b8a 100644 --- a/doc/spec/address-spec.txt +++ b/doc/spec/address-spec.txt @@ -1,4 +1,3 @@ -$Id$ Special Hostnames in Tor Nick Mathewson @@ -34,10 +33,13 @@ $Id$ "www.google.com.foo.exit=64.233.161.99.foo.exit" to speed subsequent lookups. + The .exit notation is disabled by default as of Tor 0.2.2.1-alpha, due + to potential application-level attacks. + EXAMPLES: www.example.com.exampletornode.exit - Connect to www.example.com from the node called "exampletornode." + Connect to www.example.com from the node called "exampletornode". exampletornode.exit @@ -54,15 +56,3 @@ $Id$ When Tor sees an address in this format, it tries to look up and connect to the specified hidden service. See rend-spec.txt for full details. -4. .noconnect - - SYNTAX: [string].noconnect - - When Tor sees an address in this format, it immediately closes the - connection without attaching it to any circuit. This is useful for - controllers that want to test whether a given application is indeed using - the same instance of Tor that they're controlling. - -5. [XXX Is there a ".virtual" address that we expose too, or is that -just intended to be internal? -RD] - diff --git a/doc/spec/bridges-spec.txt b/doc/spec/bridges-spec.txt index 4a9b373c8e..647118815c 100644 --- a/doc/spec/bridges-spec.txt +++ b/doc/spec/bridges-spec.txt @@ -1,4 +1,3 @@ -$Id$ Tor bridges specification diff --git a/doc/spec/control-spec-v0.txt b/doc/spec/control-spec-v0.txt index faf75a64a4..3515d395a6 100644 --- a/doc/spec/control-spec-v0.txt +++ b/doc/spec/control-spec-v0.txt @@ -1,4 +1,3 @@ -$Id$ TC: A Tor control protocol (Version 0) diff --git a/doc/spec/control-spec.txt b/doc/spec/control-spec.txt index cf92e2b9e3..73f0b020a9 100644 --- a/doc/spec/control-spec.txt +++ b/doc/spec/control-spec.txt @@ -1,4 +1,3 @@ -$Id$ TC: A Tor control protocol (Version 1) @@ -220,7 +219,7 @@ $Id$ "INFO" / "NOTICE" / "WARN" / "ERR" / "NEWDESC" / "ADDRMAP" / "AUTHDIR_NEWDESCS" / "DESCCHANGED" / "STATUS_GENERAL" / "STATUS_CLIENT" / "STATUS_SERVER" / "GUARD" / "NS" / "STREAM_BW" / - "CLIENTS_SEEN" + "CLIENTS_SEEN" / "NEWCONSENSUS" Any events *not* listed in the SETEVENTS line are turned off; thus, sending SETEVENTS with an empty body turns off all event reporting. @@ -271,6 +270,9 @@ $Id$ returns "250 OK" if successful, or "551 Unable to write configuration to disk" if it can't write the file or some other error occurs. + See also the "getinfo config-text" command, if the controller wants + to write the torrc file itself. + 3.7. SIGNAL Sent from the client to the server. The syntax is: @@ -379,6 +381,10 @@ $Id$ "config-file" -- The location of Tor's configuration file ("torrc"). + "config-text" -- The contents that Tor would write if you send it + a SAVECONF command, so the controller can write the file to + disk itself. [First implemented in 0.2.2.7-alpha.] + ["exit-policy/prepend" -- The default exit policy lines that Tor will *prepend* to the ExitPolicy config option. -- Never implemented. Useful?] @@ -503,7 +509,7 @@ $Id$ start and the rest of the interval respectively. The 'interval-start' and 'interval-end' fields are the borders of the current interval; the 'interval-wake' field is the time within the current interval (if any) - where we plan[ned] to start being active. + where we plan[ned] to start being active. The times are GMT. "config/names" A series of lines listing the available configuration options. Each is @@ -564,14 +570,14 @@ $Id$ states. See Section 4.1.10 for explanations. (Only a few of the status events are available as getinfo's currently. Let us know if you want more exposed.) - "status/reachability/or" + "status/reachability-succeeded/or" 0 or 1, depending on whether we've found our ORPort reachable. - "status/reachability/dir" + "status/reachability-succeeded/dir" 0 or 1, depending on whether we've found our DirPort reachable. - "status/reachability" + "status/reachability-succeeded" "OR=" ("0"/"1") SP "DIR=" ("0"/"1") - Combines status/reachability/*; controllers MUST ignore unrecognized - elements in this entry. + Combines status/reachability-succeeded/*; controllers MUST ignore + unrecognized elements in this entry. "status/bootstrap-phase" Returns the most recent bootstrap phase status event sent. Specifically, it returns a string starting with either @@ -775,9 +781,8 @@ $Id$ Same as passing 'EXTENDED' to SETEVENTS; this is the preferred way to request the extended event syntax. - This will not be always-enabled until at least two stable releases - after 0.1.2.3-alpha, the release where it was first used for - anything. + This feature was first used in 0.1.2.3-alpha. It is always-on in + Tor 0.2.2.1-alpha and later. VERBOSE_NAMES @@ -788,8 +793,9 @@ $Id$ LongName format includes a Fingerprint, an indication of Named status, and a Nickname (if one is known). - This will not be always-enabled until at least two stable releases - after 0.1.2.2-alpha, the release where it was first available. + This will not be always-enabled until at least two stable + releases after 0.1.2.2-alpha, the release where it was first + available. It is always-on in Tor 0.2.2.1-alpha and later. 3.20. RESOLVE @@ -1497,6 +1503,23 @@ $Id$ should just look at ACCEPTED_SERVER_DESCRIPTOR and should ignore this event for now.} + SERVER_DESCRIPTOR_STATUS + "STATUS=" "LISTED" / "UNLISTED" + We just got a new networkstatus consensus, and whether we're in + it or not in it has changed. Specifically, status is "listed" + if we're listed in it but previous to this point we didn't know + we were listed in a consensus; and status is "unlisted" if we + thought we should have been listed in it (e.g. we were listed in + the last one), but we're not. + + {Moving from listed to unlisted is not necessarily cause for + alarm. The relay might have failed a few reachability tests, + or the Internet might have had some routing problems. So this + feature is mainly to let relay operators know when their relay + has successfully been listed in the consensus.} + + [Not implemented yet. We should do this in 0.2.2.x. -RD] + NAMESERVER_STATUS "NS=addr" "STATUS=" "UP" / "DOWN" @@ -1581,17 +1604,21 @@ $Id$ 4.1.13. Bandwidth used on an application stream The syntax is: - "650" SP "STREAM_BW" SP StreamID SP BytesRead SP BytesWritten CRLF - BytesRead = 1*DIGIT + "650" SP "STREAM_BW" SP StreamID SP BytesWritten SP BytesRead CRLF BytesWritten = 1*DIGIT + BytesRead = 1*DIGIT + + BytesWritten and BytesRead are the number of bytes written and read + by the application since the last STREAM_BW event on this stream. - BytesRead and BytesWritten are the number of bytes read and written since - the last STREAM_BW event on this stream. These events are generated about - once per second per stream; no events are generated for streams that have - not read or written. + Note that from Tor's perspective, *reading* a byte on a stream means + that the application *wrote* the byte. That's why the order of "written" + vs "read" is opposite for stream_bw events compared to bw events. - These events apply only to streams entering Tor (such as on a SOCKSPort, - TransPort, or so on). They are not generated for exiting streams. + These events are generated about once per second per stream; no events + are generated for streams that have not written or read. These events + apply only to streams entering Tor (such as on a SOCKSPort, TransPort, + or so on). They are not generated for exiting streams. 4.1.14. Per-country client stats @@ -1610,11 +1637,11 @@ $Id$ TimeStarted is a quoted string indicating when the reported summary counts from (in GMT). - The CountrySummary keyword has as its argument a comma-separated - set of "countrycode=count" pairs. For example, - 650-CLIENTS_SEEN TimeStarted="Thu Dec 25 23:50:43 EST 2008" - 650 CountrySummary=us=16,de=8,uk=8 -[XXX Matt Edman informs me that the time format above is wrong. -RD] + The CountrySummary keyword has as its argument a comma-separated, + possibly empty set of "countrycode=count" pairs. For example (without + linebreak), + 650-CLIENTS_SEEN TimeStarted="2008-12-25 23:50:43" + CountrySummary=us=16,de=8,uk=8 4.1.15. New consensus networkstatus has arrived. diff --git a/doc/spec/dir-spec-v1.txt b/doc/spec/dir-spec-v1.txt index 286df664e2..a92fc7999a 100644 --- a/doc/spec/dir-spec-v1.txt +++ b/doc/spec/dir-spec-v1.txt @@ -1,4 +1,3 @@ -$Id$ Tor Protocol Specification diff --git a/doc/spec/dir-spec-v2.txt b/doc/spec/dir-spec-v2.txt index 4873c4a728..d1be27f3db 100644 --- a/doc/spec/dir-spec-v2.txt +++ b/doc/spec/dir-spec-v2.txt @@ -1,4 +1,3 @@ -$Id$ Tor directory protocol, version 2 diff --git a/doc/spec/dir-spec.txt b/doc/spec/dir-spec.txt index 9a2a62bc46..19a32027af 100644 --- a/doc/spec/dir-spec.txt +++ b/doc/spec/dir-spec.txt @@ -1,4 +1,3 @@ -$Id$ Tor directory protocol, version 3 @@ -594,7 +593,7 @@ $Id$ "allow-single-hop-exits" - [At most one.] + [At most once.] Present only if the router allows single-hop circuits to make exit connections. Most Tor servers do not support this: this is @@ -628,8 +627,8 @@ $Id$ As documented in 2.1 above. See migration notes in section 2.2.1. - "geoip-start" YYYY-MM-DD HH:MM:SS NL - "geoip-client-origins" CC=N,CC=N,... NL + ("geoip-start" YYYY-MM-DD HH:MM:SS NL) + ("geoip-client-origins" CC=N,CC=N,... NL) Only generated by bridge routers (see blocking.pdf), and only when they have been configured with a geoip database. @@ -642,6 +641,227 @@ $Id$ "geoip-start" is the time at which we began collecting geoip statistics. + "geoip-start" and "geoip-client-origins" have been replaced by + "bridge-stats-end" and "bridge-stats-ips" in 0.2.2.4-alpha. The + reason is that the measurement interval with "geoip-stats" as + determined by subtracting "geoip-start" from "published" could + have had a variable length, whereas the measurement interval in + 0.2.2.4-alpha and later is set to be exactly 24 hours long. In + order to clearly distinguish the new measurement intervals from + the old ones, the new keywords have been introduced. + + "bridge-stats-end" YYYY-MM-DD HH:MM:SS (NSEC s) NL + [At most once.] + + YYYY-MM-DD HH:MM:SS defines the end of the included measurement + interval of length NSEC seconds (86400 seconds by default). + + A "bridge-stats-end" line, as well as any other "bridge-*" line, + is only added when the relay has been running as a bridge for at + least 24 hours. + + "bridge-ips" CC=N,CC=N,... NL + [At most once.] + + List of mappings from two-letter country codes to the number of + unique IP addresses that have connected from that country to the + bridge and which are no known relays, rounded up to the nearest + multiple of 8. + + "dirreq-stats-end" YYYY-MM-DD HH:MM:SS (NSEC s) NL + [At most once.] + + YYYY-MM-DD HH:MM:SS defines the end of the included measurement + interval of length NSEC seconds (86400 seconds by default). + + A "dirreq-stats-end" line, as well as any other "dirreq-*" line, + is only added when the relay has opened its Dir port and after 24 + hours of measuring directory requests. + + "dirreq-v2-ips" CC=N,CC=N,... NL + [At most once.] + "dirreq-v3-ips" CC=N,CC=N,... NL + [At most once.] + + List of mappings from two-letter country codes to the number of + unique IP addresses that have connected from that country to + request a v2/v3 network status, rounded up to the nearest multiple + of 8. Only those IP addresses are counted that the directory can + answer with a 200 OK status code. + + "dirreq-v2-reqs" CC=N,CC=N,... NL + [At most once.] + "dirreq-v3-reqs" CC=N,CC=N,... NL + [At most once.] + + List of mappings from two-letter country codes to the number of + requests for v2/v3 network statuses from that country, rounded up + to the nearest multiple of 8. Only those requests are counted that + the directory can answer with a 200 OK status code. + + "dirreq-v2-share" num% NL + [At most once.] + "dirreq-v3-share" num% NL + [At most once.] + + The share of v2/v3 network status requests that the directory + expects to receive from clients based on its advertised bandwidth + compared to the overall network bandwidth capacity. Shares are + formatted in percent with two decimal places. Shares are + calculated as means over the whole 24-hour interval. + + "dirreq-v2-resp" status=num,... NL + [At most once.] + "dirreq-v3-resp" status=nul,... NL + [At most once.] + + List of mappings from response statuses to the number of requests + for v2/v3 network statuses that were answered with that response + status, rounded up to the nearest multiple of 4. Only response + statuses with at least 1 response are reported. New response + statuses can be added at any time. The current list of response + statuses is as follows: + + "ok": a network status request is answered; this number + corresponds to the sum of all requests as reported in + "dirreq-v2-reqs" or "dirreq-v3-reqs", respectively, before + rounding up. + "not-enough-sigs: a version 3 network status is not signed by a + sufficient number of requested authorities. + "unavailable": a requested network status object is unavailable. + "not-found": a requested network status is not found. + "not-modified": a network status has not been modified since the + If-Modified-Since time that is included in the request. + "busy": the directory is busy. + + "dirreq-v2-direct-dl" key=val,... NL + [At most once.] + "dirreq-v3-direct-dl" key=val,... NL + [At most once.] + "dirreq-v2-tunneled-dl" key=val,... NL + [At most once.] + "dirreq-v3-tunneled-dl" key=val,... NL + [At most once.] + + List of statistics about possible failures in the download process + of v2/v3 network statuses. Requests are either "direct" + HTTP-encoded requests over the relay's directory port, or + "tunneled" requests using a BEGIN_DIR cell over the relay's OR + port. The list of possible statistics can change, and statistics + can be left out from reporting. The current list of statistics is + as follows: + + Successful downloads and failures: + + "complete": a client has finished the download successfully. + "timeout": a download did not finish within 10 minutes after + starting to send the response. + "running": a download is still running at the end of the + measurement period for less than 10 minutes after starting to + send the response. + + Download times: + + "min", "max": smallest and largest measured bandwidth in B/s. + "d[1-4,6-9]": 1st to 4th and 6th to 9th decile of measured + bandwidth in B/s. For a given decile i, i/10 of all downloads + had a smaller bandwidth than di, and (10-i)/10 of all downloads + had a larger bandwidth than di. + "q[1,3]": 1st and 3rd quartile of measured bandwidth in B/s. One + fourth of all downloads had a smaller bandwidth than q1, one + fourth of all downloads had a larger bandwidth than q3, and the + remaining half of all downloads had a bandwidth between q1 and + q3. + "md": median of measured bandwidth in B/s. Half of the downloads + had a smaller bandwidth than md, the other half had a larger + bandwidth than md. + + "entry-stats-end" YYYY-MM-DD HH:MM:SS (NSEC s) NL + [At most once.] + + YYYY-MM-DD HH:MM:SS defines the end of the included measurement + interval of length NSEC seconds (86400 seconds by default). + + An "entry-stats-end" line, as well as any other "entry-*" + line, is first added after the relay has been running for at least + 24 hours. + + "entry-ips" CC=N,CC=N,... NL + [At most once.] + + List of mappings from two-letter country codes to the number of + unique IP addresses that have connected from that country to the + relay and which are no known other relays, rounded up to the + nearest multiple of 8. + + "cell-stats-end" YYYY-MM-DD HH:MM:SS (NSEC s) NL + [At most once.] + + YYYY-MM-DD HH:MM:SS defines the end of the included measurement + interval of length NSEC seconds (86400 seconds by default). + + A "cell-stats-end" line, as well as any other "cell-*" line, + is first added after the relay has been running for at least 24 + hours. + + "cell-processed-cells" num,...,num NL + [At most once.] + + Mean number of processed cells per circuit, subdivided into + deciles of circuits by the number of cells they have processed in + descending order from loudest to quietest circuits. + + "cell-queued-cells" num,...,num NL + [At most once.] + + Mean number of cells contained in queues by circuit decile. These + means are calculated by 1) determining the mean number of cells in + a single circuit between its creation and its termination and 2) + calculating the mean for all circuits in a given decile as + determined in "cell-processed-cells". Numbers have a precision of + two decimal places. + + "cell-time-in-queue" num,...,num NL + [At most once.] + + Mean time cells spend in circuit queues in milliseconds. Times are + calculated by 1) determining the mean time cells spend in the + queue of a single circuit and 2) calculating the mean for all + circuits in a given decile as determined in + "cell-processed-cells". + + "cell-circuits-per-decile" num NL + [At most once.] + + Mean number of circuits that are included in any of the deciles, + rounded up to the next integer. + + "exit-stats-end" YYYY-MM-DD HH:MM:SS (NSEC s) NL + [At most once.] + + YYYY-MM-DD HH:MM:SS defines the end of the included measurement + interval of length NSEC seconds (86400 seconds by default). + + An "exit-stats-end" line, as well as any other "exit-*" line, is + first added after the relay has been running for at least 24 hours + and only if the relay permits exiting (where exiting to a single + port and IP address is sufficient). + + "exit-kibibytes-written" port=N,port=N,... NL + [At most once.] + "exit-kibibytes-read" port=N,port=N,... NL + [At most once.] + + List of mappings from ports to the number of kibibytes that the + relay has written to or read from exit connections to that port, + rounded up to the next full kibibyte. + + "exit-streams-opened" port=N,port=N,... NL + [At most once.] + + List of mappings from ports to the number of opened exit streams + to that port, rounded up to the nearest multiple of 4. + "router-signature" NL Signature NL [At end, exactly once.] @@ -798,7 +1018,7 @@ $Id$ documents are described in section XXX below. Status documents contain a preamble, an authority section, a list of - router status entries, and one more footers signature, in that order. + router status entries, and one or more footer signature, in that order. Unlike other formats described above, a SP in these documents must be a single space character (hex 20). @@ -905,6 +1125,32 @@ $Id$ enough votes were counted for the consensus for an authoritative opinion to have been formed about their status. + "params" SP [Parameters] NL + + [At most once] + + Parameter ::= Keyword '=' Int32 + Int32 ::= A decimal integer between -2147483648 and 2147483647. + Parameters ::= Parameter | Parameters SP Parameter + + The parameters list, if present, contains a space-separated list of + key-value pairs, sorted in lexical order by their keyword. Each + parameter has its own meaning. + + (Only included when the vote is generated with consensus-method 7 or + later.) + + Commonly used "param" arguments at this point include: + + "CircWindow" -- the default package window that circuits should + be established with. It started out at 1000 cells, but some + research indicates that a lower value would mean fewer cells in + transit in the network at any given time. Obeyed by Tor 0.2.1.20 + and later. + + "CircPriorityHalflifeMsec" -- the halflife parameter used when + weighting which circuit will send the next cell. Obeyed by Tor + 0.2.2.7-alpha and later. The authority section of a vote contains the following items, followed in turn by the authority's current key certificate: @@ -1030,13 +1276,20 @@ $Id$ descriptors if they would cause "v" lines to be over 128 characters long. - "w" SP "Bandwidth=" INT NL + "w" SP "Bandwidth=" INT [SP "Measured=" INT] NL [At most once.] An estimate of the bandwidth of this server, in an arbitrary unit (currently kilobytes per second). Used to weight router - selection. Other weighting keywords may be added later. + selection. + + Additionally, the Measured= keyword is present in votes by + participating bandwidth measurement authorities to indicate + a measured bandwidth currently produced by measuring stream + capacities. + + Other weighting keywords may be added later. Clients MUST ignore keywords they do not recognize. "p" SP ("accept" / "reject") SP PortList NL @@ -1128,7 +1381,7 @@ $Id$ least one /8 address space. "Fast" -- A router is 'Fast' if it is active, and its bandwidth is - either in the top 7/8ths for known active routers or at least 100KB/s. + either in the top 7/8ths for known active routers or at least 20KB/s. "Guard" -- A router is a possible 'Guard' if its Weighted Fractional Uptime is at least the median for "familiar" active routers, and if @@ -1179,6 +1432,13 @@ $Id$ rate limit from the router descriptor. It is given in kilobytes per second, and capped at some arbitrary value (currently 10 MB/s). + The Measured= keyword on a "w" line vote is currently computed + by multiplying the previous published consensus bandwidth by the + ratio of the measured average node stream capacity to the network + average. If 3 or more authorities provide a Measured= keyword for + a router, the authorities produce a consensus containing a "w" + Bandwidth= keyword equal to the median of the Measured= votes. + The ports listed in a "p" line should be taken as those ports for which the router's exit policy permits 'most' addresses, ignoring any accept not for all addresses, ignoring all rejects for private @@ -1199,6 +1459,10 @@ $Id$ Known-flags is the union of all flags known by any voter. + Entries are given on the "params" line for every keyword on which any + authority voted. The values given are the low-median of all votes on + that keyword. + "client-versions" and "server-versions" are sorted in ascending order; A version is recommended in the consensus if it is recommended by more than half of the voting authorities that included a @@ -1261,6 +1525,14 @@ $Id$ one, breaking ties in favor of the lexicographically larger vote.) The port list is encoded as specified in 3.4.2. + * If consensus-method 6 or later is in use and if 3 or more + authorities provide a Measured= keyword in their votes for + a router, the authorities produce a consensus containing a + Bandwidth= keyword equal to the median of the Measured= votes. + + * If consensus-method 7 or later is in use, the params line is + included in the output. + The signatures at the end of a consensus document are sorted in ascending order by identity digest. @@ -1281,6 +1553,7 @@ $Id$ "3" -- Added legacy ID key support to aid in authority ID key rollovers "4" -- No longer list routers that are not running in the consensus "5" -- adds support for "w" and "p" lines. + "6" -- Prefers measured bandwidth values rather than advertised Before generating a consensus, an authority must decide which consensus method to use. To do this, it looks for the highest version number @@ -1884,7 +2157,6 @@ $Id$ A. Consensus-negotiation timeline. - Period begins: this is the Published time. Everybody sends votes Reconciliation: everybody tries to fetch missing votes. diff --git a/doc/spec/path-spec.txt b/doc/spec/path-spec.txt index dceb21dad7..78f3b63bcb 100644 --- a/doc/spec/path-spec.txt +++ b/doc/spec/path-spec.txt @@ -1,4 +1,3 @@ -$Id$ Tor Path Specification @@ -72,6 +71,24 @@ of their choices. is unknown (usually its target IP), but we believe the path probably supports the request according to the rules given below. +1.1. A server's bandwidth + + Old versions of Tor did not report bandwidths in network status + documents, so clients had to learn them from the routers' advertised + server descriptors. + + For versions of Tor prior to 0.2.1.17-rc, everywhere below where we + refer to a server's "bandwidth", we mean its clipped advertised + bandwidth, computed by taking the smaller of the 'rate' and + 'observed' arguments to the "bandwidth" element in the server's + descriptor. If a router's advertised bandwidth is greater than + MAX_BELIEVABLE_BANDWIDTH (currently 10 MB/s), we clipped to that + value. + + For more recent versions of Tor, we take the bandwidth value declared + in the consensus, and fall back to the clipped advertised bandwidth + only if the consensus does not have bandwidths listed. + 2. Building circuits 2.1. When we build @@ -179,16 +196,13 @@ of their choices. multiple candidates for a path element, we choose randomly. For "fast" circuits, we pick a given router as an exit with probability - proportional to its advertised bandwidth [the smaller of the 'rate' and - 'observed' arguments to the "bandwidth" element in its descriptor]. If a - router's advertised bandwidth is greater than MAX_BELIEVABLE_BANDWIDTH - (currently 10 MB/s), we clip to that value. + proportional to its bandwidth. For non-exit positions on "fast" circuits, we pick routers as above, but - we weight the clipped advertised bandwidth of Exit-flagged nodes depending + we weight the bandwidth of Exit-flagged nodes depending on the fraction of bandwidth available from non-Exit nodes. Call the - total clipped advertised bandwidth for Exit nodes under consideration E, - and the total clipped advertised bandwidth for all nodes under + total bandwidth for Exit nodes under consideration E, + and the total bandwidth for all nodes under consideration T. If E<T/3, we do not consider Exit-flagged nodes. Otherwise, we weight their bandwidth with the factor (E-T/3)/E. This ensures that bandwidth is evenly distributed over nodes in 3-hop paths. @@ -306,7 +320,7 @@ of their choices. We use Guard nodes (also called "helper nodes" in the literature) to prevent certain profiling attacks. Here's the risk: if we choose entry and exit nodes at random, and an attacker controls C out of N servers - (ignoring advertised bandwidth), then the + (ignoring bandwidth), then the attacker will control the entry and exit node of any given circuit with probability (C/N)^2. But as we make many different circuits over time, then the probability that the attacker will see a sample of about (C/N)^2 diff --git a/doc/spec/proposals/000-index.txt b/doc/spec/proposals/000-index.txt index d75157650d..93dc0299dc 100644 --- a/doc/spec/proposals/000-index.txt +++ b/doc/spec/proposals/000-index.txt @@ -1,7 +1,5 @@ Filename: 000-index.txt Title: Index of Tor Proposals -Version: $Revision$ -Last-Modified: $Date$ Author: Nick Mathewson Created: 26-Jan-2007 Status: Meta @@ -56,7 +54,7 @@ Proposals by number: 131 Help users to verify they are using Tor [NEEDS-REVISION] 132 A Tor Web Service For Verifying Correct Browser Configuration [DRAFT] 133 Incorporate Unreachable ORs into the Tor Network [DRAFT] -134 More robust consensus voting with diverse authority sets [ACCEPTED] +134 More robust consensus voting with diverse authority sets [REJECTED] 135 Simplify Configuration of Private Tor Networks [CLOSED] 136 Mass authority migration with legacy keys [CLOSED] 137 Keep controllers informed as Tor bootstraps [CLOSED] @@ -82,6 +80,15 @@ Proposals by number: 157 Make certificate downloads specific [ACCEPTED] 158 Clients download consensus + microdescriptors [OPEN] 159 Exit Scanning [OPEN] +160 Authorities vote for bandwidth offsets in consensus [OPEN] +161 Computing Bandwidth Adjustments [OPEN] +162 Publish the consensus in multiple flavors [OPEN] +163 Detecting whether a connection comes from a client [OPEN] +164 Reporting the status of server votes [OPEN] +165 Easy migration for voting authority sets [OPEN] +166 Including Network Statistics in Extra-Info Documents [ACCEPTED] +167 Vote on network parameters in consensus [CLOSED] +168 Reduce default circuit window [OPEN] Proposals by status: @@ -103,14 +110,21 @@ Proposals by status: 156 Tracking blocked ports on the client side [for 0.2.?] 158 Clients download consensus + microdescriptors 159 Exit Scanning + 160 Authorities vote for bandwidth offsets in consensus [for 0.2.2.x] + 161 Computing Bandwidth Adjustments [for 0.2.2.x] + 162 Publish the consensus in multiple flavors [for 0.2.2] + 163 Detecting whether a connection comes from a client [for 0.2.2] + 164 Reporting the status of server votes [for 0.2.2] + 165 Easy migration for voting authority sets + 168 Reduce default circuit window [for 0.2.2] ACCEPTED: 110 Avoiding infinite length circuits [for 0.2.1.x] [in 0.2.1.3-alpha] 117 IPv6 exits [for 0.2.1.x] 118 Advertising multiple ORPorts at once [for 0.2.1.x] - 134 More robust consensus voting with diverse authority sets [for 0.2.2.x] 140 Provide diffs between consensuses [for 0.2.2.x] 147 Eliminate the need for v2 directories in generating v3 directories [for 0.2.1.x] 157 Make certificate downloads specific [for 0.2.1.x] + 166 Including Network Statistics in Extra-Info Documents [for 0.2.2] META: 000 Index of Tor Proposals 001 The Tor Proposal Process @@ -146,6 +160,7 @@ Proposals by status: 148 Stream end reasons from the client side should be uniform [in 0.2.1.9-alpha] 150 Exclude Exit Nodes from a circuit [in 0.2.1.3-alpha] 152 Optionally allow exit from single-hop circuits [in 0.2.1.6-alpha] + 167 Vote on network parameters in consensus [in 0.2.2] SUPERSEDED: 112 Bring Back Pathlen Coin Weight 113 Simplifying directory authority administration @@ -159,3 +174,5 @@ Proposals by status: 120 Shutdown descriptors when Tor servers stop 128 Families of private bridges 142 Combine Introduction and Rendezvous Points + REJECTED: + 134 More robust consensus voting with diverse authority sets diff --git a/doc/spec/proposals/001-process.txt b/doc/spec/proposals/001-process.txt index 3a767b5fa4..636ba2c2fa 100644 --- a/doc/spec/proposals/001-process.txt +++ b/doc/spec/proposals/001-process.txt @@ -1,7 +1,5 @@ Filename: 001-process.txt Title: The Tor Proposal Process -Version: $Revision$ -Last-Modified: $Date$ Author: Nick Mathewson Created: 30-Jan-2007 Status: Meta @@ -47,7 +45,7 @@ How to change the specs now: Like an RFC, every proposal gets a number. Unlike RFCs, proposals can change over time and keep the same number, until they are finally accepted or rejected. The history for each proposal - will be stored in the Tor Subversion repository. + will be stored in the Tor repository. Once a proposal is in the repository, we should discuss and improve it until we've reached consensus that it's a good idea, and that it's @@ -82,9 +80,7 @@ How new proposals get added: What should go in a proposal: Every proposal should have a header containing these fields: - Filename, Title, Version, Last-Modified, Author, Created, Status. - The Version and Last-Modified fields should use the SVN Revision and Date - tags respectively. + Filename, Title, Author, Created, Status. These fields are optional but recommended: Target, Implemented-In. @@ -97,7 +93,7 @@ What should go in a proposal: what the proposal's about, what it does, and about what state it's in. After the Overview, the proposal becomes more free-form. Depending on its - the length and complexity, the proposal can break into sections as + length and complexity, the proposal can break into sections as appropriate, or follow a short discursive format. Every proposal should contain at least the following information before it is "ACCEPTED", though the information does not need to be in sections with these names. diff --git a/doc/spec/proposals/098-todo.txt b/doc/spec/proposals/098-todo.txt index e891ea890c..a0bbbeb568 100644 --- a/doc/spec/proposals/098-todo.txt +++ b/doc/spec/proposals/098-todo.txt @@ -1,7 +1,5 @@ Filename: 098-todo.txt Title: Proposals that should be written -Version: $Revision$ -Last-Modified: $Date$ Author: Nick Mathewson, Roger Dingledine Created: 26-Jan-2007 Status: Meta diff --git a/doc/spec/proposals/099-misc.txt b/doc/spec/proposals/099-misc.txt index ba13ea2a71..a3621dd25f 100644 --- a/doc/spec/proposals/099-misc.txt +++ b/doc/spec/proposals/099-misc.txt @@ -1,7 +1,5 @@ Filename: 099-misc.txt Title: Miscellaneous proposals -Version: $Revision$ -Last-Modified: $Date$ Author: Various Created: 26-Jan-2007 Status: Meta diff --git a/doc/spec/proposals/100-tor-spec-udp.txt b/doc/spec/proposals/100-tor-spec-udp.txt index 8224682ec8..7f062222c5 100644 --- a/doc/spec/proposals/100-tor-spec-udp.txt +++ b/doc/spec/proposals/100-tor-spec-udp.txt @@ -1,7 +1,5 @@ Filename: 100-tor-spec-udp.txt Title: Tor Unreliable Datagram Extension Proposal -Version: $Revision$ -Last-Modified: $Date$ Author: Marc Liberatore Created: 23 Feb 2006 Status: Dead diff --git a/doc/spec/proposals/101-dir-voting.txt b/doc/spec/proposals/101-dir-voting.txt index be900a641e..634d3f1948 100644 --- a/doc/spec/proposals/101-dir-voting.txt +++ b/doc/spec/proposals/101-dir-voting.txt @@ -1,7 +1,5 @@ Filename: 101-dir-voting.txt Title: Voting on the Tor Directory System -Version: $Revision$ -Last-Modified: $Date$ Author: Nick Mathewson Created: Nov 2006 Status: Closed diff --git a/doc/spec/proposals/102-drop-opt.txt b/doc/spec/proposals/102-drop-opt.txt index 8f6a38ae6c..490376bb53 100644 --- a/doc/spec/proposals/102-drop-opt.txt +++ b/doc/spec/proposals/102-drop-opt.txt @@ -1,7 +1,5 @@ Filename: 102-drop-opt.txt Title: Dropping "opt" from the directory format -Version: $Revision$ -Last-Modified: $Date$ Author: Nick Mathewson Created: Jan 2007 Status: Closed diff --git a/doc/spec/proposals/103-multilevel-keys.txt b/doc/spec/proposals/103-multilevel-keys.txt index ef51e18047..c8a7a6677b 100644 --- a/doc/spec/proposals/103-multilevel-keys.txt +++ b/doc/spec/proposals/103-multilevel-keys.txt @@ -1,7 +1,5 @@ Filename: 103-multilevel-keys.txt Title: Splitting identity key from regularly used signing key. -Version: $Revision$ -Last-Modified: $Date$ Author: Nick Mathewson Created: Jan 2007 Status: Closed diff --git a/doc/spec/proposals/104-short-descriptors.txt b/doc/spec/proposals/104-short-descriptors.txt index a1c42c8ff7..90e0764fe6 100644 --- a/doc/spec/proposals/104-short-descriptors.txt +++ b/doc/spec/proposals/104-short-descriptors.txt @@ -1,7 +1,5 @@ Filename: 104-short-descriptors.txt Title: Long and Short Router Descriptors -Version: $Revision$ -Last-Modified: $Date$ Author: Nick Mathewson Created: Jan 2007 Status: Closed diff --git a/doc/spec/proposals/105-handshake-revision.txt b/doc/spec/proposals/105-handshake-revision.txt index f6c209e71b..791a016c26 100644 --- a/doc/spec/proposals/105-handshake-revision.txt +++ b/doc/spec/proposals/105-handshake-revision.txt @@ -1,7 +1,5 @@ Filename: 105-handshake-revision.txt Title: Version negotiation for the Tor protocol. -Version: $Revision$ -Last-Modified: $Date$ Author: Nick Mathewson, Roger Dingledine Created: Jan 2007 Status: Closed diff --git a/doc/spec/proposals/106-less-tls-constraint.txt b/doc/spec/proposals/106-less-tls-constraint.txt index 35d6bf1066..7e7621df69 100644 --- a/doc/spec/proposals/106-less-tls-constraint.txt +++ b/doc/spec/proposals/106-less-tls-constraint.txt @@ -1,7 +1,5 @@ Filename: 106-less-tls-constraint.txt Title: Checking fewer things during TLS handshakes -Version: $Revision$ -Last-Modified: $Date$ Author: Nick Mathewson Created: 9-Feb-2007 Status: Closed diff --git a/doc/spec/proposals/107-uptime-sanity-checking.txt b/doc/spec/proposals/107-uptime-sanity-checking.txt index b11be89380..922129b21d 100644 --- a/doc/spec/proposals/107-uptime-sanity-checking.txt +++ b/doc/spec/proposals/107-uptime-sanity-checking.txt @@ -1,7 +1,5 @@ Filename: 107-uptime-sanity-checking.txt Title: Uptime Sanity Checking -Version: $Revision$ -Last-Modified: $Date$ Author: Kevin Bauer & Damon McCoy Created: 8-March-2007 Status: Closed diff --git a/doc/spec/proposals/108-mtbf-based-stability.txt b/doc/spec/proposals/108-mtbf-based-stability.txt index 2c66481530..294103760b 100644 --- a/doc/spec/proposals/108-mtbf-based-stability.txt +++ b/doc/spec/proposals/108-mtbf-based-stability.txt @@ -1,7 +1,5 @@ Filename: 108-mtbf-based-stability.txt Title: Base "Stable" Flag on Mean Time Between Failures -Version: $Revision$ -Last-Modified: $Date$ Author: Nick Mathewson Created: 10-Mar-2007 Status: Closed diff --git a/doc/spec/proposals/109-no-sharing-ips.txt b/doc/spec/proposals/109-no-sharing-ips.txt index 1a88b00c0f..5438cf049a 100644 --- a/doc/spec/proposals/109-no-sharing-ips.txt +++ b/doc/spec/proposals/109-no-sharing-ips.txt @@ -1,7 +1,5 @@ Filename: 109-no-sharing-ips.txt Title: No more than one server per IP address. -Version: $Revision$ -Last-Modified: $Date$ Author: Kevin Bauer & Damon McCoy Created: 9-March-2007 Status: Closed diff --git a/doc/spec/proposals/110-avoid-infinite-circuits.txt b/doc/spec/proposals/110-avoid-infinite-circuits.txt index 1834cd34a7..fffc41c25a 100644 --- a/doc/spec/proposals/110-avoid-infinite-circuits.txt +++ b/doc/spec/proposals/110-avoid-infinite-circuits.txt @@ -1,7 +1,5 @@ Filename: 110-avoid-infinite-circuits.txt Title: Avoiding infinite length circuits -Version: $Revision$ -Last-Modified: $Date$ Author: Roger Dingledine Created: 13-Mar-2007 Status: Accepted diff --git a/doc/spec/proposals/111-local-traffic-priority.txt b/doc/spec/proposals/111-local-traffic-priority.txt index f8a37efc94..9411463c21 100644 --- a/doc/spec/proposals/111-local-traffic-priority.txt +++ b/doc/spec/proposals/111-local-traffic-priority.txt @@ -1,7 +1,5 @@ Filename: 111-local-traffic-priority.txt Title: Prioritizing local traffic over relayed traffic -Version: $Revision$ -Last-Modified: $Date$ Author: Roger Dingledine Created: 14-Mar-2007 Status: Closed diff --git a/doc/spec/proposals/112-bring-back-pathlencoinweight.txt b/doc/spec/proposals/112-bring-back-pathlencoinweight.txt index e7cc6b4e36..3f6c3376f0 100644 --- a/doc/spec/proposals/112-bring-back-pathlencoinweight.txt +++ b/doc/spec/proposals/112-bring-back-pathlencoinweight.txt @@ -1,7 +1,5 @@ Filename: 112-bring-back-pathlencoinweight.txt Title: Bring Back Pathlen Coin Weight -Version: $Revision$ -Last-Modified: $Date$ Author: Mike Perry Created: Status: Superseded diff --git a/doc/spec/proposals/113-fast-authority-interface.txt b/doc/spec/proposals/113-fast-authority-interface.txt index 20cf33e429..8912b53220 100644 --- a/doc/spec/proposals/113-fast-authority-interface.txt +++ b/doc/spec/proposals/113-fast-authority-interface.txt @@ -1,7 +1,5 @@ Filename: 113-fast-authority-interface.txt Title: Simplifying directory authority administration -Version: $Revision$ -Last-Modified: $Date$ Author: Nick Mathewson Created: Status: Superseded diff --git a/doc/spec/proposals/114-distributed-storage.txt b/doc/spec/proposals/114-distributed-storage.txt index e9271fb82d..91a787d301 100644 --- a/doc/spec/proposals/114-distributed-storage.txt +++ b/doc/spec/proposals/114-distributed-storage.txt @@ -1,7 +1,5 @@ Filename: 114-distributed-storage.txt Title: Distributed Storage for Tor Hidden Service Descriptors -Version: $Revision$ -Last-Modified: $Date$ Author: Karsten Loesing Created: 13-May-2007 Status: Closed diff --git a/doc/spec/proposals/115-two-hop-paths.txt b/doc/spec/proposals/115-two-hop-paths.txt index ee10d949c4..9854c9ad55 100644 --- a/doc/spec/proposals/115-two-hop-paths.txt +++ b/doc/spec/proposals/115-two-hop-paths.txt @@ -1,7 +1,5 @@ Filename: 115-two-hop-paths.txt Title: Two Hop Paths -Version: $Revision$ -Last-Modified: $Date$ Author: Mike Perry Created: Status: Dead diff --git a/doc/spec/proposals/116-two-hop-paths-from-guard.txt b/doc/spec/proposals/116-two-hop-paths-from-guard.txt index 454b344abf..f45625350b 100644 --- a/doc/spec/proposals/116-two-hop-paths-from-guard.txt +++ b/doc/spec/proposals/116-two-hop-paths-from-guard.txt @@ -1,7 +1,5 @@ Filename: 116-two-hop-paths-from-guard.txt Title: Two hop paths from entry guards -Version: $Revision$ -Last-Modified: $Date$ Author: Michael Lieberman Created: 26-Jun-2007 Status: Dead diff --git a/doc/spec/proposals/117-ipv6-exits.txt b/doc/spec/proposals/117-ipv6-exits.txt index c8402821ed..00cd7cef10 100644 --- a/doc/spec/proposals/117-ipv6-exits.txt +++ b/doc/spec/proposals/117-ipv6-exits.txt @@ -1,7 +1,5 @@ Filename: 117-ipv6-exits.txt Title: IPv6 exits -Version: $Revision$ -Last-Modified: $Date$ Author: coderman Created: 10-Jul-2007 Status: Accepted diff --git a/doc/spec/proposals/118-multiple-orports.txt b/doc/spec/proposals/118-multiple-orports.txt index 1bef2504d9..2381ec7ca3 100644 --- a/doc/spec/proposals/118-multiple-orports.txt +++ b/doc/spec/proposals/118-multiple-orports.txt @@ -1,7 +1,5 @@ Filename: 118-multiple-orports.txt Title: Advertising multiple ORPorts at once -Version: $Revision$ -Last-Modified: $Date$ Author: Nick Mathewson Created: 09-Jul-2007 Status: Accepted diff --git a/doc/spec/proposals/119-controlport-auth.txt b/doc/spec/proposals/119-controlport-auth.txt index dc57a27368..9ed1cc1cbe 100644 --- a/doc/spec/proposals/119-controlport-auth.txt +++ b/doc/spec/proposals/119-controlport-auth.txt @@ -1,7 +1,5 @@ Filename: 119-controlport-auth.txt Title: New PROTOCOLINFO command for controllers -Version: $Revision$ -Last-Modified: $Date$ Author: Roger Dingledine Created: 14-Aug-2007 Status: Closed diff --git a/doc/spec/proposals/120-shutdown-descriptors.txt b/doc/spec/proposals/120-shutdown-descriptors.txt index dc1265b03b..5cfe2b5bc6 100644 --- a/doc/spec/proposals/120-shutdown-descriptors.txt +++ b/doc/spec/proposals/120-shutdown-descriptors.txt @@ -1,7 +1,5 @@ Filename: 120-shutdown-descriptors.txt Title: Shutdown descriptors when Tor servers stop -Version: $Revision$ -Last-Modified: $Date$ Author: Roger Dingledine Created: 15-Aug-2007 Status: Dead diff --git a/doc/spec/proposals/121-hidden-service-authentication.txt b/doc/spec/proposals/121-hidden-service-authentication.txt index 828bf3c92d..0d92b53a8c 100644 --- a/doc/spec/proposals/121-hidden-service-authentication.txt +++ b/doc/spec/proposals/121-hidden-service-authentication.txt @@ -1,7 +1,5 @@ Filename: 121-hidden-service-authentication.txt Title: Hidden Service Authentication -Version: $Revision$ -Last-Modified: $Date$ Author: Tobias Kamm, Thomas Lauterbach, Karsten Loesing, Ferdinand Rieger, Christoph Weingarten Created: 10-Sep-2007 diff --git a/doc/spec/proposals/122-unnamed-flag.txt b/doc/spec/proposals/122-unnamed-flag.txt index 6502b9c560..2ce7bb22b9 100644 --- a/doc/spec/proposals/122-unnamed-flag.txt +++ b/doc/spec/proposals/122-unnamed-flag.txt @@ -1,7 +1,5 @@ Filename: 122-unnamed-flag.txt Title: Network status entries need a new Unnamed flag -Version: $Revision$ -Last-Modified: $Date$ Author: Roger Dingledine Created: 04-Oct-2007 Status: Closed diff --git a/doc/spec/proposals/123-autonaming.txt b/doc/spec/proposals/123-autonaming.txt index 6cd25329f8..74c486985d 100644 --- a/doc/spec/proposals/123-autonaming.txt +++ b/doc/spec/proposals/123-autonaming.txt @@ -1,7 +1,5 @@ Filename: 123-autonaming.txt Title: Naming authorities automatically create bindings -Version: $Revision$ -Last-Modified: $Date$ Author: Peter Palfrader Created: 2007-10-11 Status: Closed diff --git a/doc/spec/proposals/124-tls-certificates.txt b/doc/spec/proposals/124-tls-certificates.txt index 0a47772732..9472d14af8 100644 --- a/doc/spec/proposals/124-tls-certificates.txt +++ b/doc/spec/proposals/124-tls-certificates.txt @@ -1,7 +1,5 @@ Filename: 124-tls-certificates.txt Title: Blocking resistant TLS certificate usage -Version: $Revision$ -Last-Modified: $Date$ Author: Steven J. Murdoch Created: 2007-10-25 Status: Superseded diff --git a/doc/spec/proposals/125-bridges.txt b/doc/spec/proposals/125-bridges.txt index 8bb3169780..9d95729d42 100644 --- a/doc/spec/proposals/125-bridges.txt +++ b/doc/spec/proposals/125-bridges.txt @@ -1,7 +1,5 @@ Filename: 125-bridges.txt Title: Behavior for bridge users, bridge relays, and bridge authorities -Version: $Revision$ -Last-Modified: $Date$ Author: Roger Dingledine Created: 11-Nov-2007 Status: Closed diff --git a/doc/spec/proposals/126-geoip-reporting.txt b/doc/spec/proposals/126-geoip-reporting.txt index d48a08ba38..9f3b21c670 100644 --- a/doc/spec/proposals/126-geoip-reporting.txt +++ b/doc/spec/proposals/126-geoip-reporting.txt @@ -1,7 +1,5 @@ Filename: 126-geoip-reporting.txt Title: Getting GeoIP data and publishing usage summaries -Version: $Revision$ -Last-Modified: $Date$ Author: Roger Dingledine Created: 2007-11-24 Status: Closed diff --git a/doc/spec/proposals/127-dirport-mirrors-downloads.txt b/doc/spec/proposals/127-dirport-mirrors-downloads.txt index 1b55a02d61..72d6c0cb9f 100644 --- a/doc/spec/proposals/127-dirport-mirrors-downloads.txt +++ b/doc/spec/proposals/127-dirport-mirrors-downloads.txt @@ -1,7 +1,5 @@ Filename: 127-dirport-mirrors-downloads.txt Title: Relaying dirport requests to Tor download site / website -Version: $Revision$ -Last-Modified: $Date$ Author: Roger Dingledine Created: 2007-12-02 Status: Draft diff --git a/doc/spec/proposals/128-bridge-families.txt b/doc/spec/proposals/128-bridge-families.txt index e8a0050c3c..e5bdcf95cb 100644 --- a/doc/spec/proposals/128-bridge-families.txt +++ b/doc/spec/proposals/128-bridge-families.txt @@ -1,7 +1,5 @@ Filename: 128-bridge-families.txt Title: Families of private bridges -Version: $Revision$ -Last-Modified: $Date$ Author: Roger Dingledine Created: 2007-12-xx Status: Dead diff --git a/doc/spec/proposals/129-reject-plaintext-ports.txt b/doc/spec/proposals/129-reject-plaintext-ports.txt index d4767d03d8..8080ff5b75 100644 --- a/doc/spec/proposals/129-reject-plaintext-ports.txt +++ b/doc/spec/proposals/129-reject-plaintext-ports.txt @@ -1,7 +1,5 @@ Filename: 129-reject-plaintext-ports.txt Title: Block Insecure Protocols by Default -Version: $Revision$ -Last-Modified: $Date$ Author: Kevin Bauer & Damon McCoy Created: 2008-01-15 Status: Closed diff --git a/doc/spec/proposals/130-v2-conn-protocol.txt b/doc/spec/proposals/130-v2-conn-protocol.txt index 16f5bf2844..60e742a622 100644 --- a/doc/spec/proposals/130-v2-conn-protocol.txt +++ b/doc/spec/proposals/130-v2-conn-protocol.txt @@ -1,7 +1,5 @@ Filename: 130-v2-conn-protocol.txt Title: Version 2 Tor connection protocol -Version: $Revision$ -Last-Modified: $Date$ Author: Nick Mathewson Created: 2007-10-25 Status: Closed diff --git a/doc/spec/proposals/131-verify-tor-usage.txt b/doc/spec/proposals/131-verify-tor-usage.txt index 2687139189..d3c6efe75a 100644 --- a/doc/spec/proposals/131-verify-tor-usage.txt +++ b/doc/spec/proposals/131-verify-tor-usage.txt @@ -1,7 +1,5 @@ Filename: 131-verify-tor-usage.txt Title: Help users to verify they are using Tor -Version: $Revision$ -Last-Modified: $Date$ Author: Steven J. Murdoch Created: 2008-01-25 Status: Needs-Revision diff --git a/doc/spec/proposals/132-browser-check-tor-service.txt b/doc/spec/proposals/132-browser-check-tor-service.txt index d07a10dcde..6132e5d060 100644 --- a/doc/spec/proposals/132-browser-check-tor-service.txt +++ b/doc/spec/proposals/132-browser-check-tor-service.txt @@ -1,7 +1,5 @@ Filename: 132-browser-check-tor-service.txt Title: A Tor Web Service For Verifying Correct Browser Configuration -Version: $Revision$ -Last-Modified: $Date$ Author: Robert Hogan Created: 2008-03-08 Status: Draft diff --git a/doc/spec/proposals/134-robust-voting.txt b/doc/spec/proposals/134-robust-voting.txt index 5d5e77fa3b..c5dfb3b47f 100644 --- a/doc/spec/proposals/134-robust-voting.txt +++ b/doc/spec/proposals/134-robust-voting.txt @@ -2,8 +2,10 @@ Filename: 134-robust-voting.txt Title: More robust consensus voting with diverse authority sets Author: Peter Palfrader Created: 2008-04-01 -Status: Accepted -Target: 0.2.2.x +Status: Rejected + +History: + 2009 May 27: Added note on rejecting this proposal -- Nick Overview: @@ -103,3 +105,19 @@ Possible Attacks/Open Issues/Some thinking required: Q: Can this ever force us to build a consensus with authorities we do not recognize? A: No, we can never build a fully connected set with them in step 3. + +------------------------------ + +I'm rejecting this proposal as insecure. + +Suppose that we have a clique of size N, and M hostile members in the +clique. If these hostile members stop declaring trust for up to M-1 +good members of the clique, the clique with the hostile members will +in it will be larger than the one without them. + +The M hostile members will constitute a majority of this new clique +when M > (N-(M-1)) / 2, or when M > (N + 1) / 3. This breaks our +requirement that an adversary must compromise a majority of authorities +in order to control the consensus. + +-- Nick diff --git a/doc/spec/proposals/135-private-tor-networks.txt b/doc/spec/proposals/135-private-tor-networks.txt index 131bbb9068..19ef68b7b1 100644 --- a/doc/spec/proposals/135-private-tor-networks.txt +++ b/doc/spec/proposals/135-private-tor-networks.txt @@ -1,7 +1,5 @@ Filename: 135-private-tor-networks.txt Title: Simplify Configuration of Private Tor Networks -Version: $Revision$ -Last-Modified: $Date$ Author: Karsten Loesing Created: 29-Apr-2008 Status: Closed diff --git a/doc/spec/proposals/137-bootstrap-phases.txt b/doc/spec/proposals/137-bootstrap-phases.txt index 18d3dfae12..ebe044c707 100644 --- a/doc/spec/proposals/137-bootstrap-phases.txt +++ b/doc/spec/proposals/137-bootstrap-phases.txt @@ -1,7 +1,5 @@ Filename: 137-bootstrap-phases.txt Title: Keep controllers informed as Tor bootstraps -Version: $Revision$ -Last-Modified: $Date$ Author: Roger Dingledine Created: 07-Jun-2008 Status: Closed diff --git a/doc/spec/proposals/138-remove-down-routers-from-consensus.txt b/doc/spec/proposals/138-remove-down-routers-from-consensus.txt index a07764d536..776911b5c9 100644 --- a/doc/spec/proposals/138-remove-down-routers-from-consensus.txt +++ b/doc/spec/proposals/138-remove-down-routers-from-consensus.txt @@ -1,7 +1,5 @@ Filename: 138-remove-down-routers-from-consensus.txt Title: Remove routers that are not Running from consensus documents -Version: $Revision$ -Last-Modified: $Date$ Author: Peter Palfrader Created: 11-Jun-2008 Status: Closed diff --git a/doc/spec/proposals/140-consensus-diffs.txt b/doc/spec/proposals/140-consensus-diffs.txt index da63bfe23c..8bc4070bfe 100644 --- a/doc/spec/proposals/140-consensus-diffs.txt +++ b/doc/spec/proposals/140-consensus-diffs.txt @@ -1,12 +1,15 @@ Filename: 140-consensus-diffs.txt Title: Provide diffs between consensuses -Version: $Revision$ -Last-Modified: $Date$ Author: Peter Palfrader Created: 13-Jun-2008 Status: Accepted Target: 0.2.2.x +0. History + + 22-May-2009: Restricted the ed format even more strictly for ease of + implementation. -nickm + 1. Overview. Tor clients and servers need a list of which relays are on the @@ -135,6 +138,10 @@ Target: 0.2.2.x Note that line numbers always apply to the file after all previous commands have already been applied. + The commands MUST apply to the file from back to front, such that + lines are only ever referred to by their position in the original + file. + The "current line" is either the first line of the file, if this is the first command, the last line of a block we added in an append or change command, or the line immediate following a set of lines we just diff --git a/doc/spec/proposals/141-jit-sd-downloads.txt b/doc/spec/proposals/141-jit-sd-downloads.txt index b0c2b2cbcd..2ac7a086b7 100644 --- a/doc/spec/proposals/141-jit-sd-downloads.txt +++ b/doc/spec/proposals/141-jit-sd-downloads.txt @@ -1,7 +1,5 @@ Filename: 141-jit-sd-downloads.txt Title: Download server descriptors on demand -Version: $Revision$ -Last-Modified: $Date$ Author: Peter Palfrader Created: 15-Jun-2008 Status: Draft @@ -63,8 +61,8 @@ Status: Draft which tries to convey a server's capacity to clients. Currently we weigh servers differently for different purposes. There - is a weigh for when we use a server as a guard node (our entry to the - Tor network), there is one weigh we assign servers for exit duties, + is a weight for when we use a server as a guard node (our entry to the + Tor network), there is one weight we assign servers for exit duties, and a third for when we need intermediate (middle) nodes. 2.2 Exit information @@ -80,7 +78,7 @@ Status: Draft 2.3 Capability information - Server descriptors contain information about the specific version or + Server descriptors contain information about the specific version of the Tor protocol they understand [proposal 105]. Furthermore the server descriptor also contains the exact version of diff --git a/doc/spec/proposals/142-combine-intro-and-rend-points.txt b/doc/spec/proposals/142-combine-intro-and-rend-points.txt index 3456b285a9..3abd5c863d 100644 --- a/doc/spec/proposals/142-combine-intro-and-rend-points.txt +++ b/doc/spec/proposals/142-combine-intro-and-rend-points.txt @@ -1,7 +1,5 @@ Filename: 142-combine-intro-and-rend-points.txt Title: Combine Introduction and Rendezvous Points -Version: $Revision$ -Last-Modified: $Date$ Author: Karsten Loesing, Christian Wilms Created: 27-Jun-2008 Status: Dead diff --git a/doc/spec/proposals/143-distributed-storage-improvements.txt b/doc/spec/proposals/143-distributed-storage-improvements.txt index 8789d84663..0f7468f1dc 100644 --- a/doc/spec/proposals/143-distributed-storage-improvements.txt +++ b/doc/spec/proposals/143-distributed-storage-improvements.txt @@ -1,7 +1,5 @@ Filename: 143-distributed-storage-improvements.txt Title: Improvements of Distributed Storage for Tor Hidden Service Descriptors -Version: $Revision$ -Last-Modified: $Date$ Author: Karsten Loesing Created: 28-Jun-2008 Status: Open diff --git a/doc/spec/proposals/145-newguard-flag.txt b/doc/spec/proposals/145-newguard-flag.txt index 31d707d725..9e61e30be9 100644 --- a/doc/spec/proposals/145-newguard-flag.txt +++ b/doc/spec/proposals/145-newguard-flag.txt @@ -1,7 +1,5 @@ Filename: 145-newguard-flag.txt Title: Separate "suitable as a guard" from "suitable as a new guard" -Version: $Revision$ -Last-Modified: $Date$ Author: Nick Mathewson Created: 1-Jul-2008 Status: Open diff --git a/doc/spec/proposals/146-long-term-stability.txt b/doc/spec/proposals/146-long-term-stability.txt index 7cfd58f564..9af0017441 100644 --- a/doc/spec/proposals/146-long-term-stability.txt +++ b/doc/spec/proposals/146-long-term-stability.txt @@ -1,7 +1,5 @@ Filename: 146-long-term-stability.txt Title: Add new flag to reflect long-term stability -Version: $Revision$ -Last-Modified: $Date$ Author: Nick Mathewson Created: 19-Jun-2008 Status: Open diff --git a/doc/spec/proposals/147-prevoting-opinions.txt b/doc/spec/proposals/147-prevoting-opinions.txt index 2b8cf30e46..3d9659c984 100644 --- a/doc/spec/proposals/147-prevoting-opinions.txt +++ b/doc/spec/proposals/147-prevoting-opinions.txt @@ -1,7 +1,5 @@ Filename: 147-prevoting-opinions.txt Title: Eliminate the need for v2 directories in generating v3 directories -Version: $Revision$ -Last-Modified: $Date$ Author: Nick Mathewson Created: 2-Jul-2008 Status: Accepted diff --git a/doc/spec/proposals/148-uniform-client-end-reason.txt b/doc/spec/proposals/148-uniform-client-end-reason.txt index cec81253ea..1db3b3e596 100644 --- a/doc/spec/proposals/148-uniform-client-end-reason.txt +++ b/doc/spec/proposals/148-uniform-client-end-reason.txt @@ -1,7 +1,5 @@ Filename: 148-uniform-client-end-reason.txt Title: Stream end reasons from the client side should be uniform -Version: $Revision$ -Last-Modified: $Date$ Author: Roger Dingledine Created: 2-Jul-2008 Status: Closed diff --git a/doc/spec/proposals/149-using-netinfo-data.txt b/doc/spec/proposals/149-using-netinfo-data.txt index 4919514b4c..8bf8375d5d 100644 --- a/doc/spec/proposals/149-using-netinfo-data.txt +++ b/doc/spec/proposals/149-using-netinfo-data.txt @@ -1,7 +1,5 @@ Filename: 149-using-netinfo-data.txt Title: Using data from NETINFO cells -Version: $Revision$ -Last-Modified: $Date$ Author: Nick Mathewson Created: 2-Jul-2008 Status: Open @@ -24,14 +22,14 @@ Motivation idea of their own IP addresses, so they can publish correct descriptors. This is also in NETINFO cells. -Learning the time and IP +Learning the time and IP address We need to think about attackers here. Just because a router tells us that we have a given IP or a given clock skew doesn't mean that it's true. We believe this information only if we've heard it from a majority of the routers we've connected to recently, including at least 3 routers. Routers only believe this information if the - majority inclues at least one authority. + majority includes at least one authority. Avoiding MITM attacks diff --git a/doc/spec/proposals/150-exclude-exit-nodes.txt b/doc/spec/proposals/150-exclude-exit-nodes.txt index b73a9cc4d1..b497ae62c1 100644 --- a/doc/spec/proposals/150-exclude-exit-nodes.txt +++ b/doc/spec/proposals/150-exclude-exit-nodes.txt @@ -1,6 +1,5 @@ Filename: 150-exclude-exit-nodes.txt Title: Exclude Exit Nodes from a circuit -Version: $Revision$ Author: Mfr Created: 2008-06-15 Status: Closed diff --git a/doc/spec/proposals/151-path-selection-improvements.txt b/doc/spec/proposals/151-path-selection-improvements.txt index e3c8f35451..e0e9e07432 100644 --- a/doc/spec/proposals/151-path-selection-improvements.txt +++ b/doc/spec/proposals/151-path-selection-improvements.txt @@ -1,16 +1,14 @@ Filename: 151-path-selection-improvements.txt Title: Improving Tor Path Selection -Version: -Last-Modified: Author: Fallon Chen, Mike Perry Created: 5-Jul-2008 -Status: Draft +Status: Implemented Overview The performance of paths selected can be improved by adjusting the CircuitBuildTimeout and avoiding failing guard nodes. This proposal - describes a method of tracking buildtime statistics at the client, and + describes a method of tracking buildtime statistics at the client, and using those statistics to adjust the CircuitBuildTimeout. Motivation @@ -22,121 +20,123 @@ Motivation Implementation - Storing Build Times + Gathering Build Times - Circuit build times will be stored in the circular array - 'circuit_build_times' consisting of uint16_t elements as milliseconds. - The total size of this array will be based on the number of circuits + Circuit build times are stored in the circular array + 'circuit_build_times' consisting of uint32_t elements as milliseconds. + The total size of this array is based on the number of circuits it takes to converge on a good fit of the long term distribution of the circuit builds for a fixed link. We do not want this value to be too large, because it will make it difficult for clients to adapt to moving between different links. - From our initial observations, this value appears to be on the order - of 1000, but will be configurable in a #define NCIRCUITS_TO_OBSERVE. - The exact value for this #define will be determined by performing - goodness of fit tests using measurments obtained from the shufflebt.py - script from TorFlow. - + From our observations, the minimum value for a reasonable fit appears + to be on the order of 500 (MIN_CIRCUITS_TO_OBSERVE). However, to keep + a good fit over the long term, we store 5000 most recent circuits in + the array (NCIRCUITS_TO_OBSERVE). + + The Tor client will build test circuits at a rate of one per + minute (BUILD_TIMES_TEST_FREQUENCY) up to the point of + MIN_CIRCUITS_TO_OBSERVE. This allows a fresh Tor to have + a CircuitBuildTimeout estimated within 8 hours after install, + upgrade, or network change (see below). + Long Term Storage - The long-term storage representation will be implemented by storing a - histogram with BUILDTIME_BIN_WIDTH millisecond buckets (default 50) when - writing out the statistics to disk. The format of this histogram on disk - is yet to be finalized, but it will likely be of the format - 'CircuitBuildTime <bin> <count>', with the total specified as - 'TotalBuildTimes <total>' + The long-term storage representation is implemented by storing a + histogram with BUILDTIME_BIN_WIDTH millisecond buckets (default 50) when + writing out the statistics to disk. The format this takes in the + state file is 'CircuitBuildTime <bin-ms> <count>', with the total + specified as 'TotalBuildTimes <total>' Example: TotalBuildTimes 100 - CircuitBuildTimeBin 1 50 - CircuitBuildTimeBin 2 25 - CircuitBuildTimeBin 3 13 + CircuitBuildTimeBin 25 50 + CircuitBuildTimeBin 75 25 + CircuitBuildTimeBin 125 13 ... - Reading the histogram in will entail multiplying each bin by the - BUILDTIME_BIN_WIDTH and then inserting <count> values into the - circuit_build_times array each with the value of - <bin>*BUILDTIME_BIN_WIDTH. In order to evenly distribute the - values in the circular array, a form of index skipping must - be employed. Values from bin #N with bin count C and total T - will occupy indexes specified by N+((T/C)*k)-1, where k is the - set of integers ranging from 0 to C-1. - - For example, this would mean that the values from bin 1 would - occupy indexes 1+(100/50)*k-1, or 0, 2, 4, 6, 8, 10 and so on. - The values for bin 2 would occupy positions 1, 5, 9, 13. Collisions - will be inserted at the first empty position in the array greater - than the selected index (which may requiring looping around the - array back to index 0). + Reading the histogram in will entail inserting <count> values + into the circuit_build_times array each with the value of + <bin-ms> milliseconds. In order to evenly distribute the values + in the circular array, the Fisher-Yates shuffle will be performed + after reading values from the bins. Learning the CircuitBuildTimeout Based on studies of build times, we found that the distribution of - circuit buildtimes appears to be a Pareto distribution. + circuit buildtimes appears to be a Frechet distribution. However, + estimators and quantile functions of the Frechet distribution are + difficult to work with and slow to converge. So instead, since we + are only interested in the accuracy of the tail, we approximate + the tail of the distribution with a Pareto curve starting at + the mode of the circuit build time sample set. We will calculate the parameters for a Pareto distribution fitting the data using the estimators at http://en.wikipedia.org/wiki/Pareto_distribution#Parameter_estimation. - The timeout itself will be calculated by solving the CDF for the - a percentile cutoff BUILDTIME_PERCENT_CUTOFF. This value - represents the percentage of paths the Tor client will accept out of - the total number of paths. We have not yet determined a good - cutoff for this mathematically, but 85% seems a good choice for now. + The timeout itself is calculated by using the Quartile function (the + inverted CDF) to give us the value on the CDF such that + BUILDTIME_PERCENT_CUTOFF (80%) of the mass of the distribution is + below the timeout value. + + Thus, we expect that the Tor client will accept the fastest 80% of + the total number of paths on the network. + + Detecting Changing Network Conditions - From http://en.wikipedia.org/wiki/Pareto_distribution#Definition, - the calculation we need is pow(BUILDTIME_PERCENT_CUTOFF/100.0, k)/Xm. + We attempt to detect both network connectivity loss and drastic + changes in the timeout characteristics. + + We assume that we've had network connectivity loss if 3 circuits + timeout and we've received no cells or TLS handshakes since those + circuits began. We then set the timeout to 60 seconds and stop + counting timeouts. + + If 3 more circuits timeout and the network still has not been + live within this new 60 second timeout window, we then discard + the previous timeouts during this period from our history. + + To detect changing network conditions, we keep a history of + the timeout or non-timeout status of the past RECENT_CIRCUITS (20) + that successfully completed at least one hop. If more than 75% + of these circuits timeout, we discard all buildtimes history, + reset the timeout to 60, and then begin recomputing the timeout. Testing After circuit build times, storage, and learning are implemented, the resulting histogram should be checked for consistency by - verifying it persists across successive Tor invocations where + verifying it persists across successive Tor invocations where no circuits are built. In addition, we can also use the existing - buildtime scripts to record build times, and verify that the histogram + buildtime scripts to record build times, and verify that the histogram the python produces matches that which is output to the state file in Tor, and verify that the Pareto parameters and cutoff points also match. - - Soft timeout vs Hard Timeout - - At some point, it may be desirable to change the cutoff from a - single hard cutoff that destroys the circuit to a soft cutoff and - a hard cutoff, where the soft cutoff merely triggers the building - of a new circuit, and the hard cutoff triggers destruction of the - circuit. - - Good values for hard and soft cutoffs seem to be 85% and 65% - respectively, but we should eventually justify this with observation. - - When to Begin Calculation - The number of circuits to observe (NCIRCUITS_TO_CUTOFF) before - changing the CircuitBuildTimeout will be tunable via a #define. From - our measurements, a good value for NCIRCUITS_TO_CUTOFF appears to be - on the order of 100. + We will also verify that there are no unexpected large deviations from + node selection, such as nodes from distant geographical locations being + completely excluded. Dealing with Timeouts - Timeouts should be counted as the expectation of the region of - of the Pareto distribution beyond the cutoff. The proposal will - be updated with this value soon. + Timeouts should be counted as the expectation of the region of + of the Pareto distribution beyond the cutoff. This is done by + generating a random sample for each timeout at points on the + curve beyond the current timeout cutoff. - Also, in the event of network failure, the observation mechanism - should stop collecting timeout data. + Future Work - Client Hints + At some point, it may be desirable to change the cutoff from a + single hard cutoff that destroys the circuit to a soft cutoff and + a hard cutoff, where the soft cutoff merely triggers the building + of a new circuit, and the hard cutoff triggers destruction of the + circuit. - Some research still needs to be done to provide initial values - for CircuitBuildTimeout based on values learned from modem - users, DSL users, Cable Modem users, and dedicated links. A - radiobutton in Vidalia should eventually be provided that - sets CircuitBuildTimeout to one of these values and also - provide the option of purging all learned data, should any exist. + It may also be beneficial to learn separate timeouts for each + guard node, as they will have slightly different distributions. + This will take longer to generate initial values though. - These values can either be published in the directory, or - shipped hardcoded for a particular Tor version. - Issues Impact on anonymity diff --git a/doc/spec/proposals/152-single-hop-circuits.txt b/doc/spec/proposals/152-single-hop-circuits.txt index e49a4250e0..d0b28b1c72 100644 --- a/doc/spec/proposals/152-single-hop-circuits.txt +++ b/doc/spec/proposals/152-single-hop-circuits.txt @@ -1,7 +1,5 @@ Filename: 152-single-hop-circuits.txt Title: Optionally allow exit from single-hop circuits -Version: -Last-Modified: Author: Geoff Goodell Created: 13-Jul-2008 Status: Closed diff --git a/doc/spec/proposals/153-automatic-software-update-protocol.txt b/doc/spec/proposals/153-automatic-software-update-protocol.txt index 7bc809d440..c2979bb695 100644 --- a/doc/spec/proposals/153-automatic-software-update-protocol.txt +++ b/doc/spec/proposals/153-automatic-software-update-protocol.txt @@ -1,7 +1,5 @@ Filename: 153-automatic-software-update-protocol.txt Title: Automatic software update protocol -Version: $Revision$ -Last-Modified: $Date$ Author: Jacob Appelbaum Created: 14-July-2008 Status: Superseded diff --git a/doc/spec/proposals/154-automatic-updates.txt b/doc/spec/proposals/154-automatic-updates.txt index 00a820de08..4c2c6d3899 100644 --- a/doc/spec/proposals/154-automatic-updates.txt +++ b/doc/spec/proposals/154-automatic-updates.txt @@ -1,7 +1,5 @@ Filename: 154-automatic-updates.txt Title: Automatic Software Update Protocol -Version: $Revision$ -Last-Modified: $Date$ Author: Matt Edman Created: 30-July-2008 Status: Superseded diff --git a/doc/spec/proposals/155-four-hidden-service-improvements.txt b/doc/spec/proposals/155-four-hidden-service-improvements.txt index f528f8baf2..e342bf1c39 100644 --- a/doc/spec/proposals/155-four-hidden-service-improvements.txt +++ b/doc/spec/proposals/155-four-hidden-service-improvements.txt @@ -1,7 +1,5 @@ Filename: 155-four-hidden-service-improvements.txt Title: Four Improvements of Hidden Service Performance -Version: $Revision$ -Last-Modified: $Date$ Author: Karsten Loesing, Christian Wilms Created: 25-Sep-2008 Status: Finished diff --git a/doc/spec/proposals/156-tracking-blocked-ports.txt b/doc/spec/proposals/156-tracking-blocked-ports.txt index 1e7b0d963f..419de7e74c 100644 --- a/doc/spec/proposals/156-tracking-blocked-ports.txt +++ b/doc/spec/proposals/156-tracking-blocked-ports.txt @@ -1,7 +1,5 @@ Filename: 156-tracking-blocked-ports.txt Title: Tracking blocked ports on the client side -Version: $Revision$ -Last-Modified: $Date$ Author: Robert Hogan Created: 14-Oct-2008 Status: Open diff --git a/doc/spec/proposals/157-specific-cert-download.txt b/doc/spec/proposals/157-specific-cert-download.txt index e54a987277..204b20973a 100644 --- a/doc/spec/proposals/157-specific-cert-download.txt +++ b/doc/spec/proposals/157-specific-cert-download.txt @@ -1,7 +1,5 @@ Filename: 157-specific-cert-download.txt Title: Make certificate downloads specific -Version: $Revision$ -Last-Modified: $Date$ Author: Nick Mathewson Created: 2-Dec-2008 Status: Accepted diff --git a/doc/spec/proposals/158-microdescriptors.txt b/doc/spec/proposals/158-microdescriptors.txt index f478a3c834..e6966c0cef 100644 --- a/doc/spec/proposals/158-microdescriptors.txt +++ b/doc/spec/proposals/158-microdescriptors.txt @@ -1,11 +1,20 @@ Filename: 158-microdescriptors.txt Title: Clients download consensus + microdescriptors -Version: $Revision$ -Last-Modified: $Date$ Author: Roger Dingledine Created: 17-Jan-2009 Status: Open +0. History + + 15 May 2009: Substantially revised based on discussions on or-dev + from late January. Removed the notion of voting on how to choose + microdescriptors; made it just a function of the consensus method. + (This lets us avoid the possibility of "desynchronization.") + Added suggestion to use a new consensus flavor. Specified use of + SHA256 for new hashes. -nickm + + 15 June 2009: Cleaned up based on comments from Roger. -nickm + 1. Overview This proposal replaces section 3.2 of proposal 141, which was @@ -13,9 +22,7 @@ Status: Open circuit-building protocol to fetch a server descriptor inline at each circuit extend, we instead put all of the information that clients need either into the consensus itself, or into a new set of data about each - relay called a microdescriptor. The microdescriptor is a direct - transform from the relay descriptor, so relays don't even need to know - this is happening. + relay called a microdescriptor. Descriptor elements that are small and frequently changing should go in the consensus itself, and descriptor elements that are small and @@ -24,6 +31,10 @@ Status: Open them, we'll need to resume considering some design like the one in proposal 141. + Note also that any descriptor element which clients need to use to + decide which servers to fetch info about, or which servers to fetch + info from, needs to stay in the consensus. + 2. Motivation See @@ -36,99 +47,91 @@ Status: Open 3. Design There are three pieces to the proposal. First, authorities will list in - their votes (and thus in the consensus) what relay descriptor elements - are included in the microdescriptor, and also list the expected hash - of microdescriptor for each relay. Second, directory mirrors will serve - microdescriptors. Third, clients will ask for them and cache them. + their votes (and thus in the consensus) the expected hash of + microdescriptor for each relay. Second, authorities will serve + microdescriptors, directory mirrors will cache and serve + them. Third, clients will ask for them and cache them. 3.1. Consensus changes - V3 votes should include a new line: - microdescriptor-elements bar baz foo - listing each descriptor element (sorted alphabetically) that authority - included when it calculated its expected microdescriptor hashes. + If the authorities choose a consensus method of a given version or + later, a microdescriptor format is implicit in that version. + A microdescriptor should in every case be a pure function of the + router descriptor and the consensus method. + + In votes, we need to include the hash of each expected microdescriptor + in the routerstatus section. I suggest a new "m" line for each stanza, + with the base64 of the SHA256 hash of the router's microdescriptor. + + For every consensus method that an authority supports, it includes a + separate "m" line in each router section of its vote, containing: + "m" SP methods 1*(SP AlgorithmName "=" digest) NL + where methods is a comma-separated list of the consensus methods + that the authority believes will produce "digest". - We also need to include the hash of each expected microdescriptor in - the routerstatus section. I suggest a new "m" line for each stanza, - with the base64 of the hash of the elements that the authority voted - for above. + (As with base64 encoding of SHA1 hashes in consensuses, let's + omit the trailing =s) The consensus microdescriptor-elements and "m" lines are then computed as described in Section 3.1.2 below. - I believe that means we need a new consensus-method "6" that knows - how to compute the microdescriptor-elements and add "m" lines. + (This means we need a new consensus-method that knows + how to compute the microdescriptor-elements and add "m" lines.) -3.1.1. Descriptor elements to include for now + The microdescriptor consensus uses the directory-signature format from + proposal 162, with the "sha256" algorithm. - To start, the element list that authorities suggest should be - family onion-key - (Note that the or-dev posts above only mention onion-key, but if - we don't also include family then clients will never learn it. It - seemed like it should be relatively static, so putting it in the - microdescriptor is smarter than trying to fit it into the consensus.) +3.1.1. Descriptor elements to include for now - We could imagine a config option "family,onion-key" so authorities - could change their voted preferences without needing to upgrade. + In the first version, the microdescriptor should contain the + onion-key element, and the family element from the router descriptor, + and the exit policy summary as currently specified in dir-spec.txt. 3.1.2. Computing consensus for microdescriptor-elements and "m" lines - One approach is for the consensus microdescriptor-elements line to - include every element listed by a majority of authorities, sorted. The - problem here is that it will no longer be deterministic what the correct - hash for the "m" line should be. We could imagine telling the authority - to go look in its descriptor and produce the right hash itself, but - we don't want consensus calculation to be based on external data like - that. (Plus, the authority may not have the descriptor that everybody - else voted to use.) - - The better approach is to take the exact set that has the most votes - (breaking ties by the set that has the most elements, and breaking - ties after that by whichever is alphabetically first). That will - increase the odds that we actually get a microdescriptor hash that - is both a) for the descriptor we're putting in the consensus, and b) - over the elements that we're declaring it should be for. - - Then the "m" line for a given relay is the one that gets the most votes - from authorities that both a) voted for the microdescriptor-elements - line we're using, and b) voted for the descriptor we're using. - - (If there's a tie, use the smaller hash. But really, if there are - multiple such votes and they differ about a microdescriptor, we caught - one of them lying or being buggy. We should log it to track down why.) - - If there are no such votes, then we leave out the "m" line for that - relay. That means clients should avoid it for this time period. (As - an extension it could instead mean that clients should fetch the - descriptor and figure out its microdescriptor themselves. But let's - not get ahead of ourselves.) - - It would be nice to have a more foolproof way to agree on what - microdescriptor hash each authority should vote for, so we can avoid - missing "m" lines. Just switching to a new consensus-method each time - we change the set of microdescriptor-elements won't help though, since - each authority will still have to decide what hash to vote for before - knowing what consensus-method will be used. - - Here's one way we could do it. Each vote / consensus includes - the microdescriptor-elements that were used to compute the hashes, - and also a preferred-microdescriptor-elements set. If an authority - has a consensus from the previous period, then it should use the - consensus preferred-microdescriptor-elements when computing its votes - for microdescriptor-elements and the appropriate hashes in the upcoming - period. (If it has no previous consensus, then it just writes its - own preferences in both lines.) - -3.2. Directory mirrors serve microdescriptors - - Directory mirrors should then read the microdescriptor-elements line - from the consensus, and learn how to answer requests. (Directory mirrors - continue to serve normal relay descriptors too, a) to serve old clients - and b) to be able to construct microdescriptors on the fly.) - - The microdescriptors with hashes <D1>,<D2>,<D3> should be available at: - http://<hostname>/tor/micro/d/<D1>+<D2>+<D3>.z + When we are generating a consensus, we use whichever m line + unambiguously corresponds to the descriptor digest that will be + included in the consensus. + + (If different votes have different microdescriptor digests for a + single <descriptor-digest, consensus-method> pair, then at least one + of the authorities is broken. If this happens, the consensus should + contain whichever microdescriptor digest is most common. If there is + no winner, we break ties in the favor of the lexically earliest. + Either way, we should log a warning: there is definitely a bug.) + + The "m" lines in a consensus contain only the digest, not a list of + consensus methods. + +3.1.3. A new flavor of consensus + + Rather than inserting "m" lines in the current consensus format, + they should be included in a new consensus flavor (see proposal + 162). + + This flavor can safely omit descriptor digests. + + When we implement this voting method, we can remove the exit policy + summary from the current "ns" flavor of consensus, since no current + clients use them, and they take up about 5% of the compressed + consensus. + + This new consensus flavor should be signed with the sha256 signature + format as documented in proposal 162. + +3.2. Directory mirrors fetch, cache, and serve microdescriptors + + Directory mirrors should fetch, catch, and serve each microdescriptor + from the authorities. (They need to continue to serve normal relay + descriptors too, to handle old clients.) + + The microdescriptors with base64 hashes <D1>,<D2>,<D3> should be + available at: + http://<hostname>/tor/micro/d/<D1>-<D2>-<D3>.z + (We use base64 for size and for consistency with the consensus + format. We use -s instead of +s to separate these items, since + the + character is used in base64 encoding.) All the microdescriptors from the current consensus should also be available at: @@ -136,24 +139,9 @@ Status: Open so a client that's bootstrapping doesn't need to send a 70KB URL just to name every microdescriptor it's looking for. - The format of a microdescriptor is the header line - "microdescriptor-header" - followed by each element (keyword and body), alphabetically. There's - no need to mention what hash it's for, since it's self-identifying: - you can hash the elements to learn this. - - (Do we need a footer line to show that it's over, or is the next - microdescriptor line or EOF enough of a hint? A footer line wouldn't - hurt much. Also, no fair voting for the microdescriptor-element - "microdescriptor-header".) - + Microdescriptors have no header or footer. The hash of the microdescriptor is simply the hash of the concatenated - elements -- not counting the header line or hypothetical footer line. - Unless you prefer that? - - Is there a reasonable way to version these things? We could say that - the microdescriptor-header line can contain arguments which clients - must ignore if they don't understand them. Any better ways? + elements. Directory mirrors should check to make sure that the microdescriptors they're about to serve match the right hashes (either the hashes from @@ -170,10 +158,14 @@ Status: Open When a client gets a new consensus, it looks to see if there are any microdescriptors it needs to learn. If it needs to learn more than some threshold of the microdescriptors (half?), it requests 'all', - else it requests only the missing ones. + else it requests only the missing ones. Clients MAY try to + determine whether the upload bandwidth for listing the + microdescriptors they want is more or less than the download + bandwidth for the microdescriptors they do not want. Clients maintain a cache of microdescriptors along with metadata like - when it was last referenced by a consensus. They keep a microdescriptor + when it was last referenced by a consensus, and which identity key + it corresponds to. They keep a microdescriptor until it hasn't been mentioned in any consensus for a week. Future clients might cache them for longer or shorter times. @@ -190,18 +182,17 @@ Status: Open Another future option would be to fetch some of the microdescriptors anonymously (via a Tor circuit). + Another crazy option (Roger's phrasing) is to do decoy fetches as + well. + 4. Transition and deployment Phase one, the directory authorities should start voting on - microdescriptors and microdescriptor elements, and putting them in the - consensus. This should happen during the 0.2.1.x series, and should - be relatively easy to do. + microdescriptors, and putting them in the consensus. Phase two, directory mirrors should learn how to serve them, and learn - how to read the consensus to find out what they should be serving. This - phase could be done either in 0.2.1.x or early in 0.2.2.x, depending - on how messy it turns out to be and how quickly we get around to it. + how to read the consensus to find out what they should be serving. Phase three, clients should start fetching and caching them instead - of normal descriptors. This should happen post 0.2.1.x. + of normal descriptors. diff --git a/doc/spec/proposals/159-exit-scanning.txt b/doc/spec/proposals/159-exit-scanning.txt index fbc69aa9e6..7090f2ed08 100644 --- a/doc/spec/proposals/159-exit-scanning.txt +++ b/doc/spec/proposals/159-exit-scanning.txt @@ -1,7 +1,5 @@ Filename: 159-exit-scanning.txt Title: Exit Scanning -Version: $Revision$ -Last-Modified: $Date$ Author: Mike Perry Created: 13-Feb-2009 Status: Open diff --git a/doc/spec/proposals/160-bandwidth-offset.txt b/doc/spec/proposals/160-bandwidth-offset.txt new file mode 100644 index 0000000000..7ca74dfae3 --- /dev/null +++ b/doc/spec/proposals/160-bandwidth-offset.txt @@ -0,0 +1,105 @@ +Filename: 160-bandwidth-offset.txt +Title: Authorities vote for bandwidth offsets in consensus +Author: Roger Dingledine +Created: 4-May-2009 +Status: Open +Target: 0.2.2.x + +1. Motivation + + As part of proposal 141, we moved the bandwidth value for each relay + into the consensus. Now clients can know how they should load balance + even before they've fetched the corresponding relay descriptors. + + Putting the bandwidth in the consensus also lets the directory + authorities choose more accurate numbers to advertise, if we come up + with a better algorithm for deciding weightings. + + Our original plan was to teach directory authorities how to measure + bandwidth themselves; then every authority would vote for the bandwidth + it prefers, and we'd take the median of votes as usual. + + The problem comes when we have 7 authorities, and only a few of them + have smarter bandwidth allocation algorithms. So long as the majority + of them are voting for the number in the relay descriptor, the minority + that have better numbers will be ignored. + +2. Options + + One fix would be to demand that every authority also run the + new bandwidth measurement algorithms: in that case, part of the + responsibility of being an authority operator is that you need to run + this code too. But in practice we can't really require all current + authority operators to do that; and if we want to expand the set of + authority operators even further, it will become even more impractical. + Also, bandwidth testing adds load to the network, so we don't really + want to require that the number of concurrent bandwidth tests match + the number of authorities we have. + + The better fix is to allow certain authorities to specify that they are + voting on bandwidth measurements: more accurate bandwidth values that + have actually been evaluated. In this way, authorities can vote on + the median measured value if sufficient measured votes exist for a router, + and otherwise fall back to the median value taken from the published router + descriptors. + +3. Security implications + + If only some authorities choose to vote on an offset, then a majority of + those voting authorities can arbitrarily change the bandwidth weighting + for the relay. At the extreme, if there's only one offset-voting + authority, then that authority can dictate which relays clients will + find attractive. + + This problem isn't entirely new: we already have the worry wrt + the subset of authorities that vote for BadExit. + + To make it not so bad, we should deploy at least three offset-voting + authorities. + + Also, authorities that know how to vote for offsets should vote for + an offset of zero for new nodes, rather than choosing not to vote on + any offset in those cases. + +4. Design + + First, we need a new consensus method to support this new calculation. + + Now v3 votes can have an additional value on the "w" line: + "w Bandwidth=X Measured=" INT. + + Once we're using the new consensus method, the new way to compute the + Bandwidth weight is by checking if there are at least 3 "Measured" + votes. If so, the median of these is taken. Otherwise, the median + of the "Bandwidth=" values are taken, as described in Proposal 141. + + Then the actual consensus looks just the same as it did before, + so clients never have to know that this additional calculation is + happening. + +5. Implementation + + The Measured values will be read from a file provided by the scanners + described in proposal 161. Files with a timestamp older than 3 days + will be ignored. + + The file will be read in from dirserv_generate_networkstatus_vote_obj() + in a location specified by a new config option "V3MeasuredBandwidths". + A helper function will be called to populate new 'measured' and + 'has_measured' fields of the routerstatus_t 'routerstatuses' list with + values read from this file. + + An additional for_vote flag will be passed to + routerstatus_format_entry() from format_networkstatus_vote(), which will + indicate that the "Measured=" string should be appended to the "w Bandwith=" + line with the measured value in the struct. + + routerstatus_parse_entry_from_string() will be modified to parse the + "Measured=" lines into routerstatus_t struct fields. + + Finally, networkstatus_compute_consensus() will set rs_out.bandwidth + to the median of the measured values if there are more than 3, otherwise + it will use the bandwidth value median as normal. + + + diff --git a/doc/spec/proposals/161-computing-bandwidth-adjustments.txt b/doc/spec/proposals/161-computing-bandwidth-adjustments.txt new file mode 100644 index 0000000000..786e1afebd --- /dev/null +++ b/doc/spec/proposals/161-computing-bandwidth-adjustments.txt @@ -0,0 +1,174 @@ +Title: Computing Bandwidth Adjustments +Filename: 161-computing-bandwidth-adjustments.txt +Author: Mike Perry +Created: 12-May-2009 +Target: 0.2.2.x +Status: Open + + +1. Motivation + + There is high variance in the performance of the Tor network. Despite + our efforts to balance load evenly across the Tor nodes, some nodes are + significantly slower and more overloaded than others. + + Proposal 160 describes how we can augment the directory authorities to + vote on measured bandwidths for routers. This proposal describes what + goes into the measuring process. + + +2. Measurement Selection + + The general idea is to determine a load factor representing the ratio + of the capacity of measured nodes to the rest of the network. This load + factor could be computed from three potentially relevant statistics: + circuit failure rates, circuit extend times, or stream capacity. + + Circuit failure rates and circuit extend times appear to be + non-linearly proportional to node load. We've observed that the same + nodes when scanned at US nighttime hours (when load is presumably + lower) exhibit almost no circuit failure, and significantly faster + extend times than when scanned during the day. + + Stream capacity, however, is much more uniform, even during US + nighttime hours. Moreover, it is a more intuitive representation of + node capacity, and also less dependent upon distance and latency + if amortized over large stream fetches. + + +3. Average Stream Bandwidth Calculation + + The average stream bandwidths are obtained by dividing the network into + slices of 50 nodes each, grouped according to advertised node bandwidth. + + Two hop circuits are built using nodes from the same slice, and a large + file is downloaded via these circuits. The file sizes are set based + on node percentile rank as follows: + + 0-10: 2M + 10-20: 1M + 20-30: 512k + 30-50: 256k + 50-100: 128k + + These sizes are based on measurements performed during test scans. + + This process is repeated until each node has been chosen to participate + in at least 5 circuits. + + +4. Ratio Calculation + + The ratios are calculated by dividing each measured value by the + network-wide average. + + +5. Ratio Filtering + + After the base ratios are calculated, a second pass is performed + to remove any streams with nodes of ratios less than X=0.5 from + the results of other nodes. In addition, all outlying streams + with capacity of one standard deviation below a node's average + are also removed. + + The final ratio result will be greater of the unfiltered ratio + and the filtered ratio. + + +6. Pseudocode for Ratio Calculation Algorithm + + Here is the complete pseudocode for the ratio algorithm: + + Slices = {S | S is 50 nodes of similar consensus capacity} + for S in Slices: + while exists node N in S with circ_chosen(N) < 7: + fetch_slice_file(build_2hop_circuit(N, (exit in S))) + for N in S: + BW_measured(N) = MEAN(b | b is bandwidth of a stream through N) + Bw_stddev(N) = STDDEV(b | b is bandwidth of a stream through N) + Bw_avg(S) = MEAN(b | b = BW_measured(N) for all N in S) + for N in S: + Normal_Streams(N) = {stream via N | bandwidth >= BW_measured(N)} + BW_Norm_measured(N) = MEAN(b | b is a bandwidth of Normal_Streams(N)) + + Bw_net_avg(Slices) = MEAN(BW_measured(N) for all N in Slices) + Bw_Norm_net_avg(Slices) = MEAN(BW_Norm_measured(N) for all N in Slices) + + for N in all Slices: + Bw_net_ratio(N) = Bw_measured(N)/Bw_net_avg(Slices) + Bw_Norm_net_ratio(N) = BW_Norm_measured(N)/Bw_Norm_net_avg(Slices) + + ResultRatio(N) = MAX(Bw_net_ratio(N), Bw_Norm_net_ratio(N)) + + +7. Security implications + + The ratio filtering will deal with cases of sabotage by dropping + both very slow outliers in stream average calculations, as well + as dropping streams that used very slow nodes from the calculation + of other nodes. + + This scheme will not address nodes that try to game the system by + providing better service to scanners. The scanners can be detected + at the entry by IP address, and at the exit by the destination fetch + IP. + + Measures can be taken to obfuscate and separate the scanners' source + IP address from the directory authority IP address. For instance, + scans can happen offsite and the results can be rsynced into the + authorities. The destination server IP can also change. + + Neither of these methods are foolproof, but such nodes can already + lie about their bandwidth to attract more traffic, so this solution + does not set us back any in that regard. + + +8. Parallelization + + Because each slice takes as long as 6 hours to complete, we will want + to parallelize as much as possible. This will be done by concurrently + running multiple scanners from each authority to deal with different + segments of the network. Each scanner piece will continually loop + over a portion of the network, outputting files of the form: + + node_id=<idhex> SP strm_bw=<BW_measured(N)> SP + filt_bw=<BW_Norm_measured(N)> ns_bw=<CurrentConsensusBw(N)> NL + + The most recent file from each scanner will be periodically gathered + by another script that uses them to produce network-wide averages + and calculate ratios as per the algorithm in section 6. Because nodes + may shift in capacity, they may appear in more than one slice and/or + appear more than once in the file set. The most recently measured + line will be chosen in this case. + + +9. Integration with Proposal 160 + + The final results will be produced for the voting mechanism + described in Proposal 160 by multiplying the derived ratio by + the average published consensus bandwidth during the course of the + scan, and taking the weighted average with the previous consensus + bandwidth: + + Bw_new = Round((Bw_current * Alpha + Bw_scan_avg*Bw_ratio)/(Alpha + 1)) + + The Alpha parameter is a smoothing parameter intended to prevent + rapid oscillation between loaded and unloaded conditions. It is + currently fixed at 0.333. + + The Round() step consists of rounding to the 3 most significant figures + in base10, and then rounding that result to the nearest 1000, with + a minimum value of 1000. + + This will produce a new bandwidth value that will be output into a + file consisting of lines of the form: + + node_id=<idhex> SP bw=<Bw_new> NL + + The first line of the file will contain a timestamp in UNIX time() + seconds. This will be used by the authority to decide if the + measured values are too old to use. + + This file can be either copied or rsynced into a directory readable + by the directory authority. + diff --git a/doc/spec/proposals/162-consensus-flavors.txt b/doc/spec/proposals/162-consensus-flavors.txt new file mode 100644 index 0000000000..e3b697afee --- /dev/null +++ b/doc/spec/proposals/162-consensus-flavors.txt @@ -0,0 +1,188 @@ +Filename: 162-consensus-flavors.txt +Title: Publish the consensus in multiple flavors +Author: Nick Mathewson +Created: 14-May-2009 +Target: 0.2.2 +Status: Open + +Overview: + + This proposal describes a way to publish each consensus in + multiple simultaneous formats, or "flavors". This will reduce the + amount of time needed to deploy new consensus-like documents, and + reduce the size of consensus documents in the long term. + +Motivation: + + In the future, we will almost surely want different fields and + data in the network-status document. Examples include: + - Publishing hashes of microdescriptors instead of hashes of + full descriptors (Proposal 158). + - Including different digests of descriptors, instead of the + perhaps-soon-to-be-totally-broken SHA1. + + Note that in both cases, from the client's point of view, this + information _replaces_ older information. If we're using a + SHA256 hash, we don't need to see the SHA1. If clients only want + microdescriptors, they don't (necessarily) need to see hashes of + other things. + + Our past approach to cases like this has been to shovel all of + the data into the consensus document. But this is rather poor + for bandwidth. Adding a single SHA256 hash to a consensus for + each router increases the compressed consensus size by 47%. In + comparison, replacing a single SHA1 hash with a SHA256 hash for + each listed router increases the consensus size by only 18%. + +Design in brief: + + Let the voting process remain as it is, until a consensus is + generated. With future versions of the voting algorithm, instead + of just a single consensus being generated, multiple consensus + "flavors" are produced. + + Consensuses (all of them) include a list of which flavors are + being generated. Caches fetch and serve all flavors of consensus + that are listed, regardless of whether they can parse or validate + them, and serve them to clients. Thus, once this design is in + place, we won't need to deploy more cache changes in order to get + new flavors of consensus to be cached. + + Clients download only the consensus flavor they want. + +A note on hashes: + + Everything in this document is specified to use SHA256, and to be + upgradeable to use better hashes in the future. + +Spec modifications: + + 1. URLs and changes to the current consensus format. + + Every consensus flavor has a name consisting of a sequence of one + or more alphanumeric characters and dashes. For compatibility + current descriptor flavor is called "ns". + + The supported consensus flavors are defined as part of the + authorities' consensus method. + + For each supported flavor, every authority calculates another + consensus document of as-yet-unspecified format, and exchanges + detached signatures for these documents as in the current consensus + design. + + In addition to the consensus currently served at + /tor/status-vote/(current|next)/consensus.z and + /tor/status-vote/(current|next)/consensus/<FP1>+<FP2>+<FP3>+....z , + authorities serve another consensus of each flavor "F" from the + locations /tor/status-vote/(current|next)/consensus-F.z. and + /tor/status-vote/(current|next)/consensus-F/<FP1>+....z. + + When caches serve these documents, they do so from the same + locations. + + 2. Document format: generic consensus. + + The format of a flavored consensus is as-yet-unspecified, except + that the first line is: + "network-status-version" SP version SP flavor NL + + where version is 3 or higher, and the flavor is a string + consisting of alphanumeric characters and dashes, matching the + corresponding flavor listed in the unflavored consensus. + + 3. Document format: detached signatures. + + We amend the detached signature format to include more than one + consensus-digest line, and more than one set of signatures. + + After the consensus-digest line, we allow more lines of the form: + "additional-digest" SP flavor SP algname SP digest NL + + Before the directory-signature lines, we allow more entries of the form: + "additional-signature" SP flavor SP algname SP identity SP + signing-key-digest NL signature. + + [We do not use "consensus-digest" or "directory-signature" for flavored + consensuses, since this could confuse older Tors.] + + The consensus-signatures URL should contain the signatures + for _all_ flavors of consensus. + + 4. The consensus index: + + Authorities additionally generate and serve a consensus-index + document. Its format is: + + Header ValidAfter ValidUntil Documents Signatures + + Header = "consensus-index" SP version NL + ValidAfter = as in a consensus + ValidUntil = as in a consensus + Documents = Document* + Document = "document" SP flavor SP SignedLength + 1*(SP AlgorithmName "=" Digest) NL + Signatures = Signature* + Signature = "directory-signature" SP algname SP identity + SP signing-key-digest NL signature + + There must be one Document line for each generated consensus flavor. + Each Document line describes the length of the signed portion of + a consensus (the signatures themselves are not included), along + with one or more digests of that signed portion. Digests are + given in hex. The algorithm "sha256" MUST be included; others + are allowed. + + The algname part of a signature describes what algorithm was + used to hash the identity and signing keys, and to compute the + signature. The algorithm "sha256" MUST be recognized; + signatures with unrecognized algorithms MUST be ignored. + (See below). + + The consensus index is made available at + /tor/status-vote/(current|next)/consensus-index.z. + + Caches should fetch this document so they can check the + correctness of the different consensus documents they fetch. + They do not need to check anything about an unrecognized + consensus document beyond its digest and length. + + 4.1. The "sha256" signature format. + + The 'SHA256' signature format for directory objects is defined as + the RSA signature of the OAEP+-padded SHA256 digest of the item to + be signed. When checking signatures, the signature MUST be treated + as valid if the signature material begins with SHA256(document); + this allows us to add other data later. + +Considerations: + + - We should not create a new flavor of consensus when adding a + field instead wouldn't be too onerous. + + - We should not proliferate flavors lightly: clients will be + distinguishable based on which flavor they download. + +Migration: + + - Stage one: authorities begin generating and serving + consensus-index files. + + - Stage two: Caches begin downloading consensus-index files, + validating them, and using them to decide what flavors of + consensus documents to cache. They download all listed + documents, and compare them to the digests given in the + consensus. + + - Stage three: Once we want to make a significant change to the + consensus format, we deploy another flavor of consensus at the + authorities. This will immediately start getting cached by the + caches, and clients can start fetching the new flavor without + waiting a version or two for enough caches to begin supporting + it. + +Acknowledgements: + + Aspects of this design and its applications to hash migration were + heavily influenced by IRC conversations with Marian. + diff --git a/doc/spec/proposals/163-detecting-clients.txt b/doc/spec/proposals/163-detecting-clients.txt new file mode 100644 index 0000000000..d838b17063 --- /dev/null +++ b/doc/spec/proposals/163-detecting-clients.txt @@ -0,0 +1,115 @@ +Filename: 163-detecting-clients.txt +Title: Detecting whether a connection comes from a client +Author: Nick Mathewson +Created: 22-May-2009 +Target: 0.2.2 +Status: Open + + +Overview: + + Some aspects of Tor's design require relays to distinguish + connections from clients from connections that come from relays. + The existing means for doing this is easy to spoof. We propose + a better approach. + +Motivation: + + There are at least two reasons for which Tor servers want to tell + which connections come from clients and which come from other + servers: + + 1) Some exits, proposal 152 notwithstanding, want to disallow + their use as single-hop proxies. + 2) Some performance-related proposals involve prioritizing + traffic from relays, or limiting traffic per client (but not + per relay). + + Right now, we detect client vs server status based on how the + client opens circuits. (Check out the code that implements the + AllowSingleHopExits option if you want all the details.) This + method is depressingly easy to fake, though. This document + proposes better means. + +Goals: + + To make grabbing relay privileges at least as difficult as just + running a relay. + + In the analysis below, "using server privileges" means taking any + action that only servers are supposed to do, like delivering a + BEGIN cell to an exit node that doesn't allow single hop exits, + or claiming server-like amounts of bandwidth. + +Passive detection: + + A connection is definitely a client connection if it takes one of + the TLS methods during setup that does not establish an identity + key. + + A circuit is definitely a client circuit if it is initiated with + a CREATE_FAST cell, though the node could be a client or a server. + + A node that's listed in a recent consensus is probably a server. + + A node to which we have successfully extended circuits from + multiple origins is probably a server. + +Active detection: + + If a node doesn't try to use server privileges at all, we never + need to care whether it's a server. + + When a node or circuit tries to use server privileges, if it is + "definitely a client" as per above, we can refuse it immediately. + + If it's "probably a server" as per above, we can accept it. + + Otherwise, we have either a client, or a server that is neither + listed in any consensus or used by any other clients -- in other + words, a new or private server. + + For these servers, we should attempt to build one or more test + circuits through them. If enough of the circuits succeed, the + node is a real relay. If not, it is probably a client. + + While we are waiting for the test circuits to succeed, we should + allow a short grace period in which server privileges are + permitted. When a test is done, we should remember its outcome + for a while, so we don't need to do it again. + +Why it's hard to do good testing: + + Doing a test circuit starting with an unlisted router requires + only that we have an open connection for it. Doing a test + circuit starting elsewhere _through_ an unlisted router--though + more reliable-- would require that we have a known address, port, + identity key, and onion key for the router. Only the address and + identity key are easily available via the current Tor protocol in + all cases. + + We could fix this part by requiring that all servers support + BEGIN_DIR and support downloading at least a current descriptor + for themselves. + +Open questions: + + What are the thresholds for the needed numbers of circuits + for us to decide that a node is a relay? + + [Suggested answer: two circuits from two distinct hosts.] + + How do we pick grace periods? How long do we remember the + outcome of a test? + + [Suggested answer: 10 minute grace period; 48 hour memory of + test outcomes.] + + If we can build circuits starting at a suspect node, but we don't + have enough information to try extending circuits elsewhere + through the node, should we conclude that the node is + "server-like" or not? + + [Suggested answer: for now, just try making circuits through + the node. Extend this to extending circuits as needed.] + diff --git a/doc/spec/proposals/164-reporting-server-status.txt b/doc/spec/proposals/164-reporting-server-status.txt new file mode 100644 index 0000000000..705f5f1a84 --- /dev/null +++ b/doc/spec/proposals/164-reporting-server-status.txt @@ -0,0 +1,91 @@ +Filename: 164-reporting-server-status.txt +Title: Reporting the status of server votes +Author: Nick Mathewson +Created: 22-May-2009 +Target: 0.2.2 +Status: Open + + +Overview: + + When a given node isn't listed in the directory, it isn't always easy + to tell why. This proposal suggest a quick-and-dirty way for + authorities to export not only how they voted, but why, and a way to + collate the information. + +Motivation: + + Right now, if you want to know the reason why your server was listed + a certain way in the Tor directory, the following steps are + recommended: + + - Look through your log for reports of what the authority said + when you tried to upload. + + - Look at the consensus; see if you're listed. + + - Wait a while, see if things get better. + + - Download the votes from all the authorities, and see how they + voted. Try to figure out why. + + - If you think they'll listen to you, ask some authority + operators to look you up in their mtbf files and logs to see + why they voted as they did. + + This is far too hard. + +Solution: + + We should add a new vote-like information-only document that + authorities serve on request. Call it a "vote info". It is + generated at the same time as a vote, but used only for + determining why a server voted as it did. It is served from + /tor/status-vote-info/current/authority[.z] + + It differs from a vote in that: + + * Its vote-status field is 'vote-info'. + + * It includes routers that the authority would not include + in its vote. + + For these, it includes an "omitted" line with an English + message explaining why they were omitted. + + * For each router, it includes a line describing its WFU and + MTBF. The format is: + + "stability <mtbf> up-since='date'" + "uptime <wfu> down-since='date'" + + * It describes the WFU and MTBF thresholds it requires to + vote for a given router in various roles in the header. + The format is: + + "flag-requirement <flag-name> <field> <op> <value>" + + e.g. + + "flag-requirement Guard uptime > 80" + + * It includes info on routers all of whose descriptors that + were uploaded but rejected over the past few hours. The + "r" lines for these are the same as for regular routers. + The other lines are omitted for these routers, and are + replaced with a single "rejected" line, explaining (in + English) why the router was rejected. + + + A status site (like Torweather or Torstatus or another + tool) can poll these files when they are generated, collate + the data, and make it available to server operators. + +Risks: + + This document makes no provisions for caching these "vote + info" documents. If many people wind up fetching them + aggressively from the authorities, that would be bad. + + + diff --git a/doc/spec/proposals/165-simple-robust-voting.txt b/doc/spec/proposals/165-simple-robust-voting.txt new file mode 100644 index 0000000000..f813285a83 --- /dev/null +++ b/doc/spec/proposals/165-simple-robust-voting.txt @@ -0,0 +1,133 @@ +Filename: 165-simple-robust-voting.txt +Title: Easy migration for voting authority sets +Author: Nick Mathewson +Created: 2009-05-28 +Status: Open + +Overview: + + This proposal describes any easy-to-implement, easy-to-verify way to + change the set of authorities without creating a "flag day" situation. + +Motivation: + + From proposal 134 ("More robust consensus voting with diverse + authority sets") by Peter Palfrader: + + Right now there are about five authoritative directory servers + in the Tor network, tho this number is expected to rise to about + 15 eventually. + + Adding a new authority requires synchronized action from all + operators of directory authorities so that at any time during the + update at least half of all authorities are running and agree on + who is an authority. The latter requirement is there so that the + authorities can arrive at a common consensus: Each authority + builds the consensus based on the votes from all authorities it + recognizes, and so a different set of recognized authorities will + lead to a different consensus document. + + In response to this problem, proposal 134 suggested that every + candidate authority list in its vote whom it believes to be an + authority. These A-says-B-is-an-authority relationships form a + directed graph. Each authority then iteratively finds the largest + clique in the graph and remove it, until they find one containing + them. They vote with this clique. + + Proposal 134 had some problems: + + - It had a security problem in that M hostile authorities in a + clique could effectively kick out M-1 honest authorities. This + could enable a minority of the original authorities to take over. + + - It was too complex in its implications to analyze well: it took us + over a year to realize that it was insecure. + + - It tried to solve a bigger problem: general fragmentation of + authority trust. Really, all we wanted to have was the ability to + add and remove authorities without forcing a flag day. + +Proposed protocol design: + + A "Voting Set" is a set of authorities. Each authority has a list of + the voting sets it considers acceptable. These sets are chosen + manually by the authority operators. They must always contain the + authority itself. Each authority lists all of these voting sets in + its votes. + + Authorities exchange votes with every other authority in any of their + voting sets. + + When it is time to calculate a consensus, an authority votes with + whichever voting set it lists that is listed by the most members of + that set. In other words, given two sets S1 and S2 that an authority + lists, that authority will prefer to vote with S1 over S2 whenever + the number of other authorities in S1 that themselves list S1 is + higher than the number of other authorities in S2 that themselves + list S2. + + For example, suppose authority A recognizes two sets, "A B C D" and + "A E F G H". Suppose that the first set is recognized by all of A, + B, C, and D, whereas the second set is recognized only by A, E, and + F. Because the first set is recognize by more of the authorities in + it than the other one, A will vote with the first set. + + Ties are broken in favor of some arbitrary function of the identity + keys of the authorities in the set. + +How to migrate authority sets: + + In steady state, each authority operator should list only the current + actual voting set as accepted. + + When we want to add an authority, each authority operator configures + his or her server to list two voting sets: one containing all the old + authorities, and one containing the old authorities and the new + authority too. Once all authorities are listing the new set of + authorities, they will start voting with that set because of its + size. + + What if one or two authority operators are slow to list the new set? + Then the other operators can stop listing the old set once there are + enough authorities listing the new set to make its voting successful. + (Note that these authorities not listing the new set will still have + their votes counted, since they themselves will be members of the new + set. They will only fail to sign the consensus generated by the + other authorities who are using the new set.) + + When we want to remove an authority, the operators list two voting + sets: one containing all the authorities, and one omitting the + authority we want to remove. Once enough authorities list the new + set as acceptable, we start having authority operators stop listing + the old set. Once there are more listing the new set than the old + set, the new set will win. + +Data format changes: + + Add a new 'voting-set' line to the vote document format. Allow it to + occur any number of times. Its format is: + + voting-set SP 'fingerprint' SP 'fingerprint' ... NL + + where each fingerprint is the hex fingerprint of an identity key of + an authority. Sort fingerprints in ascending order. + + When the consensus method is at least 'X' (decide this when we + implement the proposal), add this line to the consensus format as + well, before the first dir-source line. [This information is not + redundant with the dir-source sections in the consensus: If an + authority is recognized but didn't vote, that authority will appear in + the voting-set line but not in the dir-source sections.] + + We don't need to list other information about authorities in our + vote. + +Migration issues: + + We should keep track somewhere of which Tor client versions + recognized which authorities. + +Acknowledgments: + + The design came out of an IRC conversation with Peter Palfrader. He + had the basic idea first. diff --git a/doc/spec/proposals/166-statistics-extra-info-docs.txt b/doc/spec/proposals/166-statistics-extra-info-docs.txt new file mode 100644 index 0000000000..ab2716a71c --- /dev/null +++ b/doc/spec/proposals/166-statistics-extra-info-docs.txt @@ -0,0 +1,391 @@ +Filename: 166-statistics-extra-info-docs.txt +Title: Including Network Statistics in Extra-Info Documents +Author: Karsten Loesing +Created: 21-Jul-2009 +Target: 0.2.2 +Status: Accepted + +Change history: + + 21-Jul-2009 Initial proposal for or-dev + + +Overview: + + The Tor network has grown to almost two thousand relays and millions + of casual users over the past few years. With growth has come + increasing performance problems and attempts by some countries to + block access to the Tor network. In order to address these problems, + we need to learn more about the Tor network. This proposal suggests to + measure additional statistics and include them in extra-info documents + to help us understand the Tor network better. + + +Introduction: + + As of May 2009, relays, bridges, and directories gather the following + data for statistical purposes: + + - Relays and bridges count the number of bytes that they have pushed + in 15-minute intervals over the past 24 hours. Relays and bridges + include these data in extra-info documents that they send to the + directory authorities whenever they publish their server descriptor. + + - Bridges further include a rough number of clients per country that + they have seen in the past 48 hours in their extra-info documents. + + - Directories can be configured to count the number of clients they + see per country in the past 24 hours and to write them to a local + file. + + Since then we extended the network statistics in Tor. These statistics + include: + + - Directories now gather more precise statistics about connecting + clients. Fixes include measuring in intervals of exactly 24 hours, + counting unsuccessful requests, measuring download times, etc. The + directories append their statistics to a local file every 24 hours. + + - Entry guards count the number of clients per country per day like + bridges do and write them to a local file every 24 hours. + + - Relays measure statistics of the number of cells in their circuit + queues and how much time these cells spend waiting there. Relays + write these statistics to a local file every 24 hours. + + - Exit nodes count the number of read and written bytes on exit + connections per port as well as the number of opened exit streams + per port in 24-hour intervals. Exit nodes write their statistics to + a local file. + + The following four sections contain descriptions for adding these + statistics to the relays' extra-info documents. + + +Directory request statistics: + + The first type of statistics aims at measuring directory requests sent + by clients to a directory mirror or directory authority. More + precisely, these statistics aim at requests for v2 and v3 network + statuses only. These directory requests are sent non-anonymously, + either via HTTP-like requests to a directory's Dir port or tunneled + over a 1-hop circuit. + + Measuring directory request statistics is useful for several reasons: + First, the number of locally seen directory requests can be used to + estimate the total number of clients in the Tor network. Second, the + country-wise classification of requests using a GeoIP database can + help counting the relative and absolute number of users per country. + Third, the download times can give hints on the available bandwidth + capacity at clients. + + Directory requests do not give any hints on the contents that clients + send or receive over the Tor network. Every client requests network + statuses from the directories, so that there are no anonymity-related + concerns to gather these statistics. It might be, though, that clients + wish to hide the fact that they are connecting to the Tor network. + Therefore, IP addresses are resolved to country codes in memory, + events are accumulated over 24 hours, and numbers are rounded up to + multiples of 4 or 8. + + "dirreq-stats-end" YYYY-MM-DD HH:MM:SS (NSEC s) NL + [At most once.] + + YYYY-MM-DD HH:MM:SS defines the end of the included measurement + interval of length NSEC seconds (86400 seconds by default). + + A "dirreq-stats-end" line, as well as any other "dirreq-*" line, + is only added when the relay has opened its Dir port and after 24 + hours of measuring directory requests. + + "dirreq-v2-ips" CC=N,CC=N,... NL + [At most once.] + "dirreq-v3-ips" CC=N,CC=N,... NL + [At most once.] + + List of mappings from two-letter country codes to the number of + unique IP addresses that have connected from that country to + request a v2/v3 network status, rounded up to the nearest multiple + of 8. Only those IP addresses are counted that the directory can + answer with a 200 OK status code. + + "dirreq-v2-reqs" CC=N,CC=N,... NL + [At most once.] + "dirreq-v3-reqs" CC=N,CC=N,... NL + [At most once.] + + List of mappings from two-letter country codes to the number of + requests for v2/v3 network statuses from that country, rounded up + to the nearest multiple of 8. Only those requests are counted that + the directory can answer with a 200 OK status code. + + "dirreq-v2-share" num% NL + [At most once.] + "dirreq-v3-share" num% NL + [At most once.] + + The share of v2/v3 network status requests that the directory + expects to receive from clients based on its advertised bandwidth + compared to the overall network bandwidth capacity. Shares are + formatted in percent with two decimal places. Shares are + calculated as means over the whole 24-hour interval. + + "dirreq-v2-resp" status=num,... NL + [At most once.] + "dirreq-v3-resp" status=nul,... NL + [At most once.] + + List of mappings from response statuses to the number of requests + for v2/v3 network statuses that were answered with that response + status, rounded up to the nearest multiple of 4. Only response + statuses with at least 1 response are reported. New response + statuses can be added at any time. The current list of response + statuses is as follows: + + "ok": a network status request is answered; this number + corresponds to the sum of all requests as reported in + "dirreq-v2-reqs" or "dirreq-v3-reqs", respectively, before + rounding up. + "not-enough-sigs: a version 3 network status is not signed by a + sufficient number of requested authorities. + "unavailable": a requested network status object is unavailable. + "not-found": a requested network status is not found. + "not-modified": a network status has not been modified since the + If-Modified-Since time that is included in the request. + "busy": the directory is busy. + + "dirreq-v2-direct-dl" key=val,... NL + [At most once.] + "dirreq-v3-direct-dl" key=val,... NL + [At most once.] + "dirreq-v2-tunneled-dl" key=val,... NL + [At most once.] + "dirreq-v3-tunneled-dl" key=val,... NL + [At most once.] + + List of statistics about possible failures in the download process + of v2/v3 network statuses. Requests are either "direct" + HTTP-encoded requests over the relay's directory port, or + "tunneled" requests using a BEGIN_DIR cell over the relay's OR + port. The list of possible statistics can change, and statistics + can be left out from reporting. The current list of statistics is + as follows: + + Successful downloads and failures: + + "complete": a client has finished the download successfully. + "timeout": a download did not finish within 10 minutes after + starting to send the response. + "running": a download is still running at the end of the + measurement period for less than 10 minutes after starting to + send the response. + + Download times: + + "min", "max": smallest and largest measured bandwidth in B/s. + "d[1-4,6-9]": 1st to 4th and 6th to 9th decile of measured + bandwidth in B/s. For a given decile i, i/10 of all downloads + had a smaller bandwidth than di, and (10-i)/10 of all downloads + had a larger bandwidth than di. + "q[1,3]": 1st and 3rd quartile of measured bandwidth in B/s. One + fourth of all downloads had a smaller bandwidth than q1, one + fourth of all downloads had a larger bandwidth than q3, and the + remaining half of all downloads had a bandwidth between q1 and + q3. + "md": median of measured bandwidth in B/s. Half of the downloads + had a smaller bandwidth than md, the other half had a larger + bandwidth than md. + + +Entry guard statistics: + + Entry guard statistics include the number of clients per country and + per day that are connecting directly to an entry guard. + + Entry guard statistics are important to learn more about the + distribution of clients to countries. In the future, this knowledge + can be useful to detect if there are or start to be any restrictions + for clients connecting from specific countries. + + The information which client connects to a given entry guard is very + sensitive. This information must not be combined with the information + what contents are leaving the network at the exit nodes. Therefore, + entry guard statistics need to be aggregated to prevent them from + becoming useful for de-anonymization. Aggregation includes resolving + IP addresses to country codes, counting events over 24-hour intervals, + and rounding up numbers to the next multiple of 8. + + "entry-stats-end" YYYY-MM-DD HH:MM:SS (NSEC s) NL + [At most once.] + + YYYY-MM-DD HH:MM:SS defines the end of the included measurement + interval of length NSEC seconds (86400 seconds by default). + + An "entry-stats-end" line, as well as any other "entry-*" + line, is first added after the relay has been running for at least + 24 hours. + + "entry-ips" CC=N,CC=N,... NL + [At most once.] + + List of mappings from two-letter country codes to the number of + unique IP addresses that have connected from that country to the + relay and which are no known other relays, rounded up to the + nearest multiple of 8. + + +Cell statistics: + + The third type of statistics have to do with the time that cells spend + in circuit queues. In order to gather these statistics, the relay + memorizes when it puts a given cell in a circuit queue and when this + cell is flushed. The relay further notes the life time of the circuit. + These data are sufficient to determine the mean number of cells in a + queue over time and the mean time that cells spend in a queue. + + Cell statistics are necessary to learn more about possible reasons for + the poor network performance of the Tor network, especially high + latencies. The same statistics are also useful to determine the + effects of design changes by comparing today's data with future data. + + There are basically no privacy concerns from measuring cell + statistics, regardless of a node being an entry, middle, or exit node. + + "cell-stats-end" YYYY-MM-DD HH:MM:SS (NSEC s) NL + [At most once.] + + YYYY-MM-DD HH:MM:SS defines the end of the included measurement + interval of length NSEC seconds (86400 seconds by default). + + A "cell-stats-end" line, as well as any other "cell-*" line, + is first added after the relay has been running for at least 24 + hours. + + "cell-processed-cells" num,...,num NL + [At most once.] + + Mean number of processed cells per circuit, subdivided into + deciles of circuits by the number of cells they have processed in + descending order from loudest to quietest circuits. + + "cell-queued-cells" num,...,num NL + [At most once.] + + Mean number of cells contained in queues by circuit decile. These + means are calculated by 1) determining the mean number of cells in + a single circuit between its creation and its termination and 2) + calculating the mean for all circuits in a given decile as + determined in "cell-processed-cells". Numbers have a precision of + two decimal places. + + "cell-time-in-queue" num,...,num NL + [At most once.] + + Mean time cells spend in circuit queues in milliseconds. Times are + calculated by 1) determining the mean time cells spend in the + queue of a single circuit and 2) calculating the mean for all + circuits in a given decile as determined in + "cell-processed-cells". + + "cell-circuits-per-decile" num NL + [At most once.] + + Mean number of circuits that are included in any of the deciles, + rounded up to the next integer. + + +Exit statistics: + + The last type of statistics affects exit nodes counting the number of + bytes written and read and the number of streams opened per port and + per 24 hours. Exit port statistics can be measured from looking at + headers of BEGIN and DATA cells. A BEGIN cell contains the exit port + that is required for the exit node to open a new exit stream. + Subsequent DATA cells coming from the client or being sent back to the + client contain a length field stating how many bytes of application + data are contained in the cell. + + Exit port statistics are important to measure in order to identify + possible load-balancing problems with respect to exit policies. Exit + nodes that permit more ports than others are very likely overloaded + with traffic for those ports plus traffic for other ports. Improving + load balancing in the Tor network improves the overall utilization of + bandwidth capacity. + + Exit traffic is one of the most sensitive parts of network data in the + Tor network. Even though these statistics do not require looking at + traffic contents, statistics are aggregated so that they are not + useful for de-anonymizing users. Only those ports are reported that + have seen at least 0.1% of exiting or incoming bytes, numbers of bytes + are rounded up to full kibibytes (KiB), and stream numbers are rounded + up to the next multiple of 4. + + "exit-stats-end" YYYY-MM-DD HH:MM:SS (NSEC s) NL + [At most once.] + + YYYY-MM-DD HH:MM:SS defines the end of the included measurement + interval of length NSEC seconds (86400 seconds by default). + + An "exit-stats-end" line, as well as any other "exit-*" line, is + first added after the relay has been running for at least 24 hours + and only if the relay permits exiting (where exiting to a single + port and IP address is sufficient). + + "exit-kibibytes-written" port=N,port=N,... NL + [At most once.] + "exit-kibibytes-read" port=N,port=N,... NL + [At most once.] + + List of mappings from ports to the number of kibibytes that the + relay has written to or read from exit connections to that port, + rounded up to the next full kibibyte. + + "exit-streams-opened" port=N,port=N,... NL + [At most once.] + + List of mappings from ports to the number of opened exit streams + to that port, rounded up to the nearest multiple of 4. + + +Implementation notes: + + Right now, relays that are configured accordingly write similar + statistics to those described in this proposal to disk every 24 hours. + With this proposal being implemented, relays include the contents of + these files in extra-info documents. + + The following steps are necessary to implement this proposal: + + 1. The current format of [dirreq|entry|buffer|exit]-stats files needs + to be adapted to the description in this proposal. This step + basically means renaming keywords. + + 2. The timing of writing the four *-stats files should be unified, so + that they are written exactly 24 hours after starting the + relay. Right now, the measurement intervals for dirreq, entry, and + exit stats starts with the first observed request, and files are + written when observing the first request that occurs more than 24 + hours after the beginning of the measurement interval. With this + proposal, the measurement intervals should all start at the same + time, and files should be written exactly 24 hours later. + + 3. It is advantageous to cache statistics in local files in the data + directory until they are included in extra-info documents. The + reason is that the 24-hour measurement interval can be very + different from the 18-hour publication interval of extra-info + documents. When a relay crashes after finishing a measurement + interval, but before publishing the next extra-info document, + statistics would get lost. Therefore, statistics are written to + disk when finishing a measurement interval and read from disk when + generating an extra-info document. Only the statistics that were + appended to the *-stats files within the past 24 hours are included + in extra-info documents. Further, the contents of the *-stats files + need to be checked in the process of generating extra-info documents. + + 4. With the statistics patches being tested, the ./configure options + should be removed and the statistics code be compiled by default. + It is still required for relay operators to add configuration + options (DirReqStatistics, ExitPortStatistics, etc.) to enable + gathering statistics. However, in the near future, statistics shall + be enabled gathered by all relays by default, where requiring a + ./configure option would be a barrier for many relay operators. diff --git a/doc/spec/proposals/167-params-in-consensus.txt b/doc/spec/proposals/167-params-in-consensus.txt new file mode 100644 index 0000000000..d23bc9c01e --- /dev/null +++ b/doc/spec/proposals/167-params-in-consensus.txt @@ -0,0 +1,47 @@ +Filename: 167-params-in-consensus.txt +Title: Vote on network parameters in consensus +Author: Roger Dingledine +Created: 18-Aug-2009 +Status: Closed +Implemented-In: 0.2.2 + +0. History + + +1. Overview + + Several of our new performance plans involve guessing how to tune + clients and relays, yet we won't be able to learn whether we guessed + the right tuning parameters until many people have upgraded. Instead, + we should have directory authorities vote on the parameters, and teach + Tors to read the currently recommended values out of the consensus. + +2. Design + + V3 votes should include a new "params" line after the known-flags + line. It contains key=value pairs, where value is an integer. + + Consensus documents that are generated with a sufficiently new consensus + method (7?) then include a params line that includes every key listed + in any vote, and the median value for that key (in case of ties, + we use the median closer to zero). + +2.1. Planned keys. + + The first planned parameter is "circwindow=101", which is the initial + circuit packaging window that clients and relays should use. Putting + it in the consensus will let us perform experiments with different + values once enough Tors have upgraded -- see proposal 168. + + Later parameters might include a weighting for how much to favor quiet + circuits over loud circuits in our round-robin algorithm; a weighting + for how much to prioritize relays over clients if we use an incentive + scheme like the gold-star design; and what fraction of circuits we + should throw out from proposal 151. + +2.2. What about non-integers? + + I'm not sure how we would do median on non-integer values. Further, + I don't have any non-integer values in mind yet. So I say we cross + that bridge when we get to it. + diff --git a/doc/spec/proposals/168-reduce-circwindow.txt b/doc/spec/proposals/168-reduce-circwindow.txt new file mode 100644 index 0000000000..c10cf41e2e --- /dev/null +++ b/doc/spec/proposals/168-reduce-circwindow.txt @@ -0,0 +1,134 @@ +Filename: 168-reduce-circwindow.txt +Title: Reduce default circuit window +Author: Roger Dingledine +Created: 12-Aug-2009 +Status: Open +Target: 0.2.2 + +0. History + + +1. Overview + + We should reduce the starting circuit "package window" from 1000 to + 101. The lower package window will mean that clients will only be able + to receive 101 cells (~50KB) on a circuit before they need to send a + 'sendme' acknowledgement cell to request 100 more. + + Starting with a lower package window on exit relays should save on + buffer sizes (and thus memory requirements for the exit relay), and + should save on queue sizes (and thus latency for users). + + Lowering the package window will induce an extra round-trip for every + additional 50298 bytes of the circuit. This extra step is clearly a + slow-down for large streams, but ultimately we hope that a) clients + fetching smaller streams will see better response, and b) slowing + down the large streams in this way will produce lower e2e latencies, + so the round-trips won't be so bad. + +2. Motivation + + Karsten's torperf graphs show that the median download time for a 50KB + file over Tor in mid 2009 is 7.7 seconds, whereas the median download + time for 1MB and 5MB are around 50s and 150s respectively. The 7.7 + second figure is way too high, whereas the 50s and 150s figures are + surprisingly low. + + The median round-trip latency appears to be around 2s, with 25% of + the data points taking more than 5s. That's a lot of variance. + + We designed Tor originally with the original goal of maximizing + throughput. We figured that would also optimize other network properties + like round-trip latency. Looks like we were wrong. + +3. Design + + Wherever we initialize the circuit package window, initialize it to + 101 rather than 1000. Reducing it should be safe even when interacting + with old Tors: the old Tors will receive the 101 cells and send back + a sendme ack cell. They'll still have much higher deliver windows, + but the rest of their deliver window will go unused. + + You can find the patch at arma/circwindow. It seems to work. + +3.1. Why not 100? + + Tor 0.0.0 through 0.2.1.19 have a bug where they only send the sendme + ack cell after 101 cells rather than the intended 100 cells. + + Once 0.2.1.19 is obsolete we can change it back to 100 if we like. But + hopefully we'll have moved to some datagram protocol long before + 0.2.1.19 becomes obsolete. + +3.2. What about stream packaging windows? + + Right now the stream packaging windows start at 500. The goal was to + set the stream window to half the circuit window, to provide a crude + load balancing between streams on the same circuit. Once we lower + the circuit packaging window, the stream packaging window basically + becomes redundant. + + We could leave it in -- it isn't hurting much in either case. Or we + could take it out -- people building other Tor clients would thank us + for that step. Alas, people building other Tor clients are going to + have to be compatible with current Tor clients, so in practice there's + no point taking out the stream packaging windows. + +3.3. What about variable circuit windows? + + Once upon a time we imagined adapting the circuit package window to + the network conditions. That is, we would start the window small, + and raise it based on the latency and throughput we see. + + In theory that crude imitation of TCP's windowing system would allow + us to adapt to fill the network better. In practice, I think we want + to stick with the small window and never raise it. The low cap reduces + the total throughput you can get from Tor for a given circuit. But + that's a feature, not a bug. + +4. Evaluation + + How do we know this change is actually smart? It seems intuitive that + it's helpful, and some smart systems people have agreed that it's + a good idea (or said another way, they were shocked at how big the + default package window was before). + + To get a more concrete sense of the benefit, though, Karsten has been + running torperf side-by-side on exit relays with the old package window + vs the new one. The results are mixed currently -- it is slightly faster + for fetching 40KB files, and slightly slower for fetching 50KB files. + + I think it's going to be tough to get a clear conclusion that this is + a good design just by comparing one exit relay running the patch. The + trouble is that the other hops in the circuits are still getting bogged + down by other clients introducing too much traffic into the network. + + Ultimately, we'll want to put the circwindow parameter into the + consensus so we can test a broader range of values once enough relays + have upgraded. + +5. Transition and deployment + + We should put the circwindow in the consensus (see proposal 167), + with an initial value of 101. Then as more exit relays upgrade, + clients should seamlessly get the better behavior. + + Note that upgrading the exit relay will only affect the "download" + package window. An old client that's uploading lots of bytes will + continue to use the old package window at the client side, and we + can't throttle that window at the exit side without breaking protocol. + + The real question then is what we should backport to 0.2.1. Assuming + this could be a big performance win, we can't afford to wait until + 0.2.2.x comes out before starting to see the changes here. So we have + two options as I see them: + a) once clients in 0.2.2.x know how to read the value out of the + consensus, and it's been tested for a bit, backport that part to + 0.2.1.x. + b) if it's too complex to backport, just pick a number, like 101, and + backport that number. + + Clearly choice (a) is the better one if the consensus parsing part + isn't very complex. Let's shoot for that, and fall back to (b) if the + patch turns out to be so big that we reconsider. + diff --git a/doc/spec/proposals/ideas/xxx-bwrate-algs.txt b/doc/spec/proposals/ideas/xxx-bwrate-algs.txt new file mode 100644 index 0000000000..757f5bc55e --- /dev/null +++ b/doc/spec/proposals/ideas/xxx-bwrate-algs.txt @@ -0,0 +1,106 @@ +# The following two algorithms + + +# Algorithm 1 +# TODO: Burst and Relay/Regular differentiation + +BwRate = Bandwidth Rate in Bytes Per Second +GlobalWriteBucket = 0 +GlobalReadBucket = 0 +Epoch = Token Fill Rate in seconds: suggest 50ms=.050 +SecondCounter = 0 +MinWriteBytes = Minimum amount bytes per write + +Every Epoch Seconds: + UseMinWriteBytes = MinWriteBytes + WriteCnt = 0 + ReadCnt = 0 + BytesRead = 0 + + For Each Open OR Conn with pending write data: + WriteCnt++ + For Each Open OR Conn: + ReadCnt++ + + BytesToRead = (BwRate*Epoch + GlobalReadBucket)/ReadCnt + BytesToWrite = (BwRate*Epoch + GlobalWriteBucket)/WriteCnt + + if BwRate/WriteCnt < MinWriteBytes: + # If we aren't likely to accumulate enough bytes in a second to + # send a whole cell for our connections, send partials + Log(NOTICE, "Too many ORCons to write full blocks. Sending short packets.") + UseMinWriteBytes = 1 + # Other option: We could switch to plan 2 here + + # Service each writable ORConn. If there are any partial writes, + # return remaining bytes from this epoch to the global pool + For Each Open OR Conn with pending write data: + ORConn->write_bucket += BytesToWrite + if ORConn->write_bucket > UseMinWriteBytes: + w = write(ORConn, MIN(len(ORConn->write_data), ORConn->write_bucket)) + # possible that w < ORConn->write_data here due to TCP pushback. + # We should restore the rest of the write_bucket to the global + # buffer + GlobalWriteBucket += (ORConn->write_bucket - w) + ORConn->write_bucket = 0 + + For Each Open OR Conn: + r = read_nonblock(ORConn, BytesToRead) + BytesRead += r + + SecondCounter += Epoch + if SecondCounter < 1: + # Save unused bytes from this epoch to be used later in the second + GlobalReadBucket += (BwRate*Epoch - BytesRead) + else: + SecondCounter = 0 + GlobalReadBucket = 0 + GlobalWriteBucket = 0 + For Each ORConn: + ORConn->write_bucket = 0 + + + +# Alternate plan for Writing fairly. Reads would still be covered +# by plan 1 as there is no additional network overhead for short reads, +# so we don't need to try to avoid them. +# +# I think this is actually pretty similar to what we do now, but +# with the addition that the bytes accumulate up to the second mark +# and we try to keep track of our position in the write list here +# (unless libevent is doing that for us already and I just don't see it) +# +# TODO: Burst and Relay/Regular differentiation + +# XXX: The inability to send single cells will cause us to block +# on EXTEND cells for low-bandwidth node pairs.. +BwRate = Bandwidth Rate in Bytes Per Second +WriteBytes = Bytes per write +Epoch = MAX(MIN(WriteBytes/BwRate, .333s), .050s) + +SecondCounter = 0 +GlobalWriteBucket = 0 + +# New connections are inserted at Head-1 (the 'tail' of this circular list) +# This is not 100% fifo for all node data, but it is the best we can do +# without insane amounts of additional queueing complexity. +WriteConnList = List of Open OR Conns with pending write data > WriteBytes +WriteConnHead = 0 + +Every Epoch Seconds: + GlobalWriteBucket += BwRate*Epoch + WriteListEnd = WriteConnHead + + do + ORCONN = WriteConnList[WriteConnHead] + w = write(ORConn, WriteBytes) + GlobalWriteBucket -= w + WriteConnHead += 1 + while GlobalWriteBucket > 0 and WriteConnHead != WriteListEnd + + SecondCounter += Epoch + if SecondCounter >= 1: + SecondCounter = 0 + GlobalWriteBucket = 0 + + diff --git a/doc/spec/proposals/ideas/xxx-choosing-crypto-in-tor-protocol.txt b/doc/spec/proposals/ideas/xxx-choosing-crypto-in-tor-protocol.txt new file mode 100644 index 0000000000..e8489570f7 --- /dev/null +++ b/doc/spec/proposals/ideas/xxx-choosing-crypto-in-tor-protocol.txt @@ -0,0 +1,138 @@ +Filename: xxx-choosing-crypto-in-tor-protocol.txt +Title: Picking cryptographic standards in the Tor wire protocol +Author: Marian +Created: 2009-05-16 +Status: Draft + +Motivation: + + SHA-1 is horribly outdated and not suited for security critical + purposes. SHA-2, RIPEMD-160, Whirlpool and Tigerare good options + for a short-term replacement, but in the long run, we will + probably want to upgrade to the winner or a semi-finalist of the + SHA-3 competition. + + For a 2006 comparison of different hash algorithms, read: + http://www.sane.nl/sane2006/program/final-papers/R10.pdf + + Other reading about SHA-1: + http://www.schneier.com/blog/archives/2005/02/sha1_broken.html + http://www.schneier.com/blog/archives/2005/08/new_cryptanalyt.html + http://www.schneier.com/paper-preimages.html + + Additionally, AES has been theoretically broken for years. While + the attack is still not efficient enough that the public sector + has been able to prove that it works, we should probably consider + the time between a theoretical attack and a practical attack as an + opportunity to figure out how to upgrade to a better algorithm, + such as Twofish. + + See: + http://schneier.com/crypto-gram-0209.html#1 + +Design: + + I suggest that nodes should publish in directories which + cryptographic standards, such as hash algorithms and ciphers, + they support. Clients communicating with nodes will then + pick whichever of those cryptographic standards they prefer + the most. In the case that the node does not publish which + cryptographic standards it supports, the client should assume + that the server supports the older standards, such as SHA-1 + and AES, until such time as we choose to desupport those + standards. + + Node to node communications could work similarly. However, in + case they both support a set of algorithms but have different + preferences, the disagreement would have to be resolved + somehow. Two possibilities include: + * the node requesting communications presents which + cryptographic standards it supports in the request. The + other node picks. + * both nodes send each other lists of what they support and + what version of Tor they are using. The newer node picks, + based on the assumption that the newer node has the most up + to date information about which hash algorithm is the best. + Of course, the node could lie about its version, but then + again, it could also maliciously choose only to support older + algorithms. + + Using this method, we could potentially add server side support + to hash algorithms and ciphers before we instruct clients to + begin preferring those hash algorithms and ciphers. In this way, + the clients could upgrade and the servers would already support + the newly preferred hash algorithms and ciphers, even if the + servers were still using older versions of Tor, so long as the + older versions of Tor were at least new enough to have server + side support. + + This would make quickly upgrading to new hash algorithms and + ciphers easier. This could be very useful when new attacks + are published. + + One concern is that client preferences could expose the client + to segmentation attacks. To mitigate this, we suggest hardcoding + preferences in the client, to prevent the client from choosing + to use a new hash algorithm or cipher that no one else is using + yet. While offering a preference might be useful in case a client + with an older version of Tor wants to start using the newer hash + algorithm or cipher that everyone else is using, if the client + cares enough, he or she can just upgrade Tor. + + We may also have to worry about nodes which, through laziness or + maliciousness, refuse to start supporting new hash algorithms or + ciphers. This must be balanced with the need to maintain + backward compatibility so the client will have a large selection + of nodes to pick from. Adding new hash algorithms and ciphers + long before we suggest nodes start using them can help mitigate + this. However, eventually, once sufficient nodes support new + standards, client side support for older standards should be + disabled, particularly if there are practical rather than merely + theoretical attacks. + + Server side support for older standards can be kept much longer + than client side support, since clients using older hashes and + ciphers are really only hurting theirselvse. + + If server side support for a hash algorithm or cipher is added + but never preferred before we decide we don't really want it, + support can be removed without having to worry about backward + compatibility. + +Security implications: + Improving cryptography will improve Tor's security. However, if + clients pick different cryptographic standards, they could be + partitioned based on their cryptographic preferences. We also + need to worry about nodes refusing to support new standards. + These issues are detailed above. + +Specification: + + Todo. Need better understanding of how Tor currently works or + help from someone who does. + +Compatibility: + + This idea is intended to allow easier upgrading of cryptographic + hash algorithms and ciphers while maintaining backwards + compatibility. However, at some point, backwards compatibility + with very old hashes and ciphers should be dropped for security + reasons. + +Implementation: + + Todo. + +Performance and scalability nodes: + + Better hashes and cipher are someimes a little more CPU intensive + than weaker ones. For instance, on most computers AES is a little + faster than Twofish. However, in that example, I consider Twofish's + additional security worth the tradeoff. + +Acknowledgements: + + Discussed this on IRC with a few people, mostly Nick Mathewson. + Nick was particularly helpful in explaining how Tor works, + explaining goals, and providing various links to Tor + specifications. diff --git a/doc/spec/proposals/ideas/xxx-encrypted-services.txt b/doc/spec/proposals/ideas/xxx-encrypted-services.txt new file mode 100644 index 0000000000..3414f3c4fb --- /dev/null +++ b/doc/spec/proposals/ideas/xxx-encrypted-services.txt @@ -0,0 +1,18 @@ + +the basic idea might be to generate a keypair, and sign little statements +like "this key corresponds to this relay id", and publish them on karsten's +hs dht. + +so if you want to talk to it, you look it up, then go to that exit. +and by 'go to' i mean 'build a tor circuit like normal except you're sure +where to exit' + +connecting to it is slower than usual, but once you're connected, it's no +slower than normal tor. +and you get what wikileaks wants from its hidden service, which is really +just the UI piece. +indymedia also wants this. + +might be interesting to let an encrypted service list more than one relay, +too. + diff --git a/doc/spec/proposals/ideas/xxx-hide-platform.txt b/doc/spec/proposals/ideas/xxx-hide-platform.txt index 3fed5cfbd4..ad19fb1fd4 100644 --- a/doc/spec/proposals/ideas/xxx-hide-platform.txt +++ b/doc/spec/proposals/ideas/xxx-hide-platform.txt @@ -1,7 +1,5 @@ Filename: xxx-hide-platform.txt Title: Hide Tor Platform Information -Version: $Revision$ -Last-Modified: $Date$ Author: Jacob Appelbaum Created: 24-July-2008 Status: Draft diff --git a/doc/spec/proposals/ideas/xxx-port-knocking.txt b/doc/spec/proposals/ideas/xxx-port-knocking.txt index 9fbcdf3545..85c27ec52d 100644 --- a/doc/spec/proposals/ideas/xxx-port-knocking.txt +++ b/doc/spec/proposals/ideas/xxx-port-knocking.txt @@ -1,7 +1,5 @@ Filename: xxx-port-knocking.txt Title: Port knocking for bridge scanning resistance -Version: $Revision$ -Last-Modified: $Date$ Author: Jacob Appelbaum Created: 19-April-2009 Status: Draft diff --git a/doc/spec/proposals/ideas/xxx-separate-streams-by-port.txt b/doc/spec/proposals/ideas/xxx-separate-streams-by-port.txt index cebde65a9b..f26c1e580f 100644 --- a/doc/spec/proposals/ideas/xxx-separate-streams-by-port.txt +++ b/doc/spec/proposals/ideas/xxx-separate-streams-by-port.txt @@ -1,7 +1,5 @@ Filename: xxx-separate-streams-by-port.txt Title: Separate streams across circuits by destination port -Version: $Revision$ -Last-Modified: $Date$ Author: Robert Hogan Created: 21-Oct-2008 Status: Draft diff --git a/doc/spec/proposals/ideas/xxx-what-uses-sha1.txt b/doc/spec/proposals/ideas/xxx-what-uses-sha1.txt index 9b6e20c586..b3ca3eea5a 100644 --- a/doc/spec/proposals/ideas/xxx-what-uses-sha1.txt +++ b/doc/spec/proposals/ideas/xxx-what-uses-sha1.txt @@ -1,8 +1,6 @@ Filename: xxx-what-uses-sha1.txt Title: Where does Tor use SHA-1 today? -Version: $Revision$ -Last-Modified: $Date$ -Author: Nick Mathewson +Authors: Nick Mathewson, Marian Created: 30-Dec-2008 Status: Meta @@ -15,9 +13,15 @@ Introduction: too long. According to smart crypto people, the SHA-2 functions (SHA-256, etc) - share too much of SHA-1's structure to be very good. Some people - like other hash functions; most of these have not seen enough - analysis to be widely regarded as an extra-good idea. + share too much of SHA-1's structure to be very good. RIPEMD-160 is + also based on flawed past hashes. Some people think other hash + functions (e.g. Whirlpool and Tiger) are not as bad; most of these + have not seen enough analysis to be used yet. + + Here is a 2006 paper about hash algorithms. + http://www.sane.nl/sane2006/program/final-papers/R10.pdf + + (Todo: Ask smart crypto people.) By 2012, the NIST SHA-3 competition will be done, and with luck we'll have something good to switch too. But it's probably a bad idea to @@ -54,50 +58,138 @@ Why now? one look silly. +Triage + + How severe are these problems? Let's divide them into these + categories, where H(x) is the SHA-1 hash of x: + PREIMAGE -- find any x such that a H(x) has a chosen value + -- A SHA-1 usage that only depends on preimage + resistance + * Also SECOND PREIMAGE. Given x, find a y not equal to + x such that H(x) = H(y) + COLLISION<role> -- A SHA-1 usage that depends on collision + resistance, but the only party who could mount a + collision-based attack is already in a trusted role + (like a distribution signer or a directory authority). + COLLISION -- find any x and y such that H(x) = H(y) -- A + SHA-1 usage that depends on collision resistance + and doesn't need the attacker to have any special keys. + + There is no need to put much effort into fixing PREIMAGE and SECOND + PREIMAGE usages in the near-term: while there have been some + theoretical results doing these attacks against SHA-1, they don't + seem to be close to practical yet. To fix COLLISION<code-signing> + usages is not too important either, since anyone who has the key to + sign the code can mount far worse attacks. It would be good to fix + COLLISION<authority> usages, since we try to resist bad authorities + to a limited extent. The COLLISION usages are the most important + to fix. + + Kelsey and Schneier published a theoretical second preimage attack + against SHA-1 in 2005, so it would be a good idea to fix PREIMAGE + and SECOND PREIMAGE usages after fixing COLLISION usages or where fixes + require minimal effort. + + http://www.schneier.com/paper-preimages.html + + Additionally, we need to consider the impact of a successful attack + in each of these cases. SHA-1 collisions are still expensive even + if recent results are verified, and anybody with the resources to + compute one also has the resources to mount a decent Sybil attack. + + Let's be pessimistic, and not assume that producing collisions of + a given format is actually any harder than producing collisions at + all. + What Tor uses hashes for today: 1. Infrastructure. A. Our X.509 certificates are signed with SHA-1. + COLLSION B. TLS uses SHA-1 (and MD5) internally to generate keys. + PREIMAGE? + * At least breaking SHA-1 and MD5 simultaneously is + much more difficult than breaking either + independently. C. Some of the TLS ciphersuites we allow use SHA-1. + PREIMAGE? D. When we sign our code with GPG, it might be using SHA-1. + COLLISION<code-signing> + * GPG 1.4 and up have writing support for SHA-2 hashes. + This blog has help for converting: + http://www.schwer.us/journal/2005/02/19/sha-1-broken-and-gnupg-gpg/ E. Our GPG keys might be authenticated with SHA-1. + COLLISION<code-signing-key-signing> F. OpenSSL's random number generator uses SHA-1, I believe. + PREIMAGE 2. The Tor protocol A. Everything we sign, we sign using SHA-1-based OAEP-MGF1. + PREIMAGE? B. Our CREATE cell format uses SHA-1 for: OAEP padding. + PREIMAGE? C. Our EXTEND cells use SHA-1 to hash the identity key of the target server. + COLLISION D. Our CREATED cells use SHA-1 to hash the derived key data. + ?? E. The data we use in CREATE_FAST cells to generate a key is the length of a SHA-1. + NONE F. The data we send back in a CREATED/CREATED_FAST cell is the length of a SHA-1. - G. We use SHA-1 to derive our circuit keys from the negotiated g^xy value. + NONE + G. We use SHA-1 to derive our circuit keys from the negotiated g^xy + value. + NONE H. We use SHA-1 to derive the digest field of each RELAY cell, but that's used more as a checksum than as a strong digest. + NONE 3. Directory services + [All are COLLISION or COLLISION<authority> ] + A. All signatures are generated on the SHA-1 of their corresponding documents, using PKCS1 padding. + * In dir-spec.txt, section 1.3, it states, + "SIGNATURE" Object contains a signature (using the signing key) + of the PKCS1-padded digest of the entire document, taken from + the beginning of the Initial item, through the newline after + the Signature Item's keyword and its arguments." + So our attacker, Malcom, could generate a collision for the hash + that is signed. Thus, a second pre-image attack is possible. + Vulnerable to regular collision attack only if key is stolen. + If the key is stolen, Malcom could distribute two different + copies of the document which have the same hash. Maybe useful + for a partitioning attack? B. Router descriptors identify their corresponding extra-info documents by their SHA-1 digest. + * A third party might use a second pre-image attack to generate a + false extra-info document that has the same hash. The router + itself might use a regular collision attack to generate multiple + extra-info documents with the same hash, which might be useful + for a partitioning attack. C. Fingerprints in router descriptors are taken using SHA-1. - D. Fingerprints in authority certs are taken using SHA-1. - E. Fingerprints in dir-source lines of votes and consensuses are taken + * The fingerprint must match the public key. Not sure what would + happen if two routers had different public keys but the same + fingerprint. There could perhaps be unpredictable behaviour. + D. In router descriptors, routers in the same "Family" may be listed + by server nicknames or hexdigests. + * Does not seem critical. + E. Fingerprints in authority certs are taken using SHA-1. + F. Fingerprints in dir-source lines of votes and consensuses are taken using SHA-1. - F. Networkstatuses refer to routers identity keys and descriptors by their + G. Networkstatuses refer to routers identity keys and descriptors by their SHA-1 digests. - G. Directory-signature lines identify which key is doing the signing by + H. Directory-signature lines identify which key is doing the signing by the SHA-1 digests of the authority's signing key and its identity key. - H. The following items are downloaded by the SHA-1 of their contents: + I. The following items are downloaded by the SHA-1 of their contents: XXXX list them - I. The following items are downloaded by the SHA-1 of an identity key: + J. The following items are downloaded by the SHA-1 of an identity key: XXXX list them too. 4. The rendezvous protocol @@ -107,6 +199,12 @@ What Tor uses hashes for today: establishment requests. B. Hidden servers use SHA-1 in multiple places when generating hidden service descriptors. + * The permanent-id is the first 80 bits of the SHA-1 hash of the + public key + ** time-period performs caclulations using the permanent-id + * The secret-id-part is the SHA-1 has of the time period, the + descriptor-cookie, and replica. + * Hash of introduction point's identity key. C. Hidden servers performing basic-type client authorization for their services use SHA-1 when encrypting introduction points contained in hidden service descriptors. @@ -115,26 +213,35 @@ What Tor uses hashes for today: identifier or not. E. Hidden servers use SHA-1 to derive .onion addresses of their services. + * What's worse, it only uses the first 80 bits of the SHA-1 hash. + However, the rend-spec.txt says we aren't worried about arbitrary + collisons? F. Clients use SHA-1 to generate the current hidden service descriptor identifiers for a given .onion address. G. Hidden servers use SHA-1 to remember digests of the first parts of Diffie-Hellman handshakes contained in introduction requests in order - to detect replays. + to detect replays. See the RELAY_ESTABLISH_INTRO cell. We seem to be + taking a hash of a hash here. H. Hidden servers use SHA-1 during the Diffie-Hellman key exchange with a connecting client. 5. The bridge protocol XXXX write me + + A. Client may attempt to query for bridges where he knows a digest + (probably SHA-1) before a direct query. 6. The Tor user interface A. We log information about servers based on SHA-1 hashes of their identity keys. + COLLISION B. The controller identifies servers based on SHA-1 hashes of their identity keys. + COLLISION C. Nearly all of our configuration options that list servers allow SHA-1 hashes of their identity keys. + COLLISION E. The deprecated .exit notation uses SHA-1 hashes of identity keys - - + COLLISION diff --git a/doc/spec/proposals/reindex.py b/doc/spec/proposals/reindex.py index 2b4c02516b..980bc0659f 100755 --- a/doc/spec/proposals/reindex.py +++ b/doc/spec/proposals/reindex.py @@ -4,7 +4,7 @@ import re, os class Error(Exception): pass STATUSES = """DRAFT NEEDS-REVISION NEEDS-RESEARCH OPEN ACCEPTED META FINISHED - CLOSED SUPERSEDED DEAD""".split() + CLOSED SUPERSEDED DEAD REJECTED""".split() REQUIRED_FIELDS = [ "Filename", "Status", "Title" ] CONDITIONAL_FIELDS = { "OPEN" : [ "Target" ], "ACCEPTED" : [ "Target "], diff --git a/doc/spec/rend-spec.txt b/doc/spec/rend-spec.txt index e3fbe2253b..f030092679 100644 --- a/doc/spec/rend-spec.txt +++ b/doc/spec/rend-spec.txt @@ -1,4 +1,3 @@ -$Id$ Tor Rendezvous Specification @@ -145,33 +144,10 @@ $Id$ 1.2. Bob's OP generates service descriptors. The first time the OP provides an advertised service, it generates - a public/private keypair (stored locally). Periodically, the OP - generates and publishes a descriptor of type "V0". + a public/private keypair (stored locally). - The "V0" descriptor contains: - - KL Key length [2 octets] - PK Bob's public key [KL octets] - TS A timestamp [4 octets] - NI Number of introduction points [2 octets] - Ipt A list of NUL-terminated ORs [variable] - SIG Signature of above fields [variable] - - KL is the length of PK, in octets. - TS is the number of seconds elapsed since Jan 1, 1970. - - The members of Ipt may be either (a) nicknames, or (b) identity key - digests, encoded in hex, and prefixed with a '$'. Clients must - accept both forms. Services must only generate the second form. - Once 0.0.9.x is obsoleted, we can drop the first form. - - [It's ok for Bob to advertise 0 introduction points. He might want - to do that if he previously advertised some introduction points, - and now he doesn't have any. -RD] - - Beginning with 0.2.0.10-alpha, Bob's OP encodes "V2" descriptors in - addition to "V0" descriptors. The format of a "V2" descriptor is as - follows: + Beginning with 0.2.0.10-alpha, Bob's OP encodes "V2" descriptors. The + format of a "V2" descriptor is as follows: "rendezvous-service-descriptor" descriptor-id NL @@ -340,6 +316,10 @@ $Id$ (This ends the fields in the encrypted portion of the descriptor.) + [It's ok for Bob to advertise 0 introduction points. He might want + to do that if he previously advertised some introduction points, + and now he doesn't have any. -RD] + "signature" NL signature-string [At end, exactly once] @@ -349,6 +329,21 @@ $Id$ 1.2.1. Other descriptor formats we don't use. + Support for the V0 descriptor format was dropped in 0.2.2.0-alpha-dev: + + KL Key length [2 octets] + PK Bob's public key [KL octets] + TS A timestamp [4 octets] + NI Number of introduction points [2 octets] + Ipt A list of NUL-terminated ORs [variable] + SIG Signature of above fields [variable] + + KL is the length of PK, in octets. + TS is the number of seconds elapsed since Jan 1, 1970. + + The members of Ipt may be either (a) nicknames, or (b) identity key + digests, encoded in hex, and prefixed with a '$'. + The V1 descriptor format was understood and accepted from 0.1.1.5-alpha-cvs to 0.2.0.6-alpha-dev, but no Tors generated it and it was removed: @@ -409,7 +404,7 @@ $Id$ RELAY_ESTABLISH_INTRO cell, containing: KL Key length [2 octets] - PK Bob's public key [KL octets] + PK Introduction public key [KL octets] HS Hash of session info [20 octets] SIG Signature of above information [variable] @@ -431,16 +426,13 @@ $Id$ currently associated with PK. On success, the OR sends Bob a RELAY_INTRO_ESTABLISHED cell with an empty payload. - If a hidden service is configured to publish only v2 hidden service - descriptors, Bob's OP does not include its own public key in the - RELAY_ESTABLISH_INTRO cell, but the public key of a freshly generated - key pair. The OP also includes these fresh public keys in the v2 hidden - service descriptor together with the other introduction point - information. The reason is that the introduction point does not need to - and therefore should not know for which hidden service it works, so as - to prevent it from tracking the hidden service's activity. If the hidden - service is configured to publish both, v0 and v2 descriptors, two - separate sets of introduction points are established. + Bob's OP does not include its own public key in the RELAY_ESTABLISH_INTRO + cell, but the public key of a freshly generated introduction key pair. + The OP also includes these fresh public keys in the v2 hidden service + descriptor together with the other introduction point information. The + reason is that the introduction point does not need to and therefore + should not know for which hidden service it works, so as to prevent it + from tracking the hidden service's activity. 1.4. Bob's OP advertises his service descriptor(s). @@ -464,10 +456,8 @@ $Id$ after its timestamp. At least every 18 hours, Bob's OP uploads a fresh descriptor. - If Bob's OP is configured to publish v2 descriptors instead of or in - addition to v0 descriptors, it does so to a changing subset of all v2 - hidden service directories instead of the authoritative directory - servers. Therefore, Bob's OP opens a stream via Tor to each + Bob's OP publishes v2 descriptors to a changing subset of all v2 hidden + service directories. Therefore, Bob's OP opens a stream via Tor to each responsible hidden service directory. (He may re-use old circuits for this.) Over this stream, Bob's OP makes an HTTP 'POST' request to a URL "/tor/rendezvous2/publish" relative to the hidden service @@ -520,12 +510,21 @@ $Id$ 1.6. Alice's OP retrieves a service descriptor. - Alice opens a stream to a directory server via Tor, and makes an HTTP GET - request for the document '/tor/rendezvous/<z>', where '<z>' is replaced - with the encoding of Bob's public key as described above. (She may re-use - old circuits for this.) The directory replies with a 404 HTTP response if - it does not recognize <z>, and otherwise returns Bob's most recently - uploaded service descriptor. + Similarly to the description in section 1.4, Alice's OP fetches a v2 + descriptor from a randomly chosen hidden service directory out of the + changing subset of 6 nodes. If the request is unsuccessful, Alice retries + the other remaining responsible hidden service directories in a random + order. Alice relies on Bob to care about a potential clock skew between + the two by possibly storing two sets of descriptors (see end of section + 1.4). + + Alice's OP opens a stream via Tor to the chosen v2 hidden service + directory. (She may re-use old circuits for this.) Over this stream, + Alice's OP makes an HTTP 'GET' request for the document + "/tor/rendezvous2/<z>", where z is replaced with the encoding of the + descriptor ID. The directory replies with a 404 HTTP response if it does + not recognize <z>, and otherwise returns Bob's most recently uploaded + service descriptor. If Alice's OP receives a 404 response, it tries the other directory servers, and only fails the lookup if none recognize the public key hash. @@ -541,22 +540,6 @@ $Id$ [Caching may make her partitionable, but she fetched it anonymously, and we can't very well *not* cache it. -RD] - Alice's OP fetches v2 descriptors in parallel to v0 descriptors. Similarly - to the description in section 1.4, the OP fetches a v2 descriptor from a - randomly chosen hidden service directory out of the changing subset of - 6 nodes. If the request is unsuccessful, Alice retries the other - remaining responsible hidden service directories in a random order. - Alice relies on Bob to care about a potential clock skew between the two - by possibly storing two sets of descriptors (see end of section 1.4). - - Alice's OP opens a stream via Tor to the chosen v2 hidden service - directory. (She may re-use old circuits for this.) Over this stream, - Alice's OP makes an HTTP 'GET' request for the document - "/tor/rendezvous2/<z>", where z is replaced with the encoding of the - descriptor ID. The directory replies with a 404 HTTP response if it does - not recognize <z>, and otherwise returns Bob's most recently uploaded - service descriptor. - 1.7. Alice's OP establishes a rendezvous point. When Alice requests a connection to a given location-hidden service, diff --git a/doc/spec/socks-extensions.txt b/doc/spec/socks-extensions.txt index 8d58987f35..62d86acd9f 100644 --- a/doc/spec/socks-extensions.txt +++ b/doc/spec/socks-extensions.txt @@ -1,4 +1,3 @@ -$Id$ Tor's extensions to the SOCKS protocol 1. Overview diff --git a/doc/spec/tor-spec.txt b/doc/spec/tor-spec.txt index a321aa8694..e94ac7faaa 100644 --- a/doc/spec/tor-spec.txt +++ b/doc/spec/tor-spec.txt @@ -1,4 +1,3 @@ -$Id$ Tor Protocol Specification @@ -21,7 +20,7 @@ see tor-design.pdf. PK -- a public key. SK -- a private key. - K -- a key for a symmetric cypher. + K -- a key for a symmetric cipher. a|b -- concatenation of 'a' and 'b'. @@ -172,8 +171,8 @@ see tor-design.pdf. In "renegotiation", the connection initiator sends no certificates, and the responder sends a single connection certificate. Once the TLS handshake is complete, the initiator renegotiates the handshake, with each - parties sending a two-certificate chain as in "certificates up-front". - The initiator's ClientHello MUST include at least once ciphersuite not in + party sending a two-certificate chain as in "certificates up-front". + The initiator's ClientHello MUST include at least one ciphersuite not in the list above. The responder SHOULD NOT select any ciphersuite besides those in the list above. [The above "should not" is because some of the ciphers that @@ -201,9 +200,9 @@ see tor-design.pdf. to decide which to use. In all of the above handshake variants, certificates sent in the clear - SHOULD NOT include any strings to identify the host as a Tor server. In - the "renegotation" and "backwards-compatible renegotiation", the - initiator SHOULD chose a list of ciphersuites and TLS extensions chosen + SHOULD NOT include any strings to identify the host as a Tor server. In + the "renegotiation" and "backwards-compatible renegotiation" steps, the + initiator SHOULD choose a list of ciphersuites and TLS extensions to mimic one used by a popular web browser. Responders MUST NOT select any TLS ciphersuite that lacks ephemeral keys, @@ -289,7 +288,7 @@ see tor-design.pdf. 6 -- CREATED_FAST (Circuit created, no PK) (See Sec 5.1) 7 -- VERSIONS (Negotiate proto version) (See Sec 4) 8 -- NETINFO (Time and address info) (See Sec 4) - 9 -- RELAY_EARLY (End-to-end data; limited) (See sec 5.6) + 9 -- RELAY_EARLY (End-to-end data; limited)(See Sec 5.6) The interpretation of 'Payload' depends on the type of the cell. PADDING: Payload is unused. @@ -357,7 +356,7 @@ see tor-design.pdf. The address format is a type/length/value sequence as given in section 6.4 below. The timestamp is a big-endian unsigned integer number of - seconds since the unix epoch. + seconds since the Unix epoch. Implementations MAY use the timestamp value to help decide if their clocks are skewed. Initiators MAY use "other OR's address" to help @@ -399,7 +398,7 @@ see tor-design.pdf. Onion skin [DH_LEN+KEY_LEN+PK_PAD_LEN bytes] Identity fingerprint [HASH_LEN bytes] - The port and address field denote the IPV4 address and port of the next + The port and address field denote the IPv4 address and port of the next onion router in the circuit; the public key hash is the hash of the PKCS#1 ASN1 encoding of the next onion router's identity (signing) key. (See 0.3 above.) Including this hash allows the extending OR verify that it is @@ -886,7 +885,7 @@ see tor-design.pdf. 6.4. Remote hostname lookup To find the address associated with a hostname, the OP sends a - RELAY_RESOLVE cell containing the hostname to be resolved with a nul + RELAY_RESOLVE cell containing the hostname to be resolved with a NUL terminating byte. (For a reverse lookup, the OP sends a RELAY_RESOLVE cell containing an in-addr.arpa address.) The OR replies with a RELAY_RESOLVED cell containing a status byte, and any number of diff --git a/doc/spec/version-spec.txt b/doc/spec/version-spec.txt index 842271ae19..265717f409 100644 --- a/doc/spec/version-spec.txt +++ b/doc/spec/version-spec.txt @@ -1,4 +1,3 @@ -$Id$ HOW TOR VERSION NUMBERS WORK diff --git a/doc/tor.1.in b/doc/tor.1.in index 1a72ebd09f..8cd9e1e00e 100644 --- a/doc/tor.1.in +++ b/doc/tor.1.in @@ -1,4 +1,4 @@ -.TH TOR 1 "January 2009" "TOR" +.TH TOR 1 "August 2009" "TOR" .SH NAME tor \- The second-generation onion router .SH SYNOPSIS @@ -41,7 +41,7 @@ Verify the configuration file is valid. \fB--nt-service\fP \fB--service [install|remove|start|stop]\fP Manage the Tor Windows NT/2000/XP service. Current instructions can -be found at http://wiki.noreply.org/noreply/TheOnionRouter/TorFAQ#WinNTService +be found at https://wiki.torproject.org/noreply/TheOnionRouter/TorFAQ#WinNTService .LP .TP \fB--list-torrc-options\fP @@ -97,6 +97,20 @@ _relayed traffic_ to the given number of bytes in each direction. (Default: 0) .LP .TP +\fBPerConnBWRate \fR\fIN\fR \fBbytes\fR|\fBKB\fR|\fBMB\fR|\fBGB\fR|\fBTB\fP +If set, do separate rate limiting for each connection from a non-relay. +You should never need to change this value, since a network-wide value +is published in the consensus and your relay will use that value. +(Default: 0) +.LP +.TP +\fBPerConnBWBurst \fR\fIN\fR \fBbytes\fR|\fBKB\fR|\fBMB\fR|\fBGB\fR|\fBTB\fP +If set, do separate rate limiting for each connection from a non-relay. +You should never need to change this value, since a network-wide value +is published in the consensus and your relay will use that value. +(Default: 0) +.LP +.TP \fBConnLimit \fR\fINUM\fP The minimum number of file descriptors that must be available to the Tor process before it will start. Tor will ask the OS for as @@ -234,6 +248,19 @@ the default hidden service authorities, but not the directory or bridge authorities. .LP .TP +\fBDisableAllSwap \fR\fB0\fR|\fB1\fR\fP +If set to 1, Tor will attempt to lock all current and future memory pages. +On supported platforms, this should effectively disable any and all attempts +to page out memory. Under the hood, DisableAllSwap uses mlockall() on unix-like +platforms. Windows is currently unsupported. We believe that this feature works +on modern Gnu/Linux distributions. Mac OS X appears to be broken by design. On +reasonable *BSD systems it should also be supported but this is untested. This +option requires that you start your Tor as root. If you use DisableAllSwap, +please consider using the User option to properly reduce the privileges of +your Tor. +(Default: 0) +.LP +.TP \fBFetchDirInfoEarly \fR\fB0\fR|\fB1\fR\fP If set to 1, Tor will always fetch directory information like other directory caches, even if you don't meet the normal criteria for @@ -241,6 +268,13 @@ fetching early. Normal users should leave it off. (Default: 0) .LP .TP +\fBFetchDirInfoExtraEarly \fR\fB0\fR|\fB1\fR\fP +If set to 1, Tor will fetch directory information before other +directory caches. It will attempt to download directory information closer to +the start of the consensus period. Normal users should leave it off. +(Default: 0) +.LP +.TP \fBFetchHidServDescriptors \fR\fB0\fR|\fB1\fR\fP If set to 0, Tor will never fetch any hidden service descriptors from the rendezvous directories. This option is only useful if you're using @@ -292,6 +326,25 @@ HTTPS proxy authentication that Tor supports; feel free to submit a patch if you want it to support others. .LP .TP +\fBSocks4Proxy\fR \fIhost\fR[:\fIport\fR]\fP +Tor will make all OR connections through the SOCKS 4 proxy at host:port +(or host:1080 if port is not specified). +.LP +.TP +\fBSocks5Proxy\fR \fIhost\fR[:\fIport\fR]\fP +Tor will make all OR connections through the SOCKS 5 proxy at host:port +(or host:1080 if port is not specified). +.LP +.TP +\fBSocks5ProxyUsername\fR \fIusername\fP +.LP +.TP +\fBSocks5ProxyPassword\fR \fIpassword\fP +If defined, authenticate to the SOCKS 5 server using username and password +in accordance to RFC 1929. Both username and password must be between 1 and 255 +characters. +.LP +.TP \fBKeepalivePeriod \fR\fINUM\fP To keep firewalls from expiring connections, send a padding keepalive cell every NUM seconds on open connections that are in use. If the @@ -338,11 +391,16 @@ no effect on Windows; instead you should use the --service command-line option. (Default: 0) .LP .TP -\fBSafeLogging \fR\fB0\fR|\fB1\fP -If 1, Tor replaces potentially sensitive strings in the logs -(e.g. addresses) with the string [scrubbed]. This way logs can still be +\fBSafeLogging \fR\fB0\fR|\fB1\fR|\fBrelay\fP +Tor can scrub potentially sensitive strings from log messages (e.g. addresses) +by replacing them with the string [scrubbed]. This way logs can still be useful, but they don't leave behind personally identifying information -about what sites a user might have visited. (Default: 1) +about what sites a user might have visited. + +If this option is set to 0, Tor will not perform any scrubbing, if it is set +to 1, all potentially sensitive strings are replaced. If it is set to +relay, all log messages generated when acting as a relay are sanitized, but all +messages generated when acting as a client are not. (Default: 1) .LP .TP \fBUser \fR\fIUID\fP @@ -350,8 +408,19 @@ On startup, setuid to this user and setgid to their primary group. .LP .TP \fBHardwareAccel \fR\fB0\fR|\fB1\fP -If non-zero, try to use crypto hardware acceleration when -available. This is untested and probably buggy. (Default: 0) +If non-zero, try to use built-in (static) crypto hardware acceleration when +available. (Default: 0) +.LP +.TP +\fBAccelName \fR\fINAME\fP +When using OpenSSL hardware crypto acceleration attempt to load the dynamic +engine of this name. This must be used for any dynamic hardware engine. Names +can be verified with the openssl engine command. +.LP +.TP +\fBAccelDir \fR\fIDIR\fP +Specify this option if using dynamic hardware acceleration and the engine +implementation library resides somewhere other than the OpenSSL default. .LP .TP \fBAvoidDiskWrites \fR\fB0\fR|\fB1\fP @@ -369,6 +438,19 @@ ORPort. (Default: 1) \fBPreferTunneledDirConns \fR\fB0\fR|\fB1\fP If non-zero, we will avoid directory servers that don't support tunneled directory connections, when possible. (Default: 1) +.LP +.TP +\fBCircuitPriorityHalflife \fR\fBNUM\fB1\fP +If this value is set, we override the default algorithm for choosing which +circuit's cell to deliver or relay next. When the value is 0, we +round-robin between the active circuits on a connection, delivering one cell +from each in turn. When the value is positive, we prefer delivering cells +from whichever connection has the lowest weighted cell count, where cells are +weighted exponentially according to the supplied CircuitPriorityHalflife +value (in seconds). If this option is not set at all, we use the behavior +recommended in the current consensus networkstatus. +This is an advanced option; you generally shouldn't have mess with it. +(Default: not set.) .SH CLIENT OPTIONS .PP @@ -416,6 +498,13 @@ circuit list. (Default: 1 hour.) .LP .TP +\fBCircuitStreamTimeout \fR\fINUM\fP +If non-zero, this option overrides our internal timeout schedule for +how many seconds until we detach a stream from a circuit and try a new +circuit. If your network is particularly slow, you might want to set +this to a number like 60. (Default: 0) +.LP +.TP \fBClientOnly \fR\fB0\fR|\fB1\fR\fP If set to 1, Tor will under no circumstances run as a server or serve directory requests. The default @@ -440,26 +529,25 @@ list. .TP \fBEntryNodes \fR\fInode\fR,\fInode\fR,\fI...\fP A list of identity fingerprints, nicknames, country codes and address patterns -of nodes to use for the first hop in the circuit. -These are treated only as preferences unless StrictEntryNodes (see +of nodes to use for the first hop in normal circuits. +These are treated only as preferences unless StrictNodes (see below) is also set. .LP .TP \fBExitNodes \fR\fInode\fR,\fInode\fR,\fI...\fP A list of identity fingerprints, nicknames, country codes and address patterns -of nodes to use for the last hop in the circuit. -These are treated only as preferences unless StrictExitNodes (see +of nodes to use for the last hop in normal exit circuits. +These are treated only as preferences unless StrictNodes (see below) is also set. .LP .TP -\fBStrictEntryNodes \fR\fB0\fR|\fB1\fR\fP -If 1, Tor will never use any nodes besides those listed in "EntryNodes" for -the first hop of a circuit. -.LP -.TP -\fBStrictExitNodes \fR\fB0\fR|\fB1\fR\fP -If 1, Tor will never use any nodes besides those listed in "ExitNodes" for -the last hop of a circuit. +\fBStrictNodes \fR\fB0\fR|\fB1\fR\fP +If 1 and EntryNodes config option is set, Tor will never use any +nodes besides those listed in EntryNodes for the first hop of a normal +circuit. If 1 and ExitNodes config option is set, Tor will never use any +nodes besides those listed in ExitNodes for the last hop of a normal exit +circuit. Note that Tor might still use these nodes for non-exit circuits +such as one-hop directory fetches or hidden service support circuits. .LP .TP \fBFascistFirewall \fR\fB0\fR|\fB1\fR\fP @@ -674,6 +762,13 @@ resolved. This helps trap accidental attempts to resolve URLs and so on. (Default: 0) .LP .TP +\fBAllowDotExit \fR\fB0\fR|\fB1\fR\fP +If enabled, we convert "www.google.com.foo.exit" addresses on the +SocksPort/TransPort/NatdPort into "www.google.com" addresses that exit +from the node "foo". Disabled by default since attacking websites and +exit relays can use it to manipulate your path selection. (Default: 0) +.LP +.TP \fBFastFirstHopPK \fR\fB0\fR|\fB1\fR\fP When this option is disabled, Tor uses the public key step for the first hop of creating circuits. Skipping it is generally safe since we have @@ -1031,6 +1126,36 @@ behalf of clients. .TP \fBGeoIPFile \fR\fIfilename\fP A filename containing GeoIP data, for use with BridgeRecordUsageByCountry. +.LP +.TP +\fBCellStatistics \fR\fB0\fR|\fB1\fR\fP +When this option is enabled, Tor writes statistics on the mean time that +cells spend in circuit queues to disk every 24 hours. Cannot be changed +while Tor is running. (Default: 0) +.LP +.TP +\fBDirReqStatistics \fR\fB0\fR|\fB1\fR\fP +When this option is enabled, Tor writes statistics on the number and +response time of network status requests to disk every 24 hours. Cannot be +changed while Tor is running. (Default: 0) +.LP +.TP +\fBEntryStatistics \fR\fB0\fR|\fB1\fR\fP +When this option is enabled, Tor writes statistics on the number of +directly connecting clients to disk every 24 hours. Cannot be changed +while Tor is running. (Default: 0) +.LP +.TP +\fBExitPortStatistics \fR\fB0\fR|\fB1\fR\fP +When this option is enabled, Tor writes statistics on the number of +relayed bytes and opened stream per exit port to disk every 24 hours. +Cannot be changed while Tor is running. (Default: 0) +.LP +.TP +\fBExtraInfoStatistics \fR\fB0\fR|\fB1\fR\fP +When this option is enabled, Tor includes previously gathered statistics +in its extra-info documents that it uploads to the directory authorities. +(Default: 0) .SH DIRECTORY SERVER OPTIONS .PP @@ -1098,12 +1223,6 @@ When this option is set in addition to \fBAuthoritativeDirectory\fP, Tor also accepts and serves hidden service descriptors. (Default: 0) .LP .TP -\fBHSAuthorityRecordStats \fR\fB0\fR|\fB1\fR\fP -When this option is set in addition to \fBHSAuthoritativeDir\fP, Tor -periodically (every 15 minutes) writes statistics about hidden service -usage to a file \fBhsusage\fP in its data directory. (Default: 0) -.LP -.TP \fBHidServDirectoryV2 \fR\fB0\fR|\fB1\fR\fP When this option is set, Tor accepts and serves v2 hidden service descriptors. Setting DirPort is not required for this, because clients @@ -1169,6 +1288,11 @@ When this is set then \fBVersioningAuthoritativeDirectory\fP should be set too. .LP .TP +\fBConsensusParams \fR\fISTRING\fP +STRING is a space-separated list of key=value pairs that Tor will +include in the "params" line of its networkstatus vote. +.LP +.TP \fBDirAllowPrivateAddresses \fR\fB0\fR|\fB1\fR\fP If set to 1, Tor will accept router descriptors with arbitrary "Address" elements. Otherwise, if the address is not an IP address or is a private @@ -1295,7 +1419,7 @@ if you're using a Tor controller that handles hidserv publishing for you. .TP \fBHiddenServiceVersion \fR\fIversion\fR,\fIversion\fR,\fI...\fP A list of rendezvous service descriptor versions to publish for the hidden -service. Possible version numbers are 0 and 2. (Default: 0, 2) +service. Currently, only version 2 is supported. (Default: 2) .LP .TP \fBHiddenServiceAuthorizeClient \fR\fIauth-type\fR \fR\fIclient-name\fR,\fIclient-name\fR,\fI...\fP @@ -1440,7 +1564,7 @@ The most recently downloaded network status document for each authority. Each f .LP .TP .B \fIDataDirectory\fB/cached-descriptors\fR and \fBcached-descriptors.new\fR -These files hold downloaded router statuses. Some routers may appear more than once; if so, the most recently published descriptor is used. Lines beginning with @-signs are annotations that contain more information about a given router. The ".new" file is an append-only journal; when it gets too large, all entries are merged into a new cached-routers file. +These files hold downloaded router statuses. Some routers may appear more than once; if so, the most recently published descriptor is used. Lines beginning with @-signs are annotations that contain more information about a given router. The ".new" file is an append-only journal; when it gets too large, all entries are merged into a new cached-descriptors file. .LP .TP .B \fIDataDirectory\fB/cached-routers\fR and \fBcached-routers.new\fR diff --git a/doc/translations.txt b/doc/translations.txt index 874abe1bc1..06d16f4462 100644 --- a/doc/translations.txt +++ b/doc/translations.txt @@ -31,7 +31,7 @@ The current pootle configuration is checked into subversion as well: TorCheck uses our translation portal to accept translations. Users use the portal to check in their changes. To make use of the translations -that users have commited to the translations/ subversion module, you'll +that users have committed to the translations/ subversion module, you'll need to ensure that you have a current checked out copy of TorCheck: cd check/trunk/i18n/ @@ -75,59 +75,32 @@ And finally check in the changes: Torbutton uses our translation portal to accept translations. Users use the portal to check in their changes. -To make use of the translations that users have commited to the translations/ +To make use of the translations that users have committed to the translations/ subversion module, you'll need to ensure that you have a current checked out -copy of Torbutton: +copy of them in your torbutton git checkout: - cd torbutton/trans_tools - torbutton/trans_tools$ svn up + cd torbutton.git/trans_tools + torbutton.git/trans_tools$ svn co https://tor-svn.freehaven.net/svn/translation/trunk/projects/torbutton pootle You should see something like the following: - Fetching external item into 'pootle' - External at revision 15300. - - At revision 15300. - -Now if you had changes, you need to convert from .po and move -the newly updated mozilla files into the current stable locale -directory. First convert them with the 'mkmoz.sh' script and then -move the proper mozilla files from 'torbutton/trans_tools/moz/' into -'torbutton/src/chrome/locale/' directory while properly naming the files -for their respective locale. - -Here's an example of how to move all of the current pootle translations into -the svn trunk area of Torbutton: - - cd torbutton/trans_tools - ./mkmoz.sh - for locale in `ls -1 moz/`; - do - mv -v moz/$locale/*.{dtd,properties} ../src/chrome/locale/$locale/; - done - -Now check the differences (ensure the output looks reasonable): + Checked out revision 21092. - svn diff +If you made changes to strings in Torbutton, you need to rebuild the +templates in torbutton.git/trans_tools/pootle/templates. This is done with +the following command from within the torbutton.git checkout directory: -And finally check in the changes: - - svn commit - - -If you make changes to strings in Torbutton, you need to rebuild the -templates in torbutton/trans_tools/pootle/templates. This is done via: - - moz2po -P -i torbutton/src/chrome/locale/en/ -o torbutton/trans_tools/templates/ + moz2po -P -i src/chrome/locale/en/ -o trans_tools/pootle/templates/ You now have two options: -Option 1 (The Pootle Web UI Way): +Option 1 (The [shitty] Pootle Web UI Way): View then commit the changes to the template with: - svn diff torbutton/trans_tools/templates/ - svn commit torbutton/trans_tools/templates/ + cd trans_tools/pootle + svn diff templates + svn commit templates Then poke Jake to 'svn up' on the Pootle side. If you do this enough times, he may give you a button to click to update templates in Pootle, @@ -150,7 +123,7 @@ Option 2 (Use your own msgmerge: YMMV, may change .po flags and formatting): Run msgmerge yourself for each language: - cd torbutton/trans_tools + cd trans_tools for i in `ls -1 pootle` do msgmerge -U ./pootle/$i/torbutton.dtd.po ./pootle/templates/torbutton.dtd.pot @@ -171,6 +144,36 @@ breaks :) After this process is done, you then need to regenerate the mozilla .dtd and .properties files as specified above. + +Regardless of whether or not you had changes in the torbutton strings, if there +were updated strings in pootle that you checked out from svn you now need to +convert from .po and move the newly updated mozilla files into the current +stable locale directory. First convert them with the 'mkmoz.sh' script and +then move the proper mozilla files from 'torbutton.git/trans_tools/moz/' into +'torbutton.git/src/chrome/locale/' directory while properly naming the files +for their respective locale. + +Here's an example of how to move all of the current pootle translations into +the svn trunk area of Torbutton: + + cd trans_tools + ./mkmoz.sh + for locale in `ls -1 moz/`; + do + mv -v moz/$locale/*.{dtd,properties} ../src/chrome/locale/$locale/ + done + +Now check the differences to your git branch to ensure the output looks +reasonable: + + cd .. + git diff + +And finally check in the changes: + + cd src/chrome/locale + git commit . + ---------------------------- Vidalia ------------------------------- Vidalia uses our translation portal to accept translations. Users use the diff --git a/src/Makefile.am b/src/Makefile.am index ae647b2d61..fa2dd560a6 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1,5 +1,5 @@ # leave in dependency order, since common must be built first -SUBDIRS = common or tools win32 config -DIST_SUBDIRS = common or tools win32 config +SUBDIRS = common or test tools win32 config +DIST_SUBDIRS = common or test tools win32 config diff --git a/src/common/Makefile.am b/src/common/Makefile.am index 105c413343..eecfeb83fe 100644 --- a/src/common/Makefile.am +++ b/src/common/Makefile.am @@ -1,5 +1,7 @@ -noinst_LIBRARIES = libor.a libor-crypto.a +noinst_LIBRARIES = libor.a libor-crypto.a libor-event.a + +EXTRA_DIST = common_sha1.i sha256.c #CFLAGS = -Wall -Wpointer-arith -O2 @@ -10,7 +12,21 @@ libor_extra_source= endif libor_a_SOURCES = address.c log.c util.c compat.c container.c mempool.c \ - memarea.c $(libor_extra_source) + memarea.c util_codedigest.c $(libor_extra_source) libor_crypto_a_SOURCES = crypto.c aes.c tortls.c torgzip.c +libor_event_a_SOURCES = compat_libevent.c + +noinst_HEADERS = address.h log.h crypto.h util.h compat.h aes.h torint.h tortls.h strlcpy.c strlcat.c torgzip.h container.h ht.h mempool.h memarea.h ciphers.inc compat_libevent.h tortls_states.h + +common_sha1.i: $(libor_SOURCES) $(libor_crypto_a_SOURCES) $(noinst_HEADERS) + if test "@SHA1SUM@" != none; then \ + @SHA1SUM@ $(libor_SOURCES) $(libor_crypto_a_SOURCES) $(noinst_HEADERS) | @SED@ -n 's/^\(.*\)$$/"\1\\n"/p' > common_sha1.i; \ + elif test "@OPENSSL@" != none; then \ + @OPENSSL@ sha1 $(libor_SOURCES) $(libor_crypto_a_SOURCES) $(noinst_HEADERS) | @SED@ -n 's/SHA1(\(.*\))= \(.*\)/"\2 \1\\n"/p' > common_sha1.i; \ + else \ + rm common_sha1.i; \ + touch common_sha1.i; \ + fi -noinst_HEADERS = address.h log.h crypto.h test.h util.h compat.h aes.h torint.h tortls.h strlcpy.c strlcat.c torgzip.h container.h ht.h mempool.h memarea.h ciphers.inc +util_codedigest.o: common_sha1.i +crypto.o: sha256.c diff --git a/src/common/aes.c b/src/common/aes.c index e07665635b..451c31f02a 100644 --- a/src/common/aes.c +++ b/src/common/aes.c @@ -263,7 +263,8 @@ aes_set_key(aes_cnt_cipher_t *cipher, const char *key, int key_bits) void aes_free_cipher(aes_cnt_cipher_t *cipher) { - tor_assert(cipher); + if (!cipher) + return; #ifdef USE_OPENSSL_EVP EVP_CIPHER_CTX_cleanup(&cipher->key); #endif diff --git a/src/common/compat.c b/src/common/compat.c index d62b1ce1f4..d45fda0be3 100644 --- a/src/common/compat.c +++ b/src/common/compat.c @@ -398,6 +398,37 @@ const char TOR_TOLOWER_TABLE[256] = { 240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255, }; +/** Implementation of strtok_r for platforms whose coders haven't figured out + * how to write one. Hey guys! You can use this code here for free! */ +char * +tor_strtok_r_impl(char *str, const char *sep, char **lasts) +{ + char *cp, *start; + if (str) + start = cp = *lasts = str; + else if (!*lasts) + return NULL; + else + start = cp = *lasts; + + tor_assert(*sep); + if (sep[1]) { + while (*cp && !strchr(sep, *cp)) + ++cp; + } else { + tor_assert(strlen(sep) == 1); + cp = strchr(cp, *sep); + } + + if (!cp || !*cp) { + *lasts = NULL; + } else { + *cp++ = '\0'; + *lasts = cp; + } + return start; +} + #ifdef MS_WINDOWS /** Take a filename and return a pointer to its final element. This * function is called on __FILE__ to fix a MSVC nit where __FILE__ @@ -449,8 +480,8 @@ get_uint32(const char *cp) return v; } /** - * Read a 32-bit value beginning at <b>cp</b>. Equivalent to - * *(uint32_t*)(cp), but will not cause segfaults on platforms that forbid + * Read a 64-bit value beginning at <b>cp</b>. Equivalent to + * *(uint64_t*)(cp), but will not cause segfaults on platforms that forbid * unaligned memory access. */ uint64_t @@ -1577,7 +1608,6 @@ get_uname(void) #ifdef MS_WINDOWS OSVERSIONINFOEX info; int i; - unsigned int leftover_mask; const char *plat = NULL; const char *extra = NULL; static struct { @@ -1594,25 +1624,6 @@ get_uname(void) { 3, 51, "Windows NT 3.51" }, { 0, 0, NULL } }; -#ifdef VER_SUITE_BACKOFFICE - static struct { - unsigned int mask; const char *str; - } win_mask_table[] = { - { VER_SUITE_BACKOFFICE, " {backoffice}" }, - { VER_SUITE_BLADE, " {\"blade\" (2003, web edition)}" }, - { VER_SUITE_DATACENTER, " {datacenter}" }, - { VER_SUITE_ENTERPRISE, " {enterprise}" }, - { VER_SUITE_EMBEDDEDNT, " {embedded}" }, - { VER_SUITE_PERSONAL, " {personal}" }, - { VER_SUITE_SINGLEUSERTS, - " {terminal services, single user}" }, - { VER_SUITE_SMALLBUSINESS, " {small business}" }, - { VER_SUITE_SMALLBUSINESS_RESTRICTED, - " {small business, restricted}" }, - { VER_SUITE_TERMINAL, " {terminal services}" }, - { 0, NULL }, - }; -#endif memset(&info, 0, sizeof(info)); info.dwOSVersionInfoSize = sizeof(info); if (! GetVersionEx((LPOSVERSIONINFO)&info)) { @@ -1671,18 +1682,6 @@ get_uname(void) } else if (info.wProductType == VER_NT_WORKSTATION) { strlcat(uname_result, " [workstation]", sizeof(uname_result)); } - leftover_mask = info.wSuiteMask; - for (i = 0; win_mask_table[i].mask; ++i) { - if (info.wSuiteMask & win_mask_table[i].mask) { - strlcat(uname_result, win_mask_table[i].str, sizeof(uname_result)); - leftover_mask &= ~win_mask_table[i].mask; - } - } - if (leftover_mask) { - size_t len = strlen(uname_result); - tor_snprintf(uname_result+len, sizeof(uname_result)-len, - " {0x%x}", info.wSuiteMask); - } #endif #else strlcpy(uname_result, "Unknown platform", sizeof(uname_result)); @@ -2013,6 +2012,8 @@ tor_mutex_new(void) void tor_mutex_free(tor_mutex_t *m) { + if (!m) + return; tor_mutex_uninit(m); tor_free(m); } @@ -2040,7 +2041,8 @@ tor_cond_new(void) void tor_cond_free(tor_cond_t *cond) { - tor_assert(cond); + if (!cond) + return; if (pthread_cond_destroy(&cond->cond)) { log_warn(LD_GENERAL,"Error freeing condition: %s", strerror(errno)); return; @@ -2097,7 +2099,8 @@ tor_cond_new(void) void tor_cond_free(tor_cond_t *cond) { - tor_assert(cond); + if (!cond) + return; DeleteCriticalSection(&cond->mutex); /* XXXX notify? */ smartlist_free(cond->events); @@ -2173,6 +2176,102 @@ tor_threads_init(void) } #endif +#if defined(HAVE_MLOCKALL) && HAVE_DECL_MLOCKALL && defined(RLIMIT_MEMLOCK) +/** Attempt to raise the current and max rlimit to infinity for our process. + * This only needs to be done once and can probably only be done when we have + * not already dropped privileges. + */ +static int +tor_set_max_memlock(void) +{ + /* Future consideration for Windows is probably SetProcessWorkingSetSize + * This is similar to setting the memory rlimit of RLIMIT_MEMLOCK + * http://msdn.microsoft.com/en-us/library/ms686234(VS.85).aspx + */ + + struct rlimit limit; + int ret; + + /* Do we want to report current limits first? This is not really needed. */ + ret = getrlimit(RLIMIT_MEMLOCK, &limit); + if (ret == -1) { + log_warn(LD_GENERAL, "Could not get RLIMIT_MEMLOCK: %s", strerror(errno)); + return -1; + } + + /* RLIM_INFINITY is -1 on some platforms. */ + limit.rlim_cur = RLIM_INFINITY; + limit.rlim_max = RLIM_INFINITY; + + ret = setrlimit(RLIMIT_MEMLOCK, &limit); + if (ret == -1) { + if (errno == EPERM) { + log_warn(LD_GENERAL, "You appear to lack permissions to change memory " + "limits. Are you root?"); + log_warn(LD_GENERAL, "Unable to raise RLIMIT_MEMLOCK: %s", + strerror(errno)); + } else { + log_warn(LD_GENERAL, "Could not raise RLIMIT_MEMLOCK: %s", + strerror(errno)); + } + return -1; + } + + return 0; +} +#endif + +/** Attempt to lock all current and all future memory pages. + * This should only be called once and while we're privileged. + * Like mlockall() we return 0 when we're successful and -1 when we're not. + * Unlike mlockall() we return 1 if we've already attempted to lock memory. + */ +int +tor_mlockall(void) +{ + static int memory_lock_attempted = 0; + + if (memory_lock_attempted) { + return 1; + } + + memory_lock_attempted = 1; + + /* + * Future consideration for Windows may be VirtualLock + * VirtualLock appears to implement mlock() but not mlockall() + * + * http://msdn.microsoft.com/en-us/library/aa366895(VS.85).aspx + */ + +#if defined(HAVE_MLOCKALL) && HAVE_DECL_MLOCKALL && defined(RLIMIT_MEMLOCK) + if (tor_set_max_memlock() == 0) { + /* Perhaps we only want to log this if we're in a verbose mode? */ + log_notice(LD_GENERAL, "RLIMIT_MEMLOCK is now set to RLIM_INFINITY."); + } + + if (mlockall(MCL_CURRENT|MCL_FUTURE) == 0) { + log_notice(LD_GENERAL, "Insecure OS paging is effectively disabled."); + return 0; + } else { + if (errno == ENOSYS) { + /* Apple - it's 2009! I'm looking at you. Grrr. */ + log_notice(LD_GENERAL, "It appears that mlockall() is not available on " + "your platform."); + } else if (errno == EPERM) { + log_notice(LD_GENERAL, "It appears that you lack the permissions to " + "lock memory. Are you root?"); + } + log_notice(LD_GENERAL, "Unable to lock all current and future memory " + "pages: %s", strerror(errno)); + return -1; + } +#else + log_warn(LD_GENERAL, "Unable to lock memory pages. mlockall() unsupported?"); + return -1; +#endif +} + /** Identity of the "main" thread */ static unsigned long main_thread_id = -1; diff --git a/src/common/compat.h b/src/common/compat.h index 4d5a016cf2..554ae8919f 100644 --- a/src/common/compat.h +++ b/src/common/compat.h @@ -267,6 +267,13 @@ extern const char TOR_TOLOWER_TABLE[]; #define TOR_TOLOWER(c) (TOR_TOLOWER_TABLE[(uint8_t)c]) #define TOR_TOUPPER(c) (TOR_TOUPPER_TABLE[(uint8_t)c]) +char *tor_strtok_r_impl(char *str, const char *sep, char **lasts); +#ifdef HAVE_STRTOK_R +#define tor_strtok_r(str, sep, lasts) strtok_r(str, sep, lasts) +#else +#define tor_strtok_r(str, sep, lasts) tor_strtok_r_impl(str, sep, lasts) +#endif + #ifdef MS_WINDOWS #define _SHORT_FILE_ (tor_fix_source_file(__FILE__)) const char *tor_fix_source_file(const char *fname); @@ -502,6 +509,8 @@ typedef struct tor_mutex_t { #endif } tor_mutex_t; +int tor_mlockall(void); + #ifdef TOR_IS_MULTITHREADED tor_mutex_t *tor_mutex_new(void); void tor_mutex_init(tor_mutex_t *m); diff --git a/src/common/compat_libevent.c b/src/common/compat_libevent.c new file mode 100644 index 0000000000..dcf51cbbd3 --- /dev/null +++ b/src/common/compat_libevent.c @@ -0,0 +1,471 @@ +/* Copyright (c) 2009, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compat_libevent.c + * \brief Wrappers to handle porting between different versions of libevent. + * + * In an ideal world, we'd just use Libevent 2.0 from now on. But as of June + * 2009, Libevent 2.0 is still in alpha, and we will have old versions of + * Libevent for the forseeable future. + **/ + +#include "orconfig.h" +#include "compat_libevent.h" + +#include "compat.h" +#include "util.h" +#include "log.h" + +#ifdef HAVE_EVENT2_EVENT_H +#include <event2/event.h> +#else +#include <event.h> +#endif + +/** A number representing a version of Libevent. + + This is a 4-byte number, with the first three bytes representing the + major, minor, and patchlevel respectively of the library. The fourth + byte is unused. + + This is equivalent to the format of LIBEVENT_VERSION_NUMBER on Libevent + 2.0.1 or later. For versions of Libevent before 1.4.0, which followed the + format of "1.0, 1.0a, 1.0b", we define 1.0 to be equivalent to 1.0.0, 1.0a + to be equivalent to 1.0.1, and so on. +*/ +typedef uint32_t le_version_t; + +/* Macros: returns the number of a libevent version. */ +#define V(major, minor, patch) \ + (((major) << 24) | ((minor) << 16) | ((patch) << 8)) +#define V_OLD(major, minor, patch) \ + V((major), (minor), (patch)-'a'+1) + +#define LE_OLD V(0,0,0) +#define LE_OTHER V(0,0,99) + +static le_version_t tor_get_libevent_version(const char **v_out); + +#ifdef HAVE_EVENT_SET_LOG_CALLBACK +/** A string which, if it appears in a libevent log, should be ignored. */ +static const char *suppress_msg = NULL; +/** Callback function passed to event_set_log() so we can intercept + * log messages from libevent. */ +static void +libevent_logging_callback(int severity, const char *msg) +{ + char buf[1024]; + size_t n; + if (suppress_msg && strstr(msg, suppress_msg)) + return; + n = strlcpy(buf, msg, sizeof(buf)); + if (n && n < sizeof(buf) && buf[n-1] == '\n') { + buf[n-1] = '\0'; + } + switch (severity) { + case _EVENT_LOG_DEBUG: + log(LOG_DEBUG, LD_NET, "Message from libevent: %s", buf); + break; + case _EVENT_LOG_MSG: + log(LOG_INFO, LD_NET, "Message from libevent: %s", buf); + break; + case _EVENT_LOG_WARN: + log(LOG_WARN, LD_GENERAL, "Warning from libevent: %s", buf); + break; + case _EVENT_LOG_ERR: + log(LOG_ERR, LD_GENERAL, "Error from libevent: %s", buf); + break; + default: + log(LOG_WARN, LD_GENERAL, "Message [%d] from libevent: %s", + severity, buf); + break; + } +} +/** Set hook to intercept log messages from libevent. */ +void +configure_libevent_logging(void) +{ + event_set_log_callback(libevent_logging_callback); +} +/** Ignore any libevent log message that contains <b>msg</b>. */ +void +suppress_libevent_log_msg(const char *msg) +{ + suppress_msg = msg; +} +#else +void +configure_libevent_logging(void) +{ +} +void +suppress_libevent_log_msg(const char *msg) +{ + (void)msg; +} +#endif + +#ifndef HAVE_EVENT2_EVENT_H +/** Work-alike replacement for event_new() on pre-Libevent-2.0 systems. */ +struct event * +tor_event_new(struct event_base *base, int sock, short what, + void (*cb)(int, short, void *), void *arg) +{ + struct event *e = tor_malloc_zero(sizeof(struct event)); + event_set(e, sock, what, cb, arg); + if (! base) + base = tor_libevent_get_base(); + event_base_set(base, e); + return e; +} +/** Work-alike replacement for evtimer_new() on pre-Libevent-2.0 systems. */ +struct event * +tor_evtimer_new(struct event_base *base, + void (*cb)(int, short, void *), void *arg) +{ + return tor_event_new(base, -1, 0, cb, arg); +} +/** Work-alike replacement for evsignal_new() on pre-Libevent-2.0 systems. */ +struct event * +tor_evsignal_new(struct event_base * base, int sig, + void (*cb)(int, short, void *), void *arg) +{ + return tor_event_new(base, sig, EV_SIGNAL|EV_PERSIST, cb, arg); +} +/** Work-alike replacement for event_free() on pre-Libevent-2.0 systems. */ +void +tor_event_free(struct event *ev) +{ + event_del(ev); + tor_free(ev); +} +#endif + +/** Global event base for use by the main thread. */ +struct event_base *the_event_base = NULL; + +/* This is what passes for version detection on OSX. We set + * MACOSX_KQUEUE_IS_BROKEN to true iff we're on a version of OSX before + * 10.4.0 (aka 1040). */ +#ifdef __APPLE__ +#ifdef __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ +#define MACOSX_KQUEUE_IS_BROKEN \ + (__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 1040) +#else +#define MACOSX_KQUEUE_IS_BROKEN 0 +#endif +#endif + +/** Initialize the Libevent library and set up the event base. */ +void +tor_libevent_initialize(void) +{ + tor_assert(the_event_base == NULL); + +#ifdef __APPLE__ + if (MACOSX_KQUEUE_IS_BROKEN || + tor_get_libevent_version(NULL) < V_OLD(1,1,'b')) { + setenv("EVENT_NOKQUEUE","1",1); + } +#endif + +#ifdef HAVE_EVENT2_EVENT_H + the_event_base = event_base_new(); +#else + the_event_base = event_init(); +#endif + +#if defined(HAVE_EVENT_GET_VERSION) && defined(HAVE_EVENT_GET_METHOD) + /* Making this a NOTICE for now so we can link bugs to a libevent versions + * or methods better. */ + log(LOG_NOTICE, LD_GENERAL, + "Initialized libevent version %s using method %s. Good.", + event_get_version(), tor_libevent_get_method()); +#else + log(LOG_NOTICE, LD_GENERAL, + "Initialized old libevent (version 1.0b or earlier)."); + log(LOG_WARN, LD_GENERAL, + "You have a *VERY* old version of libevent. It is likely to be buggy; " + "please build Tor with a more recent version."); +#endif +} + +/** Return the current Libevent event base that we're set up to use. */ +struct event_base * +tor_libevent_get_base(void) +{ + return the_event_base; +} + +#ifndef HAVE_EVENT_BASE_LOOPEXIT +/* Replacement for event_base_loopexit on some very old versions of Libevent + that we are not yet brave enough to deprecate. */ +int +tor_event_base_loopexit(struct event_base *base, struct timeval *tv) +{ + tor_assert(base == the_event_base); + return event_loopexit(tv); +} +#endif + +/** Return the name of the Libevent backend we're using. */ +const char * +tor_libevent_get_method(void) +{ +#ifdef HAVE_EVENT2_EVENT_H + return event_base_get_method(the_event_base); +#elif defined(HAVE_EVENT_GET_METHOD) + return event_get_method(); +#else + return "<unknown>"; +#endif +} + +/** Return the le_version_t for the current version of libevent. If the + * version is very new, return LE_OTHER. If the version is so old that it + * doesn't support event_get_version(), return LE_OLD. DOCDOC */ +static le_version_t +tor_decode_libevent_version(const char *v) +{ + unsigned major, minor, patchlevel; + char c, extra; + int fields; + + /* Try the new preferred "1.4.11-stable" format. */ + fields = sscanf(v, "%u.%u.%u%c", &major, &minor, &patchlevel, &c); + if (fields == 3 || + (fields == 4 && (c == '-' || c == '_'))) { + return V(major,minor,patchlevel); + } + + /* Try the old "1.3e" format. */ + fields = sscanf(v, "%u.%u%c%c", &major, &minor, &c, &extra); + if (fields == 3 && TOR_ISALPHA(c)) { + return V_OLD(major, minor, c); + } else if (fields == 2) { + return V(major, minor, 0); + } + + return LE_OTHER; +} + +/** Return an integer representing the binary interface of a Libevent library. + * Two different versions with different numbers are sure not to be binary + * compatible. Two different versions with the same numbers have a decent + * chance of binary compatibility.*/ +static int +le_versions_compatibility(le_version_t v) +{ + if (v == LE_OTHER) + return 0; + if (v < V_OLD(1,0,'c')) + return 1; + else if (v < V(1,4,0)) + return 2; + else if (v < V(1,4,99)) + return 3; + else if (v < V(2,0,1)) + return 4; + else /* Everything 2.0 and later should be compatible. */ + return 5; +} + +/** Return the version number of the currently running version of Libevent. + See le_version_t for info on the format. + */ +static le_version_t +tor_get_libevent_version(const char **v_out) +{ + const char *v; + le_version_t r; +#if defined(HAVE_EVENT_GET_VERSION_NUMBER) + v = event_get_version(); + r = event_get_version_number(); +#elif defined (HAVE_EVENT_GET_VERSION) + v = event_get_version(); + r = tor_decode_libevent_version(v); +#else + v = "pre-1.0c"; + r = LE_OLD; +#endif + if (v_out) + *v_out = v; + return r; +} + +/** Return a string representation of the version of the currently running + * version of Libevent. */ +const char * +tor_libevent_get_version_str(void) +{ +#ifdef HAVE_EVENT_GET_VERSION + return event_get_version(); +#else + return "pre-1.0c"; +#endif +} + +/** + * Compare the current Libevent method and version to a list of versions + * which are known not to work. Warn the user as appropriate. + */ +void +tor_check_libevent_version(const char *m, int server, + const char **badness_out) +{ + int buggy = 0, iffy = 0, slow = 0, thread_unsafe = 0; + le_version_t version; + const char *v = NULL; + const char *badness = NULL; + const char *sad_os = ""; + + version = tor_get_libevent_version(&v); + + /* XXX Would it be worthwhile disabling the methods that we know + * are buggy, rather than just warning about them and then proceeding + * to use them? If so, we should probably not wrap this whole thing + * in HAVE_EVENT_GET_VERSION and HAVE_EVENT_GET_METHOD. -RD */ + /* XXXX The problem is that it's not trivial to get libevent to change it's + * method once it's initialized, and it's not trivial to tell what method it + * will use without initializing it. I guess we could preemptively disable + * buggy libevent modes based on the version _before_ initializing it, + * though, but then there's no good way (afaict) to warn "I would have used + * kqueue, but instead I'm using select." -NM */ + /* XXXX022 revist the above; it is fixable now. */ + if (!strcmp(m, "kqueue")) { + if (version < V_OLD(1,1,'b')) + buggy = 1; + } else if (!strcmp(m, "epoll")) { + if (version < V(1,1,0)) + iffy = 1; + } else if (!strcmp(m, "poll")) { + if (version < V_OLD(1,0,'e')) + buggy = 1; + if (version < V(1,1,0)) + slow = 1; + } else if (!strcmp(m, "select")) { + if (version < V(1,1,0)) + slow = 1; + } else if (!strcmp(m, "win32")) { + if (version < V_OLD(1,1,'b')) + buggy = 1; + } + + /* Libevent versions before 1.3b do very badly on operating systems with + * user-space threading implementations. */ +#if defined(__OpenBSD__) || defined(__FreeBSD__) || defined(__NetBSD__) + if (server && version < V_OLD(1,3,'b')) { + thread_unsafe = 1; + sad_os = "BSD variants"; + } +#elif defined(__APPLE__) || defined(__darwin__) + if (server && version < V_OLD(1,3,'b')) { + thread_unsafe = 1; + sad_os = "Mac OS X"; + } +#endif + + if (thread_unsafe) { + log(LOG_WARN, LD_GENERAL, + "Libevent version %s often crashes when running a Tor server with %s. " + "Please use the latest version of libevent (1.3b or later)",v,sad_os); + badness = "BROKEN"; + } else if (buggy) { + log(LOG_WARN, LD_GENERAL, + "There are serious bugs in using %s with libevent %s. " + "Please use the latest version of libevent.", m, v); + badness = "BROKEN"; + } else if (iffy) { + log(LOG_WARN, LD_GENERAL, + "There are minor bugs in using %s with libevent %s. " + "You may want to use the latest version of libevent.", m, v); + badness = "BUGGY"; + } else if (slow && server) { + log(LOG_WARN, LD_GENERAL, + "libevent %s can be very slow with %s. " + "When running a server, please use the latest version of libevent.", + v,m); + badness = "SLOW"; + } + + *badness_out = badness; +} + +#if defined(LIBEVENT_VERSION) +#define HEADER_VERSION LIBEVENT_VERSION +#elif defined(_EVENT_VERSION) +#define HEADER_VERSION _EVENT_VERSION +#endif + +/** See whether the headers we were built against differ from the library we + * linked against so much that we're likely to crash. If so, warn the + * user. */ +void +tor_check_libevent_header_compatibility(void) +{ + (void) le_versions_compatibility; + (void) tor_decode_libevent_version; + + /* In libevent versions before 2.0, it's hard to keep binary compatibility + * between upgrades, and unpleasant to detect when the version we compiled + * against is unlike the version we have linked against. Here's how. */ +#if defined(HEADER_VERSION) && defined(HAVE_EVENT_GET_VERSION) + /* We have a header-file version and a function-call version. Easy. */ + if (strcmp(HEADER_VERSION, event_get_version())) { + le_version_t v1, v2; + int compat1 = -1, compat2 = -1; + int verybad; + v1 = tor_decode_libevent_version(HEADER_VERSION); + v2 = tor_decode_libevent_version(event_get_version()); + compat1 = le_versions_compatibility(v1); + compat2 = le_versions_compatibility(v2); + + verybad = compat1 != compat2; + + log(verybad ? LOG_WARN : LOG_NOTICE, + LD_GENERAL, "We were compiled with headers from version %s " + "of Libevent, but we're using a Libevent library that says it's " + "version %s.", HEADER_VERSION, event_get_version()); + if (verybad) + log_warn(LD_GENERAL, "This will almost certainly make Tor crash."); + else + log_info(LD_GENERAL, "I think these versions are binary-compatible."); + } +#elif defined(HAVE_EVENT_GET_VERSION) + /* event_get_version but no _EVENT_VERSION. We might be in 1.4.0-beta or + earlier, where that's normal. To see whether we were compiled with an + earlier version, let's see whether the struct event defines MIN_HEAP_IDX. + */ +#ifdef HAVE_STRUCT_EVENT_MIN_HEAP_IDX + /* The header files are 1.4.0-beta or later. If the version is not + * 1.4.0-beta, we are incompatible. */ + { + if (strcmp(event_get_version(), "1.4.0-beta")) { + log_warn(LD_GENERAL, "It's a little hard to tell, but you seem to have " + "Libevent 1.4.0-beta header files, whereas you have linked " + "against Libevent %s. This will probably make Tor crash.", + event_get_version()); + } + } +#else + /* Our headers are 1.3e or earlier. If the library version is not 1.4.x or + later, we're probably fine. */ + { + const char *v = event_get_version(); + if ((v[0] == '1' && v[2] == '.' && v[3] > '3') || v[0] > '1') { + log_warn(LD_GENERAL, "It's a little hard to tell, but you seem to have " + "Libevent header file from 1.3e or earlier, whereas you have " + "linked against Libevent %s. This will probably make Tor " + "crash.", event_get_version()); + } + } +#endif + +#elif defined(HEADER_VERSION) +#warn "_EVENT_VERSION is defined but not get_event_version(): Libevent is odd." +#else + /* Your libevent is ancient. */ +#endif +} + diff --git a/src/common/compat_libevent.h b/src/common/compat_libevent.h new file mode 100644 index 0000000000..d2e76ce4e2 --- /dev/null +++ b/src/common/compat_libevent.h @@ -0,0 +1,54 @@ +/* Copyright (c) 2009, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef _TOR_COMPAT_LIBEVENT_H +#define _TOR_COMPAT_LIBEVENT_H + +#include "orconfig.h" + +struct event; +struct event_base; + +#ifdef HAVE_EVENT2_EVENT_H +#include <event2/util.h> +#else +#define evutil_socket_t int +#endif + +void configure_libevent_logging(void); +void suppress_libevent_log_msg(const char *msg); + +#ifdef HAVE_EVENT2_EVENT_H +#define tor_event_new event_new +#define tor_evtimer_new evtimer_new +#define tor_evsignal_new evsignal_new +#define tor_event_free event_free +#else +struct event *tor_event_new(struct event_base * base, evutil_socket_t sock, + short what, void (*cb)(evutil_socket_t, short, void *), void *arg); +struct event *tor_evtimer_new(struct event_base * base, + void (*cb)(evutil_socket_t, short, void *), void *arg); +struct event *tor_evsignal_new(struct event_base * base, int sig, + void (*cb)(evutil_socket_t, short, void *), void *arg); +void tor_event_free(struct event *ev); +#endif + +/* XXXX022 If we can drop support for Libevent before 1.1, we can + * do without this wrapper. */ +#ifdef HAVE_EVENT_BASE_LOOPEXIT +#define tor_event_base_loopexit event_base_loopexit +#else +struct timeval; +int tor_event_base_loopexit(struct event_base *base, struct timeval *tv); +#endif + +void tor_libevent_initialize(void); +struct event_base *tor_libevent_get_base(void); +const char *tor_libevent_get_method(void); +void tor_check_libevent_version(const char *m, int server, + const char **badness_out); +void tor_check_libevent_header_compatibility(void); +const char *tor_libevent_get_version_str(void); + +#endif + diff --git a/src/common/container.c b/src/common/container.c index c649787c0f..f452a51e42 100644 --- a/src/common/container.c +++ b/src/common/container.c @@ -44,7 +44,8 @@ smartlist_create(void) void smartlist_free(smartlist_t *sl) { - tor_assert(sl != NULL); + if (!sl) + return; tor_free(sl->list); tor_free(sl); } @@ -459,6 +460,42 @@ smartlist_sort(smartlist_t *sl, int (*compare)(const void **a, const void **b)) (int (*)(const void *,const void*))compare); } +/** Given a smartlist <b>sl</b> sorted with the function <b>compare</b>, + * return the most frequent member in the list. Break ties in favor of + * later elements. If the list is empty, return NULL. + */ +void * +smartlist_get_most_frequent(const smartlist_t *sl, + int (*compare)(const void **a, const void **b)) +{ + const void *most_frequent = NULL; + int most_frequent_count = 0; + + const void *cur = NULL; + int i, count=0; + + if (!sl->num_used) + return NULL; + for (i = 0; i < sl->num_used; ++i) { + const void *item = sl->list[i]; + if (cur && 0 == compare(&cur, &item)) { + ++count; + } else { + if (cur && count >= most_frequent_count) { + most_frequent = cur; + most_frequent_count = count; + } + cur = item; + count = 1; + } + } + if (cur && count >= most_frequent_count) { + most_frequent = cur; + most_frequent_count = count; + } + return (void*)most_frequent; +} + /** Given a sorted smartlist <b>sl</b> and the comparison function used to * sort it, remove all duplicate members. If free_fn is provided, calls * free_fn on each duplicate. Otherwise, just removes them. Preserves order. @@ -550,6 +587,13 @@ smartlist_sort_strings(smartlist_t *sl) smartlist_sort(sl, _compare_string_ptrs); } +/** Return the most frequent string in the sorted list <b>sl</b> */ +char * +smartlist_get_most_frequent_string(smartlist_t *sl) +{ + return smartlist_get_most_frequent(sl, _compare_string_ptrs); +} + /** Remove duplicate strings from a sorted list, and free them with tor_free(). */ void @@ -561,6 +605,38 @@ smartlist_uniq_strings(smartlist_t *sl) /* Heap-based priority queue implementation for O(lg N) insert and remove. * Recall that the heap property is that, for every index I, h[I] < * H[LEFT_CHILD[I]] and h[I] < H[RIGHT_CHILD[I]]. + * + * For us to remove items other than the topmost item, each item must store + * its own index within the heap. When calling the pqueue functions, tell + * them about the offset of the field that stores the index within the item. + * + * Example: + * + * typedef struct timer_t { + * struct timeval tv; + * int heap_index; + * } timer_t; + * + * static int compare(const void *p1, const void *p2) { + * const timer_t *t1 = p1, *t2 = p2; + * if (t1->tv.tv_sec < t2->tv.tv_sec) { + * return -1; + * } else if (t1->tv.tv_sec > t2->tv.tv_sec) { + * return 1; + * } else { + * return t1->tv.tv_usec - t2->tv_usec; + * } + * } + * + * void timer_heap_insert(smartlist_t *heap, timer_t *timer) { + * smartlist_pqueue_add(heap, compare, STRUCT_OFFSET(timer_t, heap_index), + * timer); + * } + * + * void timer_heap_pop(smartlist_t *heap) { + * return smartlist_pqueue_pop(heap, compare, + * STRUCT_OFFSET(timer_t, heap_index)); + * } */ /* For a 1-indexed array, we would use LEFT_CHILD[x] = 2*x and RIGHT_CHILD[x] @@ -572,12 +648,22 @@ smartlist_uniq_strings(smartlist_t *sl) #define RIGHT_CHILD(i) ( 2*(i) + 2 ) #define PARENT(i) ( ((i)-1) / 2 ) +#define IDXP(p) ((int*)STRUCT_VAR_P(p, idx_field_offset)) + +#define UPDATE_IDX(i) do { \ + void *updated = sl->list[i]; \ + *IDXP(updated) = i; \ + } while (0) + +#define IDX_OF_ITEM(p) (*IDXP(p)) + /** Helper. <b>sl</b> may have at most one violation of the heap property: * the item at <b>idx</b> may be greater than one or both of its children. * Restore the heap property. */ static INLINE void smartlist_heapify(smartlist_t *sl, int (*compare)(const void *a, const void *b), + int idx_field_offset, int idx) { while (1) { @@ -600,21 +686,28 @@ smartlist_heapify(smartlist_t *sl, void *tmp = sl->list[idx]; sl->list[idx] = sl->list[best_idx]; sl->list[best_idx] = tmp; + UPDATE_IDX(idx); + UPDATE_IDX(best_idx); idx = best_idx; } } } -/** Insert <b>item</b> into the heap stored in <b>sl</b>, where order - * is determined by <b>compare</b>. */ +/** Insert <b>item</b> into the heap stored in <b>sl</b>, where order is + * determined by <b>compare</b> and the offset of the item in the heap is + * stored in an int-typed field at position <b>idx_field_offset</b> within + * item. + */ void smartlist_pqueue_add(smartlist_t *sl, int (*compare)(const void *a, const void *b), + int idx_field_offset, void *item) { int idx; smartlist_add(sl,item); + UPDATE_IDX(sl->num_used-1); for (idx = sl->num_used - 1; idx; ) { int parent = PARENT(idx); @@ -622,6 +715,8 @@ smartlist_pqueue_add(smartlist_t *sl, void *tmp = sl->list[parent]; sl->list[parent] = sl->list[idx]; sl->list[idx] = tmp; + UPDATE_IDX(parent); + UPDATE_IDX(idx); idx = parent; } else { return; @@ -630,32 +725,63 @@ smartlist_pqueue_add(smartlist_t *sl, } /** Remove and return the top-priority item from the heap stored in <b>sl</b>, - * where order is determined by <b>compare</b>. <b>sl</b> must not be - * empty. */ + * where order is determined by <b>compare</b> and the item's position is + * stored at position <b>idx_field_offset</b> within the item. <b>sl</b> must + * not be empty. */ void * smartlist_pqueue_pop(smartlist_t *sl, - int (*compare)(const void *a, const void *b)) + int (*compare)(const void *a, const void *b), + int idx_field_offset) { void *top; tor_assert(sl->num_used); top = sl->list[0]; + *IDXP(top)=-1; if (--sl->num_used) { sl->list[0] = sl->list[sl->num_used]; - smartlist_heapify(sl, compare, 0); + UPDATE_IDX(0); + smartlist_heapify(sl, compare, idx_field_offset, 0); } return top; } +/** Remove the item <b>item</b> from the heap stored in <b>sl</b>, + * where order is determined by <b>compare</b> and the item's position is + * stored at position <b>idx_field_offset</b> within the item. <b>sl</b> must + * not be empty. */ +void +smartlist_pqueue_remove(smartlist_t *sl, + int (*compare)(const void *a, const void *b), + int idx_field_offset, + void *item) +{ + int idx = IDX_OF_ITEM(item); + tor_assert(idx >= 0); + tor_assert(sl->list[idx] == item); + --sl->num_used; + *IDXP(item) = -1; + if (idx == sl->num_used) { + return; + } else { + sl->list[idx] = sl->list[sl->num_used]; + UPDATE_IDX(idx); + smartlist_heapify(sl, compare, idx_field_offset, idx); + } +} + /** Assert that the heap property is correctly maintained by the heap stored * in <b>sl</b>, where order is determined by <b>compare</b>. */ void smartlist_pqueue_assert_ok(smartlist_t *sl, - int (*compare)(const void *a, const void *b)) + int (*compare)(const void *a, const void *b), + int idx_field_offset) { int i; - for (i = sl->num_used - 1; i > 0; --i) { - tor_assert(compare(sl->list[PARENT(i)], sl->list[i]) <= 0); + for (i = sl->num_used - 1; i >= 0; --i) { + if (i>0) + tor_assert(compare(sl->list[PARENT(i)], sl->list[i]) <= 0); + tor_assert(IDX_OF_ITEM(sl->list[i]) == i); } } @@ -681,6 +807,37 @@ smartlist_uniq_digests(smartlist_t *sl) smartlist_uniq(sl, _compare_digests, _tor_free); } +/** Helper: compare two DIGEST256_LEN digests. */ +static int +_compare_digests256(const void **_a, const void **_b) +{ + return memcmp((const char*)*_a, (const char*)*_b, DIGEST256_LEN); +} + +/** Sort the list of DIGEST256_LEN-byte digests into ascending order. */ +void +smartlist_sort_digests256(smartlist_t *sl) +{ + smartlist_sort(sl, _compare_digests256); +} + +/** Return the most frequent member of the sorted list of DIGEST256_LEN + * digests in <b>sl</b> */ +char * +smartlist_get_most_frequent_digest256(smartlist_t *sl) +{ + return smartlist_get_most_frequent(sl, _compare_digests256); +} + +/** Remove duplicate 256-bit digests from a sorted list, and free them with + * tor_free(). + */ +void +smartlist_uniq_digests256(smartlist_t *sl) +{ + smartlist_uniq(sl, _compare_digests256, _tor_free); +} + /** Helper: Declare an entry type and a map type to implement a mapping using * ht.h. The map type will be called <b>maptype</b>. The key part of each * entry is declared using the C declaration <b>keydecl</b>. All functions @@ -1113,6 +1270,9 @@ void strmap_free(strmap_t *map, void (*free_val)(void*)) { strmap_entry_t **ent, **next, *this; + if (!map) + return; + for (ent = HT_START(strmap_impl, &map->head); ent != NULL; ent = next) { this = *ent; next = HT_NEXT_RMV(strmap_impl, &map->head, ent); @@ -1134,6 +1294,8 @@ void digestmap_free(digestmap_t *map, void (*free_val)(void*)) { digestmap_entry_t **ent, **next, *this; + if (!map) + return; for (ent = HT_START(digestmap_impl, &map->head); ent != NULL; ent = next) { this = *ent; next = HT_NEXT_RMV(digestmap_impl, &map->head, ent); @@ -1220,6 +1382,7 @@ IMPLEMENT_ORDER_FUNC(find_nth_int, int) IMPLEMENT_ORDER_FUNC(find_nth_time, time_t) IMPLEMENT_ORDER_FUNC(find_nth_double, double) IMPLEMENT_ORDER_FUNC(find_nth_uint32, uint32_t) +IMPLEMENT_ORDER_FUNC(find_nth_int32, int32_t) IMPLEMENT_ORDER_FUNC(find_nth_long, long) /** Return a newly allocated digestset_t, optimized to hold a total of @@ -1248,6 +1411,8 @@ digestset_new(int max_elements) void digestset_free(digestset_t *set) { + if (!set) + return; bitarray_free(set->ba); tor_free(set); } diff --git a/src/common/container.h b/src/common/container.h index e626552467..8077d56ebd 100644 --- a/src/common/container.h +++ b/src/common/container.h @@ -93,13 +93,22 @@ void smartlist_del_keeporder(smartlist_t *sl, int idx); void smartlist_insert(smartlist_t *sl, int idx, void *val); void smartlist_sort(smartlist_t *sl, int (*compare)(const void **a, const void **b)); +void *smartlist_get_most_frequent(const smartlist_t *sl, + int (*compare)(const void **a, const void **b)); void smartlist_uniq(smartlist_t *sl, int (*compare)(const void **a, const void **b), void (*free_fn)(void *elt)); + void smartlist_sort_strings(smartlist_t *sl); void smartlist_sort_digests(smartlist_t *sl); +void smartlist_sort_digests256(smartlist_t *sl); + +char *smartlist_get_most_frequent_string(smartlist_t *sl); +char *smartlist_get_most_frequent_digest256(smartlist_t *sl); + void smartlist_uniq_strings(smartlist_t *sl); void smartlist_uniq_digests(smartlist_t *sl); +void smartlist_uniq_digests256(smartlist_t *sl); void *smartlist_bsearch(smartlist_t *sl, const void *key, int (*compare)(const void *key, const void **member)) ATTR_PURE; @@ -109,11 +118,18 @@ int smartlist_bsearch_idx(const smartlist_t *sl, const void *key, void smartlist_pqueue_add(smartlist_t *sl, int (*compare)(const void *a, const void *b), + int idx_field_offset, void *item); void *smartlist_pqueue_pop(smartlist_t *sl, - int (*compare)(const void *a, const void *b)); + int (*compare)(const void *a, const void *b), + int idx_field_offset); +void smartlist_pqueue_remove(smartlist_t *sl, + int (*compare)(const void *a, const void *b), + int idx_field_offset, + void *item); void smartlist_pqueue_assert_ok(smartlist_t *sl, - int (*compare)(const void *a, const void *b)); + int (*compare)(const void *a, const void *b), + int idx_field_offset); #define SPLIT_SKIP_SPACE 0x01 #define SPLIT_IGNORE_BLANK 0x02 @@ -627,6 +643,7 @@ void digestset_free(digestset_t* set); int find_nth_int(int *array, int n_elements, int nth); time_t find_nth_time(time_t *array, int n_elements, int nth); double find_nth_double(double *array, int n_elements, int nth); +int32_t find_nth_int32(int32_t *array, int n_elements, int nth); uint32_t find_nth_uint32(uint32_t *array, int n_elements, int nth); long find_nth_long(long *array, int n_elements, int nth); static INLINE int @@ -649,6 +666,11 @@ median_uint32(uint32_t *array, int n_elements) { return find_nth_uint32(array, n_elements, (n_elements-1)/2); } +static INLINE int32_t +median_int32(int32_t *array, int n_elements) +{ + return find_nth_int32(array, n_elements, (n_elements-1)/2); +} static INLINE long median_long(long *array, int n_elements) { diff --git a/src/common/crypto.c b/src/common/crypto.c index 8676b6bc7b..e7b0ff194f 100644 --- a/src/common/crypto.c +++ b/src/common/crypto.c @@ -27,6 +27,7 @@ #include <openssl/rsa.h> #include <openssl/pem.h> #include <openssl/evp.h> +#include <openssl/engine.h> #include <openssl/rand.h> #include <openssl/opensslv.h> #include <openssl/bn.h> @@ -49,9 +50,9 @@ #define CRYPTO_PRIVATE #include "crypto.h" -#include "log.h" +#include "../common/log.h" #include "aes.h" -#include "util.h" +#include "../common/util.h" #include "container.h" #include "compat.h" @@ -61,6 +62,33 @@ #include <openssl/engine.h> +#ifdef ANDROID +/* Android's OpenSSL seems to have removed all of its Engine support. */ +#define DISABLE_ENGINES +#endif + +#if OPENSSL_VERSION_NUMBER < 0x00908000l +/* On OpenSSL versions before 0.9.8, there is no working SHA256 + * implementation, so we use Tom St Denis's nice speedy one, slightly adapted + * to our needs */ +#define SHA256_CTX sha256_state +#define SHA256_Init sha256_init +#define SHA256_Update sha256_process +#define LTC_ARGCHK(x) tor_assert(x) +#include "sha256.c" +#define SHA256_Final(a,b) sha256_done(b,a) + +static unsigned char * +SHA256(const unsigned char *m, size_t len, unsigned char *d) +{ + SHA256_CTX ctx; + SHA256_Init(&ctx); + SHA256_Update(&ctx, m, len); + SHA256_Final(d, &ctx); + return d; +} +#endif + /** Macro: is k a valid RSA public or private key? */ #define PUBLIC_KEY_OK(k) ((k) && (k)->key && (k)->key->n) /** Macro: is k a valid RSA private key? */ @@ -94,7 +122,7 @@ struct crypto_dh_env_t { }; static int setup_openssl_threading(void); -static int tor_check_dh_key(BIGNUM *bn); +static int tor_check_dh_key(int severity, BIGNUM *bn); /** Return the number of bytes added by padding method <b>padding</b>. */ @@ -151,6 +179,7 @@ crypto_log_errors(int severity, const char *doing) } } +#ifndef DISABLE_ENGINES /** Log any OpenSSL engines we're using at NOTICE. */ static void log_engine(const char *fn, ENGINE *e) @@ -165,37 +194,82 @@ log_engine(const char *fn, ENGINE *e) log(LOG_INFO, LD_CRYPTO, "Using default implementation for %s", fn); } } +#endif + +#ifndef DISABLE_ENGINES +/** Try to load an engine in a shared library via fully qualified path. + */ +static ENGINE * +try_load_engine(const char *path, const char *engine) +{ + ENGINE *e = ENGINE_by_id("dynamic"); + if (e) { + if (!ENGINE_ctrl_cmd_string(e, "ID", engine, 0) || + !ENGINE_ctrl_cmd_string(e, "DIR_LOAD", "2", 0) || + !ENGINE_ctrl_cmd_string(e, "DIR_ADD", path, 0) || + !ENGINE_ctrl_cmd_string(e, "LOAD", NULL, 0)) { + ENGINE_free(e); + e = NULL; + } + } + return e; +} +#endif /** Initialize the crypto library. Return 0 on success, -1 on failure. */ int -crypto_global_init(int useAccel) +crypto_global_init(int useAccel, const char *accelName, const char *accelDir) { if (!_crypto_global_initialized) { ERR_load_crypto_strings(); OpenSSL_add_all_algorithms(); _crypto_global_initialized = 1; setup_openssl_threading(); - /* XXX the below is a bug, since we can't know if we're supposed - * to be using hardware acceleration or not. we should arrange - * for this function to be called before init_keys. But make it - * not complain loudly, at least until we make acceleration work. */ - if (useAccel < 0) { - log_info(LD_CRYPTO, "Initializing OpenSSL via tor_tls_init()."); - } if (useAccel > 0) { +#ifdef DISABLE_ENGINES + (void)accelName; + (void)accelDir; + log_warn(LD_CRYPTO, "No OpenSSL hardware acceleration support enabled."); +#else + ENGINE *e = NULL; + log_info(LD_CRYPTO, "Initializing OpenSSL engine support."); ENGINE_load_builtin_engines(); - if (!ENGINE_register_all_complete()) - return -1; - - /* XXXX make sure this isn't leaking. */ + ENGINE_register_all_complete(); + + if (accelName) { + if (accelDir) { + log_info(LD_CRYPTO, "Trying to load dynamic OpenSSL engine \"%s\"" + " via path \"%s\".", accelName, accelDir); + e = try_load_engine(accelName, accelDir); + } else { + log_info(LD_CRYPTO, "Initializing dynamic OpenSSL engine \"%s\"" + " acceleration support.", accelName); + e = ENGINE_by_id(accelName); + } + if (!e) { + log_warn(LD_CRYPTO, "Unable to load dynamic OpenSSL engine \"%s\".", + accelName); + } else { + log_info(LD_CRYPTO, "Loaded dynamic OpenSSL engine \"%s\".", + accelName); + } + } + if (e) { + log_info(LD_CRYPTO, "Loaded OpenSSL hardware acceleration engine," + " setting default ciphers."); + ENGINE_set_default(e, ENGINE_METHOD_ALL); + } log_engine("RSA", ENGINE_get_default_RSA()); log_engine("DH", ENGINE_get_default_DH()); log_engine("RAND", ENGINE_get_default_RAND()); log_engine("SHA1", ENGINE_get_digest_engine(NID_sha1)); log_engine("3DES", ENGINE_get_cipher_engine(NID_des_ede3_ecb)); log_engine("AES", ENGINE_get_cipher_engine(NID_aes_128_ecb)); +#endif + } else { + log_info(LD_CRYPTO, "NOT using OpenSSL engine support."); } return crypto_seed_rng(1); } @@ -217,7 +291,11 @@ crypto_global_cleanup(void) EVP_cleanup(); ERR_remove_state(0); ERR_free_strings(); + +#ifndef DISABLE_ENGINES ENGINE_cleanup(); +#endif + CONF_modules_unload(1); CRYPTO_cleanup_all_ex_data(); #ifdef TOR_IS_MULTITHREADED @@ -259,7 +337,8 @@ _crypto_new_pk_env_evp_pkey(EVP_PKEY *pkey) return _crypto_new_pk_env_rsa(rsa); } -/** Helper, used by tor-checkkey.c. Return the RSA from a crypto_pk_env_t. */ +/** Helper, used by tor-checkkey.c and tor-gencert.c. Return the RSA from a + * crypto_pk_env_t. */ RSA * _crypto_pk_env_get_rsa(crypto_pk_env_t *env) { @@ -321,7 +400,8 @@ crypto_new_pk_env(void) void crypto_free_pk_env(crypto_pk_env_t *env) { - tor_assert(env); + if (!env) + return; if (--env->refs > 0) return; @@ -347,10 +427,7 @@ crypto_create_init_cipher(const char *key, int encrypt_mode) return NULL; } - if (crypto_cipher_set_key(crypto, key)) { - crypto_log_errors(LOG_WARN, "setting symmetric key"); - goto error; - } + crypto_cipher_set_key(crypto, key); if (encrypt_mode) r = crypto_cipher_encrypt_init_cipher(crypto); @@ -384,7 +461,8 @@ crypto_new_cipher_env(void) void crypto_free_cipher_env(crypto_cipher_env_t *env) { - tor_assert(env); + if (!env) + return; tor_assert(env->cipher); aes_free_cipher(env->cipher); @@ -394,11 +472,11 @@ crypto_free_cipher_env(crypto_cipher_env_t *env) /* public key crypto */ -/** Generate a new public/private keypair in <b>env</b>. Return 0 on - * success, -1 on failure. +/** Generate a <b>bits</b>-bit new public/private keypair in <b>env</b>. + * Return 0 on success, -1 on failure. */ int -crypto_pk_generate_key(crypto_pk_env_t *env) +crypto_pk_generate_key_with_bits(crypto_pk_env_t *env, int bits) { tor_assert(env); @@ -406,7 +484,7 @@ crypto_pk_generate_key(crypto_pk_env_t *env) RSA_free(env->key); #if OPENSSL_VERSION_NUMBER < 0x00908000l /* In OpenSSL 0.9.7, RSA_generate_key is all we have. */ - env->key = RSA_generate_key(PK_BYTES*8,65537, NULL, NULL); + env->key = RSA_generate_key(bits, 65537, NULL, NULL); #else /* In OpenSSL 0.9.8, RSA_generate_key is deprecated. */ { @@ -419,7 +497,7 @@ crypto_pk_generate_key(crypto_pk_env_t *env) r = RSA_new(); if (!r) goto done; - if (RSA_generate_key_ex(r, PK_BYTES*8, e, NULL) == -1) + if (RSA_generate_key_ex(r, bits, e, NULL) == -1) goto done; env->key = r; @@ -1172,19 +1250,14 @@ crypto_cipher_generate_key(crypto_cipher_env_t *env) /** Set the symmetric key for the cipher in <b>env</b> to the first * CIPHER_KEY_LEN bytes of <b>key</b>. Does not initialize the cipher. - * Return 0 on success, -1 on failure. */ -int +void crypto_cipher_set_key(crypto_cipher_env_t *env, const char *key) { tor_assert(env); tor_assert(key); - if (!env->key) - return -1; - memcpy(env->key, key, CIPHER_KEY_LEN); - return 0; } /** Generate an initialization vector for our AES-CTR cipher; store it @@ -1358,9 +1431,69 @@ crypto_digest(char *digest, const char *m, size_t len) return (SHA1((const unsigned char*)m,len,(unsigned char*)digest) == NULL); } +int +crypto_digest256(char *digest, const char *m, size_t len, + digest_algorithm_t algorithm) +{ + tor_assert(m); + tor_assert(digest); + tor_assert(algorithm == DIGEST_SHA256); + return (SHA256((const unsigned char*)m,len,(unsigned char*)digest) == NULL); +} + +/** Set the digests_t in <b>ds_out</b> to contain every digest on the + * <b>len</b> bytes in <b>m</b> that we know how to compute. Return 0 on + * success, -1 on failure. */ +int +crypto_digest_all(digests_t *ds_out, const char *m, size_t len) +{ + digest_algorithm_t i; + tor_assert(ds_out); + memset(ds_out, 0, sizeof(*ds_out)); + if (crypto_digest(ds_out->d[DIGEST_SHA1], m, len) < 0) + return -1; + for (i = DIGEST_SHA256; i < N_DIGEST_ALGORITHMS; ++i) { + if (crypto_digest256(ds_out->d[i], m, len, i) < 0) + return -1; + } + return 0; +} + +/** Return the name of an algorithm, as used in directory documents. */ +const char * +crypto_digest_algorithm_get_name(digest_algorithm_t alg) +{ + switch (alg) { + case DIGEST_SHA1: + return "sha1"; + case DIGEST_SHA256: + return "sha256"; + default: + tor_fragile_assert(); + return "??unknown_digest??"; + } +} + +/** Given the name of a digest algorithm, return its integer value, or -1 if + * the name is not recognized. */ +int +crypto_digest_algorithm_parse_name(const char *name) +{ + if (!strcmp(name, "sha1")) + return DIGEST_SHA1; + else if (!strcmp(name, "sha256")) + return DIGEST_SHA256; + else + return -1; +} + /** Intermediate information about the digest of a stream of data. */ struct crypto_digest_env_t { - SHA_CTX d; + union { + SHA_CTX sha1; + SHA256_CTX sha2; + } d; + digest_algorithm_t algorithm : 8; }; /** Allocate and return a new digest object. @@ -1370,7 +1503,19 @@ crypto_new_digest_env(void) { crypto_digest_env_t *r; r = tor_malloc(sizeof(crypto_digest_env_t)); - SHA1_Init(&r->d); + SHA1_Init(&r->d.sha1); + r->algorithm = DIGEST_SHA1; + return r; +} + +crypto_digest_env_t * +crypto_new_digest256_env(digest_algorithm_t algorithm) +{ + crypto_digest_env_t *r; + tor_assert(algorithm == DIGEST_SHA256); + r = tor_malloc(sizeof(crypto_digest_env_t)); + SHA256_Init(&r->d.sha2); + r->algorithm = algorithm; return r; } @@ -1379,6 +1524,8 @@ crypto_new_digest_env(void) void crypto_free_digest_env(crypto_digest_env_t *digest) { + if (!digest) + return; memset(digest, 0, sizeof(crypto_digest_env_t)); tor_free(digest); } @@ -1391,30 +1538,51 @@ crypto_digest_add_bytes(crypto_digest_env_t *digest, const char *data, { tor_assert(digest); tor_assert(data); - /* Using the SHA1_*() calls directly means we don't support doing - * SHA1 in hardware. But so far the delay of getting the question + /* Using the SHA*_*() calls directly means we don't support doing + * SHA in hardware. But so far the delay of getting the question * to the hardware, and hearing the answer, is likely higher than * just doing it ourselves. Hashes are fast. */ - SHA1_Update(&digest->d, (void*)data, len); + switch (digest->algorithm) { + case DIGEST_SHA1: + SHA1_Update(&digest->d.sha1, (void*)data, len); + break; + case DIGEST_SHA256: + SHA256_Update(&digest->d.sha2, (void*)data, len); + break; + default: + tor_fragile_assert(); + break; + } } /** Compute the hash of the data that has been passed to the digest * object; write the first out_len bytes of the result to <b>out</b>. - * <b>out_len</b> must be \<= DIGEST_LEN. + * <b>out_len</b> must be \<= DIGEST256_LEN. */ void crypto_digest_get_digest(crypto_digest_env_t *digest, char *out, size_t out_len) { - unsigned char r[DIGEST_LEN]; - SHA_CTX tmpctx; + unsigned char r[DIGEST256_LEN]; + crypto_digest_env_t tmpenv; tor_assert(digest); tor_assert(out); - tor_assert(out_len <= DIGEST_LEN); - /* memcpy into a temporary ctx, since SHA1_Final clears the context */ - memcpy(&tmpctx, &digest->d, sizeof(SHA_CTX)); - SHA1_Final(r, &tmpctx); + /* memcpy into a temporary ctx, since SHA*_Final clears the context */ + memcpy(&tmpenv, digest, sizeof(crypto_digest_env_t)); + switch (digest->algorithm) { + case DIGEST_SHA1: + tor_assert(out_len <= DIGEST_LEN); + SHA1_Final(r, &tmpenv.d.sha1); + break; + case DIGEST_SHA256: + tor_assert(out_len <= DIGEST256_LEN); + SHA256_Final(r, &tmpenv.d.sha2); + break; + default: + tor_fragile_assert(); + break; + } memcpy(out, r, out_len); memset(r, 0, sizeof(r)); } @@ -1550,7 +1718,7 @@ crypto_dh_generate_public(crypto_dh_env_t *dh) crypto_log_errors(LOG_WARN, "generating DH key"); return -1; } - if (tor_check_dh_key(dh->dh->pub_key)<0) { + if (tor_check_dh_key(LOG_WARN, dh->dh->pub_key)<0) { log_warn(LD_CRYPTO, "Weird! Our own DH key was invalid. I guess once-in-" "the-universe chances really do happen. Trying again."); /* Free and clear the keys, so OpenSSL will actually try again. */ @@ -1597,7 +1765,7 @@ crypto_dh_get_public(crypto_dh_env_t *dh, char *pubkey, size_t pubkey_len) * See http://www.cl.cam.ac.uk/ftp/users/rja14/psandqs.ps.gz for some tips. */ static int -tor_check_dh_key(BIGNUM *bn) +tor_check_dh_key(int severity, BIGNUM *bn) { BIGNUM *x; char *s; @@ -1608,13 +1776,13 @@ tor_check_dh_key(BIGNUM *bn) init_dh_param(); BN_set_word(x, 1); if (BN_cmp(bn,x)<=0) { - log_warn(LD_CRYPTO, "DH key must be at least 2."); + log_fn(severity, LD_CRYPTO, "DH key must be at least 2."); goto err; } BN_copy(x,dh_param_p); BN_sub_word(x, 1); if (BN_cmp(bn,x)>=0) { - log_warn(LD_CRYPTO, "DH key must be at most p-2."); + log_fn(severity, LD_CRYPTO, "DH key must be at most p-2."); goto err; } BN_free(x); @@ -1622,7 +1790,7 @@ tor_check_dh_key(BIGNUM *bn) err: BN_free(x); s = BN_bn2hex(bn); - log_warn(LD_CRYPTO, "Rejecting insecure DH key [%s]", s); + log_fn(severity, LD_CRYPTO, "Rejecting insecure DH key [%s]", s); OPENSSL_free(s); return -1; } @@ -1640,7 +1808,7 @@ tor_check_dh_key(BIGNUM *bn) * where || is concatenation.) */ ssize_t -crypto_dh_compute_secret(crypto_dh_env_t *dh, +crypto_dh_compute_secret(int severity, crypto_dh_env_t *dh, const char *pubkey, size_t pubkey_len, char *secret_out, size_t secret_bytes_out) { @@ -1655,9 +1823,9 @@ crypto_dh_compute_secret(crypto_dh_env_t *dh, if (!(pubkey_bn = BN_bin2bn((const unsigned char*)pubkey, (int)pubkey_len, NULL))) goto error; - if (tor_check_dh_key(pubkey_bn)<0) { + if (tor_check_dh_key(severity, pubkey_bn)<0) { /* Check for invalid public keys. */ - log_warn(LD_CRYPTO,"Rejected invalid g^x"); + log_fn(severity, LD_CRYPTO,"Rejected invalid g^x"); goto error; } secret_tmp = tor_malloc(crypto_dh_get_bytes(dh)); @@ -1729,7 +1897,8 @@ crypto_expand_key_material(const char *key_in, size_t key_in_len, void crypto_dh_free(crypto_dh_env_t *dh) { - tor_assert(dh); + if (!dh) + return; tor_assert(dh->dh); DH_free(dh->dh); tor_free(dh); @@ -2147,6 +2316,44 @@ digest_from_base64(char *digest, const char *d64) #endif } +/** Base-64 encode DIGEST256_LINE bytes from <b>digest</b>, remove the + * trailing = and newline characters, and store the nul-terminated result in + * the first BASE64_DIGEST256_LEN+1 bytes of <b>d64</b>. */ +int +digest256_to_base64(char *d64, const char *digest) +{ + char buf[256]; + base64_encode(buf, sizeof(buf), digest, DIGEST256_LEN); + buf[BASE64_DIGEST256_LEN] = '\0'; + memcpy(d64, buf, BASE64_DIGEST256_LEN+1); + return 0; +} + +/** Given a base-64 encoded, nul-terminated digest in <b>d64</b> (without + * trailing newline or = characters), decode it and store the result in the + * first DIGEST256_LEN bytes at <b>digest</b>. */ +int +digest256_from_base64(char *digest, const char *d64) +{ +#ifdef USE_OPENSSL_BASE64 + char buf_in[BASE64_DIGEST256_LEN+3]; + char buf[256]; + if (strlen(d64) != BASE64_DIGEST256_LEN) + return -1; + memcpy(buf_in, d64, BASE64_DIGEST256_LEN); + memcpy(buf_in+BASE64_DIGEST256_LEN, "=\n\0", 3); + if (base64_decode(buf, sizeof(buf), buf_in, strlen(buf_in)) != DIGEST256_LEN) + return -1; + memcpy(digest, buf, DIGEST256_LEN); + return 0; +#else + if (base64_decode(digest, DIGEST256_LEN, d64, strlen(d64)) == DIGEST256_LEN) + return 0; + else + return -1; +#endif +} + /** Implements base32 encoding as in rfc3548. Limitation: Requires * that srclen*8 is a multiple of 5. */ diff --git a/src/common/crypto.h b/src/common/crypto.h index dd353ef030..239acb5871 100644 --- a/src/common/crypto.h +++ b/src/common/crypto.h @@ -18,6 +18,9 @@ /** Length of the output of our message digest. */ #define DIGEST_LEN 20 +/** Length of the output of our second (improved) message digests. (For now + * this is just sha256, but any it can be any other 256-byte digest). */ +#define DIGEST256_LEN 32 /** Length of our symmetric cipher's keys. */ #define CIPHER_KEY_LEN 16 /** Length of our symmetric cipher's IV. */ @@ -27,9 +30,12 @@ /** Length of our DH keys. */ #define DH_BYTES (1024/8) -/** Length of a message digest when encoded in base64 with trailing = signs - * removed. */ +/** Length of a sha1 message digest when encoded in base64 with trailing = + * signs removed. */ #define BASE64_DIGEST_LEN 27 +/** Length of a sha256 message digest when encoded in base64 with trailing = + * signs removed. */ +#define BASE64_DIGEST256_LEN 43 /** Constants used to indicate no padding for public-key encryption */ #define PK_NO_PADDING 60000 @@ -48,6 +54,26 @@ #define FINGERPRINT_LEN 49 /** Length of hex encoding of SHA1 digest, not including final NUL. */ #define HEX_DIGEST_LEN 40 +/** Length of hex encoding of SHA256 digest, not including final NUL. */ +#define HEX_DIGEST256_LEN 64 + +typedef enum { + DIGEST_SHA1 = 0, + DIGEST_SHA256 = 1, +} digest_algorithm_t; +#define N_DIGEST_ALGORITHMS (DIGEST_SHA256+1) + +/** A set of all the digests we know how to compute, taken on a single + * string. Any digests that are shorter than 256 bits are right-padded + * with 0 bits. + * + * Note that this representation wastes 12 bytes for the SHA1 case, so + * don't use it for anything where we need to allocate a whole bunch at + * once. + **/ +typedef struct { + char d[N_DIGEST_ALGORITHMS][DIGEST256_LEN]; +} digests_t; typedef struct crypto_pk_env_t crypto_pk_env_t; typedef struct crypto_cipher_env_t crypto_cipher_env_t; @@ -55,7 +81,9 @@ typedef struct crypto_digest_env_t crypto_digest_env_t; typedef struct crypto_dh_env_t crypto_dh_env_t; /* global state */ -int crypto_global_init(int hardwareAccel); +int crypto_global_init(int hardwareAccel, + const char *accelName, + const char *accelPath); void crypto_thread_cleanup(void); int crypto_global_cleanup(void); @@ -71,7 +99,9 @@ crypto_cipher_env_t *crypto_new_cipher_env(void); void crypto_free_cipher_env(crypto_cipher_env_t *env); /* public key crypto */ -int crypto_pk_generate_key(crypto_pk_env_t *env); +int crypto_pk_generate_key_with_bits(crypto_pk_env_t *env, int bits); +#define crypto_pk_generate_key(env) \ + crypto_pk_generate_key_with_bits((env), (PK_BYTES*8)) int crypto_pk_read_private_key_from_filename(crypto_pk_env_t *env, const char *keyfile); @@ -121,7 +151,7 @@ int crypto_pk_check_fingerprint_syntax(const char *s); /* symmetric crypto */ int crypto_cipher_generate_key(crypto_cipher_env_t *env); -int crypto_cipher_set_key(crypto_cipher_env_t *env, const char *key); +void crypto_cipher_set_key(crypto_cipher_env_t *env, const char *key); void crypto_cipher_generate_iv(char *iv_out); int crypto_cipher_set_iv(crypto_cipher_env_t *env, const char *iv); const char *crypto_cipher_get_key(crypto_cipher_env_t *env); @@ -141,9 +171,15 @@ int crypto_cipher_decrypt_with_iv(crypto_cipher_env_t *env, char *to, size_t tolen, const char *from, size_t fromlen); -/* SHA-1 */ +/* SHA-1 and other digests. */ int crypto_digest(char *digest, const char *m, size_t len); +int crypto_digest256(char *digest, const char *m, size_t len, + digest_algorithm_t algorithm); +int crypto_digest_all(digests_t *ds_out, const char *m, size_t len); +const char *crypto_digest_algorithm_get_name(digest_algorithm_t alg); +int crypto_digest_algorithm_parse_name(const char *name); crypto_digest_env_t *crypto_new_digest_env(void); +crypto_digest_env_t *crypto_new_digest256_env(digest_algorithm_t algorithm); void crypto_free_digest_env(crypto_digest_env_t *digest); void crypto_digest_add_bytes(crypto_digest_env_t *digest, const char *data, size_t len); @@ -162,7 +198,7 @@ int crypto_dh_get_bytes(crypto_dh_env_t *dh); int crypto_dh_generate_public(crypto_dh_env_t *dh); int crypto_dh_get_public(crypto_dh_env_t *dh, char *pubkey_out, size_t pubkey_out_len); -ssize_t crypto_dh_compute_secret(crypto_dh_env_t *dh, +ssize_t crypto_dh_compute_secret(int severity, crypto_dh_env_t *dh, const char *pubkey, size_t pubkey_len, char *secret_out, size_t secret_out_len); void crypto_dh_free(crypto_dh_env_t *dh); @@ -191,6 +227,8 @@ int base32_decode(char *dest, size_t destlen, const char *src, size_t srclen); int digest_to_base64(char *d64, const char *digest); int digest_from_base64(char *digest, const char *d64); +int digest256_to_base64(char *d64, const char *digest); +int digest256_from_base64(char *digest, const char *d64); /** Length of RFC2440-style S2K specifier: the first 8 bytes are a salt, the * 9th describes how much iteration to do. */ diff --git a/src/common/ht.h b/src/common/ht.h index b31492ec3c..5187c90e6f 100644 --- a/src/common/ht.h +++ b/src/common/ht.h @@ -42,6 +42,10 @@ #define HT_SIZE(head) \ ((head)->hth_n_entries) +/* Return memory usage for a hashtable (not counting the entries themselves) */ +#define HT_MEM_USAGE(head) \ + (sizeof(*head) + (head)->hth_table_length * sizeof(void*)) + #define HT_FIND(name, head, elm) name##_HT_FIND((head), (elm)) #define HT_INSERT(name, head, elm) name##_HT_INSERT((head), (elm)) #define HT_REPLACE(name, head, elm) name##_HT_REPLACE((head), (elm)) diff --git a/src/common/log.c b/src/common/log.c index 423a687a51..ef65be8a3d 100644 --- a/src/common/log.c +++ b/src/common/log.c @@ -36,8 +36,6 @@ #include "log.h" #include "container.h" -#include <event.h> - #define TRUNCATED_STR "[...truncated]" #define TRUNCATED_STR_LEN 14 @@ -83,7 +81,7 @@ should_log_function_name(log_domain_mask_t domain, int severity) /* All debugging messages occur in interesting places. */ return 1; case LOG_NOTICE: - case LOG_WARN: + case LOG_WARN: case LOG_ERR: /* We care about places where bugs occur. */ return (domain == LD_BUG); @@ -330,7 +328,7 @@ logv(int severity, log_domain_mask_t domain, const char *funcname, /** Output a message to the log. */ void -_log(int severity, log_domain_mask_t domain, const char *format, ...) +tor_log(int severity, log_domain_mask_t domain, const char *format, ...) { va_list ap; if (severity > _log_global_min_severity) @@ -428,6 +426,8 @@ _log_err(log_domain_mask_t domain, const char *format, ...) static void log_free(logfile_t *victim) { + if (!victim) + return; tor_free(victim->severities); tor_free(victim->filename); tor_free(victim); @@ -747,7 +747,7 @@ log_level_to_string(int level) static const char *domain_list[] = { "GENERAL", "CRYPTO", "NET", "CONFIG", "FS", "PROTOCOL", "MM", "HTTP", "APP", "CONTROL", "CIRC", "REND", "BUG", "DIR", "DIRSERV", - "OR", "EDGE", "ACCT", "HIST", NULL + "OR", "EDGE", "ACCT", "HIST", "HANDSHAKE", NULL }; /** Return a bitmask for the log domain for which <b>domain</b> is the name, @@ -925,65 +925,6 @@ switch_logs_debug(void) UNLOCK_LOGS(); } -#ifdef HAVE_EVENT_SET_LOG_CALLBACK -/** A string which, if it appears in a libevent log, should be ignored. */ -static const char *suppress_msg = NULL; -/** Callback function passed to event_set_log() so we can intercept - * log messages from libevent. */ -static void -libevent_logging_callback(int severity, const char *msg) -{ - char buf[1024]; - size_t n; - if (suppress_msg && strstr(msg, suppress_msg)) - return; - n = strlcpy(buf, msg, sizeof(buf)); - if (n && n < sizeof(buf) && buf[n-1] == '\n') { - buf[n-1] = '\0'; - } - switch (severity) { - case _EVENT_LOG_DEBUG: - log(LOG_DEBUG, LD_NET, "Message from libevent: %s", buf); - break; - case _EVENT_LOG_MSG: - log(LOG_INFO, LD_NET, "Message from libevent: %s", buf); - break; - case _EVENT_LOG_WARN: - log(LOG_WARN, LD_GENERAL, "Warning from libevent: %s", buf); - break; - case _EVENT_LOG_ERR: - log(LOG_ERR, LD_GENERAL, "Error from libevent: %s", buf); - break; - default: - log(LOG_WARN, LD_GENERAL, "Message [%d] from libevent: %s", - severity, buf); - break; - } -} -/** Set hook to intercept log messages from libevent. */ -void -configure_libevent_logging(void) -{ - event_set_log_callback(libevent_logging_callback); -} -/** Ignore any libevent log message that contains <b>msg</b>. */ -void -suppress_libevent_log_msg(const char *msg) -{ - suppress_msg = msg; -} -#else -void -configure_libevent_logging(void) -{ -} -void -suppress_libevent_log_msg(const char *msg) -{ - (void)msg; -} -#endif - #if 0 static void dump_log_info(logfile_t *lf) diff --git a/src/common/log.h b/src/common/log.h index 834b1724b3..9f9a4277fb 100644 --- a/src/common/log.h +++ b/src/common/log.h @@ -90,9 +90,10 @@ #define LD_ACCT (1u<<17) /** Router history */ #define LD_HIST (1u<<18) - +/** OR handshaking */ +#define LD_HANDSHAKE (1u<<19) /** Number of logging domains in the code. */ -#define N_LOGGING_DOMAINS 19 +#define N_LOGGING_DOMAINS 20 typedef uint32_t log_domain_mask_t; @@ -134,16 +135,14 @@ void add_temp_log(int min_severity); void close_temp_logs(void); void rollback_log_changes(void); void mark_logs_temp(void); -void configure_libevent_logging(void); -void suppress_libevent_log_msg(const char *msg); void change_callback_log_severity(int loglevelMin, int loglevelMax, log_callback cb); void log_set_application_name(const char *name); /* Outputs a message to stdout */ -void _log(int severity, log_domain_mask_t domain, const char *format, ...) +void tor_log(int severity, log_domain_mask_t domain, const char *format, ...) CHECK_PRINTF(3,4); -#define log _log /* hack it so we don't conflict with log() as much */ +#define log tor_log /* hack it so we don't conflict with log() as much */ #ifdef __GNUC__ extern int _log_global_min_severity; diff --git a/src/common/memarea.c b/src/common/memarea.c index 1c81e2fd78..661bd85da8 100644 --- a/src/common/memarea.c +++ b/src/common/memarea.c @@ -13,6 +13,10 @@ #include "compat.h" #include "log.h" +/** If true, we try to detect any attempts to write beyond the length of a + * memarea. */ +#define USE_SENTINELS + /** All returned pointers should be aligned to the nearest multiple of this * value. */ #define MEMAREA_ALIGN SIZEOF_VOID_P @@ -25,6 +29,24 @@ #error "void* is neither 4 nor 8 bytes long. I don't know how to align stuff." #endif +#ifdef USE_SENTINELS +#define SENTINEL_VAL 0x90806622u +#define SENTINEL_LEN sizeof(uint32_t) +#define SET_SENTINEL(chunk) \ + STMT_BEGIN \ + set_uint32( &(chunk)->u.mem[chunk->mem_size], SENTINEL_VAL ); \ + STMT_END +#define CHECK_SENTINEL(chunk) \ + STMT_BEGIN \ + uint32_t sent_val = get_uint32(&(chunk)->u.mem[chunk->mem_size]); \ + tor_assert(sent_val == SENTINEL_VAL); \ + STMT_END +#else +#define SENTINEL_LEN 0 +#define SET_SENTINEL(chunk) STMT_NIL +#define CHECK_SENTINEL(chunk) STMT_NIL +#endif + /** Increment <b>ptr</b> until it is aligned to MEMAREA_ALIGN. */ static INLINE void * realign_pointer(void *ptr) @@ -78,15 +100,20 @@ alloc_chunk(size_t sz, int freelist_ok) freelist = res->next_chunk; res->next_chunk = NULL; --freelist_len; + CHECK_SENTINEL(res); return res; } else { size_t chunk_size = freelist_ok ? CHUNK_SIZE : sz; - memarea_chunk_t *res = tor_malloc_roundup(&chunk_size); + memarea_chunk_t *res; + chunk_size += SENTINEL_LEN; + res = tor_malloc_roundup(&chunk_size); res->next_chunk = NULL; - res->mem_size = chunk_size - CHUNK_HEADER_SIZE; + res->mem_size = chunk_size - CHUNK_HEADER_SIZE - SENTINEL_LEN; res->next_mem = res->u.mem; - tor_assert(res->next_mem+res->mem_size == ((char*)res)+chunk_size); + tor_assert(res->next_mem+res->mem_size+SENTINEL_LEN == + ((char*)res)+chunk_size); tor_assert(realign_pointer(res->next_mem) == res->next_mem); + SET_SENTINEL(res); return res; } } @@ -94,8 +121,9 @@ alloc_chunk(size_t sz, int freelist_ok) /** Release <b>chunk</b> from a memarea, either by adding it to the freelist * or by freeing it if the freelist is already too big. */ static void -chunk_free(memarea_chunk_t *chunk) +chunk_free_unchecked(memarea_chunk_t *chunk) { + CHECK_SENTINEL(chunk); if (freelist_len < MAX_FREELIST_LEN) { ++freelist_len; chunk->next_chunk = freelist; @@ -123,7 +151,7 @@ memarea_drop_all(memarea_t *area) memarea_chunk_t *chunk, *next; for (chunk = area->first; chunk; chunk = next) { next = chunk->next_chunk; - chunk_free(chunk); + chunk_free_unchecked(chunk); } area->first = NULL; /*fail fast on */ tor_free(area); @@ -139,7 +167,7 @@ memarea_clear(memarea_t *area) if (area->first->next_chunk) { for (chunk = area->first->next_chunk; chunk; chunk = next) { next = chunk->next_chunk; - chunk_free(chunk); + chunk_free_unchecked(chunk); } area->first->next_chunk = NULL; } @@ -182,6 +210,7 @@ memarea_alloc(memarea_t *area, size_t sz) memarea_chunk_t *chunk = area->first; char *result; tor_assert(chunk); + CHECK_SENTINEL(chunk); if (sz == 0) sz = 1; if (chunk->next_mem+sz > chunk->u.mem+chunk->mem_size) { @@ -258,6 +287,7 @@ memarea_get_stats(memarea_t *area, size_t *allocated_out, size_t *used_out) size_t a = 0, u = 0; memarea_chunk_t *chunk; for (chunk = area->first; chunk; chunk = chunk->next_chunk) { + CHECK_SENTINEL(chunk); a += CHUNK_HEADER_SIZE + chunk->mem_size; tor_assert(chunk->next_mem >= chunk->u.mem); u += CHUNK_HEADER_SIZE + (chunk->next_mem - chunk->u.mem); @@ -274,6 +304,7 @@ memarea_assert_ok(memarea_t *area) tor_assert(area->first); for (chunk = area->first; chunk; chunk = chunk->next_chunk) { + CHECK_SENTINEL(chunk); tor_assert(chunk->next_mem >= chunk->u.mem); tor_assert(chunk->next_mem <= (char*) realign_pointer(chunk->u.mem+chunk->mem_size)); diff --git a/src/common/sha256.c b/src/common/sha256.c new file mode 100644 index 0000000000..d445ae4e6d --- /dev/null +++ b/src/common/sha256.c @@ -0,0 +1,331 @@ +/* Copyright (c) 2009, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ +/* This SHA256 implementation is adapted from the public domain one in + LibTomCrypt, version 1.6. Tor uses it on platforms where OpenSSL doesn't + have a SHA256. */ + + +typedef struct sha256_state { + uint64_t length; + uint32_t state[8], curlen; + unsigned char buf[64]; +} sha256_state; + +#define CRYPT_OK 0 +#define CRYPT_NOP -1 +#define CRYPT_INVALID_ARG -2 + +#define LOAD32H(x,y) STMT_BEGIN x = ntohl(get_uint32((const char*)y)); STMT_END +#define STORE32H(x,y) STMT_BEGIN set_uint32((char*)y, htonl(x)); STMT_END +#define STORE64H(x,y) STMT_BEGIN \ + set_uint32((char*)y, htonl((uint32_t)((x)>>32))); \ + set_uint32(((char*)y)+4, htonl((uint32_t)((x)&0xffffffff))); \ + STMT_END +#define RORc(x, y) ( ((((unsigned long)(x)&0xFFFFFFFFUL)>>(unsigned long)((y)&31)) | ((unsigned long)(x)<<(unsigned long)(32-((y)&31)))) & 0xFFFFFFFFUL) +#ifndef MIN + #define MIN(x, y) ( ((x)<(y))?(x):(y) ) +#endif + + +/* LibTomCrypt, modular cryptographic library -- Tom St Denis + * + * LibTomCrypt is a library that provides various cryptographic + * algorithms in a highly modular and flexible manner. + * + * The library is free for all purposes without any express + * guarantee it works. + * + * Tom St Denis, tomstdenis@gmail.com, http://libtomcrypt.com + */ + +/** + @file sha256.c + SHA256 by Tom St Denis +*/ + + +#ifdef LTC_SMALL_CODE +/* the K array */ +static const uint32_t K[64] = { + 0x428a2f98UL, 0x71374491UL, 0xb5c0fbcfUL, 0xe9b5dba5UL, 0x3956c25bUL, + 0x59f111f1UL, 0x923f82a4UL, 0xab1c5ed5UL, 0xd807aa98UL, 0x12835b01UL, + 0x243185beUL, 0x550c7dc3UL, 0x72be5d74UL, 0x80deb1feUL, 0x9bdc06a7UL, + 0xc19bf174UL, 0xe49b69c1UL, 0xefbe4786UL, 0x0fc19dc6UL, 0x240ca1ccUL, + 0x2de92c6fUL, 0x4a7484aaUL, 0x5cb0a9dcUL, 0x76f988daUL, 0x983e5152UL, + 0xa831c66dUL, 0xb00327c8UL, 0xbf597fc7UL, 0xc6e00bf3UL, 0xd5a79147UL, + 0x06ca6351UL, 0x14292967UL, 0x27b70a85UL, 0x2e1b2138UL, 0x4d2c6dfcUL, + 0x53380d13UL, 0x650a7354UL, 0x766a0abbUL, 0x81c2c92eUL, 0x92722c85UL, + 0xa2bfe8a1UL, 0xa81a664bUL, 0xc24b8b70UL, 0xc76c51a3UL, 0xd192e819UL, + 0xd6990624UL, 0xf40e3585UL, 0x106aa070UL, 0x19a4c116UL, 0x1e376c08UL, + 0x2748774cUL, 0x34b0bcb5UL, 0x391c0cb3UL, 0x4ed8aa4aUL, 0x5b9cca4fUL, + 0x682e6ff3UL, 0x748f82eeUL, 0x78a5636fUL, 0x84c87814UL, 0x8cc70208UL, + 0x90befffaUL, 0xa4506cebUL, 0xbef9a3f7UL, 0xc67178f2UL +}; +#endif + +/* Various logical functions */ +#define Ch(x,y,z) (z ^ (x & (y ^ z))) +#define Maj(x,y,z) (((x | y) & z) | (x & y)) +#define S(x, n) RORc((x),(n)) +#define R(x, n) (((x)&0xFFFFFFFFUL)>>(n)) +#define Sigma0(x) (S(x, 2) ^ S(x, 13) ^ S(x, 22)) +#define Sigma1(x) (S(x, 6) ^ S(x, 11) ^ S(x, 25)) +#define Gamma0(x) (S(x, 7) ^ S(x, 18) ^ R(x, 3)) +#define Gamma1(x) (S(x, 17) ^ S(x, 19) ^ R(x, 10)) + +/* compress 512-bits */ +#ifdef LTC_CLEAN_STACK +static int _sha256_compress(sha256_state * md, unsigned char *buf) +#else +static int sha256_compress(sha256_state * md, unsigned char *buf) +#endif +{ + uint32_t S[8], W[64], t0, t1; +#ifdef LTC_SMALL_CODE + uint32_t t; +#endif + int i; + + /* copy state into S */ + for (i = 0; i < 8; i++) { + S[i] = md->state[i]; + } + + /* copy the state into 512-bits into W[0..15] */ + for (i = 0; i < 16; i++) { + LOAD32H(W[i], buf + (4*i)); + } + + /* fill W[16..63] */ + for (i = 16; i < 64; i++) { + W[i] = Gamma1(W[i - 2]) + W[i - 7] + Gamma0(W[i - 15]) + W[i - 16]; + } + + /* Compress */ +#ifdef LTC_SMALL_CODE +#define RND(a,b,c,d,e,f,g,h,i) \ + t0 = h + Sigma1(e) + Ch(e, f, g) + K[i] + W[i]; \ + t1 = Sigma0(a) + Maj(a, b, c); \ + d += t0; \ + h = t0 + t1; + + for (i = 0; i < 64; ++i) { + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],i); + t = S[7]; S[7] = S[6]; S[6] = S[5]; S[5] = S[4]; + S[4] = S[3]; S[3] = S[2]; S[2] = S[1]; S[1] = S[0]; S[0] = t; + } +#else +#define RND(a,b,c,d,e,f,g,h,i,ki) \ + t0 = h + Sigma1(e) + Ch(e, f, g) + ki + W[i]; \ + t1 = Sigma0(a) + Maj(a, b, c); \ + d += t0; \ + h = t0 + t1; + + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],0,0x428a2f98); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],1,0x71374491); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],2,0xb5c0fbcf); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],3,0xe9b5dba5); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],4,0x3956c25b); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],5,0x59f111f1); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],6,0x923f82a4); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],7,0xab1c5ed5); + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],8,0xd807aa98); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],9,0x12835b01); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],10,0x243185be); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],11,0x550c7dc3); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],12,0x72be5d74); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],13,0x80deb1fe); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],14,0x9bdc06a7); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],15,0xc19bf174); + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],16,0xe49b69c1); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],17,0xefbe4786); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],18,0x0fc19dc6); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],19,0x240ca1cc); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],20,0x2de92c6f); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],21,0x4a7484aa); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],22,0x5cb0a9dc); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],23,0x76f988da); + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],24,0x983e5152); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],25,0xa831c66d); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],26,0xb00327c8); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],27,0xbf597fc7); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],28,0xc6e00bf3); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],29,0xd5a79147); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],30,0x06ca6351); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],31,0x14292967); + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],32,0x27b70a85); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],33,0x2e1b2138); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],34,0x4d2c6dfc); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],35,0x53380d13); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],36,0x650a7354); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],37,0x766a0abb); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],38,0x81c2c92e); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],39,0x92722c85); + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],40,0xa2bfe8a1); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],41,0xa81a664b); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],42,0xc24b8b70); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],43,0xc76c51a3); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],44,0xd192e819); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],45,0xd6990624); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],46,0xf40e3585); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],47,0x106aa070); + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],48,0x19a4c116); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],49,0x1e376c08); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],50,0x2748774c); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],51,0x34b0bcb5); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],52,0x391c0cb3); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],53,0x4ed8aa4a); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],54,0x5b9cca4f); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],55,0x682e6ff3); + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],56,0x748f82ee); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],57,0x78a5636f); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],58,0x84c87814); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],59,0x8cc70208); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],60,0x90befffa); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],61,0xa4506ceb); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],62,0xbef9a3f7); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],63,0xc67178f2); + +#undef RND + +#endif + + /* feedback */ + for (i = 0; i < 8; i++) { + md->state[i] = md->state[i] + S[i]; + } + return CRYPT_OK; +} + +#ifdef LTC_CLEAN_STACK +static int sha256_compress(sha256_state * md, unsigned char *buf) +{ + int err; + err = _sha256_compress(md, buf); + burn_stack(sizeof(uint32_t) * 74); + return err; +} +#endif + +/** + Initialize the hash state + @param md The hash state you wish to initialize + @return CRYPT_OK if successful +*/ +static int sha256_init(sha256_state * md) +{ + LTC_ARGCHK(md != NULL); + + md->curlen = 0; + md->length = 0; + md->state[0] = 0x6A09E667UL; + md->state[1] = 0xBB67AE85UL; + md->state[2] = 0x3C6EF372UL; + md->state[3] = 0xA54FF53AUL; + md->state[4] = 0x510E527FUL; + md->state[5] = 0x9B05688CUL; + md->state[6] = 0x1F83D9ABUL; + md->state[7] = 0x5BE0CD19UL; + return CRYPT_OK; +} + +/** + Process a block of memory though the hash + @param md The hash state + @param in The data to hash + @param inlen The length of the data (octets) + @return CRYPT_OK if successful +*/ +static int sha256_process (sha256_state * md, const unsigned char *in, unsigned long inlen) +{ + unsigned long n; + int err; + LTC_ARGCHK(md != NULL); + LTC_ARGCHK(in != NULL); + if (md->curlen > sizeof(md->buf)) { + return CRYPT_INVALID_ARG; + } + while (inlen > 0) { + if (md->curlen == 0 && inlen >= 64) { + if ((err = sha256_compress (md, (unsigned char *)in)) != CRYPT_OK) { + return err; + } + md->length += 64 * 8; + in += 64; + inlen -= 64; + } else { + n = MIN(inlen, (64 - md->curlen)); + memcpy(md->buf + md->curlen, in, (size_t)n); + md->curlen += n; + in += n; + inlen -= n; + if (md->curlen == 64) { + if ((err = sha256_compress (md, md->buf)) != CRYPT_OK) { + return err; + } + md->length += 8*64; + md->curlen = 0; + } + } + } + return CRYPT_OK; +} + +/** + Terminate the hash to get the digest + @param md The hash state + @param out [out] The destination of the hash (32 bytes) + @return CRYPT_OK if successful +*/ +static int sha256_done(sha256_state * md, unsigned char *out) +{ + int i; + + LTC_ARGCHK(md != NULL); + LTC_ARGCHK(out != NULL); + + if (md->curlen >= sizeof(md->buf)) { + return CRYPT_INVALID_ARG; + } + + + /* increase the length of the message */ + md->length += md->curlen * 8; + + /* append the '1' bit */ + md->buf[md->curlen++] = (unsigned char)0x80; + + /* if the length is currently above 56 bytes we append zeros + * then compress. Then we can fall back to padding zeros and length + * encoding like normal. + */ + if (md->curlen > 56) { + while (md->curlen < 64) { + md->buf[md->curlen++] = (unsigned char)0; + } + sha256_compress(md, md->buf); + md->curlen = 0; + } + + /* pad upto 56 bytes of zeroes */ + while (md->curlen < 56) { + md->buf[md->curlen++] = (unsigned char)0; + } + + /* store length */ + STORE64H(md->length, md->buf+56); + sha256_compress(md, md->buf); + + /* copy output */ + for (i = 0; i < 8; i++) { + STORE32H(md->state[i], out+(4*i)); + } +#ifdef LTC_CLEAN_STACK + zeromem(md, sizeof(sha256_state)); +#endif + return CRYPT_OK; +} + +/* $Source: /cvs/libtom/libtomcrypt/src/hashes/sha2/sha256.c,v $ */ +/* $Revision: 1.9 $ */ +/* $Date: 2006/11/01 09:28:17 $ */ diff --git a/src/common/test.h b/src/common/test.h deleted file mode 100644 index 52a249ccb0..0000000000 --- a/src/common/test.h +++ /dev/null @@ -1,184 +0,0 @@ -/* Copyright (c) 2001-2003, Roger Dingledine. - * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2009, The Tor Project, Inc. */ -/* See LICENSE for licensing information */ - -#ifndef _TOR_TEST_H -#define _TOR_TEST_H - -/** - * \file test.h - * \brief Macros used by unit tests. - */ - -#include "compat.h" - -#ifdef __GNUC__ -#define PRETTY_FUNCTION __PRETTY_FUNCTION__ -#else -#define PRETTY_FUNCTION "" -#endif - -#define test_fail_msg(msg) \ - STMT_BEGIN \ - have_failed = 1; \ - printf("\nFile %s: line %d (%s): %s", \ - _SHORT_FILE_, \ - __LINE__, \ - PRETTY_FUNCTION, \ - msg); \ - goto done; \ - STMT_END - -#define test_fail() test_fail_msg("Assertion failed.") - -#define test_assert(expr) \ - STMT_BEGIN \ - if (expr) { printf("."); fflush(stdout); } else { \ - have_failed = 1; \ - printf("\nFile %s: line %d (%s): assertion failed: (%s)\n", \ - _SHORT_FILE_, \ - __LINE__, \ - PRETTY_FUNCTION, \ - #expr); \ - goto done; \ - } STMT_END - -#define test_eq_type(tp, fmt, expr1, expr2) \ - STMT_BEGIN \ - tp _test_v1=(tp)(expr1); \ - tp _test_v2=(tp)(expr2); \ - if (_test_v1==_test_v2) { printf("."); fflush(stdout); } else { \ - have_failed = 1; \ - printf("\nFile %s: line %d (%s): Assertion failed: (%s==%s)\n" \ - " "fmt "!="fmt"\n", \ - _SHORT_FILE_, \ - __LINE__, \ - PRETTY_FUNCTION, \ - #expr1, #expr2, \ - _test_v1, _test_v2); \ - goto done; \ - } STMT_END - -#define test_eq(expr1, expr2) \ - test_eq_type(long, "%ld", expr1, expr2) - -#define test_eq_ptr(expr1, expr2) \ - test_eq_type(void*, "%p", expr1, expr2) - -#define test_neq_type(tp, fmt, expr1, expr2) \ - STMT_BEGIN \ - tp _test_v1=(tp)(expr1); \ - tp _test_v2=(tp)(expr2); \ - if (_test_v1!=_test_v2) { printf("."); fflush(stdout); } else { \ - have_failed = 1; \ - printf("\nFile %s: line %d (%s): Assertion failed: (%s!=%s)\n" \ - " ("fmt" == "fmt")\n", \ - _SHORT_FILE_, \ - __LINE__, \ - PRETTY_FUNCTION, \ - #expr1, #expr2, \ - _test_v1, _test_v2); \ - goto done; \ - } STMT_END - -#define test_neq(expr1, expr2) \ - test_neq_type(long, "%ld", expr1, expr2) - -#define test_neq_ptr(expr1, expr2) \ - test_neq_type(void *, "%p", expr1, expr2) - -#define test_streq(expr1, expr2) \ - STMT_BEGIN \ - const char *_test_v1=(expr1), *_test_v2=(expr2); \ - if (!strcmp(_test_v1,_test_v2)) { printf("."); fflush(stdout); } else { \ - have_failed = 1; \ - printf("\nFile %s: line %d (%s): Assertion failed: (%s==%s)\n"\ - " (\"%s\" != \"%s\")\n", \ - _SHORT_FILE_, \ - __LINE__, \ - PRETTY_FUNCTION, \ - #expr1, #expr2, \ - _test_v1, _test_v2); \ - goto done; \ - } STMT_END - -#define test_strneq(expr1, expr2) \ - STMT_BEGIN \ - const char *_test_v1=(expr1), *_test_v2=(expr2); \ - if (strcmp(_test_v1,_test_v2)) { printf("."); fflush(stdout); } else { \ - have_failed = 1; \ - printf("\nFile %s: line %d (%s): Assertion failed: (%s!=%s)\n"\ - " (\"%s\" == \"%s\")\n", \ - _SHORT_FILE_, \ - __LINE__, \ - PRETTY_FUNCTION, \ - #expr1, #expr2, \ - _test_v1, _test_v2); \ - goto done; \ - } STMT_END - -#define test_memeq(expr1, expr2, len) \ - STMT_BEGIN \ - const void *_test_v1=(expr1), *_test_v2=(expr2); \ - char *mem1, *mem2; \ - if (!memcmp(_test_v1,_test_v2,(len))) { \ - printf("."); fflush(stdout); } else { \ - have_failed = 1; \ - mem1 = tor_malloc(len*2+1); \ - mem2 = tor_malloc(len*2+1); \ - base16_encode(mem1, len*2+1, _test_v1, len); \ - base16_encode(mem2, len*2+1, _test_v2, len); \ - printf("\nFile %s: line %d (%s): Assertion failed: (%s==%s)\n" \ - " %s != %s\n", \ - _SHORT_FILE_, \ - __LINE__, \ - PRETTY_FUNCTION, \ - #expr1, #expr2, mem1, mem2); \ - tor_free(mem1); \ - tor_free(mem2); \ - goto done; \ - } STMT_END - -#define test_memeq_hex(expr1, hex) \ - STMT_BEGIN \ - const char *_test_v1 = (char*)(expr1); \ - const char *_test_v2 = (hex); \ - size_t _len_v2 = strlen(_test_v2); \ - char *_mem2 = tor_malloc(_len_v2/2); \ - tor_assert((_len_v2 & 1) == 0); \ - base16_decode(_mem2, _len_v2/2, _test_v2, _len_v2); \ - if (!memcmp(_mem2, _test_v1, _len_v2/2)) { \ - printf("."); fflush(stdout); } else { \ - char *_mem1 = tor_malloc(_len_v2+1); \ - base16_encode(_mem1, _len_v2+1, _test_v1, _len_v2/2); \ - printf("\nFile %s: line %d (%s): Assertion failed: (%s==%s)\n" \ - " %s != %s\n", \ - _SHORT_FILE_, \ - __LINE__, \ - PRETTY_FUNCTION, \ - #expr1, _test_v2, _mem1, _test_v2); \ - tor_free(_mem1); \ - tor_free(_mem2); \ - goto done; \ - } \ - tor_free(_mem2); \ - STMT_END - -#define test_memneq(expr1, expr2, len) \ - STMT_BEGIN \ - void *_test_v1=(expr1), *_test_v2=(expr2); \ - if (memcmp(_test_v1,_test_v2,(len))) { \ - printf("."); fflush(stdout); \ - } else { \ - have_failed = 1; \ - printf("\nFile %s: line %d (%s): Assertion failed: (%s!=%s)\n", \ - _SHORT_FILE_, \ - __LINE__, \ - PRETTY_FUNCTION, \ - #expr1, #expr2); \ - goto done; \ - } STMT_END - -#endif - diff --git a/src/common/torgzip.c b/src/common/torgzip.c index 762f2e71bf..4f1b46adde 100644 --- a/src/common/torgzip.c +++ b/src/common/torgzip.c @@ -165,9 +165,7 @@ tor_gzip_compress(char **out, size_t *out_len, deflateEnd(stream); tor_free(stream); } - if (*out) { - tor_free(*out); - } + tor_free(*out); return -1; } @@ -423,7 +421,8 @@ tor_zlib_process(tor_zlib_state_t *state, void tor_zlib_free(tor_zlib_state_t *state) { - tor_assert(state); + if (!state) + return; if (state->compress) deflateEnd(&state->stream); diff --git a/src/common/torint.h b/src/common/torint.h index f8441859a9..be624e04c6 100644 --- a/src/common/torint.h +++ b/src/common/torint.h @@ -117,11 +117,10 @@ typedef unsigned int uint32_t; #ifndef INT32_MAX #define INT32_MAX 0x7fffffffu #endif -#endif - #ifndef INT32_MIN #define INT32_MIN (-2147483647-1) #endif +#endif #if (SIZEOF_LONG == 4) #ifndef HAVE_INT32_T diff --git a/src/common/tortls.c b/src/common/tortls.c index beab5356c5..4bb2291b1f 100644 --- a/src/common/tortls.c +++ b/src/common/tortls.c @@ -166,30 +166,51 @@ static int tls_library_is_initialized = 0; #define _TOR_TLS_SYSCALL (_MIN_TOR_TLS_ERROR_VAL - 2) #define _TOR_TLS_ZERORETURN (_MIN_TOR_TLS_ERROR_VAL - 1) +#include "tortls_states.h" + +/** Return the symbolic name of an OpenSSL state. */ +static const char * +ssl_state_to_string(int ssl_state) +{ + static char buf[40]; + int i; + for (i = 0; state_map[i].name; ++i) { + if (state_map[i].state == ssl_state) + return state_map[i].name; + } + tor_snprintf(buf, sizeof(buf), "Unknown state %d", ssl_state); + return buf; +} + /** Log all pending tls errors at level <b>severity</b>. Use * <b>doing</b> to describe our current activities. */ static void -tls_log_errors(tor_tls_t *tls, int severity, const char *doing) +tls_log_errors(tor_tls_t *tls, int severity, int domain, const char *doing) { + const char *state = NULL; + int st; unsigned long err; const char *msg, *lib, *func, *addr; addr = tls ? tls->address : NULL; + st = (tls && tls->ssl) ? tls->ssl->state : -1; while ((err = ERR_get_error()) != 0) { msg = (const char*)ERR_reason_error_string(err); lib = (const char*)ERR_lib_error_string(err); func = (const char*)ERR_func_error_string(err); + if (!state) + state = (st>=0)?ssl_state_to_string(st):"---"; if (!msg) msg = "(null)"; if (!lib) lib = "(null)"; if (!func) func = "(null)"; if (doing) { - log(severity, LD_NET, "TLS error while %s%s%s: %s (in %s:%s)", + log(severity, domain, "TLS error while %s%s%s: %s (in %s:%s:%s)", doing, addr?" with ":"", addr?addr:"", - msg, lib, func); + msg, lib, func, state); } else { - log(severity, LD_NET, "TLS error%s%s: %s (in %s:%s)", + log(severity, domain, "TLS error%s%s: %s (in %s:%s:%s)", addr?" with ":"", addr?addr:"", - msg, lib, func); + msg, lib, func, state); } } } @@ -265,7 +286,7 @@ tor_tls_err_to_string(int err) */ static int tor_tls_get_error(tor_tls_t *tls, int r, int extra, - const char *doing, int severity) + const char *doing, int severity, int domain) { int err = SSL_get_error(tls->ssl, r); int tor_error = TOR_TLS_ERROR_MISC; @@ -280,25 +301,28 @@ tor_tls_get_error(tor_tls_t *tls, int r, int extra, if (extra&CATCH_SYSCALL) return _TOR_TLS_SYSCALL; if (r == 0) { - log(severity, LD_NET, "TLS error: unexpected close while %s", doing); + log(severity, LD_NET, "TLS error: unexpected close while %s (%s)", + doing, ssl_state_to_string(tls->ssl->state)); tor_error = TOR_TLS_ERROR_IO; } else { int e = tor_socket_errno(tls->socket); log(severity, LD_NET, - "TLS error: <syscall error while %s> (errno=%d: %s)", - doing, e, tor_socket_strerror(e)); + "TLS error: <syscall error while %s> (errno=%d: %s; state=%s)", + doing, e, tor_socket_strerror(e), + ssl_state_to_string(tls->ssl->state)); tor_error = tor_errno_to_tls_error(e); } - tls_log_errors(tls, severity, doing); + tls_log_errors(tls, severity, domain, doing); return tor_error; case SSL_ERROR_ZERO_RETURN: if (extra&CATCH_ZERO) return _TOR_TLS_ZERORETURN; - log(severity, LD_NET, "TLS connection closed while %s", doing); - tls_log_errors(tls, severity, doing); + log(severity, LD_NET, "TLS connection closed while %s in state %s", + doing, ssl_state_to_string(tls->ssl->state)); + tls_log_errors(tls, severity, domain, doing); return TOR_TLS_CLOSE; default: - tls_log_errors(tls, severity, doing); + tls_log_errors(tls, severity, domain, doing); return TOR_TLS_ERROR_MISC; } } @@ -311,7 +335,6 @@ tor_tls_init(void) if (!tls_library_is_initialized) { SSL_library_init(); SSL_load_error_strings(); - crypto_global_init(-1); tls_library_is_initialized = 1; } } @@ -431,7 +454,7 @@ tor_tls_create_certificate(crypto_pk_env_t *rsa, x509 = NULL; } done: - tls_log_errors(NULL, LOG_WARN, "generating certificate"); + tls_log_errors(NULL, LOG_WARN, LD_NET, "generating certificate"); if (sign_pkey) EVP_PKEY_free(sign_pkey); if (pkey) @@ -631,7 +654,7 @@ tor_tls_context_new(crypto_pk_env_t *identity, unsigned int key_lifetime) return 0; error: - tls_log_errors(NULL, LOG_WARN, "creating TLS context"); + tls_log_errors(NULL, LOG_WARN, LD_NET, "creating TLS context"); tor_free(nickname); tor_free(nn2); if (pkey) @@ -827,7 +850,7 @@ tor_tls_new(int sock, int isServer) tor_assert(global_tls_context); /* make sure somebody made it first */ if (!(result->ssl = SSL_new(global_tls_context->ctx))) { - tls_log_errors(NULL, LOG_WARN, "generating TLS context"); + tls_log_errors(NULL, LOG_WARN, LD_NET, "generating TLS context"); tor_free(result); return NULL; } @@ -843,7 +866,7 @@ tor_tls_new(int sock, int isServer) if (!SSL_set_cipher_list(result->ssl, isServer ? SERVER_CIPHER_LIST : CLIENT_CIPHER_LIST)) { - tls_log_errors(NULL, LOG_WARN, "setting ciphers"); + tls_log_errors(NULL, LOG_WARN, LD_NET, "setting ciphers"); #ifdef SSL_set_tlsext_host_name SSL_set_tlsext_host_name(result->ssl, NULL); #endif @@ -856,7 +879,7 @@ tor_tls_new(int sock, int isServer) result->socket = sock; bio = BIO_new_socket(sock, BIO_NOCLOSE); if (! bio) { - tls_log_errors(NULL, LOG_WARN, "opening BIO"); + tls_log_errors(NULL, LOG_WARN, LD_NET, "opening BIO"); #ifdef SSL_set_tlsext_host_name SSL_set_tlsext_host_name(result->ssl, NULL); #endif @@ -883,7 +906,7 @@ tor_tls_new(int sock, int isServer) } #endif /* Not expected to get called. */ - tls_log_errors(NULL, LOG_WARN, "generating TLS context"); + tls_log_errors(NULL, LOG_WARN, LD_NET, "generating TLS context"); return result; } @@ -965,7 +988,9 @@ void tor_tls_free(tor_tls_t *tls) { tor_tls_t *removed; - tor_assert(tls && tls->ssl); + if (!tls) + return; + tor_assert(tls->ssl); removed = HT_REMOVE(tlsmap, &tlsmap_root, tls); if (!removed) { log_warn(LD_BUG, "Freeing a TLS that was not in the ssl->tls map."); @@ -1008,7 +1033,7 @@ tor_tls_read(tor_tls_t *tls, char *cp, size_t len) #endif return r; } - err = tor_tls_get_error(tls, r, CATCH_ZERO, "reading", LOG_DEBUG); + err = tor_tls_get_error(tls, r, CATCH_ZERO, "reading", LOG_DEBUG, LD_NET); if (err == _TOR_TLS_ZERORETURN || err == TOR_TLS_CLOSE) { log_debug(LD_NET,"read returned r=%d; TLS is closed",r); tls->state = TOR_TLS_ST_CLOSED; @@ -1044,7 +1069,7 @@ tor_tls_write(tor_tls_t *tls, const char *cp, size_t n) tls->wantwrite_n = 0; } r = SSL_write(tls->ssl, cp, (int)n); - err = tor_tls_get_error(tls, r, 0, "writing", LOG_INFO); + err = tor_tls_get_error(tls, r, 0, "writing", LOG_INFO, LD_NET); if (err == TOR_TLS_DONE) { return r; } @@ -1062,21 +1087,30 @@ int tor_tls_handshake(tor_tls_t *tls) { int r; + int oldstate; tor_assert(tls); tor_assert(tls->ssl); tor_assert(tls->state == TOR_TLS_ST_HANDSHAKE); check_no_tls_errors(); + oldstate = tls->ssl->state; if (tls->isServer) { + log_debug(LD_HANDSHAKE, "About to call SSL_accept on %p (%s)", tls, + ssl_state_to_string(tls->ssl->state)); r = SSL_accept(tls->ssl); } else { + log_debug(LD_HANDSHAKE, "About to call SSL_connect on %p (%s)", tls, + ssl_state_to_string(tls->ssl->state)); r = SSL_connect(tls->ssl); } + if (oldstate != tls->ssl->state) + log_debug(LD_HANDSHAKE, "After call, %p was in state %s", + tls, ssl_state_to_string(tls->ssl->state)); /* We need to call this here and not earlier, since OpenSSL has a penchant * for clearing its flags when you say accept or connect. */ tor_tls_unblock_renegotiation(tls); - r = tor_tls_get_error(tls,r,0, "handshaking", LOG_INFO); + r = tor_tls_get_error(tls,r,0, "handshaking", LOG_INFO, LD_HANDSHAKE); if (ERR_peek_error() != 0) { - tls_log_errors(tls, tls->isServer ? LOG_INFO : LOG_WARN, + tls_log_errors(tls, tls->isServer ? LOG_INFO : LOG_WARN, LD_HANDSHAKE, "handshaking"); return TOR_TLS_ERROR_MISC; } @@ -1097,7 +1131,8 @@ tor_tls_handshake(tor_tls_t *tls) " get set. Fixing that."); } tls->wasV2Handshake = 1; - log_debug(LD_NET, "Completed V2 TLS handshake with client; waiting " + log_debug(LD_HANDSHAKE, + "Completed V2 TLS handshake with client; waiting " "for renegotiation."); } else { tls->wasV2Handshake = 0; @@ -1109,10 +1144,13 @@ tor_tls_handshake(tor_tls_t *tls) X509 *cert = SSL_get_peer_certificate(tls->ssl); STACK_OF(X509) *chain = SSL_get_peer_cert_chain(tls->ssl); int n_certs = sk_X509_num(chain); - if (n_certs > 1 || (n_certs == 1 && cert != sk_X509_value(chain, 0))) + if (n_certs > 1 || (n_certs == 1 && cert != sk_X509_value(chain, 0))) { + log_debug(LD_HANDSHAKE, "Server sent back multiple certificates; it " + "looks like a v1 handshake on %p", tls); tls->wasV2Handshake = 0; - else { - log_debug(LD_NET, "Server sent back a single certificate; looks like " + } else { + log_debug(LD_HANDSHAKE, + "Server sent back a single certificate; looks like " "a v2 handshake on %p.", tls); tls->wasV2Handshake = 1; } @@ -1120,7 +1158,7 @@ tor_tls_handshake(tor_tls_t *tls) X509_free(cert); #endif if (SSL_set_cipher_list(tls->ssl, SERVER_CIPHER_LIST) == 0) { - tls_log_errors(NULL, LOG_WARN, "re-setting ciphers"); + tls_log_errors(NULL, LOG_WARN, LD_HANDSHAKE, "re-setting ciphers"); r = TOR_TLS_ERROR_MISC; } } @@ -1143,7 +1181,8 @@ tor_tls_renegotiate(tor_tls_t *tls) if (tls->state != TOR_TLS_ST_RENEGOTIATE) { int r = SSL_renegotiate(tls->ssl); if (r <= 0) { - return tor_tls_get_error(tls, r, 0, "renegotiating", LOG_WARN); + return tor_tls_get_error(tls, r, 0, "renegotiating", LOG_WARN, + LD_HANDSHAKE); } tls->state = TOR_TLS_ST_RENEGOTIATE; } @@ -1152,7 +1191,8 @@ tor_tls_renegotiate(tor_tls_t *tls) tls->state = TOR_TLS_ST_OPEN; return TOR_TLS_DONE; } else - return tor_tls_get_error(tls, r, 0, "renegotiating handshake", LOG_INFO); + return tor_tls_get_error(tls, r, 0, "renegotiating handshake", LOG_INFO, + LD_HANDSHAKE); } /** Shut down an open tls connection <b>tls</b>. When finished, returns @@ -1176,7 +1216,7 @@ tor_tls_shutdown(tor_tls_t *tls) r = SSL_read(tls->ssl, buf, 128); } while (r>0); err = tor_tls_get_error(tls, r, CATCH_ZERO, "reading to shut down", - LOG_INFO); + LOG_INFO, LD_NET); if (err == _TOR_TLS_ZERORETURN) { tls->state = TOR_TLS_ST_GOTCLOSE; /* fall through... */ @@ -1192,7 +1232,7 @@ tor_tls_shutdown(tor_tls_t *tls) return TOR_TLS_DONE; } err = tor_tls_get_error(tls, r, CATCH_SYSCALL|CATCH_ZERO, "shutting down", - LOG_INFO); + LOG_INFO, LD_NET); if (err == _TOR_TLS_SYSCALL) { /* The underlying TCP connection closed while we were shutting down. */ tls->state = TOR_TLS_ST_CLOSED; @@ -1224,7 +1264,7 @@ tor_tls_peer_has_cert(tor_tls_t *tls) { X509 *cert; cert = SSL_get_peer_certificate(tls->ssl); - tls_log_errors(tls, LOG_WARN, "getting peer certificate"); + tls_log_errors(tls, LOG_WARN, LD_HANDSHAKE, "getting peer certificate"); if (!cert) return 0; X509_free(cert); @@ -1251,7 +1291,7 @@ log_cert_lifetime(X509 *cert, const char *problem) log_warn(LD_GENERAL, "Couldn't allocate BIO!"); goto end; } if (!(ASN1_TIME_print(bio, X509_get_notBefore(cert)))) { - tls_log_errors(NULL, LOG_WARN, "printing certificate lifetime"); + tls_log_errors(NULL, LOG_WARN, LD_NET, "printing certificate lifetime"); goto end; } BIO_get_mem_ptr(bio, &buf); @@ -1259,7 +1299,7 @@ log_cert_lifetime(X509 *cert, const char *problem) (void)BIO_reset(bio); if (!(ASN1_TIME_print(bio, X509_get_notAfter(cert)))) { - tls_log_errors(NULL, LOG_WARN, "printing certificate lifetime"); + tls_log_errors(NULL, LOG_WARN, LD_NET, "printing certificate lifetime"); goto end; } BIO_get_mem_ptr(bio, &buf); @@ -1273,13 +1313,11 @@ log_cert_lifetime(X509 *cert, const char *problem) end: /* Not expected to get invoked */ - tls_log_errors(NULL, LOG_WARN, "getting certificate lifetime"); + tls_log_errors(NULL, LOG_WARN, LD_NET, "getting certificate lifetime"); if (bio) BIO_free(bio); - if (s1) - tor_free(s1); - if (s2) - tor_free(s2); + tor_free(s1); + tor_free(s2); } /** Helper function: try to extract a link certificate and an identity @@ -1347,7 +1385,7 @@ tor_tls_verify(int severity, tor_tls_t *tls, crypto_pk_env_t **identity_key) if (!(id_pkey = X509_get_pubkey(id_cert)) || X509_verify(cert, id_pkey) <= 0) { log_fn(severity,LD_PROTOCOL,"X509_verify on cert and pkey returned <= 0"); - tls_log_errors(tls, severity,"verifying certificate"); + tls_log_errors(tls, severity, LD_HANDSHAKE, "verifying certificate"); goto done; } @@ -1366,7 +1404,7 @@ tor_tls_verify(int severity, tor_tls_t *tls, crypto_pk_env_t **identity_key) /* This should never get invoked, but let's make sure in case OpenSSL * acts unexpectedly. */ - tls_log_errors(tls, LOG_WARN, "finishing tor_tls_verify"); + tls_log_errors(tls, LOG_WARN, LD_HANDSHAKE, "finishing tor_tls_verify"); return r; } @@ -1405,7 +1443,7 @@ tor_tls_check_lifetime(tor_tls_t *tls, int tolerance) if (cert) X509_free(cert); /* Not expected to get invoked */ - tls_log_errors(tls, LOG_WARN, "checking certificate lifetime"); + tls_log_errors(tls, LOG_WARN, LD_NET, "checking certificate lifetime"); return r; } @@ -1473,7 +1511,7 @@ _check_no_tls_errors(const char *fname, int line) return; log(LOG_WARN, LD_CRYPTO, "Unhandled OpenSSL errors found at %s:%d: ", tor_fix_source_file(fname), line); - tls_log_errors(NULL, LOG_WARN, NULL); + tls_log_errors(NULL, LOG_WARN, LD_NET, NULL); } /** Return true iff the initial TLS connection at <b>tls</b> did not use a v2 diff --git a/src/common/tortls_states.h b/src/common/tortls_states.h new file mode 100644 index 0000000000..986b5a8a0d --- /dev/null +++ b/src/common/tortls_states.h @@ -0,0 +1,414 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2009, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/* Helper file: included only in tortls.c */ + +#ifndef _TORTLS_STATES_H +#define _TORTLS_STATES_H + +/* The main body of this file was mechanically generated with this + perl script: + + my %keys = (); + for $fn (@ARGV) { + open(F, $fn); + while (<F>) { + next unless /^#define ((?:SSL|DTLS)\w*_ST_\w*)/; + $keys{$1} = 1; + } + close(F); + } + for $k (sort keys %keys) { + print "#ifdef $k\n S($k),\n#endif\n" + } +*/ + +/** Mapping from allowed value of SSL.state to the name of C macro for that + * state. Used for debugging an openssl connection. */ +static const struct { int state; const char *name; } state_map[] = { +#define S(state) { state, #state } +#ifdef DTLS1_ST_CR_HELLO_VERIFY_REQUEST_A + S(DTLS1_ST_CR_HELLO_VERIFY_REQUEST_A), +#endif +#ifdef DTLS1_ST_CR_HELLO_VERIFY_REQUEST_B + S(DTLS1_ST_CR_HELLO_VERIFY_REQUEST_B), +#endif +#ifdef DTLS1_ST_SW_HELLO_VERIFY_REQUEST_A + S(DTLS1_ST_SW_HELLO_VERIFY_REQUEST_A), +#endif +#ifdef DTLS1_ST_SW_HELLO_VERIFY_REQUEST_B + S(DTLS1_ST_SW_HELLO_VERIFY_REQUEST_B), +#endif +#ifdef SSL23_ST_CR_SRVR_HELLO_A + S(SSL23_ST_CR_SRVR_HELLO_A), +#endif +#ifdef SSL23_ST_CR_SRVR_HELLO_B + S(SSL23_ST_CR_SRVR_HELLO_B), +#endif +#ifdef SSL23_ST_CW_CLNT_HELLO_A + S(SSL23_ST_CW_CLNT_HELLO_A), +#endif +#ifdef SSL23_ST_CW_CLNT_HELLO_B + S(SSL23_ST_CW_CLNT_HELLO_B), +#endif +#ifdef SSL23_ST_SR_CLNT_HELLO_A + S(SSL23_ST_SR_CLNT_HELLO_A), +#endif +#ifdef SSL23_ST_SR_CLNT_HELLO_B + S(SSL23_ST_SR_CLNT_HELLO_B), +#endif +#ifdef SSL2_ST_CLIENT_START_ENCRYPTION + S(SSL2_ST_CLIENT_START_ENCRYPTION), +#endif +#ifdef SSL2_ST_GET_CLIENT_FINISHED_A + S(SSL2_ST_GET_CLIENT_FINISHED_A), +#endif +#ifdef SSL2_ST_GET_CLIENT_FINISHED_B + S(SSL2_ST_GET_CLIENT_FINISHED_B), +#endif +#ifdef SSL2_ST_GET_CLIENT_HELLO_A + S(SSL2_ST_GET_CLIENT_HELLO_A), +#endif +#ifdef SSL2_ST_GET_CLIENT_HELLO_B + S(SSL2_ST_GET_CLIENT_HELLO_B), +#endif +#ifdef SSL2_ST_GET_CLIENT_HELLO_C + S(SSL2_ST_GET_CLIENT_HELLO_C), +#endif +#ifdef SSL2_ST_GET_CLIENT_MASTER_KEY_A + S(SSL2_ST_GET_CLIENT_MASTER_KEY_A), +#endif +#ifdef SSL2_ST_GET_CLIENT_MASTER_KEY_B + S(SSL2_ST_GET_CLIENT_MASTER_KEY_B), +#endif +#ifdef SSL2_ST_GET_SERVER_FINISHED_A + S(SSL2_ST_GET_SERVER_FINISHED_A), +#endif +#ifdef SSL2_ST_GET_SERVER_FINISHED_B + S(SSL2_ST_GET_SERVER_FINISHED_B), +#endif +#ifdef SSL2_ST_GET_SERVER_HELLO_A + S(SSL2_ST_GET_SERVER_HELLO_A), +#endif +#ifdef SSL2_ST_GET_SERVER_HELLO_B + S(SSL2_ST_GET_SERVER_HELLO_B), +#endif +#ifdef SSL2_ST_GET_SERVER_VERIFY_A + S(SSL2_ST_GET_SERVER_VERIFY_A), +#endif +#ifdef SSL2_ST_GET_SERVER_VERIFY_B + S(SSL2_ST_GET_SERVER_VERIFY_B), +#endif +#ifdef SSL2_ST_SEND_CLIENT_CERTIFICATE_A + S(SSL2_ST_SEND_CLIENT_CERTIFICATE_A), +#endif +#ifdef SSL2_ST_SEND_CLIENT_CERTIFICATE_B + S(SSL2_ST_SEND_CLIENT_CERTIFICATE_B), +#endif +#ifdef SSL2_ST_SEND_CLIENT_CERTIFICATE_C + S(SSL2_ST_SEND_CLIENT_CERTIFICATE_C), +#endif +#ifdef SSL2_ST_SEND_CLIENT_CERTIFICATE_D + S(SSL2_ST_SEND_CLIENT_CERTIFICATE_D), +#endif +#ifdef SSL2_ST_SEND_CLIENT_FINISHED_A + S(SSL2_ST_SEND_CLIENT_FINISHED_A), +#endif +#ifdef SSL2_ST_SEND_CLIENT_FINISHED_B + S(SSL2_ST_SEND_CLIENT_FINISHED_B), +#endif +#ifdef SSL2_ST_SEND_CLIENT_HELLO_A + S(SSL2_ST_SEND_CLIENT_HELLO_A), +#endif +#ifdef SSL2_ST_SEND_CLIENT_HELLO_B + S(SSL2_ST_SEND_CLIENT_HELLO_B), +#endif +#ifdef SSL2_ST_SEND_CLIENT_MASTER_KEY_A + S(SSL2_ST_SEND_CLIENT_MASTER_KEY_A), +#endif +#ifdef SSL2_ST_SEND_CLIENT_MASTER_KEY_B + S(SSL2_ST_SEND_CLIENT_MASTER_KEY_B), +#endif +#ifdef SSL2_ST_SEND_REQUEST_CERTIFICATE_A + S(SSL2_ST_SEND_REQUEST_CERTIFICATE_A), +#endif +#ifdef SSL2_ST_SEND_REQUEST_CERTIFICATE_B + S(SSL2_ST_SEND_REQUEST_CERTIFICATE_B), +#endif +#ifdef SSL2_ST_SEND_REQUEST_CERTIFICATE_C + S(SSL2_ST_SEND_REQUEST_CERTIFICATE_C), +#endif +#ifdef SSL2_ST_SEND_REQUEST_CERTIFICATE_D + S(SSL2_ST_SEND_REQUEST_CERTIFICATE_D), +#endif +#ifdef SSL2_ST_SEND_SERVER_FINISHED_A + S(SSL2_ST_SEND_SERVER_FINISHED_A), +#endif +#ifdef SSL2_ST_SEND_SERVER_FINISHED_B + S(SSL2_ST_SEND_SERVER_FINISHED_B), +#endif +#ifdef SSL2_ST_SEND_SERVER_HELLO_A + S(SSL2_ST_SEND_SERVER_HELLO_A), +#endif +#ifdef SSL2_ST_SEND_SERVER_HELLO_B + S(SSL2_ST_SEND_SERVER_HELLO_B), +#endif +#ifdef SSL2_ST_SEND_SERVER_VERIFY_A + S(SSL2_ST_SEND_SERVER_VERIFY_A), +#endif +#ifdef SSL2_ST_SEND_SERVER_VERIFY_B + S(SSL2_ST_SEND_SERVER_VERIFY_B), +#endif +#ifdef SSL2_ST_SEND_SERVER_VERIFY_C + S(SSL2_ST_SEND_SERVER_VERIFY_C), +#endif +#ifdef SSL2_ST_SERVER_START_ENCRYPTION + S(SSL2_ST_SERVER_START_ENCRYPTION), +#endif +#ifdef SSL2_ST_X509_GET_CLIENT_CERTIFICATE + S(SSL2_ST_X509_GET_CLIENT_CERTIFICATE), +#endif +#ifdef SSL2_ST_X509_GET_SERVER_CERTIFICATE + S(SSL2_ST_X509_GET_SERVER_CERTIFICATE), +#endif +#ifdef SSL3_ST_CR_CERT_A + S(SSL3_ST_CR_CERT_A), +#endif +#ifdef SSL3_ST_CR_CERT_B + S(SSL3_ST_CR_CERT_B), +#endif +#ifdef SSL3_ST_CR_CERT_REQ_A + S(SSL3_ST_CR_CERT_REQ_A), +#endif +#ifdef SSL3_ST_CR_CERT_REQ_B + S(SSL3_ST_CR_CERT_REQ_B), +#endif +#ifdef SSL3_ST_CR_CERT_STATUS_A + S(SSL3_ST_CR_CERT_STATUS_A), +#endif +#ifdef SSL3_ST_CR_CERT_STATUS_B + S(SSL3_ST_CR_CERT_STATUS_B), +#endif +#ifdef SSL3_ST_CR_CHANGE_A + S(SSL3_ST_CR_CHANGE_A), +#endif +#ifdef SSL3_ST_CR_CHANGE_B + S(SSL3_ST_CR_CHANGE_B), +#endif +#ifdef SSL3_ST_CR_FINISHED_A + S(SSL3_ST_CR_FINISHED_A), +#endif +#ifdef SSL3_ST_CR_FINISHED_B + S(SSL3_ST_CR_FINISHED_B), +#endif +#ifdef SSL3_ST_CR_KEY_EXCH_A + S(SSL3_ST_CR_KEY_EXCH_A), +#endif +#ifdef SSL3_ST_CR_KEY_EXCH_B + S(SSL3_ST_CR_KEY_EXCH_B), +#endif +#ifdef SSL3_ST_CR_SESSION_TICKET_A + S(SSL3_ST_CR_SESSION_TICKET_A), +#endif +#ifdef SSL3_ST_CR_SESSION_TICKET_B + S(SSL3_ST_CR_SESSION_TICKET_B), +#endif +#ifdef SSL3_ST_CR_SRVR_DONE_A + S(SSL3_ST_CR_SRVR_DONE_A), +#endif +#ifdef SSL3_ST_CR_SRVR_DONE_B + S(SSL3_ST_CR_SRVR_DONE_B), +#endif +#ifdef SSL3_ST_CR_SRVR_HELLO_A + S(SSL3_ST_CR_SRVR_HELLO_A), +#endif +#ifdef SSL3_ST_CR_SRVR_HELLO_B + S(SSL3_ST_CR_SRVR_HELLO_B), +#endif +#ifdef SSL3_ST_CW_CERT_A + S(SSL3_ST_CW_CERT_A), +#endif +#ifdef SSL3_ST_CW_CERT_B + S(SSL3_ST_CW_CERT_B), +#endif +#ifdef SSL3_ST_CW_CERT_C + S(SSL3_ST_CW_CERT_C), +#endif +#ifdef SSL3_ST_CW_CERT_D + S(SSL3_ST_CW_CERT_D), +#endif +#ifdef SSL3_ST_CW_CERT_VRFY_A + S(SSL3_ST_CW_CERT_VRFY_A), +#endif +#ifdef SSL3_ST_CW_CERT_VRFY_B + S(SSL3_ST_CW_CERT_VRFY_B), +#endif +#ifdef SSL3_ST_CW_CHANGE_A + S(SSL3_ST_CW_CHANGE_A), +#endif +#ifdef SSL3_ST_CW_CHANGE_B + S(SSL3_ST_CW_CHANGE_B), +#endif +#ifdef SSL3_ST_CW_CLNT_HELLO_A + S(SSL3_ST_CW_CLNT_HELLO_A), +#endif +#ifdef SSL3_ST_CW_CLNT_HELLO_B + S(SSL3_ST_CW_CLNT_HELLO_B), +#endif +#ifdef SSL3_ST_CW_FINISHED_A + S(SSL3_ST_CW_FINISHED_A), +#endif +#ifdef SSL3_ST_CW_FINISHED_B + S(SSL3_ST_CW_FINISHED_B), +#endif +#ifdef SSL3_ST_CW_FLUSH + S(SSL3_ST_CW_FLUSH), +#endif +#ifdef SSL3_ST_CW_KEY_EXCH_A + S(SSL3_ST_CW_KEY_EXCH_A), +#endif +#ifdef SSL3_ST_CW_KEY_EXCH_B + S(SSL3_ST_CW_KEY_EXCH_B), +#endif +#ifdef SSL3_ST_SR_CERT_A + S(SSL3_ST_SR_CERT_A), +#endif +#ifdef SSL3_ST_SR_CERT_B + S(SSL3_ST_SR_CERT_B), +#endif +#ifdef SSL3_ST_SR_CERT_VRFY_A + S(SSL3_ST_SR_CERT_VRFY_A), +#endif +#ifdef SSL3_ST_SR_CERT_VRFY_B + S(SSL3_ST_SR_CERT_VRFY_B), +#endif +#ifdef SSL3_ST_SR_CHANGE_A + S(SSL3_ST_SR_CHANGE_A), +#endif +#ifdef SSL3_ST_SR_CHANGE_B + S(SSL3_ST_SR_CHANGE_B), +#endif +#ifdef SSL3_ST_SR_CLNT_HELLO_A + S(SSL3_ST_SR_CLNT_HELLO_A), +#endif +#ifdef SSL3_ST_SR_CLNT_HELLO_B + S(SSL3_ST_SR_CLNT_HELLO_B), +#endif +#ifdef SSL3_ST_SR_CLNT_HELLO_C + S(SSL3_ST_SR_CLNT_HELLO_C), +#endif +#ifdef SSL3_ST_SR_FINISHED_A + S(SSL3_ST_SR_FINISHED_A), +#endif +#ifdef SSL3_ST_SR_FINISHED_B + S(SSL3_ST_SR_FINISHED_B), +#endif +#ifdef SSL3_ST_SR_KEY_EXCH_A + S(SSL3_ST_SR_KEY_EXCH_A), +#endif +#ifdef SSL3_ST_SR_KEY_EXCH_B + S(SSL3_ST_SR_KEY_EXCH_B), +#endif +#ifdef SSL3_ST_SW_CERT_A + S(SSL3_ST_SW_CERT_A), +#endif +#ifdef SSL3_ST_SW_CERT_B + S(SSL3_ST_SW_CERT_B), +#endif +#ifdef SSL3_ST_SW_CERT_REQ_A + S(SSL3_ST_SW_CERT_REQ_A), +#endif +#ifdef SSL3_ST_SW_CERT_REQ_B + S(SSL3_ST_SW_CERT_REQ_B), +#endif +#ifdef SSL3_ST_SW_CERT_STATUS_A + S(SSL3_ST_SW_CERT_STATUS_A), +#endif +#ifdef SSL3_ST_SW_CERT_STATUS_B + S(SSL3_ST_SW_CERT_STATUS_B), +#endif +#ifdef SSL3_ST_SW_CHANGE_A + S(SSL3_ST_SW_CHANGE_A), +#endif +#ifdef SSL3_ST_SW_CHANGE_B + S(SSL3_ST_SW_CHANGE_B), +#endif +#ifdef SSL3_ST_SW_FINISHED_A + S(SSL3_ST_SW_FINISHED_A), +#endif +#ifdef SSL3_ST_SW_FINISHED_B + S(SSL3_ST_SW_FINISHED_B), +#endif +#ifdef SSL3_ST_SW_FLUSH + S(SSL3_ST_SW_FLUSH), +#endif +#ifdef SSL3_ST_SW_HELLO_REQ_A + S(SSL3_ST_SW_HELLO_REQ_A), +#endif +#ifdef SSL3_ST_SW_HELLO_REQ_B + S(SSL3_ST_SW_HELLO_REQ_B), +#endif +#ifdef SSL3_ST_SW_HELLO_REQ_C + S(SSL3_ST_SW_HELLO_REQ_C), +#endif +#ifdef SSL3_ST_SW_KEY_EXCH_A + S(SSL3_ST_SW_KEY_EXCH_A), +#endif +#ifdef SSL3_ST_SW_KEY_EXCH_B + S(SSL3_ST_SW_KEY_EXCH_B), +#endif +#ifdef SSL3_ST_SW_SESSION_TICKET_A + S(SSL3_ST_SW_SESSION_TICKET_A), +#endif +#ifdef SSL3_ST_SW_SESSION_TICKET_B + S(SSL3_ST_SW_SESSION_TICKET_B), +#endif +#ifdef SSL3_ST_SW_SRVR_DONE_A + S(SSL3_ST_SW_SRVR_DONE_A), +#endif +#ifdef SSL3_ST_SW_SRVR_DONE_B + S(SSL3_ST_SW_SRVR_DONE_B), +#endif +#ifdef SSL3_ST_SW_SRVR_HELLO_A + S(SSL3_ST_SW_SRVR_HELLO_A), +#endif +#ifdef SSL3_ST_SW_SRVR_HELLO_B + S(SSL3_ST_SW_SRVR_HELLO_B), +#endif +#ifdef SSL_ST_ACCEPT + S(SSL_ST_ACCEPT), +#endif +#ifdef SSL_ST_BEFORE + S(SSL_ST_BEFORE), +#endif +#ifdef SSL_ST_CONNECT + S(SSL_ST_CONNECT), +#endif +#ifdef SSL_ST_INIT + S(SSL_ST_INIT), +#endif +#ifdef SSL_ST_MASK + S(SSL_ST_MASK), +#endif +#ifdef SSL_ST_OK + S(SSL_ST_OK), +#endif +#ifdef SSL_ST_READ_BODY + S(SSL_ST_READ_BODY), +#endif +#ifdef SSL_ST_READ_DONE + S(SSL_ST_READ_DONE), +#endif +#ifdef SSL_ST_READ_HEADER + S(SSL_ST_READ_HEADER), +#endif +#ifdef SSL_ST_RENEGOTIATE + S(SSL_ST_RENEGOTIATE), +#endif + { 0, NULL } +}; + +#endif + diff --git a/src/common/util.c b/src/common/util.c index 9dcf9fba64..a15af7ed57 100644 --- a/src/common/util.c +++ b/src/common/util.c @@ -16,6 +16,7 @@ #include "orconfig.h" #include "util.h" #include "log.h" +#undef log #include "crypto.h" #include "torint.h" #include "container.h" @@ -30,6 +31,11 @@ #include <pwd.h> #endif +/* math.h needs this on Linux */ +#ifndef __USE_ISOC99 +#define __USE_ISOC99 1 +#endif +#include <math.h> #include <stdlib.h> #include <stdio.h> #include <string.h> @@ -278,7 +284,7 @@ tor_log_mallinfo(int severity) struct mallinfo mi; memset(&mi, 0, sizeof(mi)); mi = mallinfo(); - log(severity, LD_MM, + tor_log(severity, LD_MM, "mallinfo() said: arena=%d, ordblks=%d, smblks=%d, hblks=%d, " "hblkhd=%d, usmblks=%d, fsmblks=%d, uordblks=%d, fordblks=%d, " "keepcost=%d", @@ -301,6 +307,25 @@ tor_log_mallinfo(int severity) * Math * ===== */ +/** + * Returns the natural logarithm of d base 2. We define this wrapper here so + * as to make it easier not to conflict with Tor's log() macro. + */ +double +tor_mathlog(double d) +{ + return log(d); +} + +/** Return the long integer closest to d. We define this wrapper here so + * that not all users of math.h need to use the right incancations to get + * the c99 functions. */ +long +tor_lround(double d) +{ + return lround(d); +} + /** Returns floor(log2(u64)). If u64 is 0, (incorrectly) returns 0. */ int tor_log2(uint64_t u64) @@ -345,6 +370,36 @@ round_to_power_of_2(uint64_t u64) return low; } +/** Return the lowest x such that x is at least <b>number</b>, and x modulo + * <b>divisor</b> == 0. */ +unsigned +round_to_next_multiple_of(unsigned number, unsigned divisor) +{ + number += divisor - 1; + number -= number % divisor; + return number; +} + +/** Return the lowest x such that x is at least <b>number</b>, and x modulo + * <b>divisor</b> == 0. */ +uint32_t +round_uint32_to_next_multiple_of(uint32_t number, uint32_t divisor) +{ + number += divisor - 1; + number -= number % divisor; + return number; +} + +/** Return the lowest x such that x is at least <b>number</b>, and x modulo + * <b>divisor</b> == 0. */ +uint64_t +round_uint64_to_next_multiple_of(uint64_t number, uint64_t divisor) +{ + number += divisor - 1; + number -= number % divisor; + return number; +} + /* ===== * String manipulation * ===== */ @@ -627,6 +682,29 @@ find_whitespace_eos(const char *s, const char *eos) return s; } +/** Return the first occurrence of <b>needle</b> in <b>haystack</b> that + * occurs at the start of a line (that is, at the beginning of <b>haystack</b> + * or immediately after a newline). Return NULL if no such string is found. + */ +const char * +find_str_at_start_of_line(const char *haystack, const char *needle) +{ + size_t needle_len = strlen(needle); + + do { + if (!strncmp(haystack, needle, needle_len)) + return haystack; + + haystack = strchr(haystack, '\n'); + if (!haystack) + return NULL; + else + ++haystack; + } while (*haystack); + + return NULL; +} + /** Return true iff the 'len' bytes at 'mem' are all zero. */ int tor_mem_is_zero(const char *mem, size_t len) @@ -654,6 +732,13 @@ tor_digest_is_zero(const char *digest) return tor_mem_is_zero(digest, DIGEST_LEN); } +/** Return true iff the DIGEST256_LEN bytes in digest are all zero. */ +int +tor_digest256_is_zero(const char *digest) +{ + return tor_mem_is_zero(digest, DIGEST256_LEN); +} + /* Helper: common code to check whether the result of a strtol or strtoul or * strtoll is correct. */ #define CHECK_STRTOX_RESULT() \ @@ -705,7 +790,18 @@ tor_parse_ulong(const char *s, int base, unsigned long min, CHECK_STRTOX_RESULT(); } -/** As tor_parse_log, but return a unit64_t. Only base 10 is guaranteed to +/** As tor_parse_long(), but return a double. */ +double +tor_parse_double(const char *s, double min, double max, int *ok, char **next) +{ + char *endptr; + double r; + + r = strtod(s, &endptr); + CHECK_STRTOX_RESULT(); +} + +/** As tor_parse_long, but return a uint64_t. Only base 10 is guaranteed to * work for now. */ uint64_t tor_parse_uint64(const char *s, int base, uint64_t min, @@ -905,8 +1001,7 @@ const char * escaped(const char *s) { static char *_escaped_val = NULL; - if (_escaped_val) - tor_free(_escaped_val); + tor_free(_escaped_val); if (s) _escaped_val = esc_for_log(s); @@ -993,6 +1088,42 @@ wrap_string(smartlist_t *out, const char *string, size_t width, * Time * ===== */ +/** + * Converts struct timeval to a double value. + * Preserves microsecond precision, but just barely. + * Error is approx +/- 0.1 usec when dealing with epoch values. + */ +double +tv_to_double(const struct timeval *tv) +{ + double conv = tv->tv_sec; + conv += tv->tv_usec/1000000.0; + return conv; +} + +/** + * Converts timeval to milliseconds. + */ +int64_t +tv_to_msec(const struct timeval *tv) +{ + int64_t conv = ((int64_t)tv->tv_sec)*1000L; + /* Round ghetto-style */ + conv += ((int64_t)tv->tv_usec+500)/1000L; + return conv; +} + +/** + * Converts timeval to microseconds. + */ +int64_t +tv_to_usec(const struct timeval *tv) +{ + int64_t conv = ((int64_t)tv->tv_sec)*1000000L; + conv += tv->tv_usec; + return conv; +} + /** Return the number of microseconds elapsed between *start and *end. */ long @@ -1002,7 +1133,8 @@ tv_udiff(const struct timeval *start, const struct timeval *end) long secdiff = end->tv_sec - start->tv_sec; if (labs(secdiff+1) > LONG_MAX/1000000) { - log_warn(LD_GENERAL, "comparing times too far apart."); + log_warn(LD_GENERAL, "comparing times on microsecond detail too far " + "apart: %ld seconds", secdiff); return LONG_MAX; } @@ -1010,6 +1142,26 @@ tv_udiff(const struct timeval *start, const struct timeval *end) return udiff; } +/** Return the number of milliseconds elapsed between *start and *end. + */ +long +tv_mdiff(const struct timeval *start, const struct timeval *end) +{ + long mdiff; + long secdiff = end->tv_sec - start->tv_sec; + + if (labs(secdiff+1) > LONG_MAX/1000) { + log_warn(LD_GENERAL, "comparing times on millisecond detail too far " + "apart: %ld seconds", secdiff); + return LONG_MAX; + } + + /* Subtract and round */ + mdiff = secdiff*1000L + + ((long)end->tv_usec - (long)start->tv_usec + 500L) / 1000L; + return mdiff; +} + /** Yield true iff <b>y</b> is a leap-year. */ #define IS_LEAPYEAR(y) (!(y % 4) && ((y % 100) || !(y % 400))) /** Helper: Return the number of leap-days between Jan 1, y1 and Jan 1, y2. */ @@ -1085,7 +1237,7 @@ format_rfc1123_time(char *buf, time_t t) memcpy(buf+8, MONTH_NAMES[tm.tm_mon], 3); } -/** Parse the the RFC1123 encoding of some time (in GMT) from <b>buf</b>, +/** Parse the RFC1123 encoding of some time (in GMT) from <b>buf</b>, * and store the result in *<b>t</b>. * * Return 0 on success, -1 on failure. @@ -1546,12 +1698,12 @@ check_private_dir(const char *dirname, cpd_check_t check) tor_free(f); if (r) { if (errno != ENOENT) { - log(LOG_WARN, LD_FS, "Directory %s cannot be read: %s", dirname, - strerror(errno)); + log_warn(LD_FS, "Directory %s cannot be read: %s", dirname, + strerror(errno)); return -1; } if (check == CPD_NONE) { - log(LOG_WARN, LD_FS, "Directory %s does not exist.", dirname); + log_warn(LD_FS, "Directory %s does not exist.", dirname); return -1; } else if (check == CPD_CREATE) { log_info(LD_GENERAL, "Creating directory %s", dirname); @@ -1561,7 +1713,7 @@ check_private_dir(const char *dirname, cpd_check_t check) r = mkdir(dirname, 0700); #endif if (r) { - log(LOG_WARN, LD_FS, "Error creating directory %s: %s", dirname, + log_warn(LD_FS, "Error creating directory %s: %s", dirname, strerror(errno)); return -1; } @@ -1571,7 +1723,7 @@ check_private_dir(const char *dirname, cpd_check_t check) return 0; } if (!(st.st_mode & S_IFDIR)) { - log(LOG_WARN, LD_FS, "%s is not a directory", dirname); + log_warn(LD_FS, "%s is not a directory", dirname); return -1; } #ifndef MS_WINDOWS @@ -1584,7 +1736,7 @@ check_private_dir(const char *dirname, cpd_check_t check) pw = getpwuid(st.st_uid); - log(LOG_WARN, LD_FS, "%s is not owned by this user (%s, %d) but by " + log_warn(LD_FS, "%s is not owned by this user (%s, %d) but by " "%s (%d). Perhaps you are running Tor as the wrong user?", dirname, process_ownername, (int)getuid(), pw ? pw->pw_name : "<unknown>", (int)st.st_uid); @@ -1593,9 +1745,9 @@ check_private_dir(const char *dirname, cpd_check_t check) return -1; } if (st.st_mode & 0077) { - log(LOG_WARN, LD_FS, "Fixing permissions on directory %s", dirname); + log_warn(LD_FS, "Fixing permissions on directory %s", dirname); if (chmod(dirname, 0700)) { - log(LOG_WARN, LD_FS, "Could not chmod directory %s: %s", dirname, + log_warn(LD_FS, "Could not chmod directory %s: %s", dirname, strerror(errno)); return -1; } else { @@ -1626,12 +1778,13 @@ write_str_to_file(const char *fname, const char *str, int bin) } /** Represents a file that we're writing to, with support for atomic commit: - * we can write into a a temporary file, and either remove the file on + * we can write into a temporary file, and either remove the file on * failure, or replace the original file on success. */ struct open_file_t { char *tempname; /**< Name of the temporary file. */ char *filename; /**< Name of the original file. */ - int rename_on_close; /**< Are we using the temporary file or not? */ + unsigned rename_on_close:1; /**< Are we using the temporary file or not? */ + unsigned binary:1; /**< Did we open in binary mode? */ int fd; /**< fd for the open file. */ FILE *stdio_file; /**< stdio wrapper for <b>fd</b>. */ }; @@ -1679,7 +1832,7 @@ start_writing_to_file(const char *fname, int open_flags, int mode, } else { open_name = new_file->tempname = tor_malloc(tempname_len); if (tor_snprintf(new_file->tempname, tempname_len, "%s.tmp", fname)<0) { - log(LOG_WARN, LD_GENERAL, "Failed to generate filename"); + log_warn(LD_GENERAL, "Failed to generate filename"); goto err; } /* We always replace an existing temporary file if there is one. */ @@ -1687,9 +1840,11 @@ start_writing_to_file(const char *fname, int open_flags, int mode, open_flags &= ~O_EXCL; new_file->rename_on_close = 1; } + if (open_flags & O_BINARY) + new_file->binary = 1; if ((new_file->fd = open(open_name, open_flags, mode)) < 0) { - log(LOG_WARN, LD_FS, "Couldn't open \"%s\" (%s) for writing: %s", + log_warn(LD_FS, "Couldn't open \"%s\" (%s) for writing: %s", open_name, fname, strerror(errno)); goto err; } @@ -1725,7 +1880,8 @@ fdopen_file(open_file_t *file_data) if (file_data->stdio_file) return file_data->stdio_file; tor_assert(file_data->fd >= 0); - if (!(file_data->stdio_file = fdopen(file_data->fd, "a"))) { + if (!(file_data->stdio_file = fdopen(file_data->fd, + file_data->binary?"ab":"a"))) { log_warn(LD_FS, "Couldn't fdopen \"%s\" [%d]: %s", file_data->filename, file_data->fd, strerror(errno)); } @@ -1825,7 +1981,7 @@ write_chunks_to_file_impl(const char *fname, const smartlist_t *chunks, { result = write_all(fd, chunk->bytes, chunk->len, 0); if (result < 0) { - log(LOG_WARN, LD_FS, "Error writing to \"%s\": %s", fname, + log_warn(LD_FS, "Error writing to \"%s\": %s", fname, strerror(errno)); goto err; } @@ -2209,7 +2365,8 @@ expand_filename(const char *filename) #define MAX_SCANF_WIDTH 9999 -/** DOCDOC */ +/** Helper: given an ASCII-encoded decimal digit, return its numeric value. + * NOTE: requires that its input be in-bounds. */ static int digit_to_num(char d) { @@ -2218,7 +2375,10 @@ digit_to_num(char d) return num; } -/** DOCDOC */ +/** Helper: Read an unsigned int from *<b>bufp</b> of up to <b>width</b> + * characters. (Handle arbitrary width if <b>width</b> is less than 0.) On + * success, store the result in <b>out</b>, advance bufp to the next + * character, and return 0. On failure, return -1. */ static int scan_unsigned(const char **bufp, unsigned *out, int width) { @@ -2245,7 +2405,9 @@ scan_unsigned(const char **bufp, unsigned *out, int width) return 0; } -/** DOCDOC */ +/** Helper: copy up to <b>width</b> non-space characters from <b>bufp</b> to + * <b>out</b>. Make sure <b>out</b> is nul-terminated. Advance <b>bufp</b> + * to the next non-space character or the EOS. */ static int scan_string(const char **bufp, char *out, int width) { @@ -2334,7 +2496,12 @@ tor_vsscanf(const char *buf, const char *pattern, va_list ap) * and store the results in the corresponding argument fields. Differs from * sscanf in that it: Only handles %u and %Ns. Does not handle arbitrarily * long widths. %u does not consume any space. Is locale-independent. - * Returns -1 on malformed patterns. */ + * Returns -1 on malformed patterns. + * + * (As with other locale-independent functions, we need this to parse data that + * is in ASCII without worrying that the C library's locale-handling will make + * miscellaneous characters look like numbers, spaces, and so on.) + */ int tor_sscanf(const char *buf, const char *pattern, ...) { diff --git a/src/common/util.h b/src/common/util.h index dca2f86cd1..b55bb91c51 100644 --- a/src/common/util.h +++ b/src/common/util.h @@ -43,7 +43,7 @@ * stderr. */ #define tor_assert(expr) STMT_BEGIN \ if (PREDICT_UNLIKELY(!(expr))) { \ - log(LOG_ERR, LD_BUG, "%s:%d: %s: Assertion %s failed; aborting.", \ + log_err(LD_BUG, "%s:%d: %s: Assertion %s failed; aborting.", \ _SHORT_FILE_, __LINE__, __func__, #expr); \ fprintf(stderr,"%s:%d %s: Assertion %s failed; aborting.\n", \ _SHORT_FILE_, __LINE__, __func__, #expr); \ @@ -152,8 +152,13 @@ void tor_log_mallinfo(int severity); #define bool_neq(a,b) (!(a)!=!(b)) /* Math functions */ +double tor_mathlog(double d) ATTR_CONST; +long tor_lround(double d) ATTR_CONST; int tor_log2(uint64_t u64) ATTR_CONST; uint64_t round_to_power_of_2(uint64_t u64); +unsigned round_to_next_multiple_of(unsigned number, unsigned divisor); +uint32_t round_uint32_to_next_multiple_of(uint32_t number, uint32_t divisor); +uint64_t round_uint64_to_next_multiple_of(uint64_t number, uint64_t divisor); /* String manipulation */ @@ -179,6 +184,8 @@ long tor_parse_long(const char *s, int base, long min, long max, int *ok, char **next); unsigned long tor_parse_ulong(const char *s, int base, unsigned long min, unsigned long max, int *ok, char **next); +double tor_parse_double(const char *s, double min, double max, int *ok, + char **next); uint64_t tor_parse_uint64(const char *s, int base, uint64_t min, uint64_t max, int *ok, char **next); const char *hex_str(const char *from, size_t fromlen) ATTR_NONNULL((1)); @@ -188,8 +195,11 @@ const char *eat_whitespace_no_nl(const char *s) ATTR_PURE; const char *eat_whitespace_eos_no_nl(const char *s, const char *eos) ATTR_PURE; const char *find_whitespace(const char *s) ATTR_PURE; const char *find_whitespace_eos(const char *s, const char *eos) ATTR_PURE; +const char *find_str_at_start_of_line(const char *haystack, const char *needle) + ATTR_PURE; int tor_mem_is_zero(const char *mem, size_t len) ATTR_PURE; int tor_digest_is_zero(const char *digest) ATTR_PURE; +int tor_digest256_is_zero(const char *digest) ATTR_PURE; char *esc_for_log(const char *string) ATTR_MALLOC; const char *escaped(const char *string); struct smartlist_t; @@ -207,7 +217,11 @@ void base16_encode(char *dest, size_t destlen, const char *src, size_t srclen); int base16_decode(char *dest, size_t destlen, const char *src, size_t srclen); /* Time helpers */ +double tv_to_double(const struct timeval *tv); +int64_t tv_to_msec(const struct timeval *tv); +int64_t tv_to_usec(const struct timeval *tv); long tv_udiff(const struct timeval *start, const struct timeval *end); +long tv_mdiff(const struct timeval *start, const struct timeval *end); time_t tor_timegm(struct tm *tm); #define RFC1123_TIME_LEN 29 void format_rfc1123_time(char *buf, time_t t); @@ -294,5 +308,7 @@ void start_daemon(void); void finish_daemon(const char *desired_cwd); void write_pidfile(char *filename); +const char *libor_get_digests(void); + #endif diff --git a/src/common/util_codedigest.c b/src/common/util_codedigest.c new file mode 100644 index 0000000000..88fe508b92 --- /dev/null +++ b/src/common/util_codedigest.c @@ -0,0 +1,11 @@ + +#include "util.h" + +const char * +libor_get_digests(void) +{ + return "" +#include "common_sha1.i" + ; +} + diff --git a/src/config/torrc.complete.in b/src/config/torrc.complete.in index 310458a5c0..6dbec2fbf9 100644 --- a/src/config/torrc.complete.in +++ b/src/config/torrc.complete.in @@ -1,5 +1,3 @@ -# $Id$ -# Last updated on $Date$ #################################################################### ## This config file is divided into four sections. They are: ## 1. Global Options (clients and servers) @@ -81,6 +79,9 @@ #DirServer moria2 v1 18.244.0.114:80 719B E45D E224 B607 C537 07D0 E214 3E2D 423E 74CF #DirServer tor26 v1 86.59.21.38:80 847B 1F85 0344 D787 6491 A548 92F9 0493 4E4E B85D +## Attempt to lock current and future memory pages and effectively disable swap +# DisableAllSwap 0|1 + ## On startup, setgid to this user. #Group GID diff --git a/src/config/torrc.sample.in b/src/config/torrc.sample.in index d0b1ee1591..c3b458543a 100644 --- a/src/config/torrc.sample.in +++ b/src/config/torrc.sample.in @@ -1,5 +1,5 @@ ## Configuration file for a typical Tor user -## Last updated 12 April 2009 for Tor 0.2.1.14-rc. +## Last updated 16 July 2009 for Tor 0.2.2.1-alpha. ## (May or may not work for much older or much newer versions of Tor.) ## ## Lines that begin with "## " try to explain what's going on. Lines @@ -99,6 +99,19 @@ SocksListenAddress 127.0.0.1 # accept connections only from localhost #RelayBandwidthRate 100 KBytes # Throttle traffic to 100KB/s (800Kbps) #RelayBandwidthBurst 200 KBytes # But allow bursts up to 200KB/s (1600Kbps) +## Use these to restrict the maximum traffic per day, week, or month. +## Note that this threshold applies to sent _and_ to received bytes, +## not to their sum: Setting "4 GBytes" may allow up to 8 GBytes +## total before hibernating. +## +## Set a maximum of 4 gigabytes each way per period. +#AccountingMax 4 GBytes +## Each period starts daily at midnight (AccountingMax is per day) +#AccountingStart day 00:00 +## Each period starts on the 3rd of the month at 15:00 (AccountingMax +## is per month) +#AccountingStart month 3 15:00 + ## Contact info to be published in the directory, so we can contact you ## if your relay is misconfigured or something else goes wrong. Google ## indexes this, so spammers might also collect it. diff --git a/src/or/Makefile.am b/src/or/Makefile.am index ad2476ff15..cfa8a035ad 100644 --- a/src/or/Makefile.am +++ b/src/or/Makefile.am @@ -1,8 +1,5 @@ -TESTS = test - -noinst_PROGRAMS = test - bin_PROGRAMS = tor +noinst_LIBRARIES = libtor.a if BUILD_NT_SERVICES tor_platform_source=ntmain.c @@ -10,18 +7,30 @@ else tor_platform_source= endif -EXTRA_DIST=ntmain.c +EXTRA_DIST=ntmain.c or_sha1.i + +if USE_EXTERNAL_EVDNS +evdns_source= +else +evdns_source=eventdns.c +endif -tor_SOURCES = buffers.c circuitbuild.c circuitlist.c \ +libtor_a_SOURCES = buffers.c circuitbuild.c circuitlist.c \ circuituse.c command.c config.c \ connection.c connection_edge.c connection_or.c control.c \ cpuworker.c directory.c dirserv.c dirvote.c \ dns.c dnsserv.c geoip.c hibernate.c main.c $(tor_platform_source) \ + microdesc.c \ networkstatus.c onion.c policies.c \ reasons.c relay.c rendcommon.c rendclient.c rendmid.c \ rendservice.c rephist.c router.c routerlist.c routerparse.c \ - eventdns.c \ - tor_main.c + $(evdns_source) config_codedigest.c + +#libtor_a_LIBADD = ../common/libor.a ../common/libor-crypto.a \ +# ../common/libor-event.a + + +tor_SOURCES = tor_main.c AM_CPPFLAGS = -DSHARE_DATADIR="\"$(datadir)\"" \ -DLOCALSTATEDIR="\"$(localstatedir)\"" \ @@ -32,76 +41,40 @@ AM_CPPFLAGS = -DSHARE_DATADIR="\"$(datadir)\"" \ # matters a lot there, and is quite hard to debug if you forget to do it. tor_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ @TOR_LDFLAGS_libevent@ -tor_LDADD = ../common/libor.a ../common/libor-crypto.a \ - -lz @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ -test_SOURCES = buffers.c circuitbuild.c circuitlist.c \ - circuituse.c command.c config.c \ - connection.c connection_edge.c connection_or.c control.c \ - cpuworker.c directory.c dirserv.c dirvote.c \ - dns.c dnsserv.c geoip.c hibernate.c main.c $(tor_platform_source) \ - networkstatus.c onion.c policies.c \ - reasons.c relay.c rendcommon.c rendclient.c rendmid.c \ - rendservice.c rephist.c router.c routerlist.c routerparse.c \ - eventdns.c \ - test_data.c test.c - -test_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ \ - @TOR_LDFLAGS_libevent@ -test_LDADD = ../common/libor.a ../common/libor-crypto.a \ - -lz @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ +tor_LDADD = ./libtor.a ../common/libor.a ../common/libor-crypto.a \ + ../common/libor-event.a \ + -lz -lm @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ noinst_HEADERS = or.h eventdns.h eventdns_tor.h micro-revision.i +config_codedigest.o: or_sha1.i + tor_main.o: micro-revision.i micro-revision.i: FORCE - @svkdir=$$SVKROOT; \ - if test "x$$svkdir" = x ; then \ - svkdir=$$HOME/.svk; \ - fi; \ - if test -d ../../.git && test -x "`which git 2>&1;true`" ; then \ - if test -d ../../.git/svn && test -x "`which git-svn 2>&1;true`" ; then \ - git-svn info ../../README | \ - sed -n 's/^Revision: \([0-9][0-9]*\).*/"\1"/p' \ - > micro-revision.tmp \ - || true; \ - fi; \ - elif test -d ../../.svn && test -x "`which svn 2>&1;true`" ; then \ - svn info ../.. | \ - sed -n 's/^Revision: \([0-9][0-9]*\).*/"\1"/p' > micro-revision.tmp \ - || true; \ - elif test -x "`which svk 2>&1;true`" && test -d $$svkdir/local; then \ - location=../..; \ - rev=x; \ - while test x$$rev = xx; do \ - x=`svk info $$location | \ - sed -n 's/^Mirrored From:.*, Rev\. \([0-9][0-9]*\)/\1/p'`; \ - if test x$$x != x; then \ - rev=$$x; \ - break; \ - else \ - loc=`svk info $$location | \ - sed -n 's/^Copied From: \(.*\), Rev\. [0-9][0-9]*/\1/p' | \ - head -1`; \ - if test x$$loc = x; then \ - break; \ - else \ - location=/$$loc; \ - fi; \ - fi; \ - done; \ - if test x$$rev != xx; then \ - echo \"$$rev\" > micro-revision.tmp; \ - fi; \ - fi; \ - if test ! -f micro-revision.tmp ; then \ - if test ! -f micro-revision.i ; then \ - echo '""' > micro-revision.i; \ - fi; \ - elif test ! -f micro-revision.i || \ + @rm -f micro-revision.tmp; \ + if test -d ../../.git && test -x "`which git 2>&1;true`"; then \ + HASH="`git rev-parse --short=16 HEAD`"; \ + echo \"$$HASH\" > micro-revision.tmp; \ + fi; \ + if test ! -f micro-revision.tmp ; then \ + if test ! -f micro-revision.i ; then \ + echo '""' > micro-revision.i; \ + fi; \ + elif test ! -f micro-revision.i || \ test x"`cat micro-revision.tmp`" != x"`cat micro-revision.i`"; then \ - mv micro-revision.tmp micro-revision.i; \ + mv micro-revision.tmp micro-revision.i; \ fi; true +or_sha1.i: $(tor_SOURCES) + if test "@SHA1SUM@" != none; then \ + @SHA1SUM@ $(tor_SOURCES) | @SED@ -n 's/^\(.*\)$$/"\1\\n"/p' > or_sha1.i; \ + elif test "@OPENSSL@" != none; then \ + @OPENSSL@ sha1 $(tor_SOURCES) | @SED@ -n 's/SHA1(\(.*\))= \(.*\)/"\2 \1\\n"/p' > or_sha1.i; \ + else \ + rm or_sha1.i; \ + touch or_sha1.i; \ + fi + #Dummy target to ensure that micro-revision.i _always_ gets built. FORCE: diff --git a/src/or/buffers.c b/src/or/buffers.c index ed39bc0f82..c990b6619a 100644 --- a/src/or/buffers.c +++ b/src/or/buffers.c @@ -12,6 +12,8 @@ **/ #define BUFFERS_PRIVATE #include "or.h" +#include "../common/util.h" +#include "../common/log.h" #ifdef HAVE_UNISTD_H #include <unistd.h> #endif @@ -145,10 +147,13 @@ get_freelist(size_t alloc) /** Deallocate a chunk or put it on a freelist */ static void -chunk_free(chunk_t *chunk) +chunk_free_unchecked(chunk_t *chunk) { - size_t alloc = CHUNK_ALLOC_SIZE(chunk->memlen); - chunk_freelist_t *freelist = get_freelist(alloc); + size_t alloc; + chunk_freelist_t *freelist; + + alloc = CHUNK_ALLOC_SIZE(chunk->memlen); + freelist = get_freelist(alloc); if (freelist && freelist->cur_length < freelist->max_length) { chunk->next = freelist->head; freelist->head = chunk; @@ -193,7 +198,7 @@ chunk_new_with_alloc_size(size_t alloc) } #else static void -chunk_free(chunk_t *chunk) +chunk_free_unchecked(chunk_t *chunk) { tor_free(chunk); } @@ -401,7 +406,7 @@ buf_pullup(buf_t *buf, size_t bytes, int nulterminate) dest->next = src->next; if (buf->tail == src) buf->tail = dest; - chunk_free(src); + chunk_free_unchecked(src); } else { memcpy(CHUNK_WRITE_PTR(dest), src->data, n); dest->datalen += n; @@ -447,7 +452,7 @@ buf_remove_from_front(buf_t *buf, size_t n) buf->head = victim->next; if (buf->tail == victim) buf->tail = NULL; - chunk_free(victim); + chunk_free_unchecked(victim); } } check(); @@ -481,7 +486,7 @@ buf_clear(buf_t *buf) buf->datalen = 0; for (chunk = buf->head; chunk; chunk = next) { next = chunk->next; - chunk_free(chunk); + chunk_free_unchecked(chunk); } buf->head = buf->tail = NULL; } @@ -520,6 +525,8 @@ buf_slack(const buf_t *buf) void buf_free(buf_t *buf) { + if (!buf) + return; buf_clear(buf); buf->magic = 0xdeadbeef; tor_free(buf); @@ -1400,7 +1407,7 @@ fetch_from_buf_socks(buf_t *buf, socks_request_t *req, "Tor only an IP address. Applications that do DNS resolves " "themselves may leak information. Consider using Socks4A " "(e.g. via privoxy or socat) instead. For more information, " - "please see http://wiki.noreply.org/noreply/TheOnionRouter/" + "please see https://wiki.torproject.org/TheOnionRouter/" "TorFAQ#SOCKSAndDNS.%s", req->port, safe_socks ? " Rejecting." : ""); /*have_warned_about_unsafe_socks = 1;*/ @@ -1488,7 +1495,8 @@ fetch_from_buf_socks(buf_t *buf, socks_request_t *req, return -1; } log_debug(LD_APP, - "socks4: successfully read destip (%s)", safe_str(tmpbuf)); + "socks4: successfully read destip (%s)", + safe_str_client(tmpbuf)); socks4_prot = socks4; } @@ -1513,7 +1521,7 @@ fetch_from_buf_socks(buf_t *buf, socks_request_t *req, "only an IP address. Applications that do DNS resolves " "themselves may leak information. Consider using Socks4A " "(e.g. via privoxy or socat) instead. For more information, " - "please see http://wiki.noreply.org/noreply/TheOnionRouter/" + "please see https://wiki.torproject.org/TheOnionRouter/" "TorFAQ#SOCKSAndDNS.%s", req->port, safe_socks ? " Rejecting." : ""); /*have_warned_about_unsafe_socks = 1;*/ /*(for now, warn every time)*/ @@ -1611,6 +1619,132 @@ fetch_from_buf_socks(buf_t *buf, socks_request_t *req, } } +/** Inspect a reply from SOCKS server stored in <b>buf</b> according + * to <b>state</b>, removing the protocol data upon success. Return 0 on + * incomplete response, 1 on success and -1 on error, in which case + * <b>reason</b> is set to a descriptive message (free() when finished + * with it). + * + * As a special case, 2 is returned when user/pass is required + * during SOCKS5 handshake and user/pass is configured. + */ +int +fetch_from_buf_socks_client(buf_t *buf, int state, char **reason) +{ + unsigned char *data; + size_t addrlen; + + if (buf->datalen < 2) + return 0; + + buf_pullup(buf, 128, 0); + tor_assert(buf->head && buf->head->datalen >= 2); + + data = (unsigned char *) buf->head->data; + + switch (state) { + case PROXY_SOCKS4_WANT_CONNECT_OK: + /* Wait for the complete response */ + if (buf->head->datalen < 8) + return 0; + + if (data[1] != 0x5a) { + *reason = tor_strdup(socks4_response_code_to_string(data[1])); + return -1; + } + + /* Success */ + buf_remove_from_front(buf, 8); + return 1; + + case PROXY_SOCKS5_WANT_AUTH_METHOD_NONE: + /* we don't have any credentials */ + if (data[1] != 0x00) { + *reason = tor_strdup("server doesn't support any of our " + "available authentication methods"); + return -1; + } + + log_info(LD_NET, "SOCKS 5 client: continuing without authentication"); + buf_clear(buf); + return 1; + + case PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929: + /* we have a username and password. return 1 if we can proceed without + * providing authentication, or 2 otherwise. */ + switch (data[1]) { + case 0x00: + log_info(LD_NET, "SOCKS 5 client: we have auth details but server " + "doesn't require authentication."); + buf_clear(buf); + return 1; + case 0x02: + log_info(LD_NET, "SOCKS 5 client: need authentication."); + buf_clear(buf); + return 2; + /* fall through */ + } + + *reason = tor_strdup("server doesn't support any of our available " + "authentication methods"); + return -1; + + case PROXY_SOCKS5_WANT_AUTH_RFC1929_OK: + /* handle server reply to rfc1929 authentication */ + if (data[1] != 0x00) { + *reason = tor_strdup("authentication failed"); + return -1; + } + + log_info(LD_NET, "SOCKS 5 client: authentication successful."); + buf_clear(buf); + return 1; + + case PROXY_SOCKS5_WANT_CONNECT_OK: + /* response is variable length. BND.ADDR, etc, isn't needed + * (don't bother with buf_pullup()), but make sure to eat all + * the data used */ + + /* wait for address type field to arrive */ + if (buf->datalen < 4) + return 0; + + switch (data[3]) { + case 0x01: /* ip4 */ + addrlen = 4; + break; + case 0x04: /* ip6 */ + addrlen = 16; + break; + case 0x03: /* fqdn (can this happen here?) */ + if (buf->datalen < 5) + return 0; + addrlen = 1 + data[4]; + break; + default: + *reason = tor_strdup("invalid response to connect request"); + return -1; + } + + /* wait for address and port */ + if (buf->datalen < 6 + addrlen) + return 0; + + if (data[1] != 0x00) { + *reason = tor_strdup(socks5_response_code_to_string(data[1])); + return -1; + } + + buf_remove_from_front(buf, 6 + addrlen); + return 1; + } + + /* shouldn't get here... */ + tor_assert(0); + + return -1; +} + /** Return 1 iff buf looks more like it has an (obsolete) v0 controller * command on it than any valid v1 controller command. */ int diff --git a/src/or/circuitbuild.c b/src/or/circuitbuild.c index 3441c30f90..300da7eed0 100644 --- a/src/or/circuitbuild.c +++ b/src/or/circuitbuild.c @@ -9,9 +9,25 @@ * \brief The actual details of building circuits. **/ +#define CIRCUIT_PRIVATE + #include "or.h" +#include "crypto.h" +#undef log +#include <math.h> + +#ifndef MIN +#define MIN(a,b) ((a)<(b)?(a):(b)) +#endif /********* START VARIABLES **********/ +/** Global list of circuit build times */ +// FIXME: Add this as a member for entry_guard_t instead of global? +// Then we could do per-guard statistics, as guards are likely to +// vary in their own latency. The downside of this is that guards +// can change frequently, so we'd be building a lot more circuits +// most likely. +circuit_build_times_t circ_times; /** A global list of all circuits at this hop. */ extern circuit_t *global_circuitlist; @@ -47,6 +63,10 @@ static smartlist_t *entry_guards = NULL; * and those changes need to be flushed to disk. */ static int entry_guards_dirty = 0; +/** If set, we're running the unit tests: we should avoid clobbering + * our state file or accessing get_options() or get_or_state() */ +static int unit_tests = 0; + /********* END VARIABLES ************/ static int circuit_deliver_create_cell(circuit_t *circ, @@ -59,6 +79,827 @@ static int onion_append_hop(crypt_path_t **head_ptr, extend_info_t *choice); static void entry_guards_changed(void); +/** Make a note that we're running unit tests (rather than running Tor + * itself), so we avoid clobbering our state file. */ +void +circuitbuild_running_unit_tests(void) +{ + unit_tests = 1; +} + +/** + * Return the initial default or configured timeout in milliseconds + */ +static double +circuit_build_times_get_initial_timeout(void) +{ + double timeout; + if (!unit_tests && get_options()->CircuitBuildTimeout) { + timeout = get_options()->CircuitBuildTimeout*1000; + if (timeout < BUILD_TIMEOUT_MIN_VALUE) { + log_warn(LD_CIRC, "Config CircuitBuildTimeout too low. Setting to %ds", + BUILD_TIMEOUT_MIN_VALUE/1000); + timeout = BUILD_TIMEOUT_MIN_VALUE; + } + } else { + timeout = BUILD_TIMEOUT_INITIAL_VALUE; + } + return timeout; +} + +/** + * Reset the build time state. + * + * Leave estimated parameters, timeout and network liveness intact + * for future use. + */ +void +circuit_build_times_reset(circuit_build_times_t *cbt) +{ + memset(cbt->circuit_build_times, 0, sizeof(cbt->circuit_build_times)); + cbt->pre_timeouts = 0; + cbt->total_build_times = 0; + cbt->build_times_idx = 0; + cbt->have_computed_timeout = 0; +} + +/** + * Initialize the buildtimes structure for first use. + * + * Sets the initial timeout value based to either the + * config setting or BUILD_TIMEOUT_INITIAL_VALUE. + */ +void +circuit_build_times_init(circuit_build_times_t *cbt) +{ + memset(cbt, 0, sizeof(*cbt)); + cbt->timeout_ms = circuit_build_times_get_initial_timeout(); +} + +/** + * Rewind our timeout history by n timeout positions. + */ +static void +circuit_build_times_rewind_history(circuit_build_times_t *cbt, int n) +{ + int i = 0; + + if (cbt->pre_timeouts) { + /* If we have pre-timeouts, it means we're not yet storing + * timeouts in our normal array. Only rewind the counter. */ + if (cbt->pre_timeouts > n) { + cbt->pre_timeouts -= n; + } else { + cbt->pre_timeouts = 0; + } + log_info(LD_CIRC, + "Rewound history by %d places. Current index: %d. Total: %d. " + "Pre-timeouts: %d", n, cbt->build_times_idx, + cbt->total_build_times, cbt->pre_timeouts); + + return; + } + + cbt->build_times_idx -= n; + cbt->build_times_idx %= NCIRCUITS_TO_OBSERVE; + + for (i = 0; i < n; i++) { + cbt->circuit_build_times[(i+cbt->build_times_idx)%NCIRCUITS_TO_OBSERVE]=0; + } + + if (cbt->total_build_times > n) { + cbt->total_build_times -= n; + } else { + cbt->total_build_times = 0; + } + + log_info(LD_CIRC, + "Rewound history by %d places. Current index: %d. " + "Total: %d", n, cbt->build_times_idx, cbt->total_build_times); +} + +/** + * Add a new timeout value <b>time</b> to the set of build times. Time + * units are milliseconds. + * + * circuit_build_times <b>cbt</a> is a circular array, so loop around when + * array is full. + */ +int +circuit_build_times_add_time(circuit_build_times_t *cbt, build_time_t time) +{ + tor_assert(time <= BUILD_TIME_MAX); + if (time <= 0) { + log_warn(LD_CIRC, "Circuit build time is %u!", time); + return -1; + } + + // XXX: Probably want to demote this to debug for the release. + log_info(LD_CIRC, "Adding circuit build time %u", time); + + cbt->circuit_build_times[cbt->build_times_idx] = time; + cbt->build_times_idx = (cbt->build_times_idx + 1) % NCIRCUITS_TO_OBSERVE; + if (cbt->total_build_times < NCIRCUITS_TO_OBSERVE) + cbt->total_build_times++; + + if ((cbt->total_build_times % BUILD_TIMES_SAVE_STATE_EVERY) == 0) { + /* Save state every n circuit builds */ + if (!unit_tests && !get_options()->AvoidDiskWrites) + or_state_mark_dirty(get_or_state(), 0); + } + + return 0; +} + +/** + * Return maximum circuit build time + */ +static build_time_t +circuit_build_times_max(circuit_build_times_t *cbt) +{ + int i = 0; + build_time_t max_build_time = 0; + for (i = 0; i < NCIRCUITS_TO_OBSERVE; i++) { + if (cbt->circuit_build_times[i] > max_build_time) + max_build_time = cbt->circuit_build_times[i]; + } + return max_build_time; +} + +#if 0 +/** Return minimum circuit build time */ +build_time_t +circuit_build_times_min(circuit_build_times_t *cbt) +{ + int i = 0; + build_time_t min_build_time = BUILD_TIME_MAX; + for (i = 0; i < NCIRCUITS_TO_OBSERVE; i++) { + if (cbt->circuit_build_times[i] && /* 0 <-> uninitialized */ + cbt->circuit_build_times[i] < min_build_time) + min_build_time = cbt->circuit_build_times[i]; + } + if (min_build_time == BUILD_TIME_MAX) { + log_warn(LD_CIRC, "No build times less than BUILD_TIME_MAX!"); + } + return min_build_time; +} +#endif + +/** + * Calculate and return a histogram for the set of build times. + * + * Returns an allocated array of histrogram bins representing + * the frequency of index*BUILDTIME_BIN_WIDTH millisecond + * build times. Also outputs the number of bins in nbins. + * + * The return value must be freed by the caller. + */ +static uint32_t * +circuit_build_times_create_histogram(circuit_build_times_t *cbt, + build_time_t *nbins) +{ + uint32_t *histogram; + build_time_t max_build_time = circuit_build_times_max(cbt); + int i, c; + + *nbins = 1 + (max_build_time / BUILDTIME_BIN_WIDTH); + histogram = tor_malloc_zero(*nbins * sizeof(build_time_t)); + + // calculate histogram + for (i = 0; i < NCIRCUITS_TO_OBSERVE; i++) { + if (cbt->circuit_build_times[i] == 0) continue; /* 0 <-> uninitialized */ + + c = (cbt->circuit_build_times[i] / BUILDTIME_BIN_WIDTH); + histogram[c]++; + } + + return histogram; +} + +/** + * Return the most frequent build time (rounded to BUILDTIME_BIN_WIDTH ms). + * + * Ties go in favor of the slower time. + */ +static build_time_t +circuit_build_times_mode(circuit_build_times_t *cbt) +{ + build_time_t i, nbins, max_bin=0; + uint32_t *histogram = circuit_build_times_create_histogram(cbt, &nbins); + + for (i = 0; i < nbins; i++) { + if (histogram[i] >= histogram[max_bin]) { + max_bin = i; + } + } + + tor_free(histogram); + + return max_bin*BUILDTIME_BIN_WIDTH+BUILDTIME_BIN_WIDTH/2; +} + +/** + * Output a histogram of current circuit build times to + * the or_state_t state structure. + */ +void +circuit_build_times_update_state(circuit_build_times_t *cbt, + or_state_t *state) +{ + uint32_t *histogram; + build_time_t i = 0; + build_time_t nbins = 0; + config_line_t **next, *line; + + histogram = circuit_build_times_create_histogram(cbt, &nbins); + // write to state + config_free_lines(state->BuildtimeHistogram); + next = &state->BuildtimeHistogram; + *next = NULL; + + state->TotalBuildTimes = cbt->total_build_times; + + for (i = 0; i < nbins; i++) { + // compress the histogram by skipping the blanks + if (histogram[i] == 0) continue; + *next = line = tor_malloc_zero(sizeof(config_line_t)); + line->key = tor_strdup("CircuitBuildTimeBin"); + line->value = tor_malloc(25); + tor_snprintf(line->value, 25, "%d %d", + i*BUILDTIME_BIN_WIDTH+BUILDTIME_BIN_WIDTH/2, histogram[i]); + next = &(line->next); + } + + if (!unit_tests) { + if (!get_options()->AvoidDiskWrites) + or_state_mark_dirty(get_or_state(), 0); + } + + tor_free(histogram); +} + +/** + * Shuffle the build times array. + * + * Stolen from http://en.wikipedia.org/wiki/Fisher\u2013Yates_shuffle + */ +static void +circuit_build_times_shuffle_and_store_array(circuit_build_times_t *cbt, + build_time_t *raw_times, + int num_times) +{ + int n = num_times; + if (num_times > NCIRCUITS_TO_OBSERVE) { + log_notice(LD_CIRC, "Decreasing circuit_build_times size from %d to %d", + num_times, NCIRCUITS_TO_OBSERVE); + } + + /* This code can only be run on a compact array */ + while (n-- > 1) { + int k = crypto_rand_int(n + 1); /* 0 <= k <= n. */ + build_time_t tmp = raw_times[k]; + raw_times[k] = raw_times[n]; + raw_times[n] = tmp; + } + + /* Since the times are now shuffled, take a random NCIRCUITS_TO_OBSERVE + * subset (ie the first NCIRCUITS_TO_OBSERVE values) */ + for (n = 0; n < MIN(num_times, NCIRCUITS_TO_OBSERVE); n++) { + circuit_build_times_add_time(cbt, raw_times[n]); + } +} + +/** + * Load histogram from <b>state</b>, shuffling the resulting array + * after we do so. Use this result to estimate parameters and + * calculate the timeout. + * + * Returns -1 and sets msg on error. Msg must be freed by the caller. + */ +int +circuit_build_times_parse_state(circuit_build_times_t *cbt, + or_state_t *state, char **msg) +{ + int tot_values = 0; + uint32_t loaded_cnt = 0, N = 0; + config_line_t *line; + int i; + build_time_t *loaded_times = tor_malloc(sizeof(build_time_t) + * state->TotalBuildTimes); + circuit_build_times_init(cbt); + *msg = NULL; + + for (line = state->BuildtimeHistogram; line; line = line->next) { + smartlist_t *args = smartlist_create(); + smartlist_split_string(args, line->value, " ", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + if (smartlist_len(args) < 2) { + *msg = tor_strdup("Unable to parse circuit build times: " + "Too few arguments to CircuitBuildTime"); + SMARTLIST_FOREACH(args, char*, cp, tor_free(cp)); + smartlist_free(args); + break; + } else { + const char *ms_str = smartlist_get(args,0); + const char *count_str = smartlist_get(args,1); + uint32_t count, k; + build_time_t ms; + int ok; + ms = (build_time_t)tor_parse_ulong(ms_str, 0, 0, + BUILD_TIME_MAX, &ok, NULL); + if (!ok) { + *msg = tor_strdup("Unable to parse circuit build times: " + "Unparsable bin number"); + SMARTLIST_FOREACH(args, char*, cp, tor_free(cp)); + smartlist_free(args); + break; + } + count = (uint32_t)tor_parse_ulong(count_str, 0, 0, + UINT32_MAX, &ok, NULL); + if (!ok) { + *msg = tor_strdup("Unable to parse circuit build times: " + "Unparsable bin count"); + SMARTLIST_FOREACH(args, char*, cp, tor_free(cp)); + smartlist_free(args); + break; + } + + if (loaded_cnt+count > state->TotalBuildTimes) { + log_warn(LD_CIRC, + "Too many build times in state file. " + "Stopping short before %d", + loaded_cnt+count); + SMARTLIST_FOREACH(args, char*, cp, tor_free(cp)); + smartlist_free(args); + break; + } + + for (k = 0; k < count; k++) { + loaded_times[loaded_cnt++] = ms; + } + N++; + SMARTLIST_FOREACH(args, char*, cp, tor_free(cp)); + smartlist_free(args); + } + } + + if (loaded_cnt != state->TotalBuildTimes) { + log_warn(LD_CIRC, + "Corrupt state file? Build times count mismatch. " + "Read %d, file says %d", loaded_cnt, state->TotalBuildTimes); + } + + circuit_build_times_shuffle_and_store_array(cbt, loaded_times, loaded_cnt); + + /* Verify that we didn't overwrite any indexes */ + for (i=0; i < NCIRCUITS_TO_OBSERVE; i++) { + if (!cbt->circuit_build_times[i]) + break; + tot_values++; + } + log_info(LD_CIRC, + "Loaded %d/%d values from %d lines in circuit time histogram", + tot_values, cbt->total_build_times, N); + tor_assert(cbt->total_build_times == tot_values); + tor_assert(cbt->total_build_times <= NCIRCUITS_TO_OBSERVE); + circuit_build_times_set_timeout(cbt); + tor_free(loaded_times); + return *msg ? -1 : 0; +} + +/** + * Estimates the Xm and Alpha parameters using + * http://en.wikipedia.org/wiki/Pareto_distribution#Parameter_estimation + * + * The notable difference is that we use mode instead of min to estimate Xm. + * This is because our distribution is frechet-like. We claim this is + * an acceptable approximation because we are only concerned with the + * accuracy of the CDF of the tail. + */ +void +circuit_build_times_update_alpha(circuit_build_times_t *cbt) +{ + build_time_t *x=cbt->circuit_build_times; + double a = 0; + int n=0,i=0; + + /* http://en.wikipedia.org/wiki/Pareto_distribution#Parameter_estimation */ + /* We sort of cheat here and make our samples slightly more pareto-like + * and less frechet-like. */ + cbt->Xm = circuit_build_times_mode(cbt); + + for (i=0; i< NCIRCUITS_TO_OBSERVE; i++) { + if (!x[i]) { + continue; + } + + if (x[i] < cbt->Xm) { + a += tor_mathlog(cbt->Xm); + } else { + a += tor_mathlog(x[i]); + } + n++; + } + + if (n!=cbt->total_build_times) { + log_err(LD_CIRC, "Discrepancy in build times count: %d vs %d", n, + cbt->total_build_times); + } + tor_assert(n==cbt->total_build_times); + + a -= n*tor_mathlog(cbt->Xm); + a = n/a; + + cbt->alpha = a; +} + +/** + * This is the Pareto Quantile Function. It calculates the point x + * in the distribution such that F(x) = quantile (ie quantile*100% + * of the mass of the density function is below x on the curve). + * + * We use it to calculate the timeout and also to generate synthetic + * values of time for circuits that timeout before completion. + * + * See http://en.wikipedia.org/wiki/Quantile_function, + * http://en.wikipedia.org/wiki/Inverse_transform_sampling and + * http://en.wikipedia.org/wiki/Pareto_distribution#Generating_a_ + * random_sample_from_Pareto_distribution + * That's right. I'll cite wikipedia all day long. + * + * Return value is in milliseconds. + */ +double +circuit_build_times_calculate_timeout(circuit_build_times_t *cbt, + double quantile) +{ + double ret; + tor_assert(quantile >= 0); + tor_assert(1.0-quantile > 0); + tor_assert(cbt->Xm > 0); + + ret = cbt->Xm/pow(1.0-quantile,1.0/cbt->alpha); + if (ret > INT32_MAX) { + ret = INT32_MAX; + } + tor_assert(ret > 0); + return ret; +} + +/** Pareto CDF */ +double +circuit_build_times_cdf(circuit_build_times_t *cbt, double x) +{ + double ret; + tor_assert(cbt->Xm > 0); + ret = 1.0-pow(cbt->Xm/x,cbt->alpha); + tor_assert(0 <= ret && ret <= 1.0); + return ret; +} + +/** + * Generate a synthetic time using our distribution parameters. + * + * The return value will be within the [q_lo, q_hi) quantile points + * on the CDF. + */ +build_time_t +circuit_build_times_generate_sample(circuit_build_times_t *cbt, + double q_lo, double q_hi) +{ + uint64_t r = crypto_rand_uint64(UINT64_MAX-1); + build_time_t ret; + double u; + + /* Generate between [q_lo, q_hi) */ + q_hi -= 1.0/(INT32_MAX); + + tor_assert(q_lo >= 0); + tor_assert(q_hi < 1); + tor_assert(q_lo < q_hi); + + u = q_lo + ((q_hi-q_lo)*r)/(1.0*UINT64_MAX); + + tor_assert(0 <= u && u < 1.0); + /* circuit_build_times_calculate_timeout returns <= INT32_MAX */ + ret = (build_time_t) + tor_lround(circuit_build_times_calculate_timeout(cbt, u)); + tor_assert(ret > 0); + return ret; +} + +/** Generate points in [cutoff, 1.0) on the CDF. */ +void +circuit_build_times_add_timeout_worker(circuit_build_times_t *cbt, + double quantile_cutoff) +{ + build_time_t gentime = circuit_build_times_generate_sample(cbt, + quantile_cutoff, MAX_SYNTHETIC_QUANTILE); + + if (gentime < (build_time_t)tor_lround(cbt->timeout_ms)) { + log_warn(LD_CIRC, + "Generated a synthetic timeout LESS than the current timeout: " + "%ums vs %lfms using Xm: %d a: %lf, q: %lf", + gentime, cbt->timeout_ms, cbt->Xm, cbt->alpha, quantile_cutoff); + } else if (gentime > BUILD_TIME_MAX) { + log_info(LD_CIRC, + "Generated a synthetic timeout larger than the max: %u", + gentime); + gentime = BUILD_TIME_MAX; + } else { + log_info(LD_CIRC, "Generated synthetic circuit build time %u for timeout", + gentime); + } + + circuit_build_times_add_time(cbt, gentime); +} + +/** + * Estimate an initial alpha parameter by solving the quantile + * function with a quantile point and a specific timeout value. + */ +void +circuit_build_times_initial_alpha(circuit_build_times_t *cbt, + double quantile, double timeout_ms) +{ + // Q(u) = Xm/((1-u)^(1/a)) + // Q(0.8) = Xm/((1-0.8))^(1/a)) = CircBuildTimeout + // CircBuildTimeout = Xm/((1-0.8))^(1/a)) + // CircBuildTimeout = Xm*((1-0.8))^(-1/a)) + // ln(CircBuildTimeout) = ln(Xm)+ln(((1-0.8)))*(-1/a) + // -ln(1-0.8)/(ln(CircBuildTimeout)-ln(Xm))=a + tor_assert(quantile >= 0); + tor_assert(cbt->Xm > 0); + cbt->alpha = tor_mathlog(1.0-quantile)/ + (tor_mathlog(cbt->Xm)-tor_mathlog(timeout_ms)); + tor_assert(cbt->alpha > 0); +} + +/** + * Generate synthetic timeout values for the timeouts + * that have happened before we estimated our parameters. + */ +static void +circuit_build_times_count_pretimeouts(circuit_build_times_t *cbt) +{ + /* Store a timeout as a random position past the current + * cutoff on the pareto curve */ + if (cbt->pre_timeouts) { + double timeout_quantile = 1.0- + ((double)cbt->pre_timeouts)/ + (cbt->pre_timeouts+cbt->total_build_times); + /* Make sure it doesn't exceed the synthetic max */ + timeout_quantile *= MAX_SYNTHETIC_QUANTILE; + cbt->Xm = circuit_build_times_mode(cbt); + tor_assert(cbt->Xm > 0); + /* Use current timeout to get an estimate on alpha */ + circuit_build_times_initial_alpha(cbt, timeout_quantile, + cbt->timeout_ms); + while (cbt->pre_timeouts-- != 0) { + circuit_build_times_add_timeout_worker(cbt, timeout_quantile); + } + cbt->pre_timeouts = 0; + } +} + +/** + * Returns true if we need circuits to be built + */ +int +circuit_build_times_needs_circuits(circuit_build_times_t *cbt) +{ + /* Return true if < MIN_CIRCUITS_TO_OBSERVE */ + if (cbt->total_build_times < MIN_CIRCUITS_TO_OBSERVE) + return 1; + return 0; +} + +/** + * Returns true if we should build a timeout test circuit + * right now. + */ +int +circuit_build_times_needs_circuits_now(circuit_build_times_t *cbt) +{ + return circuit_build_times_needs_circuits(cbt) && + approx_time()-cbt->last_circ_at > BUILD_TIMES_TEST_FREQUENCY; +} + +/** + * Called to indicate that the network showed some signs of liveness. + */ +void +circuit_build_times_network_is_live(circuit_build_times_t *cbt) +{ + cbt->liveness.network_last_live = approx_time(); + cbt->liveness.nonlive_discarded = 0; + cbt->liveness.nonlive_timeouts = 0; +} + +/** + * Called to indicate that we completed a circuit. Because this circuit + * succeeded, it doesn't count as a timeout-after-the-first-hop. + */ +void +circuit_build_times_network_circ_success(circuit_build_times_t *cbt) +{ + cbt->liveness.timeouts_after_firsthop[cbt->liveness.after_firsthop_idx] = 0; + cbt->liveness.after_firsthop_idx++; + cbt->liveness.after_firsthop_idx %= RECENT_CIRCUITS; +} + +/** + * A circuit just timed out. If there has been no recent network activity + * at all, but this circuit was launched back when we thought the network + * was live, increment the number of "nonlive" circuit timeouts. + * + * Also distinguish between whether it failed before the first hop + * and record that in our history for later deciding if the network has + * changed. + */ +static void +circuit_build_times_network_timeout(circuit_build_times_t *cbt, + int did_onehop, time_t start_time) +{ + time_t now = time(NULL); + /* + * Check if this is a timeout that was for a circuit that spent its + * entire existence during a time where we have had no network activity. + * + * Also double check that it is a valid timeout after we have possibly + * just recently reset cbt->timeout_ms. + */ + if (cbt->liveness.network_last_live <= start_time && + start_time <= (now - cbt->timeout_ms/1000.0)) { + cbt->liveness.nonlive_timeouts++; + } else if (did_onehop) { + /* Count a one-hop timeout */ + cbt->liveness.timeouts_after_firsthop[cbt->liveness.after_firsthop_idx]=1; + cbt->liveness.after_firsthop_idx++; + cbt->liveness.after_firsthop_idx %= RECENT_CIRCUITS; + } +} + +/** + * Returns false if the network has not received a cell or tls handshake + * in the past NETWORK_NOTLIVE_TIMEOUT_COUNT circuits. + * + * Also has the side effect of rewinding the circuit time history + * in the case of recent liveness changes. + */ +int +circuit_build_times_network_check_live(circuit_build_times_t *cbt) +{ + time_t now = approx_time(); + if (cbt->liveness.nonlive_timeouts >= NETWORK_NONLIVE_DISCARD_COUNT) { + if (!cbt->liveness.nonlive_discarded) { + cbt->liveness.nonlive_discarded = 1; + log_notice(LD_CIRC, "Network is no longer live (too many recent " + "circuit timeouts). Dead for %ld seconds.", + (long int)(now - cbt->liveness.network_last_live)); + /* Only discard NETWORK_NONLIVE_TIMEOUT_COUNT-1 because we stopped + * counting after that */ + circuit_build_times_rewind_history(cbt, NETWORK_NONLIVE_TIMEOUT_COUNT-1); + } + return 0; + } else if (cbt->liveness.nonlive_timeouts >= NETWORK_NONLIVE_TIMEOUT_COUNT) { + if (cbt->timeout_ms < circuit_build_times_get_initial_timeout()) { + log_notice(LD_CIRC, + "Network is flaky. No activity for %ld seconds. " + "Temporarily raising timeout to %lds.", + (long int)(now - cbt->liveness.network_last_live), + tor_lround(circuit_build_times_get_initial_timeout()/1000)); + cbt->timeout_ms = circuit_build_times_get_initial_timeout(); + } + + return 0; + } + + return 1; +} + +/** + * Returns true if we have seen more than MAX_RECENT_TIMEOUT_COUNT of + * the past RECENT_CIRCUITS time out after the first hop. Used to detect + * if the network connection has changed significantly. + * + * Also resets the entire timeout history in this case and causes us + * to restart the process of building test circuits and estimating a + * new timeout. + */ +int +circuit_build_times_network_check_changed(circuit_build_times_t *cbt) +{ + int total_build_times = cbt->total_build_times; + int timeout_count=0; + int i; + + /* how many of our recent circuits made it to the first hop but then + * timed out? */ + for (i = 0; i < RECENT_CIRCUITS; i++) { + timeout_count += cbt->liveness.timeouts_after_firsthop[i]; + } + + /* If 80% of our recent circuits are timing out after the first hop, + * we need to re-estimate a new initial alpha and timeout. */ + if (timeout_count < MAX_RECENT_TIMEOUT_COUNT) { + return 0; + } + + circuit_build_times_reset(cbt); + memset(cbt->liveness.timeouts_after_firsthop, 0, + sizeof(cbt->liveness.timeouts_after_firsthop)); + cbt->liveness.after_firsthop_idx = 0; + + /* Check to see if this has happened before. If so, double the timeout + * to give people on abysmally bad network connections a shot at access */ + if (cbt->timeout_ms >= circuit_build_times_get_initial_timeout()) { + cbt->timeout_ms *= 2; + } else { + cbt->timeout_ms = circuit_build_times_get_initial_timeout(); + } + + log_notice(LD_CIRC, + "Network connection speed appears to have changed. Resetting " + "timeout to %lds after %d timeouts and %d buildtimes.", + tor_lround(cbt->timeout_ms/1000), timeout_count, + total_build_times); + + return 1; +} + +/** + * Store a timeout as a synthetic value. + * + * Returns true if the store was successful and we should possibly + * update our timeout estimate. + */ +int +circuit_build_times_add_timeout(circuit_build_times_t *cbt, + int did_onehop, + time_t start_time) +{ + circuit_build_times_network_timeout(cbt, did_onehop, start_time); + + /* Only count timeouts if network is live.. */ + if (!circuit_build_times_network_check_live(cbt)) { + return 0; + } + + /* If there are a ton of timeouts, we should reduce + * the circuit build timeout */ + if (circuit_build_times_network_check_changed(cbt)) { + return 0; + } + + if (!cbt->have_computed_timeout) { + /* Store a timeout before we have enough data */ + cbt->pre_timeouts++; + log_info(LD_CIRC, + "Not enough circuits yet to calculate a new build timeout." + " Need %d more.", + MIN_CIRCUITS_TO_OBSERVE-cbt->total_build_times); + return 0; + } + + circuit_build_times_count_pretimeouts(cbt); + circuit_build_times_add_timeout_worker(cbt, BUILDTIMEOUT_QUANTILE_CUTOFF); + + return 1; +} + +/** + * Estimate a new timeout based on history and set our timeout + * variable accordingly. + */ +void +circuit_build_times_set_timeout(circuit_build_times_t *cbt) +{ + if (cbt->total_build_times < MIN_CIRCUITS_TO_OBSERVE) { + return; + } + + circuit_build_times_count_pretimeouts(cbt); + circuit_build_times_update_alpha(cbt); + + cbt->timeout_ms = circuit_build_times_calculate_timeout(cbt, + BUILDTIMEOUT_QUANTILE_CUTOFF); + + cbt->have_computed_timeout = 1; + + if (cbt->timeout_ms < BUILD_TIMEOUT_MIN_VALUE) { + log_warn(LD_CIRC, "Set buildtimeout to low value %lfms. Setting to %dms", + cbt->timeout_ms, BUILD_TIMEOUT_MIN_VALUE); + cbt->timeout_ms = BUILD_TIMEOUT_MIN_VALUE; + } + + log_info(LD_CIRC, + "Set circuit build timeout to %lds (%lfms, Xm: %d, a: %lf) " + "based on %d circuit times", tor_lround(cbt->timeout_ms/1000), + cbt->timeout_ms, cbt->Xm, cbt->alpha, cbt->total_build_times); + +} + /** Iterate over values of circ_id, starting from conn-\>next_circ_id, * and with the high bit specified by conn-\>circ_id_type, until we get * a circ_id that is not in use by any other circuit on that conn. @@ -148,8 +989,7 @@ circuit_list_path_impl(origin_circuit_t *circ, int verbose, int verbose_names) router_get_verbose_nickname(elt, ri); } else if ((rs = router_get_consensus_status_by_id(id))) { routerstatus_get_verbose_nickname(elt, rs); - } else if (hop->extend_info->nickname && - is_legal_nickname(hop->extend_info->nickname)) { + } else if (is_legal_nickname(hop->extend_info->nickname)) { elt[0] = '$'; base16_encode(elt+1, HEX_DIGEST_LEN+1, id, DIGEST_LEN); elt[HEX_DIGEST_LEN+1]= '~'; @@ -217,7 +1057,7 @@ void circuit_log_path(int severity, unsigned int domain, origin_circuit_t *circ) { char *s = circuit_list_path(circ,1); - log(severity,domain,"%s",s); + tor_log(severity,domain,"%s",s); tor_free(s); } @@ -361,9 +1201,10 @@ circuit_handle_first_hop(origin_circuit_t *circ) if (!n_conn) { /* not currently connected in a useful way. */ - const char *name = firsthop->extend_info->nickname ? + const char *name = strlen(firsthop->extend_info->nickname) ? firsthop->extend_info->nickname : fmt_addr(&firsthop->extend_info->addr); - log_info(LD_CIRC, "Next router is %s: %s ", safe_str(name), msg?msg:"???"); + log_info(LD_CIRC, "Next router is %s: %s ", + safe_str_client(name), msg?msg:"???"); circ->_base.n_hop = extend_info_dup(firsthop->extend_info); if (should_launch) { @@ -536,7 +1377,7 @@ inform_testing_reachability(void) "CHECKING_REACHABILITY DIRADDRESS=%s:%d", me->address, me->dir_port); } - log(LOG_NOTICE, LD_OR, "Now checking whether ORPort %s:%d%s %s reachable... " + log_notice(LD_OR, "Now checking whether ORPort %s:%d%s %s reachable... " "(this may take up to %d minutes -- look for log " "messages indicating success)", me->address, me->or_port, @@ -642,6 +1483,17 @@ circuit_send_next_onion_skin(origin_circuit_t *circ) if (!hop) { /* done building the circuit. whew. */ circuit_set_state(TO_CIRCUIT(circ), CIRCUIT_STATE_OPEN); + if (!circ->build_state->onehop_tunnel) { + struct timeval end; + long timediff; + tor_gettimeofday(&end); + timediff = tv_mdiff(&circ->_base.highres_created, &end); + if (timediff > INT32_MAX) + timediff = INT32_MAX; + circuit_build_times_add_time(&circ_times, (build_time_t)timediff); + circuit_build_times_network_circ_success(&circ_times); + circuit_build_times_set_timeout(&circ_times); + } log_info(LD_CIRC,"circuit built!"); circuit_reset_failure_count(0); if (circ->build_state->onehop_tunnel) @@ -650,7 +1502,7 @@ circuit_send_next_onion_skin(origin_circuit_t *circ) or_options_t *options = get_options(); has_completed_circuit=1; /* FFFF Log a count of known routers here */ - log(LOG_NOTICE, LD_GENERAL, + log_notice(LD_GENERAL, "Tor has successfully opened a circuit. " "Looks like client functionality is working."); control_event_bootstrap(BOOTSTRAP_STATUS_DONE, 0); @@ -705,7 +1557,7 @@ void circuit_note_clock_jumped(int seconds_elapsed) { int severity = server_mode(get_options()) ? LOG_WARN : LOG_NOTICE; - log(severity, LD_GENERAL, "Your system clock just jumped %d seconds %s; " + tor_log(severity, LD_GENERAL, "Your system clock just jumped %d seconds %s; " "assuming established circuits no longer work.", seconds_elapsed >=0 ? seconds_elapsed : -seconds_elapsed, seconds_elapsed >=0 ? "forward" : "backward"); @@ -942,10 +1794,9 @@ circuit_finish_handshake(origin_circuit_t *circ, uint8_t reply_type, return -END_CIRC_REASON_TORPROTOCOL; } - if (hop->dh_handshake_state) { - crypto_dh_free(hop->dh_handshake_state); /* don't need it anymore */ - hop->dh_handshake_state = NULL; - } + crypto_dh_free(hop->dh_handshake_state); /* don't need it anymore */ + hop->dh_handshake_state = NULL; + memset(hop->fast_handshake_state, 0, sizeof(hop->fast_handshake_state)); if (circuit_init_cpath_crypto(hop, keys, 0)<0) { @@ -1253,9 +2104,16 @@ choose_good_exit_server_general(routerlist_t *dir, int need_uptime, n_supported[i] = -1; continue; /* skip routers that are known to be down or bad exits */ } - if (router_is_unreliable(router, need_uptime, need_capacity, 0)) { + if (router_is_unreliable(router, need_uptime, need_capacity, 0) && + (!options->ExitNodes || + !routerset_contains_router(options->ExitNodes, router))) { + /* FFFF Someday, differentiate between a routerset that names + * routers, and a routerset that names countries, and only do this + * check if they've asked for specific exit relays. Or if the country + * they ask for is rare. Or something. */ n_supported[i] = -1; - continue; /* skip routers that are not suitable */ + continue; /* skip routers that are not suitable, unless we have + * ExitNodes set, in which case we asked for it */ } if (!(router->is_valid || options->_AllowInvalid & ALLOW_INVALID_EXIT)) { /* if it's invalid and we don't want it */ @@ -1280,7 +2138,7 @@ choose_good_exit_server_general(routerlist_t *dir, int need_uptime, { if (!ap_stream_wants_exit_attention(conn)) continue; /* Skip everything but APs in CIRCUIT_WAIT */ - if (connection_ap_can_use_exit(TO_EDGE_CONN(conn), router)) { + if (connection_ap_can_use_exit(TO_EDGE_CONN(conn), router, 1)) { ++n_supported[i]; // log_fn(LOG_DEBUG,"%s is supported. n_supported[%d] now %d.", // router->nickname, i, n_supported[i]); @@ -1322,7 +2180,8 @@ choose_good_exit_server_general(routerlist_t *dir, int need_uptime, routersets_get_disjunction(use, supporting, options->ExitNodes, options->_ExcludeExitNodesUnion, 1); - if (smartlist_len(use) == 0 && !options->StrictExitNodes) { + if (smartlist_len(use) == 0 && options->ExitNodes && + !options->StrictNodes) { /* give up on exitnodes and try again */ routersets_get_disjunction(use, supporting, NULL, options->_ExcludeExitNodesUnion, 1); } @@ -1347,8 +2206,9 @@ choose_good_exit_server_general(routerlist_t *dir, int need_uptime, tor_free(n_supported); return choose_good_exit_server_general(dir, 0, 0); } - log_notice(LD_CIRC, "All routers are down or won't exit -- choosing a " - "doomed exit at random."); + log_notice(LD_CIRC, "All routers are down or won't exit%s -- " + "choosing a doomed exit at random.", + options->_ExcludeExitNodesUnion ? " or are Excluded" : ""); } supporting = smartlist_create(); use = smartlist_create(); @@ -1368,12 +2228,14 @@ choose_good_exit_server_general(routerlist_t *dir, int need_uptime, routersets_get_disjunction(use, supporting, options->ExitNodes, options->_ExcludeExitNodesUnion, 1); - if (smartlist_len(use) == 0 && !options->StrictExitNodes) { + if (smartlist_len(use) == 0 && options->ExitNodes && + !options->StrictNodes) { /* give up on exitnodes and try again */ routersets_get_disjunction(use, supporting, NULL, options->_ExcludeExitNodesUnion, 1); } - /* XXX sometimes the above results in null, when the requested - * exit node is down. we should pick it anyway. */ + /* FFF sometimes the above results in null, when the requested + * exit node is considered down by the consensus. we should pick + * it anyway, since the user asked for it. */ router = routerlist_sl_choose_by_bandwidth(use, WEIGHT_FOR_EXIT); if (router) break; @@ -1391,10 +2253,10 @@ choose_good_exit_server_general(routerlist_t *dir, int need_uptime, log_info(LD_CIRC, "Chose exit server '%s'", router->nickname); return router; } - if (options->StrictExitNodes) { + if (options->ExitNodes && options->StrictNodes) { log_warn(LD_CIRC, "No specified exit routers seem to be running, and " - "StrictExitNodes is set: can't choose an exit."); + "StrictNodes is set: can't choose an exit."); } return NULL; } @@ -1425,15 +2287,13 @@ choose_good_exit_server(uint8_t purpose, routerlist_t *dir, if (options->_AllowInvalid & ALLOW_INVALID_MIDDLE) flags |= CRN_ALLOW_INVALID; if (is_internal) /* pick it like a middle hop */ - return router_choose_random_node(NULL, NULL, - options->ExcludeNodes, flags); + return router_choose_random_node(NULL, options->ExcludeNodes, flags); else return choose_good_exit_server_general(dir,need_uptime,need_capacity); case CIRCUIT_PURPOSE_C_ESTABLISH_REND: if (options->_AllowInvalid & ALLOW_INVALID_RENDEZVOUS) flags |= CRN_ALLOW_INVALID; - return router_choose_random_node(NULL, NULL, - options->ExcludeNodes, flags); + return router_choose_random_node(NULL, options->ExcludeNodes, flags); } log_warn(LD_BUG,"Unhandled purpose %d", purpose); tor_fragile_assert(); @@ -1553,8 +2413,7 @@ circuit_append_new_exit(origin_circuit_t *circ, extend_info_t *exit) state = circ->build_state; tor_assert(state); - if (state->chosen_exit) - extend_info_free(state->chosen_exit); + extend_info_free(state->chosen_exit); state->chosen_exit = extend_info_dup(exit); ++circ->build_state->desired_path_len; @@ -1676,8 +2535,7 @@ choose_good_middle_server(uint8_t purpose, flags |= CRN_NEED_CAPACITY; if (options->_AllowInvalid & ALLOW_INVALID_MIDDLE) flags |= CRN_ALLOW_INVALID; - choice = router_choose_random_node(NULL, - excluded, options->ExcludeNodes, flags); + choice = router_choose_random_node(excluded, options->ExcludeNodes, flags); smartlist_free(excluded); return choice; } @@ -1741,11 +2599,7 @@ choose_good_entry_server(uint8_t purpose, cpath_build_state_t *state) if (options->_AllowInvalid & ALLOW_INVALID_ENTRY) flags |= CRN_ALLOW_INVALID; - choice = router_choose_random_node( - NULL, - excluded, - options->ExcludeNodes, - flags); + choice = router_choose_random_node(excluded, options->ExcludeNodes, flags); smartlist_free(excluded); return choice; } @@ -1866,9 +2720,9 @@ extend_info_from_router(routerinfo_t *r) void extend_info_free(extend_info_t *info) { - tor_assert(info); - if (info->onion_key) - crypto_free_pk_env(info->onion_key); + if (!info) + return; + crypto_free_pk_env(info->onion_key); tor_free(info); } @@ -1991,35 +2845,58 @@ entry_is_time_to_retry(entry_guard_t *e, time_t now) * - Listed as either up or never yet contacted; * - Present in the routerlist; * - Listed as 'stable' or 'fast' by the current dirserver consensus, - * if demanded by <b>need_uptime</b> or <b>need_capacity</b>; - * (This check is currently redundant with the Guard flag, but in - * the future that might change. Best to leave it in for now.) + * if demanded by <b>need_uptime</b> or <b>need_capacity</b> + * (unless it's a configured EntryNode); * - Allowed by our current ReachableORAddresses config option; and - * - Currently thought to be reachable by us (unless assume_reachable + * - Currently thought to be reachable by us (unless <b>assume_reachable</b> * is true). + * + * If the answer is no, set *<b>msg</b> to an explanation of why. */ static INLINE routerinfo_t * entry_is_live(entry_guard_t *e, int need_uptime, int need_capacity, - int assume_reachable) + int assume_reachable, const char **msg) { routerinfo_t *r; - if (e->bad_since) + or_options_t *options = get_options(); + tor_assert(msg); + + if (e->bad_since) { + *msg = "bad"; return NULL; + } /* no good if it's unreachable, unless assume_unreachable or can_retry. */ if (!assume_reachable && !e->can_retry && - e->unreachable_since && !entry_is_time_to_retry(e, time(NULL))) + e->unreachable_since && !entry_is_time_to_retry(e, time(NULL))) { + *msg = "unreachable"; return NULL; + } r = router_get_by_digest(e->identity); - if (!r) + if (!r) { + *msg = "no descriptor"; return NULL; - if (get_options()->UseBridges && r->purpose != ROUTER_PURPOSE_BRIDGE) + } + if (get_options()->UseBridges && r->purpose != ROUTER_PURPOSE_BRIDGE) { + *msg = "not a bridge"; return NULL; - if (!get_options()->UseBridges && r->purpose != ROUTER_PURPOSE_GENERAL) + } + if (!get_options()->UseBridges && r->purpose != ROUTER_PURPOSE_GENERAL) { + *msg = "not general-purpose"; return NULL; - if (router_is_unreliable(r, need_uptime, need_capacity, 0)) + } + if (options->EntryNodes && + routerset_contains_router(options->EntryNodes, r)) { + /* they asked for it, they get it */ + need_uptime = need_capacity = 0; + } + if (router_is_unreliable(r, need_uptime, need_capacity, 0)) { + *msg = "not fast/stable"; return NULL; - if (!fascist_firewall_allows_or(r)) + } + if (!fascist_firewall_allows_or(r)) { + *msg = "unreachable by config"; return NULL; + } return r; } @@ -2028,11 +2905,12 @@ static int num_live_entry_guards(void) { int n = 0; + const char *msg; if (! entry_guards) return 0; SMARTLIST_FOREACH(entry_guards, entry_guard_t *, entry, { - if (entry_is_live(entry, 0, 1, 0)) + if (entry_is_live(entry, 0, 1, 0, &msg)) ++n; }); return n; @@ -2061,10 +2939,15 @@ log_entry_guards(int severity) SMARTLIST_FOREACH(entry_guards, entry_guard_t *, e, { - tor_snprintf(buf, sizeof(buf), "%s (%s%s)", - e->nickname, - entry_is_live(e, 0, 1, 0) ? "up " : "down ", - e->made_contact ? "made-contact" : "never-contacted"); + const char *msg = NULL; + if (entry_is_live(e, 0, 1, 0, &msg)) + tor_snprintf(buf, sizeof(buf), "%s (up %s)", + e->nickname, + e->made_contact ? "made-contact" : "never-contacted"); + else + tor_snprintf(buf, sizeof(buf), "%s (%s, %s)", + e->nickname, msg, + e->made_contact ? "made-contact" : "never-contacted"); smartlist_add(elements, tor_strdup(buf)); }); @@ -2089,12 +2972,13 @@ control_event_guard_deferred(void) **/ #if 0 int n = 0; + const char *msg; or_options_t *options = get_options(); if (!entry_guards) return; SMARTLIST_FOREACH(entry_guards, entry_guard_t *, entry, { - if (entry_is_live(entry, 0, 1, 0)) { + if (entry_is_live(entry, 0, 1, 0, &msg)) { if (n++ == options->NumEntryGuards) { control_event_guard(entry->nickname, entry->identity, "DEFERRED"); return; @@ -2180,7 +3064,8 @@ pick_entry_guards(void) static void entry_guard_free(entry_guard_t *e) { - tor_assert(e); + if (!e) + return; tor_free(e->chosen_by_version); tor_free(e); } @@ -2298,10 +3183,13 @@ entry_guards_compute_status(void) int severity = LOG_DEBUG; or_options_t *options; digestmap_t *reasons; + if (! entry_guards) return; options = get_options(); + if (options->EntryNodes) /* reshuffle the entry guard list if needed */ + entry_nodes_should_be_added(); now = time(NULL); @@ -2328,13 +3216,16 @@ entry_guards_compute_status(void) if (changed) { SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, entry) { const char *reason = digestmap_get(reasons, entry->identity); - log_info(LD_CIRC, "Summary: Entry '%s' is %s, %s%s%s, and %s.", + const char *live_msg = ""; + routerinfo_t *r = entry_is_live(entry, 0, 1, 0, &live_msg); + log_info(LD_CIRC, "Summary: Entry '%s' is %s, %s%s%s, and %s%s.", entry->nickname, entry->unreachable_since ? "unreachable" : "reachable", entry->bad_since ? "unusable" : "usable", reason ? ", ": "", reason ? reason : "", - entry_is_live(entry, 0, 1, 0) ? "live" : "not live"); + r ? "live" : "not live / ", + r ? "" : live_msg); } SMARTLIST_FOREACH_END(entry); log_info(LD_CIRC, " (%d/%d entry guards are usable/new)", num_live_entry_guards(), smartlist_len(entry_guards)); @@ -2441,7 +3332,8 @@ entry_guard_register_connect_status(const char *digest, int succeeded, if (e == entry) break; if (e->made_contact) { - routerinfo_t *r = entry_is_live(e, 0, 1, 1); + const char *msg; + routerinfo_t *r = entry_is_live(e, 0, 1, 1, &msg); if (r && e->unreachable_since) { refuse_conn = 1; e->can_retry = 1; @@ -2472,7 +3364,8 @@ static int should_add_entry_nodes = 0; void entry_nodes_should_be_added(void) { - log_info(LD_CIRC, "New EntryNodes config option detected. Will use."); + log_info(LD_CIRC, "EntryNodes config option set. Putting configured " + "relays at the front of the entry guard list."); should_add_entry_nodes = 1; } @@ -2496,7 +3389,7 @@ entry_guards_prepend_from_config(void) return; } - if (options->EntryNodes) { + { char *string = routerset_to_string(options->EntryNodes); log_info(LD_CIRC,"Adding configured EntryNodes '%s'.", string); tor_free(string); @@ -2540,8 +3433,9 @@ entry_guards_prepend_from_config(void) SMARTLIST_FOREACH(entry_routers, routerinfo_t *, ri, { add_an_entry_guard(ri, 0); }); - /* Finally, the remaining EntryNodes, unless we're strict */ - if (options->StrictEntryNodes) { + /* Finally, the remaining previously configured guards that are not in + * EntryNodes, unless we're strict in which case we drop them */ + if (options->StrictNodes) { SMARTLIST_FOREACH(old_entry_guards_not_on_list, entry_guard_t *, e, entry_guard_free(e)); } else { @@ -2555,16 +3449,30 @@ entry_guards_prepend_from_config(void) entry_guards_changed(); } -/** Return 1 if we're fine adding arbitrary routers out of the - * directory to our entry guard list. Else return 0. */ +/** Return 0 if we're fine adding arbitrary routers out of the + * directory to our entry guard list, or return 1 if we have a + * list already and we'd prefer to stick to it. + */ int -entry_list_can_grow(or_options_t *options) +entry_list_is_constrained(or_options_t *options) { - if (options->StrictEntryNodes) - return 0; + if (options->EntryNodes) + return 1; if (options->UseBridges) - return 0; - return 1; + return 1; + return 0; +} + +/* Are we dead set against changing our entry guard list, or would we + * change it if it means keeping Tor usable? */ +static int +entry_list_is_totally_static(or_options_t *options) +{ + if (options->EntryNodes && options->StrictNodes) + return 1; + if (options->UseBridges) + return 1; + return 0; } /** Pick a live (up and listed) entry guard from entry_guards. If @@ -2582,7 +3490,7 @@ choose_random_entry(cpath_build_state_t *state) routerinfo_t *r = NULL; int need_uptime = state ? state->need_uptime : 0; int need_capacity = state ? state->need_capacity : 0; - int consider_exit_family = 0; + int preferred_min, consider_exit_family = 0; if (chosen_exit) { smartlist_add(exit_family, chosen_exit); @@ -2596,36 +3504,60 @@ choose_random_entry(cpath_build_state_t *state) if (should_add_entry_nodes) entry_guards_prepend_from_config(); - if (entry_list_can_grow(options) && - (! entry_guards || - smartlist_len(entry_guards) < options->NumEntryGuards)) + if (!entry_list_is_constrained(options) && + smartlist_len(entry_guards) < options->NumEntryGuards) pick_entry_guards(); retry: smartlist_clear(live_entry_guards); SMARTLIST_FOREACH(entry_guards, entry_guard_t *, entry, { - r = entry_is_live(entry, need_uptime, need_capacity, 0); - if (r && (!consider_exit_family || !smartlist_isin(exit_family, r))) { - smartlist_add(live_entry_guards, r); - if (!entry->made_contact) { - /* Always start with the first not-yet-contacted entry - * guard. Otherwise we might add several new ones, pick - * the second new one, and now we've expanded our entry - * guard list without needing to. */ - goto choose_and_finish; + const char *msg; + r = entry_is_live(entry, need_uptime, need_capacity, 0, &msg); + if (!r) + continue; /* down, no point */ + if (consider_exit_family && smartlist_isin(exit_family, r)) + continue; /* avoid relays that are family members of our exit */ + if (options->EntryNodes && + !routerset_contains_router(options->EntryNodes, r)) { + /* We've come to the end of our preferred entry nodes. */ + if (smartlist_len(live_entry_guards)) + goto choose_and_finish; /* only choose from the ones we like */ + if (options->StrictNodes) { + /* in theory this case should never happen, since + * entry_guards_prepend_from_config() drops unwanted relays */ + tor_fragile_assert(); + } else { + log_info(LD_CIRC, + "No relays from EntryNodes available. Using others."); } - if (smartlist_len(live_entry_guards) >= options->NumEntryGuards) - break; /* we have enough */ } + smartlist_add(live_entry_guards, r); + if (!entry->made_contact) { + /* Always start with the first not-yet-contacted entry + * guard. Otherwise we might add several new ones, pick + * the second new one, and now we've expanded our entry + * guard list without needing to. */ + goto choose_and_finish; + } + if (smartlist_len(live_entry_guards) >= options->NumEntryGuards) + break; /* we have enough */ }); - /* Try to have at least 2 choices available. This way we don't - * get stuck with a single live-but-crummy entry and just keep - * using him. - * (We might get 2 live-but-crummy entry guards, but so be it.) */ - if (smartlist_len(live_entry_guards) < 2) { - if (entry_list_can_grow(options)) { + if (entry_list_is_constrained(options)) { + /* If we prefer the entry nodes we've got, and we have at least + * one choice, that's great. Use it. */ + preferred_min = 1; + } else { + /* Try to have at least 2 choices available. This way we don't + * get stuck with a single live-but-crummy entry and just keep + * using him. + * (We might get 2 live-but-crummy entry guards, but so be it.) */ + preferred_min = 2; + } + + if (smartlist_len(live_entry_guards) < preferred_min) { + if (!entry_list_is_totally_static(options)) { /* still no? try adding a new entry then */ /* XXX if guard doesn't imply fast and stable, then we need * to tell add_an_entry_guard below what we want, or it might @@ -2650,7 +3582,7 @@ choose_random_entry(cpath_build_state_t *state) need_capacity = 0; goto retry; } - if (!r && !entry_list_can_grow(options) && consider_exit_family) { + if (!r && entry_list_is_constrained(options) && consider_exit_family) { /* still no? if we're using bridges or have strictentrynodes * set, and our chosen exit is in the same family as all our * bridges/entry guards, then be flexible about families. */ @@ -2661,15 +3593,15 @@ choose_random_entry(cpath_build_state_t *state) } choose_and_finish: - if (entry_list_can_grow(options)) { + if (entry_list_is_constrained(options)) { + /* We need to weight by bandwidth, because our bridges or entryguards + * were not already selected proportional to their bandwidth. */ + r = routerlist_sl_choose_by_bandwidth(live_entry_guards, WEIGHT_FOR_GUARD); + } else { /* We choose uniformly at random here, because choose_good_entry_server() * already weights its choices by bandwidth, so we don't want to * *double*-weight our guard selection. */ r = smartlist_choose(live_entry_guards); - } else { - /* We need to weight by bandwidth, because our bridges or entryguards - * were not already selected proportional to their bandwidth. */ - r = routerlist_sl_choose_by_bandwidth(live_entry_guards, WEIGHT_FOR_GUARD); } smartlist_free(live_entry_guards); smartlist_free(exit_family); @@ -2904,8 +3836,7 @@ int getinfo_helper_entry_guards(control_connection_t *conn, const char *question, char **answer) { - int use_long_names = conn->use_long_names; - + (void) conn; if (!strcmp(question,"entry-guards") || !strcmp(question,"helper-nodes")) { smartlist_t *sl = smartlist_create(); @@ -2913,12 +3844,13 @@ getinfo_helper_entry_guards(control_connection_t *conn, char nbuf[MAX_VERBOSE_NICKNAME_LEN+1]; if (!entry_guards) entry_guards = smartlist_create(); - SMARTLIST_FOREACH(entry_guards, entry_guard_t *, e, - { + SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, e) { size_t len = MAX_VERBOSE_NICKNAME_LEN+ISO_TIME_LEN+32; char *c = tor_malloc(len); const char *status = NULL; time_t when = 0; + routerinfo_t *ri; + if (!e->made_contact) { status = "never-connected"; } else if (e->bad_since) { @@ -2927,19 +3859,17 @@ getinfo_helper_entry_guards(control_connection_t *conn, } else { status = "up"; } - if (use_long_names) { - routerinfo_t *ri = router_get_by_digest(e->identity); - if (ri) { - router_get_verbose_nickname(nbuf, ri); - } else { - nbuf[0] = '$'; - base16_encode(nbuf+1, sizeof(nbuf)-1, e->identity, DIGEST_LEN); - /* e->nickname field is not very reliable if we don't know about - * this router any longer; don't include it. */ - } + + ri = router_get_by_digest(e->identity); + if (ri) { + router_get_verbose_nickname(nbuf, ri); } else { - base16_encode(nbuf, sizeof(nbuf), e->identity, DIGEST_LEN); + nbuf[0] = '$'; + base16_encode(nbuf+1, sizeof(nbuf)-1, e->identity, DIGEST_LEN); + /* e->nickname field is not very reliable if we don't know about + * this router any longer; don't include it. */ } + if (when) { format_iso_time(tbuf, when); tor_snprintf(c, len, "%s %s %s\n", nbuf, status, tbuf); @@ -2947,7 +3877,7 @@ getinfo_helper_entry_guards(control_connection_t *conn, tor_snprintf(c, len, "%s %s\n", nbuf, status); } smartlist_add(sl, c); - }); + } SMARTLIST_FOREACH_END(e); *answer = smartlist_join_strings(sl, "", 0, NULL); SMARTLIST_FOREACH(sl, char *, c, tor_free(c)); smartlist_free(sl); diff --git a/src/or/circuitlist.c b/src/or/circuitlist.c index c55ba4dee4..fed1ccf9e6 100644 --- a/src/or/circuitlist.c +++ b/src/or/circuitlist.c @@ -380,10 +380,17 @@ static void init_circuit_base(circuit_t *circ) { circ->timestamp_created = time(NULL); + tor_gettimeofday(&circ->highres_created); circ->package_window = circuit_initial_package_window(); circ->deliver_window = CIRCWINDOW_START; + /* Initialize the cell_ewma_t structure */ + circ->n_cell_ewma.last_adjusted_tick = cell_ewma_get_tick(); + circ->n_cell_ewma.cell_count = 0.0; + circ->n_cell_ewma.heap_index = -1; + circ->n_cell_ewma.is_for_p_conn = 0; + circuit_add(circ); } @@ -408,6 +415,8 @@ origin_circuit_new(void) init_circuit_base(TO_CIRCUIT(circ)); + circ_times.last_circ_at = approx_time(); + return circ; } @@ -429,6 +438,16 @@ or_circuit_new(circid_t p_circ_id, or_connection_t *p_conn) init_circuit_base(TO_CIRCUIT(circ)); + /* Initialize the cell_ewma_t structure */ + + /* Initialize the cell counts to 0 */ + circ->p_cell_ewma.cell_count = 0.0; + circ->p_cell_ewma.last_adjusted_tick = cell_ewma_get_tick(); + circ->p_cell_ewma.is_for_p_conn = 1; + + /* It's not in any heap yet. */ + circ->p_cell_ewma.heap_index = -1; + return circ; } @@ -439,39 +458,37 @@ circuit_free(circuit_t *circ) { void *mem; size_t memlen; - tor_assert(circ); + if (!circ) + return; + if (CIRCUIT_IS_ORIGIN(circ)) { origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); mem = ocirc; memlen = sizeof(origin_circuit_t); tor_assert(circ->magic == ORIGIN_CIRCUIT_MAGIC); if (ocirc->build_state) { - if (ocirc->build_state->chosen_exit) extend_info_free(ocirc->build_state->chosen_exit); - if (ocirc->build_state->pending_final_cpath) circuit_free_cpath_node(ocirc->build_state->pending_final_cpath); } tor_free(ocirc->build_state); circuit_free_cpath(ocirc->cpath); - if (ocirc->intro_key) - crypto_free_pk_env(ocirc->intro_key); - if (ocirc->rend_data) - rend_data_free(ocirc->rend_data); + + crypto_free_pk_env(ocirc->intro_key); + rend_data_free(ocirc->rend_data); } else { or_circuit_t *ocirc = TO_OR_CIRCUIT(circ); + /* Remember cell statistics for this circuit before deallocating. */ + if (get_options()->CellStatistics) + rep_hist_buffer_stats_add_circ(circ, time(NULL)); mem = ocirc; memlen = sizeof(or_circuit_t); tor_assert(circ->magic == OR_CIRCUIT_MAGIC); - if (ocirc->p_crypto) - crypto_free_cipher_env(ocirc->p_crypto); - if (ocirc->p_digest) - crypto_free_digest_env(ocirc->p_digest); - if (ocirc->n_crypto) - crypto_free_cipher_env(ocirc->n_crypto); - if (ocirc->n_digest) - crypto_free_digest_env(ocirc->n_digest); + crypto_free_cipher_env(ocirc->p_crypto); + crypto_free_digest_env(ocirc->p_digest); + crypto_free_cipher_env(ocirc->n_crypto); + crypto_free_digest_env(ocirc->n_digest); if (ocirc->rend_splice) { or_circuit_t *other = ocirc->rend_splice; @@ -487,8 +504,7 @@ circuit_free(circuit_t *circ) cell_queue_clear(ô->p_conn_cells); } - if (circ->n_hop) - extend_info_free(circ->n_hop); + extend_info_free(circ->n_hop); tor_free(circ->n_conn_onionskin); /* Remove from map. */ @@ -498,7 +514,7 @@ circuit_free(circuit_t *circ) * "active" checks will be violated. */ cell_queue_clear(&circ->n_conn_cells); - memset(circ, 0xAA, memlen); /* poison memory */ + memset(mem, 0xAA, memlen); /* poison memory */ tor_free(mem); } @@ -541,10 +557,10 @@ circuit_free_all(void) circuit_free(global_circuitlist); global_circuitlist = next; } - if (circuits_pending_or_conns) { - smartlist_free(circuits_pending_or_conns); - circuits_pending_or_conns = NULL; - } + + smartlist_free(circuits_pending_or_conns); + circuits_pending_or_conns = NULL; + HT_CLEAR(orconn_circid_map, &orconn_circid_circuit_map); } @@ -552,18 +568,15 @@ circuit_free_all(void) static void circuit_free_cpath_node(crypt_path_t *victim) { - if (victim->f_crypto) - crypto_free_cipher_env(victim->f_crypto); - if (victim->b_crypto) - crypto_free_cipher_env(victim->b_crypto); - if (victim->f_digest) - crypto_free_digest_env(victim->f_digest); - if (victim->b_digest) - crypto_free_digest_env(victim->b_digest); - if (victim->dh_handshake_state) - crypto_dh_free(victim->dh_handshake_state); - if (victim->extend_info) - extend_info_free(victim->extend_info); + if (!victim) + return; + + crypto_free_cipher_env(victim->f_crypto); + crypto_free_cipher_env(victim->b_crypto); + crypto_free_digest_env(victim->f_digest); + crypto_free_digest_env(victim->b_digest); + crypto_dh_free(victim->dh_handshake_state); + extend_info_free(victim->extend_info); memset(victim, 0xBB, sizeof(crypt_path_t)); /* poison memory */ tor_free(victim); @@ -1083,9 +1096,9 @@ _circuit_mark_for_close(circuit_t *circ, int reason, int line, tor_assert(ocirc->rend_data); /* treat this like getting a nack from it */ log_info(LD_REND, "Failed intro circ %s to %s (awaiting ack). " - "Removing from descriptor.", - safe_str(ocirc->rend_data->onion_address), - safe_str(build_state_get_exit_nickname(ocirc->build_state))); + "Removing from descriptor.", + safe_str_client(ocirc->rend_data->onion_address), + safe_str_client(build_state_get_exit_nickname(ocirc->build_state))); rend_client_remove_intro_point(ocirc->build_state->chosen_exit, ocirc->rend_data); } @@ -1236,11 +1249,6 @@ assert_circuit_ok(const circuit_t *c) tor_assert(c == c2); } } -#if 0 /* false now that rendezvous exits are attached to p_streams */ - if (origin_circ) - for (conn = origin_circ->p_streams; conn; conn = conn->next_stream) - tor_assert(conn->_base.type == CONN_TYPE_AP); -#endif if (or_circ) for (conn = or_circ->n_streams; conn; conn = conn->next_stream) tor_assert(conn->_base.type == CONN_TYPE_EXIT); diff --git a/src/or/circuituse.c b/src/or/circuituse.c index 3acc0e9a74..59b6998b99 100644 --- a/src/or/circuituse.c +++ b/src/or/circuituse.c @@ -20,6 +20,8 @@ extern circuit_t *global_circuitlist; /* from circuitlist.c */ static void circuit_expire_old_circuits(time_t now); static void circuit_increment_failure_count(void); +long int lround(double x); + /** Return 1 if <b>circ</b> could be returned by circuit_get_best(). * Else return 0. */ @@ -113,7 +115,7 @@ circuit_is_acceptable(circuit_t *circ, edge_connection_t *conn, return 0; } } - if (exitrouter && !connection_ap_can_use_exit(conn, exitrouter)) { + if (exitrouter && !connection_ap_can_use_exit(conn, exitrouter, 0)) { /* can't exit from this router */ return 0; } @@ -263,16 +265,18 @@ circuit_conforms_to_options(const origin_circuit_t *circ, void circuit_expire_building(time_t now) { - circuit_t *victim, *circ = global_circuitlist; - time_t general_cutoff = now - get_options()->CircuitBuildTimeout; - time_t begindir_cutoff = now - get_options()->CircuitBuildTimeout/2; + circuit_t *victim, *next_circ = global_circuitlist; + /* circ_times.timeout is BUILD_TIMEOUT_INITIAL_VALUE if we haven't + * decided on a customized one yet */ + time_t general_cutoff = now - lround(circ_times.timeout_ms/1000); + time_t begindir_cutoff = now - lround(circ_times.timeout_ms/2000); time_t introcirc_cutoff = begindir_cutoff; cpath_build_state_t *build_state; - while (circ) { + while (next_circ) { time_t cutoff; - victim = circ; - circ = circ->next; + victim = next_circ; + next_circ = next_circ->next; if (!CIRCUIT_IS_ORIGIN(victim) || /* didn't originate here */ victim->marked_for_close) /* don't mess with marked circs */ continue; @@ -343,6 +347,12 @@ circuit_expire_building(time_t now) continue; break; } + } else { /* circuit not open, consider recording failure as timeout */ + int first_hop_succeeded = TO_ORIGIN_CIRCUIT(victim)->cpath && + TO_ORIGIN_CIRCUIT(victim)->cpath->state == CPATH_STATE_OPEN; + if (circuit_build_times_add_timeout(&circ_times, first_hop_succeeded, + victim->timestamp_created)) + circuit_build_times_set_timeout(&circ_times); } if (victim->n_conn) @@ -414,7 +424,7 @@ circuit_stream_is_being_handled(edge_connection_t *conn, if (exitrouter && (!need_uptime || build_state->need_uptime)) { int ok; if (conn) { - ok = connection_ap_can_use_exit(conn, exitrouter); + ok = connection_ap_can_use_exit(conn, exitrouter, 0); } else { addr_policy_result_t r = compare_addr_to_addr_policy( 0, port, exitrouter->exit_policy); @@ -431,11 +441,11 @@ circuit_stream_is_being_handled(edge_connection_t *conn, } /** Don't keep more than this many unused open circuits around. */ -#define MAX_UNUSED_OPEN_CIRCUITS 12 +#define MAX_UNUSED_OPEN_CIRCUITS 14 /** Figure out how many circuits we have open that are clean. Make * sure it's enough for all the upcoming behaviors we predict we'll have. - * But if we have too many, close the not-so-useful ones. + * But put an upper bound on the total number of circuits. */ static void circuit_predict_and_launch_new(void) @@ -517,6 +527,19 @@ circuit_predict_and_launch_new(void) circuit_launch_by_router(CIRCUIT_PURPOSE_C_GENERAL, NULL, flags); return; } + + /* Finally, check to see if we still need more circuits to learn + * a good build timeout. But if we're close to our max number we + * want, don't do another -- we want to leave a few slots open so + * we can still build circuits preemptively as needed. */ + if (num < MAX_UNUSED_OPEN_CIRCUITS-2 && + circuit_build_times_needs_circuits_now(&circ_times)) { + flags = CIRCLAUNCH_NEED_CAPACITY; + log_info(LD_CIRC, + "Have %d clean circs need another buildtime test circ.", num); + circuit_launch_by_router(CIRCUIT_PURPOSE_C_GENERAL, NULL, flags); + return; + } } /** Build a new test circuit every 5 minutes */ @@ -624,6 +647,11 @@ circuit_detach_stream(circuit_t *circ, edge_connection_t *conn) tor_fragile_assert(); } +/** If we haven't yet decided on a good timeout value for circuit + * building, we close idles circuits aggressively so we can get more + * data points. */ +#define IDLE_TIMEOUT_WHILE_LEARNING (10*60) + /** Find each circuit that has been unused for too long, or dirty * for too long and has no streams on it: mark it for close. */ @@ -631,7 +659,15 @@ static void circuit_expire_old_circuits(time_t now) { circuit_t *circ; - time_t cutoff = now - get_options()->CircuitIdleTimeout; + time_t cutoff; + + if (circuit_build_times_needs_circuits(&circ_times)) { + /* Circuits should be shorter lived if we need more of them + * for learning a good build timeout */ + cutoff = now - IDLE_TIMEOUT_WHILE_LEARNING; + } else { + cutoff = now - get_options()->CircuitIdleTimeout; + } for (circ = global_circuitlist; circ; circ = circ->next) { if (circ->marked_for_close || ! CIRCUIT_IS_ORIGIN(circ)) @@ -806,6 +842,9 @@ circuit_build_failed(origin_circuit_t *circ) "(%s:%d). I'm going to try to rotate to a better connection.", n_conn->_base.address, n_conn->_base.port); n_conn->is_bad_for_new_circs = 1; + } else { + log_info(LD_OR, + "Our circuit died before the first hop with no connection"); } if (n_conn_id) { entry_guard_register_connect_status(n_conn_id, 0, 1, time(NULL)); @@ -889,8 +928,8 @@ circuit_launch_by_router(uint8_t purpose, if (exit) info = extend_info_from_router(exit); circ = circuit_launch_by_extend_info(purpose, info, flags); - if (info) - extend_info_free(info); + + extend_info_free(info); return circ; } @@ -1064,7 +1103,7 @@ circuit_get_open_circ_or_launch(edge_connection_t *conn, need_uptime)) { log_notice(LD_APP, "No Tor server allows exit to %s:%d. Rejecting.", - safe_str(conn->socks_request->address), + safe_str_client(conn->socks_request->address), conn->socks_request->port); return -1; } @@ -1072,7 +1111,7 @@ circuit_get_open_circ_or_launch(edge_connection_t *conn, /* XXXX022 Duplicates checks in connection_ap_handshake_attach_circuit */ routerinfo_t *router = router_get_by_nickname(conn->chosen_exit_name, 1); int opt = conn->chosen_exit_optional; - if (router && !connection_ap_can_use_exit(conn, router)) { + if (router && !connection_ap_can_use_exit(conn, router, 0)) { log_fn(opt ? LOG_INFO : LOG_WARN, LD_APP, "Requested exit point '%s' would refuse request. %s.", conn->chosen_exit_name, opt ? "Trying others" : "Closing"); @@ -1105,19 +1144,14 @@ circuit_get_open_circ_or_launch(edge_connection_t *conn, if (!extend_info) { log_info(LD_REND, "No intro points for '%s': re-fetching service descriptor.", - safe_str(conn->rend_data->onion_address)); - /* Fetch both, v0 and v2 rend descriptors in parallel. Use whichever - * arrives first. Exception: When using client authorization, only - * fetch v2 descriptors.*/ + safe_str_client(conn->rend_data->onion_address)); rend_client_refetch_v2_renddesc(conn->rend_data); - if (conn->rend_data->auth_type == REND_NO_AUTH) - rend_client_refetch_renddesc(conn->rend_data->onion_address); conn->_base.state = AP_CONN_STATE_RENDDESC_WAIT; return 0; } log_info(LD_REND,"Chose '%s' as intro point for '%s'.", extend_info->nickname, - safe_str(conn->rend_data->onion_address)); + safe_str_client(conn->rend_data->onion_address)); } /* If we have specified a particular exit node for our @@ -1146,7 +1180,7 @@ circuit_get_open_circ_or_launch(edge_connection_t *conn, } if (tor_addr_from_str(&addr, conn->socks_request->address) < 0) { log_info(LD_DIR, "Broken address %s on tunnel conn. Closing.", - escaped_safe_str(conn->socks_request->address)); + escaped_safe_str_client(conn->socks_request->address)); return -1; } extend_info = extend_info_alloc(conn->chosen_exit_name+1, @@ -1188,8 +1222,7 @@ circuit_get_open_circ_or_launch(edge_connection_t *conn, flags); } - if (extend_info) - extend_info_free(extend_info); + extend_info_free(extend_info); if (desired_circuit_purpose != CIRCUIT_PURPOSE_C_GENERAL) { /* help predict this next time */ @@ -1371,7 +1404,7 @@ connection_ap_handshake_attach_circuit(edge_connection_t *conn) LOG_INFO : LOG_NOTICE; log_fn(severity, LD_APP, "Tried for %d seconds to get a connection to %s:%d. Giving up.", - conn_age, safe_str(conn->socks_request->address), + conn_age, safe_str_client(conn->socks_request->address), conn->socks_request->port); return -1; } @@ -1398,7 +1431,7 @@ connection_ap_handshake_attach_circuit(edge_connection_t *conn) } return -1; } - if (router && !connection_ap_can_use_exit(conn, router)) { + if (router && !connection_ap_can_use_exit(conn, router, 0)) { log_fn(opt ? LOG_INFO : LOG_WARN, LD_APP, "Requested exit point '%s' would refuse request. %s.", conn->chosen_exit_name, opt ? "Trying others" : "Closing"); diff --git a/src/or/command.c b/src/or/command.c index 67e463723f..b1e6fa23a3 100644 --- a/src/or/command.c +++ b/src/or/command.c @@ -395,15 +395,18 @@ command_process_relay_cell(cell_t *cell, or_connection_t *conn) * gotten no more than MAX_RELAY_EARLY_CELLS_PER_CIRCUIT of them. */ if (cell->command == CELL_RELAY_EARLY) { if (direction == CELL_DIRECTION_IN) { - /* XXX Allow an unlimited number of inbound relay_early cells for - * now, for hidden service compatibility. See bug 1038. -RD */ + /* Allow an unlimited number of inbound relay_early cells, + * for hidden service compatibility. There isn't any way to make + * a long circuit through inbound relay_early cells anyway. See + * bug 1038. -RD */ } else { or_circuit_t *or_circ = TO_OR_CIRCUIT(circ); if (or_circ->remaining_relay_early_cells == 0) { log_fn(LOG_PROTOCOL_WARN, LD_OR, "Received too many RELAY_EARLY cells on circ %d from %s:%d." " Closing circuit.", - cell->circ_id, safe_str(conn->_base.address), conn->_base.port); + cell->circ_id, safe_str(conn->_base.address), + conn->_base.port); circuit_mark_for_close(circ, END_CIRC_REASON_TORPROTOCOL); return; } @@ -511,7 +514,8 @@ command_process_versions_cell(var_cell_t *cell, or_connection_t *conn) conn->handshake_state->received_versions = 1; log_info(LD_OR, "Negotiated version %d with %s:%d; sending NETINFO.", - highest_supported_version, safe_str(conn->_base.address), + highest_supported_version, + safe_str_client(conn->_base.address), conn->_base.port); tor_assert(conn->link_proto >= 2); @@ -625,8 +629,8 @@ command_process_netinfo_cell(cell_t *cell, or_connection_t *conn) else log_info(LD_OR, "Got good NETINFO cell from %s:%d; OR connection is now " "open, using protocol version %d", - safe_str(conn->_base.address), conn->_base.port, - (int)conn->link_proto); + safe_str_client(conn->_base.address), + conn->_base.port, (int)conn->link_proto); assert_connection_ok(TO_CONN(conn),time(NULL)); } diff --git a/src/or/config.c b/src/or/config.c index 9d9e029b81..b8356e2fc0 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -66,6 +66,7 @@ static config_abbrev_t _option_abbrevs[] = { PLURAL(RendExcludeNode), PLURAL(StrictEntryNode), PLURAL(StrictExitNode), + PLURAL(StrictNode), { "l", "Log", 1, 0}, { "AllowUnverifiedNodes", "AllowInvalidNodes", 0, 0}, { "AutomapHostSuffixes", "AutomapHostsSuffixes", 0, 0}, @@ -83,10 +84,12 @@ static config_abbrev_t _option_abbrevs[] = { { "NumEntryNodes", "NumEntryGuards", 0, 0}, { "ResolvConf", "ServerDNSResolvConfFile", 0, 1}, { "SearchDomains", "ServerDNSSearchDomains", 0, 1}, - { "ServerDNSAllowBrokenResolvConf", "ServerDNSAllowBrokenConfig", 0, 0 }, + { "ServerDNSAllowBrokenResolvConf", "ServerDNSAllowBrokenConfig", 0, 0}, { "PreferTunnelledDirConns", "PreferTunneledDirConns", 0, 0}, { "BridgeAuthoritativeDirectory", "BridgeAuthoritativeDir", 0, 0}, { "HashedControlPassword", "__HashedControlSessionPassword", 1, 0}, + { "StrictEntryNodes", "StrictNodes", 0, 1}, + { "StrictExitNodes", "StrictNodes", 0, 1}, { NULL, NULL, 0, 0}, }; @@ -134,6 +137,7 @@ static config_var_t _option_vars[] = { V(AccountingMax, MEMUNIT, "0 bytes"), V(AccountingStart, STRING, NULL), V(Address, STRING, NULL), + V(AllowDotExit, BOOL, "0"), V(AllowInvalidNodes, CSV, "middle,rendezvous"), V(AllowNonRFC953Hostnames, BOOL, "0"), V(AllowSingleHopCircuits, BOOL, "0"), @@ -162,10 +166,14 @@ static config_var_t _option_vars[] = { V(BridgePassword, STRING, NULL), V(BridgeRecordUsageByCountry, BOOL, "1"), V(BridgeRelay, BOOL, "0"), - V(CircuitBuildTimeout, INTERVAL, "1 minute"), + V(CellStatistics, BOOL, "0"), + V(CircuitBuildTimeout, INTERVAL, "0"), V(CircuitIdleTimeout, INTERVAL, "1 hour"), + V(CircuitStreamTimeout, INTERVAL, "0"), + V(CircuitPriorityHalflife, DOUBLE, "-100.0"), /*negative:'Use default'*/ V(ClientDNSRejectInternalAddresses, BOOL,"1"), V(ClientOnly, BOOL, "0"), + V(ConsensusParams, STRING, NULL), V(ConnLimit, UINT, "1000"), V(ConstrainedSockets, BOOL, "0"), V(ConstrainedSockSize, MEMUNIT, "8192"), @@ -186,18 +194,19 @@ static config_var_t _option_vars[] = { V(DirPort, UINT, "0"), V(DirPortFrontPage, FILENAME, NULL), OBSOLETE("DirPostPeriod"), -#ifdef ENABLE_GEOIP_STATS - V(DirRecordUsageByCountry, BOOL, "0"), - V(DirRecordUsageGranularity, UINT, "4"), - V(DirRecordUsageRetainIPs, INTERVAL, "14 days"), - V(DirRecordUsageSaveInterval, INTERVAL, "6 hours"), -#endif + OBSOLETE("DirRecordUsageByCountry"), + OBSOLETE("DirRecordUsageGranularity"), + OBSOLETE("DirRecordUsageRetainIPs"), + OBSOLETE("DirRecordUsageSaveInterval"), + V(DirReqStatistics, BOOL, "0"), VAR("DirServer", LINELIST, DirServers, NULL), + V(DisableAllSwap, BOOL, "0"), V(DNSPort, UINT, "0"), V(DNSListenAddress, LINELIST, NULL), V(DownloadExtraInfo, BOOL, "0"), V(EnforceDistinctSubnets, BOOL, "1"), V(EntryNodes, ROUTERSET, NULL), + V(EntryStatistics, BOOL, "0"), V(TestingEstimatedDescriptorPropagationTime, INTERVAL, "10 minutes"), V(ExcludeNodes, ROUTERSET, NULL), V(ExcludeExitNodes, ROUTERSET, NULL), @@ -205,12 +214,16 @@ static config_var_t _option_vars[] = { V(ExitNodes, ROUTERSET, NULL), V(ExitPolicy, LINELIST, NULL), V(ExitPolicyRejectPrivate, BOOL, "1"), + V(ExitPortStatistics, BOOL, "0"), + V(ExtraInfoStatistics, BOOL, "0"), + V(FallbackNetworkstatusFile, FILENAME, SHARE_DATADIR PATH_SEPARATOR "tor" PATH_SEPARATOR "fallback-consensus"), V(FascistFirewall, BOOL, "0"), V(FirewallPorts, CSV, ""), V(FastFirstHopPK, BOOL, "1"), V(FetchDirInfoEarly, BOOL, "0"), + V(FetchDirInfoExtraEarly, BOOL, "0"), V(FetchServerDescriptors, BOOL, "1"), V(FetchHidServDescriptors, BOOL, "1"), V(FetchUselessDescriptors, BOOL, "0"), @@ -222,6 +235,8 @@ static config_var_t _option_vars[] = { #endif OBSOLETE("Group"), V(HardwareAccel, BOOL, "0"), + V(AccelName, STRING, NULL), + V(AccelDir, FILENAME, NULL), V(HashedControlPassword, LINELIST, NULL), V(HidServDirectoryV2, BOOL, "1"), VAR("HiddenServiceDir", LINELIST_S, RendConfigLines, NULL), @@ -233,11 +248,15 @@ static config_var_t _option_vars[] = { VAR("HiddenServiceAuthorizeClient",LINELIST_S,RendConfigLines, NULL), V(HidServAuth, LINELIST, NULL), V(HSAuthoritativeDir, BOOL, "0"), - V(HSAuthorityRecordStats, BOOL, "0"), + OBSOLETE("HSAuthorityRecordStats"), V(HttpProxy, STRING, NULL), V(HttpProxyAuthenticator, STRING, NULL), V(HttpsProxy, STRING, NULL), V(HttpsProxyAuthenticator, STRING, NULL), + V(Socks4Proxy, STRING, NULL), + V(Socks5Proxy, STRING, NULL), + V(Socks5ProxyUsername, STRING, NULL), + V(Socks5ProxyPassword, STRING, NULL), OBSOLETE("IgnoreVersion"), V(KeepalivePeriod, INTERVAL, "5 minutes"), VAR("Log", LINELIST, Logs, NULL), @@ -265,6 +284,8 @@ static config_var_t _option_vars[] = { V(ORPort, UINT, "0"), V(OutboundBindAddress, STRING, NULL), OBSOLETE("PathlenCoinWeight"), + V(PerConnBWBurst, MEMUNIT, "0"), + V(PerConnBWRate, MEMUNIT, "0"), V(PidFile, STRING, NULL), V(TestingTorNetwork, BOOL, "0"), V(PreferTunneledDirConns, BOOL, "1"), @@ -288,7 +309,7 @@ static config_var_t _option_vars[] = { OBSOLETE("RouterFile"), V(RunAsDaemon, BOOL, "0"), V(RunTesting, BOOL, "0"), - V(SafeLogging, BOOL, "1"), + V(SafeLogging, STRING, "1"), V(SafeSocks, BOOL, "0"), V(ServerDNSAllowBrokenConfig, BOOL, "1"), V(ServerDNSAllowNonRFC953Hostnames, BOOL,"0"), @@ -304,8 +325,7 @@ static config_var_t _option_vars[] = { V(SocksPort, UINT, "9050"), V(SocksTimeout, INTERVAL, "2 minutes"), OBSOLETE("StatusFetchPeriod"), - V(StrictEntryNodes, BOOL, "0"), - V(StrictExitNodes, BOOL, "0"), + V(StrictNodes, BOOL, "0"), OBSOLETE("SysLog"), V(TestSocks, BOOL, "0"), OBSOLETE("TestVia"), @@ -330,6 +350,7 @@ static config_var_t _option_vars[] = { V(V3AuthDistDelay, INTERVAL, "5 minutes"), V(V3AuthNIntervalsValid, UINT, "3"), V(V3AuthUseLegacyKey, BOOL, "0"), + V(V3BandwidthsFile, FILENAME, NULL), VAR("VersioningAuthoritativeDirectory",BOOL,VersioningAuthoritativeDir, "0"), V(VirtualAddrNetwork, STRING, "127.192.0.0/10"), V(WarnPlaintextPorts, CSV, "23,109,110,143"), @@ -340,6 +361,7 @@ static config_var_t _option_vars[] = { VAR("__HashedControlSessionPassword", LINELIST, HashedControlSessionPassword, NULL), V(MinUptimeHidServDirectoryV2, INTERVAL, "24 hours"), + { NULL, CONFIG_TYPE_OBSOLETE, 0, NULL } }; @@ -396,6 +418,10 @@ static config_var_t _state_vars[] = { V(LastRotatedOnionKey, ISOTIME, NULL), V(LastWritten, ISOTIME, NULL), + V(TotalBuildTimes, UINT, NULL), + VAR("CircuitBuildTimeBin", LINELIST_S, BuildtimeHistogram, NULL), + VAR("BuildtimeHistogram", LINELIST_V, BuildtimeHistogram, NULL), + { NULL, CONFIG_TYPE_OBSOLETE, 0, NULL } }; @@ -410,213 +436,6 @@ typedef struct config_var_description_t { const char *description; } config_var_description_t; -/** Descriptions of the configuration options, to be displayed by online - * option browsers */ -/* XXXX022 did anybody want this? at all? If not, kill it.*/ -static config_var_description_t options_description[] = { - /* ==== general options */ - { "AvoidDiskWrites", "If non-zero, try to write to disk less frequently than" - " we would otherwise." }, - { "BandwidthRate", "A token bucket limits the average incoming bandwidth on " - "this node to the specified number of bytes per second." }, - { "BandwidthBurst", "Limit the maximum token buffer size (also known as " - "burst) to the given number of bytes." }, - { "ConnLimit", "Minimum number of simultaneous sockets we must have." }, - { "ConstrainedSockets", "Shrink tx and rx buffers for sockets to avoid " - "system limits on vservers and related environments. See man page for " - "more information regarding this option." }, - { "ConstrainedSockSize", "Limit socket buffers to this size when " - "ConstrainedSockets is enabled." }, - /* ControlListenAddress */ - { "ControlPort", "If set, Tor will accept connections from the same machine " - "(localhost only) on this port, and allow those connections to control " - "the Tor process using the Tor Control Protocol (described in " - "control-spec.txt).", }, - { "CookieAuthentication", "If this option is set to 1, don't allow any " - "connections to the control port except when the connecting process " - "can read a file that Tor creates in its data directory." }, - { "DataDirectory", "Store working data, state, keys, and caches here." }, - { "DirServer", "Tor only trusts directories signed with one of these " - "servers' keys. Used to override the standard list of directory " - "authorities." }, - /* { "FastFirstHopPK", "" }, */ - /* FetchServerDescriptors, FetchHidServDescriptors, - * FetchUselessDescriptors */ - { "HardwareAccel", "If set, Tor tries to use hardware crypto accelerators " - "when it can." }, - /* HashedControlPassword */ - { "HTTPProxy", "Force Tor to make all HTTP directory requests through this " - "host:port (or host:80 if port is not set)." }, - { "HTTPProxyAuthenticator", "A username:password pair to be used with " - "HTTPProxy." }, - { "HTTPSProxy", "Force Tor to make all TLS (SSL) connections through this " - "host:port (or host:80 if port is not set)." }, - { "HTTPSProxyAuthenticator", "A username:password pair to be used with " - "HTTPSProxy." }, - { "KeepalivePeriod", "Send a padding cell every N seconds to keep firewalls " - "from closing our connections while Tor is not in use." }, - { "Log", "Where to send logging messages. Format is " - "minSeverity[-maxSeverity] (stderr|stdout|syslog|file FILENAME)." }, - { "OutboundBindAddress", "Make all outbound connections originate from the " - "provided IP address (only useful for multiple network interfaces)." }, - { "PIDFile", "On startup, write our PID to this file. On clean shutdown, " - "remove the file." }, - { "PreferTunneledDirConns", "If non-zero, avoid directory servers that " - "don't support tunneled connections." }, - /* PreferTunneledDirConns */ - /* ProtocolWarnings */ - /* RephistTrackTime */ - { "RunAsDaemon", "If set, Tor forks and daemonizes to the background when " - "started. Unix only." }, - { "SafeLogging", "If set to 0, Tor logs potentially sensitive strings " - "rather than replacing them with the string [scrubbed]." }, - { "TunnelDirConns", "If non-zero, when a directory server we contact " - "supports it, we will build a one-hop circuit and make an encrypted " - "connection via its ORPort." }, - { "User", "On startup, setuid to this user." }, - - /* ==== client options */ - { "AllowInvalidNodes", "Where on our circuits should Tor allow servers " - "that the directory authorities haven't called \"valid\"?" }, - { "AllowNonRFC953Hostnames", "If set to 1, we don't automatically reject " - "hostnames for having invalid characters." }, - /* CircuitBuildTimeout, CircuitIdleTimeout */ - { "ClientOnly", "If set to 1, Tor will under no circumstances run as a " - "server, even if ORPort is enabled." }, - { "EntryNodes", "A list of preferred entry nodes to use for the first hop " - "in circuits, when possible." }, - /* { "EnforceDistinctSubnets" , "" }, */ - { "ExitNodes", "A list of preferred nodes to use for the last hop in " - "circuits, when possible." }, - { "ExcludeNodes", "A list of nodes never to use when building a circuit." }, - { "FascistFirewall", "If set, Tor will only create outgoing connections to " - "servers running on the ports listed in FirewallPorts." }, - { "FirewallPorts", "A list of ports that we can connect to. Only used " - "when FascistFirewall is set." }, - { "LongLivedPorts", "A list of ports for services that tend to require " - "high-uptime connections." }, - { "MapAddress", "Force Tor to treat all requests for one address as if " - "they were for another." }, - { "NewCircuitPeriod", "Force Tor to consider whether to build a new circuit " - "every NUM seconds." }, - { "MaxCircuitDirtiness", "Do not attach new streams to a circuit that has " - "been used more than this many seconds ago." }, - /* NatdPort, NatdListenAddress */ - { "NodeFamily", "A list of servers that constitute a 'family' and should " - "never be used in the same circuit." }, - { "NumEntryGuards", "How many entry guards should we keep at a time?" }, - /* PathlenCoinWeight */ - { "ReachableAddresses", "Addresses we can connect to, as IP/bits:port-port. " - "By default, we assume all addresses are reachable." }, - /* reachablediraddresses, reachableoraddresses. */ - /* SafeSOCKS */ - { "SOCKSPort", "The port where we listen for SOCKS connections from " - "applications." }, - { "SOCKSListenAddress", "Bind to this address to listen to connections from " - "SOCKS-speaking applications." }, - { "SOCKSPolicy", "Set an entry policy to limit which addresses can connect " - "to the SOCKSPort." }, - /* SocksTimeout */ - { "StrictExitNodes", "If set, Tor will fail to operate when none of the " - "configured ExitNodes can be used." }, - { "StrictEntryNodes", "If set, Tor will fail to operate when none of the " - "configured EntryNodes can be used." }, - /* TestSocks */ - { "TrackHostsExit", "Hosts and domains which should, if possible, be " - "accessed from the same exit node each time we connect to them." }, - { "TrackHostsExitExpire", "Time after which we forget which exit we were " - "using to connect to hosts in TrackHostsExit." }, - /* "TransPort", "TransListenAddress */ - { "UseEntryGuards", "Set to 0 if we want to pick from the whole set of " - "servers for the first position in each circuit, rather than picking a " - "set of 'Guards' to prevent profiling attacks." }, - - /* === server options */ - { "Address", "The advertised (external) address we should use." }, - /* Accounting* options. */ - /* AssumeReachable */ - { "ContactInfo", "Administrative contact information to advertise for this " - "server." }, - { "ExitPolicy", "Address/port ranges for which to accept or reject outgoing " - "connections on behalf of Tor users." }, - /* { "ExitPolicyRejectPrivate, "" }, */ - { "MaxAdvertisedBandwidth", "If set, we will not advertise more than this " - "amount of bandwidth for our bandwidth rate, regardless of how much " - "bandwidth we actually detect." }, - { "MaxOnionsPending", "Reject new attempts to extend circuits when we " - "already have this many pending." }, - { "MyFamily", "Declare a list of other servers as belonging to the same " - "family as this one, so that clients will not use two from the same " - "family in the same circuit." }, - { "Nickname", "Set the server nickname." }, - { "NoPublish", "{DEPRECATED}" }, - { "NumCPUs", "How many processes to use at once for public-key crypto." }, - { "ORPort", "Advertise this port to listen for connections from Tor clients " - "and servers." }, - { "ORListenAddress", "Bind to this address to listen for connections from " - "clients and servers, instead of the default 0.0.0.0:ORPort." }, - { "PublishServerDescriptor", "Set to 0 to keep the server from " - "uploading info to the directory authorities." }, - /* ServerDNS: DetectHijacking, ResolvConfFile, SearchDomains */ - { "ShutdownWaitLength", "Wait this long for clients to finish when " - "shutting down because of a SIGINT." }, - - /* === directory cache options */ - { "DirPort", "Serve directory information from this port, and act as a " - "directory cache." }, - { "DirPortFrontPage", "Serve a static html disclaimer on DirPort." }, - { "DirListenAddress", "Bind to this address to listen for connections from " - "clients and servers, instead of the default 0.0.0.0:DirPort." }, - { "DirPolicy", "Set a policy to limit who can connect to the directory " - "port." }, - - /* Authority options: AuthDirBadExit, AuthDirInvalid, AuthDirReject, - * AuthDirRejectUnlisted, AuthDirListBadExits, AuthoritativeDirectory, - * DirAllowPrivateAddresses, HSAuthoritativeDir, - * NamingAuthoritativeDirectory, RecommendedVersions, - * RecommendedClientVersions, RecommendedServerVersions, RendPostPeriod, - * RunTesting, V1AuthoritativeDirectory, VersioningAuthoritativeDirectory, */ - - /* Hidden service options: HiddenService: dir,excludenodes, nodes, - * options, port. PublishHidServDescriptor */ - - /* Nonpersistent options: __LeaveStreamsUnattached, __AllDirActionsPrivate */ - { NULL, NULL }, -}; - -/** Online description of state variables. */ -static config_var_description_t state_description[] = { - { "AccountingBytesReadInInterval", - "How many bytes have we read in this accounting period?" }, - { "AccountingBytesWrittenInInterval", - "How many bytes have we written in this accounting period?" }, - { "AccountingExpectedUsage", - "How many bytes did we expect to use per minute? (0 for no estimate.)" }, - { "AccountingIntervalStart", "When did this accounting period begin?" }, - { "AccountingSecondsActive", "How long have we been awake in this period?" }, - - { "BWHistoryReadEnds", "When does the last-recorded read-interval end?" }, - { "BWHistoryReadInterval", "How long is each read-interval (in seconds)?" }, - { "BWHistoryReadValues", "Number of bytes read in each interval." }, - { "BWHistoryWriteEnds", "When does the last-recorded write-interval end?" }, - { "BWHistoryWriteInterval", "How long is each write-interval (in seconds)?"}, - { "BWHistoryWriteValues", "Number of bytes written in each interval." }, - - { "EntryGuard", "One of the nodes we have chosen as a fixed entry" }, - { "EntryGuardDownSince", - "The last entry guard has been unreachable since this time." }, - { "EntryGuardUnlistedSince", - "The last entry guard has been unusable since this time." }, - - { "LastRotatedOnionKey", - "The last time at which we changed the medium-term private key used for " - "building circuits." }, - { "LastWritten", "When was this state file last regenerated?" }, - - { "TorVersion", "Which version of Tor generated this state file?" }, - { NULL, NULL }, -}; - /** Type of a callback to validate whether a given configuration is * well-formed and consistent. See options_trial_assign() for documentation * of arguments. */ @@ -635,8 +454,6 @@ typedef struct { config_var_t *vars; /**< List of variables we recognize, their default * values, and where we stick them in the structure. */ validate_fn_t validate_fn; /**< Function to validate config. */ - /** Documentation for configuration variables. */ - config_var_description_t *descriptions; /** If present, extra is a LINELIST variable for unrecognized * lines. Otherwise, unrecognized lines are an error. */ config_var_t *extra; @@ -700,20 +517,6 @@ static uint64_t config_parse_memunit(const char *s, int *ok); static int config_parse_interval(const char *s, int *ok); static void init_libevent(void); static int opt_streq(const char *s1, const char *s2); -/** Versions of libevent. */ -typedef enum { - /* Note: we compare these, so it's important that "old" precede everything, - * and that "other" come last. */ - LE_OLD=0, LE_10C, LE_10D, LE_10E, LE_11, LE_11A, LE_11B, LE_12, LE_12A, - LE_13, LE_13A, LE_13B, LE_13C, LE_13D, LE_13E, - LE_140, LE_141, LE_142, LE_143, LE_144, LE_145, LE_146, LE_147, LE_148, - LE_1499, - LE_OTHER -} le_version_t; -static le_version_t decode_libevent_version(const char *v, int *bincompat_out); -#if defined(HAVE_EVENT_GET_VERSION) && defined(HAVE_EVENT_GET_METHOD) -static void check_libevent_version(const char *m, int server); -#endif /** Magic value for or_options_t. */ #define OR_OPTIONS_MAGIC 9090909 @@ -726,7 +529,6 @@ static config_format_t options_format = { _option_abbrevs, _option_vars, (validate_fn_t)options_validate, - options_description, NULL }; @@ -747,7 +549,6 @@ static config_format_t state_format = { _state_abbrevs, _state_vars, (validate_fn_t)or_state_validate, - state_description, &state_extra_var, }; @@ -812,13 +613,13 @@ set_options(or_options_t *new_val, char **msg) "Acting on config options left us in a broken state. Dying."); exit(1); } - if (old_options) - config_free(&options_format, old_options); + + config_free(&options_format, old_options); return 0; } -extern const char tor_svn_revision[]; /* from tor_main.c */ +extern const char tor_git_revision[]; /* from tor_main.c */ /** The version of this Tor process, as parsed. */ static char *_version = NULL; @@ -828,10 +629,10 @@ const char * get_version(void) { if (_version == NULL) { - if (strlen(tor_svn_revision)) { - size_t len = strlen(VERSION)+strlen(tor_svn_revision)+8; + if (strlen(tor_git_revision)) { + size_t len = strlen(VERSION)+strlen(tor_git_revision)+16; _version = tor_malloc(len); - tor_snprintf(_version, len, "%s (r%s)", VERSION, tor_svn_revision); + tor_snprintf(_version, len, "%s (git-%s)", VERSION, tor_git_revision); } else { _version = tor_strdup(VERSION); } @@ -844,8 +645,10 @@ get_version(void) static void or_options_free(or_options_t *options) { - if (options->_ExcludeExitNodesUnion) - routerset_free(options->_ExcludeExitNodesUnion); + if (!options) + return; + + routerset_free(options->_ExcludeExitNodesUnion); config_free(&options_format, options); } @@ -854,43 +657,72 @@ or_options_free(or_options_t *options) void config_free_all(void) { - if (global_options) { - or_options_free(global_options); - global_options = NULL; - } - if (global_state) { - config_free(&state_format, global_state); - global_state = NULL; - } - if (global_cmdline_options) { - config_free_lines(global_cmdline_options); - global_cmdline_options = NULL; - } + or_options_free(global_options); + global_options = NULL; + + config_free(&state_format, global_state); + global_state = NULL; + + config_free_lines(global_cmdline_options); + global_cmdline_options = NULL; + tor_free(torrc_fname); tor_free(_version); tor_free(global_dirfrontpagecontents); } -/** If options->SafeLogging is on, return a not very useful string, - * else return address. +/** Make <b>address</b> -- a piece of information related to our operation as + * a client -- safe to log according to the settings in options->SafeLogging, + * and return it. + * + * (We return "[scrubbed]" if SafeLogging is "1", and address otherwise.) + */ +const char * +safe_str_client(const char *address) +{ + tor_assert(address); + if (get_options()->_SafeLogging == SAFELOG_SCRUB_ALL) + return "[scrubbed]"; + else + return address; +} + +/** Make <b>address</b> -- a piece of information of unspecified sensitivity + * -- safe to log according to the settings in options->SafeLogging, and + * return it. + * + * (We return "[scrubbed]" if SafeLogging is anything besides "0", and address + * otherwise.) */ const char * safe_str(const char *address) { tor_assert(address); - if (get_options()->SafeLogging) + if (get_options()->_SafeLogging != SAFELOG_SCRUB_NONE) return "[scrubbed]"; else return address; } +/** Equivalent to escaped(safe_str_client(address)). See reentrancy note on + * escaped(): don't use this outside the main thread, or twice in the same + * log statement. */ +const char * +escaped_safe_str_client(const char *address) +{ + if (get_options()->_SafeLogging == SAFELOG_SCRUB_ALL) + return "[scrubbed]"; + else + return escaped(address); +} + /** Equivalent to escaped(safe_str(address)). See reentrancy note on * escaped(): don't use this outside the main thread, or twice in the same * log statement. */ const char * escaped_safe_str(const char *address) { - if (get_options()->SafeLogging) + if (get_options()->_SafeLogging != SAFELOG_SCRUB_NONE) return "[scrubbed]"; else return escaped(address); @@ -1105,6 +937,15 @@ options_act_reversible(or_options_t *old_options, char **msg) } #endif + /* Attempt to lock all current and future memory with mlockall() only once */ + if (options->DisableAllSwap) { + if (tor_mlockall() == -1) { + *msg = tor_strdup("DisableAllSwap failure. Do you have proper " + "permissions?"); + goto done; + } + } + /* Setuid/setgid as appropriate */ if (options->User) { if (switch_id(options->User) != 0) { @@ -1235,7 +1076,6 @@ get_effective_bwrate(or_options_t *options) bw = options->MaxAdvertisedBandwidth; if (options->RelayBandwidthRate > 0 && bw > options->RelayBandwidthRate) bw = options->RelayBandwidthRate; - /* ensure_bandwidth_cap() makes sure that this cast can't overflow. */ return (uint32_t)bw; } @@ -1313,14 +1153,14 @@ options_act(or_options_t *old_options) return 0; /* Finish backgrounding the process */ - if (running_tor && options->RunAsDaemon) { + if (options->RunAsDaemon) { /* We may be calling this for the n'th time (on SIGHUP), but it's safe. */ finish_daemon(options->DataDirectory); } /* Write our PID to the PID file. If we do not have write permissions we * will log a warning */ - if (running_tor && options->PidFile) + if (options->PidFile) write_pidfile(options->PidFile); /* Register addressmap directives */ @@ -1353,11 +1193,26 @@ options_act(or_options_t *old_options) if (accounting_is_enabled(options)) configure_accounting(time(NULL)); + /* Change the cell EWMA settings */ + cell_ewma_set_scale_factor(options, networkstatus_get_latest_consensus()); + /* Check for transitions that need action. */ if (old_options) { - if (options->UseEntryGuards && !old_options->UseEntryGuards) { + + if ((options->UseEntryGuards && !old_options->UseEntryGuards) || + (options->ExcludeNodes && + !routerset_equal(old_options->ExcludeNodes,options->ExcludeNodes)) || + (options->ExcludeExitNodes && + !routerset_equal(old_options->ExcludeExitNodes, + options->ExcludeExitNodes)) || + (options->EntryNodes && + !routerset_equal(old_options->EntryNodes, options->EntryNodes)) || + (options->ExitNodes && + !routerset_equal(old_options->ExitNodes, options->ExitNodes)) || + options->StrictNodes != old_options->StrictNodes) { log_info(LD_CIRC, - "Switching to entry guards; abandoning previous circuits"); + "Changed to using entry guards, or changed preferred or " + "excluded node lists. Abandoning previous circuits."); circuit_mark_all_unused_circs(); circuit_expire_all_dirty_circs(); } @@ -1411,13 +1266,29 @@ options_act(or_options_t *old_options) geoip_load_file(actual_fname, options); tor_free(actual_fname); } -#ifdef ENABLE_GEOIP_STATS - log_warn(LD_CONFIG, "We are configured to measure GeoIP statistics, but " - "the way these statistics are measured has changed " - "significantly in later versions of Tor. The results may not be " - "as expected if you are used to later versions. Be sure you " - "know what you are doing."); -#endif + + if (options->DirReqStatistics && !geoip_is_loaded()) { + /* Check if GeoIP database could be loaded. */ + log_warn(LD_CONFIG, "Configured to measure directory request " + "statistics, but no GeoIP database found!"); + return -1; + } + + if (options->EntryStatistics) { + if (should_record_bridge_info(options)) { + /* Don't allow measuring statistics on entry guards when configured + * as bridge. */ + log_warn(LD_CONFIG, "Bridges cannot be configured to measure " + "additional GeoIP statistics as entry guards."); + return -1; + } else if (!geoip_is_loaded()) { + /* Check if GeoIP database could be loaded. */ + log_warn(LD_CONFIG, "Configured to measure entry node statistics, " + "but no GeoIP database found!"); + return -1; + } + } + /* Check if we need to parse and add the EntryNodes config option. */ if (options->EntryNodes && (!old_options || @@ -1490,7 +1361,10 @@ expand_abbrev(config_format_t *fmt, const char *option, int command_line, fmt->abbrevs[i].abbreviated, fmt->abbrevs[i].full); } - return fmt->abbrevs[i].full; + /* Keep going through the list in case we want to rewrite it more. + * (We could imagine recursing here, but I don't want to get the + * user into an infinite loop if we craft our list wrong.) */ + option = fmt->abbrevs[i].full; } } return option; @@ -1535,7 +1409,10 @@ config_get_commandlines(int argc, char **argv, config_line_t **result) *new = tor_malloc_zero(sizeof(config_line_t)); s = argv[i]; - while (*s == '-') + /* Each keyword may be prefixed with one or two dashes. */ + if (*s == '-') + s++; + if (*s == '-') s++; (*new)->key = tor_strdup(expand_abbrev(&options_format, s, 1, 1)); @@ -1627,19 +1504,6 @@ config_free_lines(config_line_t *front) } } -/** Return the description for a given configuration variable, or NULL if no - * description exists. */ -static const char * -config_find_description(config_format_t *fmt, const char *name) -{ - int i; - for (i=0; fmt->descriptions[i].name; ++i) { - if (!strcasecmp(name, fmt->descriptions[i].name)) - return fmt->descriptions[i].description; - } - return NULL; -} - /** If <b>key</b> is a configuration option, return the corresponding * config_var_t. Otherwise, if <b>key</b> is a non-standard abbreviation, * warn, and return the corresponding config_var_t. Otherwise return NULL. @@ -2307,20 +2171,10 @@ list_torrc_options(void) smartlist_t *lines = smartlist_create(); for (i = 0; _option_vars[i].name; ++i) { config_var_t *var = &_option_vars[i]; - const char *desc; if (var->type == CONFIG_TYPE_OBSOLETE || var->type == CONFIG_TYPE_LINELIST_V) continue; - desc = config_find_description(&options_format, var->name); printf("%s\n", var->name); - if (desc) { - wrap_string(lines, desc, 76, " ", " "); - SMARTLIST_FOREACH(lines, char *, cp, { - printf("%s", cp); - tor_free(cp); - }); - smartlist_clear(lines); - } } smartlist_free(lines); } @@ -2568,7 +2422,10 @@ config_free(config_format_t *fmt, void *options) { int i; - tor_assert(options); + if (!options) + return; + + tor_assert(fmt); for (i=0; fmt->vars[i].name; ++i) option_clear(fmt, options, &(fmt->vars[i])); @@ -2670,6 +2527,8 @@ is_listening_on_low_port(uint16_t port_option, const config_line_t *listen_options) { #ifdef MS_WINDOWS + (void) port_option; + (void) listen_options; return 0; /* No port is too low for windows. */ #else const config_line_t *l; @@ -2719,7 +2578,6 @@ config_dump(config_format_t *fmt, void *options, int minimal, config_line_t *line, *assigned; char *result; int i; - const char *desc; char *msg = NULL; defaults = config_alloc(fmt); @@ -2747,14 +2605,8 @@ config_dump(config_format_t *fmt, void *options, int minimal, option_is_same(fmt, options, defaults, fmt->vars[i].name)) comment_option = 1; - desc = config_find_description(fmt, fmt->vars[i].name); line = assigned = get_assigned_option(fmt, options, fmt->vars[i].name, 1); - if (line && desc) { - /* Only dump the description if there's something to describe. */ - wrap_string(elements, desc, 78, "# ", "# "); - } - for (; line; line = line->next) { size_t len = strlen(line->key) + strlen(line->value) + 5; char *tmp; @@ -2795,7 +2647,7 @@ config_dump(config_format_t *fmt, void *options, int minimal, * the configuration in <b>options</b>. If <b>minimal</b> is true, do not * include options that are the same as Tor's defaults. */ -static char * +char * options_dump(or_options_t *options, int minimal) { return config_dump(&options_format, options, minimal, 0); @@ -2896,15 +2748,14 @@ compute_publishserverdescriptor(or_options_t *options) /** Highest allowable value for RendPostPeriod. */ #define MAX_DIR_PERIOD (MIN_ONION_KEY_LIFETIME/2) -/** Lowest allowable value for CircuitBuildTimeout; values too low will - * increase network load because of failing connections being retried, and - * might prevent users from connecting to the network at all. */ -#define MIN_CIRCUIT_BUILD_TIMEOUT 30 - /** Lowest allowable value for MaxCircuitDirtiness; if this is too low, Tor * will generate too many circuits and potentially overload the network. */ #define MIN_MAX_CIRCUIT_DIRTINESS 10 +/** Lowest allowable value for CircuitStreamTimeout; if this is too low, Tor + * will generate too many circuits and potentially overload the network. */ +#define MIN_CIRCUIT_STREAM_TIMEOUT 10 + /** Return 0 if every setting in <b>options</b> is reasonable, and a * permissible transition from <b>old_options</b>. Else return -1. * Should have no side effects, except for normalizing the contents of @@ -2941,7 +2792,7 @@ options_validate(or_options_t *old_options, or_options_t *options, !strcmpstart(uname, "Windows Me"))) { log(LOG_WARN, LD_CONFIG, "Tor is running as a server, but you are " "running %s; this probably won't work. See " - "http://wiki.noreply.org/noreply/TheOnionRouter/TorFAQ#ServerOS " + "https://wiki.torproject.org/TheOnionRouter/TorFAQ#ServerOS " "for details.", uname); } @@ -3109,19 +2960,11 @@ options_validate(or_options_t *old_options, or_options_t *options, routerset_union(options->_ExcludeExitNodesUnion,options->ExcludeNodes); } - if (options->StrictExitNodes && - (!options->ExitNodes) && - (!old_options || - (old_options->StrictExitNodes != options->StrictExitNodes) || - (!routerset_equal(old_options->ExitNodes,options->ExitNodes)))) - COMPLAIN("StrictExitNodes set, but no ExitNodes listed."); - - if (options->StrictEntryNodes && - (!options->EntryNodes) && - (!old_options || - (old_options->StrictEntryNodes != options->StrictEntryNodes) || - (!routerset_equal(old_options->EntryNodes,options->EntryNodes)))) - COMPLAIN("StrictEntryNodes set, but no EntryNodes listed."); + if (options->ExcludeNodes && options->StrictNodes) { + COMPLAIN("You have asked to exclude certain relays from all positions " + "in your circuits. Expect hidden services and other Tor " + "features to be broken in unpredictable ways."); + } if (options->EntryNodes && !routerset_is_list(options->EntryNodes)) { /* XXXX fix this; see entry_guards_prepend_from_config(). */ @@ -3159,6 +3002,10 @@ options_validate(or_options_t *old_options, or_options_t *options, options->V3AuthoritativeDir)) REJECT("AuthoritativeDir is set, but none of " "(Bridge/HS/V1/V2/V3)AuthoritativeDir is set."); + /* If we have a v3bandwidthsfile and it's broken, complain on startup */ + if (options->V3BandwidthsFile && !old_options) { + dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL); + } } if (options->AuthoritativeDir && !options->DirPort) @@ -3170,9 +3017,9 @@ options_validate(or_options_t *old_options, or_options_t *options, if (options->AuthoritativeDir && options->ClientOnly) REJECT("Running as authoritative directory, but ClientOnly also set."); - if (options->HSAuthorityRecordStats && !options->HSAuthoritativeDir) - REJECT("HSAuthorityRecordStats is set but we're not running as " - "a hidden service authority."); + if (options->FetchDirInfoExtraEarly && !options->FetchDirInfoEarly) + REJECT("FetchDirInfoExtraEarly requires that you also set " + "FetchDirInfoEarly"); if (options->ConnLimit <= 0) { r = tor_snprintf(buf, sizeof(buf), @@ -3305,6 +3152,21 @@ options_validate(or_options_t *old_options, or_options_t *options, }); } + if (!options->SafeLogging || + !strcasecmp(options->SafeLogging, "0")) { + options->_SafeLogging = SAFELOG_SCRUB_NONE; + } else if (!strcasecmp(options->SafeLogging, "relay")) { + options->_SafeLogging = SAFELOG_SCRUB_RELAY; + } else if (!strcasecmp(options->SafeLogging, "1")) { + options->_SafeLogging = SAFELOG_SCRUB_ALL; + } else { + r = tor_snprintf(buf, sizeof(buf), + "Unrecognized value '%s' in SafeLogging", + escaped(options->SafeLogging)); + *msg = tor_strdup(r >= 0 ? buf : "internal error"); + return -1; + } + if (compute_publishserverdescriptor(options) < 0) { r = tor_snprintf(buf, sizeof(buf), "Unrecognized value in PublishServerDescriptor"); @@ -3328,29 +3190,30 @@ options_validate(or_options_t *old_options, or_options_t *options, } if (options->RendPostPeriod < MIN_REND_POST_PERIOD) { - log(LOG_WARN,LD_CONFIG,"RendPostPeriod option is too short; " - "raising to %d seconds.", MIN_REND_POST_PERIOD); + log_warn(LD_CONFIG, "RendPostPeriod option is too short; " + "raising to %d seconds.", MIN_REND_POST_PERIOD); options->RendPostPeriod = MIN_REND_POST_PERIOD; } if (options->RendPostPeriod > MAX_DIR_PERIOD) { - log(LOG_WARN, LD_CONFIG, "RendPostPeriod is too large; clipping to %ds.", - MAX_DIR_PERIOD); + log_warn(LD_CONFIG, "RendPostPeriod is too large; clipping to %ds.", + MAX_DIR_PERIOD); options->RendPostPeriod = MAX_DIR_PERIOD; } - if (options->CircuitBuildTimeout < MIN_CIRCUIT_BUILD_TIMEOUT) { - log(LOG_WARN, LD_CONFIG, "CircuitBuildTimeout option is too short; " - "raising to %d seconds.", MIN_CIRCUIT_BUILD_TIMEOUT); - options->CircuitBuildTimeout = MIN_CIRCUIT_BUILD_TIMEOUT; - } - if (options->MaxCircuitDirtiness < MIN_MAX_CIRCUIT_DIRTINESS) { - log(LOG_WARN, LD_CONFIG, "MaxCircuitDirtiness option is too short; " - "raising to %d seconds.", MIN_MAX_CIRCUIT_DIRTINESS); + log_warn(LD_CONFIG, "MaxCircuitDirtiness option is too short; " + "raising to %d seconds.", MIN_MAX_CIRCUIT_DIRTINESS); options->MaxCircuitDirtiness = MIN_MAX_CIRCUIT_DIRTINESS; } + if (options->CircuitStreamTimeout && + options->CircuitStreamTimeout < MIN_CIRCUIT_STREAM_TIMEOUT) { + log_warn(LD_CONFIG, "CircuitStreamTimeout option is too short; " + "raising to %d seconds.", MIN_CIRCUIT_STREAM_TIMEOUT); + options->CircuitStreamTimeout = MIN_CIRCUIT_STREAM_TIMEOUT; + } + if (options->KeepalivePeriod < 1) REJECT("KeepalivePeriod option must be positive."); @@ -3422,7 +3285,7 @@ options_validate(or_options_t *old_options, or_options_t *options, REJECT("Failed to parse accounting options. See logs for details."); if (options->HttpProxy) { /* parse it now */ - if (parse_addr_port(LOG_WARN, options->HttpProxy, NULL, + if (tor_addr_port_parse(options->HttpProxy, &options->HttpProxyAddr, &options->HttpProxyPort) < 0) REJECT("HttpProxy failed to parse or resolve. Please fix."); if (options->HttpProxyPort == 0) { /* give it a default */ @@ -3436,7 +3299,7 @@ options_validate(or_options_t *old_options, or_options_t *options, } if (options->HttpsProxy) { /* parse it now */ - if (parse_addr_port(LOG_WARN, options->HttpsProxy, NULL, + if (tor_addr_port_parse(options->HttpsProxy, &options->HttpsProxyAddr, &options->HttpsProxyPort) <0) REJECT("HttpsProxy failed to parse or resolve. Please fix."); if (options->HttpsProxyPort == 0) { /* give it a default */ @@ -3449,6 +3312,45 @@ options_validate(or_options_t *old_options, or_options_t *options, REJECT("HttpsProxyAuthenticator is too long (>= 48 chars)."); } + if (options->Socks4Proxy) { /* parse it now */ + if (tor_addr_port_parse(options->Socks4Proxy, + &options->Socks4ProxyAddr, + &options->Socks4ProxyPort) <0) + REJECT("Socks4Proxy failed to parse or resolve. Please fix."); + if (options->Socks4ProxyPort == 0) { /* give it a default */ + options->Socks4ProxyPort = 1080; + } + } + + if (options->Socks5Proxy) { /* parse it now */ + if (tor_addr_port_parse(options->Socks5Proxy, + &options->Socks5ProxyAddr, + &options->Socks5ProxyPort) <0) + REJECT("Socks5Proxy failed to parse or resolve. Please fix."); + if (options->Socks5ProxyPort == 0) { /* give it a default */ + options->Socks5ProxyPort = 1080; + } + } + + if (options->Socks4Proxy && options->Socks5Proxy) + REJECT("You cannot specify both Socks4Proxy and SOCKS5Proxy"); + + if (options->Socks5ProxyUsername) { + size_t len; + + len = strlen(options->Socks5ProxyUsername); + if (len < 1 || len > 255) + REJECT("Socks5ProxyUsername must be between 1 and 255 characters."); + + if (!options->Socks5ProxyPassword) + REJECT("Socks5ProxyPassword must be included with Socks5ProxyUsername."); + + len = strlen(options->Socks5ProxyPassword); + if (len < 1 || len > 255) + REJECT("Socks5ProxyPassword must be between 1 and 255 characters."); + } else if (options->Socks5ProxyPassword) + REJECT("Socks5ProxyPassword must be included with Socks5ProxyUsername."); + if (options->HashedControlPassword) { smartlist_t *sl = decode_hashed_passwords(options->HashedControlPassword); if (!sl) { @@ -3597,6 +3499,12 @@ options_validate(or_options_t *old_options, or_options_t *options, if (options->PreferTunneledDirConns && !options->TunnelDirConns) REJECT("Must set TunnelDirConns if PreferTunneledDirConns is set."); + if ((options->Socks4Proxy || options->Socks5Proxy) && + !options->HttpProxy && !options->PreferTunneledDirConns) + REJECT("When Socks4Proxy or Socks5Proxy is configured, " + "PreferTunneledDirConns and TunnelDirConns must both be " + "set to 1, or HttpProxy must be configured."); + if (options->AutomapHostsSuffixes) { SMARTLIST_FOREACH(options->AutomapHostsSuffixes, char *, suf, { @@ -3676,6 +3584,11 @@ options_validate(or_options_t *old_options, or_options_t *options, "testing Tor network!"); } + if (options->AccelName && !options->HardwareAccel) + options->HardwareAccel = 1; + if (options->AccelDir && !options->AccelName) + REJECT("Can't use hardware crypto accelerator dir without engine name."); + return 0; #undef REJECT #undef COMPLAIN @@ -3733,9 +3646,11 @@ options_transition_allowed(or_options_t *old, or_options_t *new_val, return -1; } - if (old->HardwareAccel != new_val->HardwareAccel) { - *msg = tor_strdup("While Tor is running, changing HardwareAccel is " - "not allowed."); + if ((old->HardwareAccel != new_val->HardwareAccel) + || !opt_streq(old->AccelName, new_val->AccelName) + || !opt_streq(old->AccelDir, new_val->AccelDir)) { + *msg = tor_strdup("While Tor is running, changing OpenSSL hardware " + "acceleration engine is not allowed."); return -1; } @@ -3745,6 +3660,22 @@ options_transition_allowed(or_options_t *old, or_options_t *new_val, return -1; } + if (old->CellStatistics != new_val->CellStatistics || + old->DirReqStatistics != new_val->DirReqStatistics || + old->EntryStatistics != new_val->EntryStatistics || + old->ExitPortStatistics != new_val->ExitPortStatistics) { + *msg = tor_strdup("While Tor is running, changing either " + "CellStatistics, DirReqStatistics, EntryStatistics, " + "or ExitPortStatistics is not allowed."); + return -1; + } + + if (old->DisableAllSwap != new_val->DisableAllSwap) { + *msg = tor_strdup("While Tor is running, changing DisableAllSwap " + "is not allowed."); + return -1; + } + return 0; } @@ -4024,6 +3955,12 @@ options_init_from_torrc(int argc, char **argv) printf("Tor version %s.\n",get_version()); exit(0); } + if (argc > 1 && (!strcmp(argv[1],"--digests"))) { + printf("Tor version %s.\n",get_version()); + printf("%s", libor_get_digests()); + printf("%s", tor_get_digests()); + exit(0); + } /* Go through command-line variables */ if (!global_cmdline_options) { @@ -4584,7 +4521,7 @@ normalize_data_directory(or_options_t *options) } /** Check and normalize the value of options->DataDirectory; return 0 if it - * sane, -1 otherwise. */ + * is sane, -1 otherwise. */ static int validate_data_directory(or_options_t *options) { @@ -4768,30 +4705,47 @@ static struct unit_table_t time_units[] = { static uint64_t config_parse_units(const char *val, struct unit_table_t *u, int *ok) { - uint64_t v; + uint64_t v = 0; + double d = 0; + int use_float = 0; char *cp; tor_assert(ok); v = tor_parse_uint64(val, 10, 0, UINT64_MAX, ok, &cp); - if (!*ok) - return 0; + if (!*ok || (cp && *cp == '.')) { + d = tor_parse_double(val, 0, UINT64_MAX, ok, &cp); + if (!*ok) + goto done; + use_float = 1; + } + if (!cp) { *ok = 1; - return v; + v = use_float ? DBL_TO_U64(d) : v; + goto done; } - while (TOR_ISSPACE(*cp)) - ++cp; + + cp = (char*) eat_whitespace(cp); + for ( ;u->unit;++u) { if (!strcasecmp(u->unit, cp)) { - v *= u->multiplier; + if (use_float) + v = u->multiplier * d; + else + v *= u->multiplier; *ok = 1; - return v; + goto done; } } log_warn(LD_CONFIG, "Unknown unit '%s'.", cp); *ok = 0; - return 0; + done: + + if (*ok) + return v; + else + return 0; } /** Parse a string in the format "number unit", where unit is a unit of @@ -4801,7 +4755,8 @@ config_parse_units(const char *val, struct unit_table_t *u, int *ok) static uint64_t config_parse_memunit(const char *s, int *ok) { - return config_parse_units(s, memory_units, ok); + uint64_t u = config_parse_units(s, memory_units, ok); + return u; } /** Parse a string in the format "number unit", where unit is a unit of time. @@ -4823,256 +4778,37 @@ config_parse_interval(const char *s, int *ok) return (int)r; } -/* This is what passes for version detection on OSX. We set - * MACOSX_KQUEUE_IS_BROKEN to true iff we're on a version of OSX before - * 10.4.0 (aka 1040). */ -#ifdef __APPLE__ -#ifdef __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ -#define MACOSX_KQUEUE_IS_BROKEN \ - (__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 1040) -#else -#define MACOSX_KQUEUE_IS_BROKEN 0 -#endif -#endif - /** * Initialize the libevent library. */ static void init_libevent(void) { + const char *badness=NULL; + configure_libevent_logging(); /* If the kernel complains that some method (say, epoll) doesn't * exist, we don't care about it, since libevent will cope. */ suppress_libevent_log_msg("Function not implemented"); -#ifdef __APPLE__ - if (MACOSX_KQUEUE_IS_BROKEN || - decode_libevent_version(event_get_version(), NULL) < LE_11B) { - setenv("EVENT_NOKQUEUE","1",1); - } -#endif - /* In libevent versions before 2.0, it's hard to keep binary compatibility - * between upgrades, and unpleasant to detect when the version we compiled - * against is unlike the version we have linked against. Here's how. */ -#if defined(_EVENT_VERSION) && defined(HAVE_EVENT_GET_VERSION) - /* We have a header-file version and a function-call version. Easy. */ - if (strcmp(_EVENT_VERSION, event_get_version())) { - int compat1 = -1, compat2 = -1; - int verybad, prettybad ; - decode_libevent_version(_EVENT_VERSION, &compat1); - decode_libevent_version(event_get_version(), &compat2); - verybad = compat1 != compat2; - prettybad = (compat1 == -1 || compat2 == -1) && compat1 != compat2; - - log(verybad ? LOG_WARN : (prettybad ? LOG_NOTICE : LOG_INFO), - LD_GENERAL, "We were compiled with headers from version %s " - "of Libevent, but we're using a Libevent library that says it's " - "version %s.", _EVENT_VERSION, event_get_version()); - if (verybad) - log_warn(LD_GENERAL, "This will almost certainly make Tor crash."); - else if (prettybad) - log_notice(LD_GENERAL, "If Tor crashes, this might be why."); - else - log_info(LD_GENERAL, "I think these versions are binary-compatible."); - } -#elif defined(HAVE_EVENT_GET_VERSION) - /* event_get_version but no _EVENT_VERSION. We might be in 1.4.0-beta or - earlier, where that's normal. To see whether we were compiled with an - earlier version, let's see whether the struct event defines MIN_HEAP_IDX. - */ -#ifdef HAVE_STRUCT_EVENT_MIN_HEAP_IDX - /* The header files are 1.4.0-beta or later. If the version is not - * 1.4.0-beta, we are incompatible. */ - { - if (strcmp(event_get_version(), "1.4.0-beta")) { - log_warn(LD_GENERAL, "It's a little hard to tell, but you seem to have " - "Libevent 1.4.0-beta header files, whereas you have linked " - "against Libevent %s. This will probably make Tor crash.", - event_get_version()); - } - } -#else - /* Our headers are 1.3e or earlier. If the library version is not 1.4.x or - later, we're probably fine. */ - { - const char *v = event_get_version(); - if ((v[0] == '1' && v[2] == '.' && v[3] > '3') || v[0] > '1') { - log_warn(LD_GENERAL, "It's a little hard to tell, but you seem to have " - "Libevent header file from 1.3e or earlier, whereas you have " - "linked against Libevent %s. This will probably make Tor " - "crash.", event_get_version()); - } - } -#endif + tor_check_libevent_header_compatibility(); -#elif defined(_EVENT_VERSION) -#warn "_EVENT_VERSION is defined but not get_event_version(): Libevent is odd." -#else - /* Your libevent is ancient. */ -#endif + tor_libevent_initialize(); - event_init(); suppress_libevent_log_msg(NULL); -#if defined(HAVE_EVENT_GET_VERSION) && defined(HAVE_EVENT_GET_METHOD) - /* Making this a NOTICE for now so we can link bugs to a libevent versions - * or methods better. */ - log(LOG_NOTICE, LD_GENERAL, - "Initialized libevent version %s using method %s. Good.", - event_get_version(), event_get_method()); - check_libevent_version(event_get_method(), get_options()->ORPort != 0); -#else - log(LOG_NOTICE, LD_GENERAL, - "Initialized old libevent (version 1.0b or earlier)."); - log(LOG_WARN, LD_GENERAL, - "You have a *VERY* old version of libevent. It is likely to be buggy; " - "please build Tor with a more recent version."); -#endif -} - -/** Table mapping return value of event_get_version() to le_version_t. */ -static const struct { - const char *name; le_version_t version; int bincompat; -} le_version_table[] = { - /* earlier versions don't have get_version. */ - { "1.0c", LE_10C, 1}, - { "1.0d", LE_10D, 1}, - { "1.0e", LE_10E, 1}, - { "1.1", LE_11, 1 }, - { "1.1a", LE_11A, 1 }, - { "1.1b", LE_11B, 1 }, - { "1.2", LE_12, 1 }, - { "1.2a", LE_12A, 1 }, - { "1.3", LE_13, 1 }, - { "1.3a", LE_13A, 1 }, - { "1.3b", LE_13B, 1 }, - { "1.3c", LE_13C, 1 }, - { "1.3d", LE_13D, 1 }, - { "1.3e", LE_13E, 1 }, - { "1.4.0-beta", LE_140, 2 }, - { "1.4.1-beta", LE_141, 2 }, - { "1.4.2-rc", LE_142, 2 }, - { "1.4.3-stable", LE_143, 2 }, - { "1.4.4-stable", LE_144, 2 }, - { "1.4.5-stable", LE_145, 2 }, - { "1.4.6-stable", LE_146, 2 }, - { "1.4.7-stable", LE_147, 2 }, - { "1.4.8-stable", LE_148, 2 }, - { "1.4.99-trunk", LE_1499, 3 }, - { NULL, LE_OTHER, 0 } -}; -/** Return the le_version_t for the current version of libevent. If the - * version is very new, return LE_OTHER. If the version is so old that it - * doesn't support event_get_version(), return LE_OLD. */ -static le_version_t -decode_libevent_version(const char *v, int *bincompat_out) -{ - int i; - for (i=0; le_version_table[i].name; ++i) { - if (!strcmp(le_version_table[i].name, v)) { - if (bincompat_out) - *bincompat_out = le_version_table[i].bincompat; - return le_version_table[i].version; - } - } - if (v[0] != '1' && bincompat_out) - *bincompat_out = 100; - else if (!strcmpstart(v, "1.4") && bincompat_out) - *bincompat_out = 2; - return LE_OTHER; -} - -#if defined(HAVE_EVENT_GET_VERSION) && defined(HAVE_EVENT_GET_METHOD) -/** - * Compare the given libevent method and version to a list of versions - * which are known not to work. Warn the user as appropriate. - */ -static void -check_libevent_version(const char *m, int server) -{ - int buggy = 0, iffy = 0, slow = 0, thread_unsafe = 0; - le_version_t version; - const char *v = event_get_version(); - const char *badness = NULL; - const char *sad_os = ""; - - version = decode_libevent_version(v, NULL); - - /* XXX Would it be worthwhile disabling the methods that we know - * are buggy, rather than just warning about them and then proceeding - * to use them? If so, we should probably not wrap this whole thing - * in HAVE_EVENT_GET_VERSION and HAVE_EVENT_GET_METHOD. -RD */ - /* XXXX The problem is that it's not trivial to get libevent to change it's - * method once it's initialized, and it's not trivial to tell what method it - * will use without initializing it. I guess we could preemptively disable - * buggy libevent modes based on the version _before_ initializing it, - * though, but then there's no good way (afaict) to warn "I would have used - * kqueue, but instead I'm using select." -NM */ - if (!strcmp(m, "kqueue")) { - if (version < LE_11B) - buggy = 1; - } else if (!strcmp(m, "epoll")) { - if (version < LE_11) - iffy = 1; - } else if (!strcmp(m, "poll")) { - if (version < LE_10E) - buggy = 1; - else if (version < LE_11) - slow = 1; - } else if (!strcmp(m, "select")) { - if (version < LE_11) - slow = 1; - } else if (!strcmp(m, "win32")) { - if (version < LE_11B) - buggy = 1; - } - - /* Libevent versions before 1.3b do very badly on operating systems with - * user-space threading implementations. */ -#if defined(__OpenBSD__) || defined(__FreeBSD__) || defined(__NetBSD__) - if (server && version < LE_13B) { - thread_unsafe = 1; - sad_os = "BSD variants"; - } -#elif defined(__APPLE__) || defined(__darwin__) - if (server && version < LE_13B) { - thread_unsafe = 1; - sad_os = "Mac OS X"; - } -#endif - - if (thread_unsafe) { - log(LOG_WARN, LD_GENERAL, - "Libevent version %s often crashes when running a Tor server with %s. " - "Please use the latest version of libevent (1.3b or later)",v,sad_os); - badness = "BROKEN"; - } else if (buggy) { - log(LOG_WARN, LD_GENERAL, - "There are serious bugs in using %s with libevent %s. " - "Please use the latest version of libevent.", m, v); - badness = "BROKEN"; - } else if (iffy) { - log(LOG_WARN, LD_GENERAL, - "There are minor bugs in using %s with libevent %s. " - "You may want to use the latest version of libevent.", m, v); - badness = "BUGGY"; - } else if (slow && server) { - log(LOG_WARN, LD_GENERAL, - "libevent %s can be very slow with %s. " - "When running a server, please use the latest version of libevent.", - v,m); - badness = "SLOW"; - } + tor_check_libevent_version(tor_libevent_get_method(), + get_options()->ORPort != 0, + &badness); if (badness) { + const char *v = tor_libevent_get_version_str(); + const char *m = tor_libevent_get_method(); control_event_general_status(LOG_WARN, "BAD_LIBEVENT VERSION=%s METHOD=%s BADNESS=%s RECOVERED=NO", v, m, badness); } - } -#endif /** Return the persistent state struct for this Tor. */ or_state_t * @@ -5159,8 +4895,7 @@ or_state_set(or_state_t *new_state) { char *err = NULL; tor_assert(new_state); - if (global_state) - config_free(&state_format, global_state); + config_free(&state_format, global_state); global_state = new_state; if (entry_guards_parse_state(global_state, 1, &err)<0) { log_warn(LD_GENERAL,"%s",err); @@ -5170,6 +4905,10 @@ or_state_set(or_state_t *new_state) log_warn(LD_GENERAL,"Unparseable bandwidth history state: %s",err); tor_free(err); } + if (circuit_build_times_parse_state(&circ_times, global_state, &err) < 0) { + log_warn(LD_GENERAL,"%s",err); + tor_free(err); + } } /** Reload the persistent state from disk, generating a new state as needed. @@ -5302,6 +5041,7 @@ or_state_save(time_t now) * to avoid redundant writes. */ entry_guards_update_state(global_state); rep_hist_update_state(global_state); + circuit_build_times_update_state(&circ_times, global_state); if (accounting_is_enabled(get_options())) accounting_run_housekeeping(now); @@ -5367,10 +5107,9 @@ getinfo_helper_config(control_connection_t *conn, int i; for (i = 0; _option_vars[i].name; ++i) { config_var_t *var = &_option_vars[i]; - const char *type, *desc; + const char *type; char *line; size_t len; - desc = config_find_description(&options_format, var->name); switch (var->type) { case CONFIG_TYPE_STRING: type = "String"; break; case CONFIG_TYPE_FILENAME: type = "Filename"; break; @@ -5392,13 +5131,8 @@ getinfo_helper_config(control_connection_t *conn, if (!type) continue; len = strlen(var->name)+strlen(type)+16; - if (desc) - len += strlen(desc); line = tor_malloc(len); - if (desc) - tor_snprintf(line, len, "%s %s %s\n",var->name,type,desc); - else - tor_snprintf(line, len, "%s %s\n",var->name,type); + tor_snprintf(line, len, "%s %s\n",var->name,type); smartlist_add(sl, line); } *answer = smartlist_join_strings(sl, "", 0, NULL); diff --git a/src/or/config_codedigest.c b/src/or/config_codedigest.c new file mode 100644 index 0000000000..be9eaa331d --- /dev/null +++ b/src/or/config_codedigest.c @@ -0,0 +1,11 @@ + +const char *tor_get_digests(void); + +const char * +tor_get_digests(void) +{ + return "" +#include "or_sha1.i" + ; +} + diff --git a/src/or/connection.c b/src/or/connection.c index 32ae259cf1..eeb25c1828 100644 --- a/src/or/connection.c +++ b/src/or/connection.c @@ -21,7 +21,8 @@ static void connection_init(time_t now, connection_t *conn, int type, static int connection_init_accepted_conn(connection_t *conn, uint8_t listener_type); static int connection_handle_listener_read(connection_t *conn, int new_type); -static int connection_read_bucket_should_increase(or_connection_t *conn); +static int connection_bucket_should_increase(int bucket, + or_connection_t *conn); static int connection_finished_flushing(connection_t *conn); static int connection_flushed_some(connection_t *conn); static int connection_finished_connecting(connection_t *conn); @@ -32,6 +33,10 @@ static int connection_process_inbuf(connection_t *conn, int package_partial); static void client_check_address_changed(int sock); static void set_constrained_socket_buffers(int sock, int size); +static const char *connection_proxy_state_to_string(int state); +static int connection_read_https_proxy_response(connection_t *conn); +static void connection_send_socks5_connect(connection_t *conn); + /** The last IPv4 address that our network interface seemed to have been * binding to, in host order. We use this to detect when our IP changes. */ static uint32_t last_interface_ip = 0; @@ -92,8 +97,7 @@ conn_state_to_string(int type, int state) case CONN_TYPE_OR: switch (state) { case OR_CONN_STATE_CONNECTING: return "connect()ing"; - case OR_CONN_STATE_PROXY_FLUSHING: return "proxy flushing"; - case OR_CONN_STATE_PROXY_READING: return "proxy reading"; + case OR_CONN_STATE_PROXY_HANDSHAKING: return "handshaking (proxy)"; case OR_CONN_STATE_TLS_HANDSHAKING: return "handshaking (TLS)"; case OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING: return "renegotiating (TLS)"; @@ -177,6 +181,9 @@ or_connection_new(int socket_family) or_conn->timestamp_last_added_nonpadding = time(NULL); or_conn->next_circ_id = crypto_rand_int(1<<15); + or_conn->active_circuit_pqueue = smartlist_create(); + or_conn->active_circuit_pqueue_last_recalibrated = cell_ewma_get_tick(); + return or_conn; } @@ -202,6 +209,7 @@ control_connection_new(int socket_family) tor_malloc_zero(sizeof(control_connection_t)); connection_init(time(NULL), TO_CONN(control_conn), CONN_TYPE_CONTROL, socket_family); + log_notice(LD_CONTROL, "New control connection opened."); return control_conn; } @@ -299,25 +307,6 @@ connection_link_connections(connection_t *conn_a, connection_t *conn_b) conn_b->linked_conn = conn_a; } -/** Tell libevent that we don't care about <b>conn</b> any more. */ -void -connection_unregister_events(connection_t *conn) -{ - if (conn->read_event) { - if (event_del(conn->read_event)) - log_warn(LD_BUG, "Error removing read event for %d", conn->s); - tor_free(conn->read_event); - } - if (conn->write_event) { - if (event_del(conn->write_event)) - log_warn(LD_BUG, "Error removing write event for %d", conn->s); - tor_free(conn->write_event); - } - if (conn->dns_server_port) { - dnsserv_close_listener(conn); - } -} - /** Deallocate memory used by <b>conn</b>. Deallocate its buffers if * necessary, close its socket if necessary, and mark the directory as dirty * if <b>conn</b> is an OR or OP connection. @@ -327,6 +316,9 @@ _connection_free(connection_t *conn) { void *mem; size_t memlen; + if (!conn) + return; + switch (conn->type) { case CONN_TYPE_OR: tor_assert(conn->magic == OR_CONNECTION_MAGIC); @@ -384,14 +376,11 @@ _connection_free(connection_t *conn) if (connection_speaks_cells(conn)) { or_connection_t *or_conn = TO_OR_CONN(conn); - if (or_conn->tls) { - tor_tls_free(or_conn->tls); - or_conn->tls = NULL; - } - if (or_conn->handshake_state) { - or_handshake_state_free(or_conn->handshake_state); - or_conn->handshake_state = NULL; - } + tor_tls_free(or_conn->tls); + or_conn->tls = NULL; + or_handshake_state_free(or_conn->handshake_state); + or_conn->handshake_state = NULL; + smartlist_free(or_conn->active_circuit_pqueue); tor_free(or_conn->nickname); } if (CONN_IS_EDGE(conn)) { @@ -401,8 +390,8 @@ _connection_free(connection_t *conn) memset(edge_conn->socks_request, 0xcc, sizeof(socks_request_t)); tor_free(edge_conn->socks_request); } - if (edge_conn->rend_data) - rend_data_free(edge_conn->rend_data); + + rend_data_free(edge_conn->rend_data); } if (conn->type == CONN_TYPE_CONTROL) { control_connection_t *control_conn = TO_CONTROL_CONN(conn); @@ -415,16 +404,15 @@ _connection_free(connection_t *conn) if (conn->type == CONN_TYPE_DIR) { dir_connection_t *dir_conn = TO_DIR_CONN(conn); tor_free(dir_conn->requested_resource); - if (dir_conn->zlib_state) - tor_zlib_free(dir_conn->zlib_state); + + tor_zlib_free(dir_conn->zlib_state); if (dir_conn->fingerprint_stack) { SMARTLIST_FOREACH(dir_conn->fingerprint_stack, char *, cp, tor_free(cp)); smartlist_free(dir_conn->fingerprint_stack); } - if (dir_conn->cached_dir) - cached_dir_decref(dir_conn->cached_dir); - if (dir_conn->rend_data) - rend_data_free(dir_conn->rend_data); + + cached_dir_decref(dir_conn->cached_dir); + rend_data_free(dir_conn->rend_data); } if (conn->s >= 0) { @@ -439,7 +427,7 @@ _connection_free(connection_t *conn) connection_or_remove_from_identity_map(TO_OR_CONN(conn)); } - memset(conn, 0xAA, memlen); /* poison memory */ + memset(mem, 0xCC, memlen); /* poison memory */ tor_free(mem); } @@ -448,7 +436,8 @@ _connection_free(connection_t *conn) void connection_free(connection_t *conn) { - tor_assert(conn); + if (!conn) + return; tor_assert(!connection_is_on_closeable_list(conn)); tor_assert(!connection_in_array(conn)); if (conn->linked_conn) { @@ -544,13 +533,6 @@ connection_about_to_close_connection(connection_t *conn) * failed: forget about this router, and maybe try again. */ connection_dir_request_failed(dir_conn); } - if (conn->purpose == DIR_PURPOSE_FETCH_RENDDESC && dir_conn->rend_data) { - /* Give it a try. However, there is no re-fetching for v0 rend - * descriptors; if the response is empty or the descriptor is - * unusable, close pending connections (unless a v2 request is - * still in progress). */ - rend_client_desc_trynow(dir_conn->rend_data->onion_address, 0); - } /* If we were trying to fetch a v2 rend desc and did not succeed, * retry as needed. (If a fetch is successful, the connection state * is changed to DIR_PURPOSE_HAS_FETCHED_RENDDESC to mark that @@ -589,7 +571,7 @@ connection_about_to_close_connection(connection_t *conn) rep_hist_note_disconnect(or_conn->identity_digest, now); control_event_or_conn_status(or_conn, OR_CONN_EVENT_CLOSED, tls_error_to_orconn_end_reason(or_conn->tls_error)); - } else if (or_conn->identity_digest) { + } else if (!tor_digest_is_zero(or_conn->identity_digest)) { rep_hist_note_connection_died(or_conn->identity_digest, now); control_event_or_conn_status(or_conn, OR_CONN_EVENT_CLOSED, tls_error_to_orconn_end_reason(or_conn->tls_error)); @@ -1254,7 +1236,7 @@ connection_connect(connection_t *conn, const char *address, return -1; } - if (options->OutboundBindAddress) { + if (options->OutboundBindAddress && !tor_addr_is_loopback(addr)) { struct sockaddr_in ext_addr; memset(&ext_addr, 0, sizeof(ext_addr)); @@ -1285,7 +1267,8 @@ connection_connect(connection_t *conn, const char *address, dest_addr_len = tor_addr_to_sockaddr(addr, port, dest_addr, sizeof(addrbuf)); tor_assert(dest_addr_len > 0); - log_debug(LD_NET,"Connecting to %s:%u.",escaped_safe_str(address),port); + log_debug(LD_NET, "Connecting to %s:%u.", + escaped_safe_str_client(address), port); if (connect(s, dest_addr, dest_addr_len) < 0) { int e = tor_socket_errno(s); @@ -1293,7 +1276,8 @@ connection_connect(connection_t *conn, const char *address, /* yuck. kill it. */ *socket_error = e; log_info(LD_NET, - "connect() to %s:%u failed: %s",escaped_safe_str(address), + "connect() to %s:%u failed: %s", + escaped_safe_str_client(address), port, tor_socket_strerror(e)); tor_close_socket(s); return -1; @@ -1307,7 +1291,8 @@ connection_connect(connection_t *conn, const char *address, /* it succeeded. we're connected. */ log_fn(inprogress?LOG_DEBUG:LOG_INFO, LD_NET, - "Connection to %s:%u %s (sock %d).",escaped_safe_str(address), + "Connection to %s:%u %s (sock %d).", + escaped_safe_str_client(address), port, inprogress?"in progress":"established", s); conn->s = s; if (connection_add(conn) < 0) /* no space, forget it */ @@ -1315,6 +1300,353 @@ connection_connect(connection_t *conn, const char *address, return inprogress ? 0 : 1; } +/** Convert state number to string representation for logging purposes. + */ +static const char * +connection_proxy_state_to_string(int state) +{ + static const char *unknown = "???"; + static const char *states[] = { + "PROXY_NONE", + "PROXY_HTTPS_WANT_CONNECT_OK", + "PROXY_SOCKS4_WANT_CONNECT_OK", + "PROXY_SOCKS5_WANT_AUTH_METHOD_NONE", + "PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929", + "PROXY_SOCKS5_WANT_AUTH_RFC1929_OK", + "PROXY_SOCKS5_WANT_CONNECT_OK", + "PROXY_CONNECTED", + }; + + if (state < PROXY_NONE || state > PROXY_CONNECTED) + return unknown; + + return states[state]; +} + +/** Write a proxy request of <b>type</b> (socks4, socks5, https) to conn + * for conn->addr:conn->port, authenticating with the auth details given + * in the configuration (if available). SOCKS 5 and HTTP CONNECT proxies + * support authentication. + * + * Returns -1 if conn->addr is incompatible with the proxy protocol, and + * 0 otherwise. + * + * Use connection_read_proxy_handshake() to complete the handshake. + */ +int +connection_proxy_connect(connection_t *conn, int type) +{ + or_options_t *options; + + tor_assert(conn); + + options = get_options(); + + switch (type) { + case PROXY_CONNECT: { + char buf[1024]; + char *base64_authenticator=NULL; + const char *authenticator = options->HttpsProxyAuthenticator; + + /* Send HTTP CONNECT and authentication (if available) in + * one request */ + + if (authenticator) { + base64_authenticator = alloc_http_authenticator(authenticator); + if (!base64_authenticator) + log_warn(LD_OR, "Encoding https authenticator failed"); + } + + if (base64_authenticator) { + tor_snprintf(buf, sizeof(buf), "CONNECT %s:%d HTTP/1.1\r\n" + "Proxy-Authorization: Basic %s\r\n\r\n", + fmt_addr(&conn->addr), + conn->port, base64_authenticator); + tor_free(base64_authenticator); + } else { + tor_snprintf(buf, sizeof(buf), "CONNECT %s:%d HTTP/1.0\r\n\r\n", + fmt_addr(&conn->addr), conn->port); + } + + connection_write_to_buf(buf, strlen(buf), conn); + conn->proxy_state = PROXY_HTTPS_WANT_CONNECT_OK; + break; + } + + case PROXY_SOCKS4: { + unsigned char buf[9]; + uint16_t portn; + uint32_t ip4addr; + + /* Send a SOCKS4 connect request with empty user id */ + + if (tor_addr_family(&conn->addr) != AF_INET) { + log_warn(LD_NET, "SOCKS4 client is incompatible with IPv6"); + return -1; + } + + ip4addr = tor_addr_to_ipv4n(&conn->addr); + portn = htons(conn->port); + + buf[0] = 4; /* version */ + buf[1] = SOCKS_COMMAND_CONNECT; /* command */ + memcpy(buf + 2, &portn, 2); /* port */ + memcpy(buf + 4, &ip4addr, 4); /* addr */ + buf[8] = 0; /* userid (empty) */ + + connection_write_to_buf((char *)buf, sizeof(buf), conn); + conn->proxy_state = PROXY_SOCKS4_WANT_CONNECT_OK; + break; + } + + case PROXY_SOCKS5: { + unsigned char buf[4]; /* fields: vers, num methods, method list */ + + /* Send a SOCKS5 greeting (connect request must wait) */ + + buf[0] = 5; /* version */ + + /* number of auth methods */ + if (options->Socks5ProxyUsername) { + buf[1] = 2; + buf[2] = 0x00; /* no authentication */ + buf[3] = 0x02; /* rfc1929 Username/Passwd auth */ + conn->proxy_state = PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929; + } else { + buf[1] = 1; + buf[2] = 0x00; /* no authentication */ + conn->proxy_state = PROXY_SOCKS5_WANT_AUTH_METHOD_NONE; + } + + connection_write_to_buf((char *)buf, 2 + buf[1], conn); + break; + } + + default: + log_err(LD_BUG, "Invalid proxy protocol, %d", type); + tor_fragile_assert(); + return -1; + } + + log_debug(LD_NET, "set state %s", + connection_proxy_state_to_string(conn->proxy_state)); + + return 0; +} + +/** Read conn's inbuf. If the http response from the proxy is all + * here, make sure it's good news, then return 1. If it's bad news, + * return -1. Else return 0 and hope for better luck next time. + */ +static int +connection_read_https_proxy_response(connection_t *conn) +{ + char *headers; + char *reason=NULL; + int status_code; + time_t date_header; + + switch (fetch_from_buf_http(conn->inbuf, + &headers, MAX_HEADERS_SIZE, + NULL, NULL, 10000, 0)) { + case -1: /* overflow */ + log_warn(LD_PROTOCOL, + "Your https proxy sent back an oversized response. Closing."); + return -1; + case 0: + log_info(LD_NET,"https proxy response not all here yet. Waiting."); + return 0; + /* case 1, fall through */ + } + + if (parse_http_response(headers, &status_code, &date_header, + NULL, &reason) < 0) { + log_warn(LD_NET, + "Unparseable headers from proxy (connecting to '%s'). Closing.", + conn->address); + tor_free(headers); + return -1; + } + if (!reason) reason = tor_strdup("[no reason given]"); + + if (status_code == 200) { + log_info(LD_NET, + "HTTPS connect to '%s' successful! (200 %s) Starting TLS.", + conn->address, escaped(reason)); + tor_free(reason); + return 1; + } + /* else, bad news on the status code */ + log_warn(LD_NET, + "The https proxy sent back an unexpected status code %d (%s). " + "Closing.", + status_code, escaped(reason)); + tor_free(reason); + return -1; +} + +/** Send SOCKS5 CONNECT command to <b>conn</b>, copying <b>conn->addr</b> + * and <b>conn->port</b> into the request. + */ +static void +connection_send_socks5_connect(connection_t *conn) +{ + unsigned char buf[1024]; + size_t reqsize = 6; + uint16_t port = htons(conn->port); + + buf[0] = 5; /* version */ + buf[1] = SOCKS_COMMAND_CONNECT; /* command */ + buf[2] = 0; /* reserved */ + + if (tor_addr_family(&conn->addr) == AF_INET) { + uint32_t addr = tor_addr_to_ipv4n(&conn->addr); + + buf[3] = 1; + reqsize += 4; + memcpy(buf + 4, &addr, 4); + memcpy(buf + 8, &port, 2); + } else { /* AF_INET6 */ + buf[3] = 4; + reqsize += 16; + memcpy(buf + 4, tor_addr_to_in6(&conn->addr), 16); + memcpy(buf + 20, &port, 2); + } + + connection_write_to_buf((char *)buf, reqsize, conn); + + conn->proxy_state = PROXY_SOCKS5_WANT_CONNECT_OK; +} + +/** Call this from connection_*_process_inbuf() to advance the proxy + * handshake. + * + * No matter what proxy protocol is used, if this function returns 1, the + * handshake is complete, and the data remaining on inbuf may contain the + * start of the communication with the requested server. + * + * Returns 0 if the current buffer contains an incomplete response, and -1 + * on error. + */ +int +connection_read_proxy_handshake(connection_t *conn) +{ + int ret = 0; + char *reason = NULL; + + log_debug(LD_NET, "enter state %s", + connection_proxy_state_to_string(conn->proxy_state)); + + switch (conn->proxy_state) { + case PROXY_HTTPS_WANT_CONNECT_OK: + ret = connection_read_https_proxy_response(conn); + if (ret == 1) + conn->proxy_state = PROXY_CONNECTED; + break; + + case PROXY_SOCKS4_WANT_CONNECT_OK: + ret = fetch_from_buf_socks_client(conn->inbuf, + conn->proxy_state, + &reason); + if (ret == 1) + conn->proxy_state = PROXY_CONNECTED; + break; + + case PROXY_SOCKS5_WANT_AUTH_METHOD_NONE: + ret = fetch_from_buf_socks_client(conn->inbuf, + conn->proxy_state, + &reason); + /* no auth needed, do connect */ + if (ret == 1) { + connection_send_socks5_connect(conn); + ret = 0; + } + break; + + case PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929: + ret = fetch_from_buf_socks_client(conn->inbuf, + conn->proxy_state, + &reason); + + /* send auth if needed, otherwise do connect */ + if (ret == 1) { + connection_send_socks5_connect(conn); + ret = 0; + } else if (ret == 2) { + unsigned char buf[1024]; + size_t reqsize, usize, psize; + const char *user, *pass; + + user = get_options()->Socks5ProxyUsername; + pass = get_options()->Socks5ProxyPassword; + tor_assert(user && pass); + + /* XXX len of user and pass must be <= 255 !!! */ + usize = strlen(user); + psize = strlen(pass); + tor_assert(usize <= 255 && psize <= 255); + reqsize = 3 + usize + psize; + + buf[0] = 1; /* negotiation version */ + buf[1] = usize; + memcpy(buf + 2, user, usize); + buf[2 + usize] = psize; + memcpy(buf + 3 + usize, pass, psize); + + connection_write_to_buf((char *)buf, reqsize, conn); + + conn->proxy_state = PROXY_SOCKS5_WANT_AUTH_RFC1929_OK; + ret = 0; + } + break; + + case PROXY_SOCKS5_WANT_AUTH_RFC1929_OK: + ret = fetch_from_buf_socks_client(conn->inbuf, + conn->proxy_state, + &reason); + /* send the connect request */ + if (ret == 1) { + connection_send_socks5_connect(conn); + ret = 0; + } + break; + + case PROXY_SOCKS5_WANT_CONNECT_OK: + ret = fetch_from_buf_socks_client(conn->inbuf, + conn->proxy_state, + &reason); + if (ret == 1) + conn->proxy_state = PROXY_CONNECTED; + break; + + default: + log_err(LD_BUG, "Invalid proxy_state for reading, %d", + conn->proxy_state); + tor_fragile_assert(); + ret = -1; + break; + } + + log_debug(LD_NET, "leaving state %s", + connection_proxy_state_to_string(conn->proxy_state)); + + if (ret < 0) { + if (reason) { + log_warn(LD_NET, "Proxy Client: unable to connect to %s:%d (%s)", + conn->address, conn->port, escaped(reason)); + tor_free(reason); + } else { + log_warn(LD_NET, "Proxy Client: unable to connect to %s:%d", + conn->address, conn->port); + } + } else if (ret == 1) { + log_info(LD_NET, "Proxy Client: connection to %s:%d successful", + conn->address, conn->port); + } + + return ret; +} + /** * Launch any configured listener connections of type <b>type</b>. (A * listener is configured if <b>port_option</b> is non-zero. If any @@ -1643,6 +1975,7 @@ connection_bucket_write_limit(connection_t *conn, time_t now) int base = connection_speaks_cells(conn) ? CELL_NETWORK_SIZE : RELAY_PAYLOAD_SIZE; int priority = conn->type != CONN_TYPE_DIR; + int conn_bucket = (int)conn->outbuf_flushlen; int global_bucket = global_write_bucket; if (!connection_is_rate_limited(conn)) { @@ -1650,12 +1983,22 @@ connection_bucket_write_limit(connection_t *conn, time_t now) return conn->outbuf_flushlen; } + if (connection_speaks_cells(conn)) { + /* use the per-conn write limit if it's lower, but if it's less + * than zero just use zero */ + or_connection_t *or_conn = TO_OR_CONN(conn); + if (conn->state == OR_CONN_STATE_OPEN) + if (or_conn->write_bucket < conn_bucket) + conn_bucket = or_conn->write_bucket >= 0 ? + or_conn->write_bucket : 0; + } + if (connection_counts_as_relayed_traffic(conn, now) && global_relayed_write_bucket <= global_write_bucket) global_bucket = global_relayed_write_bucket; - return connection_bucket_round_robin(base, priority, global_bucket, - conn->outbuf_flushlen); + return connection_bucket_round_robin(base, priority, + global_bucket, conn_bucket); } /** Return 1 if the global write buckets are low enough that we @@ -1709,8 +2052,8 @@ global_write_bucket_low(connection_t *conn, size_t attempt, int priority) return 0; } -/** We just read num_read and wrote num_written onto conn. - * Decrement buckets appropriately. */ +/** We just read <b>num_read</b> and wrote <b>num_written</b> bytes + * onto <b>conn</b>. Decrement buckets appropriately. */ static void connection_buckets_decrement(connection_t *conn, time_t now, size_t num_read, size_t num_written) @@ -1728,10 +2071,16 @@ connection_buckets_decrement(connection_t *conn, time_t now, tor_fragile_assert(); } - if (num_read > 0) + if (num_read > 0) { + if (conn->type == CONN_TYPE_EXIT) + rep_hist_note_exit_bytes_read(conn->port, num_read); rep_hist_note_bytes_read(num_read, now); - if (num_written > 0) + } + if (num_written > 0) { + if (conn->type == CONN_TYPE_EXIT) + rep_hist_note_exit_bytes_written(conn->port, num_written); rep_hist_note_bytes_written(num_written, now); + } if (connection_counts_as_relayed_traffic(conn, now)) { global_relayed_read_bucket -= (int)num_read; @@ -1739,8 +2088,10 @@ connection_buckets_decrement(connection_t *conn, time_t now, } global_read_bucket -= (int)num_read; global_write_bucket -= (int)num_written; - if (connection_speaks_cells(conn) && conn->state == OR_CONN_STATE_OPEN) + if (connection_speaks_cells(conn) && conn->state == OR_CONN_STATE_OPEN) { TO_OR_CONN(conn)->read_bucket -= (int)num_read; + TO_OR_CONN(conn)->write_bucket -= (int)num_written; + } } /** If we have exhausted our global buckets, or the buckets for conn, @@ -1779,12 +2130,10 @@ connection_consider_empty_write_buckets(connection_t *conn) } else if (connection_counts_as_relayed_traffic(conn, approx_time()) && global_relayed_write_bucket <= 0) { reason = "global relayed write bucket exhausted. Pausing."; -#if 0 } else if (connection_speaks_cells(conn) && conn->state == OR_CONN_STATE_OPEN && TO_OR_CONN(conn)->write_bucket <= 0) { reason = "connection write bucket exhausted. Pausing."; -#endif } else return; /* all good, no need to stop it */ @@ -1880,14 +2229,19 @@ connection_bucket_refill(int seconds_elapsed, time_t now) { if (connection_speaks_cells(conn)) { or_connection_t *or_conn = TO_OR_CONN(conn); - if (connection_read_bucket_should_increase(or_conn)) { + if (connection_bucket_should_increase(or_conn->read_bucket, or_conn)) { connection_bucket_refill_helper(&or_conn->read_bucket, or_conn->bandwidthrate, or_conn->bandwidthburst, seconds_elapsed, "or_conn->read_bucket"); - //log_fn(LOG_DEBUG,"Receiver bucket %d now %d.", i, - // conn->read_bucket); + } + if (connection_bucket_should_increase(or_conn->write_bucket, or_conn)) { + connection_bucket_refill_helper(&or_conn->write_bucket, + or_conn->bandwidthrate, + or_conn->bandwidthburst, + seconds_elapsed, + "or_conn->write_bucket"); } } @@ -1908,8 +2262,10 @@ connection_bucket_refill(int seconds_elapsed, time_t now) if (conn->write_blocked_on_bw == 1 && global_write_bucket > 0 /* and we're allowed to write */ && (!connection_counts_as_relayed_traffic(conn, now) || - global_relayed_write_bucket > 0)) { - /* even if we're relayed traffic */ + global_relayed_write_bucket > 0) /* even if it's relayed traffic */ + && (!connection_speaks_cells(conn) || + conn->state != OR_CONN_STATE_OPEN || + TO_OR_CONN(conn)->write_bucket > 0)) { LOG_FN_CONN(conn, (LOG_DEBUG,LD_NET, "waking up conn (fd %d) for write", conn->s)); conn->write_blocked_on_bw = 0; @@ -1918,17 +2274,17 @@ connection_bucket_refill(int seconds_elapsed, time_t now) }); } -/** Is the receiver bucket for connection <b>conn</b> low enough that we +/** Is the <b>bucket</b> for connection <b>conn</b> low enough that we * should add another pile of tokens to it? */ static int -connection_read_bucket_should_increase(or_connection_t *conn) +connection_bucket_should_increase(int bucket, or_connection_t *conn) { tor_assert(conn); if (conn->_base.state != OR_CONN_STATE_OPEN) return 0; /* only open connections play the rate limiting game */ - if (conn->read_bucket >= conn->bandwidthburst) + if (bucket >= conn->bandwidthburst) return 0; return 1; @@ -1946,8 +2302,8 @@ connection_read_bucket_should_increase(or_connection_t *conn) * Mark the connection and return -1 if you want to close it, else * return 0. */ -int -connection_handle_read(connection_t *conn) +static int +connection_handle_read_impl(connection_t *conn) { int max_to_read=-1, try_to_read; size_t before, n_read = 0; @@ -2016,13 +2372,13 @@ loop_again: return -1; } if (conn->linked_conn) { - /* The other side's handle_write will never actually get called, so + /* The other side's handle_write() will never actually get called, so * we need to invoke the appropriate callbacks ourself. */ connection_t *linked = conn->linked_conn; if (n_read) { /* Probably a no-op, but hey. */ - connection_buckets_decrement(linked, approx_time(), 0, n_read); + connection_buckets_decrement(linked, approx_time(), n_read, 0); if (connection_flushed_some(linked) < 0) connection_mark_for_close(linked); @@ -2033,7 +2389,7 @@ loop_again: if (!buf_datalen(linked->outbuf) && conn->active_on_link) connection_stop_reading_from_linked_conn(conn); } - /* If we hit the EOF, call connection_reached_eof. */ + /* If we hit the EOF, call connection_reached_eof(). */ if (!conn->marked_for_close && conn->inbuf_reached_eof && connection_reached_eof(conn) < 0) { @@ -2042,6 +2398,16 @@ loop_again: return 0; } +int +connection_handle_read(connection_t *conn) +{ + int res; + + tor_gettimeofday_cache_clear(); + res = connection_handle_read_impl(conn); + return res; +} + /** Pull in new bytes from conn-\>s or conn-\>linked_conn onto conn-\>inbuf, * either directly or via TLS. Reduce the token buckets by the number of bytes * read. @@ -2075,7 +2441,7 @@ connection_read_to_buf(connection_t *conn, int *max_to_read, int *socket_error) } if (connection_speaks_cells(conn) && - conn->state > OR_CONN_STATE_PROXY_READING) { + conn->state > OR_CONN_STATE_PROXY_HANDSHAKING) { int pending; or_connection_t *or_conn = TO_OR_CONN(conn); size_t initial_size; @@ -2243,8 +2609,8 @@ connection_outbuf_too_full(connection_t *conn) * Mark the connection and return -1 if you want to close it, else * return 0. */ -int -connection_handle_write(connection_t *conn, int force) +static int +connection_handle_write_impl(connection_t *conn, int force) { int e; socklen_t len=(socklen_t)sizeof(e); @@ -2259,7 +2625,7 @@ connection_handle_write(connection_t *conn, int force) return 0; /* do nothing */ if (conn->in_flushed_some) { - log_warn(LD_BUG, "called recursively from inside conn->in_flushed_some()"); + log_warn(LD_BUG, "called recursively from inside conn->in_flushed_some"); return 0; } @@ -2303,7 +2669,7 @@ connection_handle_write(connection_t *conn, int force) : connection_bucket_write_limit(conn, now); if (connection_speaks_cells(conn) && - conn->state > OR_CONN_STATE_PROXY_READING) { + conn->state > OR_CONN_STATE_PROXY_HANDSHAKING) { or_connection_t *or_conn = TO_OR_CONN(conn); if (conn->state == OR_CONN_STATE_TLS_HANDSHAKING || conn->state == OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING) { @@ -2322,6 +2688,13 @@ connection_handle_write(connection_t *conn, int force) /* else open, or closing */ result = flush_buf_tls(or_conn->tls, conn->outbuf, max_to_write, &conn->outbuf_flushlen); + + /* If we just flushed the last bytes, check if this tunneled dir + * request is done. */ + if (buf_datalen(conn->outbuf) == 0 && conn->dirreq_id) + geoip_change_dirreq_state(conn->dirreq_id, DIRREQ_TUNNELED, + DIRREQ_OR_CONN_BUFFER_FLUSHED); + switch (result) { CASE_TOR_TLS_ERROR_ANY: case TOR_TLS_CLOSE: @@ -2341,8 +2714,8 @@ connection_handle_write(connection_t *conn, int force) if (!connection_is_reading(conn)) { connection_stop_writing(conn); conn->write_blocked_on_bw = 1; - /* we'll start reading again when the next second arrives, - * and then also start writing again. + /* we'll start reading again when we get more tokens in our + * read bucket; then we'll start writing again too. */ } /* else no problem, we're already reading */ @@ -2404,6 +2777,15 @@ connection_handle_write(connection_t *conn, int force) return 0; } +int +connection_handle_write(connection_t *conn, int force) +{ + int res; + tor_gettimeofday_cache_clear(); + res = connection_handle_write_impl(conn, force); + return res; +} + /** OpenSSL TLS record size is 16383; this is close. The goal here is to * push data out as soon as we know there's enough for a TLS record, so * during periods of high load we won't read entire megabytes from @@ -2577,13 +2959,11 @@ connection_get_by_type_state(int type, int state) /** Return a connection of type <b>type</b> that has rendquery equal * to <b>rendquery</b>, and that is not marked for close. If state - * is non-zero, conn must be of that state too. If rendversion is - * nonnegative, conn must be fetching that rendversion, too. + * is non-zero, conn must be of that state too. */ connection_t * connection_get_by_type_state_rendquery(int type, int state, - const char *rendquery, - int rendversion) + const char *rendquery) { smartlist_t *conns = get_connection_array(); @@ -2598,8 +2978,6 @@ connection_get_by_type_state_rendquery(int type, int state, (!state || state == conn->state)) { if (type == CONN_TYPE_DIR && TO_DIR_CONN(conn)->rend_data && - (rendversion < 0 || - rendversion == TO_DIR_CONN(conn)->rend_data->rend_desc_version) && !rend_cmp_service_ids(rendquery, TO_DIR_CONN(conn)->rend_data->onion_address)) return conn; @@ -2734,7 +3112,7 @@ client_check_address_changed(int sock) return; } - /* Okay. If we've used this address previously, we're okay. */ + /* If we've used this address previously, we're okay. */ ip_out = ntohl(out_addr.sin_addr.s_addr); SMARTLIST_FOREACH(outgoing_addrs, uint32_t*, ip_ptr, if (*ip_ptr == ip_out) return; @@ -3033,7 +3411,7 @@ assert_connection_ok(connection_t *conn, time_t now) } // tor_assert(conn->addr && conn->port); tor_assert(conn->address); - if (conn->state > OR_CONN_STATE_PROXY_READING) + if (conn->state > OR_CONN_STATE_PROXY_HANDSHAKING) tor_assert(or_conn->tls); } diff --git a/src/or/connection_edge.c b/src/or/connection_edge.c index d699591cdc..d7e8394614 100644 --- a/src/or/connection_edge.c +++ b/src/or/connection_edge.c @@ -330,11 +330,13 @@ connection_edge_finished_connecting(edge_connection_t *edge_conn) tor_assert(conn->state == EXIT_CONN_STATE_CONNECTING); log_info(LD_EXIT,"Exit connection to %s:%u (%s) established.", - escaped_safe_str(conn->address),conn->port, + escaped_safe_str(conn->address), conn->port, safe_str(fmt_addr(&conn->addr))); + rep_hist_note_exit_stream_opened(conn->port); + conn->state = EXIT_CONN_STATE_OPEN; - connection_watch_events(conn, EV_READ); /* stop writing, continue reading */ + connection_watch_events(conn, READ_EVENT); /* stop writing, keep reading */ if (connection_wants_to_flush(conn)) /* in case there are any queued relay * cells */ connection_start_writing(conn); @@ -375,13 +377,16 @@ connection_edge_finished_connecting(edge_connection_t *edge_conn) static int compute_retry_timeout(edge_connection_t *conn) { + int timeout = get_options()->CircuitStreamTimeout; + if (timeout) /* if our config options override the default, use them */ + return timeout; if (conn->num_socks_retries < 2) /* try 0 and try 1 */ return 10; return 15; } /** Find all general-purpose AP streams waiting for a response that sent their - * begin/resolve cell >=15 seconds ago. Detach from their current circuit, and + * begin/resolve cell too long ago. Detach from their current circuit, and * mark their current circuit as unsuitable for new streams. Then call * connection_ap_handshake_attach_circuit() to attach to a new circuit (if * available) or launch a new one. @@ -423,7 +428,8 @@ connection_ap_expire_beginning(void) log_fn(severity, LD_APP, "Tried for %d seconds to get a connection to %s:%d. " "Giving up. (%s)", - seconds_since_born, safe_str(conn->socks_request->address), + seconds_since_born, + safe_str_client(conn->socks_request->address), conn->socks_request->port, conn_state_to_string(CONN_TYPE_AP, conn->_base.state)); connection_mark_unattached_ap(conn, END_STREAM_REASON_TIMEOUT); @@ -440,7 +446,7 @@ connection_ap_expire_beginning(void) circ = circuit_get_by_edge_conn(conn); if (!circ) { /* it's vanished? */ log_info(LD_APP,"Conn is waiting (address %s), but lost its circ.", - safe_str(conn->socks_request->address)); + safe_str_client(conn->socks_request->address)); connection_mark_unattached_ap(conn, END_STREAM_REASON_TIMEOUT); continue; } @@ -450,7 +456,7 @@ connection_ap_expire_beginning(void) "Rend stream is %d seconds late. Giving up on address" " '%s.onion'.", seconds_idle, - safe_str(conn->socks_request->address)); + safe_str_client(conn->socks_request->address)); connection_edge_end(conn, END_STREAM_REASON_TIMEOUT); connection_mark_unattached_ap(conn, END_STREAM_REASON_TIMEOUT); } @@ -460,7 +466,8 @@ connection_ap_expire_beginning(void) log_fn(cutoff < 15 ? LOG_INFO : severity, LD_APP, "We tried for %d seconds to connect to '%s' using exit '%s'." " Retrying on a new circuit.", - seconds_idle, safe_str(conn->socks_request->address), + seconds_idle, + safe_str_client(conn->socks_request->address), conn->cpath_layer ? conn->cpath_layer->extend_info->nickname : "*unnamed*"); /* send an end down the circuit */ @@ -577,8 +584,8 @@ circuit_discard_optional_exit_enclaves(extend_info_t *info) tor_assert(edge_conn->socks_request); if (edge_conn->chosen_exit_optional) { log_info(LD_APP, "Giving up on enclave exit '%s' for destination %s.", - safe_str(edge_conn->chosen_exit_name), - escaped_safe_str(edge_conn->socks_request->address)); + safe_str_client(edge_conn->chosen_exit_name), + escaped_safe_str_client(edge_conn->socks_request->address)); edge_conn->chosen_exit_optional = 0; tor_free(edge_conn->chosen_exit_name); /* clears it */ /* if this port is dangerous, warn or reject it now that we don't @@ -683,7 +690,11 @@ addressmap_init(void) static void addressmap_ent_free(void *_ent) { - addressmap_entry_t *ent = _ent; + addressmap_entry_t *ent; + if (!_ent) + return; + + ent = _ent; tor_free(ent->new_address); tor_free(ent); } @@ -692,7 +703,11 @@ addressmap_ent_free(void *_ent) static void addressmap_virtaddress_ent_free(void *_ent) { - virtaddress_entry_t *ent = _ent; + virtaddress_entry_t *ent; + if (!_ent) + return; + + ent = _ent; tor_free(ent->ipv4_address); tor_free(ent->hostname_address); tor_free(ent); @@ -782,14 +797,11 @@ addressmap_clean(time_t now) void addressmap_free_all(void) { - if (addressmap) { - strmap_free(addressmap, addressmap_ent_free); - addressmap = NULL; - } - if (virtaddress_reversemap) { - strmap_free(virtaddress_reversemap, addressmap_virtaddress_ent_free); - virtaddress_reversemap = NULL; - } + strmap_free(addressmap, addressmap_ent_free); + addressmap = NULL; + + strmap_free(virtaddress_reversemap, addressmap_virtaddress_ent_free); + virtaddress_reversemap = NULL; } /** Look at address, and rewrite it until it doesn't want any @@ -816,9 +828,9 @@ addressmap_rewrite(char *address, size_t maxlen, time_t *expires_out) return (rewrites > 0); /* done, no rewrite needed */ } - cp = tor_strdup(escaped_safe_str(ent->new_address)); + cp = tor_strdup(escaped_safe_str_client(ent->new_address)); log_info(LD_APP, "Addressmap: rewriting %s to %s", - escaped_safe_str(address), cp); + escaped_safe_str_client(address), cp); if (ent->expires > 1 && ent->expires < expires) expires = ent->expires; tor_free(cp); @@ -826,7 +838,7 @@ addressmap_rewrite(char *address, size_t maxlen, time_t *expires_out) } log_warn(LD_CONFIG, "Loop detected: we've rewritten %s 16 times! Using it as-is.", - escaped_safe_str(address)); + escaped_safe_str_client(address)); /* it's fine to rewrite a rewrite, but don't loop forever */ if (expires_out) *expires_out = TIME_MAX; @@ -848,9 +860,9 @@ addressmap_rewrite_reverse(char *address, size_t maxlen, time_t *expires_out) tor_snprintf(s, len, "REVERSE[%s]", address); ent = strmap_get(addressmap, s); if (ent) { - cp = tor_strdup(escaped_safe_str(ent->new_address)); + cp = tor_strdup(escaped_safe_str_client(ent->new_address)); log_info(LD_APP, "Rewrote reverse lookup %s -> %s", - escaped_safe_str(s), cp); + escaped_safe_str_client(s), cp); tor_free(cp); strlcpy(address, ent->new_address, maxlen); r = 1; @@ -912,7 +924,9 @@ addressmap_register(const char *address, char *new_address, time_t expires, if (expires > 1) { log_info(LD_APP,"Temporary addressmap ('%s' to '%s') not performed, " "since it's already mapped to '%s'", - safe_str(address), safe_str(new_address), safe_str(ent->new_address)); + safe_str_client(address), + safe_str_client(new_address), + safe_str_client(ent->new_address)); tor_free(new_address); return; } @@ -931,7 +945,8 @@ addressmap_register(const char *address, char *new_address, time_t expires, ent->source = source; log_info(LD_CONFIG, "Addressmap: (re)mapped '%s' to '%s'", - safe_str(address), safe_str(ent->new_address)); + safe_str_client(address), + safe_str_client(ent->new_address)); control_event_address_mapped(address, ent->new_address, expires, NULL); } @@ -951,7 +966,8 @@ client_dns_incr_failures(const char *address) if (ent->num_resolve_failures < SHORT_MAX) ++ent->num_resolve_failures; /* don't overflow */ log_info(LD_APP, "Address %s now has %d resolve failures.", - safe_str(address), ent->num_resolve_failures); + safe_str_client(address), + ent->num_resolve_failures); return ent->num_resolve_failures; } @@ -1230,8 +1246,10 @@ addressmap_register_virtual_address(int type, char *new_address) log_warn(LD_BUG, "Internal confusion: I thought that '%s' was mapped to by " "'%s', but '%s' really maps to '%s'. This is a harmless bug.", - safe_str(new_address), safe_str(*addrp), safe_str(*addrp), - ent?safe_str(ent->new_address):"(nothing)"); + safe_str_client(new_address), + safe_str_client(*addrp), + safe_str_client(*addrp), + ent?safe_str_client(ent->new_address):"(nothing)"); } tor_free(*addrp); @@ -1252,7 +1270,8 @@ addressmap_register_virtual_address(int type, char *new_address) (type == RESOLVED_TYPE_IPV4) ? vent->ipv4_address : vent->hostname_address)); log_info(LD_APP, "Map from %s to %s okay.", - safe_str(*addrp),safe_str(new_address)); + safe_str_client(*addrp), + safe_str_client(new_address)); } #endif @@ -1400,7 +1419,7 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, tor_strlower(socks->address); /* normalize it */ strlcpy(orig_address, socks->address, sizeof(orig_address)); log_debug(LD_APP,"Client asked for %s:%d", - safe_str(socks->address), + safe_str_client(socks->address), socks->port); if (socks->command == SOCKS_COMMAND_RESOLVE && @@ -1417,7 +1436,8 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, RESOLVED_TYPE_IPV4, tor_strdup(socks->address)); tor_assert(new_addr); log_info(LD_APP, "Automapping %s to %s", - escaped_safe_str(socks->address), safe_str(new_addr)); + escaped_safe_str_client(socks->address), + safe_str_client(new_addr)); strlcpy(socks->address, new_addr, sizeof(socks->address)); } } @@ -1473,7 +1493,7 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, * information. */ log_warn(LD_APP,"Missing mapping for virtual address '%s'. Refusing.", - socks->address); /* don't safe_str() this yet. */ + safe_str_client(socks->address)); connection_mark_unattached_ap(conn, END_STREAM_REASON_INTERNAL); return -1; } @@ -1481,11 +1501,12 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, /* Parse the address provided by SOCKS. Modify it in-place if it * specifies a hidden-service (.onion) or particular exit node (.exit). */ - addresstype = parse_extended_hostname(socks->address); + addresstype = parse_extended_hostname(socks->address, + remapped_to_exit || options->AllowDotExit); if (addresstype == BAD_HOSTNAME) { log_warn(LD_APP, "Invalid onion hostname %s; rejecting", - safe_str(socks->address)); + safe_str_client(socks->address)); control_event_client_status(LOG_WARN, "SOCKS_BAD_HOSTNAME HOSTNAME=%s", escaped(socks->address)); connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL); @@ -1494,7 +1515,7 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, if (addresstype == EXIT_HOSTNAME) { /* foo.exit -- modify conn->chosen_exit_node to specify the exit - * node, and conn->address to hold only the address portion.*/ + * node, and conn->address to hold only the address portion. */ char *s = strrchr(socks->address,'.'); tor_assert(!automap); if (s) { @@ -1505,7 +1526,7 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, *s = 0; } else { log_warn(LD_APP,"Malformed exit address '%s.exit'. Refusing.", - safe_str(socks->address)); + safe_str_client(socks->address)); control_event_client_status(LOG_WARN, "SOCKS_BAD_HOSTNAME HOSTNAME=%s", escaped(socks->address)); connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL); @@ -1521,7 +1542,7 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, } else { log_warn(LD_APP, "Unrecognized server in exit address '%s.exit'. Refusing.", - safe_str(socks->address)); + safe_str_client(socks->address)); connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL); return -1; } @@ -1535,7 +1556,7 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, escaped(socks->address)); log_warn(LD_APP, "Destination '%s' seems to be an invalid hostname. Failing.", - safe_str(socks->address)); + safe_str_client(socks->address)); connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL); return -1; } @@ -1544,18 +1565,6 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, uint32_t answer; struct in_addr in; /* Reply to resolves immediately if we can. */ - if (strlen(socks->address) > RELAY_PAYLOAD_SIZE) { - log_warn(LD_APP,"Address to be resolved is too large. Failing."); - control_event_client_status(LOG_WARN, "SOCKS_BAD_HOSTNAME HOSTNAME=%s", - escaped(socks->address)); - connection_ap_handshake_socks_resolved(conn, - RESOLVED_TYPE_ERROR_TRANSIENT, - 0,NULL,-1,TIME_MAX); - connection_mark_unattached_ap(conn, - END_STREAM_REASON_SOCKSPROTOCOL | - END_STREAM_REASON_FLAG_ALREADY_SOCKS_REPLIED); - return -1; - } if (tor_inet_aton(socks->address, &in)) { /* see if it's an IP already */ /* leave it in network order */ answer = in.s_addr; @@ -1585,7 +1594,7 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, if (r) { log_info(LD_APP, "Redirecting address %s to exit at enclave router %s", - safe_str(socks->address), r->nickname); + safe_str_client(socks->address), r->nickname); /* use the hex digest, not nickname, in case there are two routers with this nickname */ conn->chosen_exit_name = @@ -1649,12 +1658,12 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, strlcpy(conn->rend_data->onion_address, socks->address, sizeof(conn->rend_data->onion_address)); log_info(LD_REND,"Got a hidden service request for ID '%s'", - safe_str(conn->rend_data->onion_address)); + safe_str_client(conn->rend_data->onion_address)); /* see if we already have it cached */ r = rend_cache_lookup_entry(conn->rend_data->onion_address, -1, &entry); if (r<0) { log_warn(LD_BUG,"Invalid service name '%s'", - safe_str(conn->rend_data->onion_address)); + safe_str_client(conn->rend_data->onion_address)); connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL); return -1; } @@ -1676,32 +1685,15 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, if (r==0) { conn->_base.state = AP_CONN_STATE_RENDDESC_WAIT; log_info(LD_REND, "Unknown descriptor %s. Fetching.", - safe_str(conn->rend_data->onion_address)); - /* Fetch both, v0 and v2 rend descriptors in parallel. Use whichever - * arrives first. Exception: When using client authorization, only - * fetch v2 descriptors.*/ + safe_str_client(conn->rend_data->onion_address)); rend_client_refetch_v2_renddesc(conn->rend_data); - if (conn->rend_data->auth_type == REND_NO_AUTH) - rend_client_refetch_renddesc(conn->rend_data->onion_address); } else { /* r > 0 */ - if (now - entry->received < NUM_SECONDS_BEFORE_HS_REFETCH) { - conn->_base.state = AP_CONN_STATE_CIRCUIT_WAIT; - log_info(LD_REND, "Descriptor is here and fresh enough. Great."); - if (connection_ap_handshake_attach_circuit(conn) < 0) { - if (!conn->_base.marked_for_close) - connection_mark_unattached_ap(conn, END_STREAM_REASON_CANT_ATTACH); - return -1; - } - } else { - conn->_base.state = AP_CONN_STATE_RENDDESC_WAIT; - log_info(LD_REND, "Stale descriptor %s. Re-fetching.", - safe_str(conn->rend_data->onion_address)); - /* Fetch both, v0 and v2 rend descriptors in parallel. Use whichever - * arrives first. Exception: When using client authorization, only - * fetch v2 descriptors.*/ - rend_client_refetch_v2_renddesc(conn->rend_data); - if (conn->rend_data->auth_type == REND_NO_AUTH) - rend_client_refetch_renddesc(conn->rend_data->onion_address); + conn->_base.state = AP_CONN_STATE_CIRCUIT_WAIT; + log_info(LD_REND, "Descriptor is here. Great."); + if (connection_ap_handshake_attach_circuit(conn) < 0) { + if (!conn->_base.marked_for_close) + connection_mark_unattached_ap(conn, END_STREAM_REASON_CANT_ATTACH); + return -1; } } return 0; @@ -1889,14 +1881,6 @@ connection_ap_handshake_process_socks(edge_connection_t *conn) return -1; } /* else socks handshake is done, continue processing */ - if (hostname_is_noconnect_address(socks->address)) - { - control_event_stream_status(conn, STREAM_EVENT_NEW, 0); - control_event_stream_status(conn, STREAM_EVENT_CLOSED, 0); - connection_mark_unattached_ap(conn, END_STREAM_REASON_DONE); - return -1; - } - if (SOCKS_COMMAND_IS_CONNECT(socks->command)) control_event_stream_status(conn, STREAM_EVENT_NEW, 0); else @@ -2160,7 +2144,7 @@ connection_ap_handshake_send_resolve(edge_connection_t *ap_conn) r = tor_addr_parse_reverse_lookup_name(&addr, a, AF_INET, 1); if (r <= 0) { log_warn(LD_APP, "Rejecting ill-formed reverse lookup of %s", - safe_str(a)); + safe_str_client(a)); connection_mark_unattached_ap(ap_conn, END_STREAM_REASON_INTERNAL); return -1; } @@ -2168,7 +2152,7 @@ connection_ap_handshake_send_resolve(edge_connection_t *ap_conn) r = tor_addr_to_reverse_lookup_name(inaddr_buf, sizeof(inaddr_buf), &addr); if (r < 0) { log_warn(LD_BUG, "Couldn't generate reverse lookup hostname of %s", - safe_str(a)); + safe_str_client(a)); connection_mark_unattached_ap(ap_conn, END_STREAM_REASON_INTERNAL); return -1; } @@ -2178,12 +2162,6 @@ connection_ap_handshake_send_resolve(edge_connection_t *ap_conn) tor_assert(payload_len <= (int)sizeof(inaddr_buf)); } - if (payload_len > RELAY_PAYLOAD_SIZE) { - /* This should be impossible: we don't accept addresses this big. */ - connection_mark_unattached_ap(ap_conn, END_STREAM_REASON_INTERNAL); - return -1; - } - log_debug(LD_APP, "Sending relay cell to begin stream %d.", ap_conn->stream_id); @@ -2215,7 +2193,8 @@ connection_ap_make_link(char *address, uint16_t port, edge_connection_t *conn; log_info(LD_APP,"Making internal %s tunnel to %s:%d ...", - want_onehop ? "direct" : "anonymized" , safe_str(address),port); + want_onehop ? "direct" : "anonymized", + safe_str_client(address), port); conn = edge_connection_new(CONN_TYPE_AP, AF_INET); conn->_base.linked = 1; /* so that we can add it safely below. */ @@ -2566,6 +2545,11 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ) log_debug(LD_EXIT,"Creating new exit connection."); n_stream = edge_connection_new(CONN_TYPE_EXIT, AF_INET); + + /* Remember the tunneled request ID in the new edge connection, so that + * we can measure download times. */ + TO_CONN(n_stream)->dirreq_id = circ->dirreq_id; + n_stream->_base.purpose = EXIT_PURPOSE_CONNECT; n_stream->stream_id = rh.stream_id; @@ -2713,7 +2697,7 @@ connection_exit_connect(edge_connection_t *edge_conn) if (!connection_edge_is_rendezvous_stream(edge_conn) && router_compare_to_my_exit_policy(edge_conn)) { log_info(LD_EXIT,"%s:%d failed exit policy. Closing.", - escaped_safe_str(conn->address), conn->port); + escaped_safe_str_client(conn->address), conn->port); connection_edge_end(edge_conn, END_STREAM_REASON_EXITPOLICY); circuit_detach_stream(circuit_get_by_edge_conn(edge_conn), edge_conn); connection_free(conn); @@ -2735,7 +2719,7 @@ connection_exit_connect(edge_connection_t *edge_conn) case 0: conn->state = EXIT_CONN_STATE_CONNECTING; - connection_watch_events(conn, EV_WRITE | EV_READ); + connection_watch_events(conn, READ_EVENT | WRITE_EVENT); /* writable indicates finish; * readable/error indicates broken link in windows-land. */ return; @@ -2748,7 +2732,7 @@ connection_exit_connect(edge_connection_t *edge_conn) log_warn(LD_BUG,"newly connected conn had data waiting!"); // connection_start_writing(conn); } - connection_watch_events(conn, EV_READ); + connection_watch_events(conn, READ_EVENT); /* also, deliver a 'connected' cell back through the circuit. */ if (connection_edge_is_rendezvous_stream(edge_conn)) { @@ -2802,6 +2786,10 @@ connection_exit_connect_dir(edge_connection_t *exitconn) dirconn->_base.purpose = DIR_PURPOSE_SERVER; dirconn->_base.state = DIR_CONN_STATE_SERVER_COMMAND_WAIT; + /* Note that the new dir conn belongs to the same tunneled request as + * the edge conn, so that we can measure download times. */ + TO_CONN(dirconn)->dirreq_id = TO_CONN(exitconn)->dirreq_id; + connection_link_connections(TO_CONN(dirconn), TO_CONN(exitconn)); if (connection_add(TO_CONN(exitconn))<0) { @@ -2852,10 +2840,16 @@ connection_edge_is_rendezvous_stream(edge_connection_t *conn) * to exit from it, or 0 if it probably will not allow it. * (We might be uncertain if conn's destination address has not yet been * resolved.) + * + * If <b>excluded_means_no</b> is 1 and Exclude*Nodes is set and excludes + * this relay, return 0. */ int -connection_ap_can_use_exit(edge_connection_t *conn, routerinfo_t *exit) +connection_ap_can_use_exit(edge_connection_t *conn, routerinfo_t *exit, + int excluded_means_no) { + or_options_t *options = get_options(); + tor_assert(conn); tor_assert(conn->_base.type == CONN_TYPE_AP); tor_assert(conn->socks_request); @@ -2901,20 +2895,35 @@ connection_ap_can_use_exit(edge_connection_t *conn, routerinfo_t *exit) if (!conn->chosen_exit_name && policy_is_reject_star(exit->exit_policy)) return 0; } + if (options->_ExcludeExitNodesUnion && + (options->StrictNodes || excluded_means_no) && + routerset_contains_router(options->_ExcludeExitNodesUnion, exit)) { + /* If we are trying to avoid this node as exit, and we have StrictNodes + * set, then this is not a suitable exit. Refuse it. + * + * If we don't have StrictNodes set, then this function gets called in + * two contexts. First, we've got a circuit open and we want to know + * whether we can use it. In that case, we somehow built this circuit + * despite having the last hop in ExcludeExitNodes, so we should be + * willing to use it. Second, we are evaluating whether this is an + * acceptable exit for a new circuit. In that case, skip it. */ + return 0; + } + return 1; } /** If address is of the form "y.onion" with a well-formed handle y: * Put a NUL after y, lower-case it, and return ONION_HOSTNAME. * - * If address is of the form "y.exit": + * If address is of the form "y.exit" and <b>allowdotexit</b> is true: * Put a NUL after y and return EXIT_HOSTNAME. * * Otherwise: * Return NORMAL_HOSTNAME and change nothing. */ hostname_type_t -parse_extended_hostname(char *address) +parse_extended_hostname(char *address, int allowdotexit) { char *s; char query[REND_SERVICE_ID_LEN_BASE32+1]; @@ -2923,8 +2932,13 @@ parse_extended_hostname(char *address) if (!s) return NORMAL_HOSTNAME; /* no dot, thus normal */ if (!strcmp(s+1,"exit")) { - *s = 0; /* NUL-terminate it */ - return EXIT_HOSTNAME; /* .exit */ + if (allowdotexit) { + *s = 0; /* NUL-terminate it */ + return EXIT_HOSTNAME; /* .exit */ + } /* else */ + log_warn(LD_APP, "The \".exit\" notation is disabled in Tor due to " + "security risks. Set AllowDotExit in your torrc to enable it."); + /* FFFF send a controller event too to notify Vidalia users */ } if (strcmp(s+1,"onion")) return NORMAL_HOSTNAME; /* neither .exit nor .onion, thus normal */ @@ -2943,11 +2957,3 @@ failed: return BAD_HOSTNAME; } -/** Check if the address is of the form "y.noconnect" - */ -int -hostname_is_noconnect_address(const char *address) -{ - return ! strcasecmpend(address, ".noconnect"); -} - diff --git a/src/or/connection_or.c b/src/or/connection_or.c index 2a52b3fcd6..1aa0bb35b7 100644 --- a/src/or/connection_or.c +++ b/src/or/connection_or.c @@ -80,10 +80,8 @@ connection_or_clear_identity_map(void) } }); - if (orconn_identity_map) { - digestmap_free(orconn_identity_map, NULL); - orconn_identity_map = NULL; - } + digestmap_free(orconn_identity_map, NULL); + orconn_identity_map = NULL; } /** Change conn->identity_digest to digest, and add conn into @@ -187,66 +185,6 @@ connection_or_reached_eof(or_connection_t *conn) return 0; } -/** Read conn's inbuf. If the http response from the proxy is all - * here, make sure it's good news, and begin the tls handshake. If - * it's bad news, close the connection and return -1. Else return 0 - * and hope for better luck next time. - */ -static int -connection_or_read_proxy_response(or_connection_t *or_conn) -{ - char *headers; - char *reason=NULL; - int status_code; - time_t date_header; - connection_t *conn = TO_CONN(or_conn); - - switch (fetch_from_buf_http(conn->inbuf, - &headers, MAX_HEADERS_SIZE, - NULL, NULL, 10000, 0)) { - case -1: /* overflow */ - log_warn(LD_PROTOCOL, - "Your https proxy sent back an oversized response. Closing."); - return -1; - case 0: - log_info(LD_OR,"https proxy response not all here yet. Waiting."); - return 0; - /* case 1, fall through */ - } - - if (parse_http_response(headers, &status_code, &date_header, - NULL, &reason) < 0) { - log_warn(LD_OR, - "Unparseable headers from proxy (connecting to '%s'). Closing.", - conn->address); - tor_free(headers); - return -1; - } - if (!reason) reason = tor_strdup("[no reason given]"); - - if (status_code == 200) { - log_info(LD_OR, - "HTTPS connect to '%s' successful! (200 %s) Starting TLS.", - conn->address, escaped(reason)); - tor_free(reason); - if (connection_tls_start_handshake(or_conn, 0) < 0) { - /* TLS handshaking error of some kind. */ - connection_mark_for_close(conn); - - return -1; - } - return 0; - } - /* else, bad news on the status code */ - log_warn(LD_OR, - "The https proxy sent back an unexpected status code %d (%s). " - "Closing.", - status_code, escaped(reason)); - tor_free(reason); - connection_mark_for_close(conn); - return -1; -} - /** Handle any new bytes that have come in on connection <b>conn</b>. * If conn is in 'open' state, hand it to * connection_or_process_cells_from_inbuf() @@ -255,11 +193,24 @@ connection_or_read_proxy_response(or_connection_t *or_conn) int connection_or_process_inbuf(or_connection_t *conn) { + int ret; tor_assert(conn); switch (conn->_base.state) { - case OR_CONN_STATE_PROXY_READING: - return connection_or_read_proxy_response(conn); + case OR_CONN_STATE_PROXY_HANDSHAKING: + ret = connection_read_proxy_handshake(TO_CONN(conn)); + + /* start TLS after handshake completion, or deal with error */ + if (ret == 1) { + tor_assert(TO_CONN(conn)->proxy_state == PROXY_CONNECTED); + if (connection_tls_start_handshake(conn, 0) < 0) + ret = -1; + } + if (ret < 0) { + connection_mark_for_close(TO_CONN(conn)); + } + + return ret; case OR_CONN_STATE_OPEN: case OR_CONN_STATE_OR_HANDSHAKING: return connection_or_process_cells_from_inbuf(conn); @@ -312,11 +263,7 @@ connection_or_finished_flushing(or_connection_t *conn) assert_connection_ok(TO_CONN(conn),0); switch (conn->_base.state) { - case OR_CONN_STATE_PROXY_FLUSHING: - log_debug(LD_OR,"finished sending CONNECT to proxy."); - conn->_base.state = OR_CONN_STATE_PROXY_READING; - connection_stop_writing(TO_CONN(conn)); - break; + case OR_CONN_STATE_PROXY_HANDSHAKING: case OR_CONN_STATE_OPEN: case OR_CONN_STATE_OR_HANDSHAKING: connection_stop_writing(TO_CONN(conn)); @@ -334,37 +281,34 @@ connection_or_finished_flushing(or_connection_t *conn) int connection_or_finished_connecting(or_connection_t *or_conn) { + int proxy_type; connection_t *conn; tor_assert(or_conn); conn = TO_CONN(or_conn); tor_assert(conn->state == OR_CONN_STATE_CONNECTING); - log_debug(LD_OR,"OR connect() to router at %s:%u finished.", + log_debug(LD_HANDSHAKE,"OR connect() to router at %s:%u finished.", conn->address,conn->port); control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE, 0); - if (get_options()->HttpsProxy) { - char buf[1024]; - char *base64_authenticator=NULL; - const char *authenticator = get_options()->HttpsProxyAuthenticator; + proxy_type = PROXY_NONE; - if (authenticator) { - base64_authenticator = alloc_http_authenticator(authenticator); - if (!base64_authenticator) - log_warn(LD_OR, "Encoding https authenticator failed"); - } - if (base64_authenticator) { - tor_snprintf(buf, sizeof(buf), "CONNECT %s:%d HTTP/1.1\r\n" - "Proxy-Authorization: Basic %s\r\n\r\n", - fmt_addr(&conn->addr), - conn->port, base64_authenticator); - tor_free(base64_authenticator); - } else { - tor_snprintf(buf, sizeof(buf), "CONNECT %s:%d HTTP/1.0\r\n\r\n", - fmt_addr(&conn->addr), conn->port); + if (get_options()->HttpsProxy) + proxy_type = PROXY_CONNECT; + else if (get_options()->Socks4Proxy) + proxy_type = PROXY_SOCKS4; + else if (get_options()->Socks5Proxy) + proxy_type = PROXY_SOCKS5; + + if (proxy_type != PROXY_NONE) { + /* start proxy handshake */ + if (connection_proxy_connect(conn, proxy_type) < 0) { + connection_mark_for_close(conn); + return -1; } - connection_write_to_buf(buf, strlen(buf), conn); - conn->state = OR_CONN_STATE_PROXY_FLUSHING; + + connection_start_reading(conn); + conn->state = OR_CONN_STATE_PROXY_HANDSHAKING; return 0; } @@ -376,6 +320,19 @@ connection_or_finished_connecting(or_connection_t *or_conn) return 0; } +/** Return 1 if identity digest <b>id_digest</b> is known to be a + * currently or recently running relay. Otherwise return 0. */ +static int +connection_or_digest_is_known_relay(const char *id_digest) +{ + if (router_get_consensus_status_by_id(id_digest)) + return 1; /* It's in the consensus: "yes" */ + if (router_get_by_digest(id_digest)) + return 1; /* Not in the consensus, but we have a descriptor for + * it. Probably it was in a recent consensus. "Yes". */ + return 0; +} + /** If we don't necessarily know the router we're connecting to, but we * have an addr/port/id_digest, then fill in as much as we can. Start * by checking to see if this describes a router we know. */ @@ -386,11 +343,31 @@ connection_or_init_conn_from_address(or_connection_t *conn, int started_here) { or_options_t *options = get_options(); + int rate, burst; /* per-connection rate limiting params */ routerinfo_t *r = router_get_by_digest(id_digest); - conn->bandwidthrate = (int)options->BandwidthRate; - conn->read_bucket = conn->bandwidthburst = (int)options->BandwidthBurst; connection_or_set_identity_digest(conn, id_digest); + if (connection_or_digest_is_known_relay(id_digest)) { + /* It's in the consensus, or we have a descriptor for it meaning it + * was probably in a recent consensus. It's a recognized relay: + * give it full bandwidth. */ + rate = (int)options->BandwidthRate; + burst = (int)options->BandwidthBurst; + } else { + /* Not a recognized relay. Squeeze it down based on the suggested + * bandwidth parameters in the consensus, but allow local config + * options to override. */ + rate = options->PerConnBWRate ? (int)options->PerConnBWRate : + (int)networkstatus_get_param(NULL, "bwconnrate", + (int)options->BandwidthRate); + burst = options->PerConnBWBurst ? (int)options->PerConnBWBurst : + (int)networkstatus_get_param(NULL, "bwconnburst", + (int)options->BandwidthBurst); + } + + conn->bandwidthrate = rate; + conn->read_bucket = conn->write_bucket = conn->bandwidthburst = burst; + conn->_base.port = port; tor_addr_copy(&conn->_base.addr, addr); tor_addr_copy(&conn->real_addr, addr); @@ -753,6 +730,7 @@ connection_or_connect(const tor_addr_t *_addr, uint16_t port, or_connection_t *conn; or_options_t *options = get_options(); int socket_error = 0; + int using_proxy = 0; tor_addr_t addr; tor_assert(_addr); @@ -771,19 +749,27 @@ connection_or_connect(const tor_addr_t *_addr, uint16_t port, conn->_base.state = OR_CONN_STATE_CONNECTING; control_event_or_conn_status(conn, OR_CONN_EVENT_LAUNCHED, 0); + /* use a proxy server if available */ if (options->HttpsProxy) { - /* we shouldn't connect directly. use the https proxy instead. */ - tor_addr_from_ipv4h(&addr, options->HttpsProxyAddr); + using_proxy = 1; + tor_addr_copy(&addr, &options->HttpsProxyAddr); port = options->HttpsProxyPort; + } else if (options->Socks4Proxy) { + using_proxy = 1; + tor_addr_copy(&addr, &options->Socks4ProxyAddr); + port = options->Socks4ProxyPort; + } else if (options->Socks5Proxy) { + using_proxy = 1; + tor_addr_copy(&addr, &options->Socks5ProxyAddr); + port = options->Socks5ProxyPort; } switch (connection_connect(TO_CONN(conn), conn->_base.address, &addr, port, &socket_error)) { case -1: /* If the connection failed immediately, and we're using - * an https proxy, our https proxy is down. Don't blame the - * Tor server. */ - if (!options->HttpsProxy) + * a proxy, our proxy is down. Don't blame the Tor server. */ + if (!using_proxy) entry_guard_register_connect_status(conn->identity_digest, 0, 1, time(NULL)); connection_or_connect_failed(conn, @@ -792,7 +778,7 @@ connection_or_connect(const tor_addr_t *_addr, uint16_t port, connection_free(TO_CONN(conn)); return NULL; case 0: - connection_watch_events(TO_CONN(conn), EV_READ | EV_WRITE); + connection_watch_events(TO_CONN(conn), READ_EVENT | WRITE_EVENT); /* writable indicates finish, readable indicates broken link, error indicates broken link on windows */ return conn; @@ -819,13 +805,14 @@ connection_tls_start_handshake(or_connection_t *conn, int receiving) { conn->_base.state = OR_CONN_STATE_TLS_HANDSHAKING; conn->tls = tor_tls_new(conn->_base.s, receiving); - tor_tls_set_logged_address(conn->tls, escaped_safe_str(conn->_base.address)); + tor_tls_set_logged_address(conn->tls, // XXX client and relay? + escaped_safe_str(conn->_base.address)); if (!conn->tls) { log_warn(LD_BUG,"tor_tls_new failed. Closing."); return -1; } connection_start_reading(TO_CONN(conn)); - log_debug(LD_OR,"starting TLS handshake on fd %d", conn->_base.s); + log_debug(LD_HANDSHAKE,"starting TLS handshake on fd %d", conn->_base.s); note_crypto_pk_op(receiving ? TLS_HANDSHAKE_S : TLS_HANDSHAKE_C); if (connection_tls_continue_handshake(conn) < 0) { @@ -932,7 +919,7 @@ connection_or_nonopen_was_started_here(or_connection_t *conn) * return -1 if he is lying, broken, or otherwise something is wrong. * * If we initiated this connection (<b>started_here</b> is true), make sure - * the other side sent sent a correctly formed certificate. If I initiated the + * the other side sent a correctly formed certificate. If I initiated the * connection, make sure it's the right guy. * * Otherwise (if we _didn't_ initiate this connection), it's okay for @@ -959,19 +946,20 @@ connection_or_check_valid_tls_handshake(or_connection_t *conn, or_options_t *options = get_options(); int severity = server_mode(options) ? LOG_PROTOCOL_WARN : LOG_WARN; const char *safe_address = - started_here ? conn->_base.address : safe_str(conn->_base.address); + started_here ? conn->_base.address : + safe_str_client(conn->_base.address); const char *conn_type = started_here ? "outgoing" : "incoming"; int has_cert = 0, has_identity=0; check_no_tls_errors(); has_cert = tor_tls_peer_has_cert(conn->tls); if (started_here && !has_cert) { - log_info(LD_PROTOCOL,"Tried connecting to router at %s:%d, but it didn't " + log_info(LD_HANDSHAKE,"Tried connecting to router at %s:%d, but it didn't " "send a cert! Closing.", safe_address, conn->_base.port); return -1; } else if (!has_cert) { - log_debug(LD_PROTOCOL,"Got incoming connection with no certificate. " + log_debug(LD_HANDSHAKE,"Got incoming connection with no certificate. " "That's ok."); } check_no_tls_errors(); @@ -980,15 +968,16 @@ connection_or_check_valid_tls_handshake(or_connection_t *conn, int v = tor_tls_verify(started_here?severity:LOG_INFO, conn->tls, &identity_rcvd); if (started_here && v<0) { - log_fn(severity,LD_OR,"Tried connecting to router at %s:%d: It" + log_fn(severity,LD_HANDSHAKE,"Tried connecting to router at %s:%d: It" " has a cert but it's invalid. Closing.", safe_address, conn->_base.port); return -1; } else if (v<0) { - log_info(LD_PROTOCOL,"Incoming connection gave us an invalid cert " + log_info(LD_HANDSHAKE,"Incoming connection gave us an invalid cert " "chain; ignoring."); } else { - log_debug(LD_OR,"The certificate seems to be valid on %s connection " + log_debug(LD_HANDSHAKE, + "The certificate seems to be valid on %s connection " "with %s:%d", conn_type, safe_address, conn->_base.port); } check_no_tls_errors(); @@ -1015,7 +1004,7 @@ connection_or_check_valid_tls_handshake(or_connection_t *conn, conn->nickname[0] = '$'; base16_encode(conn->nickname+1, HEX_DIGEST_LEN+1, conn->identity_digest, DIGEST_LEN); - log_info(LD_OR, "Connected to router %s at %s:%d without knowing " + log_info(LD_HANDSHAKE, "Connected to router %s at %s:%d without knowing " "its key. Hoping for the best.", conn->nickname, conn->_base.address, conn->_base.port); } @@ -1031,7 +1020,7 @@ connection_or_check_valid_tls_handshake(or_connection_t *conn, base16_encode(seen, sizeof(seen), digest_rcvd_out, DIGEST_LEN); base16_encode(expected, sizeof(expected), conn->identity_digest, DIGEST_LEN); - log_fn(severity, LD_OR, + log_fn(severity, LD_HANDSHAKE, "Tried connecting to router at %s:%d, but identity key was not " "as expected: wanted %s but got %s.", conn->_base.address, conn->_base.port, expected, seen); @@ -1073,8 +1062,8 @@ connection_tls_finish_handshake(or_connection_t *conn) char digest_rcvd[DIGEST_LEN]; int started_here = connection_or_nonopen_was_started_here(conn); - log_debug(LD_OR,"tls handshake with %s done. verifying.", - safe_str(conn->_base.address)); + log_debug(LD_HANDSHAKE,"tls handshake with %s done. verifying.", + safe_str_client(conn->_base.address)); directory_set_dirty(); @@ -1082,6 +1071,8 @@ connection_tls_finish_handshake(or_connection_t *conn) digest_rcvd) < 0) return -1; + circuit_build_times_network_is_live(&circ_times); + if (tor_tls_used_v1_handshake(conn->tls)) { conn->link_proto = 1; if (!started_here) { @@ -1117,7 +1108,8 @@ connection_init_or_handshake_state(or_connection_t *conn, int started_here) void or_handshake_state_free(or_handshake_state_t *state) { - tor_assert(state); + if (!state) + return; memset(state, 0xBE, sizeof(or_handshake_state_t)); tor_free(state); } @@ -1134,6 +1126,7 @@ connection_or_set_state_open(or_connection_t *conn) control_event_or_conn_status(conn, OR_CONN_EVENT_CONNECTED, 0); if (started_here) { + circuit_build_times_network_is_live(&circ_times); rep_hist_note_connect_succeeded(conn->identity_digest, now); if (entry_guard_register_connect_status(conn->identity_digest, 1, 0, now) < 0) { @@ -1158,10 +1151,10 @@ connection_or_set_state_open(or_connection_t *conn) } } } - if (conn->handshake_state) { - or_handshake_state_free(conn->handshake_state); - conn->handshake_state = NULL; - } + + or_handshake_state_free(conn->handshake_state); + conn->handshake_state = NULL; + connection_start_reading(TO_CONN(conn)); circuit_n_conn_done(conn, 1); /* send the pending creates, if any. */ @@ -1234,6 +1227,7 @@ connection_or_process_cells_from_inbuf(or_connection_t *conn) if (connection_fetch_var_cell_from_buf(conn, &var_cell)) { if (!var_cell) return 0; /* not yet. */ + circuit_build_times_network_is_live(&circ_times); command_process_var_cell(var_cell, conn); var_cell_free(var_cell); } else { @@ -1243,6 +1237,7 @@ connection_or_process_cells_from_inbuf(or_connection_t *conn) available? */ return 0; /* not yet */ + circuit_build_times_network_is_live(&circ_times); connection_fetch_from_buf(buf, CELL_NETWORK_SIZE, TO_CONN(conn)); /* retrieve cell info from buf (create the host-order struct from the diff --git a/src/or/control.c b/src/or/control.c index ad9081da68..be1e921f31 100644 --- a/src/or/control.c +++ b/src/or/control.c @@ -54,13 +54,9 @@ **/ typedef uint32_t event_mask_t; -/** An event mask of all the events that controller with the LONG_NAMES option - * set is interested in receiving. */ -static event_mask_t global_event_mask1long = 0; - -/** An event mask of all the events that controller with the SHORT_NAMES option - * set is interested in receiving. */ -static event_mask_t global_event_mask1short = 0; +/** An event mask of all the events that any controller is interested in + * receiving. */ +static event_mask_t global_event_mask = 0; /** True iff we have disabled log messages from being sent to the controller */ static int disable_log_messages = 0; @@ -68,13 +64,7 @@ static int disable_log_messages = 0; /** Macro: true if any control connection is interested in events of type * <b>e</b>. */ #define EVENT_IS_INTERESTING(e) \ - ((global_event_mask1long|global_event_mask1short) & (1<<(e))) -/** Macro: true if any control connection with the LONG_NAMES option is - * interested in events of type <b>e</b>. */ -#define EVENT_IS_INTERESTING1L(e) (global_event_mask1long & (1<<(e))) -/** Macro: true if any control connection with the SHORT_NAMES option is - * interested in events of type <b>e</b>. */ -#define EVENT_IS_INTERESTING1S(e) (global_event_mask1short & (1<<(e))) + (global_event_mask & (1<<(e))) /** If we're using cookie-type authentication, how long should our cookies be? */ @@ -95,25 +85,13 @@ static char authentication_cookie[AUTHENTICATION_COOKIE_LEN]; * of this so we can respond to getinfo status/bootstrap-phase queries. */ static char last_sent_bootstrap_message[BOOTSTRAP_MSG_LEN]; -/** Flag for event_format_t. Indicates that we should use the old - * name format of nickname|hexdigest - */ -#define SHORT_NAMES 1 -/** Flag for event_format_t. Indicates that we should use the new - * name format of $hexdigest[=~]nickname +/** Flag for event_format_t. Indicates that we should use the one standard + format. */ -#define LONG_NAMES 2 -#define ALL_NAMES (SHORT_NAMES|LONG_NAMES) -/** Flag for event_format_t. Indicates that we should use the new event - * format where extra event fields are allowed using a NAME=VAL format. */ -#define EXTENDED_FORMAT 4 -/** Flag for event_format_t. Indicates that we are using the old event format - * where extra fields aren't allowed. */ -#define NONEXTENDED_FORMAT 8 -#define ALL_FORMATS (EXTENDED_FORMAT|NONEXTENDED_FORMAT) +#define ALL_FORMATS 1 /** Bit field of flags to select how to format a controller event. Recognized - * flags are SHORT_NAMES, LONG_NAMES, EXTENDED_FORMAT, NONEXTENDED_FORMAT. */ + * flag is ALL_FORMATS. */ typedef int event_format_t; static void connection_printf_to_buf(control_connection_t *conn, @@ -123,9 +101,6 @@ static void send_control_done(control_connection_t *conn); static void send_control_event(uint16_t event, event_format_t which, const char *format, ...) CHECK_PRINTF(3,4); -static void send_control_event_extended(uint16_t event, event_format_t which, - const char *format, ...) - CHECK_PRINTF(3,4); static int handle_control_setconf(control_connection_t *conn, uint32_t len, char *body); static int handle_control_resetconf(control_connection_t *conn, uint32_t len, @@ -139,8 +114,6 @@ static int handle_control_setevents(control_connection_t *conn, uint32_t len, static int handle_control_authenticate(control_connection_t *conn, uint32_t len, const char *body); -static int handle_control_saveconf(control_connection_t *conn, uint32_t len, - const char *body); static int handle_control_signal(control_connection_t *conn, uint32_t len, const char *body); static int handle_control_mapaddress(control_connection_t *conn, uint32_t len, @@ -174,7 +147,7 @@ static int handle_control_usefeature(control_connection_t *conn, const char *body); static int write_stream_target_to_buf(edge_connection_t *conn, char *buf, size_t len); -static void orconn_target_get_name(int long_names, char *buf, size_t len, +static void orconn_target_get_name(char *buf, size_t len, or_connection_t *conn); static char *get_cookie_file(void); @@ -214,25 +187,19 @@ control_update_global_event_mask(void) { smartlist_t *conns = get_connection_array(); event_mask_t old_mask, new_mask; - old_mask = global_event_mask1short; - old_mask |= global_event_mask1long; + old_mask = global_event_mask; - global_event_mask1short = 0; - global_event_mask1long = 0; + global_event_mask = 0; SMARTLIST_FOREACH(conns, connection_t *, _conn, { if (_conn->type == CONN_TYPE_CONTROL && STATE_IS_OPEN(_conn->state)) { control_connection_t *conn = TO_CONTROL_CONN(_conn); - if (conn->use_long_names) - global_event_mask1long |= conn->event_mask; - else - global_event_mask1short |= conn->event_mask; + global_event_mask |= conn->event_mask; } }); - new_mask = global_event_mask1short; - new_mask |= global_event_mask1long; + new_mask = global_event_mask; /* Handle the aftermath. Set up the log callback to tell us only what * we want to hear...*/ @@ -312,7 +279,7 @@ connection_write_str_to_buf(const char *s, control_connection_t *conn) /** Given a <b>len</b>-character string in <b>data</b>, made of lines * terminated by CRLF, allocate a new string in *<b>out</b>, and copy the * contents of <b>data</b> into *<b>out</b>, adding a period before any period - * that that appears at the start of a line, and adding a period-CRLF line at + * that appears at the start of a line, and adding a period-CRLF line at * the end. Replace all LF characters sequences with CRLF. Return the number * of bytes in *<b>out</b>. */ @@ -542,28 +509,15 @@ send_control_event_string(uint16_t event, event_format_t which, const char *msg) { smartlist_t *conns = get_connection_array(); + (void)which; tor_assert(event >= _EVENT_MIN && event <= _EVENT_MAX); - SMARTLIST_FOREACH(conns, connection_t *, conn, - { + SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) { if (conn->type == CONN_TYPE_CONTROL && !conn->marked_for_close && conn->state == CONTROL_CONN_STATE_OPEN) { control_connection_t *control_conn = TO_CONTROL_CONN(conn); - if (control_conn->use_long_names) { - if (!(which & LONG_NAMES)) - continue; - } else { - if (!(which & SHORT_NAMES)) - continue; - } - if (control_conn->use_extended_events) { - if (!(which & EXTENDED_FORMAT)) - continue; - } else { - if (!(which & NONEXTENDED_FORMAT)) - continue; - } + if (control_conn->event_mask & (1<<event)) { int is_err = 0; connection_write_to_buf(msg, strlen(msg), TO_CONN(control_conn)); @@ -579,7 +533,7 @@ send_control_event_string(uint16_t event, event_format_t which, connection_handle_write(TO_CONN(control_conn), 1); } } - }); + } SMARTLIST_FOREACH_END(conn); } /** Helper for send_control1_event and send_control1_event_extended: @@ -587,22 +541,17 @@ send_control_event_string(uint16_t event, event_format_t which, * <b>event</b>. The event's body is created by the printf-style format in * <b>format</b>, and other arguments as provided. * - * If <b>extended</b> is true, and the format contains a single '@' character, - * it will be replaced with a space and all text after that character will be - * sent only to controllers that have enabled extended events. - * * Currently the length of the message is limited to 1024 (including the * ending \\r\\n\\0). */ static void -send_control_event_impl(uint16_t event, event_format_t which, int extended, - const char *format, va_list ap) +send_control_event_impl(uint16_t event, event_format_t which, + const char *format, va_list ap) { /* This is just a little longer than the longest allowed log message */ #define SEND_CONTROL1_EVENT_BUFFERSIZE 10064 int r; char buf[SEND_CONTROL1_EVENT_BUFFERSIZE]; size_t len; - char *cp; r = tor_vsnprintf(buf, sizeof(buf), format, ap); if (r<0) { @@ -618,15 +567,7 @@ send_control_event_impl(uint16_t event, event_format_t which, int extended, buf[SEND_CONTROL1_EVENT_BUFFERSIZE-3] = '\r'; } - if (extended && (cp = strchr(buf, '@'))) { - which &= ~ALL_FORMATS; - *cp = ' '; - send_control_event_string(event, which|EXTENDED_FORMAT, buf); - memcpy(cp, "\r\n\0", 3); - send_control_event_string(event, which|NONEXTENDED_FORMAT, buf); - } else { - send_control_event_string(event, which|ALL_FORMATS, buf); - } + send_control_event_string(event, which|ALL_FORMATS, buf); } /** Send an event to all v1 controllers that are listening for code @@ -641,27 +582,7 @@ send_control_event(uint16_t event, event_format_t which, { va_list ap; va_start(ap, format); - send_control_event_impl(event, which, 0, format, ap); - va_end(ap); -} - -/** Send an event to all v1 controllers that are listening for code - * <b>event</b>. The event's body is created by the printf-style format in - * <b>format</b>, and other arguments as provided. - * - * If the format contains a single '@' character, it will be replaced with a - * space and all text after that character will be sent only to controllers - * that have enabled extended events. - * - * Currently the length of the message is limited to 1024 (including the - * ending \\n\\r\\0. */ -static void -send_control_event_extended(uint16_t event, event_format_t which, - const char *format, ...) -{ - va_list ap; - va_start(ap, format); - send_control_event_impl(event, which, 1, format, ap); + send_control_event_impl(event, which, format, ap); va_end(ap); } @@ -907,36 +828,37 @@ handle_control_loadconf(control_connection_t *conn, uint32_t len, retval = options_init_from_string(body, CMD_RUN_TOR, NULL, &errstring); - if (retval != SETOPT_OK) { + if (retval != SETOPT_OK) log_warn(LD_CONTROL, "Controller gave us config file that didn't validate: %s", errstring); - switch (retval) { - case SETOPT_ERR_PARSE: - msg = "552 Invalid config file"; - break; - case SETOPT_ERR_TRANSITION: - msg = "553 Transition not allowed"; - break; - case SETOPT_ERR_SETTING: - msg = "553 Unable to set option"; - break; - case SETOPT_ERR_MISC: - default: - msg = "550 Unable to load config"; - break; - case SETOPT_OK: - tor_fragile_assert(); - break; - } + + switch (retval) { + case SETOPT_ERR_PARSE: + msg = "552 Invalid config file"; + break; + case SETOPT_ERR_TRANSITION: + msg = "553 Transition not allowed"; + break; + case SETOPT_ERR_SETTING: + msg = "553 Unable to set option"; + break; + case SETOPT_ERR_MISC: + default: + msg = "550 Unable to load config"; + break; + case SETOPT_OK: + break; + } + if (msg) { if (errstring) connection_printf_to_buf(conn, "%s: %s\r\n", msg, errstring); else connection_printf_to_buf(conn, "%s\r\n", msg); - tor_free(errstring); - return 0; + } else { + send_control_done(conn); } - send_control_done(conn); + tor_free(errstring); return 0; } @@ -948,7 +870,6 @@ handle_control_setevents(control_connection_t *conn, uint32_t len, { uint16_t event_code; uint32_t event_mask = 0; - unsigned int extended = 0; smartlist_t *events = smartlist_create(); (void) len; @@ -958,7 +879,6 @@ handle_control_setevents(control_connection_t *conn, uint32_t len, SMARTLIST_FOREACH_BEGIN(events, const char *, ev) { if (!strcasecmp(ev, "EXTENDED")) { - extended = 1; continue; } else if (!strcasecmp(ev, "CIRC")) event_code = EVENT_CIRCUIT_STATUS; @@ -1016,8 +936,6 @@ handle_control_setevents(control_connection_t *conn, uint32_t len, smartlist_free(events); conn->event_mask = event_mask; - if (extended) - conn->use_extended_events = 1; control_update_global_event_mask(); send_control_done(conn); @@ -1328,7 +1246,7 @@ handle_control_mapaddress(control_connection_t *conn, uint32_t len, smartlist_add(reply, ans); log_warn(LD_CONTROL, "Unable to allocate address for '%s' in MapAddress msg", - safe_str(line)); + safe_str_client(line)); } else { tor_snprintf(ans, anslen, "250-%s=%s", address, to); smartlist_add(reply, ans); @@ -1345,7 +1263,8 @@ handle_control_mapaddress(control_connection_t *conn, uint32_t len, "not of expected form 'foo=bar'.", line); smartlist_add(reply, ans); log_info(LD_CONTROL, "Skipping MapAddress '%s': wrong " - "number of items.", safe_str(line)); + "number of items.", + safe_str_client(line)); } SMARTLIST_FOREACH(elts, char *, cp, tor_free(cp)); smartlist_clear(elts); @@ -1381,6 +1300,8 @@ getinfo_helper_misc(control_connection_t *conn, const char *question, *answer = tor_strdup(get_version()); } else if (!strcmp(question, "config-file")) { *answer = tor_strdup(get_torrc_fname()); + } else if (!strcmp(question, "config-text")) { + *answer = options_dump(get_options(), 1); } else if (!strcmp(question, "info/names")) { *answer = list_getinfo_options(); } else if (!strcmp(question, "events/names")) { @@ -1463,6 +1384,7 @@ static int getinfo_helper_dir(control_connection_t *control_conn, const char *question, char **answer) { + (void) control_conn; if (!strcmpstart(question, "desc/id/")) { routerinfo_t *ri = router_get_by_hexdigest(question+strlen("desc/id/")); if (ri) { @@ -1587,7 +1509,7 @@ getinfo_helper_dir(control_connection_t *control_conn, } } else if (!strcmp(question, "dir/status-vote/current/consensus")) { /* v3 */ if (directory_caches_dir_info(get_options())) { - const cached_dir_t *consensus = dirserv_get_consensus(); + const cached_dir_t *consensus = dirserv_get_consensus("ns"); if (consensus) *answer = tor_strdup(consensus->dir); } @@ -1598,10 +1520,8 @@ getinfo_helper_dir(control_connection_t *control_conn, } } else if (!strcmp(question, "network-status")) { /* v1 */ routerlist_t *routerlist = router_get_routerlist(); - int verbose = control_conn->use_long_names; if (!routerlist || !routerlist->routers || - list_server_status_v1(routerlist->routers, answer, - verbose ? 2 : 1) < 0) { + list_server_status_v1(routerlist->routers, answer, 1) < 0) { return -1; } } else if (!strcmpstart(question, "extra-info/digest/")) { @@ -1637,6 +1557,7 @@ static int getinfo_helper_events(control_connection_t *control_conn, const char *question, char **answer) { + (void) control_conn; if (!strcmp(question, "circuit-status")) { circuit_t *circ; smartlist_t *status = smartlist_create(); @@ -1647,10 +1568,9 @@ getinfo_helper_events(control_connection_t *control_conn, const char *purpose; if (! CIRCUIT_IS_ORIGIN(circ) || circ->marked_for_close) continue; - if (control_conn->use_long_names) - path = circuit_list_path_for_controller(TO_ORIGIN_CIRCUIT(circ)); - else - path = circuit_list_path(TO_ORIGIN_CIRCUIT(circ),0); + + path = circuit_list_path_for_controller(TO_ORIGIN_CIRCUIT(circ)); + if (circ->state == CIRCUIT_STATE_OPEN) state = "BUILT"; else if (strlen(path)) @@ -1728,8 +1648,7 @@ getinfo_helper_events(control_connection_t *control_conn, } else if (!strcmp(question, "orconn-status")) { smartlist_t *conns = get_connection_array(); smartlist_t *status = smartlist_create(); - SMARTLIST_FOREACH(conns, connection_t *, base_conn, - { + SMARTLIST_FOREACH_BEGIN(conns, connection_t *, base_conn) { const char *state; char *s; char name[128]; @@ -1744,29 +1663,19 @@ getinfo_helper_events(control_connection_t *control_conn, state = "LAUNCHED"; else state = "NEW"; - orconn_target_get_name(control_conn->use_long_names, name, sizeof(name), - conn); + orconn_target_get_name(name, sizeof(name), conn); slen = strlen(name)+strlen(state)+2; s = tor_malloc(slen+1); tor_snprintf(s, slen, "%s %s", name, state); smartlist_add(status, s); - }); + } SMARTLIST_FOREACH_END(base_conn); *answer = smartlist_join_strings(status, "\r\n", 0, NULL); SMARTLIST_FOREACH(status, char *, cp, tor_free(cp)); smartlist_free(status); - } else if (!strcmpstart(question, "addr-mappings/") || - !strcmpstart(question, "address-mappings/")) { + } else if (!strcmpstart(question, "address-mappings/")) { time_t min_e, max_e; smartlist_t *mappings; - int want_expiry = !strcmpstart(question, "address-mappings/"); - if (!strcmpstart(question, "addr-mappings/")) { - /* XXXX022 This has been deprecated since 0.2.0.3-alpha, and has - generated a warning since 0.2.1.10-alpha; remove late in 0.2.2.x. */ - log_warn(LD_CONTROL, "Controller used obsolete addr-mappings/ GETINFO " - "key; use address-mappings/ instead."); - } - question += strlen(want_expiry ? "address-mappings/" - : "addr-mappings/"); + question += strlen("address-mappings/"); if (!strcmp(question, "all")) { min_e = 0; max_e = TIME_MAX; } else if (!strcmp(question, "cache")) { @@ -1779,7 +1688,7 @@ getinfo_helper_events(control_connection_t *control_conn, return 0; } mappings = smartlist_create(); - addressmap_get_mappings(mappings, min_e, max_e, want_expiry); + addressmap_get_mappings(mappings, min_e, max_e, 1); *answer = smartlist_join_strings(mappings, "\r\n", 0, NULL); SMARTLIST_FOREACH(mappings, char *, cp, tor_free(cp)); smartlist_free(mappings); @@ -1846,21 +1755,10 @@ getinfo_helper_events(control_connection_t *control_conn, "information", question); } } else if (!strcmp(question, "status/clients-seen")) { - char geoip_start[ISO_TIME_LEN+1]; - size_t answer_len; - char *geoip_summary = extrainfo_get_client_geoip_summary(time(NULL)); - - if (!geoip_summary) + char *bridge_stats = geoip_get_bridge_stats_controller(time(NULL)); + if (!bridge_stats) return -1; - - answer_len = strlen("TimeStarted=\"\" CountrySummary=") + - ISO_TIME_LEN + strlen(geoip_summary) + 1; - *answer = tor_malloc(answer_len); - format_iso_time(geoip_start, geoip_get_history_start()); - tor_snprintf(*answer, answer_len, - "TimeStarted=\"%s\" CountrySummary=%s", - geoip_start, geoip_summary); - tor_free(geoip_summary); + *answer = bridge_stats; } else { return 0; } @@ -1894,6 +1792,8 @@ typedef struct getinfo_item_t { static const getinfo_item_t getinfo_items[] = { ITEM("version", misc, "The current version of Tor."), ITEM("config-file", misc, "Current location of the \"torrc\" file."), + ITEM("config-text", misc, + "Return the string that would be written by a saveconf command."), ITEM("accounting/bytes", accounting, "Number of bytes read/written so far in the accounting interval."), ITEM("accounting/bytes-left", accounting, @@ -1933,7 +1833,6 @@ static const getinfo_item_t getinfo_items[] = { PREFIX("ns/purpose/", networkstatus, "Brief summary of router status by purpose (v2 directory format)."), - PREFIX("unregistered-servers-", dirserv_unregistered, NULL), ITEM("network-status", dir, "Brief summary of router status (v1 directory format)"), ITEM("circuit-status", events, "List of current circuits originating here."), @@ -1945,14 +1844,6 @@ static const getinfo_item_t getinfo_items[] = { DOC("address-mappings/config", "Current address mappings from configuration."), DOC("address-mappings/control", "Current address mappings from controller."), - PREFIX("addr-mappings/", events, NULL), - DOC("addr-mappings/all", "Current address mappings without expiry times."), - DOC("addr-mappings/cache", - "Current cached DNS replies without expiry times."), - DOC("addr-mappings/config", - "Current address mappings from configuration without expiry times."), - DOC("addr-mappings/control", - "Current address mappings from controller without expiry times."), PREFIX("status/", events, NULL), DOC("status/circuit-established", "Whether we think client functionality is working."), @@ -2244,8 +2135,7 @@ handle_control_extendcircuit(control_connection_t *conn, uint32_t len, done: SMARTLIST_FOREACH(router_nicknames, char *, n, tor_free(n)); smartlist_free(router_nicknames); - if (routers) - smartlist_free(routers); + smartlist_free(routers); return 0; } @@ -2365,7 +2255,7 @@ handle_control_attachstream(control_connection_t *conn, uint32_t len, char* exit_digest; if (circ->build_state && circ->build_state->chosen_exit && - circ->build_state->chosen_exit->identity_digest) { + !tor_digest_is_zero(circ->build_state->chosen_exit->identity_digest)) { exit_digest = circ->build_state->chosen_exit->identity_digest; r = router_get_by_digest(exit_digest); } @@ -2710,7 +2600,6 @@ handle_control_usefeature(control_connection_t *conn, const char *body) { smartlist_t *args; - int verbose_names = 0, extended_events = 0; int bad = 0; (void) len; /* body is nul-terminated; it's safe to ignore the length */ args = smartlist_create(); @@ -2718,9 +2607,9 @@ handle_control_usefeature(control_connection_t *conn, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); SMARTLIST_FOREACH(args, const char *, arg, { if (!strcasecmp(arg, "VERBOSE_NAMES")) - verbose_names = 1; + ; else if (!strcasecmp(arg, "EXTENDED_EVENTS")) - extended_events = 1; + ; else { connection_printf_to_buf(conn, "552 Unrecognized feature \"%s\"\r\n", arg); @@ -2730,12 +2619,6 @@ handle_control_usefeature(control_connection_t *conn, }); if (!bad) { - if (verbose_names) { - conn->use_long_names = 1; - control_update_global_event_mask(); - } - if (extended_events) - conn->use_extended_events = 1; send_control_done(conn); } @@ -3039,20 +2922,11 @@ control_event_circuit_status(origin_circuit_t *circ, circuit_status_event_t tp, tor_free(reason); } - if (EVENT_IS_INTERESTING1S(EVENT_CIRCUIT_STATUS)) { - char *path = circuit_list_path(circ,0); - const char *sp = strlen(path) ? " " : ""; - send_control_event_extended(EVENT_CIRCUIT_STATUS, SHORT_NAMES, - "650 CIRC %lu %s%s%s@%s\r\n", - (unsigned long)circ->global_identifier, - status, sp, path, extended_buf); - tor_free(path); - } - if (EVENT_IS_INTERESTING1L(EVENT_CIRCUIT_STATUS)) { + { char *vpath = circuit_list_path_for_controller(circ); const char *sp = strlen(vpath) ? " " : ""; - send_control_event_extended(EVENT_CIRCUIT_STATUS, LONG_NAMES, - "650 CIRC %lu %s%s%s@%s\r\n", + send_control_event(EVENT_CIRCUIT_STATUS, ALL_FORMATS, + "650 CIRC %lu %s%s%s %s\r\n", (unsigned long)circ->global_identifier, status, sp, vpath, extended_buf); tor_free(vpath); @@ -3188,8 +3062,8 @@ control_event_stream_status(edge_connection_t *conn, stream_status_event_t tp, circ = circuit_get_by_edge_conn(conn); if (circ && CIRCUIT_IS_ORIGIN(circ)) origin_circ = TO_ORIGIN_CIRCUIT(circ); - send_control_event_extended(EVENT_STREAM_STATUS, ALL_NAMES, - "650 STREAM "U64_FORMAT" %s %lu %s@%s%s%s\r\n", + send_control_event(EVENT_STREAM_STATUS, ALL_FORMATS, + "650 STREAM "U64_FORMAT" %s %lu %s %s%s%s\r\n", U64_PRINTF_ARG(conn->_base.global_identifier), status, origin_circ? (unsigned long)origin_circ->global_identifier : 0ul, @@ -3202,30 +3076,21 @@ control_event_stream_status(edge_connection_t *conn, stream_status_event_t tp, /** Figure out the best name for the target router of an OR connection * <b>conn</b>, and write it into the <b>len</b>-character buffer - * <b>name</b>. Use verbose names if <b>long_names</b> is set. */ + * <b>name</b>. */ static void -orconn_target_get_name(int long_names, - char *name, size_t len, or_connection_t *conn) -{ - if (! long_names) { - if (conn->nickname) - strlcpy(name, conn->nickname, len); - else - tor_snprintf(name, len, "%s:%d", - conn->_base.address, conn->_base.port); +orconn_target_get_name(char *name, size_t len, or_connection_t *conn) +{ + routerinfo_t *ri = router_get_by_digest(conn->identity_digest); + if (ri) { + tor_assert(len > MAX_VERBOSE_NICKNAME_LEN); + router_get_verbose_nickname(name, ri); + } else if (! tor_digest_is_zero(conn->identity_digest)) { + name[0] = '$'; + base16_encode(name+1, len-1, conn->identity_digest, + DIGEST_LEN); } else { - routerinfo_t *ri = router_get_by_digest(conn->identity_digest); - if (ri) { - tor_assert(len > MAX_VERBOSE_NICKNAME_LEN); - router_get_verbose_nickname(name, ri); - } else if (! tor_digest_is_zero(conn->identity_digest)) { - name[0] = '$'; - base16_encode(name+1, len-1, conn->identity_digest, - DIGEST_LEN); - } else { - tor_snprintf(name, len, "%s:%d", - conn->_base.address, conn->_base.port); - } + tor_snprintf(name, len, "%s:%d", + conn->_base.address, conn->_base.port); } } @@ -3264,24 +3129,13 @@ control_event_or_conn_status(or_connection_t *conn, or_conn_status_event_t tp, reason ? " " : "", ncircs); } - if (EVENT_IS_INTERESTING1S(EVENT_OR_CONN_STATUS)) { - orconn_target_get_name(0, name, sizeof(name), conn); - send_control_event_extended(EVENT_OR_CONN_STATUS, SHORT_NAMES, - "650 ORCONN %s %s@%s%s%s\r\n", - name, status, - reason ? "REASON=" : "", - orconn_end_reason_to_control_string(reason), - ncircs_buf); - } - if (EVENT_IS_INTERESTING1L(EVENT_OR_CONN_STATUS)) { - orconn_target_get_name(1, name, sizeof(name), conn); - send_control_event_extended(EVENT_OR_CONN_STATUS, LONG_NAMES, - "650 ORCONN %s %s@%s%s%s\r\n", - name, status, - reason ? "REASON=" : "", - orconn_end_reason_to_control_string(reason), - ncircs_buf); - } + orconn_target_get_name(name, sizeof(name), conn); + send_control_event(EVENT_OR_CONN_STATUS, ALL_FORMATS, + "650 ORCONN %s %s %s%s%s\r\n", + name, status, + reason ? "REASON=" : "", + orconn_end_reason_to_control_string(reason), + ncircs_buf); return 0; } @@ -3296,7 +3150,7 @@ control_event_stream_bandwidth(edge_connection_t *edge_conn) if (!edge_conn->n_read && !edge_conn->n_written) return 0; - send_control_event(EVENT_STREAM_BANDWIDTH_USED, ALL_NAMES, + send_control_event(EVENT_STREAM_BANDWIDTH_USED, ALL_FORMATS, "650 STREAM_BW "U64_FORMAT" %lu %lu\r\n", U64_PRINTF_ARG(edge_conn->_base.global_identifier), (unsigned long)edge_conn->n_read, @@ -3325,7 +3179,7 @@ control_event_stream_bandwidth_used(void) if (!edge_conn->n_read && !edge_conn->n_written) continue; - send_control_event(EVENT_STREAM_BANDWIDTH_USED, ALL_NAMES, + send_control_event(EVENT_STREAM_BANDWIDTH_USED, ALL_FORMATS, "650 STREAM_BW "U64_FORMAT" %lu %lu\r\n", U64_PRINTF_ARG(edge_conn->_base.global_identifier), (unsigned long)edge_conn->n_read, @@ -3345,7 +3199,7 @@ int control_event_bandwidth_used(uint32_t n_read, uint32_t n_written) { if (EVENT_IS_INTERESTING(EVENT_BANDWIDTH_USED)) { - send_control_event(EVENT_BANDWIDTH_USED, ALL_NAMES, + send_control_event(EVENT_BANDWIDTH_USED, ALL_FORMATS, "650 BW %lu %lu\r\n", (unsigned long)n_read, (unsigned long)n_written); @@ -3415,7 +3269,7 @@ control_event_logmsg(int severity, uint32_t domain, const char *msg) default: s = "UnknownLogSeverity"; break; } ++disable_log_messages; - send_control_event(event, ALL_NAMES, "650 %s %s\r\n", s, b?b:msg); + send_control_event(event, ALL_FORMATS, "650 %s %s\r\n", s, b?b:msg); --disable_log_messages; tor_free(b); } @@ -3428,31 +3282,12 @@ control_event_logmsg(int severity, uint32_t domain, const char *msg) int control_event_descriptors_changed(smartlist_t *routers) { - size_t len; char *msg; - smartlist_t *identities = NULL; - char buf[HEX_DIGEST_LEN+1]; if (!EVENT_IS_INTERESTING(EVENT_NEW_DESC)) return 0; - if (EVENT_IS_INTERESTING1S(EVENT_NEW_DESC)) { - identities = smartlist_create(); - SMARTLIST_FOREACH(routers, routerinfo_t *, r, - { - base16_encode(buf,sizeof(buf),r->cache_info.identity_digest,DIGEST_LEN); - smartlist_add(identities, tor_strdup(buf)); - }); - } - if (EVENT_IS_INTERESTING1S(EVENT_NEW_DESC)) { - char *ids = smartlist_join_strings(identities, " ", 0, &len); - size_t ids_len = strlen(ids)+32; - msg = tor_malloc(ids_len); - tor_snprintf(msg, ids_len, "650 NEWDESC %s\r\n", ids); - send_control_event_string(EVENT_NEW_DESC, SHORT_NAMES|ALL_FORMATS, msg); - tor_free(ids); - tor_free(msg); - } - if (EVENT_IS_INTERESTING1L(EVENT_NEW_DESC)) { + + { smartlist_t *names = smartlist_create(); char *ids; size_t names_len; @@ -3465,16 +3300,12 @@ control_event_descriptors_changed(smartlist_t *routers) names_len = strlen(ids)+32; msg = tor_malloc(names_len); tor_snprintf(msg, names_len, "650 NEWDESC %s\r\n", ids); - send_control_event_string(EVENT_NEW_DESC, LONG_NAMES|ALL_FORMATS, msg); + send_control_event_string(EVENT_NEW_DESC, ALL_FORMATS, msg); tor_free(ids); tor_free(msg); SMARTLIST_FOREACH(names, char *, cp, tor_free(cp)); smartlist_free(names); } - if (identities) { - SMARTLIST_FOREACH(identities, char *, cp, tor_free(cp)); - smartlist_free(identities); - } return 0; } @@ -3491,17 +3322,17 @@ control_event_address_mapped(const char *from, const char *to, time_t expires, return 0; if (expires < 3 || expires == TIME_MAX) - send_control_event_extended(EVENT_ADDRMAP, ALL_NAMES, - "650 ADDRMAP %s %s NEVER@%s\r\n", from, to, + send_control_event(EVENT_ADDRMAP, ALL_FORMATS, + "650 ADDRMAP %s %s NEVER %s\r\n", from, to, error?error:""); else { char buf[ISO_TIME_LEN+1]; char buf2[ISO_TIME_LEN+1]; format_local_iso_time(buf,expires); format_iso_time(buf2,expires); - send_control_event_extended(EVENT_ADDRMAP, ALL_NAMES, + send_control_event(EVENT_ADDRMAP, ALL_FORMATS, "650 ADDRMAP %s %s \"%s\"" - "@%s%sEXPIRES=\"%s\"\r\n", + " %s%sEXPIRES=\"%s\"\r\n", from, to, buf, error?error:"", error?" ":"", buf2); @@ -3541,9 +3372,9 @@ control_event_or_authdir_new_descriptor(const char *action, buf = tor_malloc(totallen); strlcpy(buf, firstline, totallen); strlcpy(buf+strlen(firstline), esc, totallen); - send_control_event_string(EVENT_AUTHDIR_NEWDESCS, ALL_NAMES|ALL_FORMATS, + send_control_event_string(EVENT_AUTHDIR_NEWDESCS, ALL_FORMATS, buf); - send_control_event_string(EVENT_AUTHDIR_NEWDESCS, ALL_NAMES|ALL_FORMATS, + send_control_event_string(EVENT_AUTHDIR_NEWDESCS, ALL_FORMATS, "650 OK\r\n"); tor_free(esc); tor_free(buf); @@ -3581,8 +3412,8 @@ control_event_networkstatus_changed_helper(smartlist_t *statuses, SMARTLIST_FOREACH(strs, char *, cp, tor_free(cp)); smartlist_free(strs); tor_free(s); - send_control_event_string(event, ALL_NAMES|ALL_FORMATS, esc); - send_control_event_string(event, ALL_NAMES|ALL_FORMATS, + send_control_event_string(event, ALL_FORMATS, esc); + send_control_event_string(event, ALL_FORMATS, "650 OK\r\n"); tor_free(esc); @@ -3631,7 +3462,7 @@ control_event_networkstatus_changed_single(routerstatus_t *rs) int control_event_my_descriptor_changed(void) { - send_control_event(EVENT_DESCCHANGED, ALL_NAMES, "650 DESCCHANGED\r\n"); + send_control_event(EVENT_DESCCHANGED, ALL_FORMATS, "650 DESCCHANGED\r\n"); return 0; } @@ -3679,7 +3510,7 @@ control_event_status(int type, int severity, const char *format, va_list args) return -1; } - send_control_event_impl(type, ALL_NAMES|ALL_FORMATS, 0, format_buf, args); + send_control_event_impl(type, ALL_FORMATS, format_buf, args); return 0; } @@ -3743,7 +3574,7 @@ control_event_guard(const char *nickname, const char *digest, if (!EVENT_IS_INTERESTING(EVENT_GUARD)) return 0; - if (EVENT_IS_INTERESTING1L(EVENT_GUARD)) { + { char buf[MAX_VERBOSE_NICKNAME_LEN+1]; routerinfo_t *ri = router_get_by_digest(digest); if (ri) { @@ -3751,13 +3582,9 @@ control_event_guard(const char *nickname, const char *digest, } else { tor_snprintf(buf, sizeof(buf), "$%s~%s", hbuf, nickname); } - send_control_event(EVENT_GUARD, LONG_NAMES, + send_control_event(EVENT_GUARD, ALL_FORMATS, "650 GUARD ENTRY %s %s\r\n", buf, status); } - if (EVENT_IS_INTERESTING1S(EVENT_GUARD)) { - send_control_event(EVENT_GUARD, SHORT_NAMES, - "650 GUARD ENTRY $%s %s\r\n", hbuf, status); - } return 0; } @@ -4008,10 +3835,9 @@ control_event_bootstrap_problem(const char *warn, int reason) * from recently. Send a copy to the controller in case it wants to * display it for the user. */ void -control_event_clients_seen(const char *timestarted, const char *countries) +control_event_clients_seen(const char *controller_str) { send_control_event(EVENT_CLIENTS_SEEN, 0, - "650 CLIENTS_SEEN TimeStarted=\"%s\" CountrySummary=%s\r\n", - timestarted, countries); + "650 CLIENTS_SEEN %s\r\n", controller_str); } diff --git a/src/or/directory.c b/src/or/directory.c index 42341f1040..30c08b84b2 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -92,11 +92,13 @@ static void directory_initiate_command_rend(const char *address, #define ROUTERDESC_CACHE_LIFETIME (30*60) #define ROUTERDESC_BY_DIGEST_CACHE_LIFETIME (48*60*60) #define ROBOTS_CACHE_LIFETIME (24*60*60) +#define MICRODESC_CACHE_LIFETIME (48*60*60) /********* END VARIABLES ************/ -/** Return true iff the directory purpose 'purpose' must use an - * anonymous connection to a directory. */ +/** Return true iff the directory purpose <b>dir_purpose</b> (and if it's + * fetching descriptors, it's fetching them for <b>router_purpose</b>) + * must use an anonymous connection to a directory. */ static int purpose_needs_anonymity(uint8_t dir_purpose, uint8_t router_purpose) { @@ -228,7 +230,7 @@ directories_have_accepted_server_descriptor(void) /** Start a connection to every suitable directory authority, using * connection purpose 'purpose' and uploading the payload 'payload' - * (length 'payload_len'). The purpose should be one of + * (length 'payload_len'). dir_purpose should be one of * 'DIR_PURPOSE_UPLOAD_DIR' or 'DIR_PURPOSE_UPLOAD_RENDDESC'. * * <b>type</b> specifies what sort of dir authorities (V1, V2, @@ -556,7 +558,7 @@ connection_dir_request_failed(dir_connection_t *conn) if (directory_conn_is_self_reachability_test(conn)) { return; /* this was a test fetch. don't retry. */ } - if (entry_list_can_grow(get_options())) + if (entry_list_is_constrained(get_options())) router_set_status(conn->identity_digest, 0); /* don't try him again */ if (conn->_base.purpose == DIR_PURPOSE_FETCH_V2_NETWORKSTATUS) { log_info(LD_DIR, "Giving up on directory server at '%s'; retrying", @@ -610,7 +612,7 @@ connection_dir_download_networkstatus_failed(dir_connection_t *conn, * failed, and possibly retry them later.*/ smartlist_t *failed = smartlist_create(); dir_split_resource_into_fingerprints(conn->requested_resource+3, - failed, NULL, 0, 0); + failed, NULL, 0); if (smartlist_len(failed)) { dir_networkstatus_download_failed(failed, status_code); SMARTLIST_FOREACH(failed, char *, cp, tor_free(cp)); @@ -647,7 +649,7 @@ connection_dir_download_cert_failed(dir_connection_t *conn, int status) return; failed = smartlist_create(); dir_split_resource_into_fingerprints(conn->requested_resource+3, - failed, NULL, 1, 0); + failed, NULL, DSR_HEX); SMARTLIST_FOREACH(failed, char *, cp, { authority_cert_dl_failed(cp, status); @@ -663,7 +665,7 @@ connection_dir_download_cert_failed(dir_connection_t *conn, int status) * 1) If or_port is 0, or it's a direct conn and or_port is firewalled * or we're a dir mirror, no. * 2) If we prefer to avoid begindir conns, and we're not fetching or - * publishing a bridge relay descriptor, no. + * publishing a bridge relay descriptor, no. * 3) Else yes. */ static int @@ -742,6 +744,15 @@ directory_initiate_command_rend(const char *address, const tor_addr_t *_addr, log_debug(LD_DIR, "Initiating %s", dir_conn_purpose_to_string(dir_purpose)); + /* ensure that we don't make direct connections when a SOCKS server is + * configured. */ + if (!anonymized_connection && !use_begindir && !options->HttpProxy && + (options->Socks4Proxy || options->Socks5Proxy)) { + log_warn(LD_DIR, "Cannot connect to a directory server through a " + "SOCKS proxy!"); + return; + } + conn = dir_connection_new(AF_INET); /* set up conn so it's got all the data we need to remember */ @@ -767,7 +778,7 @@ directory_initiate_command_rend(const char *address, const tor_addr_t *_addr, /* then we want to connect to dirport directly */ if (options->HttpProxy) { - tor_addr_from_ipv4h(&addr, options->HttpProxyAddr); + tor_addr_copy(&addr, &options->HttpProxyAddr); dir_port = options->HttpProxyPort; } @@ -789,7 +800,7 @@ directory_initiate_command_rend(const char *address, const tor_addr_t *_addr, payload, payload_len, supports_conditional_consensus, if_modified_since); - connection_watch_events(TO_CONN(conn), EV_READ | EV_WRITE); + connection_watch_events(TO_CONN(conn), READ_EVENT | WRITE_EVENT); /* writable indicates finish, readable indicates broken link, error indicates broken link in windowsland. */ } @@ -828,7 +839,7 @@ directory_initiate_command_rend(const char *address, const tor_addr_t *_addr, payload, payload_len, supports_conditional_consensus, if_modified_since); - connection_watch_events(TO_CONN(conn), EV_READ | EV_WRITE); + connection_watch_events(TO_CONN(conn), READ_EVENT|WRITE_EVENT); connection_start_reading(TO_CONN(linked_conn)); } } @@ -876,7 +887,7 @@ directory_get_consensus_url(int supports_conditional_consensus) if (supports_conditional_consensus) { char *authority_id_list; - smartlist_t *authority_digets = smartlist_create(); + smartlist_t *authority_digests = smartlist_create(); SMARTLIST_FOREACH(router_get_trusted_dir_servers(), trusted_dir_server_t *, ds, @@ -888,10 +899,10 @@ directory_get_consensus_url(int supports_conditional_consensus) hex = tor_malloc(2*CONDITIONAL_CONSENSUS_FPR_LEN+1); base16_encode(hex, 2*CONDITIONAL_CONSENSUS_FPR_LEN+1, ds->v3_identity_digest, CONDITIONAL_CONSENSUS_FPR_LEN); - smartlist_add(authority_digets, hex); + smartlist_add(authority_digests, hex); }); - smartlist_sort(authority_digets, _compare_strs); - authority_id_list = smartlist_join_strings(authority_digets, + smartlist_sort(authority_digests, _compare_strs); + authority_id_list = smartlist_join_strings(authority_digests, "+", 0, NULL); len = strlen(authority_id_list)+64; @@ -899,8 +910,8 @@ directory_get_consensus_url(int supports_conditional_consensus) tor_snprintf(url, len, "/tor/status-vote/current/consensus/%s.z", authority_id_list); - SMARTLIST_FOREACH(authority_digets, char *, cp, tor_free(cp)); - smartlist_free(authority_digets); + SMARTLIST_FOREACH(authority_digests, char *, cp, tor_free(cp)); + smartlist_free(authority_digests); tor_free(authority_id_list); } else { url = tor_strdup("/tor/status-vote/current/consensus.z"); @@ -909,7 +920,7 @@ directory_get_consensus_url(int supports_conditional_consensus) } /** Queue an appropriate HTTP command on conn-\>outbuf. The other args - * are as in directory_initiate_command. + * are as in directory_initiate_command(). */ static void directory_send_command(dir_connection_t *conn, @@ -1045,31 +1056,10 @@ directory_send_command(dir_connection_t *conn, httpcommand = "POST"; url = tor_strdup("/tor/post/consensus-signature"); break; - case DIR_PURPOSE_FETCH_RENDDESC: - tor_assert(resource); - tor_assert(!payload); - - /* this must be true or we wouldn't be doing the lookup */ - tor_assert(strlen(resource) <= REND_SERVICE_ID_LEN_BASE32); - /* This breaks the function abstraction. */ - conn->rend_data = tor_malloc_zero(sizeof(rend_data_t)); - strlcpy(conn->rend_data->onion_address, resource, - sizeof(conn->rend_data->onion_address)); - conn->rend_data->rend_desc_version = 0; - - httpcommand = "GET"; - /* Request the most recent versioned descriptor. */ - // (XXXX We were going to switch this to fetch rendezvous1 descriptors, - // but that never got testing, and it wasn't a good design.) - len = strlen(resource)+32; - url = tor_malloc(len); - tor_snprintf(url, len, "/tor/rendezvous/%s", resource); - break; case DIR_PURPOSE_FETCH_RENDDESC_V2: tor_assert(resource); tor_assert(strlen(resource) <= REND_DESC_ID_V2_LEN_BASE32); tor_assert(!payload); - conn->rend_data->rend_desc_version = 2; httpcommand = "GET"; len = strlen(resource) + 32; url = tor_malloc(len); @@ -1158,7 +1148,7 @@ parse_http_url(const char *headers, char **url) if (s-tmp >= 3 && !strcmpstart(tmp,"://")) { tmp = strchr(tmp+3, '/'); if (tmp && tmp < s) { - log_debug(LD_DIR,"Skipping over 'http[s]://hostname' string"); + log_debug(LD_DIR,"Skipping over 'http[s]://hostname/' string"); start = tmp; } } @@ -1474,21 +1464,22 @@ connection_dir_client_reached_eof(dir_connection_t *conn) } (void) skewed; /* skewed isn't used yet. */ - if (status_code == 503 && body_len < 16) { - routerstatus_t *rs; - trusted_dir_server_t *ds; - log_info(LD_DIR,"Received http status code %d (%s) from server " - "'%s:%d'. I'll try again soon.", - status_code, escaped(reason), conn->_base.address, - conn->_base.port); - if ((rs = router_get_consensus_status_by_id(conn->identity_digest))) - rs->last_dir_503_at = now; - if ((ds = router_get_trusteddirserver_by_digest(conn->identity_digest))) - ds->fake_status.last_dir_503_at = now; + if (status_code == 503) { + if (body_len < 16) { + routerstatus_t *rs; + trusted_dir_server_t *ds; + log_info(LD_DIR,"Received http status code %d (%s) from server " + "'%s:%d'. I'll try again soon.", + status_code, escaped(reason), conn->_base.address, + conn->_base.port); + if ((rs = router_get_consensus_status_by_id(conn->identity_digest))) + rs->last_dir_503_at = now; + if ((ds = router_get_trusteddirserver_by_digest(conn->identity_digest))) + ds->fake_status.last_dir_503_at = now; - tor_free(body); tor_free(headers); tor_free(reason); - return -1; - } else if (status_code == 503) { + tor_free(body); tor_free(headers); tor_free(reason); + return -1; + } /* XXXX022 Remove this once every server with bug 539 is obsolete. */ log_info(LD_DIR, "Server at '%s:%d' sent us a 503 response, but included " "a body anyway. We'll pretend it gave us a 200.", @@ -1560,7 +1551,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) v2_networkstatus_source_t source; char *cp; log_info(LD_DIR,"Received networkstatus objects (size %d) from server " - "'%s:%d'",(int) body_len, conn->_base.address, conn->_base.port); + "'%s:%d'", (int)body_len, conn->_base.address, conn->_base.port); if (status_code != 200) { log_warn(LD_DIR, "Received http status code %d (%s) from server " @@ -1576,7 +1567,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) source = NS_FROM_DIR_BY_FP; which = smartlist_create(); dir_split_resource_into_fingerprints(conn->requested_resource+3, - which, NULL, 0, 0); + which, NULL, 0); } else if (conn->requested_resource && !strcmpstart(conn->requested_resource, "all")) { source = NS_FROM_DIR_ALL; @@ -1634,8 +1625,8 @@ connection_dir_client_reached_eof(dir_connection_t *conn) return -1; } log_info(LD_DIR,"Received consensus directory (size %d) from server " - "'%s:%d'",(int) body_len, conn->_base.address, conn->_base.port); - if ((r=networkstatus_set_current_consensus(body, 0))<0) { + "'%s:%d'", (int)body_len, conn->_base.address, conn->_base.port); + if ((r=networkstatus_set_current_consensus(body, "ns", 0))<0) { log_fn(r<-1?LOG_WARN:LOG_INFO, LD_DIR, "Unable to load consensus directory downloaded from " "server '%s:%d'. I'll try again soon.", @@ -1662,9 +1653,11 @@ connection_dir_client_reached_eof(dir_connection_t *conn) return -1; } log_info(LD_DIR,"Received authority certificates (size %d) from server " - "'%s:%d'",(int) body_len, conn->_base.address, conn->_base.port); + "'%s:%d'", (int)body_len, conn->_base.address, conn->_base.port); if (trusted_dirs_load_certs_from_string(body, 0, 1)<0) { log_warn(LD_DIR, "Unable to parse fetched certificates"); + /* if we fetched more than one and only some failed, the successful + * ones got flushed to disk so it's safe to call this on them */ connection_dir_download_cert_failed(conn, status_code); } else { directory_info_has_arrived(now, 0); @@ -1675,7 +1668,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) const char *msg; int st; log_info(LD_DIR,"Got votes (size %d) from server %s:%d", - (int) body_len, conn->_base.address, conn->_base.port); + (int)body_len, conn->_base.address, conn->_base.port); if (status_code != 200) { log_warn(LD_DIR, "Received http status code %d (%s) from server " @@ -1695,11 +1688,11 @@ connection_dir_client_reached_eof(dir_connection_t *conn) if (conn->_base.purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES) { const char *msg = NULL; log_info(LD_DIR,"Got detached signatures (size %d) from server %s:%d", - (int) body_len, conn->_base.address, conn->_base.port); + (int)body_len, conn->_base.address, conn->_base.port); if (status_code != 200) { log_warn(LD_DIR, - "Received http status code %d (%s) from server " - "'%s:%d' while fetching \"/tor/status-vote/consensus-signatures.z\".", + "Received http status code %d (%s) from server '%s:%d' while fetching " + "\"/tor/status-vote/next/consensus-signatures.z\".", status_code, escaped(reason), conn->_base.address, conn->_base.port); tor_free(body); tor_free(headers); tor_free(reason); @@ -1727,7 +1720,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) which = smartlist_create(); dir_split_resource_into_fingerprints(conn->requested_resource + (descriptor_digests ? 2 : 3), - which, NULL, 0, 0); + which, NULL, 0); n_asked_for = smartlist_len(which); } if (status_code != 200) { @@ -1915,7 +1908,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) /* Success, or at least there's a v2 descriptor already * present. Notify pending connections about this. */ conn->_base.purpose = DIR_PURPOSE_HAS_FETCHED_RENDDESC; - rend_client_desc_trynow(conn->rend_data->onion_address, -1); + rend_client_desc_trynow(conn->rend_data->onion_address); } break; case 404: @@ -1962,7 +1955,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) log_info(LD_REND, "Successfully fetched v2 rendezvous " "descriptor."); conn->_base.purpose = DIR_PURPOSE_HAS_FETCHED_RENDDESC; - rend_client_desc_trynow(conn->rend_data->onion_address, -1); + rend_client_desc_trynow(conn->rend_data->onion_address); break; } break; @@ -2005,12 +1998,6 @@ connection_dir_client_reached_eof(dir_connection_t *conn) "'%s:%d'. Malformed rendezvous descriptor?", escaped(reason), conn->_base.address, conn->_base.port); break; - case 503: - log_info(LD_REND,"http status 503 (%s) response from dirserver " - "'%s:%d'. Node is (currently) not acting as v2 hidden " - "service directory.", - escaped(reason), conn->_base.address, conn->_base.port); - break; default: log_warn(LD_REND,"http status %d (%s) response unexpected (server " "'%s:%d').", @@ -2317,7 +2304,7 @@ directory_dump_request_log(void) } #endif -/** Decide whether a client would accept the consensus we have +/** Decide whether a client would accept the consensus we have. * * Clients can say they only want a consensus if it's signed by more * than half the authorities in a list. They pass this list in @@ -2338,9 +2325,9 @@ client_likes_consensus(networkstatus_t *v, const char *want_url) int need_at_least; int have = 0; - dir_split_resource_into_fingerprints(want_url, want_authorities, NULL, 0, 0); + dir_split_resource_into_fingerprints(want_url, want_authorities, NULL, 0); need_at_least = smartlist_len(want_authorities)/2+1; - SMARTLIST_FOREACH(want_authorities, const char *, d, { + SMARTLIST_FOREACH_BEGIN(want_authorities, const char *, d) { char want_digest[DIGEST_LEN]; size_t want_len = strlen(d)/2; if (want_len > DIGEST_LEN) @@ -2351,18 +2338,18 @@ client_likes_consensus(networkstatus_t *v, const char *want_url) continue; }; - SMARTLIST_FOREACH(v->voters, networkstatus_voter_info_t *, vi, { - if (vi->signature && + SMARTLIST_FOREACH_BEGIN(v->voters, networkstatus_voter_info_t *, vi) { + if (smartlist_len(vi->sigs) && !memcmp(vi->identity_digest, want_digest, want_len)) { have++; break; }; - }); + } SMARTLIST_FOREACH_END(vi); /* early exit, if we already have enough */ if (have >= need_at_least) break; - }); + } SMARTLIST_FOREACH_END(d); SMARTLIST_FOREACH(want_authorities, char *, d, tor_free(d)); smartlist_free(want_authorities); @@ -2509,9 +2496,12 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, /* v2 or v3 network status fetch. */ smartlist_t *dir_fps = smartlist_create(); int is_v3 = !strcmpstart(url, "/tor/status-vote"); + geoip_client_action_t act = + is_v3 ? GEOIP_CLIENT_NETWORKSTATUS : GEOIP_CLIENT_NETWORKSTATUS_V2; const char *request_type = NULL; const char *key = url + strlen("/tor/status/"); long lifetime = NETWORKSTATUS_CACHE_LIFETIME; + if (!is_v3) { dirserv_get_networkstatus_v2_fingerprints(dir_fps, key); if (!strcmpstart(key, "fp/")) @@ -2526,18 +2516,44 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, } else { networkstatus_t *v = networkstatus_get_latest_consensus(); time_t now = time(NULL); + const char *want_fps = NULL; + char *flavor = NULL; #define CONSENSUS_URL_PREFIX "/tor/status-vote/current/consensus/" - if (v && - !strcmpstart(url, CONSENSUS_URL_PREFIX) && - !client_likes_consensus(v, url + strlen(CONSENSUS_URL_PREFIX))) { + #define CONSENSUS_FLAVORED_PREFIX "/tor/status-vote/current/consensus-" + /* figure out the flavor if any, and who we wanted to sign the thing */ + if (!strcmpstart(url, CONSENSUS_FLAVORED_PREFIX)) { + const char *f, *cp; + f = url + strlen(CONSENSUS_FLAVORED_PREFIX); + cp = strchr(f, '/'); + if (cp) { + want_fps = cp+1; + flavor = tor_strndup(f, cp-f); + } else { + flavor = tor_strdup(f); + } + } else { + if (!strcmpstart(url, CONSENSUS_URL_PREFIX)) + want_fps = url+strlen(CONSENSUS_URL_PREFIX); + } + + /* XXXX MICRODESC NM NM should check document of correct flavor */ + if (v && want_fps && + !client_likes_consensus(v, want_fps)) { write_http_status_line(conn, 404, "Consensus not signed by sufficient " "number of requested authorities"); smartlist_free(dir_fps); + geoip_note_ns_response(act, GEOIP_REJECT_NOT_ENOUGH_SIGS); + tor_free(flavor); goto done; } - smartlist_add(dir_fps, tor_memdup("\0\0\0\0\0\0\0\0\0\0" - "\0\0\0\0\0\0\0\0\0\0", 20)); + { + char *fp = tor_malloc_zero(DIGEST_LEN); + if (flavor) + strlcpy(fp, flavor, DIGEST_LEN); + tor_free(flavor); + smartlist_add(dir_fps, fp); + } request_type = compressed?"v3.z":"v3"; lifetime = (v && v->fresh_until > now) ? v->fresh_until - now : 0; } @@ -2545,6 +2561,7 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, if (!smartlist_len(dir_fps)) { /* we failed to create/cache cp */ write_http_status_line(conn, 503, "Network status object unavailable"); smartlist_free(dir_fps); + geoip_note_ns_response(act, GEOIP_REJECT_UNAVAILABLE); goto done; } @@ -2552,11 +2569,13 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, write_http_status_line(conn, 404, "Not found"); SMARTLIST_FOREACH(dir_fps, char *, cp, tor_free(cp)); smartlist_free(dir_fps); + geoip_note_ns_response(act, GEOIP_REJECT_NOT_FOUND); goto done; } else if (!smartlist_len(dir_fps)) { write_http_status_line(conn, 304, "Not modified"); SMARTLIST_FOREACH(dir_fps, char *, cp, tor_free(cp)); smartlist_free(dir_fps); + geoip_note_ns_response(act, GEOIP_REJECT_NOT_MODIFIED); goto done; } @@ -2568,18 +2587,25 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, write_http_status_line(conn, 503, "Directory busy, try again later"); SMARTLIST_FOREACH(dir_fps, char *, fp, tor_free(fp)); smartlist_free(dir_fps); + geoip_note_ns_response(act, GEOIP_REJECT_BUSY); goto done; } -#ifdef ENABLE_GEOIP_STATS { - geoip_client_action_t act = - is_v3 ? GEOIP_CLIENT_NETWORKSTATUS : GEOIP_CLIENT_NETWORKSTATUS_V2; struct in_addr in; - if (tor_inet_aton((TO_CONN(conn))->address, &in)) + if (tor_inet_aton((TO_CONN(conn))->address, &in)) { geoip_note_client_seen(act, ntohl(in.s_addr), time(NULL)); + geoip_note_ns_response(act, GEOIP_SUCCESS); + /* Note that a request for a network status has started, so that we + * can measure the download time later on. */ + if (TO_CONN(conn)->dirreq_id) + geoip_start_dirreq(TO_CONN(conn)->dirreq_id, dlen, act, + DIRREQ_TUNNELED); + else + geoip_start_dirreq(TO_CONN(conn)->global_identifier, dlen, act, + DIRREQ_DIRECT); + } } -#endif // note_request(request_type,dlen); (void) request_type; @@ -2615,7 +2641,7 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, const char *item; tor_assert(!current); /* we handle current consensus specially above, * since it wants to be spooled. */ - if ((item = dirvote_get_pending_consensus())) + if ((item = dirvote_get_pending_consensus(FLAV_NS))) smartlist_add(items, (char*)item); } else if (!current && !strcmp(url, "consensus-signatures")) { /* XXXX the spec says that we should implement @@ -2641,7 +2667,8 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, flags = DGV_BY_ID | (current ? DGV_INCLUDE_PREVIOUS : DGV_INCLUDE_PENDING); } - dir_split_resource_into_fingerprints(url, fps, NULL, 1, 1); + dir_split_resource_into_fingerprints(url, fps, NULL, + DSR_HEX|DSR_SORT_UNIQ); SMARTLIST_FOREACH(fps, char *, fp, { if ((d = dirvote_get_vote(fp, flags))) smartlist_add(dir_items, (cached_dir_t*)d); @@ -2694,6 +2721,41 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, goto done; } + if (!strcmpstart(url, "/tor/micro/d/")) { + smartlist_t *fps = smartlist_create(); + + dir_split_resource_into_fingerprints(url+strlen("/tor/micro/d/"), + fps, NULL, + DSR_DIGEST256|DSR_BASE64|DSR_SORT_UNIQ); + + if (!dirserv_have_any_microdesc(fps)) { + write_http_status_line(conn, 404, "Not found"); + SMARTLIST_FOREACH(fps, char *, fp, tor_free(fp)); + smartlist_free(fps); + goto done; + } + dlen = dirserv_estimate_microdesc_size(fps, compressed); + if (global_write_bucket_low(TO_CONN(conn), dlen, 2)) { + log_info(LD_DIRSERV, + "Client asked for server descriptors, but we've been " + "writing too many bytes lately. Sending 503 Dir busy."); + write_http_status_line(conn, 503, "Directory busy, try again later"); + SMARTLIST_FOREACH(fps, char *, fp, tor_free(fp)); + smartlist_free(fps); + goto done; + } + + write_http_response_header(conn, -1, compressed, MICRODESC_CACHE_LIFETIME); + conn->dir_spool_src = DIR_SPOOL_MICRODESC; + conn->fingerprint_stack = fps; + + if (compressed) + conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD); + + connection_dirserv_flushed_some(conn); + goto done; + } + if (!strcmpstart(url,"/tor/server/") || (!options->BridgeAuthoritativeDir && !options->BridgeRelay && !strcmpstart(url,"/tor/extra/"))) { @@ -2775,7 +2837,8 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, } else if (!strcmpstart(url, "/tor/keys/fp/")) { smartlist_t *fps = smartlist_create(); dir_split_resource_into_fingerprints(url+strlen("/tor/keys/fp/"), - fps, NULL, 1, 1); + fps, NULL, + DSR_HEX|DSR_SORT_UNIQ); SMARTLIST_FOREACH(fps, char *, d, { authority_cert_t *c = authority_cert_get_newest_by_id(d); if (c) smartlist_add(certs, c); @@ -2785,7 +2848,8 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, } else if (!strcmpstart(url, "/tor/keys/sk/")) { smartlist_t *fps = smartlist_create(); dir_split_resource_into_fingerprints(url+strlen("/tor/keys/sk/"), - fps, NULL, 1, 1); + fps, NULL, + DSR_HEX|DSR_SORT_UNIQ); SMARTLIST_FOREACH(fps, char *, d, { authority_cert_t *c = authority_cert_get_by_sk_digest(d); if (c) smartlist_add(certs, c); @@ -2887,18 +2951,9 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, note_request("/tor/rendezvous?/", desc_len); /* need to send descp separately, because it may include NULs */ connection_write_to_buf(descp, desc_len, TO_CONN(conn)); - /* report successful fetch to statistic */ - if (options->HSAuthorityRecordStats) { - hs_usage_note_fetch_total(query, time(NULL)); - hs_usage_note_fetch_successful(query, time(NULL)); - } break; case 0: /* well-formed but not present */ write_http_status_line(conn, 404, "Not found"); - /* report (unsuccessful) fetch to statistic */ - if (options->HSAuthorityRecordStats) { - hs_usage_note_fetch_total(query, time(NULL)); - } break; case -1: /* not well-formed */ write_http_status_line(conn, 400, "Bad request"); @@ -3175,8 +3230,8 @@ directory_handle_command(dir_connection_t *conn) &body, &body_len, MAX_DIR_UL_SIZE, 0)) { case -1: /* overflow */ log_warn(LD_DIRSERV, - "Invalid input from address '%s'. Closing.", - conn->_base.address); + "Request too large from address '%s' to DirPort. Closing.", + safe_str(conn->_base.address)); return -1; case 0: log_debug(LD_DIRSERV,"command not all here yet."); @@ -3212,6 +3267,16 @@ connection_dir_finished_flushing(dir_connection_t *conn) tor_assert(conn); tor_assert(conn->_base.type == CONN_TYPE_DIR); + /* Note that we have finished writing the directory response. For direct + * connections this means we're done, for tunneled connections its only + * an intermediate step. */ + if (TO_CONN(conn)->dirreq_id) + geoip_change_dirreq_state(TO_CONN(conn)->dirreq_id, DIRREQ_TUNNELED, + DIRREQ_FLUSHING_DIR_CONN_FINISHED); + else + geoip_change_dirreq_state(TO_CONN(conn)->global_identifier, + DIRREQ_DIRECT, + DIRREQ_FLUSHING_DIR_CONN_FINISHED); switch (conn->_base.state) { case DIR_CONN_STATE_CLIENT_SENDING: log_debug(LD_DIR,"client finished sending command."); @@ -3511,19 +3576,37 @@ dir_split_resource_into_fingerprint_pairs(const char *res, /** Given a directory <b>resource</b> request, containing zero * or more strings separated by plus signs, followed optionally by ".z", store * the strings, in order, into <b>fp_out</b>. If <b>compressed_out</b> is - * non-NULL, set it to 1 if the resource ends in ".z", else set it to 0. If - * decode_hex is true, then delete all elements that aren't hex digests, and - * decode the rest. If sort_uniq is true, then sort the list and remove - * all duplicates. + * non-NULL, set it to 1 if the resource ends in ".z", else set it to 0. + * + * If (flags & DSR_HEX), then delete all elements that aren't hex digests, and + * decode the rest. If (flags & DSR_BASE64), then use "-" rather than "+" as + * a separator, delete all the elements that aren't base64-encoded digests, + * and decode the rest. If (flags & DSR_DIGEST256), these digests should be + * 256 bits long; else they should be 160. + * + * If (flags & DSR_SORT_UNIQ), then sort the list and remove all duplicates. */ int dir_split_resource_into_fingerprints(const char *resource, smartlist_t *fp_out, int *compressed_out, - int decode_hex, int sort_uniq) + int flags) { + const int decode_hex = flags & DSR_HEX; + const int decode_base64 = flags & DSR_BASE64; + const int digests_are_256 = flags & DSR_DIGEST256; + const int sort_uniq = flags & DSR_SORT_UNIQ; + + const int digest_len = digests_are_256 ? DIGEST256_LEN : DIGEST_LEN; + const int hex_digest_len = digests_are_256 ? + HEX_DIGEST256_LEN : HEX_DIGEST_LEN; + const int base64_digest_len = digests_are_256 ? + BASE64_DIGEST256_LEN : BASE64_DIGEST_LEN; smartlist_t *fp_tmp = smartlist_create(); + + tor_assert(!(decode_hex && decode_base64)); tor_assert(fp_out); - smartlist_split_string(fp_tmp, resource, "+", 0, 0); + + smartlist_split_string(fp_tmp, resource, decode_base64?"-":"+", 0, 0); if (compressed_out) *compressed_out = 0; if (smartlist_len(fp_tmp)) { @@ -3535,22 +3618,25 @@ dir_split_resource_into_fingerprints(const char *resource, *compressed_out = 1; } } - if (decode_hex) { + if (decode_hex || decode_base64) { + const size_t encoded_len = decode_hex ? hex_digest_len : base64_digest_len; int i; char *cp, *d = NULL; for (i = 0; i < smartlist_len(fp_tmp); ++i) { cp = smartlist_get(fp_tmp, i); - if (strlen(cp) != HEX_DIGEST_LEN) { + if (strlen(cp) != encoded_len) { log_info(LD_DIR, "Skipping digest %s with non-standard length.", escaped(cp)); smartlist_del_keeporder(fp_tmp, i--); goto again; } - d = tor_malloc_zero(DIGEST_LEN); - if (base16_decode(d, DIGEST_LEN, cp, HEX_DIGEST_LEN)<0) { - log_info(LD_DIR, "Skipping non-decodable digest %s", escaped(cp)); - smartlist_del_keeporder(fp_tmp, i--); - goto again; + d = tor_malloc_zero(digest_len); + if (decode_hex ? + (base16_decode(d, digest_len, cp, hex_digest_len)<0) : + (base64_decode(d, digest_len, cp, base64_digest_len)<0)) { + log_info(LD_DIR, "Skipping non-decodable digest %s", escaped(cp)); + smartlist_del_keeporder(fp_tmp, i--); + goto again; } smartlist_set(fp_tmp, i, d); d = NULL; @@ -3560,26 +3646,18 @@ dir_split_resource_into_fingerprints(const char *resource, } } if (sort_uniq) { - smartlist_t *fp_tmp2 = smartlist_create(); - int i; - if (decode_hex) - smartlist_sort_digests(fp_tmp); - else + if (decode_hex || decode_base64) { + if (digests_are_256) { + smartlist_sort_digests256(fp_tmp); + smartlist_uniq_digests256(fp_tmp); + } else { + smartlist_sort_digests(fp_tmp); + smartlist_uniq_digests(fp_tmp); + } + } else { smartlist_sort_strings(fp_tmp); - if (smartlist_len(fp_tmp)) - smartlist_add(fp_tmp2, smartlist_get(fp_tmp, 0)); - for (i = 1; i < smartlist_len(fp_tmp); ++i) { - char *cp = smartlist_get(fp_tmp, i); - char *last = smartlist_get(fp_tmp2, smartlist_len(fp_tmp2)-1); - - if ((decode_hex && memcmp(cp, last, DIGEST_LEN)) - || (!decode_hex && strcasecmp(cp, last))) - smartlist_add(fp_tmp2, cp); - else - tor_free(cp); + smartlist_uniq_strings(fp_tmp); } - smartlist_free(fp_tmp); - fp_tmp = fp_tmp2; } smartlist_add_all(fp_out, fp_tmp); smartlist_free(fp_tmp); diff --git a/src/or/dirserv.c b/src/or/dirserv.c index 349e383ab5..3024612357 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -41,7 +41,7 @@ static time_t the_v2_networkstatus_is_dirty = 1; static cached_dir_t *the_directory = NULL; /** For authoritative directories: the current (v1) network status. */ -static cached_dir_t the_runningrouters = { NULL, NULL, 0, 0, 0, -1 }; +static cached_dir_t the_runningrouters; static void directory_remove_invalid(void); static cached_dir_t *dirserv_regenerate_directory(void); @@ -63,13 +63,16 @@ static signed_descriptor_t *get_signed_descriptor_by_fp(const char *fp, time_t publish_cutoff); static int dirserv_add_extrainfo(extrainfo_t *ei, const char **msg); +/************** Measured Bandwidth parsing code ******/ +#define MAX_MEASUREMENT_AGE (3*24*60*60) /* 3 days */ + /************** Fingerprint handling code ************/ #define FP_NAMED 1 /**< Listed in fingerprint file. */ #define FP_INVALID 2 /**< Believed invalid. */ #define FP_REJECT 4 /**< We will not publish this router. */ #define FP_BADDIR 8 /**< We'll tell clients to avoid using this as a dir. */ -#define FP_BADEXIT 16 /**< We'll tell clients not to use this as an exit. */ +#define FP_BADEXIT 16 /**< We'll tell clients not to use this as an exit. */ #define FP_UNNAMED 32 /**< Another router has this name in fingerprint file. */ /** Encapsulate a nickname and an FP_* status; target of status_by_digest @@ -99,7 +102,7 @@ authdir_config_new(void) return list; } -/** Add the fingerprint <b>fp</b> for the nickname <b>nickname</b> to +/** Add the fingerprint <b>fp</b> for <b>nickname</b> to * the smartlist of fingerprint_entry_t's <b>list</b>. Return 0 if it's * new, or 1 if we replaced the old value. */ @@ -181,8 +184,7 @@ dirserv_add_own_fingerprint(const char *nickname, crypto_pk_env_t *pk) * file. The file format is line-based, with each non-blank holding one * nickname, some space, and a fingerprint for that nickname. On success, * replace the current fingerprint list with the new list and return 0. On - * failure, leave the current fingerprint list untouched, and - * return -1. */ + * failure, leave the current fingerprint list untouched, and return -1. */ int dirserv_load_fingerprint_file(void) { @@ -368,10 +370,10 @@ dirserv_get_status_impl(const char *id_digest, const char *nickname, strmap_size(fingerprint_list->fp_by_name), digestmap_size(fingerprint_list->status_by_digest)); - /* 0.1.1.17-rc was the first version that claimed to be stable, doesn't - * crash and drop circuits all the time, and is even vaguely compatible with - * the current network */ - if (platform && !tor_version_as_new_as(platform,"0.1.1.17-rc")) { + /* Tor 0.1.2.x is pretty old, but there are a lot of them running still, + * and there aren't any critical relay-side vulnerabilities. Once more + * of them die off, we should raise this minimum to 0.2.0.x. */ + if (platform && !tor_version_as_new_as(platform,"0.1.2.14")) { if (msg) *msg = "Tor version is far too old to work."; return FP_REJECT; @@ -520,7 +522,7 @@ authdir_wants_to_reject_router(routerinfo_t *ri, const char **msg, /* Okay. Now check whether the fingerprint is recognized. */ uint32_t status = dirserv_router_get_status(ri, msg); time_t now; - int severity = complain ? LOG_NOTICE : LOG_INFO; + int severity = (complain && ri->contact_info) ? LOG_NOTICE : LOG_INFO; tor_assert(msg); if (status & FP_REJECT) return -1; /* msg is already set. */ @@ -835,46 +837,6 @@ directory_remove_invalid(void) routerlist_assert_ok(rl); } -/** Write a list of unregistered descriptors into a newly allocated - * string and return it. Used by dirserv operators to keep track of - * fast nodes that haven't registered. - */ -int -getinfo_helper_dirserv_unregistered(control_connection_t *control_conn, - const char *question, char **answer_out) -{ - smartlist_t *answerlist; - char buf[1024]; - char *answer; - int min_bw = atoi(question); - routerlist_t *rl = router_get_routerlist(); - - (void) control_conn; - - if (strcmpstart(question, "unregistered-servers-")) - return 0; - question += strlen("unregistered-servers-"); - - answerlist = smartlist_create(); - SMARTLIST_FOREACH(rl->routers, routerinfo_t *, ent, { - uint32_t r = dirserv_router_get_status(ent, NULL); - if (router_get_advertised_bandwidth(ent) >= (size_t)min_bw && - !(r & FP_NAMED)) { - /* then log this one */ - tor_snprintf(buf, sizeof(buf), - "%s: BW %d on '%s'.", - ent->nickname, router_get_advertised_bandwidth(ent), - ent->platform ? ent->platform : ""); - smartlist_add(answerlist, tor_strdup(buf)); - } - }); - answer = smartlist_join_strings(answerlist, "\r\n", 0, NULL); - SMARTLIST_FOREACH(answerlist, char *, cp, tor_free(cp)); - smartlist_free(answerlist); - *answer_out = answer; - return 0; -} - /** Mark the directory as <b>dirty</b> -- when we're next asked for a * directory, we will rebuild it instead of reusing the most recently * generated one. @@ -933,6 +895,13 @@ list_single_server_status(routerinfo_t *desc, int is_live) return tor_strdup(buf); } +static INLINE int +running_long_enough_to_decide_unreachable(void) +{ + return time_of_process_start + + get_options()->TestingAuthDirTimeToLearnReachability < approx_time(); +} + /** Each server needs to have passed a reachability test no more * than this number of seconds ago, or he is listed as down in * the directory. */ @@ -944,6 +913,10 @@ list_single_server_status(routerinfo_t *desc, int is_live) void dirserv_set_router_is_running(routerinfo_t *router, time_t now) { + /*XXXX022 This function is a mess. Separate out the part that calculates + whether it's reachable and the part that tells rephist that the router was + unreachable. + */ int answer; if (router_is_me(router) && !we_are_hibernating()) @@ -952,7 +925,7 @@ dirserv_set_router_is_running(routerinfo_t *router, time_t now) answer = get_options()->AssumeReachable || now < router->last_reachable + REACHABLE_TIMEOUT; - if (!answer) { + if (!answer && running_long_enough_to_decide_unreachable()) { /* not considered reachable. tell rephist. */ rep_hist_note_router_unreachable(router->cache_info.identity_digest, now); } @@ -965,7 +938,6 @@ dirserv_set_router_is_running(routerinfo_t *router, time_t now) * *<b>router_status_out</b>. Return 0 on success, -1 on failure. * * If for_controller is true, include the routers with very old descriptors. - * If for_controller is >1, use the verbose nickname format. */ int list_server_status_v1(smartlist_t *routers, char **router_status_out, @@ -985,23 +957,22 @@ list_server_status_v1(smartlist_t *routers, char **router_status_out, rs_entries = smartlist_create(); - SMARTLIST_FOREACH(routers, routerinfo_t *, ri, - { + SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) { if (authdir) { /* Update router status in routerinfo_t. */ dirserv_set_router_is_running(ri, now); } - if (for_controller == 1 || ri->cache_info.published_on >= cutoff) - smartlist_add(rs_entries, list_single_server_status(ri, ri->is_running)); - else if (for_controller > 2) { + if (for_controller) { char name_buf[MAX_VERBOSE_NICKNAME_LEN+2]; char *cp = name_buf; if (!ri->is_running) *cp++ = '!'; router_get_verbose_nickname(cp, ri); smartlist_add(rs_entries, tor_strdup(name_buf)); + } else if (ri->cache_info.published_on >= cutoff) { + smartlist_add(rs_entries, list_single_server_status(ri, ri->is_running)); } - }); + } SMARTLIST_FOREACH_END(ri); *router_status_out = smartlist_join_strings(rs_entries, " ", 0, NULL); @@ -1119,7 +1090,8 @@ dirserv_dump_directory_to_string(char **dir_out, return -1; } note_crypto_pk_op(SIGN_DIR); - if (router_append_dirobj_signature(buf,buf_len,digest,private_key)<0) { + if (router_append_dirobj_signature(buf,buf_len,digest,DIGEST_LEN, + private_key)<0) { tor_free(buf); return -1; } @@ -1238,14 +1210,14 @@ directory_too_idle_to_fetch_descriptors(or_options_t *options, time_t now) static cached_dir_t *cached_directory = NULL; /** The v1 runningrouters document we'll serve (as a cache or as an authority) * if requested. */ -static cached_dir_t cached_runningrouters = { NULL, NULL, 0, 0, 0, -1 }; +static cached_dir_t cached_runningrouters; /** Used for other dirservers' v2 network statuses. Map from hexdigest to * cached_dir_t. */ static digestmap_t *cached_v2_networkstatus = NULL; -/** The v3 consensus network status that we're currently serving. */ -static cached_dir_t *cached_v3_networkstatus = NULL; +/** Map from flavor name to the v3 consensuses that we're currently serving. */ +static strmap_t *cached_consensuses = NULL; /** Possibly replace the contents of <b>d</b> with the value of * <b>directory</b> published on <b>when</b>, unless <b>when</b> is older than @@ -1319,7 +1291,11 @@ clear_cached_dir(cached_dir_t *d) static void _free_cached_dir(void *_d) { - cached_dir_t *d = (cached_dir_t *)_d; + cached_dir_t *d; + if (!_d) + return; + + d = (cached_dir_t *)_d; cached_dir_decref(d); } @@ -1413,17 +1389,26 @@ dirserv_set_cached_networkstatus_v2(const char *networkstatus, } } -/** Replace the v3 consensus networkstatus that we're serving with - * <b>networkstatus</b>, published at <b>published</b>. No validation is - * performed. */ +/** Replace the v3 consensus networkstatus of type <b>flavor_name</b> that + * we're serving with <b>networkstatus</b>, published at <b>published</b>. No + * validation is performed. */ void -dirserv_set_cached_networkstatus_v3(const char *networkstatus, - time_t published) +dirserv_set_cached_consensus_networkstatus(const char *networkstatus, + const char *flavor_name, + const digests_t *digests, + time_t published) { - if (cached_v3_networkstatus) - cached_dir_decref(cached_v3_networkstatus); - cached_v3_networkstatus = new_cached_dir( - tor_strdup(networkstatus), published); + cached_dir_t *new_networkstatus; + cached_dir_t *old_networkstatus; + if (!cached_consensuses) + cached_consensuses = strmap_new(); + + new_networkstatus = new_cached_dir(tor_strdup(networkstatus), published); + memcpy(&new_networkstatus->digests, digests, sizeof(digests_t)); + old_networkstatus = strmap_set(cached_consensuses, flavor_name, + new_networkstatus); + if (old_networkstatus) + cached_dir_decref(old_networkstatus); } /** Remove any v2 networkstatus from the directory cache that was published @@ -1577,7 +1562,8 @@ generate_runningrouters(void) goto err; } note_crypto_pk_op(SIGN_DIR); - if (router_append_dirobj_signature(s, len, digest, private_key)<0) + if (router_append_dirobj_signature(s, len, digest, DIGEST_LEN, + private_key)<0) goto err; set_cached_dir(&the_runningrouters, s, time(NULL)); @@ -1605,9 +1591,9 @@ dirserv_get_runningrouters(void) /** Return the latest downloaded consensus networkstatus in encoded, signed, * optionally compressed format, suitable for sending to clients. */ cached_dir_t * -dirserv_get_consensus(void) +dirserv_get_consensus(const char *flavor_name) { - return cached_v3_networkstatus; + return strmap_get(cached_consensuses, flavor_name); } /** For authoritative directories: the current (v2) network status. */ @@ -1897,16 +1883,20 @@ version_from_platform(const char *platform) * which has at least <b>buf_len</b> free characters. Do NUL-termination. * Use the same format as in network-status documents. If <b>version</b> is * non-NULL, add a "v" line for the platform. Return 0 on success, -1 on - * failure. If <b>first_line_only</b> is true, don't include any flags - * or version line. + * failure. + * + * The format argument has three possible values: + * NS_V2 - Output an entry suitable for a V2 NS opinion document + * NS_V3_CONSENSUS - Output the first portion of a V3 NS consensus entry + * NS_V3_CONSENSUS_MICRODESC - Output the first portion of a V3 microdesc + * consensus entry. + * NS_V3_VOTE - Output a complete V3 NS vote + * NS_CONTROL_PORT - Output a NS document for the control port */ int routerstatus_format_entry(char *buf, size_t buf_len, routerstatus_t *rs, const char *version, - int first_line_only, int v2_format) -/* XXX: first_line_only and v2_format should probably be be both - * replaced by a single purpose parameter. - */ + routerstatus_format_type_t format) { int r; struct in_addr in; @@ -1925,10 +1915,11 @@ routerstatus_format_entry(char *buf, size_t buf_len, tor_inet_ntoa(&in, ipaddr, sizeof(ipaddr)); r = tor_snprintf(buf, buf_len, - "r %s %s %s %s %s %d %d\n", + "r %s %s %s%s%s %s %d %d\n", rs->nickname, identity64, - digest64, + (format==NS_V3_CONSENSUS_MICRODESC)?"":digest64, + (format==NS_V3_CONSENSUS_MICRODESC)?"":" ", published, ipaddr, (int)rs->or_port, @@ -1937,7 +1928,12 @@ routerstatus_format_entry(char *buf, size_t buf_len, log_warn(LD_BUG, "Not enough space in buffer."); return -1; } - if (first_line_only) + + /* TODO: Maybe we want to pass in what we need to build the rest of + * this here, instead of in the caller. Then we could use the + * networkstatus_type_t values, with an additional control port value + * added -MP */ + if (format == NS_V3_CONSENSUS || format == NS_V3_CONSENSUS_MICRODESC) return 0; cp = buf + strlen(buf); @@ -1974,62 +1970,87 @@ routerstatus_format_entry(char *buf, size_t buf_len, cp += strlen(cp); } - if (!v2_format) { + if (format != NS_V2) { routerinfo_t* desc = router_get_by_digest(rs->identity_digest); + uint32_t bw; + + if (format != NS_CONTROL_PORT) { + /* Blow up more or less nicely if we didn't get anything or not the + * thing we expected. + */ + if (!desc) { + char id[HEX_DIGEST_LEN+1]; + char dd[HEX_DIGEST_LEN+1]; + + base16_encode(id, sizeof(id), rs->identity_digest, DIGEST_LEN); + base16_encode(dd, sizeof(dd), rs->descriptor_digest, DIGEST_LEN); + log_warn(LD_BUG, "Cannot get any descriptor for %s " + "(wanted descriptor %s).", + id, dd); + return -1; + }; + + /* This assert can fire for the control port, because + * it can request NS documents before all descriptors + * have been fetched. */ + if (memcmp(desc->cache_info.signed_descriptor_digest, + rs->descriptor_digest, + DIGEST_LEN)) { + char rl_d[HEX_DIGEST_LEN+1]; + char rs_d[HEX_DIGEST_LEN+1]; + char id[HEX_DIGEST_LEN+1]; + + base16_encode(rl_d, sizeof(rl_d), + desc->cache_info.signed_descriptor_digest, DIGEST_LEN); + base16_encode(rs_d, sizeof(rs_d), rs->descriptor_digest, DIGEST_LEN); + base16_encode(id, sizeof(id), rs->identity_digest, DIGEST_LEN); + log_err(LD_BUG, "descriptor digest in routerlist does not match " + "the one in routerstatus: %s vs %s " + "(router %s)\n", + rl_d, rs_d, id); + + tor_assert(!memcmp(desc->cache_info.signed_descriptor_digest, + rs->descriptor_digest, + DIGEST_LEN)); + }; + } - /* Blow up more or less nicely if we didn't get anything or not the - * thing we expected. - */ - if (!desc) { - char id[HEX_DIGEST_LEN+1]; - char dd[HEX_DIGEST_LEN+1]; - - base16_encode(id, sizeof(id), rs->identity_digest, DIGEST_LEN); - base16_encode(dd, sizeof(dd), rs->descriptor_digest, DIGEST_LEN); - log_warn(LD_BUG, "Cannot get any descriptor for %s " - "(wanted descriptor %s).", - id, dd); - return -1; - }; - if (memcmp(desc->cache_info.signed_descriptor_digest, - rs->descriptor_digest, - DIGEST_LEN)) { - char rl_d[HEX_DIGEST_LEN+1]; - char rs_d[HEX_DIGEST_LEN+1]; - char id[HEX_DIGEST_LEN+1]; - - base16_encode(rl_d, sizeof(rl_d), - desc->cache_info.signed_descriptor_digest, DIGEST_LEN); - base16_encode(rs_d, sizeof(rs_d), rs->descriptor_digest, DIGEST_LEN); - base16_encode(id, sizeof(id), rs->identity_digest, DIGEST_LEN); - log_err(LD_BUG, "descriptor digest in routerlist does not match " - "the one in routerstatus: %s vs %s " - "(router %s)\n", - rl_d, rs_d, id); - - tor_assert(!memcmp(desc->cache_info.signed_descriptor_digest, - rs->descriptor_digest, - DIGEST_LEN)); - }; - + if (format == NS_CONTROL_PORT && rs->has_bandwidth) { + bw = rs->bandwidth; + } else { + tor_assert(desc); + bw = router_get_advertised_bandwidth_capped(desc) / 1000; + } r = tor_snprintf(cp, buf_len - (cp-buf), - "w Bandwidth=%d\n", - router_get_advertised_bandwidth_capped(desc) / 1024); + "w Bandwidth=%d\n", bw); + if (r<0) { log_warn(LD_BUG, "Not enough space in buffer."); return -1; } cp += strlen(cp); + if (format == NS_V3_VOTE && rs->has_measured_bw) { + *--cp = '\0'; /* Kill "\n" */ + r = tor_snprintf(cp, buf_len - (cp-buf), + " Measured=%d\n", rs->measured_bw); + if (r<0) { + log_warn(LD_BUG, "Not enough space in buffer for weight line."); + return -1; + } + cp += strlen(cp); + } - summary = policy_summarize(desc->exit_policy); - r = tor_snprintf(cp, buf_len - (cp-buf), "p %s\n", summary); - if (r<0) { - log_warn(LD_BUG, "Not enough space in buffer."); + if (desc) { + summary = policy_summarize(desc->exit_policy); + r = tor_snprintf(cp, buf_len - (cp-buf), "p %s\n", summary); + if (r<0) { + log_warn(LD_BUG, "Not enough space in buffer."); + tor_free(summary); + return -1; + } + cp += strlen(cp); tor_free(summary); - return -1; } - cp += strlen(cp); - tor_free(summary); } return 0; @@ -2232,6 +2253,177 @@ router_clear_status_flags(routerinfo_t *router) router->is_bad_exit = router->is_bad_directory = 0; } +/** + * Helper function to parse out a line in the measured bandwidth file + * into a measured_bw_line_t output structure. Returns -1 on failure + * or 0 on success. + */ +int +measured_bw_line_parse(measured_bw_line_t *out, const char *orig_line) +{ + char *line = tor_strdup(orig_line); + char *cp = line; + int got_bw = 0; + int got_node_id = 0; + char *strtok_state; /* lame sauce d'jour */ + cp = tor_strtok_r(cp, " \t", &strtok_state); + + if (!cp) { + log_warn(LD_DIRSERV, "Invalid line in bandwidth file: %s", + escaped(orig_line)); + tor_free(line); + return -1; + } + + if (orig_line[strlen(orig_line)-1] != '\n') { + log_warn(LD_DIRSERV, "Incomplete line in bandwidth file: %s", + escaped(orig_line)); + tor_free(line); + return -1; + } + + do { + if (strcmpstart(cp, "bw=") == 0) { + int parse_ok = 0; + char *endptr; + if (got_bw) { + log_warn(LD_DIRSERV, "Double bw= in bandwidth file line: %s", + escaped(orig_line)); + tor_free(line); + return -1; + } + cp+=strlen("bw="); + + out->bw = tor_parse_long(cp, 0, 0, LONG_MAX, &parse_ok, &endptr); + if (!parse_ok || (*endptr && !TOR_ISSPACE(*endptr))) { + log_warn(LD_DIRSERV, "Invalid bandwidth in bandwidth file line: %s", + escaped(orig_line)); + tor_free(line); + return -1; + } + got_bw=1; + } else if (strcmpstart(cp, "node_id=$") == 0) { + if (got_node_id) { + log_warn(LD_DIRSERV, "Double node_id= in bandwidth file line: %s", + escaped(orig_line)); + tor_free(line); + return -1; + } + cp+=strlen("node_id=$"); + + if (strlen(cp) != HEX_DIGEST_LEN || + base16_decode(out->node_id, DIGEST_LEN, cp, HEX_DIGEST_LEN)) { + log_warn(LD_DIRSERV, "Invalid node_id in bandwidth file line: %s", + escaped(orig_line)); + tor_free(line); + return -1; + } + strncpy(out->node_hex, cp, sizeof(out->node_hex)); + got_node_id=1; + } + } while ((cp = tor_strtok_r(NULL, " \t", &strtok_state))); + + if (got_bw && got_node_id) { + tor_free(line); + return 0; + } else { + log_warn(LD_DIRSERV, "Incomplete line in bandwidth file: %s", + escaped(orig_line)); + tor_free(line); + return -1; + } +} + +/** + * Helper function to apply a parsed measurement line to a list + * of bandwidth statuses. Returns true if a line is found, + * false otherwise. + */ +int +measured_bw_line_apply(measured_bw_line_t *parsed_line, + smartlist_t *routerstatuses) +{ + routerstatus_t *rs = NULL; + if (!routerstatuses) + return 0; + + rs = smartlist_bsearch(routerstatuses, parsed_line->node_id, + compare_digest_to_routerstatus_entry); + + if (rs) { + rs->has_measured_bw = 1; + rs->measured_bw = (uint32_t)parsed_line->bw; + } else { + log_info(LD_DIRSERV, "Node ID %s not found in routerstatus list", + parsed_line->node_hex); + } + + return rs != NULL; +} + +/** + * Read the measured bandwidth file and apply it to the list of + * routerstatuses. Returns -1 on error, 0 otherwise. + */ +int +dirserv_read_measured_bandwidths(const char *from_file, + smartlist_t *routerstatuses) +{ + char line[256]; + FILE *fp = fopen(from_file, "r"); + int applied_lines = 0; + time_t file_time; + int ok; + if (fp == NULL) { + log_warn(LD_CONFIG, "Can't open bandwidth file at configured location: %s", + from_file); + return -1; + } + + if (!fgets(line, sizeof(line), fp) + || !strlen(line) || line[strlen(line)-1] != '\n') { + log_warn(LD_DIRSERV, "Long or truncated time in bandwidth file: %s", + escaped(line)); + fclose(fp); + return -1; + } + + line[strlen(line)-1] = '\0'; + file_time = tor_parse_ulong(line, 10, 0, ULONG_MAX, &ok, NULL); + if (!ok) { + log_warn(LD_DIRSERV, "Non-integer time in bandwidth file: %s", + escaped(line)); + fclose(fp); + return -1; + } + + if ((time(NULL) - file_time) > MAX_MEASUREMENT_AGE) { + log_warn(LD_DIRSERV, "Bandwidth measurement file stale. Age: %u", + (unsigned)(time(NULL) - file_time)); + fclose(fp); + return -1; + } + + if (routerstatuses) + smartlist_sort(routerstatuses, compare_routerstatus_entries); + + while (!feof(fp)) { + measured_bw_line_t parsed_line; + if (fgets(line, sizeof(line), fp) && strlen(line)) { + if (measured_bw_line_parse(&parsed_line, line) != -1) { + if (measured_bw_line_apply(&parsed_line, routerstatuses) > 0) + applied_lines++; + } + } + } + + fclose(fp); + log_info(LD_DIRSERV, + "Bandwidth measurement file successfully read. " + "Applied %d measurements.", applied_lines); + return 0; +} + /** Return a new networkstatus_t* containing our current opinion. (For v3 * authorities) */ networkstatus_t * @@ -2256,15 +2448,12 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key, networkstatus_voter_info_t *voter = NULL; vote_timing_t timing; digestmap_t *omit_as_sybil = NULL; - int vote_on_reachability = 1; + const int vote_on_reachability = running_long_enough_to_decide_unreachable(); + smartlist_t *microdescriptors = NULL; tor_assert(private_key); tor_assert(cert); - if (now - time_of_process_start < - options->TestingAuthDirTimeToLearnReachability) - vote_on_reachability = 0; - if (resolve_my_address(LOG_WARN, options, &addr, &hostname)<0) { log_warn(LD_NET, "Couldn't resolve my hostname"); return NULL; @@ -2309,11 +2498,13 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key, omit_as_sybil = get_possible_sybil_list(routers); routerstatuses = smartlist_create(); + microdescriptors = smartlist_create(); - SMARTLIST_FOREACH(routers, routerinfo_t *, ri, { + SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) { if (ri->cache_info.published_on >= cutoff) { routerstatus_t *rs; vote_routerstatus_t *vrs; + microdesc_t *md; vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); rs = &vrs->status; @@ -2328,12 +2519,39 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key, rs->is_running = 0; vrs->version = version_from_platform(ri->platform); + md = dirvote_create_microdescriptor(ri); + if (md) { + char buf[128]; + vote_microdesc_hash_t *h; + dirvote_format_microdesc_vote_line(buf, sizeof(buf), md); + h = tor_malloc(sizeof(vote_microdesc_hash_t)); + h->microdesc_hash_line = tor_strdup(buf); + h->next = NULL; + vrs->microdesc = h; + md->last_listed = now; + smartlist_add(microdescriptors, md); + } + smartlist_add(routerstatuses, vrs); } - }); + } SMARTLIST_FOREACH_END(ri); + + { + smartlist_t *added = + microdescs_add_list_to_cache(get_microdesc_cache(), + microdescriptors, SAVED_NOWHERE, 0); + smartlist_free(added); + smartlist_free(microdescriptors); + } + smartlist_free(routers); digestmap_free(omit_as_sybil, NULL); + if (options->V3BandwidthsFile) { + dirserv_read_measured_bandwidths(options->V3BandwidthsFile, + routerstatuses); + } + v3_out = tor_malloc_zero(sizeof(networkstatus_t)); v3_out->type = NS_TYPE_VOTE; @@ -2383,15 +2601,22 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key, } smartlist_sort_strings(v3_out->known_flags); + if (options->ConsensusParams) { + v3_out->net_params = smartlist_create(); + smartlist_split_string(v3_out->net_params, + options->ConsensusParams, NULL, 0, 0); + smartlist_sort_strings(v3_out->net_params); + } + voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t)); voter->nickname = tor_strdup(options->Nickname); memcpy(voter->identity_digest, identity_digest, DIGEST_LEN); + voter->sigs = smartlist_create(); voter->address = hostname; voter->addr = addr; voter->dir_port = options->DirPort; voter->or_port = options->ORPort; voter->contact = tor_strdup(contact); - memcpy(voter->signing_key_digest, signing_key_digest, DIGEST_LEN); if (options->V3AuthUseLegacyKey) { authority_cert_t *c = get_my_v3_legacy_cert(); if (c) { @@ -2537,7 +2762,7 @@ generate_v2_networkstatus_opinion(void) if (digestmap_get(omit_as_sybil, ri->cache_info.identity_digest)) clear_status_flags_on_sybil(&rs); - if (routerstatus_format_entry(outp, endp-outp, &rs, version, 0, 1)) { + if (routerstatus_format_entry(outp, endp-outp, &rs, version, NS_V2)) { log_warn(LD_BUG, "Unable to print router status."); tor_free(version); goto done; @@ -2559,7 +2784,8 @@ generate_v2_networkstatus_opinion(void) outp += strlen(outp); note_crypto_pk_op(SIGN_DIR); - if (router_append_dirobj_signature(outp,endp-outp,digest,private_key)<0) { + if (router_append_dirobj_signature(outp,endp-outp,digest,DIGEST_LEN, + private_key)<0) { log_warn(LD_BUG, "Unable to sign router status."); goto done; } @@ -2591,10 +2817,8 @@ generate_v2_networkstatus_opinion(void) tor_free(status); tor_free(hostname); tor_free(identity_pkey); - if (routers) - smartlist_free(routers); - if (omit_as_sybil) - digestmap_free(omit_as_sybil, NULL); + smartlist_free(routers); + digestmap_free(omit_as_sybil, NULL); return r; } @@ -2642,7 +2866,8 @@ dirserv_get_networkstatus_v2_fingerprints(smartlist_t *result, log_info(LD_DIRSERV, "Client requested 'all' network status objects; we have none."); } else if (!strcmpstart(key, "fp/")) { - dir_split_resource_into_fingerprints(key+3, result, NULL, 1, 1); + dir_split_resource_into_fingerprints(key+3, result, NULL, + DSR_HEX|DSR_SORT_UNIQ); } } @@ -2707,10 +2932,12 @@ dirserv_get_routerdesc_fingerprints(smartlist_t *fps_out, const char *key, } else if (!strcmpstart(key, "d/")) { by_id = 0; key += strlen("d/"); - dir_split_resource_into_fingerprints(key, fps_out, NULL, 1, 1); + dir_split_resource_into_fingerprints(key, fps_out, NULL, + DSR_HEX|DSR_SORT_UNIQ); } else if (!strcmpstart(key, "fp/")) { key += strlen("fp/"); - dir_split_resource_into_fingerprints(key, fps_out, NULL, 1, 1); + dir_split_resource_into_fingerprints(key, fps_out, NULL, + DSR_HEX|DSR_SORT_UNIQ); } else { *msg = "Key not recognized"; return -1; @@ -2775,7 +3002,8 @@ dirserv_get_routerdescs(smartlist_t *descs_out, const char *key, } else if (!strcmpstart(key, "/tor/server/d/")) { smartlist_t *digests = smartlist_create(); key += strlen("/tor/server/d/"); - dir_split_resource_into_fingerprints(key, digests, NULL, 1, 1); + dir_split_resource_into_fingerprints(key, digests, NULL, + DSR_HEX|DSR_SORT_UNIQ); SMARTLIST_FOREACH(digests, const char *, d, { signed_descriptor_t *sd = router_get_by_descriptor_digest(d); @@ -2788,7 +3016,8 @@ dirserv_get_routerdescs(smartlist_t *descs_out, const char *key, smartlist_t *digests = smartlist_create(); time_t cutoff = time(NULL) - ROUTER_MAX_AGE_TO_PUBLISH; key += strlen("/tor/server/fp/"); - dir_split_resource_into_fingerprints(key, digests, NULL, 1, 1); + dir_split_resource_into_fingerprints(key, digests, NULL, + DSR_HEX|DSR_SORT_UNIQ); SMARTLIST_FOREACH(digests, const char *, d, { if (router_digest_is_me(d)) { @@ -2904,17 +3133,20 @@ dirserv_test_reachability(time_t now, int try_all) ctr = (ctr + 1) % 128; } -/** Given a fingerprint <b>fp</b> which is either set if we're looking - * for a v2 status, or zeroes if we're looking for a v3 status, return - * a pointer to the appropriate cached dir object, or NULL if there isn't - * one available. */ +/** Given a fingerprint <b>fp</b> which is either set if we're looking for a + * v2 status, or zeroes if we're looking for a v3 status, or a NUL-padded + * flavor name if we want a flavored v3 status, return a pointer to the + * appropriate cached dir object, or NULL if there isn't one available. */ static cached_dir_t * lookup_cached_dir_by_fp(const char *fp) { cached_dir_t *d = NULL; - if (tor_digest_is_zero(fp) && cached_v3_networkstatus) - d = cached_v3_networkstatus; - else if (router_digest_is_me(fp) && the_v2_networkstatus) + if (tor_digest_is_zero(fp) && cached_consensuses) + d = strmap_get(cached_consensuses, "ns"); + else if (memchr(fp, '\0', DIGEST_LEN) && cached_consensuses && + (d = strmap_get(cached_consensuses, fp))) { + /* this here interface is a nasty hack XXXX022 */; + } else if (router_digest_is_me(fp) && the_v2_networkstatus) d = the_v2_networkstatus; else if (cached_v2_networkstatus) d = digestmap_get(cached_v2_networkstatus, fp); @@ -3000,6 +3232,18 @@ dirserv_have_any_serverdesc(smartlist_t *fps, int spool_src) return 0; } +/** Return true iff any of the 256-bit elements in <b>fps</b> is the digest of + * a microdescriptor we have. */ +int +dirserv_have_any_microdesc(const smartlist_t *fps) +{ + microdesc_cache_t *cache = get_microdesc_cache(); + SMARTLIST_FOREACH(fps, const char *, fp, + if (microdesc_cache_lookup_by_digest256(cache, fp)) + return 1); + return 0; +} + /** Return an approximate estimate of the number of bytes that will * be needed to transmit the server descriptors (if is_serverdescs -- * they can be either d/ or fp/ queries) or networkstatus objects (if @@ -3031,6 +3275,17 @@ dirserv_estimate_data_size(smartlist_t *fps, int is_serverdescs, return result; } +/** Given a list of microdescriptor hashes, guess how many bytes will be + * needed to transmit them, and return the guess. */ +size_t +dirserv_estimate_microdesc_size(const smartlist_t *fps, int compressed) +{ + size_t result = smartlist_len(fps) * microdesc_average_size(NULL); + if (compressed) + result /= 2; + return result; +} + /** When we're spooling data onto our outbuf, add more whenever we dip * below this threshold. */ #define DIRSERV_BUFFER_MIN 16384 @@ -3094,6 +3349,8 @@ connection_dirserv_add_servers_to_outbuf(dir_connection_t *conn) #endif body = signed_descriptor_get_body(sd); if (conn->zlib_state) { + /* XXXX022 This 'last' business should actually happen on the last + * routerinfo, not on the last fingerprint. */ int last = ! smartlist_len(conn->fingerprint_stack); connection_write_to_buf_zlib(body, sd->signed_descriptor_len, conn, last); @@ -3117,6 +3374,44 @@ connection_dirserv_add_servers_to_outbuf(dir_connection_t *conn) return 0; } +/** Spooling helper: called when we're sending a bunch of microdescriptors, + * and the outbuf has become too empty. Pulls some entries from + * fingerprint_stack, and writes the corresponding microdescs onto outbuf. If + * we run out of entries, flushes the zlib state and sets the spool source to + * NONE. Returns 0 on success, negative on failure. + */ +static int +connection_dirserv_add_microdescs_to_outbuf(dir_connection_t *conn) +{ + microdesc_cache_t *cache = get_microdesc_cache(); + while (smartlist_len(conn->fingerprint_stack) && + buf_datalen(conn->_base.outbuf) < DIRSERV_BUFFER_MIN) { + char *fp256 = smartlist_pop_last(conn->fingerprint_stack); + microdesc_t *md = microdesc_cache_lookup_by_digest256(cache, fp256); + tor_free(fp256); + if (!md) + continue; + if (conn->zlib_state) { + /* XXXX022 This 'last' business should actually happen on the last + * routerinfo, not on the last fingerprint. */ + int last = !smartlist_len(conn->fingerprint_stack); + connection_write_to_buf_zlib(md->body, md->bodylen, conn, last); + if (last) { + tor_zlib_free(conn->zlib_state); + conn->zlib_state = NULL; + } + } else { + connection_write_to_buf(md->body, md->bodylen, TO_CONN(conn)); + } + } + if (!smartlist_len(conn->fingerprint_stack)) { + conn->dir_spool_src = DIR_SPOOL_NONE; + smartlist_free(conn->fingerprint_stack); + conn->fingerprint_stack = NULL; + } + return 0; +} + /** Spooling helper: Called when we're sending a directory or networkstatus, * and the outbuf has become too empty. Pulls some bytes from * <b>conn</b>-\>cached_dir-\>dir_z, uncompresses them if appropriate, and @@ -3199,8 +3494,7 @@ connection_dirserv_add_networkstatus_bytes_to_outbuf(dir_connection_t *conn) } } else { connection_dirserv_finish_spooling(conn); - if (conn->fingerprint_stack) - smartlist_free(conn->fingerprint_stack); + smartlist_free(conn->fingerprint_stack); conn->fingerprint_stack = NULL; return 0; } @@ -3224,6 +3518,8 @@ connection_dirserv_flushed_some(dir_connection_t *conn) case DIR_SPOOL_SERVER_BY_DIGEST: case DIR_SPOOL_SERVER_BY_FP: return connection_dirserv_add_servers_to_outbuf(conn); + case DIR_SPOOL_MICRODESC: + return connection_dirserv_add_microdescs_to_outbuf(conn); case DIR_SPOOL_CACHED_DIR: return connection_dirserv_add_dir_bytes_to_outbuf(conn); case DIR_SPOOL_NETWORKSTATUS: @@ -3245,10 +3541,10 @@ dirserv_free_all(void) cached_dir_decref(the_v2_networkstatus); cached_dir_decref(cached_directory); clear_cached_dir(&cached_runningrouters); - if (cached_v2_networkstatus) { - digestmap_free(cached_v2_networkstatus, _free_cached_dir); - cached_v2_networkstatus = NULL; - } - cached_dir_decref(cached_v3_networkstatus); + + digestmap_free(cached_v2_networkstatus, _free_cached_dir); + cached_v2_networkstatus = NULL; + strmap_free(cached_consensuses, _free_cached_dir); + cached_consensuses = NULL; } diff --git a/src/or/dirvote.c b/src/or/dirvote.c index 0156e273be..7227ac9740 100644 --- a/src/or/dirvote.c +++ b/src/or/dirvote.c @@ -11,21 +11,48 @@ * \brief Functions to compute directory consensus, and schedule voting. **/ -static int dirvote_add_signatures_to_pending_consensus( +/** A consensus that we have built and are appending signatures to. Once it's + * time to publish it, it will become an active consensus if it accumulates + * enough signatures. */ +typedef struct pending_consensus_t { + /** The body of the consensus that we're currently building. Once we + * have it built, it goes into dirserv.c */ + char *body; + /** The parsed in-progress consensus document. */ + networkstatus_t *consensus; +} pending_consensus_t; + +static int dirvote_add_signatures_to_all_pending_consensuses( const char *detached_signatures_body, const char **msg_out); +static int dirvote_add_signatures_to_pending_consensus( + pending_consensus_t *pc, + ns_detached_signatures_t *sigs, + const char **msg_out); static char *list_v3_auth_ids(void); static void dirvote_fetch_missing_votes(void); static void dirvote_fetch_missing_signatures(void); static int dirvote_perform_vote(void); static void dirvote_clear_votes(int all_votes); -static int dirvote_compute_consensus(void); +static int dirvote_compute_consensuses(void); static int dirvote_publish_consensus(void); +static char *make_consensus_method_list(int low, int high, const char *sep); + +/** The highest consensus method that we currently support. */ +#define MAX_SUPPORTED_CONSENSUS_METHOD 8 + +#define MIN_METHOD_FOR_PARAMS 7 + +/** Lowest consensus method that generates microdescriptors */ +#define MIN_METHOD_FOR_MICRODESC 8 /* ===== * Voting * =====*/ +/* Overestimated. */ +#define MICRODESC_LINE_LEN 80 + /** Return a new string containing the string representation of the vote in * <b>v3_ns</b>, signed with our v3 signing key <b>private_signing_key</b>. * For v3 authorities. */ @@ -83,7 +110,7 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key, len = 8192; len += strlen(version_lines); - len += (RS_ENTRY_LEN)*smartlist_len(rl->routers); + len += (RS_ENTRY_LEN+MICRODESC_LINE_LEN)*smartlist_len(rl->routers); len += v3_ns->cert->cache_info.signed_descriptor_len; status = tor_malloc(len); @@ -93,17 +120,25 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key, char fu[ISO_TIME_LEN+1]; char vu[ISO_TIME_LEN+1]; char *flags = smartlist_join_strings(v3_ns->known_flags, " ", 0, NULL); + char *params; authority_cert_t *cert = v3_ns->cert; + char *methods = + make_consensus_method_list(1, MAX_SUPPORTED_CONSENSUS_METHOD, " "); format_iso_time(published, v3_ns->published); format_iso_time(va, v3_ns->valid_after); format_iso_time(fu, v3_ns->fresh_until); format_iso_time(vu, v3_ns->valid_until); + if (v3_ns->net_params) + params = smartlist_join_strings(v3_ns->net_params, " ", 0, NULL); + else + params = tor_strdup(""); + tor_assert(cert); tor_snprintf(status, len, "network-status-version 3\n" "vote-status %s\n" - "consensus-methods 1 2 3 4 5\n" + "consensus-methods %s\n" "published %s\n" "valid-after %s\n" "fresh-until %s\n" @@ -111,17 +146,22 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key, "voting-delay %d %d\n" "%s" /* versions */ "known-flags %s\n" + "params %s\n" "dir-source %s %s %s %s %d %d\n" "contact %s\n", v3_ns->type == NS_TYPE_VOTE ? "vote" : "opinion", + methods, published, va, fu, vu, v3_ns->vote_seconds, v3_ns->dist_seconds, version_lines, flags, + params, voter->nickname, fingerprint, voter->address, - ipaddr, voter->dir_port, voter->or_port, voter->contact); + ipaddr, voter->dir_port, voter->or_port, voter->contact); + tor_free(params); tor_free(flags); + tor_free(methods); outp = status + strlen(status); endp = status + len; @@ -139,15 +179,25 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key, outp += cert->cache_info.signed_descriptor_len; } - SMARTLIST_FOREACH(v3_ns->routerstatus_list, vote_routerstatus_t *, vrs, - { + SMARTLIST_FOREACH_BEGIN(v3_ns->routerstatus_list, vote_routerstatus_t *, + vrs) { + vote_microdesc_hash_t *h; if (routerstatus_format_entry(outp, endp-outp, &vrs->status, - vrs->version, 0, 0) < 0) { + vrs->version, NS_V3_VOTE) < 0) { log_warn(LD_BUG, "Unable to print router status."); goto err; } outp += strlen(outp); - }); + + for (h = vrs->microdesc; h; h = h->next) { + size_t mlen = strlen(h->microdesc_hash_line); + if (outp+mlen >= endp) { + log_warn(LD_BUG, "Can't fit microdesc line in vote."); + } + memcpy(outp, h->microdesc_hash_line, mlen+1); + outp += strlen(outp); + } + } SMARTLIST_FOREACH_END(vrs); { char signing_key_fingerprint[FINGERPRINT_LEN+1]; @@ -170,10 +220,10 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key, outp += strlen(outp); } - if (router_get_networkstatus_v3_hash(status, digest)<0) + if (router_get_networkstatus_v3_hash(status, digest, DIGEST_SHA1)<0) goto err; note_crypto_pk_op(SIGN_DIR); - if (router_append_dirobj_signature(outp,endp-outp,digest, + if (router_append_dirobj_signature(outp,endp-outp,digest, DIGEST_LEN, private_signing_key)<0) { log_warn(LD_BUG, "Unable to sign networkstatus vote."); goto err; @@ -216,6 +266,20 @@ get_voter(const networkstatus_t *vote) return smartlist_get(vote->voters, 0); } +/** Return the signature made by <b>voter</b> using the algorithm + * <b>alg</b>, or NULL if none is found. */ +document_signature_t * +voter_get_sig_by_algorithm(const networkstatus_voter_info_t *voter, + digest_algorithm_t alg) +{ + if (!voter->sigs) + return NULL; + SMARTLIST_FOREACH(voter->sigs, document_signature_t *, sig, + if (sig->alg == alg) + return sig); + return NULL; +} + /** Temporary structure used in constructing a list of dir-source entries * for a consensus. One of these is generated for every vote, and one more * for every legacy key in each vote. */ @@ -275,34 +339,8 @@ get_frequent_members(smartlist_t *out, smartlist_t *in, int min) /** Given a sorted list of strings <b>lst</b>, return the member that appears * most. Break ties in favor of later-occurring members. */ -static const char * -get_most_frequent_member(smartlist_t *lst) -{ - const char *most_frequent = NULL; - int most_frequent_count = 0; - - const char *cur = NULL; - int count = 0; - - SMARTLIST_FOREACH(lst, const char *, s, - { - if (cur && !strcmp(s, cur)) { - ++count; - } else { - if (count >= most_frequent_count) { - most_frequent = cur; - most_frequent_count = count; - } - cur = s; - count = 1; - } - }); - if (count >= most_frequent_count) { - most_frequent = cur; - most_frequent_count = count; - } - return most_frequent; -} +#define get_most_frequent_member(lst) \ + smartlist_get_most_frequent_string(lst) /** Return 0 if and only if <b>a</b> and <b>b</b> are routerstatuses * that come from the same routerinfo, with the same derived elements. @@ -344,7 +382,8 @@ _compare_vote_rs(const void **_a, const void **_b) * in favor of smaller descriptor digest. */ static vote_routerstatus_t * -compute_routerstatus_consensus(smartlist_t *votes) +compute_routerstatus_consensus(smartlist_t *votes, int consensus_method, + char *microdesc_digest256_out) { vote_routerstatus_t *most = NULL, *cur = NULL; int most_n = 0, cur_n = 0; @@ -380,18 +419,45 @@ compute_routerstatus_consensus(smartlist_t *votes) } tor_assert(most); + + if (consensus_method >= MIN_METHOD_FOR_MICRODESC && + microdesc_digest256_out) { + smartlist_t *digests = smartlist_create(); + const char *best_microdesc_digest; + SMARTLIST_FOREACH_BEGIN(votes, vote_routerstatus_t *, rs) { + char d[DIGEST256_LEN]; + if (compare_vote_rs(rs, most)) + continue; + if (!vote_routerstatus_find_microdesc_hash(d, rs, consensus_method, + DIGEST_SHA256)) + smartlist_add(digests, tor_memdup(d, sizeof(d))); + } SMARTLIST_FOREACH_END(rs); + smartlist_sort_digests256(digests); + best_microdesc_digest = smartlist_get_most_frequent_digest256(digests); + if (best_microdesc_digest) + memcpy(microdesc_digest256_out, best_microdesc_digest, DIGEST256_LEN); + SMARTLIST_FOREACH(digests, char *, cp, tor_free(cp)); + smartlist_free(digests); + } + return most; } -/** Given a list of strings in <b>lst</b>, set the DIGEST_LEN-byte digest at - * <b>digest_out</b> to the hash of the concatenation of those strings. */ +/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest + * at <b>digest_out</b> to the hash of the concatenation of those strings, + * computed with the algorithm <b>alg</b>. */ static void -hash_list_members(char *digest_out, smartlist_t *lst) +hash_list_members(char *digest_out, size_t len_out, + smartlist_t *lst, digest_algorithm_t alg) { - crypto_digest_env_t *d = crypto_new_digest_env(); + crypto_digest_env_t *d; + if (alg == DIGEST_SHA1) + d = crypto_new_digest_env(); + else + d = crypto_new_digest256_env(alg); SMARTLIST_FOREACH(lst, const char *, cp, crypto_digest_add_bytes(d, cp, strlen(cp))); - crypto_digest_get_digest(d, digest_out, DIGEST_LEN); + crypto_digest_get_digest(d, digest_out, len_out); crypto_free_digest_env(d); } @@ -455,7 +521,31 @@ compute_consensus_method(smartlist_t *votes) static int consensus_method_is_supported(int method) { - return (method >= 1) && (method <= 5); + return (method >= 1) && (method <= MAX_SUPPORTED_CONSENSUS_METHOD); +} + +/** Return a newly allocated string holding the numbers between low and high + * (inclusive) that are supported consensus methods. */ +static char * +make_consensus_method_list(int low, int high, const char *separator) +{ + char *list; + + char b[32]; + int i; + smartlist_t *lst; + lst = smartlist_create(); + for (i = low; i <= high; ++i) { + if (!consensus_method_is_supported(i)) + continue; + tor_snprintf(b, sizeof(b), "%d", i); + smartlist_add(lst, tor_strdup(b)); + } + list = smartlist_join_strings(lst, separator, 0, NULL); + tor_assert(list); + SMARTLIST_FOREACH(lst, char *, cp, tor_free(cp)); + smartlist_free(lst); + return list; } /** Helper: given <b>lst</b>, a list of version strings such that every @@ -475,6 +565,89 @@ compute_consensus_versions_list(smartlist_t *lst, int n_versioning) return result; } +/** Helper: given a list of valid networkstatus_t, return a new string + * containing the contents of the consensus network parameter set. + */ +/* private */ char * +dirvote_compute_params(smartlist_t *votes) +{ + int i; + int32_t *vals; + + int cur_param_len; + const char *cur_param; + const char *eq; + char *result; + + const int n_votes = smartlist_len(votes); + smartlist_t *output; + smartlist_t *param_list = smartlist_create(); + + /* We require that the parameter lists in the votes are well-formed: that + is, that their keywords are unique and sorted, and that their values are + between INT32_MIN and INT32_MAX inclusive. This should be guaranteed by + the parsing code. */ + + vals = tor_malloc(sizeof(int)*n_votes); + + SMARTLIST_FOREACH_BEGIN(votes, networkstatus_t *, v) { + if (!v->net_params) + continue; + smartlist_add_all(param_list, v->net_params); + } SMARTLIST_FOREACH_END(v); + + if (smartlist_len(param_list) == 0) { + tor_free(vals); + smartlist_free(param_list); + return NULL; + } + + smartlist_sort_strings(param_list); + i = 0; + cur_param = smartlist_get(param_list, 0); + eq = strchr(cur_param, '='); + tor_assert(eq); + cur_param_len = (int)(eq+1 - cur_param); + + output = smartlist_create(); + + SMARTLIST_FOREACH_BEGIN(param_list, const char *, param) { + const char *next_param; + int ok=0; + eq = strchr(param, '='); + tor_assert(i<n_votes); + vals[i++] = (int32_t) + tor_parse_long(eq+1, 10, INT32_MIN, INT32_MAX, &ok, NULL); + tor_assert(ok); + + if (param_sl_idx+1 == smartlist_len(param_list)) + next_param = NULL; + else + next_param = smartlist_get(param_list, param_sl_idx+1); + if (!next_param || strncmp(next_param, param, cur_param_len)) { + /* We've reached the end of a series. */ + int32_t median = median_int32(vals, i); + char *out_string = tor_malloc(64+cur_param_len); + memcpy(out_string, param, cur_param_len); + tor_snprintf(out_string+cur_param_len,64, "%ld", (long)median); + smartlist_add(output, out_string); + + i = 0; + if (next_param) { + eq = strchr(next_param, '='); + cur_param_len = (int)(eq+1 - next_param); + } + } + } SMARTLIST_FOREACH_END(param); + + result = smartlist_join_strings(output, " ", 0, NULL); + SMARTLIST_FOREACH(output, char *, cp, tor_free(cp)); + smartlist_free(output); + smartlist_free(param_list); + tor_free(vals); + return result; +} + /** Given a list of vote networkstatus_t in <b>votes</b>, our public * authority <b>identity_key</b>, our private authority <b>signing_key</b>, * and the number of <b>total_authorities</b> that we believe exist in our @@ -489,18 +662,25 @@ networkstatus_compute_consensus(smartlist_t *votes, crypto_pk_env_t *identity_key, crypto_pk_env_t *signing_key, const char *legacy_id_key_digest, - crypto_pk_env_t *legacy_signing_key) + crypto_pk_env_t *legacy_signing_key, + consensus_flavor_t flavor) { smartlist_t *chunks; char *result = NULL; int consensus_method; - time_t valid_after, fresh_until, valid_until; int vote_seconds, dist_seconds; char *client_versions = NULL, *server_versions = NULL; smartlist_t *flags; + const char *flavor_name; + const routerstatus_format_type_t rs_format = + flavor == FLAV_NS ? NS_V3_CONSENSUS : NS_V3_CONSENSUS_MICRODESC; + + tor_assert(flavor == FLAV_NS || flavor == FLAV_MICRODESC); tor_assert(total_authorities >= smartlist_len(votes)); + flavor_name = networkstatus_get_flavor_name(flavor); + if (!smartlist_len(votes)) { log_warn(LD_DIR, "Can't compute a consensus from no votes."); return NULL; @@ -602,8 +782,12 @@ networkstatus_compute_consensus(smartlist_t *votes, format_iso_time(vu_buf, valid_until); flaglist = smartlist_join_strings(flags, " ", 0, NULL); - smartlist_add(chunks, tor_strdup("network-status-version 3\n" - "vote-status consensus\n")); + tor_snprintf(buf, sizeof(buf), "network-status-version 3%s%s\n" + "vote-status consensus\n", + flavor == FLAV_NS ? "" : " ", + flavor == FLAV_NS ? "" : flavor_name); + + smartlist_add(chunks, tor_strdup(buf)); if (consensus_method >= 2) { tor_snprintf(buf, sizeof(buf), "consensus-method %d\n", @@ -627,13 +811,21 @@ networkstatus_compute_consensus(smartlist_t *votes, tor_free(flaglist); } + if (consensus_method >= MIN_METHOD_FOR_PARAMS) { + char *params = dirvote_compute_params(votes); + if (params) { + smartlist_add(chunks, tor_strdup("params ")); + smartlist_add(chunks, params); + smartlist_add(chunks, tor_strdup("\n")); + } + } + /* Sort the votes. */ smartlist_sort(votes, _compare_votes_by_authority_id); /* Add the authority sections. */ { smartlist_t *dir_sources = smartlist_create(); - SMARTLIST_FOREACH(votes, networkstatus_t *, v, - { + SMARTLIST_FOREACH_BEGIN(votes, networkstatus_t *, v) { dir_src_ent_t *e = tor_malloc_zero(sizeof(dir_src_ent_t)); e->v = v; e->digest = get_voter(v)->identity_digest; @@ -647,7 +839,7 @@ networkstatus_compute_consensus(smartlist_t *votes, e_legacy->is_legacy = 1; smartlist_add(dir_sources, e_legacy); } - }); + } SMARTLIST_FOREACH_END(v); smartlist_sort(dir_sources, _compare_dir_src_ents_by_authority_id); SMARTLIST_FOREACH(dir_sources, const dir_src_ent_t *, e, @@ -701,7 +893,10 @@ networkstatus_compute_consensus(smartlist_t *votes, smartlist_t *versions = smartlist_create(); smartlist_t *exitsummaries = smartlist_create(); uint32_t *bandwidths = tor_malloc(sizeof(uint32_t) * smartlist_len(votes)); + uint32_t *measured_bws = tor_malloc(sizeof(uint32_t) * + smartlist_len(votes)); int num_bandwidths; + int num_mbws; int *n_voter_flags; /* n_voter_flags[j] is the number of flags that * votes[j] knows about. */ @@ -817,6 +1012,7 @@ networkstatus_compute_consensus(smartlist_t *votes, int n_listing = 0; int i; char buf[256]; + char microdesc_digest[DIGEST256_LEN]; /* Of the next-to-be-considered digest in each voter, which is first? */ SMARTLIST_FOREACH(votes, networkstatus_t *, v, { @@ -835,6 +1031,7 @@ networkstatus_compute_consensus(smartlist_t *votes, smartlist_clear(chosen_flags); smartlist_clear(versions); num_bandwidths = 0; + num_mbws = 0; /* Okay, go through all the entries for this digest. */ SMARTLIST_FOREACH_BEGIN(votes, networkstatus_t *, v) { @@ -868,6 +1065,9 @@ networkstatus_compute_consensus(smartlist_t *votes, } /* count bandwidths */ + if (rs->status.has_measured_bw) + measured_bws[num_mbws++] = rs->status.measured_bw; + if (rs->status.has_bandwidth) bandwidths[num_bandwidths++] = rs->status.bandwidth; } SMARTLIST_FOREACH_END(v); @@ -879,7 +1079,9 @@ networkstatus_compute_consensus(smartlist_t *votes, /* Figure out the most popular opinion of what the most recent * routerinfo and its contents are. */ - rs = compute_routerstatus_consensus(matching_descs); + memset(microdesc_digest, 0, sizeof(microdesc_digest)); + rs = compute_routerstatus_consensus(matching_descs, consensus_method, + microdesc_digest); /* Copy bits of that into rs_out. */ tor_assert(!memcmp(lowest_id, rs->status.identity_digest, DIGEST_LEN)); memcpy(rs_out.identity_digest, lowest_id, DIGEST_LEN); @@ -945,7 +1147,10 @@ networkstatus_compute_consensus(smartlist_t *votes, } /* Pick a bandwidth */ - if (consensus_method >= 5 && num_bandwidths > 0) { + if (consensus_method >= 6 && num_mbws > 2) { + rs_out.has_bandwidth = 1; + rs_out.bandwidth = median_uint32(measured_bws, num_mbws); + } else if (consensus_method >= 5 && num_bandwidths > 0) { rs_out.has_bandwidth = 1; rs_out.bandwidth = median_uint32(bandwidths, num_bandwidths); } @@ -1036,9 +1241,20 @@ networkstatus_compute_consensus(smartlist_t *votes, /* Okay!! Now we can write the descriptor... */ /* First line goes into "buf". */ - routerstatus_format_entry(buf, sizeof(buf), &rs_out, NULL, 1, 0); + routerstatus_format_entry(buf, sizeof(buf), &rs_out, NULL, + rs_format); smartlist_add(chunks, tor_strdup(buf)); - /* Second line is all flags. The "\n" is missing. */ + /* Now an m line, if applicable. */ + if (flavor == FLAV_MICRODESC && + !tor_digest256_is_zero(microdesc_digest)) { + char m[BASE64_DIGEST256_LEN+1], *cp; + const size_t mlen = BASE64_DIGEST256_LEN+5; + digest256_to_base64(m, microdesc_digest); + cp = tor_malloc(mlen); + tor_snprintf(cp, mlen, "m %s\n", m); + smartlist_add(chunks, cp); + } + /* Next line is all flags. The "\n" is missing. */ smartlist_add(chunks, smartlist_join_strings(chosen_flags, " ", 0, NULL)); /* Now the version line. */ @@ -1055,10 +1271,12 @@ networkstatus_compute_consensus(smartlist_t *votes, log_warn(LD_BUG, "Not enough space in buffer for weight line."); *buf = '\0'; } + smartlist_add(chunks, tor_strdup(buf)); }; + /* Now the exitpolicy summary line. */ - if (rs_out.has_exitsummary) { + if (rs_out.has_exitsummary && flavor == FLAV_NS) { char buf[MAX_POLICY_LINE_LEN+1]; int r = tor_snprintf(buf, sizeof(buf), "p %s\n", rs_out.exitsummary); if (r<0) { @@ -1087,29 +1305,41 @@ networkstatus_compute_consensus(smartlist_t *votes, smartlist_free(versions); smartlist_free(exitsummaries); tor_free(bandwidths); + tor_free(measured_bws); } /* Add a signature. */ { - char digest[DIGEST_LEN]; + char digest[DIGEST256_LEN]; char fingerprint[HEX_DIGEST_LEN+1]; char signing_key_fingerprint[HEX_DIGEST_LEN+1]; + digest_algorithm_t digest_alg = + flavor == FLAV_NS ? DIGEST_SHA1 : DIGEST_SHA256; + size_t digest_len = + flavor == FLAV_NS ? DIGEST_LEN : DIGEST256_LEN; + const char *algname = crypto_digest_algorithm_get_name(digest_alg); char buf[4096]; smartlist_add(chunks, tor_strdup("directory-signature ")); /* Compute the hash of the chunks. */ - hash_list_members(digest, chunks); + hash_list_members(digest, digest_len, chunks, digest_alg); /* Get the fingerprints */ crypto_pk_get_fingerprint(identity_key, fingerprint, 0); crypto_pk_get_fingerprint(signing_key, signing_key_fingerprint, 0); /* add the junk that will go at the end of the line. */ - tor_snprintf(buf, sizeof(buf), "%s %s\n", fingerprint, - signing_key_fingerprint); + if (flavor == FLAV_NS) { + tor_snprintf(buf, sizeof(buf), "%s %s\n", fingerprint, + signing_key_fingerprint); + } else { + tor_snprintf(buf, sizeof(buf), "%s %s %s\n", + algname, fingerprint, + signing_key_fingerprint); + } /* And the signature. */ - if (router_append_dirobj_signature(buf, sizeof(buf), digest, + if (router_append_dirobj_signature(buf, sizeof(buf), digest, digest_len, signing_key)) { log_warn(LD_BUG, "Couldn't sign consensus networkstatus."); return NULL; /* This leaks, but it should never happen. */ @@ -1122,9 +1352,15 @@ networkstatus_compute_consensus(smartlist_t *votes, legacy_id_key_digest, DIGEST_LEN); crypto_pk_get_fingerprint(legacy_signing_key, signing_key_fingerprint, 0); - tor_snprintf(buf, sizeof(buf), "%s %s\n", fingerprint, - signing_key_fingerprint); - if (router_append_dirobj_signature(buf, sizeof(buf), digest, + if (flavor == FLAV_NS) { + tor_snprintf(buf, sizeof(buf), "%s %s\n", fingerprint, + signing_key_fingerprint); + } else { + tor_snprintf(buf, sizeof(buf), "%s %s %s\n", + algname, fingerprint, + signing_key_fingerprint); + } + if (router_append_dirobj_signature(buf, sizeof(buf), digest, digest_len, legacy_signing_key)) { log_warn(LD_BUG, "Couldn't sign consensus networkstatus."); return NULL; /* This leaks, but it should never happen. */ @@ -1171,10 +1407,14 @@ networkstatus_add_detached_signatures(networkstatus_t *target, const char **msg_out) { int r = 0; + const char *flavor; + smartlist_t *siglist; tor_assert(sigs); tor_assert(target); tor_assert(target->type == NS_TYPE_CONSENSUS); + flavor = networkstatus_get_flavor_name(target->flavor); + /* Do the times seem right? */ if (target->valid_after != sigs->valid_after) { *msg_out = "Valid-After times do not match " @@ -1191,79 +1431,179 @@ networkstatus_add_detached_signatures(networkstatus_t *target, "when adding detached signatures to consensus"; return -1; } - /* Are they the same consensus? */ - if (memcmp(target->networkstatus_digest, sigs->networkstatus_digest, - DIGEST_LEN)) { - *msg_out = "Digest mismatch when adding detached signatures to consensus"; + siglist = strmap_get(sigs->signatures, flavor); + if (!siglist) { + *msg_out = "No signatures for given consensus flavor"; return -1; } - /* For each voter in src... */ - SMARTLIST_FOREACH_BEGIN(sigs->signatures, networkstatus_voter_info_t *, - src_voter) { - char voter_identity[HEX_DIGEST_LEN+1]; - networkstatus_voter_info_t *target_voter = - networkstatus_get_voter_by_id(target, src_voter->identity_digest); - authority_cert_t *cert = NULL; - - base16_encode(voter_identity, sizeof(voter_identity), - src_voter->identity_digest, DIGEST_LEN); - log_info(LD_DIR, "Looking at signature from %s", voter_identity); - /* If the target doesn't know about this voter, then forget it. */ - if (!target_voter) { - log_info(LD_DIR, "We do not know about %s", voter_identity); - continue; + /** Make sure all the digests we know match, and at least one matches. */ + { + digests_t *digests = strmap_get(sigs->digests, flavor); + int n_matches = 0; + digest_algorithm_t alg; + if (!digests) { + *msg_out = "No digests for given consensus flavor"; + return -1; + } + for (alg = DIGEST_SHA1; alg < N_DIGEST_ALGORITHMS; ++alg) { + if (!tor_mem_is_zero(digests->d[alg], DIGEST256_LEN)) { + if (!memcmp(target->digests.d[alg], digests->d[alg], DIGEST256_LEN)) { + ++n_matches; + } else { + *msg_out = "Mismatched digest."; + return -1; + } } + } + if (!n_matches) { + *msg_out = "No regognized digests for given consensus flavor"; + } + } - /* If the target already has a good signature from this voter, then skip - * this one. */ - if (target_voter->good_signature) { - log_info(LD_DIR, "We already have a good signature from %s", - voter_identity); - continue; - } + /* For each voter in src... */ + SMARTLIST_FOREACH_BEGIN(siglist, document_signature_t *, sig) { + char voter_identity[HEX_DIGEST_LEN+1]; + networkstatus_voter_info_t *target_voter = + networkstatus_get_voter_by_id(target, sig->identity_digest); + authority_cert_t *cert = NULL; + const char *algorithm; + document_signature_t *old_sig = NULL; + + algorithm = crypto_digest_algorithm_get_name(sig->alg); + + base16_encode(voter_identity, sizeof(voter_identity), + sig->identity_digest, DIGEST_LEN); + log_info(LD_DIR, "Looking at signature from %s using %s", voter_identity, + algorithm); + /* If the target doesn't know about this voter, then forget it. */ + if (!target_voter) { + log_info(LD_DIR, "We do not know any voter with ID %s", voter_identity); + continue; + } - /* Try checking the signature if we haven't already. */ - if (!src_voter->good_signature && !src_voter->bad_signature) { - cert = authority_cert_get_by_digests(src_voter->identity_digest, - src_voter->signing_key_digest); - if (cert) { - networkstatus_check_voter_signature(target, src_voter, cert); - } + old_sig = voter_get_sig_by_algorithm(target_voter, sig->alg); + + /* If the target already has a good signature from this voter, then skip + * this one. */ + if (old_sig && old_sig->good_signature) { + log_info(LD_DIR, "We already have a good signature from %s using %s", + voter_identity, algorithm); + continue; + } + + /* Try checking the signature if we haven't already. */ + if (!sig->good_signature && !sig->bad_signature) { + cert = authority_cert_get_by_digests(sig->identity_digest, + sig->signing_key_digest); + if (cert) + networkstatus_check_document_signature(target, sig, cert); + } + + /* If this signature is good, or we don't have any signature yet, + * then maybe add it. */ + if (sig->good_signature || !old_sig || old_sig->bad_signature) { + log_info(LD_DIR, "Adding signature from %s with %s", voter_identity, + algorithm); + ++r; + if (old_sig) { + smartlist_remove(target_voter->sigs, old_sig); + document_signature_free(old_sig); } + smartlist_add(target_voter->sigs, document_signature_dup(sig)); + } else { + log_info(LD_DIR, "Not adding signature from %s", voter_identity); + } + } SMARTLIST_FOREACH_END(sig); + + return r; +} + +/** Return a newly allocated string containing all the signatures on + * <b>consensus</b> by all voters. If <b>for_detached_signatures</b> is true, + * then the signatures will be put in a detached signatures document, so + * prefix any non-NS-flavored signatures with "additional-signature" rather + * than "directory-signature". */ +static char * +networkstatus_format_signatures(networkstatus_t *consensus, + int for_detached_signatures) +{ + smartlist_t *elements; + char buf[4096]; + char *result = NULL; + int n_sigs = 0; + const consensus_flavor_t flavor = consensus->flavor; + const char *flavor_name = networkstatus_get_flavor_name(flavor); + const char *keyword; + + if (for_detached_signatures && flavor != FLAV_NS) + keyword = "additional-signature"; + else + keyword = "directory-signature"; - /* If this signature is good, or we don't have any signature yet, - * then add it. */ - if (src_voter->good_signature || !target_voter->signature) { - log_info(LD_DIR, "Adding signature from %s", voter_identity); - ++r; - tor_free(target_voter->signature); - target_voter->signature = - tor_memdup(src_voter->signature, src_voter->signature_len); - memcpy(target_voter->signing_key_digest, src_voter->signing_key_digest, - DIGEST_LEN); - target_voter->signature_len = src_voter->signature_len; - target_voter->good_signature = src_voter->good_signature; - target_voter->bad_signature = src_voter->bad_signature; + elements = smartlist_create(); + + SMARTLIST_FOREACH_BEGIN(consensus->voters, networkstatus_voter_info_t *, v) { + SMARTLIST_FOREACH_BEGIN(v->sigs, document_signature_t *, sig) { + char sk[HEX_DIGEST_LEN+1]; + char id[HEX_DIGEST_LEN+1]; + if (!sig->signature || sig->bad_signature) + continue; + ++n_sigs; + base16_encode(sk, sizeof(sk), sig->signing_key_digest, DIGEST_LEN); + base16_encode(id, sizeof(id), sig->identity_digest, DIGEST_LEN); + if (flavor == FLAV_NS) { + tor_snprintf(buf, sizeof(buf), + "%s %s %s\n-----BEGIN SIGNATURE-----\n", + keyword, id, sk); } else { - log_info(LD_DIR, "Not adding signature from %s", voter_identity); + const char *digest_name = + crypto_digest_algorithm_get_name(sig->alg); + tor_snprintf(buf, sizeof(buf), + "%s%s%s %s %s %s\n-----BEGIN SIGNATURE-----\n", + keyword, + for_detached_signatures ? " " : "", + for_detached_signatures ? flavor_name : "", + digest_name, id, sk); } - } SMARTLIST_FOREACH_END(src_voter); + smartlist_add(elements, tor_strdup(buf)); + base64_encode(buf, sizeof(buf), sig->signature, sig->signature_len); + strlcat(buf, "-----END SIGNATURE-----\n", sizeof(buf)); + smartlist_add(elements, tor_strdup(buf)); + } SMARTLIST_FOREACH_END(sig); + } SMARTLIST_FOREACH_END(v); - return r; + result = smartlist_join_strings(elements, "", 0, NULL); + SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp)); + smartlist_free(elements); + if (!n_sigs) + tor_free(result); + return result; } /** Return a newly allocated string holding the detached-signatures document - * corresponding to the signatures on <b>consensus</b>. */ + * corresponding to the signatures on <b>consensuses</b>, which must contain + * exactly one FLAV_NS consensus, and no more than one consensus for each + * other flavor. */ char * -networkstatus_get_detached_signatures(networkstatus_t *consensus) +networkstatus_get_detached_signatures(smartlist_t *consensuses) { smartlist_t *elements; char buf[4096]; - char *result = NULL; - int n_sigs = 0; - tor_assert(consensus); - tor_assert(consensus->type == NS_TYPE_CONSENSUS); + char *result = NULL, *sigs = NULL; + networkstatus_t *consensus_ns = NULL; + tor_assert(consensuses); + + SMARTLIST_FOREACH(consensuses, networkstatus_t *, ns, { + tor_assert(ns); + tor_assert(ns->type == NS_TYPE_CONSENSUS); + if (ns && ns->flavor == FLAV_NS) + consensus_ns = ns; + }); + if (!consensus_ns) { + log_warn(LD_BUG, "No NS consensus given."); + return NULL; + } elements = smartlist_create(); @@ -1272,10 +1612,11 @@ networkstatus_get_detached_signatures(networkstatus_t *consensus) vu_buf[ISO_TIME_LEN+1]; char d[HEX_DIGEST_LEN+1]; - base16_encode(d, sizeof(d), consensus->networkstatus_digest, DIGEST_LEN); - format_iso_time(va_buf, consensus->valid_after); - format_iso_time(fu_buf, consensus->fresh_until); - format_iso_time(vu_buf, consensus->valid_until); + base16_encode(d, sizeof(d), + consensus_ns->digests.d[DIGEST_SHA1], DIGEST_LEN); + format_iso_time(va_buf, consensus_ns->valid_after); + format_iso_time(fu_buf, consensus_ns->fresh_until); + format_iso_time(vu_buf, consensus_ns->valid_until); tor_snprintf(buf, sizeof(buf), "consensus-digest %s\n" @@ -1285,45 +1626,89 @@ networkstatus_get_detached_signatures(networkstatus_t *consensus) smartlist_add(elements, tor_strdup(buf)); } - SMARTLIST_FOREACH(consensus->voters, networkstatus_voter_info_t *, v, - { - char sk[HEX_DIGEST_LEN+1]; - char id[HEX_DIGEST_LEN+1]; - if (!v->signature || v->bad_signature) + /* Get all the digests for the non-FLAV_NS consensuses */ + SMARTLIST_FOREACH_BEGIN(consensuses, networkstatus_t *, ns) { + const char *flavor_name = networkstatus_get_flavor_name(ns->flavor); + int alg; + if (ns->flavor == FLAV_NS) + continue; + + /* start with SHA256; we don't include SHA1 for anything but the basic + * consensus. */ + for (alg = DIGEST_SHA256; alg < N_DIGEST_ALGORITHMS; ++alg) { + char d[HEX_DIGEST256_LEN+1]; + const char *alg_name = + crypto_digest_algorithm_get_name(alg); + if (tor_mem_is_zero(ns->digests.d[alg], DIGEST256_LEN)) continue; - ++n_sigs; - base16_encode(sk, sizeof(sk), v->signing_key_digest, DIGEST_LEN); - base16_encode(id, sizeof(id), v->identity_digest, DIGEST_LEN); - tor_snprintf(buf, sizeof(buf), - "directory-signature %s %s\n-----BEGIN SIGNATURE-----\n", - id, sk); - smartlist_add(elements, tor_strdup(buf)); - base64_encode(buf, sizeof(buf), v->signature, v->signature_len); - strlcat(buf, "-----END SIGNATURE-----\n", sizeof(buf)); + base16_encode(d, sizeof(d), ns->digests.d[alg], DIGEST256_LEN); + tor_snprintf(buf, sizeof(buf), "additional-digest %s %s %s\n", + flavor_name, alg_name, d); smartlist_add(elements, tor_strdup(buf)); - }); + } + } SMARTLIST_FOREACH_END(ns); + + /* Now get all the sigs for non-FLAV_NS consensuses */ + SMARTLIST_FOREACH_BEGIN(consensuses, networkstatus_t *, ns) { + char *sigs; + if (ns->flavor == FLAV_NS) + continue; + sigs = networkstatus_format_signatures(ns, 1); + if (!sigs) { + log_warn(LD_DIR, "Couldn't format signatures"); + goto err; + } + smartlist_add(elements, sigs); + } SMARTLIST_FOREACH_END(ns); - result = smartlist_join_strings(elements, "", 0, NULL); + /* Now add the FLAV_NS consensus signatrures. */ + sigs = networkstatus_format_signatures(consensus_ns, 1); + if (!sigs) + goto err; + smartlist_add(elements, sigs); + result = smartlist_join_strings(elements, "", 0, NULL); + err: SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp)); smartlist_free(elements); - if (!n_sigs) - tor_free(result); return result; } +/** Return a newly allocated string holding a detached-signatures document for + * all of the in-progress consensuses in the <b>n_flavors</b>-element array at + * <b>pending</b>. */ +static char * +get_detached_signatures_from_pending_consensuses(pending_consensus_t *pending, + int n_flavors) +{ + int flav; + char *signatures; + smartlist_t *c = smartlist_create(); + for (flav = 0; flav < n_flavors; ++flav) { + if (pending[flav].consensus) + smartlist_add(c, pending[flav].consensus); + } + signatures = networkstatus_get_detached_signatures(c); + smartlist_free(c); + return signatures; +} + /** Release all storage held in <b>s</b>. */ void ns_detached_signatures_free(ns_detached_signatures_t *s) { + if (!s) + return; if (s->signatures) { - SMARTLIST_FOREACH(s->signatures, networkstatus_voter_info_t *, v, - { - tor_free(v->signature); - tor_free(v); - }); - smartlist_free(s->signatures); + STRMAP_FOREACH(s->signatures, flavor, smartlist_t *, sigs) { + SMARTLIST_FOREACH(sigs, document_signature_t *, sig, + document_signature_free(sig)); + smartlist_free(sigs); + } STRMAP_FOREACH_END; + strmap_free(s->signatures, NULL); + strmap_free(s->digests, _tor_free); } + tor_free(s); } @@ -1513,7 +1898,7 @@ dirvote_act(or_options_t *options, time_t now) if (voting_schedule.voting_ends < now && !voting_schedule.have_built_consensus) { log_notice(LD_DIR, "Time to compute a consensus."); - dirvote_compute_consensus(); + dirvote_compute_consensuses(); /* XXXX We will want to try again later if we haven't got enough * votes yet. Implement this if it turns out to ever happen. */ voting_schedule.have_built_consensus = 1; @@ -1550,14 +1935,13 @@ static smartlist_t *pending_vote_list = NULL; /** List of pending_vote_t for the previous vote. After we've used them to * build a consensus, the votes go here for the next period. */ static smartlist_t *previous_vote_list = NULL; -/** The body of the consensus that we're currently building. Once we - * have it built, it goes into dirserv.c */ -static char *pending_consensus_body = NULL; + +static pending_consensus_t pending_consensuses[N_CONSENSUS_FLAVORS]; + /** The detached signatures for the consensus that we're currently * building. */ static char *pending_consensus_signatures = NULL; -/** The parsed in-progress consensus document. */ -static networkstatus_t *pending_consensus = NULL; + /** List of ns_detached_signatures_t: hold signatures that get posted to us * before we have generated the consensus on our own. */ static smartlist_t *pending_consensus_signature_list = NULL; @@ -1651,15 +2035,39 @@ dirvote_fetch_missing_votes(void) static void dirvote_fetch_missing_signatures(void) { - if (!pending_consensus) + int need_any = 0; + int i; + for (i=0; i < N_CONSENSUS_FLAVORS; ++i) { + networkstatus_t *consensus = pending_consensuses[i].consensus; + if (!consensus || + networkstatus_check_consensus_signature(consensus, -1) == 1) { + /* We have no consensus, or we have one that's signed by everybody. */ + continue; + } + need_any = 1; + } + if (!need_any) return; - if (networkstatus_check_consensus_signature(pending_consensus, -1) == 1) - return; /* we have a signature from everybody. */ directory_get_from_all_authorities(DIR_PURPOSE_FETCH_DETACHED_SIGNATURES, 0, NULL); } +/** Release all storage held by pending consensuses (those waiting for + * signatures). */ +static void +dirvote_clear_pending_consensuses(void) +{ + int i; + for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) { + pending_consensus_t *pc = &pending_consensuses[i]; + tor_free(pc->body); + + networkstatus_vote_free(pc->consensus); + pc->consensus = NULL; + } +} + /** Drop all currently pending votes, consensus, and detached signatures. */ static void dirvote_clear_votes(int all_votes) @@ -1697,12 +2105,8 @@ dirvote_clear_votes(int all_votes) tor_free(cp)); smartlist_clear(pending_consensus_signature_list); } - tor_free(pending_consensus_body); tor_free(pending_consensus_signatures); - if (pending_consensus) { - networkstatus_vote_free(pending_consensus); - pending_consensus = NULL; - } + dirvote_clear_pending_consensuses(); } /** Return a newly allocated string containing the hex-encoded v3 authority @@ -1760,7 +2164,13 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out) } tor_assert(smartlist_len(vote->voters) == 1); vi = get_voter(vote); - tor_assert(vi->good_signature == 1); + { + int any_sig_good = 0; + SMARTLIST_FOREACH(vi->sigs, document_signature_t *, sig, + if (sig->good_signature) + any_sig_good = 1); + tor_assert(any_sig_good); + } ds = trusteddirserver_get_by_v3_auth_digest(vi->identity_digest); if (!ds) { char *keys = list_v3_auth_ids(); @@ -1856,8 +2266,7 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out) *status_out = 400; discard: - if (vote) - networkstatus_vote_free(vote); + networkstatus_vote_free(vote); if (end_of_vote && !strcmpstart(end_of_vote, "network-status-version ")) { vote_body = end_of_vote; @@ -1884,14 +2293,18 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out) * pending_consensus: it won't be ready to be published until we have * everybody else's signatures collected too. (V3 Authority only) */ static int -dirvote_compute_consensus(void) +dirvote_compute_consensuses(void) { /* Have we got enough votes to try? */ - int n_votes, n_voters; + int n_votes, n_voters, n_vote_running = 0; smartlist_t *votes = NULL, *votestrings = NULL; char *consensus_body = NULL, *signatures = NULL, *votefile; networkstatus_t *consensus = NULL; authority_cert_t *my_cert; + pending_consensus_t pending[N_CONSENSUS_FLAVORS]; + int flav; + + memset(pending, 0, sizeof(pending)); if (!pending_vote_list) pending_vote_list = smartlist_create(); @@ -1903,6 +2316,19 @@ dirvote_compute_consensus(void) "%d of %d", n_votes, n_voters/2); goto err; } + tor_assert(pending_vote_list); + SMARTLIST_FOREACH(pending_vote_list, pending_vote_t *, v, { + if (smartlist_string_isin(v->vote->known_flags, "Running")) + n_vote_running++; + }); + if (!n_vote_running) { + /* See task 1066. */ + log_warn(LD_DIR, "Nobody has voted on the Running flag. Generating " + "and publishing a consensus without Running nodes " + "would make many clients stop working. Not " + "generating a consensus!"); + goto err; + } if (!(my_cert = get_my_v3_authority_cert())) { log_warn(LD_DIR, "Can't generate consensus without a certificate."); @@ -1931,6 +2357,7 @@ dirvote_compute_consensus(void) char legacy_dbuf[DIGEST_LEN]; crypto_pk_env_t *legacy_sign=NULL; char *legacy_id_digest = NULL; + int n_generated = 0; if (get_options()->V3AuthUseLegacyKey) { authority_cert_t *cert = get_my_v3_legacy_cert(); legacy_sign = get_my_v3_legacy_signing_key(); @@ -1939,39 +2366,58 @@ dirvote_compute_consensus(void) legacy_id_digest = legacy_dbuf; } } - consensus_body = networkstatus_compute_consensus( + + for (flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) { + const char *flavor_name = networkstatus_get_flavor_name(flav); + consensus_body = networkstatus_compute_consensus( votes, n_voters, my_cert->identity_key, - get_my_v3_authority_signing_key(), legacy_id_digest, legacy_sign); - } - if (!consensus_body) { - log_warn(LD_DIR, "Couldn't generate a consensus at all!"); - goto err; - } - consensus = networkstatus_parse_vote_from_string(consensus_body, NULL, - NS_TYPE_CONSENSUS); - if (!consensus) { - log_warn(LD_DIR, "Couldn't parse consensus we generated!"); - goto err; + get_my_v3_authority_signing_key(), legacy_id_digest, legacy_sign, + flav); + + if (!consensus_body) { + log_warn(LD_DIR, "Couldn't generate a %s consensus at all!", + flavor_name); + continue; + } + consensus = networkstatus_parse_vote_from_string(consensus_body, NULL, + NS_TYPE_CONSENSUS); + if (!consensus) { + log_warn(LD_DIR, "Couldn't parse %s consensus we generated!", + flavor_name); + tor_free(consensus_body); + continue; + } + + /* 'Check' our own signature, to mark it valid. */ + networkstatus_check_consensus_signature(consensus, -1); + + pending[flav].body = consensus_body; + pending[flav].consensus = consensus; + n_generated++; + consensus_body = NULL; + consensus = NULL; + } + if (!n_generated) { + log_warn(LD_DIR, "Couldn't generate any consensus flavors at all."); + goto err; + } } - /* 'Check' our own signature, to mark it valid. */ - networkstatus_check_consensus_signature(consensus, -1); - signatures = networkstatus_get_detached_signatures(consensus); + signatures = get_detached_signatures_from_pending_consensuses( + pending, N_CONSENSUS_FLAVORS); + if (!signatures) { log_warn(LD_DIR, "Couldn't extract signatures."); goto err; } - tor_free(pending_consensus_body); - pending_consensus_body = consensus_body; + dirvote_clear_pending_consensuses(); + memcpy(pending_consensuses, pending, sizeof(pending)); + tor_free(pending_consensus_signatures); pending_consensus_signatures = signatures; - if (pending_consensus) - networkstatus_vote_free(pending_consensus); - pending_consensus = consensus; - if (pending_consensus_signature_list) { int n_sigs = 0; /* we may have gotten signatures for this consensus before we built @@ -1979,7 +2425,7 @@ dirvote_compute_consensus(void) SMARTLIST_FOREACH(pending_consensus_signature_list, char *, sig, { const char *msg = NULL; - int r = dirvote_add_signatures_to_pending_consensus(sig, &msg); + int r = dirvote_add_signatures_to_all_pending_consensuses(sig, &msg); if (r >= 0) n_sigs += r; else @@ -2005,8 +2451,7 @@ dirvote_compute_consensus(void) return 0; err: - if (votes) - smartlist_free(votes); + smartlist_free(votes); tor_free(consensus_body); tor_free(signatures); networkstatus_vote_free(consensus); @@ -2015,75 +2460,60 @@ dirvote_compute_consensus(void) } /** Helper: we just got the <b>detached_signatures_body</b> sent to us as - * signatures on the currently pending consensus. Add them to the consensus + * signatures on the currently pending consensus. Add them to <b>pc</b> * as appropriate. Return the number of signatures added. (?) */ static int dirvote_add_signatures_to_pending_consensus( - const char *detached_signatures_body, + pending_consensus_t *pc, + ns_detached_signatures_t *sigs, const char **msg_out) { - ns_detached_signatures_t *sigs = NULL; + const char *flavor_name; int r = -1; - tor_assert(detached_signatures_body); - tor_assert(msg_out); - /* Only call if we have a pending consensus right now. */ - tor_assert(pending_consensus); - tor_assert(pending_consensus_body); + tor_assert(pc->consensus); + tor_assert(pc->body); tor_assert(pending_consensus_signatures); + flavor_name = networkstatus_get_flavor_name(pc->consensus->flavor); *msg_out = NULL; - if (!(sigs = networkstatus_parse_detached_signatures( - detached_signatures_body, NULL))) { - *msg_out = "Couldn't parse detached signatures."; - goto err; + { + smartlist_t *sig_list = strmap_get(sigs->signatures, flavor_name); + log_info(LD_DIR, "Have %d signatures for adding to %s consensus.", + sig_list ? smartlist_len(sig_list) : 0, flavor_name); } - - log_info(LD_DIR, "Have %d signatures for adding to consensus.", - smartlist_len(sigs->signatures)); - r = networkstatus_add_detached_signatures(pending_consensus, - sigs, msg_out); + r = networkstatus_add_detached_signatures(pc->consensus, sigs, msg_out); log_info(LD_DIR,"Added %d signatures to consensus.", r); if (r >= 1) { - char *new_detached = - networkstatus_get_detached_signatures(pending_consensus); - const char *src; + char *new_signatures = + networkstatus_format_signatures(pc->consensus, 0); char *dst, *dst_end; size_t new_consensus_len; - if (!new_detached) { + if (!new_signatures) { *msg_out = "No signatures to add"; goto err; } new_consensus_len = - strlen(pending_consensus_body) + strlen(new_detached) + 1; - pending_consensus_body = tor_realloc(pending_consensus_body, - new_consensus_len); - dst_end = pending_consensus_body + new_consensus_len; - dst = strstr(pending_consensus_body, "directory-signature "); + strlen(pc->body) + strlen(new_signatures) + 1; + pc->body = tor_realloc(pc->body, new_consensus_len); + dst_end = pc->body + new_consensus_len; + dst = strstr(pc->body, "directory-signature "); tor_assert(dst); - src = strstr(new_detached, "directory-signature "); - tor_assert(src); - strlcpy(dst, src, dst_end-dst); + strlcpy(dst, new_signatures, dst_end-dst); /* We remove this block once it has failed to crash for a while. But * unless it shows up in profiles, we're probably better leaving it in, * just in case we break detached signature processing at some point. */ { - ns_detached_signatures_t *sigs = - networkstatus_parse_detached_signatures(new_detached, NULL); networkstatus_t *v = networkstatus_parse_vote_from_string( - pending_consensus_body, NULL, + pc->body, NULL, NS_TYPE_CONSENSUS); - tor_assert(sigs); - ns_detached_signatures_free(sigs); tor_assert(v); networkstatus_vote_free(v); } - tor_free(pending_consensus_signatures); - pending_consensus_signatures = new_detached; *msg_out = "Signatures added"; } else if (r == 0) { *msg_out = "Signatures ignored"; @@ -2096,8 +2526,62 @@ dirvote_add_signatures_to_pending_consensus( if (!*msg_out) *msg_out = "Unrecognized error while adding detached signatures."; done: - if (sigs) - ns_detached_signatures_free(sigs); + return r; +} + +static int +dirvote_add_signatures_to_all_pending_consensuses( + const char *detached_signatures_body, + const char **msg_out) +{ + int r=0, i, n_added = 0, errors = 0; + ns_detached_signatures_t *sigs; + tor_assert(detached_signatures_body); + tor_assert(msg_out); + tor_assert(pending_consensus_signatures); + + if (!(sigs = networkstatus_parse_detached_signatures( + detached_signatures_body, NULL))) { + *msg_out = "Couldn't parse detached signatures."; + goto err; + } + + for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) { + int res; + pending_consensus_t *pc = &pending_consensuses[i]; + if (!pc->consensus) + continue; + res = dirvote_add_signatures_to_pending_consensus(pc, sigs, msg_out); + if (res < 0) + errors++; + else + n_added += res; + } + + if (errors && !n_added) { + r = -1; + goto err; + } + + if (n_added && pending_consensuses[FLAV_NS].consensus) { + char *new_detached = + get_detached_signatures_from_pending_consensuses( + pending_consensuses, N_CONSENSUS_FLAVORS); + if (new_detached) { + tor_free(pending_consensus_signatures); + pending_consensus_signatures = new_detached; + } + } + + r = n_added; + goto done; + err: + if (!*msg_out) + *msg_out = "Unrecognized error while adding detached signatures."; + done: + ns_detached_signatures_free(sigs); + /* XXXX NM Check how return is used. We can now have an error *and* + signatures added. */ return r; } @@ -2110,10 +2594,10 @@ dirvote_add_signatures(const char *detached_signatures_body, const char *source, const char **msg) { - if (pending_consensus) { + if (pending_consensuses[FLAV_NS].consensus) { log_notice(LD_DIR, "Got a signature from %s. " "Adding it to the pending consensus.", source); - return dirvote_add_signatures_to_pending_consensus( + return dirvote_add_signatures_to_all_pending_consensuses( detached_signatures_body, msg); } else { log_notice(LD_DIR, "Got a signature from %s. " @@ -2132,17 +2616,25 @@ dirvote_add_signatures(const char *detached_signatures_body, static int dirvote_publish_consensus(void) { - /* Can we actually publish it yet? */ - if (!pending_consensus || - networkstatus_check_consensus_signature(pending_consensus, 1)<0) { - log_warn(LD_DIR, "Not enough info to publish pending consensus"); - return -1; - } + int i; + + /* Now remember all the other consensuses as if we were a directory cache. */ + for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) { + pending_consensus_t *pending = &pending_consensuses[i]; + const char *name; + name = networkstatus_get_flavor_name(i); + tor_assert(name); + if (!pending->consensus || + networkstatus_check_consensus_signature(pending->consensus, 1)<0) { + log_warn(LD_DIR, "Not enough info to publish pending %s consensus",name); + continue; + } - if (networkstatus_set_current_consensus(pending_consensus_body, 0)) - log_warn(LD_DIR, "Error publishing consensus"); - else - log_notice(LD_DIR, "Consensus published."); + if (networkstatus_set_current_consensus(pending->body, name, 0)) + log_warn(LD_DIR, "Error publishing %s consensus", name); + else + log_notice(LD_DIR, "Published %s consensus", name); + } return 0; } @@ -2158,12 +2650,8 @@ dirvote_free_all(void) smartlist_free(previous_vote_list); previous_vote_list = NULL; - tor_free(pending_consensus_body); + dirvote_clear_pending_consensuses(); tor_free(pending_consensus_signatures); - if (pending_consensus) { - networkstatus_vote_free(pending_consensus); - pending_consensus = NULL; - } if (pending_consensus_signature_list) { /* now empty as a result of clear_pending_votes. */ smartlist_free(pending_consensus_signature_list); @@ -2177,13 +2665,14 @@ dirvote_free_all(void) /** Return the body of the consensus that we're currently trying to build. */ const char * -dirvote_get_pending_consensus(void) +dirvote_get_pending_consensus(consensus_flavor_t flav) { - return pending_consensus_body; + tor_assert(((int)flav) >= 0 && flav < N_CONSENSUS_FLAVORS); + return pending_consensuses[flav].body; } /** Return the signatures that we know for the consensus that we're currently - * trying to build */ + * trying to build. */ const char * dirvote_get_pending_detached_signatures(void) { @@ -2229,15 +2718,147 @@ dirvote_get_vote(const char *fp, int flags) } else { if (pending_vote_list && include_pending) { SMARTLIST_FOREACH(pending_vote_list, pending_vote_t *, pv, - if (!memcmp(pv->vote->networkstatus_digest, fp, DIGEST_LEN)) + if (!memcmp(pv->vote->digests.d[DIGEST_SHA1], fp, DIGEST_LEN)) return pv->vote_body); } if (previous_vote_list && include_previous) { SMARTLIST_FOREACH(previous_vote_list, pending_vote_t *, pv, - if (!memcmp(pv->vote->networkstatus_digest, fp, DIGEST_LEN)) + if (!memcmp(pv->vote->digests.d[DIGEST_SHA1], fp, DIGEST_LEN)) return pv->vote_body); } } return NULL; } +/** Construct and return a new microdescriptor from a routerinfo <b>ri</b>. + * + * XXX Right now, there is only one way to generate microdescriptors from + * router descriptors. This may change in future consensus methods. If so, + * we'll need an internal way to remember which method we used, and ask for a + * particular method. + **/ +microdesc_t * +dirvote_create_microdescriptor(const routerinfo_t *ri) +{ + microdesc_t *result = NULL; + char *key = NULL, *summary = NULL, *family = NULL; + char buf[1024]; + size_t keylen; + char *out = buf, *end = buf+sizeof(buf); + + if (crypto_pk_write_public_key_to_string(ri->onion_pkey, &key, &keylen)<0) + goto done; + summary = policy_summarize(ri->exit_policy); + if (ri->declared_family) + family = smartlist_join_strings(ri->declared_family, " ", 0, NULL); + + if (tor_snprintf(out, end-out, "onion-key\n%s", key)<0) + goto done; + out += strlen(out); + if (family) { + if (tor_snprintf(out, end-out, "family %s\n", family)<0) + goto done; + out += strlen(out); + } + if (summary && strcmp(summary, "reject 1-65535")) { + if (tor_snprintf(out, end-out, "p %s\n", summary)<0) + goto done; + out += strlen(out); + } + *out = '\0'; /* Make sure it's nul-terminated. This should be a no-op */ + + { + smartlist_t *lst = microdescs_parse_from_string(buf, out, 0, 1); + if (smartlist_len(lst) != 1) { + log_warn(LD_DIR, "We generated a microdescriptor we couldn't parse."); + SMARTLIST_FOREACH(lst, microdesc_t *, md, microdesc_free(md)); + smartlist_free(lst); + goto done; + } + result = smartlist_get(lst, 0); + smartlist_free(lst); + } + + done: + tor_free(key); + tor_free(summary); + tor_free(family); + return result; +} + +/** Cached space-separated string to hold */ +static char *microdesc_consensus_methods = NULL; + +/** Format the appropriate vote line to describe the microdescriptor <b>md</b> + * in a consensus vote document. Write it into the <b>out_len</b>-byte buffer + * in <b>out</b>. Return -1 on failure and the number of characters written + * on success. */ +ssize_t +dirvote_format_microdesc_vote_line(char *out, size_t out_len, + const microdesc_t *md) +{ + char d64[BASE64_DIGEST256_LEN+1]; + if (!microdesc_consensus_methods) { + microdesc_consensus_methods = + make_consensus_method_list(MIN_METHOD_FOR_MICRODESC, + MAX_SUPPORTED_CONSENSUS_METHOD, + ","); + tor_assert(microdesc_consensus_methods); + } + if (digest256_to_base64(d64, md->digest)<0) + return -1; + + if (tor_snprintf(out, out_len, "m %s sha256=%s\n", + microdesc_consensus_methods, d64)<0) + return -1; + + return strlen(out); +} + +/** If <b>vrs</b> has a hash made for the consensus method <b>method</b> with + * the digest algorithm <b>alg</b>, decode it and copy it into + * <b>digest256_out</b> and return 0. Otherwise return -1. */ +int +vote_routerstatus_find_microdesc_hash(char *digest256_out, + const vote_routerstatus_t *vrs, + int method, + digest_algorithm_t alg) +{ + /* XXXX only returns the sha256 method. */ + const vote_microdesc_hash_t *h; + char mstr[64]; + size_t mlen; + char dstr[64]; + + tor_snprintf(mstr, sizeof(mstr), "%d", method); + mlen = strlen(mstr); + tor_snprintf(dstr, sizeof(dstr), " %s=", + crypto_digest_algorithm_get_name(alg)); + + for (h = vrs->microdesc; h; h = h->next) { + const char *cp = h->microdesc_hash_line; + size_t num_len; + /* cp looks like \d+(,\d+)* (digesttype=val )+ . Let's hunt for mstr in + * the first part. */ + while (1) { + num_len = strspn(cp, "1234567890"); + if (num_len == mlen && !memcmp(mstr, cp, mlen)) { + /* This is the line. */ + char buf[BASE64_DIGEST256_LEN+1]; + /* XXXX ignores extraneous stuff if the digest is too long. This + * seems harmless enough, right? */ + cp = strstr(cp, dstr); + if (!cp) + return -1; + cp += strlen(dstr); + strlcpy(buf, cp, sizeof(buf)); + return digest256_from_base64(digest256_out, buf); + } + if (num_len == 0 || cp[num_len] != ',') + break; + cp += num_len + 1; + } + } + return -1; +} + diff --git a/src/or/dns.c b/src/or/dns.c index 74852b0f70..08bff763a3 100644 --- a/src/or/dns.c +++ b/src/or/dns.c @@ -13,7 +13,43 @@ #include "or.h" #include "ht.h" +#ifdef HAVE_EVENT2_DNS_H +#include <event2/event.h> +#include <event2/dns.h> +#else +#include <event.h> #include "eventdns.h" +#ifndef HAVE_EVDNS_SET_DEFAULT_OUTGOING_BIND_ADDRESS +#define HAVE_EVDNS_SET_DEFAULT_OUTGOING_BIND_ADDRESS +#endif +#endif + +#ifndef HAVE_EVENT2_DNS_H +struct evdns_base; +struct evdns_request; +#define evdns_base_new(x,y) tor_malloc(1) +#define evdns_base_clear_nameservers_and_suspend(base) \ + evdns_clear_nameservers_and_suspend() +#define evdns_base_search_clear(base) evdns_search_clear() +#define evdns_base_set_default_outgoing_bind_address(base, a, len) \ + evdns_set_default_outgoing_bind_address((a),(len)) +#define evdns_base_resolv_conf_parse(base, options, fname) \ + evdns_resolv_conf_parse((options), (fname)) +#define evdns_base_count_nameservers(base) \ + evdns_count_nameservers() +#define evdns_base_resume(base) \ + evdns_resume() +#define evdns_base_config_windows_nameservers(base) \ + evdns_config_windows_nameservers() +#define evdns_base_set_option(base, opt, val, flags) \ + evdns_set_option((opt),(val),(flags)) +#define evdns_base_resolve_ipv4(base, addr, options, cb, ptr) \ + ((evdns_resolve_ipv4(addr, options, cb, ptr)<0) ? NULL : ((void*)1)) +#define evdns_base_resolve_reverse(base, addr, options, cb, ptr) \ + ((evdns_resolve_reverse(addr, options, cb, ptr)<0) ? NULL : ((void*)1)) +#define evdns_base_resolve_reverse_ipv6(base, addr, options, cb, ptr) \ + ((evdns_resolve_reverse_ipv6(addr, options, cb, ptr)<0) ? NULL : ((void*)1)) +#endif /** Longest hostname we're willing to resolve. */ #define MAX_ADDRESSLEN 256 @@ -28,6 +64,9 @@ #define DNS_RESOLVE_FAILED_PERMANENT 2 #define DNS_RESOLVE_SUCCEEDED 3 +/** Our evdns_base; this structure handles all our name lookups. */ +static struct evdns_base *the_evdns_base = NULL; + /** Have we currently configured nameservers with eventdns? */ static int nameservers_configured = 0; /** Did our most recent attempt to configure nameservers with eventdns fail? */ @@ -89,6 +128,8 @@ typedef struct cached_resolve_t { uint32_t ttl; /**< What TTL did the nameserver tell us? */ /** Connections that want to know when we get an answer for this resolve. */ pending_connection_t *pending_connections; + /** Position of this element in the heap*/ + int minheap_idx; } cached_resolve_t; static void purge_expired_resolves(time_t now); @@ -211,8 +252,8 @@ dns_reset(void) { or_options_t *options = get_options(); if (! server_mode(options)) { - evdns_clear_nameservers_and_suspend(); - evdns_search_clear(); + evdns_base_clear_nameservers_and_suspend(the_evdns_base); + evdns_base_search_clear(the_evdns_base); nameservers_configured = 0; tor_free(resolv_conf_fname); resolv_conf_mtime = 0; @@ -262,6 +303,8 @@ dns_get_expiry_ttl(uint32_t ttl) static void _free_cached_resolve(cached_resolve_t *r) { + if (!r) + return; while (r->pending_connections) { pending_connection_t *victim = r->pending_connections; r->pending_connections = victim->next; @@ -303,6 +346,7 @@ set_expiry(cached_resolve_t *resolve, time_t expires) resolve->expire = expires; smartlist_pqueue_add(cached_resolve_pqueue, _compare_cached_resolves_by_expiry, + STRUCT_OFFSET(cached_resolve_t, minheap_idx), resolve); } @@ -325,8 +369,7 @@ dns_free_all(void) _free_cached_resolve(item); } HT_CLEAR(cache_map, &cache_root); - if (cached_resolve_pqueue) - smartlist_free(cached_resolve_pqueue); + smartlist_free(cached_resolve_pqueue); cached_resolve_pqueue = NULL; tor_free(resolv_conf_fname); } @@ -349,7 +392,8 @@ purge_expired_resolves(time_t now) if (resolve->expire > now) break; smartlist_pqueue_pop(cached_resolve_pqueue, - _compare_cached_resolves_by_expiry); + _compare_cached_resolves_by_expiry, + STRUCT_OFFSET(cached_resolve_t, minheap_idx)); if (resolve->state == CACHE_STATE_PENDING) { log_debug(LD_EXIT, @@ -711,6 +755,7 @@ dns_resolve_impl(edge_connection_t *exitconn, int is_resolve, resolve = tor_malloc_zero(sizeof(cached_resolve_t)); resolve->magic = CACHED_RESOLVE_MAGIC; resolve->state = CACHE_STATE_PENDING; + resolve->minheap_idx = -1; resolve->is_reverse = is_reverse; strlcpy(resolve->address, exitconn->_base.address, sizeof(resolve->address)); @@ -807,7 +852,8 @@ connection_dns_remove(edge_connection_t *conn) tor_free(pend); log_debug(LD_EXIT, "First connection (fd %d) no longer waiting " "for resolve of %s", - conn->_base.s, escaped_safe_str(conn->_base.address)); + conn->_base.s, + escaped_safe_str(conn->_base.address)); return; } else { for ( ; pend->next; pend = pend->next) { @@ -1108,6 +1154,14 @@ configure_nameservers(int force) conf_fname = "/etc/resolv.conf"; #endif + if (!the_evdns_base) { + if (!(the_evdns_base = evdns_base_new(tor_libevent_get_base(), 0))) { + log_err(LD_BUG, "Couldn't create an evdns_base"); + return -1; + } + } + +#ifdef HAVE_EVDNS_SET_DEFAULT_OUTGOING_BIND_ADDRESS if (options->OutboundBindAddress) { tor_addr_t addr; if (tor_addr_from_str(&addr, options->OutboundBindAddress) < 0) { @@ -1122,16 +1176,13 @@ configure_nameservers(int force) log_warn(LD_BUG, "Couldn't convert outbound bind address to sockaddr." " Ignoring."); } else { - evdns_set_default_outgoing_bind_address((struct sockaddr *)&ss, - socklen); + evdns_base_set_default_outgoing_bind_address(the_evdns_base, + (struct sockaddr *)&ss, + socklen); } } } - - if (options->ServerDNSRandomizeCase) - evdns_set_option("randomize-case:", "1", DNS_OPTIONS_ALL); - else - evdns_set_option("randomize-case:", "0", DNS_OPTIONS_ALL); +#endif evdns_set_log_fn(evdns_log_cb); if (conf_fname) { @@ -1146,16 +1197,17 @@ configure_nameservers(int force) return 0; } if (nameservers_configured) { - evdns_search_clear(); - evdns_clear_nameservers_and_suspend(); + evdns_base_search_clear(the_evdns_base); + evdns_base_clear_nameservers_and_suspend(the_evdns_base); } log_info(LD_EXIT, "Parsing resolver configuration in '%s'", conf_fname); - if ((r = evdns_resolv_conf_parse(DNS_OPTIONS_ALL, conf_fname))) { + if ((r = evdns_base_resolv_conf_parse(the_evdns_base, + DNS_OPTIONS_ALL, conf_fname))) { log_warn(LD_EXIT, "Unable to parse '%s', or no nameservers in '%s' (%d)", conf_fname, conf_fname, r); goto err; } - if (evdns_count_nameservers() == 0) { + if (evdns_base_count_nameservers(the_evdns_base) == 0) { log_warn(LD_EXIT, "Unable to find any nameservers in '%s'.", conf_fname); goto err; } @@ -1163,38 +1215,48 @@ configure_nameservers(int force) resolv_conf_fname = tor_strdup(conf_fname); resolv_conf_mtime = st.st_mtime; if (nameservers_configured) - evdns_resume(); + evdns_base_resume(the_evdns_base); } #ifdef MS_WINDOWS else { if (nameservers_configured) { - evdns_search_clear(); - evdns_clear_nameservers_and_suspend(); + evdns_base_search_clear(the_evdns_base); + evdns_base_clear_nameservers_and_suspend(the_evdns_base); } - if (evdns_config_windows_nameservers()) { + if (evdns_base_config_windows_nameservers(the_evdns_base)) { log_warn(LD_EXIT,"Could not config nameservers."); goto err; } - if (evdns_count_nameservers() == 0) { + if (evdns_base_count_nameservers(the_evdns_base) == 0) { log_warn(LD_EXIT, "Unable to find any platform nameservers in " "your Windows configuration."); goto err; } if (nameservers_configured) - evdns_resume(); + evdns_base_resume(the_evdns_base); tor_free(resolv_conf_fname); resolv_conf_mtime = 0; } #endif - if (evdns_count_nameservers() == 1) { - evdns_set_option("max-timeouts:", "16", DNS_OPTIONS_ALL); - evdns_set_option("timeout:", "10", DNS_OPTIONS_ALL); +#define SET(k,v) \ + evdns_base_set_option(the_evdns_base, (k), (v), DNS_OPTIONS_ALL) + + if (evdns_base_count_nameservers(the_evdns_base) == 1) { + SET("max-timeouts:", "16"); + SET("timeout:", "10"); } else { - evdns_set_option("max-timeouts:", "3", DNS_OPTIONS_ALL); - evdns_set_option("timeout:", "5", DNS_OPTIONS_ALL); + SET("max-timeouts:", "3"); + SET("timeout:", "5"); } + if (options->ServerDNSRandomizeCase) + SET("randomize-case:", "1"); + else + SET("randomize-case:", "0"); + +#undef SET + dns_servers_relaunch_checks(); nameservers_configured = 1; @@ -1292,6 +1354,7 @@ static int launch_resolve(edge_connection_t *exitconn) { char *addr = tor_strdup(exitconn->_base.address); + struct evdns_request *req = NULL; tor_addr_t a; int r; int options = get_options()->ServerDNSSearchDomains ? 0 @@ -1310,25 +1373,29 @@ launch_resolve(edge_connection_t *exitconn) if (r == 0) { log_info(LD_EXIT, "Launching eventdns request for %s", escaped_safe_str(exitconn->_base.address)); - r = evdns_resolve_ipv4(exitconn->_base.address, options, - evdns_callback, addr); + req = evdns_base_resolve_ipv4(the_evdns_base, + exitconn->_base.address, options, + evdns_callback, addr); } else if (r == 1) { log_info(LD_EXIT, "Launching eventdns reverse request for %s", escaped_safe_str(exitconn->_base.address)); if (tor_addr_family(&a) == AF_INET) - r = evdns_resolve_reverse(tor_addr_to_in(&a), DNS_QUERY_NO_SEARCH, + req = evdns_base_resolve_reverse(the_evdns_base, + tor_addr_to_in(&a), DNS_QUERY_NO_SEARCH, evdns_callback, addr); else - r = evdns_resolve_reverse_ipv6(tor_addr_to_in6(&a), DNS_QUERY_NO_SEARCH, + req = evdns_base_resolve_reverse_ipv6(the_evdns_base, + tor_addr_to_in6(&a), DNS_QUERY_NO_SEARCH, evdns_callback, addr); } else if (r == -1) { log_warn(LD_BUG, "Somehow a malformed in-addr.arpa address reached here."); } - if (r) { - log_warn(LD_EXIT, "eventdns rejected address %s: error %d.", - escaped_safe_str(addr), r); - r = evdns_err_is_transient(r) ? -2 : -1; + r = 0; + if (!req) { + log_warn(LD_EXIT, "eventdns rejected address %s.", + escaped_safe_str(addr)); + r = -1; tor_free(addr); /* There is no evdns request in progress; stop * addr from getting leaked. */ } @@ -1449,8 +1516,8 @@ evdns_wildcard_check_callback(int result, char type, int count, int ttl, } log(dns_wildcard_one_notice_given ? LOG_INFO : LOG_NOTICE, LD_EXIT, "Your DNS provider gave an answer for \"%s\", which " - "is not supposed to exist. Apparently they are hijacking " - "DNS failures. Trying to correct for this. We've noticed %d " + "is not supposed to exist. Apparently they are hijacking " + "DNS failures. Trying to correct for this. We've noticed %d " "possibly bad address%s so far.", string_address, strmap_size(dns_wildcard_response_count), (strmap_size(dns_wildcard_response_count) == 1) ? "" : "es"); @@ -1466,17 +1533,19 @@ static void launch_wildcard_check(int min_len, int max_len, const char *suffix) { char *addr; - int r; + struct evdns_request *req; addr = crypto_random_hostname(min_len, max_len, "", suffix); log_info(LD_EXIT, "Testing whether our DNS server is hijacking nonexistent " "domains with request for bogus hostname \"%s\"", addr); - r = evdns_resolve_ipv4(/* This "addr" tells us which address to resolve */ + req = evdns_base_resolve_ipv4( + the_evdns_base, + /* This "addr" tells us which address to resolve */ addr, DNS_QUERY_NO_SEARCH, evdns_wildcard_check_callback, /* This "addr" is an argument to the callback*/ addr); - if (r) { + if (!req) { /* There is no evdns request in progress; stop addr from getting leaked */ tor_free(addr); } @@ -1488,6 +1557,7 @@ static void launch_test_addresses(int fd, short event, void *args) { or_options_t *options = get_options(); + struct evdns_request *req; (void)fd; (void)event; (void)args; @@ -1499,14 +1569,18 @@ launch_test_addresses(int fd, short event, void *args) * be an exit server.*/ if (!options->ServerDNSTestAddresses) return; - SMARTLIST_FOREACH(options->ServerDNSTestAddresses, const char *, address, - { - int r = evdns_resolve_ipv4(address, DNS_QUERY_NO_SEARCH, evdns_callback, - tor_strdup(address)); - if (r) - log_info(LD_EXIT, "eventdns rejected test address %s: error %d", - escaped_safe_str(address), r); - }); + SMARTLIST_FOREACH_BEGIN(options->ServerDNSTestAddresses, + const char *, address) { + char *a = tor_strdup(address); + req = evdns_base_resolve_ipv4(the_evdns_base, + address, DNS_QUERY_NO_SEARCH, evdns_callback, a); + + if (!req) { + log_info(LD_EXIT, "eventdns rejected test address %s", + escaped_safe_str(address)); + tor_free(a); + } + } SMARTLIST_FOREACH_END(address); } #define N_WILDCARD_CHECKS 2 @@ -1547,7 +1621,7 @@ dns_launch_wildcard_checks(void) void dns_launch_correctness_checks(void) { - static struct event launch_event; + static struct event *launch_event = NULL; struct timeval timeout; if (!get_options()->ServerDNSDetectHijacking) return; @@ -1555,10 +1629,12 @@ dns_launch_correctness_checks(void) /* Wait a while before launching requests for test addresses, so we can * get the results from checking for wildcarding. */ - evtimer_set(&launch_event, launch_test_addresses, NULL); + if (! launch_event) + launch_event = tor_evtimer_new(tor_libevent_get_base(), + launch_test_addresses, NULL); timeout.tv_sec = 30; timeout.tv_usec = 0; - if (evtimer_add(&launch_event, &timeout)<0) { + if (evtimer_add(launch_event, &timeout)<0) { log_warn(LD_BUG, "Couldn't add timer for checking for dns hijacking"); } } @@ -1574,10 +1650,9 @@ dns_seems_to_be_broken(void) void dns_reset_correctness_checks(void) { - if (dns_wildcard_response_count) { - strmap_free(dns_wildcard_response_count, _tor_free); - dns_wildcard_response_count = NULL; - } + strmap_free(dns_wildcard_response_count, _tor_free); + dns_wildcard_response_count = NULL; + n_wildcard_requests = 0; if (dns_wildcard_list) { @@ -1622,6 +1697,30 @@ assert_resolve_ok(cached_resolve_t *resolve) } } +/** Return the number of DNS cache entries as an int */ +static int +dns_cache_entry_count(void) +{ + return HT_SIZE(&cache_root); +} + +/** Log memory information about our internal DNS cache at level 'severity'. */ +void +dump_dns_mem_usage(int severity) +{ + /* This should never be larger than INT_MAX. */ + int hash_count = dns_cache_entry_count(); + size_t hash_mem = sizeof(struct cached_resolve_t) * hash_count; + hash_mem += HT_MEM_USAGE(&cache_root); + + /* Print out the count and estimated size of our &cache_root. It undercounts + hostnames in cached reverse resolves. + */ + log(severity, LD_MM, "Our DNS cache has %d entries.", hash_count); + log(severity, LD_MM, "Our DNS cache size is approximately %u bytes.", + (unsigned)hash_mem); +} + #ifdef DEBUG_DNS_CACHE /** Exit with an assertion if the DNS cache is corrupt. */ static void @@ -1642,7 +1741,8 @@ _assert_cache_ok(void) return; smartlist_pqueue_assert_ok(cached_resolve_pqueue, - _compare_cached_resolves_by_expiry); + _compare_cached_resolves_by_expiry, + STRUCT_OFFSET(cached_resolve_t, minheap_idx)); SMARTLIST_FOREACH(cached_resolve_pqueue, cached_resolve_t *, res, { diff --git a/src/or/dnsserv.c b/src/or/dnsserv.c index 6020f9a781..256dcbd75b 100644 --- a/src/or/dnsserv.c +++ b/src/or/dnsserv.c @@ -9,7 +9,14 @@ **/ #include "or.h" +#ifdef HAVE_EVENT2_DNS_H +#include <event2/dns.h> +#include <event2/dns_compat.h> +/* XXXX022 this implies we want an improved evdns */ +#include <event2/dns_struct.h> +#else #include "eventdns.h" +#endif /** Helper function: called by evdns whenever the client sends a request to our * DNSPort. We need to eventually answer the request <b>req</b>. @@ -85,12 +92,7 @@ evdns_server_callback(struct evdns_server_request *req, void *_data) evdns_server_request_respond(req, DNS_ERR_NONE); return; } - if (q->type == EVDNS_TYPE_A) { - /* Refuse any attempt to resolve a noconnect address, right now. */ - if (hostname_is_noconnect_address(q->name)) { - err = DNS_ERR_REFUSED; - } - } else { + if (q->type != EVDNS_TYPE_A) { tor_assert(q->type == EVDNS_TYPE_PTR); } @@ -136,13 +138,13 @@ evdns_server_callback(struct evdns_server_request *req, void *_data) * immediately if it's in the cache, or completely bogus, or automapped), * and then attached to a circuit. */ log_info(LD_APP, "Passing request for %s to rewrite_and_attach.", - escaped_safe_str(q->name)); + escaped_safe_str_client(q->name)); q_name = tor_strdup(q->name); /* q could be freed in rewrite_and_attach */ connection_ap_handshake_rewrite_and_attach(conn, NULL, NULL); /* Now, the connection is marked if it was bad. */ log_info(LD_APP, "Passed request for %s to rewrite_and_attach.", - escaped_safe_str(q_name)); + escaped_safe_str_client(q_name)); tor_free(q_name); } @@ -181,13 +183,13 @@ dnsserv_launch_request(const char *name, int reverse) * immediately if it's in the cache, or completely bogus, or automapped), * and then attached to a circuit. */ log_info(LD_APP, "Passing request for %s to rewrite_and_attach.", - escaped_safe_str(name)); + escaped_safe_str_client(name)); q_name = tor_strdup(name); /* q could be freed in rewrite_and_attach */ connection_ap_handshake_rewrite_and_attach(conn, NULL, NULL); /* Now, the connection is marked if it was bad. */ log_info(LD_APP, "Passed request for %s to rewrite_and_attach.", - escaped_safe_str(q_name)); + escaped_safe_str_client(q_name)); tor_free(q_name); return 0; } diff --git a/src/or/eventdns.c b/src/or/eventdns.c index a889e803ed..a504224c54 100644 --- a/src/or/eventdns.c +++ b/src/or/eventdns.c @@ -31,6 +31,7 @@ */ #include "eventdns_tor.h" +#include "../common/util.h" #include <sys/types.h> /* #define NDEBUG */ @@ -89,6 +90,7 @@ #include <stdarg.h> #include "eventdns.h" + #ifdef WIN32 #include <windows.h> #include <winsock2.h> @@ -173,8 +175,6 @@ struct evdns_request { /* these objects are kept in a circular list */ struct evdns_request *next, *prev; - u16 timeout_event_deleted; /**< Debugging: where was timeout_event - * deleted? 0 for "it's added." */ struct event timeout_event; u16 trans_id; /* the transaction id */ @@ -214,8 +214,6 @@ struct nameserver { struct event event; /* these objects are kept in a circular list */ struct nameserver *next, *prev; - u16 timeout_event_deleted; /**< Debugging: where was timeout_event - * deleted? 0 for "it's added." */ struct event timeout_event; /* used to keep the timeout for */ /* when we next probe this server. */ /* Valid if state == 0 */ @@ -474,51 +472,10 @@ sockaddr_eq(const struct sockaddr *sa1, const struct sockaddr *sa2, return 1; } -/* for debugging bug 929. XXXX022 */ -static int -_add_timeout_event(u16 *lineno, struct event *ev, struct timeval *to) -{ - *lineno = 0; - return evtimer_add(ev, to); -} -#define add_timeout_event(s, to) \ - (_add_timeout_event(&(s)->timeout_event_deleted, &(s)->timeout_event, (to))) - -/* for debugging bug 929. XXXX022 */ -static int -_del_timeout_event(u16 *lineno, struct event *ev, int line) -{ - if (*lineno) { - log(EVDNS_LOG_DEBUG, - "Duplicate timeout event_del from line %d: first call " - "was at %d.", line, (int)*lineno); - return 0; - } else { - *lineno = (u16)line; - return event_del(ev); - } -} -#define del_timeout_event(s) \ - (_del_timeout_event(&(s)->timeout_event_deleted, &(s)->timeout_event, \ - __LINE__)) -/* For debugging bug 929/957. XXXX022 */ -static int -_del_timeout_event_if_set(u16 *lineno, struct event *ev, int line) -{ - if (*lineno == 0) { - log(EVDNS_LOG_DEBUG, - "Event that I thought was non-added as of line %d " - "was actually added on line %d", - line, (int)*lineno); - *lineno = line; - return event_del(ev); - } - return 0; -} -#define del_timeout_event_if_set(s) \ - _del_timeout_event_if_set(&(s)->timeout_event_deleted, \ - &(s)->timeout_event, \ - __LINE__) +#define add_timeout_event(s, to) \ + (event_add(&(s)->timeout_event, (to))) +#define del_timeout_event(s) \ + (event_del(&(s)->timeout_event)) /* This walks the list of inflight requests to find the */ /* one with a matching transaction id. Returns NULL on */ @@ -555,7 +512,7 @@ static void nameserver_probe_failed(struct nameserver *const ns) { const struct timeval * timeout; del_timeout_event(ns); - CLEAR(&ns->timeout_event); + if (ns->state == 1) { /* This can happen if the nameserver acts in a way which makes us mark */ /* it as bad and then starts sending good replies. */ @@ -567,8 +524,6 @@ nameserver_probe_failed(struct nameserver *const ns) { global_nameserver_timeouts_length - 1)]; ns->failed_times++; - del_timeout_event_if_set(ns); - evtimer_set(&ns->timeout_event, nameserver_prod_callback, ns); if (add_timeout_event(ns, (struct timeval *) timeout) < 0) { log(EVDNS_LOG_WARN, "Error from libevent when adding timer event for %s", @@ -597,8 +552,6 @@ nameserver_failed(struct nameserver *const ns, const char *msg) { ns->state = 0; ns->failed_times = 1; - del_timeout_event_if_set(ns); - evtimer_set(&ns->timeout_event, nameserver_prod_callback, ns); if (add_timeout_event(ns, (struct timeval *) &global_nameserver_timeouts[0]) < 0) { log(EVDNS_LOG_WARN, "Error from libevent when adding timer event for %s", @@ -634,7 +587,6 @@ nameserver_up(struct nameserver *const ns) { log(EVDNS_LOG_WARN, "Nameserver %s is back up", debug_ntop((struct sockaddr *)&ns->address)); del_timeout_event(ns); - CLEAR(&ns->timeout_event); ns->state = 1; ns->failed_times = 0; ns->timedout = 0; @@ -666,7 +618,6 @@ request_finished(struct evdns_request *const req, struct evdns_request **head) { log(EVDNS_LOG_DEBUG, "Removing timeout for request %lx", (unsigned long) req); del_timeout_event(req); - CLEAR(&req->timeout_event); search_request_finished(req); global_requests_inflight--; @@ -2046,7 +1997,6 @@ evdns_request_timeout_callback(int fd, short events, void *arg) { * request_finished; that one already deletes the timeout event. * XXXX021 port this change to libevent. */ del_timeout_event(req); - CLEAR(&req->timeout_event); evdns_request_transmit(req); } } @@ -2109,8 +2059,7 @@ evdns_request_transmit(struct evdns_request *req) { /* transmitted; we need to check for timeout. */ log(EVDNS_LOG_DEBUG, "Setting timeout for request %lx", (unsigned long) req); - del_timeout_event_if_set(req); - evtimer_set(&req->timeout_event, evdns_request_timeout_callback, req); + if (add_timeout_event(req, &global_timeout) < 0) { log(EVDNS_LOG_WARN, "Error from libevent when adding timer for request %lx", @@ -2225,7 +2174,6 @@ evdns_clear_nameservers_and_suspend(void) (void) event_del(&server->event); CLEAR(&server->event); del_timeout_event(server); - CLEAR(&server->timeout_event); if (server->socket >= 0) CLOSE_SOCKET(server->socket); CLEAR(server); @@ -2243,7 +2191,6 @@ evdns_clear_nameservers_and_suspend(void) req->ns = NULL; /* ???? What to do about searches? */ del_timeout_event(req); - CLEAR(&req->timeout_event); req->trans_id = 0; req->transmit_me = 0; @@ -2292,6 +2239,21 @@ evdns_resume(void) } static int +sockaddr_is_loopback(const struct sockaddr *addr) +{ + static const char LOOPBACK_S6[16] = + "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\1"; + if (addr->sa_family == AF_INET) { + struct sockaddr_in *sin = (struct sockaddr_in *)addr; + return (ntohl(sin->sin_addr.s_addr) & 0xff000000) == 0x7f000000; + } else if (addr->sa_family == AF_INET6) { + struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)addr; + return !memcmp(sin6->sin6_addr.s6_addr, LOOPBACK_S6, 16); + } + return 0; +} + +static int _evdns_nameserver_add_impl(const struct sockaddr *address, socklen_t addrlen) { /* first check to see if we already have this nameserver */ @@ -2318,7 +2280,8 @@ _evdns_nameserver_add_impl(const struct sockaddr *address, if (!ns) return -1; memset(ns, 0, sizeof(struct nameserver)); - ns->timeout_event_deleted = __LINE__; + + evtimer_set(&ns->timeout_event, nameserver_prod_callback, ns); ns->socket = socket(PF_INET, SOCK_DGRAM, 0); if (ns->socket < 0) { err = 1; goto out1; } @@ -2331,7 +2294,8 @@ _evdns_nameserver_add_impl(const struct sockaddr *address, fcntl(ns->socket, F_SETFL, O_NONBLOCK); #endif - if (global_bind_addr_is_set) { + if (global_bind_addr_is_set && + !sockaddr_is_loopback((struct sockaddr*)&global_bind_address)) { if (bind(ns->socket, (struct sockaddr *)&global_bind_address, global_bind_addrlen) < 0) { log(EVDNS_LOG_DEBUG, "Couldn't bind to outgoing address."); @@ -2553,7 +2517,8 @@ request_new(int type, const char *name, int flags, } memset(req, 0, sizeof(struct evdns_request)); - req->timeout_event_deleted = __LINE__; + + evtimer_set(&req->timeout_event, evdns_request_timeout_callback, req); if (global_randomize_case) { unsigned i; @@ -2941,14 +2906,6 @@ evdns_resolv_set_defaults(int flags) { if (flags & DNS_OPTION_NAMESERVERS) evdns_nameserver_ip_add("127.0.0.1"); } -#ifndef HAVE_STRTOK_R -static char * -strtok_r(char *s, const char *delim, char **state) { - (void)state; - return strtok(s, delim); -} -#endif - /* helper version of atoi which returns -1 on error */ static int strtoint(const char *const str) { @@ -3025,9 +2982,9 @@ static void resolv_conf_parse_line(char *const start, int flags) { char *strtok_state; static const char *const delims = " \t"; -#define NEXT_TOKEN strtok_r(NULL, delims, &strtok_state) +#define NEXT_TOKEN tor_strtok_r(NULL, delims, &strtok_state) - char *const first_token = strtok_r(start, delims, &strtok_state); + char *const first_token = tor_strtok_r(start, delims, &strtok_state); if (!first_token) return; if (!strcmp(first_token, "nameserver") && (flags & DNS_OPTION_NAMESERVERS)) { @@ -3384,8 +3341,7 @@ evdns_shutdown(int fail_requests) if (server->socket >= 0) CLOSE_SOCKET(server->socket); (void) event_del(&server->event); - if (server->state == 0) - del_timeout_event(server); + del_timeout_event(server); CLEAR(server); mm_free(server); if (server_next == server_head) diff --git a/src/or/geoip.c b/src/or/geoip.c index aabbe26889..a00280e39d 100644 --- a/src/or/geoip.c +++ b/src/or/geoip.c @@ -21,9 +21,9 @@ typedef struct geoip_entry_t { } geoip_entry_t; /** For how many periods should we remember per-country request history? */ -#define REQUEST_HIST_LEN 3 +#define REQUEST_HIST_LEN 1 /** How long are the periods for which we should remember request history? */ -#define REQUEST_HIST_PERIOD (8*60*60) +#define REQUEST_HIST_PERIOD (24*60*60) /** A per-country record for GeoIP request history. */ typedef struct geoip_country_t { @@ -42,7 +42,7 @@ static strmap_t *country_idxplus1_by_lc_code = NULL; static smartlist_t *geoip_entries = NULL; /** Return the index of the <b>country</b>'s entry in the GeoIP DB - * if it is a valid 2-letter country code, otherwise return zero. + * if it is a valid 2-letter country code, otherwise return -1. */ country_t geoip_get_country(const char *country) @@ -186,7 +186,14 @@ geoip_load_file(const char *filename, or_options_t *options) return -1; } if (!geoip_countries) { + geoip_country_t *geoip_unresolved; geoip_countries = smartlist_create(); + /* Add a geoip_country_t for requests that could not be resolved to a + * country as first element (index 0) to geoip_countries. */ + geoip_unresolved = tor_malloc_zero(sizeof(geoip_country_t)); + strlcpy(geoip_unresolved->countrycode, "??", + sizeof(geoip_unresolved->countrycode)); + smartlist_add(geoip_countries, geoip_unresolved); country_idxplus1_by_lc_code = strmap_new(); } if (geoip_entries) { @@ -261,8 +268,8 @@ geoip_is_loaded(void) typedef struct clientmap_entry_t { HT_ENTRY(clientmap_entry_t) node; uint32_t ipaddr; - time_t last_seen; /* The last 2 bits of this value hold the client - * operation. */ + unsigned int last_seen_in_minutes:30; + unsigned int action:2; } clientmap_entry_t; #define ACTION_MASK 3 @@ -289,7 +296,7 @@ clientmap_entry_hash(const clientmap_entry_t *a) static INLINE int clientmap_entries_eq(const clientmap_entry_t *a, const clientmap_entry_t *b) { - return a->ipaddr == b->ipaddr; + return a->ipaddr == b->ipaddr && a->action == b->action; } HT_PROTOTYPE(clientmap, clientmap_entry_t, node, clientmap_entry_hash, @@ -297,8 +304,89 @@ HT_PROTOTYPE(clientmap, clientmap_entry_t, node, clientmap_entry_hash, HT_GENERATE(clientmap, clientmap_entry_t, node, clientmap_entry_hash, clientmap_entries_eq, 0.6, malloc, realloc, free); +/** How often do we update our estimate which share of v2 and v3 directory + * requests is sent to us? We could as well trigger updates of shares from + * network status updates, but that means adding a lot of calls into code + * that is independent from geoip stats (and keeping them up-to-date). We + * are perfectly fine with an approximation of 15-minute granularity. */ +#define REQUEST_SHARE_INTERVAL (15 * 60) + +/** When did we last determine which share of v2 and v3 directory requests + * is sent to us? */ +static time_t last_time_determined_shares = 0; + +/** Sum of products of v2 shares times the number of seconds for which we + * consider these shares as valid. */ +static double v2_share_times_seconds; + +/** Sum of products of v3 shares times the number of seconds for which we + * consider these shares as valid. */ +static double v3_share_times_seconds; + +/** Number of seconds we are determining v2 and v3 shares. */ +static int share_seconds; + +/** Try to determine which fraction of v2 and v3 directory requests aimed at + * caches will be sent to us at time <b>now</b> and store that value in + * order to take a mean value later on. */ +static void +geoip_determine_shares(time_t now) +{ + double v2_share = 0.0, v3_share = 0.0; + if (router_get_my_share_of_directory_requests(&v2_share, &v3_share) < 0) + return; + if (last_time_determined_shares) { + v2_share_times_seconds += v2_share * + ((double) (now - last_time_determined_shares)); + v3_share_times_seconds += v3_share * + ((double) (now - last_time_determined_shares)); + share_seconds += (int)(now - last_time_determined_shares); + } + last_time_determined_shares = now; +} + +/** Calculate which fraction of v2 and v3 directory requests aimed at caches + * have been sent to us since the last call of this function up to time + * <b>now</b>. Set *<b>v2_share_out</b> and *<b>v3_share_out</b> to the + * fractions of v2 and v3 protocol shares we expect to have seen. Reset + * counters afterwards. Return 0 on success, -1 on failure (e.g. when zero + * seconds have passed since the last call).*/ +static int +geoip_get_mean_shares(time_t now, double *v2_share_out, + double *v3_share_out) +{ + geoip_determine_shares(now); + if (!share_seconds) + return -1; + *v2_share_out = v2_share_times_seconds / ((double) share_seconds); + *v3_share_out = v3_share_times_seconds / ((double) share_seconds); + v2_share_times_seconds = v3_share_times_seconds = 0.0; + share_seconds = 0; + return 0; +} + +/* Rotate period of v2 and v3 network status requests. */ +static void +rotate_request_period(void) +{ + SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, { +#if REQUEST_HIST_LEN > 1 + memmove(&c->n_v2_ns_requests[0], &c->n_v2_ns_requests[1], + sizeof(uint32_t)*(REQUEST_HIST_LEN-1)); + memmove(&c->n_v3_ns_requests[0], &c->n_v3_ns_requests[1], + sizeof(uint32_t)*(REQUEST_HIST_LEN-1)); +#endif + c->n_v2_ns_requests[REQUEST_HIST_LEN-1] = 0; + c->n_v3_ns_requests[REQUEST_HIST_LEN-1] = 0; + }); + current_request_period_starts += REQUEST_HIST_PERIOD; + if (n_old_request_periods < REQUEST_HIST_LEN-1) + ++n_old_request_periods; +} + /** Note that we've seen a client connect from the IP <b>addr</b> (host order) - * at time <b>now</b>. Ignored by all but bridges. */ + * at time <b>now</b>. Ignored by all but bridges and directories if + * configured accordingly. */ void geoip_note_client_seen(geoip_client_action_t action, uint32_t addr, time_t now) @@ -306,60 +394,57 @@ geoip_note_client_seen(geoip_client_action_t action, or_options_t *options = get_options(); clientmap_entry_t lookup, *ent; if (action == GEOIP_CLIENT_CONNECT) { - if (!(options->BridgeRelay && options->BridgeRecordUsageByCountry)) + /* Only remember statistics as entry guard or as bridge. */ + if (!options->EntryStatistics && + (!(options->BridgeRelay && options->BridgeRecordUsageByCountry))) return; /* Did we recently switch from bridge to relay or back? */ if (client_history_starts > now) return; } else { -#ifndef ENABLE_GEOIP_STATS - return; -#else if (options->BridgeRelay || options->BridgeAuthoritativeDir || - !options->DirRecordUsageByCountry) + !options->DirReqStatistics) return; -#endif } - /* Rotate the current request period. */ - while (current_request_period_starts + REQUEST_HIST_PERIOD < now) { - if (!geoip_countries) - geoip_countries = smartlist_create(); - if (!current_request_period_starts) { - current_request_period_starts = now; - break; + /* As a bridge that doesn't rotate request periods every 24 hours, + * possibly rotate now. */ + if (options->BridgeRelay) { + while (current_request_period_starts + REQUEST_HIST_PERIOD < now) { + if (!geoip_countries) + geoip_countries = smartlist_create(); + if (!current_request_period_starts) { + current_request_period_starts = now; + break; + } + /* Also discard all items in the client history that are too old. + * (This only works here because bridge and directory stats are + * independent. Otherwise, we'd only want to discard those items + * with action GEOIP_CLIENT_NETWORKSTATUS{_V2}.) */ + geoip_remove_old_clients(current_request_period_starts); + /* Now rotate request period */ + rotate_request_period(); } - SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, { - memmove(&c->n_v2_ns_requests[0], &c->n_v2_ns_requests[1], - sizeof(uint32_t)*(REQUEST_HIST_LEN-1)); - memmove(&c->n_v3_ns_requests[0], &c->n_v3_ns_requests[1], - sizeof(uint32_t)*(REQUEST_HIST_LEN-1)); - c->n_v2_ns_requests[REQUEST_HIST_LEN-1] = 0; - c->n_v3_ns_requests[REQUEST_HIST_LEN-1] = 0; - }); - current_request_period_starts += REQUEST_HIST_PERIOD; - if (n_old_request_periods < REQUEST_HIST_LEN-1) - ++n_old_request_periods; - } - - /* We use the low 3 bits of the time to encode the action. Since we're - * potentially remembering tons of clients, we don't want to make - * clientmap_entry_t larger than it has to be. */ - now = (now & ~ACTION_MASK) | (((int)action) & ACTION_MASK); + } + lookup.ipaddr = addr; + lookup.action = (int)action; ent = HT_FIND(clientmap, &client_history, &lookup); if (ent) { - ent->last_seen = now; + ent->last_seen_in_minutes = now / 60; } else { ent = tor_malloc_zero(sizeof(clientmap_entry_t)); ent->ipaddr = addr; - ent->last_seen = now; + ent->last_seen_in_minutes = now / 60; + ent->action = (int)action; HT_INSERT(clientmap, &client_history, ent); } if (action == GEOIP_CLIENT_NETWORKSTATUS || action == GEOIP_CLIENT_NETWORKSTATUS_V2) { int country_idx = geoip_get_country_by_ip(addr); + if (country_idx < 0) + country_idx = 0; /** unresolved requests are stored at index 0. */ if (country_idx >= 0 && country_idx < smartlist_len(geoip_countries)) { geoip_country_t *country = smartlist_get(geoip_countries, country_idx); if (action == GEOIP_CLIENT_NETWORKSTATUS) @@ -367,6 +452,10 @@ geoip_note_client_seen(geoip_client_action_t action, else ++country->n_v2_ns_requests[REQUEST_HIST_LEN-1]; } + + /* Periodically determine share of requests that we should see */ + if (last_time_determined_shares + REQUEST_SHARE_INTERVAL < now) + geoip_determine_shares(now); } if (!client_history_starts) { @@ -380,8 +469,8 @@ geoip_note_client_seen(geoip_client_action_t action, static int _remove_old_client_helper(struct clientmap_entry_t *ent, void *_cutoff) { - time_t cutoff = *(time_t*)_cutoff; - if (ent->last_seen < cutoff) { + time_t cutoff = *(time_t*)_cutoff / 60; + if (ent->last_seen_in_minutes < cutoff) { tor_free(ent); return 1; } else { @@ -403,6 +492,38 @@ geoip_remove_old_clients(time_t cutoff) client_history_starts = cutoff; } +/** How many responses are we giving to clients requesting v2 network + * statuses? */ +static uint32_t ns_v2_responses[GEOIP_NS_RESPONSE_NUM]; + +/** How many responses are we giving to clients requesting v3 network + * statuses? */ +static uint32_t ns_v3_responses[GEOIP_NS_RESPONSE_NUM]; + +/** Note that we've rejected a client's request for a v2 or v3 network + * status, encoded in <b>action</b> for reason <b>reason</b> at time + * <b>now</b>. */ +void +geoip_note_ns_response(geoip_client_action_t action, + geoip_ns_response_t response) +{ + static int arrays_initialized = 0; + if (!get_options()->DirReqStatistics) + return; + if (!arrays_initialized) { + memset(ns_v2_responses, 0, sizeof(ns_v2_responses)); + memset(ns_v3_responses, 0, sizeof(ns_v3_responses)); + arrays_initialized = 1; + } + tor_assert(action == GEOIP_CLIENT_NETWORKSTATUS || + action == GEOIP_CLIENT_NETWORKSTATUS_V2); + tor_assert(response < GEOIP_NS_RESPONSE_NUM); + if (action == GEOIP_CLIENT_NETWORKSTATUS) + ns_v3_responses[response]++; + else + ns_v2_responses[response]++; +} + /** Do not mention any country from which fewer than this number of IPs have * connected. This conceivably avoids reporting information that could * deanonymize users, though analysis is lacking. */ @@ -442,32 +563,258 @@ _c_hist_compare(const void **_a, const void **_b) return strcmp(a->country, b->country); } -/** How long do we have to have observed per-country request history before we - * are willing to talk about it? */ -#define GEOIP_MIN_OBSERVATION_TIME (12*60*60) +/** When there are incomplete directory requests at the end of a 24-hour + * period, consider those requests running for longer than this timeout as + * failed, the others as still running. */ +#define DIRREQ_TIMEOUT (10*60) -/** Return the lowest x such that x is at least <b>number</b>, and x modulo - * <b>divisor</b> == 0. */ -static INLINE unsigned -round_to_next_multiple_of(unsigned number, unsigned divisor) +/** Entry in a map from either conn->global_identifier for direct requests + * or a unique circuit identifier for tunneled requests to request time, + * response size, and completion time of a network status request. Used to + * measure download times of requests to derive average client + * bandwidths. */ +typedef struct dirreq_map_entry_t { + HT_ENTRY(dirreq_map_entry_t) node; + /** Unique identifier for this network status request; this is either the + * conn->global_identifier of the dir conn (direct request) or a new + * locally unique identifier of a circuit (tunneled request). This ID is + * only unique among other direct or tunneled requests, respectively. */ + uint64_t dirreq_id; + unsigned int state:3; /**< State of this directory request. */ + unsigned int type:1; /**< Is this a direct or a tunneled request? */ + unsigned int completed:1; /**< Is this request complete? */ + unsigned int action:2; /**< Is this a v2 or v3 request? */ + /** When did we receive the request and started sending the response? */ + struct timeval request_time; + size_t response_size; /**< What is the size of the response in bytes? */ + struct timeval completion_time; /**< When did the request succeed? */ +} dirreq_map_entry_t; + +/** Map of all directory requests asking for v2 or v3 network statuses in + * the current geoip-stats interval. Values are + * of type *<b>dirreq_map_entry_t</b>. */ +static HT_HEAD(dirreqmap, dirreq_map_entry_t) dirreq_map = + HT_INITIALIZER(); + +static int +dirreq_map_ent_eq(const dirreq_map_entry_t *a, + const dirreq_map_entry_t *b) { - number += divisor - 1; - number -= number % divisor; - return number; + return a->dirreq_id == b->dirreq_id && a->type == b->type; } -/** Return a newly allocated comma-separated string containing entries for all - * the countries from which we've seen enough clients connect. The entry - * format is cc=num where num is the number of IPs we've seen connecting from - * that country, and cc is a lowercased country code. Returns NULL if we don't - * want to export geoip data yet. */ -char * -geoip_get_client_history(time_t now, geoip_client_action_t action) +static unsigned +dirreq_map_ent_hash(const dirreq_map_entry_t *entry) +{ + unsigned u = (unsigned) entry->dirreq_id; + u += entry->type << 20; + return u; +} + +HT_PROTOTYPE(dirreqmap, dirreq_map_entry_t, node, dirreq_map_ent_hash, + dirreq_map_ent_eq); +HT_GENERATE(dirreqmap, dirreq_map_entry_t, node, dirreq_map_ent_hash, + dirreq_map_ent_eq, 0.6, malloc, realloc, free); + +/** Helper: Put <b>entry</b> into map of directory requests using + * <b>tunneled</b> and <b>dirreq_id</b> as key parts. If there is + * already an entry for that key, print out a BUG warning and return. */ +static void +_dirreq_map_put(dirreq_map_entry_t *entry, dirreq_type_t type, + uint64_t dirreq_id) +{ + dirreq_map_entry_t *old_ent; + tor_assert(entry->type == type); + tor_assert(entry->dirreq_id == dirreq_id); + + /* XXXX022 once we're sure the bug case never happens, we can switch + * to HT_INSERT */ + old_ent = HT_REPLACE(dirreqmap, &dirreq_map, entry); + if (old_ent && old_ent != entry) { + log_warn(LD_BUG, "Error when putting directory request into local " + "map. There was already an entry for the same identifier."); + return; + } +} + +/** Helper: Look up and return an entry in the map of directory requests + * using <b>tunneled</b> and <b>dirreq_id</b> as key parts. If there + * is no such entry, return NULL. */ +static dirreq_map_entry_t * +_dirreq_map_get(dirreq_type_t type, uint64_t dirreq_id) +{ + dirreq_map_entry_t lookup; + lookup.type = type; + lookup.dirreq_id = dirreq_id; + return HT_FIND(dirreqmap, &dirreq_map, &lookup); +} + +/** Note that an either direct or tunneled (see <b>type</b>) directory + * request for a network status with unique ID <b>dirreq_id</b> of size + * <b>response_size</b> and action <b>action</b> (either v2 or v3) has + * started. */ +void +geoip_start_dirreq(uint64_t dirreq_id, size_t response_size, + geoip_client_action_t action, dirreq_type_t type) +{ + dirreq_map_entry_t *ent; + if (!get_options()->DirReqStatistics) + return; + ent = tor_malloc_zero(sizeof(dirreq_map_entry_t)); + ent->dirreq_id = dirreq_id; + tor_gettimeofday(&ent->request_time); + ent->response_size = response_size; + ent->action = action; + ent->type = type; + _dirreq_map_put(ent, type, dirreq_id); +} + +/** Change the state of the either direct or tunneled (see <b>type</b>) + * directory request with <b>dirreq_id</b> to <b>new_state</b> and + * possibly mark it as completed. If no entry can be found for the given + * key parts (e.g., if this is a directory request that we are not + * measuring, or one that was started in the previous measurement period), + * or if the state cannot be advanced to <b>new_state</b>, do nothing. */ +void +geoip_change_dirreq_state(uint64_t dirreq_id, dirreq_type_t type, + dirreq_state_t new_state) +{ + dirreq_map_entry_t *ent; + if (!get_options()->DirReqStatistics) + return; + ent = _dirreq_map_get(type, dirreq_id); + if (!ent) + return; + if (new_state == DIRREQ_IS_FOR_NETWORK_STATUS) + return; + if (new_state - 1 != ent->state) + return; + ent->state = new_state; + if ((type == DIRREQ_DIRECT && + new_state == DIRREQ_FLUSHING_DIR_CONN_FINISHED) || + (type == DIRREQ_TUNNELED && + new_state == DIRREQ_OR_CONN_BUFFER_FLUSHED)) { + tor_gettimeofday(&ent->completion_time); + ent->completed = 1; + } +} + +/** Return a newly allocated comma-separated string containing statistics + * on network status downloads. The string contains the number of completed + * requests, timeouts, and still running requests as well as the download + * times by deciles and quartiles. Return NULL if we have not observed + * requests for long enough. */ +static char * +geoip_get_dirreq_history(geoip_client_action_t action, + dirreq_type_t type) +{ + char *result = NULL; + smartlist_t *dirreq_completed = NULL; + uint32_t complete = 0, timeouts = 0, running = 0; + int bufsize = 1024, written; + dirreq_map_entry_t **ptr, **next, *ent; + struct timeval now; + + tor_gettimeofday(&now); + if (action != GEOIP_CLIENT_NETWORKSTATUS && + action != GEOIP_CLIENT_NETWORKSTATUS_V2) + return NULL; + dirreq_completed = smartlist_create(); + for (ptr = HT_START(dirreqmap, &dirreq_map); ptr; ptr = next) { + ent = *ptr; + if (ent->action != action || ent->type != type) { + next = HT_NEXT(dirreqmap, &dirreq_map, ptr); + continue; + } else { + if (ent->completed) { + smartlist_add(dirreq_completed, ent); + complete++; + next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ptr); + } else { + if (tv_mdiff(&ent->request_time, &now) / 1000 > DIRREQ_TIMEOUT) + timeouts++; + else + running++; + next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ptr); + tor_free(ent); + } + } + } +#define DIR_REQ_GRANULARITY 4 + complete = round_uint32_to_next_multiple_of(complete, + DIR_REQ_GRANULARITY); + timeouts = round_uint32_to_next_multiple_of(timeouts, + DIR_REQ_GRANULARITY); + running = round_uint32_to_next_multiple_of(running, + DIR_REQ_GRANULARITY); + result = tor_malloc_zero(bufsize); + written = tor_snprintf(result, bufsize, "complete=%u,timeout=%u," + "running=%u", complete, timeouts, running); + if (written < 0) { + tor_free(result); + goto done; + } + +#define MIN_DIR_REQ_RESPONSES 16 + if (complete >= MIN_DIR_REQ_RESPONSES) { + uint32_t *dltimes; + /* We may have rounded 'completed' up. Here we want to use the + * real value. */ + complete = smartlist_len(dirreq_completed); + dltimes = tor_malloc_zero(sizeof(uint32_t) * complete); + SMARTLIST_FOREACH_BEGIN(dirreq_completed, dirreq_map_entry_t *, ent) { + uint32_t bytes_per_second; + uint32_t time_diff = (uint32_t) tv_mdiff(&ent->request_time, + &ent->completion_time); + if (time_diff == 0) + time_diff = 1; /* Avoid DIV/0; "instant" answers are impossible + * by law of nature or something, but a milisecond + * is a bit greater than "instantly" */ + bytes_per_second = (uint32_t)(1000 * ent->response_size / time_diff); + dltimes[ent_sl_idx] = bytes_per_second; + } SMARTLIST_FOREACH_END(ent); + median_uint32(dltimes, complete); /* sorts as a side effect. */ + written = tor_snprintf(result + written, bufsize - written, + ",min=%u,d1=%u,d2=%u,q1=%u,d3=%u,d4=%u,md=%u," + "d6=%u,d7=%u,q3=%u,d8=%u,d9=%u,max=%u", + dltimes[0], + dltimes[1*complete/10-1], + dltimes[2*complete/10-1], + dltimes[1*complete/4-1], + dltimes[3*complete/10-1], + dltimes[4*complete/10-1], + dltimes[5*complete/10-1], + dltimes[6*complete/10-1], + dltimes[7*complete/10-1], + dltimes[3*complete/4-1], + dltimes[8*complete/10-1], + dltimes[9*complete/10-1], + dltimes[complete-1]); + if (written<0) + tor_free(result); + tor_free(dltimes); + } + done: + SMARTLIST_FOREACH(dirreq_completed, dirreq_map_entry_t *, ent, + tor_free(ent)); + smartlist_free(dirreq_completed); + return result; +} + +/** How long do we have to have observed per-country request history before we + * are willing to talk about it? */ +#define GEOIP_MIN_OBSERVATION_TIME (12*60*60) + +/** Helper for geoip_get_client_history_dirreq() and + * geoip_get_client_history_bridge(). */ +static char * +geoip_get_client_history(time_t now, geoip_client_action_t action, + int min_observation_time, unsigned granularity) { char *result = NULL; if (!geoip_is_loaded()) return NULL; - if (client_history_starts < (now - GEOIP_MIN_OBSERVATION_TIME)) { + if (client_history_starts < (now - min_observation_time)) { char buf[32]; smartlist_t *chunks = NULL; smartlist_t *entries = NULL; @@ -476,18 +823,13 @@ geoip_get_client_history(time_t now, geoip_client_action_t action) clientmap_entry_t **ent; unsigned *counts = tor_malloc_zero(sizeof(unsigned)*n_countries); unsigned total = 0; - unsigned granularity = IP_GRANULARITY; -#ifdef ENABLE_GEOIP_STATS - if (get_options()->DirRecordUsageByCountry) - granularity = get_options()->DirRecordUsageGranularity; -#endif HT_FOREACH(ent, clientmap, &client_history) { int country; - if (((*ent)->last_seen & ACTION_MASK) != (int)action) + if ((*ent)->action != (int)action) continue; country = geoip_get_country_by_ip((*ent)->ipaddr); if (country < 0) - continue; + country = 0; /** unresolved requests are stored at index 0. */ tor_assert(0 <= country && country < n_countries); ++counts[country]; ++total; @@ -536,6 +878,34 @@ geoip_get_client_history(time_t now, geoip_client_action_t action) return result; } +/** Return a newly allocated comma-separated string containing entries for + * all the countries from which we've seen enough clients connect as a + * directory. The entry format is cc=num where num is the number of IPs + * we've seen connecting from that country, and cc is a lowercased country + * code. Returns NULL if we don't want to export geoip data yet. */ +char * +geoip_get_client_history_dirreq(time_t now, + geoip_client_action_t action) +{ + return geoip_get_client_history(now, action, + DIR_RECORD_USAGE_MIN_OBSERVATION_TIME, + DIR_RECORD_USAGE_GRANULARITY); +} + +/** Return a newly allocated comma-separated string containing entries for + * all the countries from which we've seen enough clients connect as a + * bridge. The entry format is cc=num where num is the number of IPs + * we've seen connecting from that country, and cc is a lowercased country + * code. Returns NULL if we don't want to export geoip data yet. */ +char * +geoip_get_client_history_bridge(time_t now, + geoip_client_action_t action) +{ + return geoip_get_client_history(now, action, + GEOIP_MIN_OBSERVATION_TIME, + IP_GRANULARITY); +} + /** Return a newly allocated string holding the per-country request history * for <b>action</b> in a format suitable for an extra-info document, or NULL * on failure. */ @@ -545,12 +915,9 @@ geoip_get_request_history(time_t now, geoip_client_action_t action) smartlist_t *entries, *strings; char *result; unsigned granularity = IP_GRANULARITY; -#ifdef ENABLE_GEOIP_STATS - if (get_options()->DirRecordUsageByCountry) - granularity = get_options()->DirRecordUsageGranularity; -#endif + int min_observation_time = GEOIP_MIN_OBSERVATION_TIME; - if (client_history_starts >= (now - GEOIP_MIN_OBSERVATION_TIME)) + if (client_history_starts >= (now - min_observation_time)) return NULL; if (action != GEOIP_CLIENT_NETWORKSTATUS && action != GEOIP_CLIENT_NETWORKSTATUS_V2) @@ -590,60 +957,370 @@ geoip_get_request_history(time_t now, geoip_client_action_t action) return result; } -/** Store all our geoip statistics into $DATADIR/geoip-stats. */ +/** Start time of directory request stats. */ +static time_t start_of_dirreq_stats_interval; + +/** Initialize directory request stats. */ +void +geoip_dirreq_stats_init(time_t now) +{ + start_of_dirreq_stats_interval = now; +} + +/** Write dirreq statistics to $DATADIR/stats/dirreq-stats. */ void -dump_geoip_stats(void) +geoip_dirreq_stats_write(time_t now) { -#ifdef ENABLE_GEOIP_STATS - time_t now = time(NULL); - time_t request_start; - char *filename = get_datadir_fname("geoip-stats"); + char *statsdir = NULL, *filename = NULL; char *data_v2 = NULL, *data_v3 = NULL; - char since[ISO_TIME_LEN+1], written[ISO_TIME_LEN+1]; + char written[ISO_TIME_LEN+1]; open_file_t *open_file = NULL; double v2_share = 0.0, v3_share = 0.0; FILE *out; + int i; + + if (!get_options()->DirReqStatistics) + goto done; + + /* Discard all items in the client history that are too old. */ + geoip_remove_old_clients(start_of_dirreq_stats_interval); - data_v2 = geoip_get_client_history(now, GEOIP_CLIENT_NETWORKSTATUS_V2); - data_v3 = geoip_get_client_history(now, GEOIP_CLIENT_NETWORKSTATUS); - format_iso_time(since, geoip_get_history_start()); + statsdir = get_datadir_fname("stats"); + if (check_private_dir(statsdir, CPD_CREATE) < 0) + goto done; + filename = get_datadir_fname2("stats", "dirreq-stats"); + data_v2 = geoip_get_client_history_dirreq(now, + GEOIP_CLIENT_NETWORKSTATUS_V2); + data_v3 = geoip_get_client_history_dirreq(now, + GEOIP_CLIENT_NETWORKSTATUS); format_iso_time(written, now); - out = start_writing_to_stdio_file(filename, OPEN_FLAGS_REPLACE, + out = start_writing_to_stdio_file(filename, OPEN_FLAGS_APPEND, 0600, &open_file); if (!out) goto done; - if (fprintf(out, "written %s\nstarted-at %s\nns-ips %s\nns-v2-ips %s\n", - written, since, + if (fprintf(out, "dirreq-stats-end %s (%d s)\ndirreq-v3-ips %s\n" + "dirreq-v2-ips %s\n", written, + (unsigned) (now - start_of_dirreq_stats_interval), data_v3 ? data_v3 : "", data_v2 ? data_v2 : "") < 0) goto done; tor_free(data_v2); tor_free(data_v3); - request_start = current_request_period_starts - - (n_old_request_periods * REQUEST_HIST_PERIOD); - format_iso_time(since, request_start); data_v2 = geoip_get_request_history(now, GEOIP_CLIENT_NETWORKSTATUS_V2); data_v3 = geoip_get_request_history(now, GEOIP_CLIENT_NETWORKSTATUS); - if (fprintf(out, "requests-start %s\nn-ns-reqs %s\nn-v2-ns-reqs %s\n", - since, + if (fprintf(out, "dirreq-v3-reqs %s\ndirreq-v2-reqs %s\n", data_v3 ? data_v3 : "", data_v2 ? data_v2 : "") < 0) goto done; - if (!router_get_my_share_of_directory_requests(&v2_share, &v3_share)) { - if (fprintf(out, "v2-ns-share %0.2lf%%\n", v2_share*100) < 0) + tor_free(data_v2); + tor_free(data_v3); +#define RESPONSE_GRANULARITY 8 + for (i = 0; i < GEOIP_NS_RESPONSE_NUM; i++) { + ns_v2_responses[i] = round_uint32_to_next_multiple_of( + ns_v2_responses[i], RESPONSE_GRANULARITY); + ns_v3_responses[i] = round_uint32_to_next_multiple_of( + ns_v3_responses[i], RESPONSE_GRANULARITY); + } +#undef RESPONSE_GRANULARITY + if (fprintf(out, "dirreq-v3-resp ok=%u,not-enough-sigs=%u,unavailable=%u," + "not-found=%u,not-modified=%u,busy=%u\n", + ns_v3_responses[GEOIP_SUCCESS], + ns_v3_responses[GEOIP_REJECT_NOT_ENOUGH_SIGS], + ns_v3_responses[GEOIP_REJECT_UNAVAILABLE], + ns_v3_responses[GEOIP_REJECT_NOT_FOUND], + ns_v3_responses[GEOIP_REJECT_NOT_MODIFIED], + ns_v3_responses[GEOIP_REJECT_BUSY]) < 0) + goto done; + if (fprintf(out, "dirreq-v2-resp ok=%u,unavailable=%u," + "not-found=%u,not-modified=%u,busy=%u\n", + ns_v2_responses[GEOIP_SUCCESS], + ns_v2_responses[GEOIP_REJECT_UNAVAILABLE], + ns_v2_responses[GEOIP_REJECT_NOT_FOUND], + ns_v2_responses[GEOIP_REJECT_NOT_MODIFIED], + ns_v2_responses[GEOIP_REJECT_BUSY]) < 0) + goto done; + memset(ns_v2_responses, 0, sizeof(ns_v2_responses)); + memset(ns_v3_responses, 0, sizeof(ns_v3_responses)); + if (!geoip_get_mean_shares(now, &v2_share, &v3_share)) { + if (fprintf(out, "dirreq-v2-share %0.2lf%%\n", v2_share*100) < 0) goto done; - if (fprintf(out, "v3-ns-share %0.2lf%%\n", v3_share*100) < 0) + if (fprintf(out, "dirreq-v3-share %0.2lf%%\n", v3_share*100) < 0) goto done; } + data_v2 = geoip_get_dirreq_history(GEOIP_CLIENT_NETWORKSTATUS_V2, + DIRREQ_DIRECT); + data_v3 = geoip_get_dirreq_history(GEOIP_CLIENT_NETWORKSTATUS, + DIRREQ_DIRECT); + if (fprintf(out, "dirreq-v3-direct-dl %s\ndirreq-v2-direct-dl %s\n", + data_v3 ? data_v3 : "", data_v2 ? data_v2 : "") < 0) + goto done; + tor_free(data_v2); + tor_free(data_v3); + data_v2 = geoip_get_dirreq_history(GEOIP_CLIENT_NETWORKSTATUS_V2, + DIRREQ_TUNNELED); + data_v3 = geoip_get_dirreq_history(GEOIP_CLIENT_NETWORKSTATUS, + DIRREQ_TUNNELED); + if (fprintf(out, "dirreq-v3-tunneled-dl %s\ndirreq-v2-tunneled-dl %s\n", + data_v3 ? data_v3 : "", data_v2 ? data_v2 : "") < 0) + goto done; + finish_writing_to_file(open_file); open_file = NULL; + + /* Rotate request period */ + rotate_request_period(); + + start_of_dirreq_stats_interval = now; + done: if (open_file) abort_writing_to_file(open_file); tor_free(filename); + tor_free(statsdir); tor_free(data_v2); tor_free(data_v3); -#endif +} + +/** Start time of bridge stats. */ +static time_t start_of_bridge_stats_interval; + +/** Initialize bridge stats. */ +void +geoip_bridge_stats_init(time_t now) +{ + start_of_bridge_stats_interval = now; +} + +/** Parse the bridge statistics as they are written to extra-info + * descriptors for being returned to controller clients. Return the + * controller string if successful, or NULL otherwise. */ +static char * +parse_bridge_stats_controller(const char *stats_str, time_t now) +{ + char stats_end_str[ISO_TIME_LEN+1], stats_start_str[ISO_TIME_LEN+1], + *controller_str, *eos, *eol, *summary; + + const char *BRIDGE_STATS_END = "bridge-stats-end "; + const char *BRIDGE_IPS = "bridge-ips "; + const char *BRIDGE_IPS_EMPTY_LINE = "bridge-ips\n"; + const char *tmp; + time_t stats_end_time; + size_t controller_len; + int seconds; + tor_assert(stats_str); + + /* Parse timestamp and number of seconds from + "bridge-stats-end YYYY-MM-DD HH:MM:SS (N s)" */ + tmp = find_str_at_start_of_line(stats_str, BRIDGE_STATS_END); + if (!tmp) + return NULL; + tmp += strlen(BRIDGE_STATS_END); + + if (strlen(tmp) < ISO_TIME_LEN + 6) + return NULL; + strlcpy(stats_end_str, tmp, sizeof(stats_end_str)); + if (parse_iso_time(stats_end_str, &stats_end_time) < 0) + return NULL; + if (stats_end_time < now - (25*60*60) || + stats_end_time > now + (1*60*60)) + return NULL; + seconds = (int)strtol(tmp + ISO_TIME_LEN + 2, &eos, 10); + if (!eos || seconds < 23*60*60) + return NULL; + format_iso_time(stats_start_str, stats_end_time - seconds); + + /* Parse: "bridge-ips CC=N,CC=N,..." */ + tmp = find_str_at_start_of_line(stats_str, BRIDGE_IPS); + if (tmp) { + tmp += strlen(BRIDGE_IPS); + tmp = eat_whitespace_no_nl(tmp); + eol = strchr(tmp, '\n'); + if (eol) + summary = tor_strndup(tmp, eol-tmp); + else + summary = tor_strdup(tmp); + } else { + /* Look if there is an empty "bridge-ips" line */ + tmp = find_str_at_start_of_line(stats_str, BRIDGE_IPS_EMPTY_LINE); + if (!tmp) + return NULL; + summary = tor_strdup(""); + } + + controller_len = strlen("TimeStarted=\"\" CountrySummary=") + + strlen(summary) + 42; + controller_str = tor_malloc(controller_len); + if (tor_snprintf(controller_str, controller_len, + "TimeStarted=\"%s\" CountrySummary=%s", + stats_start_str, summary) < 0) { + tor_free(controller_str); + tor_free(summary); + return NULL; + } + tor_free(summary); + return controller_str; +} + +/** Most recent bridge statistics formatted to be written to extra-info + * descriptors. */ +static char *bridge_stats_extrainfo = NULL; + +/** Most recent bridge statistics formatted to be returned to controller + * clients. */ +static char *bridge_stats_controller = NULL; + +/** Write bridge statistics to $DATADIR/stats/bridge-stats and return + * when we should next try to write statistics. */ +time_t +geoip_bridge_stats_write(time_t now) +{ + char *statsdir = NULL, *filename = NULL, *data = NULL, + written[ISO_TIME_LEN+1], *out = NULL, *controller_str; + size_t len; + + /* If we changed from relay to bridge recently, adapt starting time + * of current measurements. */ + if (start_of_bridge_stats_interval < client_history_starts) + start_of_bridge_stats_interval = client_history_starts; + + /* Check if 24 hours have passed since starting measurements. */ + if (now < start_of_bridge_stats_interval + + DIR_ENTRY_RECORD_USAGE_RETAIN_IPS) + return start_of_bridge_stats_interval + + DIR_ENTRY_RECORD_USAGE_RETAIN_IPS; + + /* Discard all items in the client history that are too old. */ + geoip_remove_old_clients(start_of_bridge_stats_interval); + + statsdir = get_datadir_fname("stats"); + if (check_private_dir(statsdir, CPD_CREATE) < 0) + goto done; + filename = get_datadir_fname2("stats", "bridge-stats"); + data = geoip_get_client_history_bridge(now, GEOIP_CLIENT_CONNECT); + format_iso_time(written, now); + len = strlen("bridge-stats-end (999999 s)\nbridge-ips \n") + + ISO_TIME_LEN + (data ? strlen(data) : 0) + 42; + out = tor_malloc(len); + if (tor_snprintf(out, len, "bridge-stats-end %s (%u s)\nbridge-ips %s\n", + written, (unsigned) (now - start_of_bridge_stats_interval), + data ? data : "") < 0) + goto done; + write_str_to_file(filename, out, 0); + controller_str = parse_bridge_stats_controller(out, now); + if (!controller_str) + goto done; + start_of_bridge_stats_interval = now; + tor_free(bridge_stats_extrainfo); + tor_free(bridge_stats_controller); + bridge_stats_extrainfo = out; + out = NULL; + bridge_stats_controller = controller_str; + control_event_clients_seen(controller_str); + done: + tor_free(filename); + tor_free(statsdir); + tor_free(data); + tor_free(out); + return start_of_bridge_stats_interval + + DIR_ENTRY_RECORD_USAGE_RETAIN_IPS; +} + +/** Try to load the most recent bridge statistics from disk, unless we + * have finished a measurement interval lately. */ +static void +load_bridge_stats(time_t now) +{ + char *statsdir, *fname=NULL, *contents, *controller_str; + if (bridge_stats_extrainfo) + return; + statsdir = get_datadir_fname("stats"); + if (check_private_dir(statsdir, CPD_CREATE) < 0) + goto done; + fname = get_datadir_fname2("stats", "bridge-stats"); + contents = read_file_to_str(fname, RFTS_IGNORE_MISSING, NULL); + if (contents) { + controller_str = parse_bridge_stats_controller(contents, now); + if (controller_str) { + bridge_stats_extrainfo = contents; + bridge_stats_controller = controller_str; + } else { + tor_free(contents); + } + } + done: + tor_free(fname); + tor_free(statsdir); +} + +/** Return most recent bridge statistics for inclusion in extra-info + * descriptors, or NULL if we don't have recent bridge statistics. */ +char * +geoip_get_bridge_stats_extrainfo(time_t now) +{ + load_bridge_stats(now); + return bridge_stats_extrainfo; +} + +/** Return most recent bridge statistics to be returned to controller + * clients, or NULL if we don't have recent bridge statistics. */ +char * +geoip_get_bridge_stats_controller(time_t now) +{ + load_bridge_stats(now); + return bridge_stats_controller; +} + +/** Start time of entry stats. */ +static time_t start_of_entry_stats_interval; + +/** Initialize entry stats. */ +void +geoip_entry_stats_init(time_t now) +{ + start_of_entry_stats_interval = now; +} + +/** Write entry statistics to $DATADIR/stats/entry-stats. */ +void +geoip_entry_stats_write(time_t now) +{ + char *statsdir = NULL, *filename = NULL; + char *data = NULL; + char written[ISO_TIME_LEN+1]; + open_file_t *open_file = NULL; + FILE *out; + + if (!get_options()->EntryStatistics) + goto done; + + /* Discard all items in the client history that are too old. */ + geoip_remove_old_clients(start_of_entry_stats_interval); + + statsdir = get_datadir_fname("stats"); + if (check_private_dir(statsdir, CPD_CREATE) < 0) + goto done; + filename = get_datadir_fname2("stats", "entry-stats"); + data = geoip_get_client_history_dirreq(now, GEOIP_CLIENT_CONNECT); + format_iso_time(written, now); + out = start_writing_to_stdio_file(filename, OPEN_FLAGS_APPEND, + 0600, &open_file); + if (!out) + goto done; + if (fprintf(out, "entry-stats-end %s (%u s)\nentry-ips %s\n", + written, (unsigned) (now - start_of_entry_stats_interval), + data ? data : "") < 0) + goto done; + + start_of_entry_stats_interval = now; + + finish_writing_to_file(open_file); + open_file = NULL; + done: + if (open_file) + abort_writing_to_file(open_file); + tor_free(filename); + tor_free(statsdir); + tor_free(data); } /** Helper used to implement GETINFO ip-to-country/... controller command. */ @@ -674,8 +1351,8 @@ clear_geoip_db(void) SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, tor_free(c)); smartlist_free(geoip_countries); } - if (country_idxplus1_by_lc_code) - strmap_free(country_idxplus1_by_lc_code, NULL); + + strmap_free(country_idxplus1_by_lc_code, NULL); if (geoip_entries) { SMARTLIST_FOREACH(geoip_entries, geoip_entry_t *, ent, tor_free(ent)); smartlist_free(geoip_entries); @@ -689,13 +1366,24 @@ clear_geoip_db(void) void geoip_free_all(void) { - clientmap_entry_t **ent, **next, *this; - for (ent = HT_START(clientmap, &client_history); ent != NULL; ent = next) { - this = *ent; - next = HT_NEXT_RMV(clientmap, &client_history, ent); - tor_free(this); + { + clientmap_entry_t **ent, **next, *this; + for (ent = HT_START(clientmap, &client_history); ent != NULL; ent = next) { + this = *ent; + next = HT_NEXT_RMV(clientmap, &client_history, ent); + tor_free(this); + } + HT_CLEAR(clientmap, &client_history); + } + { + dirreq_map_entry_t **ent, **next, *this; + for (ent = HT_START(dirreqmap, &dirreq_map); ent != NULL; ent = next) { + this = *ent; + next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ent); + tor_free(this); + } + HT_CLEAR(dirreqmap, &dirreq_map); } - HT_CLEAR(clientmap, &client_history); clear_geoip_db(); } diff --git a/src/or/hibernate.c b/src/or/hibernate.c index 89f9aa701b..d68682d730 100644 --- a/src/or/hibernate.c +++ b/src/or/hibernate.c @@ -182,6 +182,9 @@ accounting_parse_options(or_options_t *options, int validate_only) case UNIT_DAY: d = 0; break; + /* Coverity dislikes unreachable default cases; some compilers warn on + * switch statements missing a case. Tell Coverity not to worry. */ + /* coverity[dead_error_begin] */ default: tor_assert(0); } diff --git a/src/or/main.c b/src/or/main.c index ca09af0561..f4ee16ca1f 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -18,6 +18,12 @@ #endif #include "memarea.h" +#ifdef HAVE_EVENT2_EVENT_H +#include <event2/event.h> +#else +#include <event.h> +#endif + void evdns_shutdown(int); /********* PROTOTYPES **********/ @@ -127,12 +133,10 @@ connection_add(connection_t *conn) smartlist_add(connection_array, conn); if (conn->s >= 0 || conn->linked) { - conn->read_event = tor_malloc_zero(sizeof(struct event)); - conn->write_event = tor_malloc_zero(sizeof(struct event)); - event_set(conn->read_event, conn->s, EV_READ|EV_PERSIST, - conn_read_callback, conn); - event_set(conn->write_event, conn->s, EV_WRITE|EV_PERSIST, - conn_write_callback, conn); + conn->read_event = tor_event_new(tor_libevent_get_base(), + conn->s, EV_READ|EV_PERSIST, conn_read_callback, conn); + conn->write_event = tor_event_new(tor_libevent_get_base(), + conn->s, EV_WRITE|EV_PERSIST, conn_write_callback, conn); } log_debug(LD_NET,"new conn type %s, socket %d, address %s, n_conns %d.", @@ -142,6 +146,25 @@ connection_add(connection_t *conn) return 0; } +/** Tell libevent that we don't care about <b>conn</b> any more. */ +void +connection_unregister_events(connection_t *conn) +{ + if (conn->read_event) { + if (event_del(conn->read_event)) + log_warn(LD_BUG, "Error removing read event for %d", conn->s); + tor_free(conn->read_event); + } + if (conn->write_event) { + if (event_del(conn->write_event)) + log_warn(LD_BUG, "Error removing write event for %d", conn->s); + tor_free(conn->write_event); + } + if (conn->dns_server_port) { + dnsserv_close_listener(conn); + } +} + /** Remove the connection from the global list, and remove the * corresponding poll entry. Calling this function will shift the last * connection (if any) into the position occupied by conn. @@ -246,17 +269,17 @@ get_connection_array(void) } /** Set the event mask on <b>conn</b> to <b>events</b>. (The event - * mask is a bitmask whose bits are EV_READ and EV_WRITE.) + * mask is a bitmask whose bits are READ_EVENT and WRITE_EVENT) */ void -connection_watch_events(connection_t *conn, short events) +connection_watch_events(connection_t *conn, watchable_events_t events) { - if (events & EV_READ) + if (events & READ_EVENT) connection_start_reading(conn); else connection_stop_reading(conn); - if (events & EV_WRITE) + if (events & WRITE_EVENT) connection_start_writing(conn); else connection_stop_writing(conn); @@ -393,11 +416,11 @@ connection_start_reading_from_linked_conn(connection_t *conn) smartlist_add(active_linked_connection_lst, conn); if (!called_loop_once) { /* This is the first event on the list; we won't be in LOOP_ONCE mode, - * so we need to make sure that the event_loop() actually exits at the - * end of its run through the current connections and - * lets us activate read events for linked connections. */ + * so we need to make sure that the event_base_loop() actually exits at + * the end of its run through the current connections and lets us + * activate read events for linked connections. */ struct timeval tv = { 0, 0 }; - event_loopexit(&tv); + tor_event_base_loopexit(tor_libevent_get_base(), &tv); } } else { tor_assert(smartlist_isin(active_linked_connection_lst, conn)); @@ -540,7 +563,7 @@ conn_close_if_marked(int i) log_info(LD_NET, "Conn (addr %s, fd %d, type %s, state %d) marked, but wants " "to flush %d bytes. (Marked at %s:%d)", - escaped_safe_str(conn->address), + escaped_safe_str_client(conn->address), conn->s, conn_type_to_string(conn->type), conn->state, (int)conn->outbuf_flushlen, conn->marked_for_close_file, conn->marked_for_close); @@ -593,8 +616,8 @@ conn_close_if_marked(int i) "something is wrong with theirs. " "(fd %d, type %s, state %d, marked at %s:%d).", (int)buf_datalen(conn->outbuf), - escaped_safe_str(conn->address), conn->s, - conn_type_to_string(conn->type), conn->state, + escaped_safe_str_client(conn->address), + conn->s, conn_type_to_string(conn->type), conn->state, conn->marked_for_close_file, conn->marked_for_close); } @@ -623,7 +646,7 @@ directory_all_unreachable(time_t now) log_notice(LD_NET, "Is your network connection down? " "Failing connection to '%s:%d'.", - safe_str(edge_conn->socks_request->address), + safe_str_client(edge_conn->socks_request->address), edge_conn->socks_request->port); connection_mark_unattached_ap(edge_conn, END_STREAM_REASON_NET_UNREACHABLE); @@ -708,6 +731,7 @@ run_connection_housekeeping(int i, time_t now) return; /* we're all done here, the rest is just for OR conns */ or_conn = TO_OR_CONN(conn); + tor_assert(conn->outbuf); if (or_conn->is_bad_for_new_circs && !or_conn->n_circuits) { /* It's bad for new circuits, and has no unmarked circuits on it: @@ -774,7 +798,7 @@ run_connection_housekeeping(int i, time_t now) } } -/** Honor a NEWNYM request: make future requests unlinkability to past +/** Honor a NEWNYM request: make future requests unlinkable to past * requests. */ static void signewnym_impl(time_t now) @@ -800,14 +824,15 @@ run_scheduled_events(time_t now) static time_t time_to_try_getting_descriptors = 0; static time_t time_to_reset_descriptor_failures = 0; static time_t time_to_add_entropy = 0; - static time_t time_to_write_hs_statistics = 0; static time_t time_to_write_bridge_status_file = 0; static time_t time_to_downrate_stability = 0; static time_t time_to_save_stability = 0; static time_t time_to_clean_caches = 0; static time_t time_to_recheck_bandwidth = 0; static time_t time_to_check_for_expired_networkstatus = 0; - static time_t time_to_dump_geoip_stats = 0; + static time_t time_to_write_stats_files = 0; + static time_t time_to_write_bridge_stats = 0; + static int should_init_bridge_stats = 1; static time_t time_to_retry_dns_init = 0; or_options_t *options = get_options(); int i; @@ -935,11 +960,66 @@ run_scheduled_events(time_t now) time_to_check_for_expired_networkstatus = now + CHECK_EXPIRED_NS_INTERVAL; } - if (time_to_dump_geoip_stats < now) { -#define DUMP_GEOIP_STATS_INTERVAL (60*60); - if (time_to_dump_geoip_stats) - dump_geoip_stats(); - time_to_dump_geoip_stats = now + DUMP_GEOIP_STATS_INTERVAL; + /* 1g. Check whether we should write statistics to disk. + */ + if (time_to_write_stats_files >= 0 && time_to_write_stats_files < now) { +#define WRITE_STATS_INTERVAL (24*60*60) + if (options->CellStatistics || options->DirReqStatistics || + options->EntryStatistics || options->ExitPortStatistics) { + if (!time_to_write_stats_files) { + /* Initialize stats. We're doing this here and not in options_act, + * so that we know exactly when the 24 hours interval ends. */ + if (options->CellStatistics) + rep_hist_buffer_stats_init(now); + if (options->DirReqStatistics) + geoip_dirreq_stats_init(now); + if (options->EntryStatistics) + geoip_entry_stats_init(now); + if (options->ExitPortStatistics) + rep_hist_exit_stats_init(now); + log_notice(LD_CONFIG, "Configured to measure statistics. Look for " + "the *-stats files that will first be written to the " + "data directory in %d hours from now.", + WRITE_STATS_INTERVAL / (60 * 60)); + time_to_write_stats_files = now + WRITE_STATS_INTERVAL; + } else { + /* Write stats to disk. */ + if (options->CellStatistics) + rep_hist_buffer_stats_write(time_to_write_stats_files); + if (options->DirReqStatistics) + geoip_dirreq_stats_write(time_to_write_stats_files); + if (options->EntryStatistics) + geoip_entry_stats_write(time_to_write_stats_files); + if (options->ExitPortStatistics) + rep_hist_exit_stats_write(time_to_write_stats_files); + time_to_write_stats_files += WRITE_STATS_INTERVAL; + } + } else { + /* Never write stats to disk */ + time_to_write_stats_files = -1; + } + } + + /* 1h. Check whether we should write bridge statistics to disk. + */ + if (should_record_bridge_info(options)) { + if (time_to_write_bridge_stats < now) { + if (should_init_bridge_stats) { + /* (Re-)initialize bridge statistics. */ + geoip_bridge_stats_init(now); + time_to_write_bridge_stats = now + WRITE_STATS_INTERVAL; + should_init_bridge_stats = 0; + } else { + /* Possibly write bridge statistics to disk and ask when to write + * them next time. */ + time_to_write_bridge_stats = geoip_bridge_stats_write( + time_to_write_bridge_stats); + } + } + } else if (!should_init_bridge_stats) { + /* Bridge mode was turned off. Ensure that stats are re-initialized + * next time bridge mode is turned on. */ + should_init_bridge_stats = 1; } /* Remove old information from rephist and the rend cache. */ @@ -1109,12 +1189,6 @@ run_scheduled_events(time_t now) } } - /** 10. write hidden service usage statistic to disk */ - if (options->HSAuthorityRecordStats && time_to_write_hs_statistics < now) { - hs_usage_write_statistics_to_file(now); -#define WRITE_HSUSAGE_INTERVAL (30*60) - time_to_write_hs_statistics = now+WRITE_HSUSAGE_INTERVAL; - } /** 10b. write bridge networkstatus file to disk */ if (options->BridgeAuthoritativeDir && time_to_write_bridge_status_file < now) { @@ -1148,8 +1222,8 @@ second_elapsed_callback(int fd, short event, void *args) (void)event; (void)args; if (!timeout_event) { - timeout_event = tor_malloc_zero(sizeof(struct event)); - evtimer_set(timeout_event, second_elapsed_callback, NULL); + timeout_event = tor_evtimer_new(tor_libevent_get_base(), + second_elapsed_callback, NULL); one_second.tv_sec = 1; one_second.tv_usec = 0; } @@ -1222,15 +1296,7 @@ second_elapsed_callback(int fd, short event, void *args) current_second = now; /* remember which second it is, for next time */ -#if 0 - if (current_second % 300 == 0) { - rep_history_clean(current_second - options->RephistTrackTime); - dumpmemusage(get_min_log_level()<LOG_INFO ? - get_min_log_level() : LOG_INFO); - } -#endif - - if (evtimer_add(timeout_event, &one_second)) + if (event_add(timeout_event, &one_second)) log_err(LD_NET, "Error from libevent when setting one-second timeout event"); } @@ -1393,8 +1459,10 @@ do_main_loop(void) /* initialize the bootstrap status events to know we're starting up */ control_event_bootstrap(BOOTSTRAP_STATUS_STARTING, 0); - if (trusted_dirs_reload_certs()) - return -1; + if (trusted_dirs_reload_certs()) { + log_warn(LD_DIR, + "Couldn't load all cached v3 certificates. Starting anyway."); + } if (router_reload_v2_networkstatus()) { return -1; } @@ -1441,20 +1509,16 @@ do_main_loop(void) /* poll until we have an event, or the second ends, or until we have * some active linked connections to trigger events for. */ - loop_result = event_loop(called_loop_once ? EVLOOP_ONCE : 0); + loop_result = event_base_loop(tor_libevent_get_base(), + called_loop_once ? EVLOOP_ONCE : 0); /* let catch() handle things like ^c, and otherwise don't worry about it */ if (loop_result < 0) { int e = tor_socket_errno(-1); /* let the program survive things like ^z */ if (e != EINTR && !ERRNO_IS_EINPROGRESS(e)) { -#ifdef HAVE_EVENT_GET_METHOD log_err(LD_NET,"libevent call with %s failed: %s [%d]", - event_get_method(), tor_socket_strerror(e), e); -#else - log_err(LD_NET,"libevent call failed: %s [%d]", - tor_socket_strerror(e), e); -#endif + tor_libevent_get_method(), tor_socket_strerror(e), e); return -1; #ifndef MS_WINDOWS } else if (e == EINVAL) { @@ -1597,6 +1661,7 @@ dumpmemusage(int severity) U64_PRINTF_ARG(rephist_total_alloc), rephist_total_num); dump_routerlist_mem_usage(severity); dump_cell_pool_usage(severity); + dump_dns_mem_usage(severity); buf_dump_freelist_sizes(severity); tor_log_mallinfo(severity); } @@ -1623,7 +1688,8 @@ dumpstats(int severity) if (!connection_is_listener(conn)) { log(severity,LD_GENERAL, "Conn %d is to %s:%d.", i, - safe_str(conn->address), conn->port); + safe_str_client(conn->address), + conn->port); log(severity,LD_GENERAL, "Conn %d: %d bytes waiting on inbuf (len %d, last read %d secs ago)", i, @@ -1720,7 +1786,7 @@ handle_signals(int is_parent) { #ifndef MS_WINDOWS /* do signal stuff only on Unix */ int i; - static int signals[] = { + static const int signals[] = { SIGINT, /* do a controlled slow shutdown */ SIGTERM, /* to terminate now */ SIGPIPE, /* otherwise SIGPIPE kills us */ @@ -1732,12 +1798,13 @@ handle_signals(int is_parent) #endif SIGCHLD, /* handle dns/cpu workers that exit */ -1 }; - static struct event signal_events[16]; /* bigger than it has to be. */ + static struct event *signal_events[16]; /* bigger than it has to be. */ if (is_parent) { for (i = 0; signals[i] >= 0; ++i) { - signal_set(&signal_events[i], signals[i], signal_callback, - (void*)(uintptr_t)signals[i]); - if (signal_add(&signal_events[i], NULL)) + signal_events[i] = tor_evsignal_new( + tor_libevent_get_base(), signals[i], signal_callback, + (void*)(uintptr_t)signals[i]); + if (event_add(signal_events[i], NULL)) log_warn(LD_BUG, "Error from libevent when adding event for signal %d", signals[i]); } @@ -1826,7 +1893,9 @@ tor_init(int argc, char *argv[]) "and you probably shouldn't."); #endif - if (crypto_global_init(get_options()->HardwareAccel)) { + if (crypto_global_init(get_options()->HardwareAccel, + get_options()->AccelName, + get_options()->AccelDir)) { log_err(LD_BUG, "Unable to initialize OpenSSL. Exiting."); return -1; } @@ -1920,7 +1989,6 @@ tor_free_all(int postfork) rend_cache_free_all(); rend_service_authorization_free_all(); rep_hist_free_all(); - hs_usage_free_all(); dns_free_all(); clear_pending_onions(); circuit_free_all(); @@ -1928,6 +1996,7 @@ tor_free_all(int postfork) connection_free_all(); buf_shrink_freelists(1); memarea_clear_freelist(); + microdesc_free_all(); if (!postfork) { config_free_all(); router_free_all(); @@ -1938,12 +2007,10 @@ tor_free_all(int postfork) tor_tls_free_all(); } /* stuff in main.c */ - if (connection_array) - smartlist_free(connection_array); - if (closeable_connection_lst) - smartlist_free(closeable_connection_lst); - if (active_linked_connection_lst) - smartlist_free(active_linked_connection_lst); + + smartlist_free(connection_array); + smartlist_free(closeable_connection_lst); + smartlist_free(active_linked_connection_lst); tor_free(timeout_event); if (!postfork) { release_lockfile(); diff --git a/src/or/microdesc.c b/src/or/microdesc.c new file mode 100644 index 0000000000..c1f447c5df --- /dev/null +++ b/src/or/microdesc.c @@ -0,0 +1,393 @@ +/* Copyright (c) 2009, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "or.h" + +/** A data structure to hold a bunch of cached microdescriptors. There are + * two active files in the cache: a "cache file" that we mmap, and a "journal + * file" that we append to. Periodically, we rebuild the cache file to hold + * only the microdescriptors that we want to keep */ +struct microdesc_cache_t { + /** Map from sha256-digest to microdesc_t for every microdesc_t in the + * cache. */ + HT_HEAD(microdesc_map, microdesc_t) map; + + /** Name of the cache file. */ + char *cache_fname; + /** Name of the journal file. */ + char *journal_fname; + /** Mmap'd contents of the cache file, or NULL if there is none. */ + tor_mmap_t *cache_content; + /** Number of bytes used in the journal file. */ + size_t journal_len; + + /** Total bytes of microdescriptor bodies we have added to this cache */ + uint64_t total_len_seen; + /** Total number of microdescriptors we have added to this cache */ + unsigned n_seen; +}; + +/** Helper: computes a hash of <b>md</b> to place it in a hash table. */ +static INLINE unsigned int +_microdesc_hash(microdesc_t *md) +{ + unsigned *d = (unsigned*)md->digest; +#if SIZEOF_INT == 4 + return d[0] ^ d[1] ^ d[2] ^ d[3] ^ d[4] ^ d[5] ^ d[6] ^ d[7]; +#else + return d[0] ^ d[1] ^ d[2] ^ d[3]; +#endif +} + +/** Helper: compares <b>a</b> and </b> for equality for hash-table purposes. */ +static INLINE int +_microdesc_eq(microdesc_t *a, microdesc_t *b) +{ + return !memcmp(a->digest, b->digest, DIGEST256_LEN); +} + +HT_PROTOTYPE(microdesc_map, microdesc_t, node, + _microdesc_hash, _microdesc_eq); +HT_GENERATE(microdesc_map, microdesc_t, node, + _microdesc_hash, _microdesc_eq, 0.6, + _tor_malloc, _tor_realloc, _tor_free); + +/** Write the body of <b>md</b> into <b>f</b>, with appropriate annotations. + * On success, return the total number of bytes written, and set + * *<b>annotation_len_out</b> to the number of bytes written as + * annotations. */ +static size_t +dump_microdescriptor(FILE *f, microdesc_t *md, size_t *annotation_len_out) +{ + size_t r = 0; + /* XXXX drops unkown annotations. */ + if (md->last_listed) { + char buf[ISO_TIME_LEN+1]; + char annotation[ISO_TIME_LEN+32]; + format_iso_time(buf, md->last_listed); + tor_snprintf(annotation, sizeof(annotation), "@last-listed %s\n", buf); + fputs(annotation, f); + r += strlen(annotation); + *annotation_len_out = r; + } else { + *annotation_len_out = 0; + } + + md->off = (off_t) ftell(f); + fwrite(md->body, 1, md->bodylen, f); + r += md->bodylen; + return r; +} + +/** Holds a pointer to the current microdesc_cache_t object, or NULL if no + * such object has been allocated. */ +static microdesc_cache_t *the_microdesc_cache = NULL; + +/** Return a pointer to the microdescriptor cache, loading it if necessary. */ +microdesc_cache_t * +get_microdesc_cache(void) +{ + if (PREDICT_UNLIKELY(the_microdesc_cache==NULL)) { + microdesc_cache_t *cache = tor_malloc_zero(sizeof(microdesc_cache_t)); + HT_INIT(microdesc_map, &cache->map); + cache->cache_fname = get_datadir_fname("cached-microdescs"); + cache->journal_fname = get_datadir_fname("cached-microdescs.new"); + microdesc_cache_reload(cache); + the_microdesc_cache = cache; + } + return the_microdesc_cache; +} + +/* There are three sources of microdescriptors: + 1) Generated by us while acting as a directory authority. + 2) Loaded from the cache on disk. + 3) Downloaded. +*/ + +/** Decode the microdescriptors from the string starting at <b>s</b> and + * ending at <b>eos</b>, and store them in <b>cache</b>. If <b>no-save</b>, + * mark them as non-writable to disk. If <b>where</b> is SAVED_IN_CACHE, + * leave their bodies as pointers to the mmap'd cache. If where is + * <b>SAVED_NOWHERE</b>, do not allow annotations. Return a list of the added + * microdescriptors. */ +smartlist_t * +microdescs_add_to_cache(microdesc_cache_t *cache, + const char *s, const char *eos, saved_location_t where, + int no_save) +{ + /*XXXX need an argument that sets last_listed as appropriate. */ + + smartlist_t *descriptors, *added; + const int allow_annotations = (where != SAVED_NOWHERE); + const int copy_body = (where != SAVED_IN_CACHE); + + descriptors = microdescs_parse_from_string(s, eos, + allow_annotations, + copy_body); + + added = microdescs_add_list_to_cache(cache, descriptors, where, no_save); + smartlist_free(descriptors); + return added; +} + +/* As microdescs_add_to_cache, but takes a list of micrdescriptors instead of + * a string to encode. Frees any members of <b>descriptors</b> that it does + * not add. */ +smartlist_t * +microdescs_add_list_to_cache(microdesc_cache_t *cache, + smartlist_t *descriptors, saved_location_t where, + int no_save) +{ + smartlist_t *added; + open_file_t *open_file = NULL; + FILE *f = NULL; + // int n_added = 0; + size_t size = 0; + + if (where == SAVED_NOWHERE && !no_save) { + f = start_writing_to_stdio_file(cache->journal_fname, + OPEN_FLAGS_APPEND|O_BINARY, + 0600, &open_file); + if (!f) { + log_warn(LD_DIR, "Couldn't append to journal in %s: %s", + cache->journal_fname, strerror(errno)); + return NULL; + } + } + + added = smartlist_create(); + SMARTLIST_FOREACH_BEGIN(descriptors, microdesc_t *, md) { + microdesc_t *md2; + md2 = HT_FIND(microdesc_map, &cache->map, md); + if (md2) { + /* We already had this one. */ + if (md2->last_listed < md->last_listed) + md2->last_listed = md->last_listed; + microdesc_free(md); + continue; + } + + /* Okay, it's a new one. */ + if (f) { + size_t annotation_len; + size = dump_microdescriptor(f, md, &annotation_len); + md->saved_location = SAVED_IN_JOURNAL; + cache->journal_len += size; + } else { + md->saved_location = where; + } + + md->no_save = no_save; + + HT_INSERT(microdesc_map, &cache->map, md); + smartlist_add(added, md); + ++cache->n_seen; + cache->total_len_seen += md->bodylen; + } SMARTLIST_FOREACH_END(md); + + if (f) + finish_writing_to_file(open_file); /*XXX Check me.*/ + + { + size_t old_content_len = + cache->cache_content ? cache->cache_content->size : 0; + if (cache->journal_len > 16384 + old_content_len && + cache->journal_len > old_content_len * 2) { + microdesc_cache_rebuild(cache); + } + } + + return added; +} + +/** Remove every microdescriptor in <b>cache</b>. */ +void +microdesc_cache_clear(microdesc_cache_t *cache) +{ + microdesc_t **entry, **next; + for (entry = HT_START(microdesc_map, &cache->map); entry; entry = next) { + microdesc_t *md = *entry; + next = HT_NEXT_RMV(microdesc_map, &cache->map, entry); + microdesc_free(md); + } + HT_CLEAR(microdesc_map, &cache->map); + if (cache->cache_content) { + tor_munmap_file(cache->cache_content); + cache->cache_content = NULL; + } + cache->total_len_seen = 0; + cache->n_seen = 0; +} + +/** Reload the contents of <b>cache</b> from disk. If it is empty, load it + * for the first time. Return 0 on success, -1 on failure. */ +int +microdesc_cache_reload(microdesc_cache_t *cache) +{ + struct stat st; + char *journal_content; + smartlist_t *added; + tor_mmap_t *mm; + int total = 0; + + microdesc_cache_clear(cache); + + mm = cache->cache_content = tor_mmap_file(cache->cache_fname); + if (mm) { + added = microdescs_add_to_cache(cache, mm->data, mm->data+mm->size, + SAVED_IN_CACHE, 0); + if (added) { + total += smartlist_len(added); + smartlist_free(added); + } + } + + journal_content = read_file_to_str(cache->journal_fname, + RFTS_IGNORE_MISSING, &st); + if (journal_content) { + added = microdescs_add_to_cache(cache, journal_content, + journal_content+st.st_size, + SAVED_IN_JOURNAL, 0); + if (added) { + total += smartlist_len(added); + smartlist_free(added); + } + tor_free(journal_content); + } + log_notice(LD_DIR, "Reloaded microdescriptor cache. Found %d descriptors.", + total); + return 0; +} + +/** Regenerate the main cache file for <b>cache</b>, clear the journal file, + * and update every microdesc_t in the cache with pointers to its new + * location. */ +int +microdesc_cache_rebuild(microdesc_cache_t *cache) +{ + open_file_t *open_file; + FILE *f; + microdesc_t **mdp; + smartlist_t *wrote; + size_t size; + off_t off = 0; + int orig_size, new_size; + + log_info(LD_DIR, "Rebuilding the microdescriptor cache..."); + orig_size = (int)(cache->cache_content ? cache->cache_content->size : 0); + orig_size += (int)cache->journal_len; + + f = start_writing_to_stdio_file(cache->cache_fname, + OPEN_FLAGS_REPLACE|O_BINARY, + 0600, &open_file); + if (!f) + return -1; + + wrote = smartlist_create(); + + HT_FOREACH(mdp, microdesc_map, &cache->map) { + microdesc_t *md = *mdp; + size_t annotation_len; + if (md->no_save) + continue; + + size = dump_microdescriptor(f, md, &annotation_len); + md->off = off + annotation_len; + off += size; + if (md->saved_location != SAVED_IN_CACHE) { + tor_free(md->body); + md->saved_location = SAVED_IN_CACHE; + } + smartlist_add(wrote, md); + } + + finish_writing_to_file(open_file); /*XXX Check me.*/ + + if (cache->cache_content) + tor_munmap_file(cache->cache_content); + cache->cache_content = tor_mmap_file(cache->cache_fname); + + if (!cache->cache_content && smartlist_len(wrote)) { + log_err(LD_DIR, "Couldn't map file that we just wrote to %s!", + cache->cache_fname); + smartlist_free(wrote); + return -1; + } + SMARTLIST_FOREACH_BEGIN(wrote, microdesc_t *, md) { + tor_assert(md->saved_location == SAVED_IN_CACHE); + md->body = (char*)cache->cache_content->data + md->off; + tor_assert(!memcmp(md->body, "onion-key", 9)); + } SMARTLIST_FOREACH_END(md); + + smartlist_free(wrote); + + write_str_to_file(cache->journal_fname, "", 1); + cache->journal_len = 0; + + new_size = (int)cache->cache_content->size; + log_info(LD_DIR, "Done rebuilding microdesc cache. " + "Saved %d bytes; %d still used.", + orig_size-new_size, new_size); + + return 0; +} + +/** Deallocate a single microdescriptor. Note: the microdescriptor MUST have + * previously been removed from the cache if it had ever been inserted. */ +void +microdesc_free(microdesc_t *md) +{ + if (!md) + return; + /* Must be removed from hash table! */ + if (md->onion_pkey) + crypto_free_pk_env(md->onion_pkey); + if (md->body && md->saved_location != SAVED_IN_CACHE) + tor_free(md->body); + + if (md->family) { + SMARTLIST_FOREACH(md->family, char *, cp, tor_free(cp)); + smartlist_free(md->family); + } + tor_free(md->exitsummary); + + tor_free(md); +} + +/** Free all storage held in the microdesc.c module. */ +void +microdesc_free_all(void) +{ + if (the_microdesc_cache) { + microdesc_cache_clear(the_microdesc_cache); + tor_free(the_microdesc_cache->cache_fname); + tor_free(the_microdesc_cache->journal_fname); + tor_free(the_microdesc_cache); + } +} + +/** If there is a microdescriptor in <b>cache</b> whose sha256 digest is + * <b>d</b>, return it. Otherwise return NULL. */ +microdesc_t * +microdesc_cache_lookup_by_digest256(microdesc_cache_t *cache, const char *d) +{ + microdesc_t *md, search; + if (!cache) + cache = get_microdesc_cache(); + memcpy(search.digest, d, DIGEST256_LEN); + md = HT_FIND(microdesc_map, &cache->map, &search); + return md; +} + +/** Return the mean size of decriptors added to <b>cache</b> since it was last + * cleared. Used to estimate the size of large downloads. */ +size_t +microdesc_average_size(microdesc_cache_t *cache) +{ + if (!cache) + cache = get_microdesc_cache(); + if (!cache->n_seen) + return 512; + return (size_t)(cache->total_len_seen / cache->n_seen); +} + diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c index f4a0761f7b..507875ab2a 100644 --- a/src/or/networkstatus.c +++ b/src/or/networkstatus.c @@ -35,16 +35,22 @@ static networkstatus_t *current_consensus = NULL; /** A v3 consensus networkstatus that we've received, but which we don't * have enough certificates to be happy about. */ -static networkstatus_t *consensus_waiting_for_certs = NULL; -/** The encoded version of consensus_waiting_for_certs. */ -static char *consensus_waiting_for_certs_body = NULL; -/** When did we set the current value of consensus_waiting_for_certs? If this - * is too recent, we shouldn't try to fetch a new consensus for a little while, - * to give ourselves time to get certificates for this one. */ -static time_t consensus_waiting_for_certs_set_at = 0; -/** Set to 1 if we've been holding on to consensus_waiting_for_certs so long - * that we should treat it as maybe being bad. */ -static int consensus_waiting_for_certs_dl_failed = 0; +typedef struct consensus_waiting_for_certs_t { + /** The consensus itself. */ + networkstatus_t *consensus; + /** The encoded version of the consensus, nul-terminated. */ + char *body; + /** When did we set the current value of consensus_waiting_for_certs? If + * this is too recent, we shouldn't try to fetch a new consensus for a + * little while, to give ourselves time to get certificates for this one. */ + time_t set_at; + /** Set to 1 if we've been holding on to it for so long we should maybe + * treat it as being bad. */ + int dl_failed; +} consensus_waiting_for_certs_t; + +static consensus_waiting_for_certs_t + consensus_waiting_for_certs[N_CONSENSUS_FLAVORS]; /** The last time we tried to download a networkstatus, or 0 for "never". We * use this to rate-limit download attempts for directory caches (including @@ -56,7 +62,7 @@ static time_t last_networkstatus_download_attempted = 0; * before the current consensus becomes invalid. */ static time_t time_to_download_next_consensus = 0; /** Download status for the current consensus networkstatus. */ -static download_status_t consensus_dl_status = { 0, 0, DL_SCHED_CONSENSUS }; +static download_status_t consensus_dl_status[N_CONSENSUS_FLAVORS]; /** True iff we have logged a warning about this OR's version being older than * listed by the authorities. */ @@ -89,6 +95,7 @@ networkstatus_reset_warnings(void) void networkstatus_reset_download_failures(void) { + int i; const smartlist_t *networkstatus_v2_list = networkstatus_get_v2_list(); SMARTLIST_FOREACH(networkstatus_v2_list, networkstatus_v2_t *, ns, SMARTLIST_FOREACH(ns->entries, routerstatus_t *, rs, @@ -97,7 +104,8 @@ networkstatus_reset_download_failures(void) rs->need_to_mirror = 1; }));; - download_status_reset(&consensus_dl_status); + for (i=0; i < N_CONSENSUS_FLAVORS; ++i) + download_status_reset(&consensus_dl_status[i]); if (v2_download_status_map) { digestmap_iter_t *iter; digestmap_t *map = v2_download_status_map; @@ -170,7 +178,7 @@ router_reload_v2_networkstatus(void) return 0; } -/** Read the cached v3 consensus networkstatus from the disk. */ +/** Read every cached v3 consensus networkstatus from the disk. */ int router_reload_consensus_networkstatus(void) { @@ -179,31 +187,46 @@ router_reload_consensus_networkstatus(void) struct stat st; or_options_t *options = get_options(); const unsigned int flags = NSSET_FROM_CACHE | NSSET_DONT_DOWNLOAD_CERTS; + int flav; /* FFFF Suppress warnings if cached consensus is bad? */ + for (flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) { + char buf[128]; + const char *flavor = networkstatus_get_flavor_name(flav); + if (flav == FLAV_NS) { + filename = get_datadir_fname("cached-consensus"); + } else { + tor_snprintf(buf, sizeof(buf), "cached-%s-consensus", flavor); + filename = get_datadir_fname(buf); + } + s = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL); + if (s) { + if (networkstatus_set_current_consensus(s, flavor, flags) < -1) { + log_warn(LD_FS, "Couldn't load consensus %s networkstatus from \"%s\"", + flavor, filename); + } + tor_free(s); + } + tor_free(filename); - filename = get_datadir_fname("cached-consensus"); - s = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL); - if (s) { - if (networkstatus_set_current_consensus(s, flags) < -1) { - log_warn(LD_FS, "Couldn't load consensus networkstatus from \"%s\"", - filename); + if (flav == FLAV_NS) { + filename = get_datadir_fname("unverified-consensus"); + } else { + tor_snprintf(buf, sizeof(buf), "unverified-%s-consensus", flavor); + filename = get_datadir_fname(buf); } - tor_free(s); - } - tor_free(filename); - filename = get_datadir_fname("unverified-consensus"); - s = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL); - if (s) { - if (networkstatus_set_current_consensus(s, + s = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL); + if (s) { + if (networkstatus_set_current_consensus(s, flavor, flags|NSSET_WAS_WAITING_FOR_CERTS)) { - log_info(LD_FS, "Couldn't load consensus networkstatus from \"%s\"", - filename); + log_info(LD_FS, "Couldn't load consensus %s networkstatus from \"%s\"", + flavor, filename); + } + tor_free(s); } - tor_free(s); + tor_free(filename); } - tor_free(filename); if (!current_consensus || (stat(options->FallbackNetworkstatusFile, &st)==0 && @@ -211,7 +234,7 @@ router_reload_consensus_networkstatus(void) s = read_file_to_str(options->FallbackNetworkstatusFile, RFTS_IGNORE_MISSING, NULL); if (s) { - if (networkstatus_set_current_consensus(s, + if (networkstatus_set_current_consensus(s, "ns", flags|NSSET_ACCEPT_OBSOLETE)) { log_info(LD_FS, "Couldn't load consensus networkstatus from \"%s\"", options->FallbackNetworkstatusFile); @@ -242,8 +265,16 @@ router_reload_consensus_networkstatus(void) static void vote_routerstatus_free(vote_routerstatus_t *rs) { + vote_microdesc_hash_t *h, *next; + if (!rs) + return; tor_free(rs->version); tor_free(rs->status.exitsummary); + for (h = rs->microdesc; h; h = next) { + tor_free(h->microdesc_hash_line); + next = h->next; + tor_free(h); + } tor_free(rs); } @@ -251,6 +282,8 @@ vote_routerstatus_free(vote_routerstatus_t *rs) void routerstatus_free(routerstatus_t *rs) { + if (!rs) + return; tor_free(rs->exitsummary); tor_free(rs); } @@ -259,6 +292,8 @@ routerstatus_free(routerstatus_t *rs) void networkstatus_v2_free(networkstatus_v2_t *ns) { + if (!ns) + return; tor_free(ns->source_address); tor_free(ns->contact); if (ns->signing_key) @@ -273,7 +308,25 @@ networkstatus_v2_free(networkstatus_v2_t *ns) tor_free(ns); } -/** Clear all storage held in <b>ns</b>. */ +/** Free all storage held in <b>sig</b> */ +void +document_signature_free(document_signature_t *sig) +{ + tor_free(sig->signature); + tor_free(sig); +} + +/** Return a newly allocated copy of <b>sig</b> */ +document_signature_t * +document_signature_dup(const document_signature_t *sig) +{ + document_signature_t *r = tor_memdup(sig, sizeof(document_signature_t)); + if (r->signature) + r->signature = tor_memdup(sig->signature, sig->signature_len); + return r; +} + +/** Free all storage held in <b>ns</b>. */ void networkstatus_vote_free(networkstatus_t *ns) { @@ -295,18 +348,20 @@ networkstatus_vote_free(networkstatus_t *ns) smartlist_free(ns->supported_methods); } if (ns->voters) { - SMARTLIST_FOREACH(ns->voters, networkstatus_voter_info_t *, voter, - { + SMARTLIST_FOREACH_BEGIN(ns->voters, networkstatus_voter_info_t *, voter) { tor_free(voter->nickname); tor_free(voter->address); tor_free(voter->contact); - tor_free(voter->signature); + if (voter->sigs) { + SMARTLIST_FOREACH(voter->sigs, document_signature_t *, sig, + document_signature_free(sig)); + smartlist_free(voter->sigs); + } tor_free(voter); - }); + } SMARTLIST_FOREACH_END(voter); smartlist_free(ns->voters); } - if (ns->cert) - authority_cert_free(ns->cert); + authority_cert_free(ns->cert); if (ns->routerstatus_list) { if (ns->type == NS_TYPE_VOTE || ns->type == NS_TYPE_OPINION) { @@ -319,8 +374,8 @@ networkstatus_vote_free(networkstatus_t *ns) smartlist_free(ns->routerstatus_list); } - if (ns->desc_digest_map) - digestmap_free(ns->desc_digest_map, NULL); + + digestmap_free(ns->desc_digest_map, NULL); memset(ns, 11, sizeof(*ns)); tor_free(ns); @@ -341,34 +396,38 @@ networkstatus_get_voter_by_id(networkstatus_t *vote, return NULL; } -/** Check whether the signature on <b>voter</b> is correctly signed by - * the signing key of <b>cert</b>. Return -1 if <b>cert</b> doesn't match the +/** Check whether the signature <b>sig</b> is correctly signed with the + * signing key in <b>cert</b>. Return -1 if <b>cert</b> doesn't match the * signing key; otherwise set the good_signature or bad_signature flag on * <b>voter</b>, and return 0. */ -/* (private; exposed for testing.) */ int -networkstatus_check_voter_signature(networkstatus_t *consensus, - networkstatus_voter_info_t *voter, - authority_cert_t *cert) +networkstatus_check_document_signature(const networkstatus_t *consensus, + document_signature_t *sig, + const authority_cert_t *cert) { - char d[DIGEST_LEN]; + char key_digest[DIGEST_LEN]; + const int dlen = sig->alg == DIGEST_SHA1 ? DIGEST_LEN : DIGEST256_LEN; char *signed_digest; size_t signed_digest_len; - if (crypto_pk_get_digest(cert->signing_key, d)<0) + + if (crypto_pk_get_digest(cert->signing_key, key_digest)<0) return -1; - if (memcmp(voter->signing_key_digest, d, DIGEST_LEN)) + if (memcmp(sig->signing_key_digest, key_digest, DIGEST_LEN) || + memcmp(sig->identity_digest, cert->cache_info.identity_digest, + DIGEST_LEN)) return -1; + signed_digest_len = crypto_pk_keysize(cert->signing_key); signed_digest = tor_malloc(signed_digest_len); if (crypto_pk_public_checksig(cert->signing_key, signed_digest, - voter->signature, - voter->signature_len) != DIGEST_LEN || - memcmp(signed_digest, consensus->networkstatus_digest, DIGEST_LEN)) { + sig->signature, + sig->signature_len) < dlen || + memcmp(signed_digest, consensus->digests.d[sig->alg], dlen)) { log_warn(LD_DIR, "Got a bad signature on a networkstatus vote"); - voter->bad_signature = 1; + sig->bad_signature = 1; } else { - voter->good_signature = 1; + sig->good_signature = 1; } tor_free(signed_digest); return 0; @@ -401,37 +460,52 @@ networkstatus_check_consensus_signature(networkstatus_t *consensus, tor_assert(consensus->type == NS_TYPE_CONSENSUS); - SMARTLIST_FOREACH(consensus->voters, networkstatus_voter_info_t *, voter, - { - if (!voter->good_signature && !voter->bad_signature && voter->signature) { - /* we can try to check the signature. */ - int is_v3_auth = trusteddirserver_get_by_v3_auth_digest( - voter->identity_digest) != NULL; - authority_cert_t *cert = - authority_cert_get_by_digests(voter->identity_digest, - voter->signing_key_digest); - if (!is_v3_auth) { - smartlist_add(unrecognized, voter); - ++n_unknown; - continue; - } else if (!cert || cert->expires < now) { - smartlist_add(need_certs_from, voter); - ++n_missing_key; - continue; - } - if (networkstatus_check_voter_signature(consensus, voter, cert) < 0) { - smartlist_add(need_certs_from, voter); - ++n_missing_key; - continue; + SMARTLIST_FOREACH_BEGIN(consensus->voters, networkstatus_voter_info_t *, + voter) { + int good_here = 0; + int bad_here = 0; + int missing_key_here = 0; + SMARTLIST_FOREACH_BEGIN(voter->sigs, document_signature_t *, sig) { + if (!sig->good_signature && !sig->bad_signature && + sig->signature) { + /* we can try to check the signature. */ + int is_v3_auth = trusteddirserver_get_by_v3_auth_digest( + sig->identity_digest) != NULL; + authority_cert_t *cert = + authority_cert_get_by_digests(sig->identity_digest, + sig->signing_key_digest); + tor_assert(!memcmp(sig->identity_digest, voter->identity_digest, + DIGEST_LEN)); + + if (!is_v3_auth) { + smartlist_add(unrecognized, voter); + ++n_unknown; + continue; + } else if (!cert || cert->expires < now) { + smartlist_add(need_certs_from, voter); + ++missing_key_here; + continue; + } + if (networkstatus_check_document_signature(consensus, sig, cert) < 0) { + smartlist_add(need_certs_from, voter); + ++missing_key_here; + continue; + } } - } - if (voter->good_signature) + if (sig->good_signature) + ++good_here; + else if (sig->bad_signature) + ++bad_here; + } SMARTLIST_FOREACH_END(sig); + if (good_here) ++n_good; - else if (voter->bad_signature) + else if (bad_here) ++n_bad; + else if (missing_key_here) + ++n_missing_key; else ++n_no_signature; - }); + } SMARTLIST_FOREACH_END(voter); /* Now see whether we're missing any voters entirely. */ SMARTLIST_FOREACH(router_get_trusted_dir_servers(), @@ -784,8 +858,8 @@ networkstatus_v2_list_clean(time_t now) /** Helper for bsearching a list of routerstatus_t pointers: compare a * digest in the key to the identity digest of a routerstatus_t. */ -static int -_compare_digest_to_routerstatus_entry(const void *_key, const void **_member) +int +compare_digest_to_routerstatus_entry(const void *_key, const void **_member) { const char *key = _key; const routerstatus_t *rs = *_member; @@ -798,7 +872,7 @@ routerstatus_t * networkstatus_v2_find_entry(networkstatus_v2_t *ns, const char *digest) { return smartlist_bsearch(ns->entries, digest, - _compare_digest_to_routerstatus_entry); + compare_digest_to_routerstatus_entry); } /** Return the entry in <b>ns</b> for the identity digest <b>digest</b>, or @@ -807,7 +881,7 @@ routerstatus_t * networkstatus_vote_find_entry(networkstatus_t *ns, const char *digest) { return smartlist_bsearch(ns->routerstatus_list, digest, - _compare_digest_to_routerstatus_entry); + compare_digest_to_routerstatus_entry); } /*XXXX make this static once functions are moved into this file. */ @@ -819,7 +893,7 @@ networkstatus_vote_find_entry_idx(networkstatus_t *ns, const char *digest, int *found_out) { return smartlist_bsearch_idx(ns->routerstatus_list, digest, - _compare_digest_to_routerstatus_entry, + compare_digest_to_routerstatus_entry, found_out); } @@ -872,7 +946,7 @@ router_get_consensus_status_by_id(const char *digest) if (!current_consensus) return NULL; return smartlist_bsearch(current_consensus->routerstatus_list, digest, - _compare_digest_to_routerstatus_entry); + compare_digest_to_routerstatus_entry); } /** Given a nickname (possibly verbose, possibly a hexadecimal digest), return @@ -1077,27 +1151,32 @@ static void update_consensus_networkstatus_downloads(time_t now) { or_options_t *options = get_options(); + int i; if (!networkstatus_get_live_consensus(now)) time_to_download_next_consensus = now; /* No live consensus? Get one now!*/ if (time_to_download_next_consensus > now) return; /* Wait until the current consensus is older. */ if (authdir_mode_v3(options)) return; /* Authorities never fetch a consensus */ - if (!download_status_is_ready(&consensus_dl_status, now, + /* XXXXNM Microdescs: may need to download more types. */ + if (!download_status_is_ready(&consensus_dl_status[FLAV_NS], now, CONSENSUS_NETWORKSTATUS_MAX_DL_TRIES)) return; /* We failed downloading a consensus too recently. */ if (connection_get_by_type_purpose(CONN_TYPE_DIR, DIR_PURPOSE_FETCH_CONSENSUS)) return; /* There's an in-progress download.*/ - if (consensus_waiting_for_certs) { - /* XXXX make sure this doesn't delay sane downloads. */ - if (consensus_waiting_for_certs_set_at + DELAY_WHILE_FETCHING_CERTS > now) - return; /* We're still getting certs for this one. */ - else { - if (!consensus_waiting_for_certs_dl_failed) { - download_status_failed(&consensus_dl_status, 0); - consensus_waiting_for_certs_dl_failed=1; + for (i=0; i < N_CONSENSUS_FLAVORS; ++i) { + consensus_waiting_for_certs_t *waiting = &consensus_waiting_for_certs[i]; + if (waiting->consensus) { + /* XXXX make sure this doesn't delay sane downloads. */ + if (waiting->set_at + DELAY_WHILE_FETCHING_CERTS > now) + return; /* We're still getting certs for this one. */ + else { + if (!waiting->dl_failed) { + download_status_failed(&consensus_dl_status[FLAV_NS], 0); + waiting->dl_failed=1; + } } } } @@ -1113,7 +1192,8 @@ update_consensus_networkstatus_downloads(time_t now) void networkstatus_consensus_download_failed(int status_code) { - download_status_failed(&consensus_dl_status, status_code); + /* XXXXNM Microdescs: may need to handle more types. */ + download_status_failed(&consensus_dl_status[FLAV_NS], status_code); /* Retry immediately, if appropriate. */ update_consensus_networkstatus_downloads(time(NULL)); } @@ -1137,8 +1217,13 @@ update_consensus_networkstatus_fetch_time(time_t now) /* We want to cache the next one at some point after this one * is no longer fresh... */ start = c->fresh_until + CONSENSUS_MIN_SECONDS_BEFORE_CACHING; - /* But only in the first half-interval after that. */ - dl_interval = interval/2; + /* Some clients may need the consensus sooner than others. */ + if (options->FetchDirInfoExtraEarly) { + dl_interval = 60; + } else { + /* But only in the first half-interval after that. */ + dl_interval = interval/2; + } } else { /* We're an ordinary client or a bridge. Give all the caches enough * time to download the consensus. */ @@ -1158,7 +1243,7 @@ update_consensus_networkstatus_fetch_time(time_t now) } if (dl_interval < 1) dl_interval = 1; - /* We must not try to replace c while it's still the most valid: */ + /* We must not try to replace c while it's still fresh: */ tor_assert(c->fresh_until < start); /* We must download the next one before c is invalid: */ tor_assert(start+dl_interval < c->valid_until); @@ -1214,10 +1299,14 @@ update_networkstatus_downloads(time_t now) void update_certificate_downloads(time_t now) { - if (consensus_waiting_for_certs) - authority_certs_fetch_missing(consensus_waiting_for_certs, now); - else - authority_certs_fetch_missing(current_consensus, now); + int i; + for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) { + if (consensus_waiting_for_certs[i].consensus) + authority_certs_fetch_missing(consensus_waiting_for_certs[i].consensus, + now); + } + + authority_certs_fetch_missing(current_consensus, now); } /** Return 1 if we have a consensus but we don't have enough certificates @@ -1225,7 +1314,8 @@ update_certificate_downloads(time_t now) int consensus_is_waiting_for_certs(void) { - return consensus_waiting_for_certs ? 1 : 0; + return consensus_waiting_for_certs[USABLE_CONSENSUS_FLAVOR].consensus + ? 1 : 0; } /** Return the network status with a given identity digest. */ @@ -1394,16 +1484,29 @@ networkstatus_copy_old_consensus_info(networkstatus_t *new_c, * user, and -2 for more serious problems. */ int -networkstatus_set_current_consensus(const char *consensus, unsigned flags) +networkstatus_set_current_consensus(const char *consensus, + const char *flavor, + unsigned flags) { - networkstatus_t *c; + networkstatus_t *c=NULL; int r, result = -1; time_t now = time(NULL); char *unverified_fname = NULL, *consensus_fname = NULL; + int flav = networkstatus_parse_flavor_name(flavor); const unsigned from_cache = flags & NSSET_FROM_CACHE; const unsigned was_waiting_for_certs = flags & NSSET_WAS_WAITING_FOR_CERTS; const unsigned dl_certs = !(flags & NSSET_DONT_DOWNLOAD_CERTS); const unsigned accept_obsolete = flags & NSSET_ACCEPT_OBSOLETE; + const unsigned require_flavor = flags & NSSET_REQUIRE_FLAVOR; + const digests_t *current_digests = NULL; + consensus_waiting_for_certs_t *waiting = NULL; + time_t current_valid_after = 0; + + if (flav < 0) { + /* XXXX we don't handle unrecognized flavors yet. */ + log_warn(LD_BUG, "Unrecognized consensus flavor %s", flavor); + return -2; + } /* Make sure it's parseable. */ c = networkstatus_parse_vote_from_string(consensus, NULL, NS_TYPE_CONSENSUS); @@ -1413,33 +1516,70 @@ networkstatus_set_current_consensus(const char *consensus, unsigned flags) goto done; } + if ((int)c->flavor != flav) { + /* This wasn't the flavor we thought we were getting. */ + if (require_flavor) { + log_warn(LD_DIR, "Got consensus with unexpected flavor %s (wanted %s)", + networkstatus_get_flavor_name(c->flavor), flavor); + goto done; + } + flav = c->flavor; + flavor = networkstatus_get_flavor_name(flav); + } + + if (flav != USABLE_CONSENSUS_FLAVOR && + !directory_caches_dir_info(get_options())) { + /* This consensus is totally boring to us: we won't use it, and we won't + * serve it. Drop it. */ + result = -1; + goto done; + } + if (from_cache && !accept_obsolete && c->valid_until < now-OLD_ROUTER_DESC_MAX_AGE) { /* XXX022 when we try to make fallbackconsensus work again, we should * consider taking this out. Until then, believing obsolete consensuses * is causing more harm than good. See also bug 887. */ - log_info(LD_DIR, "Loaded an obsolete consensus. Discarding."); + log_info(LD_DIR, "Loaded an expired consensus. Discarding."); goto done; } - if (current_consensus && - !memcmp(c->networkstatus_digest, current_consensus->networkstatus_digest, - DIGEST_LEN)) { + if (!strcmp(flavor, "ns")) { + consensus_fname = get_datadir_fname("cached-consensus"); + unverified_fname = get_datadir_fname("unverified-consensus"); + if (current_consensus) { + current_digests = ¤t_consensus->digests; + current_valid_after = current_consensus->valid_after; + } + } else { + cached_dir_t *cur; + char buf[128]; + tor_snprintf(buf, sizeof(buf), "cached-%s-consensus", flavor); + consensus_fname = get_datadir_fname(buf); + tor_snprintf(buf, sizeof(buf), "unverified-%s-consensus", flavor); + unverified_fname = get_datadir_fname(buf); + cur = dirserv_get_consensus(flavor); + if (cur) { + current_digests = &cur->digests; + current_valid_after = cur->published; + } + } + + if (current_digests && + !memcmp(&c->digests, current_digests, sizeof(c->digests))) { /* We already have this one. That's a failure. */ - log_info(LD_DIR, "Got a consensus we already have"); + log_info(LD_DIR, "Got a %s consensus we already have", flavor); goto done; } - if (current_consensus && c->valid_after <= current_consensus->valid_after) { + if (current_valid_after && c->valid_after <= current_valid_after) { /* We have a newer one. There's no point in accepting this one, * even if it's great. */ - log_info(LD_DIR, "Got a consensus at least as old as the one we have"); + log_info(LD_DIR, "Got a %s consensus at least as old as the one we have", + flavor); goto done; } - consensus_fname = get_datadir_fname("cached-consensus"); - unverified_fname = get_datadir_fname("unverified-consensus"); - /* Make sure it's signed enough. */ if ((r=networkstatus_check_consensus_signature(c, 1))<0) { if (r == -1) { @@ -1448,16 +1588,16 @@ networkstatus_set_current_consensus(const char *consensus, unsigned flags) log_info(LD_DIR, "Not enough certificates to check networkstatus consensus"); } - if (!current_consensus || - c->valid_after > current_consensus->valid_after) { - if (consensus_waiting_for_certs) - networkstatus_vote_free(consensus_waiting_for_certs); - tor_free(consensus_waiting_for_certs_body); - consensus_waiting_for_certs = c; + if (!current_valid_after || + c->valid_after > current_valid_after) { + waiting = &consensus_waiting_for_certs[flav]; + networkstatus_vote_free(waiting->consensus); + tor_free(waiting->body); + waiting->consensus = c; c = NULL; /* Prevent free. */ - consensus_waiting_for_certs_body = tor_strdup(consensus); - consensus_waiting_for_certs_set_at = now; - consensus_waiting_for_certs_dl_failed = 0; + waiting->body = tor_strdup(consensus); + waiting->set_at = now; + waiting->dl_failed = 0; if (!from_cache) { write_str_to_file(unverified_fname, consensus, 0); } @@ -1486,56 +1626,66 @@ networkstatus_set_current_consensus(const char *consensus, unsigned flags) } } - if (!from_cache) + if (!from_cache && flav == USABLE_CONSENSUS_FLAVOR) control_event_client_status(LOG_NOTICE, "CONSENSUS_ARRIVED"); /* Are we missing any certificates at all? */ if (r != 1 && dl_certs) authority_certs_fetch_missing(c, now); - notify_control_networkstatus_changed(current_consensus, c); + if (flav == USABLE_CONSENSUS_FLAVOR) { + notify_control_networkstatus_changed(current_consensus, c); - if (current_consensus) { - networkstatus_copy_old_consensus_info(c, current_consensus); - networkstatus_vote_free(current_consensus); + if (current_consensus) { + networkstatus_copy_old_consensus_info(c, current_consensus); + networkstatus_vote_free(current_consensus); + } } - if (consensus_waiting_for_certs && - consensus_waiting_for_certs->valid_after <= c->valid_after) { - networkstatus_vote_free(consensus_waiting_for_certs); - consensus_waiting_for_certs = NULL; - if (consensus != consensus_waiting_for_certs_body) - tor_free(consensus_waiting_for_certs_body); + waiting = &consensus_waiting_for_certs[flav]; + if (waiting->consensus && + waiting->consensus->valid_after <= c->valid_after) { + networkstatus_vote_free(waiting->consensus); + waiting->consensus = NULL; + if (consensus != waiting->body) + tor_free(waiting->body); else - consensus_waiting_for_certs_body = NULL; - consensus_waiting_for_certs_set_at = 0; - consensus_waiting_for_certs_dl_failed = 0; + waiting->body = NULL; + waiting->set_at = 0; + waiting->dl_failed = 0; unlink(unverified_fname); } /* Reset the failure count only if this consensus is actually valid. */ if (c->valid_after <= now && now <= c->valid_until) { - download_status_reset(&consensus_dl_status); + download_status_reset(&consensus_dl_status[flav]); } else { if (!from_cache) - download_status_failed(&consensus_dl_status, 0); + download_status_failed(&consensus_dl_status[flav], 0); } - current_consensus = c; - c = NULL; /* Prevent free. */ + if (directory_caches_dir_info(get_options())) { + dirserv_set_cached_consensus_networkstatus(consensus, + flavor, + &c->digests, + c->valid_after); + } + + if (flav == USABLE_CONSENSUS_FLAVOR) { + current_consensus = c; + c = NULL; /* Prevent free. */ - update_consensus_networkstatus_fetch_time(now); - dirvote_recalculate_timing(get_options(), now); - routerstatus_list_update_named_server_map(); + /* XXXXNM Microdescs: needs a non-ns variant. */ + update_consensus_networkstatus_fetch_time(now); + dirvote_recalculate_timing(get_options(), now); + routerstatus_list_update_named_server_map(); + cell_ewma_set_scale_factor(get_options(), current_consensus); + } if (!from_cache) { write_str_to_file(consensus_fname, consensus, 0); } - if (directory_caches_dir_info(get_options())) - dirserv_set_cached_networkstatus_v3(consensus, - current_consensus->valid_after); - if (ftime_definitely_before(now, current_consensus->valid_after)) { char tbuf[ISO_TIME_LEN+1]; char dbuf[64]; @@ -1554,8 +1704,7 @@ networkstatus_set_current_consensus(const char *consensus, unsigned flags) result = 0; done: - if (c) - networkstatus_vote_free(c); + networkstatus_vote_free(c); tor_free(consensus_fname); tor_free(unverified_fname); return result; @@ -1566,13 +1715,17 @@ networkstatus_set_current_consensus(const char *consensus, unsigned flags) void networkstatus_note_certs_arrived(void) { - if (consensus_waiting_for_certs) { - if (networkstatus_check_consensus_signature( - consensus_waiting_for_certs, 0)>=0) { + int i; + for (i=0; i<N_CONSENSUS_FLAVORS; ++i) { + consensus_waiting_for_certs_t *waiting = &consensus_waiting_for_certs[i]; + if (!waiting->consensus) + continue; + if (networkstatus_check_consensus_signature(waiting->consensus, 0)>=0) { if (!networkstatus_set_current_consensus( - consensus_waiting_for_certs_body, + waiting->body, + networkstatus_get_flavor_name(i), NSSET_WAS_WAITING_FOR_CERTS)) { - tor_free(consensus_waiting_for_certs_body); + tor_free(waiting->body); } } } @@ -1658,10 +1811,8 @@ download_status_map_update_from_v2_networkstatus(void) v2_download_status_map = digestmap_new(); dl_status = digestmap_new(); - SMARTLIST_FOREACH(networkstatus_v2_list, networkstatus_v2_t *, ns, - { - SMARTLIST_FOREACH(ns->entries, routerstatus_t *, rs, - { + SMARTLIST_FOREACH_BEGIN(networkstatus_v2_list, networkstatus_v2_t *, ns) { + SMARTLIST_FOREACH_BEGIN(ns->entries, routerstatus_t *, rs) { const char *d = rs->descriptor_digest; download_status_t *s; if (digestmap_get(dl_status, d)) @@ -1670,8 +1821,8 @@ download_status_map_update_from_v2_networkstatus(void) s = tor_malloc_zero(sizeof(download_status_t)); } digestmap_set(dl_status, d, s); - }); - }); + } SMARTLIST_FOREACH_END(rs); + } SMARTLIST_FOREACH_END(ns); digestmap_free(v2_download_status_map, _tor_free); v2_download_status_map = dl_status; networkstatus_v2_list_has_changed = 0; @@ -1685,11 +1836,9 @@ routerstatus_list_update_named_server_map(void) if (!current_consensus) return; - if (named_server_map) - strmap_free(named_server_map, _tor_free); + strmap_free(named_server_map, _tor_free); named_server_map = strmap_new(); - if (unnamed_server_map) - strmap_free(unnamed_server_map, NULL); + strmap_free(unnamed_server_map, NULL); unnamed_server_map = strmap_new(); SMARTLIST_FOREACH(current_consensus->routerstatus_list, routerstatus_t *, rs, { @@ -1826,7 +1975,7 @@ char * networkstatus_getinfo_helper_single(routerstatus_t *rs) { char buf[RS_ENTRY_LEN+1]; - routerstatus_format_entry(buf, sizeof(buf), rs, NULL, 0, 1); + routerstatus_format_entry(buf, sizeof(buf), rs, NULL, NS_CONTROL_PORT); return tor_strdup(buf); } @@ -1909,8 +2058,8 @@ networkstatus_get_param(networkstatus_t *ns, const char *param_name, SMARTLIST_FOREACH_BEGIN(ns->net_params, const char *, p) { if (!strcmpstart(p, param_name) && p[name_len] == '=') { int ok=0; - long v = tor_parse_long(p+name_len+1, 10, INT32_MIN, INT32_MAX, &ok, - NULL); + long v = tor_parse_long(p+name_len+1, 10, INT32_MIN, + INT32_MAX, &ok, NULL); if (ok) return (int32_t) v; } @@ -1919,6 +2068,35 @@ networkstatus_get_param(networkstatus_t *ns, const char *param_name, return default_val; } +/** Return the name of the consensus flavor <b>flav</b> as used to identify + * the flavor in directory documents. */ +const char * +networkstatus_get_flavor_name(consensus_flavor_t flav) +{ + switch (flav) { + case FLAV_NS: + return "ns"; + case FLAV_MICRODESC: + return "microdesc"; + default: + tor_fragile_assert(); + return "??"; + } +} + +/** Return the consensus_flavor_t value for the flavor called <b>flavname</b>, + * or -1 if the flavor is not recognized. */ +int +networkstatus_parse_flavor_name(const char *flavname) +{ + if (!strcmp(flavname, "ns")) + return FLAV_NS; + else if (!strcmp(flavname, "microdesc")) + return FLAV_MICRODESC; + else + return -1; +} + /** If <b>question</b> is a string beginning with "ns/" in a format the * control interface expects for a GETINFO question, set *<b>answer</b> to a * newly-allocated string containing networkstatus lines for the appropriate @@ -1970,30 +2148,29 @@ getinfo_helper_networkstatus(control_connection_t *conn, void networkstatus_free_all(void) { + int i; if (networkstatus_v2_list) { SMARTLIST_FOREACH(networkstatus_v2_list, networkstatus_v2_t *, ns, networkstatus_v2_free(ns)); smartlist_free(networkstatus_v2_list); networkstatus_v2_list = NULL; } - if (v2_download_status_map) { - digestmap_free(v2_download_status_map, _tor_free); - v2_download_status_map = NULL; - } - if (current_consensus) { - networkstatus_vote_free(current_consensus); - current_consensus = NULL; - } - if (consensus_waiting_for_certs) { - networkstatus_vote_free(consensus_waiting_for_certs); - consensus_waiting_for_certs = NULL; - } - tor_free(consensus_waiting_for_certs_body); - if (named_server_map) { - strmap_free(named_server_map, _tor_free); - } - if (unnamed_server_map) { - strmap_free(unnamed_server_map, NULL); + + digestmap_free(v2_download_status_map, _tor_free); + v2_download_status_map = NULL; + networkstatus_vote_free(current_consensus); + current_consensus = NULL; + + for (i=0; i < N_CONSENSUS_FLAVORS; ++i) { + consensus_waiting_for_certs_t *waiting = &consensus_waiting_for_certs[i]; + if (waiting->consensus) { + networkstatus_vote_free(waiting->consensus); + waiting->consensus = NULL; + } + tor_free(waiting->body); } + + strmap_free(named_server_map, _tor_free); + strmap_free(unnamed_server_map, NULL); } diff --git a/src/or/ntmain.c b/src/or/ntmain.c index 4f96fbe5d7..e5a855eec4 100644 --- a/src/or/ntmain.c +++ b/src/or/ntmain.c @@ -6,6 +6,12 @@ #define MAIN_PRIVATE #include "or.h" +#ifdef HAVE_EVENT2_EVENT_H +#include <event2/event.h> +#else +#include <event.h> +#endif + #include <tchar.h> #define GENSRV_SERVICENAME TEXT("tor") #define GENSRV_DISPLAYNAME TEXT("Tor Win32 Service") @@ -218,7 +224,7 @@ nt_service_control(DWORD request) log_notice(LD_GENERAL, "Got stop/shutdown request; shutting down cleanly."); service_status.dwCurrentState = SERVICE_STOP_PENDING; - event_loopexit(&exit_now); + event_base_loopexit(tor_libevent_get_base(), &exit_now); return; } service_fns.SetServiceStatus_fn(hStatus, &service_status); diff --git a/src/or/onion.c b/src/or/onion.c index b49a86aba3..f8913cd23b 100644 --- a/src/or/onion.c +++ b/src/or/onion.c @@ -58,11 +58,17 @@ onion_pending_add(or_circuit_t *circ, char *onionskin) tor_assert(!ol_tail->next); if (ol_length >= get_options()->MaxOnionsPending) { - log_warn(LD_GENERAL, - "Your computer is too slow to handle this many circuit " - "creation requests! Please consider using the " - "MaxAdvertisedBandwidth config option or choosing a more " - "restricted exit policy."); +#define WARN_TOO_MANY_CIRC_CREATIONS_INTERVAL (60) + static time_t last_warned = 0; + time_t now = time(NULL); + if (last_warned + WARN_TOO_MANY_CIRC_CREATIONS_INTERVAL < now) { + log_warn(LD_GENERAL, + "Your computer is too slow to handle this many circuit " + "creation requests! Please consider using the " + "MaxAdvertisedBandwidth config option or choosing a more " + "restricted exit policy."); + last_warned = now; + } tor_free(tmp); return -1; } @@ -253,8 +259,9 @@ onion_skin_server_handshake(const char *onion_skin, /*ONIONSKIN_CHALLENGE_LEN*/ key_material_len = DIGEST_LEN+key_out_len; key_material = tor_malloc(key_material_len); - len = crypto_dh_compute_secret(dh, challenge, DH_KEY_LEN, - key_material, key_material_len); + len = crypto_dh_compute_secret(LOG_PROTOCOL_WARN, dh, challenge, + DH_KEY_LEN, key_material, + key_material_len); if (len < 0) { log_info(LD_GENERAL, "crypto_dh_compute_secret failed."); goto err; @@ -304,8 +311,9 @@ onion_skin_client_handshake(crypto_dh_env_t *handshake_state, key_material_len = DIGEST_LEN + key_out_len; key_material = tor_malloc(key_material_len); - len = crypto_dh_compute_secret(handshake_state, handshake_reply, DH_KEY_LEN, - key_material, key_material_len); + len = crypto_dh_compute_secret(LOG_PROTOCOL_WARN, handshake_state, + handshake_reply, DH_KEY_LEN, key_material, + key_material_len); if (len < 0) goto err; diff --git a/src/or/or.h b/src/or/or.h index ae65127e36..091d819f79 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -20,9 +20,6 @@ #ifndef INSTRUMENT_DOWNLOADS #define INSTRUMENT_DOWNLOADS 1 #endif -#ifndef ENABLE_GEOIP_STATS -#define ENABLE_GEOIP_STATS 1 -#endif #endif #ifdef MS_WINDOWS @@ -85,14 +82,14 @@ #include "crypto.h" #include "tortls.h" -#include "log.h" +#include "../common/log.h" #include "compat.h" #include "container.h" #include "util.h" #include "torgzip.h" #include "address.h" - -#include <event.h> +#include "compat_libevent.h" +#include "ht.h" /* These signals are defined to help control_signal_act work. */ @@ -221,6 +218,21 @@ typedef enum { /* !!!! If _CONN_TYPE_MAX is ever over 15, we must grow the type field in * connection_t. */ +/* Proxy client types */ +#define PROXY_NONE 0 +#define PROXY_CONNECT 1 +#define PROXY_SOCKS4 2 +#define PROXY_SOCKS5 3 + +/* Proxy client handshake states */ +#define PROXY_HTTPS_WANT_CONNECT_OK 1 +#define PROXY_SOCKS4_WANT_CONNECT_OK 2 +#define PROXY_SOCKS5_WANT_AUTH_METHOD_NONE 3 +#define PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929 4 +#define PROXY_SOCKS5_WANT_AUTH_RFC1929_OK 5 +#define PROXY_SOCKS5_WANT_CONNECT_OK 6 +#define PROXY_CONNECTED 7 + /** True iff <b>x</b> is an edge connection. */ #define CONN_IS_EDGE(x) \ ((x)->type == CONN_TYPE_EXIT || (x)->type == CONN_TYPE_AP) @@ -241,26 +253,24 @@ typedef enum { #define _OR_CONN_STATE_MIN 1 /** State for a connection to an OR: waiting for connect() to finish. */ #define OR_CONN_STATE_CONNECTING 1 -/** State for a connection to an OR: waiting for proxy command to flush. */ -#define OR_CONN_STATE_PROXY_FLUSHING 2 -/** State for a connection to an OR: waiting for proxy response. */ -#define OR_CONN_STATE_PROXY_READING 3 +/** State for a connection to an OR: waiting for proxy handshake to complete */ +#define OR_CONN_STATE_PROXY_HANDSHAKING 2 /** State for a connection to an OR or client: SSL is handshaking, not done * yet. */ -#define OR_CONN_STATE_TLS_HANDSHAKING 4 +#define OR_CONN_STATE_TLS_HANDSHAKING 3 /** State for a connection to an OR: We're doing a second SSL handshake for * renegotiation purposes. */ -#define OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING 5 +#define OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING 4 /** State for a connection at an OR: We're waiting for the client to * renegotiate. */ -#define OR_CONN_STATE_TLS_SERVER_RENEGOTIATING 6 +#define OR_CONN_STATE_TLS_SERVER_RENEGOTIATING 5 /** State for a connection to an OR: We're done with our SSL handshake, but we * haven't yet negotiated link protocol versions and sent a netinfo cell. */ -#define OR_CONN_STATE_OR_HANDSHAKING 7 +#define OR_CONN_STATE_OR_HANDSHAKING 6 /** State for a connection to an OR: Ready to send/receive cells. */ -#define OR_CONN_STATE_OPEN 8 -#define _OR_CONN_STATE_MAX 8 +#define OR_CONN_STATE_OPEN 7 +#define _OR_CONN_STATE_MAX 7 #define _EXIT_CONN_STATE_MIN 1 /** State for an exit connection: waiting for response from DNS farm. */ @@ -483,7 +493,7 @@ typedef enum { #define CIRCUIT_PURPOSE_IS_ORIGIN(p) ((p)>_CIRCUIT_PURPOSE_OR_MAX) /** True iff the circuit purpose <b>p</b> is for a circuit that originated * here to serve as a client. (Hidden services don't count here.) */ -#define CIRCUIT_PURPOSE_IS_CLIENT(p) \ +#define CIRCUIT_PURPOSE_IS_CLIENT(p) \ ((p)> _CIRCUIT_PURPOSE_OR_MAX && \ (p)<=_CIRCUIT_PURPOSE_C_MAX) /** True iff the circuit_t <b>c</b> is actually an origin_circuit_t. */ @@ -641,10 +651,6 @@ typedef enum { /** Length of a binary-encoded rendezvous service ID. */ #define REND_SERVICE_ID_LEN 10 -/** How long after we receive a hidden service descriptor do we consider - * it fresh? */ -#define NUM_SECONDS_BEFORE_HS_REFETCH (60*15) - /** Time period for which a v2 descriptor will be valid. */ #define REND_TIME_PERIOD_V2_DESC_VALIDITY (24*60*60) @@ -739,12 +745,6 @@ typedef struct rend_data_t { /** Rendezvous cookie used by both, client and service. */ char rend_cookie[REND_COOKIE_LEN]; - - /** Rendezvous descriptor version that is used by a service. Used to - * distinguish introduction and rendezvous points belonging to the same - * rendezvous service ID, but different descriptor versions. - */ - uint8_t rend_desc_version; } rend_data_t; /** Time interval for tracking possible replays of INTRODUCE2 cells. @@ -851,12 +851,28 @@ typedef struct packed_cell_t { char body[CELL_NETWORK_SIZE]; /**< Cell as packed for network. */ } packed_cell_t; +/** Number of cells added to a circuit queue including their insertion + * time on 10 millisecond detail; used for buffer statistics. */ +typedef struct insertion_time_elem_t { + struct insertion_time_elem_t *next; /**< Next element in queue. */ + uint32_t insertion_time; /**< When were cells inserted (in 10 ms steps + * starting at 0:00 of the current day)? */ + unsigned counter; /**< How many cells were inserted? */ +} insertion_time_elem_t; + +/** Queue of insertion times. */ +typedef struct insertion_time_queue_t { + struct insertion_time_elem_t *first; /**< First element in queue. */ + struct insertion_time_elem_t *last; /**< Last element in queue. */ +} insertion_time_queue_t; + /** A queue of cells on a circuit, waiting to be added to the * or_connection_t's outbuf. */ typedef struct cell_queue_t { packed_cell_t *head; /**< The first cell, or NULL if the queue is empty. */ packed_cell_t *tail; /**< The last cell, or NULL if the queue is empty. */ int n; /**< The number of cells in the queue. */ + insertion_time_queue_t *insertion_times; /**< Insertion times of cells. */ } cell_queue_t; /** Beginning of a RELAY cell payload. */ @@ -912,7 +928,7 @@ typedef struct connection_t { * again once the bandwidth throttler allows it? */ unsigned int write_blocked_on_bw:1; /**< Boolean: should we start writing * again once the bandwidth throttler allows - * reads? */ + * writes? */ unsigned int hold_open_until_flushed:1; /**< Despite this connection's being * marked for close, do we flush it * before closing it? */ @@ -937,8 +953,11 @@ typedef struct connection_t { * connection. */ unsigned int linked_conn_is_closed:1; - int s; /**< Our socket; -1 if this connection is closed, or has no - * socket. */ + /** CONNECT/SOCKS proxy client handshake state (for outgoing connections). */ + unsigned int proxy_state:4; + + /** Our socket; -1 if this connection is closed, or has no socket. */ + evutil_socket_t s; int conn_array_index; /**< Index into the global connection array. */ struct event *read_event; /**< Libevent event structure. */ struct event *write_event; /**< Libevent event structure. */ @@ -980,6 +999,8 @@ typedef struct connection_t { * to the evdns_server_port is uses to listen to and answer connections. */ struct evdns_server_port *dns_server_port; + /** Unique ID for measuring tunneled network status requests. */ + uint64_t dirreq_id; } connection_t; /** Stores flags and information related to the portion of a v2 Tor OR @@ -1041,12 +1062,13 @@ typedef struct or_connection_t { time_t timestamp_last_added_nonpadding; /** When did we last add a * non-padding cell to the outbuf? */ - /* bandwidth* and read_bucket only used by ORs in OPEN state: */ + /* bandwidth* and *_bucket only used by ORs in OPEN state: */ int bandwidthrate; /**< Bytes/s added to the bucket. (OPEN ORs only.) */ int bandwidthburst; /**< Max bucket size for this conn. (OPEN ORs only.) */ int read_bucket; /**< When this hits 0, stop receiving. Every second we * add 'bandwidthrate' to this, capping it at * bandwidthburst. (OPEN ORs only) */ + int write_bucket; /**< When this hits 0, stop writing. Like read_bucket. */ int n_circuits; /**< How many circuits use this connection as p_conn or * n_conn ? */ @@ -1054,6 +1076,17 @@ typedef struct or_connection_t { * free up on this connection's outbuf. Every time we pull cells from a * circuit, we advance this pointer to the next circuit in the ring. */ struct circuit_t *active_circuits; + /** Priority queue of cell_ewma_t for circuits with queued cells waiting for + * room to free up on this connection's outbuf. Kept in heap order + * according to EWMA. + * + * This is redundant with active_circuits; if we ever decide only to use the + * cell_ewma algorithm for choosing circuits, we can remove active_circuits. + */ + smartlist_t *active_circuit_pqueue; + /** The tick on which the cell_ewma_ts in active_circuit_pqueue last had + * their ewma values rescaled. */ + unsigned active_circuit_pqueue_last_recalibrated; struct or_connection_t *next_with_same_id; /**< Next connection with same * identity digest as this one. */ } or_connection_t; @@ -1150,7 +1183,8 @@ typedef struct dir_connection_t { enum { DIR_SPOOL_NONE=0, DIR_SPOOL_SERVER_BY_DIGEST, DIR_SPOOL_SERVER_BY_FP, DIR_SPOOL_EXTRA_BY_DIGEST, DIR_SPOOL_EXTRA_BY_FP, - DIR_SPOOL_CACHED_DIR, DIR_SPOOL_NETWORKSTATUS + DIR_SPOOL_CACHED_DIR, DIR_SPOOL_NETWORKSTATUS, + DIR_SPOOL_MICRODESC, /* NOTE: if we add another entry, add another bit. */ } dir_spool_src : 3; /** If we're fetching descriptors, what router purpose shall we assign * to them? */ @@ -1178,12 +1212,6 @@ typedef struct control_connection_t { uint32_t event_mask; /**< Bitfield: which events does this controller * care about? */ - unsigned int use_long_names:1; /**< True if we should use long nicknames - * on this (v1) connection. Only settable - * via v1 controllers. */ - /** For control connections only. If set, we send extended info with control - * events as appropriate. */ - unsigned int use_extended_events:1; /** True if we have sent a protocolinfo reply on this connection. */ unsigned int have_sent_protocolinfo:1; @@ -1267,6 +1295,7 @@ typedef struct cached_dir_t { size_t dir_len; /**< Length of <b>dir</b> (not counting its NUL). */ size_t dir_z_len; /**< Length of <b>dir_z</b>. */ time_t published; /**< When was this object published. */ + digests_t digests; /**< Digests of this object (networkstatus only) */ int refcnt; /**< Reference count for this cached_dir_t. */ } cached_dir_t; @@ -1516,6 +1545,9 @@ typedef struct routerstatus_t { unsigned int has_bandwidth:1; /**< The vote/consensus had bw info */ unsigned int has_exitsummary:1; /**< The vote/consensus had exit summaries */ + unsigned int has_measured_bw:1; /**< The vote/consensus had a measured bw */ + + uint32_t measured_bw; /**< Measured bandwidth (capacity) of the router */ uint32_t bandwidth; /**< Bandwidth (capacity) of the router as reported in * the vote/consensus, in kilobytes/sec. */ @@ -1540,6 +1572,52 @@ typedef struct routerstatus_t { } routerstatus_t; +/** A microdescriptor is the smallest amount of information needed to build a + * circuit through a router. They are generated by the directory authorities, + * using information from the uploaded routerinfo documents. They are not + * self-signed, but are rather authenticated by having their hash in a signed + * networkstatus document. */ +typedef struct microdesc_t { + /** Hashtable node, used to look up the microdesc by its digest. */ + HT_ENTRY(microdesc_t) node; + + /* Cache information */ + + /** When was this microdescriptor last listed in a consensus document? + * Once a microdesc has been unlisted long enough, we can drop it. + */ + time_t last_listed; + /** Where is this microdescriptor currently stored? */ + saved_location_t saved_location : 3; + /** If true, do not attempt to cache this microdescriptor on disk. */ + unsigned int no_save : 1; + /** If saved_location == SAVED_IN_CACHE, this field holds the offset of the + * microdescriptor in the cache. */ + off_t off; + + /* The string containing the microdesc. */ + + /** A pointer to the encoded body of the microdescriptor. If the + * saved_location is SAVED_IN_CACHE, then the body is a pointer into an + * mmap'd region. Otherwise, it is a malloc'd string. The string might not + * be NUL-terminated; take the length from <b>bodylen</b>. */ + char *body; + /** The length of the microdescriptor in <b>body</b>. */ + size_t bodylen; + /** A SHA256-digest of the microdescriptor. */ + char digest[DIGEST256_LEN]; + + /* Fields in the microdescriptor. */ + + /** As routerinfo_t.onion_pkey */ + crypto_pk_env_t *onion_pkey; + /** As routerinfo_t.family */ + smartlist_t *family; + /** Encoded exit policy summary */ + char *exitsummary; /**< exit policy summary - + * XXX this probably should not stay a string. */ +} microdesc_t; + /** How many times will we try to download a router's descriptor before giving * up? */ #define MAX_ROUTERDESC_DOWNLOAD_FAILURES 8 @@ -1582,6 +1660,11 @@ typedef struct networkstatus_v2_t { * sorted by identity_digest. */ } networkstatus_v2_t; +typedef struct vote_microdesc_hash_t { + struct vote_microdesc_hash_t *next; + char *microdesc_hash_line; +} vote_microdesc_hash_t; + /** The claim about a single router, made in a vote. */ typedef struct vote_routerstatus_t { routerstatus_t status; /**< Underlying 'status' object for this router. @@ -1590,31 +1673,45 @@ typedef struct vote_routerstatus_t { * networkstatus_t.known_flags. */ char *version; /**< The version that the authority says this router is * running. */ + vote_microdesc_hash_t *microdesc; } vote_routerstatus_t; +/** A signature of some document by an authority. */ +typedef struct document_signature_t { + /** Declared SHA-1 digest of this voter's identity key */ + char identity_digest[DIGEST_LEN]; + /** Declared SHA-1 digest of signing key used by this voter. */ + char signing_key_digest[DIGEST_LEN]; + /** Algorithm used to compute the digest of the document. */ + digest_algorithm_t alg; + /** Signature of the signed thing. */ + char *signature; + /** Length of <b>signature</b> */ + int signature_len; + unsigned int bad_signature : 1; /**< Set to true if we've tried to verify + * the sig, and we know it's bad. */ + unsigned int good_signature : 1; /**< Set to true if we've verified the sig + * as good. */ +} document_signature_t; + /** Information about a single voter in a vote or a consensus. */ typedef struct networkstatus_voter_info_t { + /** Declared SHA-1 digest of this voter's identity key */ + char identity_digest[DIGEST_LEN]; char *nickname; /**< Nickname of this voter */ - char identity_digest[DIGEST_LEN]; /**< Digest of this voter's identity key */ + /** Digest of this voter's "legacy" identity key, if any. In vote only; for + * consensuses, we treat legacy keys as additional signers. */ + char legacy_id_digest[DIGEST_LEN]; char *address; /**< Address of this voter, in string format. */ uint32_t addr; /**< Address of this voter, in IPv4, in host order. */ uint16_t dir_port; /**< Directory port of this voter */ uint16_t or_port; /**< OR port of this voter */ char *contact; /**< Contact information for this voter. */ char vote_digest[DIGEST_LEN]; /**< Digest of this voter's vote, as signed. */ - /** Digest of this voter's "legacy" identity key, if any. In vote only; for - * consensuses, we treat legacy keys as additional signers. */ - char legacy_id_digest[DIGEST_LEN]; /* Nothing from here on is signed. */ - char signing_key_digest[DIGEST_LEN]; /**< Declared digest of signing key - * used by this voter. */ - char *signature; /**< Signature from this voter. */ - int signature_len; /**< Length of <b>signature</b> */ - unsigned int bad_signature : 1; /**< Set to true if we've tried to verify - * the sig, and we know it's bad. */ - unsigned int good_signature : 1; /**< Set to true if we've verified the sig - * as good. */ + /** The signature of the document and the signature's status. */ + smartlist_t *sigs; } networkstatus_voter_info_t; /** Enumerates the possible seriousness values of a networkstatus document. */ @@ -1624,10 +1721,25 @@ typedef enum { NS_TYPE_OPINION, } networkstatus_type_t; +/** Enumerates recognized flavors of a consensus networkstatus document. All + * flavors of a consensus are generated from the same set of votes, but they + * present different types information to different versions of Tor. */ +typedef enum { + FLAV_NS = 0, + FLAV_MICRODESC = 1, +} consensus_flavor_t; + +/** Which consensus flavor do we actually want to use to build circuits? */ +#define USABLE_CONSENSUS_FLAVOR FLAV_NS + +/** How many different consensus flavors are there? */ +#define N_CONSENSUS_FLAVORS ((int)(FLAV_MICRODESC)+1) + /** A common structure to hold a v3 network status vote, or a v3 network * status consensus. */ typedef struct networkstatus_t { - networkstatus_type_t type; /**< Vote, consensus, or opinion? */ + networkstatus_type_t type : 8; /**< Vote, consensus, or opinion? */ + consensus_flavor_t flavor : 8; /**< If a consensus, what kind? */ time_t published; /**< Vote only: Time when vote was written. */ time_t valid_after; /**< Time after which this vote or consensus applies. */ time_t fresh_until; /**< Time before which this is the most recent vote or @@ -1666,8 +1778,8 @@ typedef struct networkstatus_t { struct authority_cert_t *cert; /**< Vote only: the voter's certificate. */ - /** Digest of this document, as signed. */ - char networkstatus_digest[DIGEST_LEN]; + /** Digests of this document, as signed. */ + digests_t digests; /** List of router statuses, sorted by identity digest. For a vote, * the elements are vote_routerstatus_t; for a consensus, the elements @@ -1679,14 +1791,15 @@ typedef struct networkstatus_t { digestmap_t *desc_digest_map; } networkstatus_t; -/** A set of signatures for a networkstatus consensus. All fields are as for - * networkstatus_t. */ +/** A set of signatures for a networkstatus consensus. Unless otherwise + * noted, all fields are as for networkstatus_t. */ typedef struct ns_detached_signatures_t { time_t valid_after; time_t fresh_until; time_t valid_until; - char networkstatus_digest[DIGEST_LEN]; - smartlist_t *signatures; /* list of networkstatus_voter_info_t */ + strmap_t *digests; /**< Map from flavor name to digestset_t */ + strmap_t *signatures; /**< Map from flavor name to list of + * document_signature_t */ } ns_detached_signatures_t; /** Allowable types of desc_store_t. */ @@ -1891,6 +2004,29 @@ typedef struct { time_t expiry_time; } cpath_build_state_t; +/** + * The cell_ewma_t structure keeps track of how many cells a circuit has + * transferred recently. It keeps an EWMA (exponentially weighted moving + * average) of the number of cells flushed from the circuit queue onto a + * connection in connection_or_flush_from_first_active_circuit(). + */ +typedef struct { + /** The last 'tick' at which we recalibrated cell_count. + * + * A cell sent at exactly the start of this tick has weight 1.0. Cells sent + * since the start of this tick have weight greater than 1.0; ones sent + * earlier have less weight. */ + unsigned last_adjusted_tick; + /** The EWMA of the cell count. */ + double cell_count; + /** True iff this is the cell count for a circuit's previous + * connection. */ + unsigned int is_for_p_conn : 1; + /** The position of the circuit within the OR connection's priority + * queue. */ + int heap_index; +} cell_ewma_t; + #define ORIGIN_CIRCUIT_MAGIC 0x35315243u #define OR_CIRCUIT_MAGIC 0x98ABC04Fu @@ -1960,6 +2096,7 @@ typedef struct circuit_t { time_t timestamp_created; /**< When was this circuit created? */ time_t timestamp_dirty; /**< When the circuit was first used, or 0 if the * circuit is clean. */ + struct timeval highres_created; /**< When exactly was the circuit created? */ uint16_t marked_for_close; /**< Should we close this circuit at the end of * the main loop? (If true, holds the line number @@ -1976,6 +2113,14 @@ typedef struct circuit_t { * linked to an OR connection. */ struct circuit_t *prev_active_on_n_conn; struct circuit_t *next; /**< Next circuit in linked list of all circuits. */ + + /** Unique ID for measuring tunneled network status requests. */ + uint64_t dirreq_id; + + /** The EWMA count for the number of cells flushed from the + * n_conn_cells queue. Used to determine which circuit to flush from next. + */ + cell_ewma_t n_cell_ewma; } circuit_t; /** Largest number of relay_early cells that we can send on a given @@ -2098,6 +2243,19 @@ typedef struct or_circuit_t { /** True iff this circuit was made with a CREATE_FAST cell. */ unsigned int is_first_hop : 1; + + /** Number of cells that were removed from circuit queue; reset every + * time when writing buffer stats to disk. */ + uint32_t processed_cells; + + /** Total time in milliseconds that cells spent in both app-ward and + * exit-ward queues of this circuit; reset every time when writing + * buffer stats to disk. */ + uint64_t total_cell_waiting_time; + + /** The EWMA count for the number of cells flushed from the + * p_conn_cells queue. */ + cell_ewma_t p_cell_ewma; } or_circuit_t; /** Convert a circuit subtype to a circuit_t.*/ @@ -2169,13 +2327,13 @@ typedef struct { routerset_t *EntryNodes;/**< Structure containing nicknames, digests, * country codes and IP address patterns of ORs to * consider as entry points. */ - int StrictExitNodes; /**< Boolean: When none of our ExitNodes are up, do we - * stop building circuits? */ - int StrictEntryNodes; /**< Boolean: When none of our EntryNodes are up, do we - * stop building circuits? */ + int StrictNodes; /**< Boolean: When none of our EntryNodes or ExitNodes + * are up, or we need to access a node in ExcludeNodes, + * do we just fail instead? */ routerset_t *ExcludeNodes;/**< Structure containing nicknames, digests, * country codes and IP address patterns of ORs - * not to use in circuits. */ + * not to use in circuits. But see StrictNodes + * above. */ routerset_t *ExcludeExitNodes;/**< Structure containing nicknames, digests, * country codes and IP address patterns of * ORs not to consider as exits. */ @@ -2183,6 +2341,9 @@ typedef struct { /** Union of ExcludeNodes and ExcludeExitNodes */ struct routerset_t *_ExcludeExitNodesUnion; + int DisableAllSwap; /**< Boolean: Attempt to call mlockall() on our + * process for all current and future memory. */ + /** List of "entry", "middle", "exit", "introduction", "rendezvous". */ smartlist_t *AllowInvalidNodes; /** Bitmask; derived from AllowInvalidNodes. */ @@ -2237,8 +2398,6 @@ typedef struct { * for version 3 directories? */ int HSAuthoritativeDir; /**< Boolean: does this an authoritative directory * handle hidden service requests? */ - int HSAuthorityRecordStats; /**< Boolean: does this HS authoritative - * directory record statistics? */ int NamingAuthoritativeDir; /**< Boolean: is this an authoritative directory * that's willing to bind names? */ int VersioningAuthoritativeDir; /**< Boolean: is this an authoritative @@ -2328,10 +2487,15 @@ typedef struct { * connections alive? */ int SocksTimeout; /**< How long do we let a socks connection wait * unattached before we fail it? */ - int CircuitBuildTimeout; /**< Cull non-open circuits that were born - * at least this many seconds ago. */ + int CircuitBuildTimeout; /**< If non-zero, cull non-open circuits that + * were born at least this many seconds ago. If + * zero, use the internal adaptive algorithm. */ int CircuitIdleTimeout; /**< Cull open clean circuits that were born * at least this many seconds ago. */ + int CircuitStreamTimeout; /**< If non-zero, detach streams from circuits + * and try a new circuit if the stream has been + * waiting for this many seconds. If zero, use + * our default internal timeout schedule. */ int MaxOnionsPending; /**< How many circuit CREATE requests do we allow * to wait simultaneously before we start dropping * them? */ @@ -2349,6 +2513,8 @@ typedef struct { * willing to use for all relayed conns? */ uint64_t RelayBandwidthBurst; /**< How much bandwidth, at maximum, will we * use in a second for all relayed conns? */ + uint64_t PerConnBWRate; /**< Long-term bw on a single TLS conn, if set. */ + uint64_t PerConnBWBurst; /**< Allowed burst on a single TLS conn, if set. */ int NumCpus; /**< How many CPUs should we try to use? */ int RunTesting; /**< If true, create testing circuits to measure how well the * other ORs are running. */ @@ -2359,15 +2525,25 @@ typedef struct { char *ContactInfo; /**< Contact info to be published in the directory. */ char *HttpProxy; /**< hostname[:port] to use as http proxy, if any. */ - uint32_t HttpProxyAddr; /**< Parsed IPv4 addr for http proxy, if any. */ + tor_addr_t HttpProxyAddr; /**< Parsed IPv4 addr for http proxy, if any. */ uint16_t HttpProxyPort; /**< Parsed port for http proxy, if any. */ char *HttpProxyAuthenticator; /**< username:password string, if any. */ char *HttpsProxy; /**< hostname[:port] to use as https proxy, if any. */ - uint32_t HttpsProxyAddr; /**< Parsed IPv4 addr for https proxy, if any. */ + tor_addr_t HttpsProxyAddr; /**< Parsed addr for https proxy, if any. */ uint16_t HttpsProxyPort; /**< Parsed port for https proxy, if any. */ char *HttpsProxyAuthenticator; /**< username:password string, if any. */ + char *Socks4Proxy; /**< hostname:port to use as a SOCKS4 proxy, if any. */ + tor_addr_t Socks4ProxyAddr; /**< Derived from Socks4Proxy. */ + uint16_t Socks4ProxyPort; /**< Derived from Socks4Proxy. */ + + char *Socks5Proxy; /**< hostname:port to use as a SOCKS5 proxy, if any. */ + tor_addr_t Socks5ProxyAddr; /**< Derived from Sock5Proxy. */ + uint16_t Socks5ProxyPort; /**< Derived from Socks5Proxy. */ + char *Socks5ProxyUsername; /**< Username for SOCKS5 authentication, if any */ + char *Socks5ProxyPassword; /**< Password for SOCKS5 authentication, if any */ + /** List of configuration lines for replacement directory authorities. * If you just want to replace one class of authority at a time, * use the "Alternate*Authority" options below instead. */ @@ -2429,8 +2605,13 @@ typedef struct { * or not (1)? */ int ShutdownWaitLength; /**< When we get a SIGINT and we're a server, how * long do we wait before exiting? */ - int SafeLogging; /**< Boolean: are we allowed to log sensitive strings - * such as addresses (0), or do we scrub them first (1)? */ + char *SafeLogging; /**< Contains "relay", "1", "0" (meaning no scrubbing). */ + + /* Derived from SafeLogging */ + enum { + SAFELOG_SCRUB_ALL, SAFELOG_SCRUB_RELAY, SAFELOG_SCRUB_NONE + } _SafeLogging; + int SafeSocks; /**< Boolean: should we outright refuse application * connections that use socks4 or socks5-with-local-dns? */ #define LOG_PROTOCOL_WARN (get_options()->ProtocolWarnings ? \ @@ -2441,6 +2622,8 @@ typedef struct { * log whether it was DNS-leaking or not? */ int HardwareAccel; /**< Boolean: Should we enable OpenSSL hardware * acceleration where available? */ + char *AccelName; /**< Optional hardware acceleration engine name. */ + char *AccelDir; /**< Optional hardware acceleration engine search dir. */ int UseEntryGuards; /**< Boolean: Do we try to enter from a smallish number * of fixed nodes? */ int NumEntryGuards; /**< How many entry guards do we try to establish? */ @@ -2451,6 +2634,9 @@ typedef struct { * means directly from the authorities) no matter our other config? */ int FetchDirInfoEarly; + /** Should we fetch our dir info at the start of the consensus period? */ + int FetchDirInfoExtraEarly; + char *VirtualAddrNetwork; /**< Address and mask to hand out for virtual * MAPADDRESS requests. */ int ServerDNSSearchDomains; /**< Boolean: If set, we don't force exit @@ -2499,6 +2685,29 @@ typedef struct { * exit allows it, we use it. */ int AllowSingleHopCircuits; + /** If true, we convert "www.google.com.foo.exit" addresses on the + * socks/trans/natd ports into "www.google.com" addresses that + * exit from the node "foo". Disabled by default since attacking + * websites and exit relays can use it to manipulate your path + * selection. */ + int AllowDotExit; + + /** If true, the user wants us to collect statistics on clients + * requesting network statuses from us as directory. */ + int DirReqStatistics; + + /** If true, the user wants us to collect statistics on port usage. */ + int ExitPortStatistics; + + /** If true, the user wants us to collect cell statistics. */ + int CellStatistics; + + /** If true, the user wants us to collect statistics as entry node. */ + int EntryStatistics; + + /** If true, include statistics file contents in extra-info documents. */ + int ExtraInfoStatistics; + /** If true, do not believe anybody who tells us that a domain resolves * to an internal address, or that an internal address has a PTR mapping. * Helps avoid some cross-site attacks. */ @@ -2517,6 +2726,13 @@ typedef struct { * migration purposes? */ int V3AuthUseLegacyKey; + /** Location of bandwidth measurement file */ + char *V3BandwidthsFile; + + /** Authority only: key=value pairs that we add to our networkstatus + * consensus vote on the 'params' line. */ + char *ConsensusParams; + /** The length of time that we think an initial consensus should be fresh. * Only altered on testing networks. */ int TestingV3AuthInitialVotingInterval; @@ -2553,19 +2769,6 @@ typedef struct { * the bridge authority guess which countries have blocked access to us. */ int BridgeRecordUsageByCountry; -#ifdef ENABLE_GEOIP_STATS - /** If true, and Tor is built with GEOIP_STATS support, and we're a - * directory, record how many directory requests we get from each country. */ - int DirRecordUsageByCountry; - /** Round all GeoIP results to the next multiple of this value, to avoid - * leaking information. */ - int DirRecordUsageGranularity; - /** Time interval: purge geoip stats after this long. */ - int DirRecordUsageRetainIPs; - /** Time interval: Flush geoip data to disk this often. */ - int DirRecordUsageSaveInterval; -#endif - /** Optionally, a file with GeoIP data. */ char *GeoIPFile; @@ -2573,6 +2776,21 @@ typedef struct { * to make this false. */ int ReloadTorrcOnSIGHUP; + /* The main parameter for picking circuits within a connection. + * + * If this value is positive, when picking a cell to relay on a connection, + * we always relay from the circuit whose weighted cell count is lowest. + * Cells are weighted exponentially such that if one cell is sent + * 'CircuitPriorityHalflife' seconds before another, it counts for half as + * much. + * + * If this value is zero, we're disabling the cell-EWMA algorithm. + * + * If this value is negative, we're using the default approach + * according to either Tor or a parameter set in the consensus. + */ + double CircuitPriorityHalflife; + } or_options_t; /** Persistent state for an onion router, as saved to disk. */ @@ -2609,6 +2827,10 @@ typedef struct { int BWHistoryWriteInterval; smartlist_t *BWHistoryWriteValues; + /** Build time histogram */ + config_line_t * BuildtimeHistogram; + uint16_t TotalBuildTimes; + /** What version of Tor wrote this state file? */ char *TorVersion; @@ -2703,6 +2925,7 @@ int fetch_from_buf_http(buf_t *buf, int force_complete); int fetch_from_buf_socks(buf_t *buf, socks_request_t *req, int log_sockstype, int safe_socks); +int fetch_from_buf_socks_client(buf_t *buf, int state, char **reason); int fetch_from_buf_line(buf_t *buf, char *data_out, size_t *data_len); int peek_buf_has_control0_command(buf_t *buf); @@ -2756,7 +2979,7 @@ void entry_guards_compute_status(void); int entry_guard_register_connect_status(const char *digest, int succeeded, int mark_relay_status, time_t now); void entry_nodes_should_be_added(void); -int entry_list_can_grow(or_options_t *options); +int entry_list_is_constrained(or_options_t *options); routerinfo_t *choose_random_entry(cpath_build_state_t *state); int entry_guards_parse_state(or_state_t *state, int set, char **msg); void entry_guards_update_state(or_state_t *state); @@ -2777,6 +3000,160 @@ void bridges_retry_all(void); void entry_guards_free_all(void); +/* Circuit Build Timeout "public" functions and structures. */ + +/** Maximum quantile to use to generate synthetic timeouts. + * We want to stay a bit short of 1.0, because longtail is + * loooooooooooooooooooooooooooooooooooooooooooooooooooong. */ +#define MAX_SYNTHETIC_QUANTILE 0.985 + +/** Minimum circuits before estimating a timeout */ +#define MIN_CIRCUITS_TO_OBSERVE 500 + +/** Total size of the circuit timeout history to accumulate. + * 5000 is approx 1.5 weeks worth of continual-use circuits. */ +#define NCIRCUITS_TO_OBSERVE 5000 + +/** Width of the histogram bins in milliseconds */ +#define BUILDTIME_BIN_WIDTH ((build_time_t)50) + +/** Cutoff point on the CDF for our timeout estimation. + * TODO: This should be moved to the consensus */ +#define BUILDTIMEOUT_QUANTILE_CUTOFF 0.8 + +/** A build_time_t is milliseconds */ +typedef uint32_t build_time_t; +#define BUILD_TIME_MAX ((build_time_t)(INT32_MAX)) + +/** Lowest allowable value for CircuitBuildTimeout in milliseconds */ +#define BUILD_TIMEOUT_MIN_VALUE (3*1000) + +/** Initial circuit build timeout in milliseconds */ +#define BUILD_TIMEOUT_INITIAL_VALUE (60*1000) + +/** How often in seconds should we build a test circuit */ +#define BUILD_TIMES_TEST_FREQUENCY 60 + +/** Save state every 10 circuits */ +#define BUILD_TIMES_SAVE_STATE_EVERY 10 + +/* Circuit Build Timeout network liveness constants */ + +/** + * How many circuits count as recent when considering if the + * connection has gone gimpy or changed. + */ +#define RECENT_CIRCUITS 20 + +/** + * Have we received a cell in the last N circ attempts? + * + * This tells us when to temporarily switch back to + * BUILD_TIMEOUT_INITIAL_VALUE until we start getting cells, + * at which point we switch back to computing the timeout from + * our saved history. + */ +#define NETWORK_NONLIVE_TIMEOUT_COUNT (RECENT_CIRCUITS*3/20) + +/** + * This tells us when to toss out the last streak of N timeouts. + * + * If instead we start getting cells, we switch back to computing the timeout + * from our saved history. + */ +#define NETWORK_NONLIVE_DISCARD_COUNT (NETWORK_NONLIVE_TIMEOUT_COUNT*2) + +/** + * Maximum count of timeouts that finish the first hop in the past + * RECENT_CIRCUITS before calculating a new timeout. + * + * This tells us to abandon timeout history and set + * the timeout back to BUILD_TIMEOUT_INITIAL_VALUE. + */ +#define MAX_RECENT_TIMEOUT_COUNT (RECENT_CIRCUITS*4/5) + +#if MAX_RECENT_TIMEOUT_COUNT < 1 || NETWORK_NONLIVE_DISCARD_COUNT < 1 || \ + NETWORK_NONLIVE_TIMEOUT_COUNT < 1 +#error "RECENT_CIRCUITS is set too low." +#endif + +/** Information about the state of our local network connection */ +typedef struct { + /** The timestamp we last completed a TLS handshake or received a cell */ + time_t network_last_live; + /** If the network is not live, how many timeouts has this caused? */ + int nonlive_timeouts; + /** If the network is not live, have we yet discarded our history? */ + int nonlive_discarded; + /** Circular array of circuits that have made it to the first hop. Slot is + * 1 if circuit timed out, 0 if circuit succeeded */ + int8_t timeouts_after_firsthop[RECENT_CIRCUITS]; + /** Index into circular array. */ + int after_firsthop_idx; +} network_liveness_t; + +/** Structure for circuit build times history */ +typedef struct { + /** The circular array of recorded build times in milliseconds */ + build_time_t circuit_build_times[NCIRCUITS_TO_OBSERVE]; + /** Current index in the circuit_build_times circular array */ + int build_times_idx; + /** Total number of build times accumulated. Maxes at NCIRCUITS_TO_OBSERVE */ + int total_build_times; + /** Information about the state of our local network connection */ + network_liveness_t liveness; + /** Last time we built a circuit. Used to decide to build new test circs */ + time_t last_circ_at; + /** Number of timeouts that have happened before estimating pareto + * parameters */ + int pre_timeouts; + /** "Minimum" value of our pareto distribution (actually mode) */ + build_time_t Xm; + /** alpha exponent for pareto dist. */ + double alpha; + /** Have we computed a timeout? */ + int have_computed_timeout; + /** The exact value for that timeout in milliseconds */ + double timeout_ms; +} circuit_build_times_t; + +extern circuit_build_times_t circ_times; +void circuit_build_times_update_state(circuit_build_times_t *cbt, + or_state_t *state); +int circuit_build_times_parse_state(circuit_build_times_t *cbt, + or_state_t *state, char **msg); +int circuit_build_times_add_timeout(circuit_build_times_t *cbt, + int did_onehop, time_t start_time); +void circuit_build_times_set_timeout(circuit_build_times_t *cbt); +int circuit_build_times_add_time(circuit_build_times_t *cbt, + build_time_t time); +int circuit_build_times_needs_circuits(circuit_build_times_t *cbt); +int circuit_build_times_needs_circuits_now(circuit_build_times_t *cbt); +void circuit_build_times_init(circuit_build_times_t *cbt); + +#ifdef CIRCUIT_PRIVATE +double circuit_build_times_calculate_timeout(circuit_build_times_t *cbt, + double quantile); +build_time_t circuit_build_times_generate_sample(circuit_build_times_t *cbt, + double q_lo, double q_hi); +void circuit_build_times_initial_alpha(circuit_build_times_t *cbt, + double quantile, double time_ms); +void circuit_build_times_update_alpha(circuit_build_times_t *cbt); +double circuit_build_times_cdf(circuit_build_times_t *cbt, double x); +void circuit_build_times_add_timeout_worker(circuit_build_times_t *cbt, + double quantile_cutoff); +void circuitbuild_running_unit_tests(void); +void circuit_build_times_reset(circuit_build_times_t *cbt); + +/* Network liveness functions */ +int circuit_build_times_network_check_changed(circuit_build_times_t *cbt); +#endif + +/* Network liveness functions */ +void circuit_build_times_network_is_live(circuit_build_times_t *cbt); +int circuit_build_times_network_check_live(circuit_build_times_t *cbt); +void circuit_build_times_network_circ_success(circuit_build_times_t *cbt); + /********************************* circuitlist.c ***********************/ circuit_t * _circuit_get_global_list(void); @@ -2886,7 +3263,9 @@ const char *get_dirportfrontpage(void); or_options_t *get_options(void); int set_options(or_options_t *new_val, char **msg); void config_free_all(void); +const char *safe_str_client(const char *address); const char *safe_str(const char *address); +const char *escaped_safe_str_client(const char *address); const char *escaped_safe_str(const char *address); const char *get_version(void); @@ -2898,6 +3277,7 @@ int resolve_my_address(int warn_severity, or_options_t *options, uint32_t *addr, char **hostname_out); int is_local_addr(const tor_addr_t *addr) ATTR_PURE; void options_init(or_options_t *options); +char *options_dump(or_options_t *options, int minimal); int options_init_from_torrc(int argc, char **argv); setopt_err_t options_init_from_string(const char *cf, int command, const char *command_arg, char **msg); @@ -2931,6 +3311,7 @@ int options_need_geoip_info(or_options_t *options, const char **reason_out); int getinfo_helper_config(control_connection_t *conn, const char *question, char **answer); +const char *tor_get_digests(void); uint32_t get_effective_bwrate(or_options_t *options); uint32_t get_effective_bwburst(or_options_t *options); @@ -2951,7 +3332,6 @@ control_connection_t *control_connection_new(int socket_family); connection_t *connection_new(int type, int socket_family); void connection_link_connections(connection_t *conn_a, connection_t *conn_b); -void connection_unregister_events(connection_t *conn); void connection_free(connection_t *conn); void connection_free_all(void); void connection_about_to_close_connection(connection_t *conn); @@ -2966,6 +3346,10 @@ void connection_expire_held_open(void); int connection_connect(connection_t *conn, const char *address, const tor_addr_t *addr, uint16_t port, int *socket_error); + +int connection_proxy_connect(connection_t *conn, int type); +int connection_read_proxy_handshake(connection_t *conn); + int retry_all_listeners(smartlist_t *replaced_conns, smartlist_t *new_conns); @@ -3008,8 +3392,7 @@ connection_t *connection_get_by_type_addr_port_purpose(int type, uint16_t port, int purpose); connection_t *connection_get_by_type_state(int type, int state); connection_t *connection_get_by_type_state_rendquery(int type, int state, - const char *rendquery, - int rendversion); + const char *rendquery); #define connection_speaks_cells(conn) ((conn)->type == CONN_TYPE_OR) int connection_is_listener(connection_t *conn); @@ -3059,7 +3442,8 @@ int connection_exit_begin_conn(cell_t *cell, circuit_t *circ); int connection_exit_begin_resolve(cell_t *cell, or_circuit_t *circ); void connection_exit_connect(edge_connection_t *conn); int connection_edge_is_rendezvous_stream(edge_connection_t *conn); -int connection_ap_can_use_exit(edge_connection_t *conn, routerinfo_t *exit); +int connection_ap_can_use_exit(edge_connection_t *conn, routerinfo_t *exit, + int excluded_means_no); void connection_ap_expire_beginning(void); void connection_ap_attach_pending(void); void connection_ap_fail_onehop(const char *failed_digest, @@ -3113,7 +3497,7 @@ int hostname_is_noconnect_address(const char *address); typedef enum hostname_type_t { NORMAL_HOSTNAME, ONION_HOSTNAME, EXIT_HOSTNAME, BAD_HOSTNAME } hostname_type_t; -hostname_type_t parse_extended_hostname(char *address); +hostname_type_t parse_extended_hostname(char *address, int allowdotexit); #if defined(HAVE_NET_IF_H) && defined(HAVE_NET_PFVAR_H) int get_pf_socket(void); @@ -3293,8 +3677,7 @@ typedef enum { void control_event_bootstrap(bootstrap_status_t status, int progress); void control_event_bootstrap_problem(const char *warn, int reason); -void control_event_clients_seen(const char *timestarted, - const char *countries); +void control_event_clients_seen(const char *controller_str); #ifdef CONTROL_PRIVATE /* Used only by control.c and test.c */ @@ -3363,9 +3746,13 @@ void directory_initiate_command(const char *address, const tor_addr_t *addr, const char *payload, size_t payload_len, time_t if_modified_since); +#define DSR_HEX (1<<0) +#define DSR_BASE64 (1<<1) +#define DSR_DIGEST256 (1<<2) +#define DSR_SORT_UNIQ (1<<3) int dir_split_resource_into_fingerprints(const char *resource, - smartlist_t *fp_out, int *compresseed_out, - int decode_hex, int sort_uniq); + smartlist_t *fp_out, int *compressed_out, + int flags); /** A pair of digests created by dir_split_resource_info_fingerprint_pairs() */ typedef struct { char first[DIGEST_LEN]; @@ -3417,8 +3804,8 @@ download_status_mark_impossible(download_status_t *dl) * Running Stable Unnamed V2Dir Valid\n". */ #define MAX_FLAG_LINE_LEN 96 /** Length of "w" line for weighting. Currently at most - * "w Bandwidth=<uint32t>\n" */ -#define MAX_WEIGHT_LINE_LEN (13+10) + * "w Bandwidth=<uint32t> Measured=<uint32t>\n" */ +#define MAX_WEIGHT_LINE_LEN (12+10+10+10+1) /** Maximum length of an exit policy summary line. */ #define MAX_POLICY_LINE_LEN (3+MAX_EXITPOLICY_SUMMARY_LEN) /** Amount of space to allocate for each entry: r, s, and v lines. */ @@ -3450,8 +3837,6 @@ enum was_router_added_t dirserv_add_multiple_descriptors( enum was_router_added_t dirserv_add_descriptor(routerinfo_t *ri, const char **msg, const char *source); -int getinfo_helper_dirserv_unregistered(control_connection_t *conn, - const char *question, char **answer); void dirserv_free_descriptors(void); void dirserv_set_router_is_running(routerinfo_t *router, time_t now); int list_server_status_v1(smartlist_t *routers, char **router_status_out, @@ -3472,14 +3857,16 @@ int directory_too_idle_to_fetch_descriptors(or_options_t *options, time_t now); void directory_set_dirty(void); cached_dir_t *dirserv_get_directory(void); cached_dir_t *dirserv_get_runningrouters(void); -cached_dir_t *dirserv_get_consensus(void); +cached_dir_t *dirserv_get_consensus(const char *flavor_name); void dirserv_set_cached_directory(const char *directory, time_t when, int is_running_routers); void dirserv_set_cached_networkstatus_v2(const char *directory, const char *identity, time_t published); -void dirserv_set_cached_networkstatus_v3(const char *consensus, - time_t published); +void dirserv_set_cached_consensus_networkstatus(const char *consensus, + const char *flavor_name, + const digests_t *digests, + time_t published); void dirserv_clear_old_networkstatuses(time_t cutoff); void dirserv_clear_old_v1_info(time_t now); void dirserv_get_networkstatus_v2(smartlist_t *result, const char *key); @@ -3501,15 +3888,38 @@ int authdir_wants_to_reject_router(routerinfo_t *ri, const char **msg, int dirserv_would_reject_router(routerstatus_t *rs); int dirserv_remove_old_statuses(smartlist_t *fps, time_t cutoff); int dirserv_have_any_serverdesc(smartlist_t *fps, int spool_src); +int dirserv_have_any_microdesc(const smartlist_t *fps); size_t dirserv_estimate_data_size(smartlist_t *fps, int is_serverdescs, int compressed); +size_t dirserv_estimate_microdesc_size(const smartlist_t *fps, int compressed); + +typedef enum { + NS_V2, NS_V3_CONSENSUS, NS_V3_VOTE, NS_CONTROL_PORT, + NS_V3_CONSENSUS_MICRODESC +} routerstatus_format_type_t; int routerstatus_format_entry(char *buf, size_t buf_len, routerstatus_t *rs, const char *platform, - int first_line_only, int v2_format); + routerstatus_format_type_t format); void dirserv_free_all(void); void cached_dir_decref(cached_dir_t *d); cached_dir_t *new_cached_dir(char *s, time_t published); +#ifdef DIRSERV_PRIVATE +typedef struct measured_bw_line_t { + char node_id[DIGEST_LEN]; + char node_hex[MAX_HEX_NICKNAME_LEN+1]; + long int bw; +} measured_bw_line_t; + +int measured_bw_line_parse(measured_bw_line_t *out, const char *line); + +int measured_bw_line_apply(measured_bw_line_t *parsed_line, + smartlist_t *routerstatuses); +#endif + +int dirserv_read_measured_bandwidths(const char *from_file, + smartlist_t *routerstatuses); + /********************************* dirvote.c ************************/ /** Lowest allowable value for VoteSeconds. */ @@ -3527,11 +3937,12 @@ char *networkstatus_compute_consensus(smartlist_t *votes, crypto_pk_env_t *identity_key, crypto_pk_env_t *signing_key, const char *legacy_identity_key_digest, - crypto_pk_env_t *legacy_signing_key); + crypto_pk_env_t *legacy_signing_key, + consensus_flavor_t flavor); int networkstatus_add_detached_signatures(networkstatus_t *target, ns_detached_signatures_t *sigs, const char **msg_out); -char *networkstatus_get_detached_signatures(networkstatus_t *consensus); +char *networkstatus_get_detached_signatures(smartlist_t *consensuses); void ns_detached_signatures_free(ns_detached_signatures_t *s); /* cert manipulation */ @@ -3559,7 +3970,7 @@ int dirvote_add_signatures(const char *detached_signatures_body, const char **msg_out); /* Item access */ -const char *dirvote_get_pending_consensus(void); +const char *dirvote_get_pending_consensus(consensus_flavor_t flav); const char *dirvote_get_pending_detached_signatures(void); #define DGV_BY_ID 1 #define DGV_INCLUDE_PENDING 2 @@ -3574,6 +3985,17 @@ networkstatus_t * dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key, authority_cert_t *cert); +microdesc_t *dirvote_create_microdescriptor(const routerinfo_t *ri); +ssize_t dirvote_format_microdesc_vote_line(char *out, size_t out_len, + const microdesc_t *md); +int vote_routerstatus_find_microdesc_hash(char *digest256_out, + const vote_routerstatus_t *vrs, + int method, + digest_algorithm_t alg); +document_signature_t *voter_get_sig_by_algorithm( + const networkstatus_voter_info_t *voter, + digest_algorithm_t alg); + #ifdef DIRVOTE_PRIVATE char *format_networkstatus_vote(crypto_pk_env_t *private_key, networkstatus_t *v3_ns); @@ -3595,6 +4017,7 @@ int dns_resolve(edge_connection_t *exitconn); void dns_launch_correctness_checks(void); int dns_seems_to_be_broken(void); void dns_reset_correctness_checks(void); +void dump_dns_mem_usage(int severity); /********************************* dnsserv.c ************************/ @@ -3610,6 +4033,15 @@ int dnsserv_launch_request(const char *name, int is_reverse); /********************************* geoip.c **************************/ +/** Round all GeoIP results to the next multiple of this value, to avoid + * leaking information. */ +#define DIR_RECORD_USAGE_GRANULARITY 8 +/** Time interval: Flush geoip data to disk this often. */ +#define DIR_ENTRY_RECORD_USAGE_RETAIN_IPS (24*60*60) +/** How long do we have to have observed per-country request history before + * we are willing to talk about it? */ +#define DIR_RECORD_USAGE_MIN_OBSERVATION_TIME (12*60*60) + #ifdef GEOIP_PRIVATE int geoip_parse_entry(const char *line); #endif @@ -3625,7 +4057,7 @@ country_t geoip_get_country(const char *countrycode); * the others, we're not. */ typedef enum { - /** We've noticed a connection as a bridge relay. */ + /** We've noticed a connection as a bridge relay or entry guard. */ GEOIP_CLIENT_CONNECT = 0, /** We've served a networkstatus consensus as a directory server. */ GEOIP_CLIENT_NETWORKSTATUS = 1, @@ -3635,13 +4067,78 @@ typedef enum { void geoip_note_client_seen(geoip_client_action_t action, uint32_t addr, time_t now); void geoip_remove_old_clients(time_t cutoff); +/** Indicates either a positive reply or a reason for rejectng a network + * status request that will be included in geoip statistics. */ +typedef enum { + /** Request is answered successfully. */ + GEOIP_SUCCESS = 0, + /** V3 network status is not signed by a sufficient number of requested + * authorities. */ + GEOIP_REJECT_NOT_ENOUGH_SIGS = 1, + /** Requested network status object is unavailable. */ + GEOIP_REJECT_UNAVAILABLE = 2, + /** Requested network status not found. */ + GEOIP_REJECT_NOT_FOUND = 3, + /** Network status has not been modified since If-Modified-Since time. */ + GEOIP_REJECT_NOT_MODIFIED = 4, + /** Directory is busy. */ + GEOIP_REJECT_BUSY = 5, +} geoip_ns_response_t; +#define GEOIP_NS_RESPONSE_NUM 6 +void geoip_note_ns_response(geoip_client_action_t action, + geoip_ns_response_t response); time_t geoip_get_history_start(void); -char *geoip_get_client_history(time_t now, geoip_client_action_t action); +char *geoip_get_client_history_dirreq(time_t now, + geoip_client_action_t action); +char *geoip_get_client_history_bridge(time_t now, + geoip_client_action_t action); char *geoip_get_request_history(time_t now, geoip_client_action_t action); int getinfo_helper_geoip(control_connection_t *control_conn, const char *question, char **answer); void geoip_free_all(void); -void dump_geoip_stats(void); + +/** Directory requests that we are measuring can be either direct or + * tunneled. */ +typedef enum { + DIRREQ_DIRECT = 0, + DIRREQ_TUNNELED = 1, +} dirreq_type_t; + +/** Possible states for either direct or tunneled directory requests that + * are relevant for determining network status download times. */ +typedef enum { + /** Found that the client requests a network status; applies to both + * direct and tunneled requests; initial state of a request that we are + * measuring. */ + DIRREQ_IS_FOR_NETWORK_STATUS = 0, + /** Finished writing a network status to the directory connection; + * applies to both direct and tunneled requests; completes a direct + * request. */ + DIRREQ_FLUSHING_DIR_CONN_FINISHED = 1, + /** END cell sent to circuit that initiated a tunneled request. */ + DIRREQ_END_CELL_SENT = 2, + /** Flushed last cell from queue of the circuit that initiated a + * tunneled request to the outbuf of the OR connection. */ + DIRREQ_CIRC_QUEUE_FLUSHED = 3, + /** Flushed last byte from buffer of the OR connection belonging to the + * circuit that initiated a tunneled request; completes a tunneled + * request. */ + DIRREQ_OR_CONN_BUFFER_FLUSHED = 4 +} dirreq_state_t; + +void geoip_start_dirreq(uint64_t dirreq_id, size_t response_size, + geoip_client_action_t action, dirreq_type_t type); +void geoip_change_dirreq_state(uint64_t dirreq_id, dirreq_type_t type, + dirreq_state_t new_state); + +void geoip_dirreq_stats_init(time_t now); +void geoip_dirreq_stats_write(time_t now); +void geoip_entry_stats_init(time_t now); +void geoip_entry_stats_write(time_t now); +void geoip_bridge_stats_init(time_t now); +time_t geoip_bridge_stats_write(time_t now); +char *geoip_get_bridge_stats_extrainfo(time_t); +char *geoip_get_bridge_stats_controller(time_t); /********************************* hibernate.c **********************/ @@ -3664,13 +4161,18 @@ extern int has_completed_circuit; int connection_add(connection_t *conn); int connection_remove(connection_t *conn); +void connection_unregister_events(connection_t *conn); int connection_in_array(connection_t *conn); void add_connection_to_closeable_list(connection_t *conn); int connection_is_on_closeable_list(connection_t *conn); smartlist_t *get_connection_array(void); -void connection_watch_events(connection_t *conn, short events); +typedef enum watchable_events { + READ_EVENT=0x02, + WRITE_EVENT=0x04 +} watchable_events_t; +void connection_watch_events(connection_t *conn, watchable_events_t events); int connection_is_reading(connection_t *conn); void connection_stop_reading(connection_t *conn); void connection_start_reading(connection_t *conn); @@ -3706,6 +4208,31 @@ void do_hash_password(void); int tor_init(int argc, char **argv); #endif +/********************************* microdesc.c *************************/ + +typedef struct microdesc_cache_t microdesc_cache_t; + +microdesc_cache_t *get_microdesc_cache(void); + +smartlist_t *microdescs_add_to_cache(microdesc_cache_t *cache, + const char *s, const char *eos, saved_location_t where, + int no_save); +smartlist_t *microdescs_add_list_to_cache(microdesc_cache_t *cache, + smartlist_t *descriptors, saved_location_t where, + int no_save); + +int microdesc_cache_rebuild(microdesc_cache_t *cache); +int microdesc_cache_reload(microdesc_cache_t *cache); +void microdesc_cache_clear(microdesc_cache_t *cache); + +microdesc_t *microdesc_cache_lookup_by_digest256(microdesc_cache_t *cache, + const char *d); + +size_t microdesc_average_size(microdesc_cache_t *cache); + +void microdesc_free(microdesc_t *md); +void microdesc_free_all(void); + /********************************* networkstatus.c *********************/ /** How old do we allow a v2 network-status to get before removing it @@ -3743,14 +4270,16 @@ networkstatus_voter_info_t *networkstatus_get_voter_by_id( const char *identity); int networkstatus_check_consensus_signature(networkstatus_t *consensus, int warn); -int networkstatus_check_voter_signature(networkstatus_t *consensus, - networkstatus_voter_info_t *voter, - authority_cert_t *cert); +int networkstatus_check_document_signature(const networkstatus_t *consensus, + document_signature_t *sig, + const authority_cert_t *cert); char *networkstatus_get_cache_filename(const char *identity_digest); int router_set_networkstatus_v2(const char *s, time_t arrived_at, v2_networkstatus_source_t source, smartlist_t *requested_fingerprints); void networkstatus_v2_list_clean(time_t now); +int compare_digest_to_routerstatus_entry(const void *_key, + const void **_member); routerstatus_t *networkstatus_v2_find_entry(networkstatus_v2_t *ns, const char *digest); routerstatus_t *networkstatus_vote_find_entry(networkstatus_t *ns, @@ -3780,7 +4309,10 @@ networkstatus_t *networkstatus_get_reasonably_live_consensus(time_t now); #define NSSET_WAS_WAITING_FOR_CERTS 2 #define NSSET_DONT_DOWNLOAD_CERTS 4 #define NSSET_ACCEPT_OBSOLETE 8 -int networkstatus_set_current_consensus(const char *consensus, unsigned flags); +#define NSSET_REQUIRE_FLAVOR 16 +int networkstatus_set_current_consensus(const char *consensus, + const char *flavor, + unsigned flags); void networkstatus_note_certs_arrived(void); void routers_update_all_from_networkstatus(time_t now, int dir_version); void routerstatus_list_update_from_consensus_networkstatus(time_t now); @@ -3796,6 +4328,10 @@ int32_t networkstatus_get_param(networkstatus_t *ns, const char *param_name, int32_t default_val); int getinfo_helper_networkstatus(control_connection_t *conn, const char *question, char **answer); +const char *networkstatus_get_flavor_name(consensus_flavor_t flav); +int networkstatus_parse_flavor_name(const char *flavname); +void document_signature_free(document_signature_t *sig); +document_signature_t *document_signature_dup(const document_signature_t *sig); void networkstatus_free_all(void); /********************************* ntmain.c ***************************/ @@ -3888,7 +4424,8 @@ addr_policy_result_t compare_tor_addr_to_addr_policy(const tor_addr_t *addr, addr_policy_result_t compare_addr_to_addr_policy(uint32_t addr, uint16_t port, const smartlist_t *policy); int policies_parse_exit_policy(config_line_t *cfg, smartlist_t **dest, - int rejectprivate, const char *local_address); + int rejectprivate, const char *local_address, + int add_default_policy); void policies_set_router_exitpolicy_to_reject_all(routerinfo_t *exitrouter); int exit_policy_is_general_exit(smartlist_t *policy); int policy_is_reject_star(const smartlist_t *policy); @@ -3915,6 +4452,8 @@ int tls_error_to_orconn_end_reason(int e); int errno_to_orconn_end_reason(int e); const char *circuit_end_reason_to_control_string(int reason); +const char *socks4_response_code_to_string(uint8_t code); +const char *socks5_response_code_to_string(uint8_t code); /********************************* relay.c ***************************/ @@ -3963,6 +4502,9 @@ int append_address_to_payload(char *payload_out, const tor_addr_t *addr); const char *decode_address_from_payload(tor_addr_t *addr_out, const char *payload, int payload_len); +unsigned cell_ewma_get_tick(void); +void cell_ewma_set_scale_factor(or_options_t *options, + networkstatus_t *consensus); /********************************* rephist.c ***************************/ @@ -3977,6 +4519,11 @@ void rep_hist_note_extend_failed(const char *from_name, const char *to_name); void rep_hist_dump_stats(time_t now, int severity); void rep_hist_note_bytes_read(size_t num_bytes, time_t when); void rep_hist_note_bytes_written(size_t num_bytes, time_t when); +void rep_hist_note_exit_bytes_read(uint16_t port, size_t num_bytes); +void rep_hist_note_exit_bytes_written(uint16_t port, size_t num_bytes); +void rep_hist_note_exit_stream_opened(uint16_t port); +void rep_hist_exit_stats_init(time_t now); +void rep_hist_exit_stats_write(time_t now); int rep_hist_bandwidth_assess(void); char *rep_hist_get_bandwidth_lines(int for_extrainfo); void rep_hist_update_state(or_state_t *state); @@ -4028,13 +4575,17 @@ void hs_usage_note_fetch_successful(const char *service_id, time_t now); void hs_usage_write_statistics_to_file(time_t now); void hs_usage_free_all(void); +void rep_hist_buffer_stats_init(time_t now); +void rep_hist_buffer_stats_add_circ(circuit_t *circ, + time_t end_of_interval); +void rep_hist_buffer_stats_write(time_t now); + /********************************* rendclient.c ***************************/ void rend_client_introcirc_has_opened(origin_circuit_t *circ); void rend_client_rendcirc_has_opened(origin_circuit_t *circ); int rend_client_introduction_acked(origin_circuit_t *circ, const char *request, size_t request_len); -void rend_client_refetch_renddesc(const char *query); void rend_client_refetch_v2_renddesc(const rend_data_t *rend_query); int rend_client_remove_intro_point(extend_info_t *failed_intro, const rend_data_t *rend_query); @@ -4042,7 +4593,7 @@ int rend_client_rendezvous_acked(origin_circuit_t *circ, const char *request, size_t request_len); int rend_client_receive_rendezvous(origin_circuit_t *circ, const char *request, size_t request_len); -void rend_client_desc_trynow(const char *query, int rend_version); +void rend_client_desc_trynow(const char *query); extend_info_t *rend_client_get_random_intro(const rend_data_t *rend_query); @@ -4109,10 +4660,6 @@ void rend_process_relay_cell(circuit_t *circ, const crypt_path_t *layer_hint, int command, size_t length, const char *payload); void rend_service_descriptor_free(rend_service_descriptor_t *desc); -int rend_encode_service_descriptor(rend_service_descriptor_t *desc, - crypto_pk_env_t *key, - char **str_out, - size_t *len_out); rend_service_descriptor_t *rend_parse_service_descriptor(const char *str, size_t len); int rend_get_service_id(crypto_pk_env_t *pk, char *out); @@ -4255,7 +4802,6 @@ int router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router, crypto_pk_env_t *ident_key); int extrainfo_dump_to_string(char *s, size_t maxlen, extrainfo_t *extrainfo, crypto_pk_env_t *ident_key); -char *extrainfo_get_client_geoip_summary(time_t); int is_legal_nickname(const char *s); int is_legal_nickname_or_hexdigest(const char *s); int is_legal_hexdigest(const char *s); @@ -4394,13 +4940,10 @@ typedef enum { CRN_NEED_GUARD = 1<<2, CRN_ALLOW_INVALID = 1<<3, /* XXXX not used, apparently. */ - CRN_STRICT_PREFERRED = 1<<4, - /* XXXX not used, apparently. */ CRN_WEIGHT_AS_EXIT = 1<<5 } router_crn_flags_t; -routerinfo_t *router_choose_random_node(const char *preferred, - smartlist_t *excludedsmartlist, +routerinfo_t *router_choose_random_node(smartlist_t *excludedsmartlist, struct routerset_t *excludedset, router_crn_flags_t flags); @@ -4564,16 +5107,22 @@ typedef struct tor_version_t { int patchlevel; char status_tag[MAX_STATUS_TAG_LEN]; int svn_revision; + + int git_tag_len; + char git_tag[DIGEST_LEN]; } tor_version_t; int router_get_router_hash(const char *s, char *digest); int router_get_dir_hash(const char *s, char *digest); int router_get_runningrouters_hash(const char *s, char *digest); int router_get_networkstatus_v2_hash(const char *s, char *digest); -int router_get_networkstatus_v3_hash(const char *s, char *digest); +int router_get_networkstatus_v3_hash(const char *s, char *digest, + digest_algorithm_t algorithm); +int router_get_networkstatus_v3_hashes(const char *s, digests_t *digests); int router_get_extrainfo_hash(const char *s, char *digest); int router_append_dirobj_signature(char *buf, size_t buf_len, const char *digest, + size_t digest_len, crypto_pk_env_t *private_key); int router_parse_list_from_string(const char **s, const char *eos, smartlist_t *dest, @@ -4605,6 +5154,7 @@ void sort_version_list(smartlist_t *lst, int remove_duplicates); void assert_addr_policy_ok(smartlist_t *t); void dump_distinct_digest_count(int severity); +int compare_routerstatus_entries(const void **_a, const void **_b); networkstatus_v2_t *networkstatus_v2_parse_from_string(const char *s); networkstatus_t *networkstatus_parse_vote_from_string(const char *s, const char **eos_out, @@ -4612,6 +5162,10 @@ networkstatus_t *networkstatus_parse_vote_from_string(const char *s, ns_detached_signatures_t *networkstatus_parse_detached_signatures( const char *s, const char *eos); +smartlist_t *microdescs_parse_from_string(const char *s, const char *eos, + int allow_annotations, + int copy_body); + authority_cert_t *authority_cert_parse_from_string(const char *s, const char **end_of_string); int rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out, @@ -4630,5 +5184,7 @@ int rend_parse_introduction_points(rend_service_descriptor_t *parsed, size_t intro_points_encoded_size); int rend_parse_client_keys(strmap_t *parsed_clients, const char *str); +void tor_gettimeofday_cache_clear(void); + #endif diff --git a/src/or/policies.c b/src/or/policies.c index d55e86c184..453138eb46 100644 --- a/src/or/policies.c +++ b/src/or/policies.c @@ -344,7 +344,8 @@ validate_addr_policies(or_options_t *options, char **msg) *msg = NULL; if (policies_parse_exit_policy(options->ExitPolicy, &addr_policy, - options->ExitPolicyRejectPrivate, NULL)) + options->ExitPolicyRejectPrivate, NULL, + !options->BridgeRelay)) REJECT("Error in ExitPolicy entry."); /* The rest of these calls *append* to addr_policy. So don't actually @@ -375,12 +376,6 @@ validate_addr_policies(or_options_t *options, char **msg) if (parse_addr_policy(options->ReachableDirAddresses, &addr_policy, ADDR_POLICY_ACCEPT)) REJECT("Error in ReachableDirAddresses entry."); - if (parse_addr_policy(options->AuthDirReject, &addr_policy, - ADDR_POLICY_REJECT)) - REJECT("Error in AuthDirReject entry."); - if (parse_addr_policy(options->AuthDirInvalid, &addr_policy, - ADDR_POLICY_REJECT)) - REJECT("Error in AuthDirInvalid entry."); err: addr_policy_list_free(addr_policy); @@ -829,14 +824,16 @@ exit_policy_remove_redundancies(smartlist_t *dest) "reject *:6346-6429,reject *:6699,reject *:6881-6999,accept *:*" /** Parse the exit policy <b>cfg</b> into the linked list *<b>dest</b>. If - * cfg doesn't end in an absolute accept or reject, add the default exit + * cfg doesn't end in an absolute accept or reject and if + * <b>add_default_policy</b> is true, add the default exit * policy afterwards. If <b>rejectprivate</b> is true, prepend * "reject private:*" to the policy. Return -1 if we can't parse cfg, * else return 0. */ int policies_parse_exit_policy(config_line_t *cfg, smartlist_t **dest, - int rejectprivate, const char *local_address) + int rejectprivate, const char *local_address, + int add_default_policy) { if (rejectprivate) { append_exit_policy_string(dest, "reject private:*"); @@ -848,8 +845,10 @@ policies_parse_exit_policy(config_line_t *cfg, smartlist_t **dest, } if (parse_addr_policy(cfg, dest, -1)) return -1; - append_exit_policy_string(dest, DEFAULT_EXIT_POLICY); - + if (add_default_policy) + append_exit_policy_string(dest, DEFAULT_EXIT_POLICY); + else + append_exit_policy_string(dest, "reject *:*"); exit_policy_remove_redundancies(*dest); return 0; @@ -1271,7 +1270,8 @@ getinfo_helper_policies(control_connection_t *conn, void addr_policy_list_free(smartlist_t *lst) { - if (!lst) return; + if (!lst) + return; SMARTLIST_FOREACH(lst, addr_policy_t *, policy, addr_policy_free(policy)); smartlist_free(lst); } @@ -1280,19 +1280,20 @@ addr_policy_list_free(smartlist_t *lst) void addr_policy_free(addr_policy_t *p) { - if (p) { - if (--p->refcnt <= 0) { - if (p->is_canonical) { - policy_map_ent_t search, *found; - search.policy = p; - found = HT_REMOVE(policy_map, &policy_root, &search); - if (found) { - tor_assert(p == found->policy); - tor_free(found); - } + if (!p) + return; + + if (--p->refcnt <= 0) { + if (p->is_canonical) { + policy_map_ent_t search, *found; + search.policy = p; + found = HT_REMOVE(policy_map, &policy_root, &search); + if (found) { + tor_assert(p == found->policy); + tor_free(found); } - tor_free(p); } + tor_free(p); } } diff --git a/src/or/reasons.c b/src/or/reasons.c index a252f83198..0544990679 100644 --- a/src/or/reasons.c +++ b/src/or/reasons.c @@ -326,3 +326,49 @@ circuit_end_reason_to_control_string(int reason) } } +/** Return a string corresponding to a SOCKS4 reponse code. */ +const char * +socks4_response_code_to_string(uint8_t code) +{ + switch (code) { + case 0x5a: + return "connection accepted"; + case 0x5b: + return "server rejected connection"; + case 0x5c: + return "server cannot connect to identd on this client"; + case 0x5d: + return "user id does not match identd"; + default: + return "invalid SOCKS 4 response code"; + } +} + +/** Return a string corresponding to a SOCKS5 reponse code. */ +const char * +socks5_response_code_to_string(uint8_t code) +{ + switch (code) { + case 0x00: + return "connection accepted"; + case 0x01: + return "general SOCKS server failure"; + case 0x02: + return "connection not allowed by ruleset"; + case 0x03: + return "Network unreachable"; + case 0x04: + return "Host unreachable"; + case 0x05: + return "Connection refused"; + case 0x06: + return "TTL expired"; + case 0x07: + return "Command not supported"; + case 0x08: + return "Address type not supported"; + default: + return "unknown reason"; + } +} + diff --git a/src/or/relay.c b/src/or/relay.c index b26c582b82..ae1b062cf6 100644 --- a/src/or/relay.c +++ b/src/or/relay.c @@ -10,6 +10,7 @@ * receiving from circuits, plus queuing on circuits. **/ +#include <math.h> #include "or.h" #include "mempool.h" @@ -35,6 +36,26 @@ circuit_resume_edge_reading_helper(edge_connection_t *conn, static int circuit_consider_stop_edge_reading(circuit_t *circ, crypt_path_t *layer_hint); +/** Cache the current hi-res time; the cache gets reset when libevent + * calls us. */ + +static struct timeval cached_time_hires = {0, 0}; + +static void +tor_gettimeofday_cached(struct timeval *tv) +{ + if (cached_time_hires.tv_sec == 0) { + tor_gettimeofday(&cached_time_hires); + } + *tv = cached_time_hires; +} + +void +tor_gettimeofday_cache_clear(void) +{ + cached_time_hires.tv_sec = 0; +} + /** Stats: how many relay cells have originated at this hop, or have * been relayed onward (not recognized at this hop)? */ @@ -533,6 +554,12 @@ relay_send_command_from_edge(uint16_t stream_id, circuit_t *circ, log_debug(LD_OR,"delivering %d cell %s.", relay_command, cell_direction == CELL_DIRECTION_OUT ? "forward" : "backward"); + /* If we are sending an END cell and this circuit is used for a tunneled + * directory request, advance its state. */ + if (relay_command == RELAY_COMMAND_END && circ->dirreq_id) + geoip_change_dirreq_state(circ->dirreq_id, DIRREQ_TUNNELED, + DIRREQ_END_CELL_SENT); + if (cell_direction == CELL_DIRECTION_OUT && circ->n_conn) { /* if we're using relaybandwidthrate, this conn wants priority */ circ->n_conn->client_used = approx_time(); @@ -550,9 +577,9 @@ relay_send_command_from_edge(uint16_t stream_id, circuit_t *circ, * Don't worry about the conn protocol version: * append_cell_to_circuit_queue will fix it up. */ /* XXX For now, clients don't use RELAY_EARLY cells when sending - * relay cells on rendezvous circuits. See bug 1038. Eventually, - * we can take this behavior away in favor of having clients avoid - * rendezvous points running 0.2.1.3-alpha through 0.2.1.18. -RD */ + * relay cells on rendezvous circuits. See bug 1038. Once no relays + * (and thus no rendezvous points) are running 0.2.1.3-alpha through + * 0.2.1.18, we can take out that exception. -RD */ cell.command = CELL_RELAY_EARLY; --origin_circ->remaining_relay_early_cells; log_debug(LD_OR, "Sending a RELAY_EARLY cell; %d remaining.", @@ -999,7 +1026,8 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, relay_header_unpack(&rh, cell->payload); // log_fn(LOG_DEBUG,"command %d stream %d", rh.command, rh.stream_id); num_seen++; - log_debug(domain, "Now seen %d relay cells here.", num_seen); + log_debug(domain, "Now seen %d relay cells here (command %d, stream %d).", + num_seen, rh.command, rh.stream_id); if (rh.length > RELAY_PAYLOAD_SIZE) { log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, @@ -1038,6 +1066,16 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, "Begin cell for known stream. Dropping."); return 0; } + if (rh.command == RELAY_COMMAND_BEGIN_DIR) { + /* Assign this circuit and its app-ward OR connection a unique ID, + * so that we can measure download times. The local edge and dir + * connection will be assigned the same ID when they are created + * and linked. */ + static uint64_t next_id = 0; + circ->dirreq_id = ++next_id; + TO_CONN(TO_OR_CIRCUIT(circ)->p_conn)->dirreq_id = circ->dirreq_id; + } + return connection_exit_begin_conn(cell, circ); case RELAY_COMMAND_DATA: ++stats_n_data_cells_received; @@ -1508,6 +1546,10 @@ static int total_cells_allocated = 0; /** A memory pool to allocate packed_cell_t objects. */ static mp_pool_t *cell_pool = NULL; +/** Memory pool to allocate insertion_time_elem_t objects used for cell + * statistics. */ +static mp_pool_t *it_pool = NULL; + /** Allocate structures to hold cells. */ void init_cell_pool(void) @@ -1516,7 +1558,8 @@ init_cell_pool(void) cell_pool = mp_pool_new(sizeof(packed_cell_t), 128*1024); } -/** Free all storage used to hold cells. */ +/** Free all storage used to hold cells (and insertion times if we measure + * cell statistics). */ void free_cell_pool(void) { @@ -1525,6 +1568,10 @@ free_cell_pool(void) mp_pool_destroy(cell_pool); cell_pool = NULL; } + if (it_pool) { + mp_pool_destroy(it_pool); + it_pool = NULL; + } } /** Free excess storage in cell pool. */ @@ -1537,7 +1584,7 @@ clean_cell_pool(void) /** Release storage held by <b>cell</b>. */ static INLINE void -packed_cell_free(packed_cell_t *cell) +packed_cell_free_unchecked(packed_cell_t *cell) { --total_cells_allocated; mp_pool_release(cell); @@ -1599,7 +1646,38 @@ cell_queue_append(cell_queue_t *queue, packed_cell_t *cell) void cell_queue_append_packed_copy(cell_queue_t *queue, const cell_t *cell) { - cell_queue_append(queue, packed_cell_copy(cell)); + packed_cell_t *copy = packed_cell_copy(cell); + /* Remember the time when this cell was put in the queue. */ + if (get_options()->CellStatistics) { + struct timeval now; + uint32_t added; + insertion_time_queue_t *it_queue = queue->insertion_times; + if (!it_pool) + it_pool = mp_pool_new(sizeof(insertion_time_elem_t), 1024); + tor_gettimeofday_cached(&now); +#define SECONDS_IN_A_DAY 86400L + added = (uint32_t)(((now.tv_sec % SECONDS_IN_A_DAY) * 100L) + + ((uint32_t)now.tv_usec / (uint32_t)10000L)); + if (!it_queue) { + it_queue = tor_malloc_zero(sizeof(insertion_time_queue_t)); + queue->insertion_times = it_queue; + } + if (it_queue->last && it_queue->last->insertion_time == added) { + it_queue->last->counter++; + } else { + insertion_time_elem_t *elem = mp_pool_get(it_pool); + elem->next = NULL; + elem->insertion_time = added; + elem->counter = 1; + if (it_queue->last) { + it_queue->last->next = elem; + it_queue->last = elem; + } else { + it_queue->first = it_queue->last = elem; + } + } + } + cell_queue_append(queue, copy); } /** Remove and free every cell in <b>queue</b>. */ @@ -1610,11 +1688,19 @@ cell_queue_clear(cell_queue_t *queue) cell = queue->head; while (cell) { next = cell->next; - packed_cell_free(cell); + packed_cell_free_unchecked(cell); cell = next; } queue->head = queue->tail = NULL; queue->n = 0; + if (queue->insertion_times) { + while (queue->insertion_times->first) { + insertion_time_elem_t *elem = queue->insertion_times->first; + queue->insertion_times->first = elem->next; + mp_pool_release(elem); + } + tor_free(queue->insertion_times); + } } /** Extract and return the cell at the head of <b>queue</b>; return NULL if @@ -1666,8 +1752,226 @@ prev_circ_on_conn_p(circuit_t *circ, or_connection_t *conn) } } +/** Helper for sorting cell_ewma_t values in their priority queue. */ +static int +compare_cell_ewma_counts(const void *p1, const void *p2) +{ + const cell_ewma_t *e1=p1, *e2=p2; + if (e1->cell_count < e2->cell_count) + return -1; + else if (e1->cell_count > e2->cell_count) + return 1; + else + return 0; +} + +/** Given a cell_ewma_t, return a pointer to the circuit containing it. */ +static circuit_t * +cell_ewma_to_circuit(cell_ewma_t *ewma) +{ + if (ewma->is_for_p_conn) { + /* This is an or_circuit_t's p_cell_ewma. */ + or_circuit_t *orcirc = SUBTYPE_P(ewma, or_circuit_t, p_cell_ewma); + return TO_CIRCUIT(orcirc); + } else { + /* This is some circuit's n_cell_ewma. */ + return SUBTYPE_P(ewma, circuit_t, n_cell_ewma); + } +} + +/* ==== Functions for scaling cell_ewma_t ==== + + When choosing which cells to relay first, we favor circuits that have been + quiet recently. This gives better latency on connections that aren't + pushing lots of data, and makes the network feel more interactive. + + Conceptually, we take an exponentially weighted mean average of the number + of cells a circuit has sent, and allow active circuits (those with cells to + relay) to send cells in reverse order of their exponentially-weighted mean + average (EWMA) cell count. [That is, a cell sent N seconds ago 'counts' + F^N times as much as a cell sent now, for 0<F<1.0, and we favor the + circuit that has sent the fewest cells] + + If 'double' had infinite precision, we could do this simply by counting a + cell sent at startup as having weight 1.0, and a cell sent N seconds later + as having weight F^-N. This way, we would never need to re-scale + any already-sent cells. + + To prevent double from overflowing, we could count a cell sent now as + having weight 1.0 and a cell sent N seconds ago as having weight F^N. + This, however, would mean we'd need to re-scale *ALL* old circuits every + time we wanted to send a cell. + + So as a compromise, we divide time into 'ticks' (currently, 10-second + increments) and say that a cell sent at the start of a current tick is + worth 1.0, a cell sent N seconds before the start of the current tick is + worth F^N, and a cell sent N seconds after the start of the current tick is + worth F^-N. This way we don't overflow, and we don't need to constantly + rescale. + */ + +/** How long does a tick last (seconds)? */ +#define EWMA_TICK_LEN 10 + +/** The default per-tick scale factor, if it hasn't been overridden by a + * consensus or a configuration setting. zero means "disabled". */ +#define EWMA_DEFAULT_HALFLIFE 0.0 + +/** Given a timeval <b>now</b>, compute the cell_ewma tick in which it occurs + * and the fraction of the tick that has elapsed between the start of the tick + * and <b>now</b>. Return the former and store the latter in + * *<b>remainder_out</b>. + * + * These tick values are not meant to be shared between Tor instances, or used + * for other purposes. */ +static unsigned +cell_ewma_tick_from_timeval(const struct timeval *now, + double *remainder_out) +{ + unsigned res = (unsigned) (now->tv_sec / EWMA_TICK_LEN); + /* rem */ + double rem = (now->tv_sec % EWMA_TICK_LEN) + + ((double)(now->tv_usec)) / 1.0e6; + *remainder_out = rem / EWMA_TICK_LEN; + return res; +} + +/** Compute and return the current cell_ewma tick. */ +unsigned +cell_ewma_get_tick(void) +{ + return ((unsigned)approx_time() / EWMA_TICK_LEN); +} + +/** The per-tick scale factor to be used when computing cell-count EWMA + * values. (A cell sent N ticks before the start of the current tick + * has value ewma_scale_factor ** N.) + */ +static double ewma_scale_factor = 0.1; +static int ewma_enabled = 0; + +#define EPSILON 0.00001 +#define LOG_ONEHALF -0.69314718055994529 + +/** Adjust the global cell scale factor based on <b>options</b> */ +void +cell_ewma_set_scale_factor(or_options_t *options, networkstatus_t *consensus) +{ + int32_t halflife_ms; + double halflife; + const char *source; + if (options && options->CircuitPriorityHalflife >= -EPSILON) { + halflife = options->CircuitPriorityHalflife; + source = "CircuitPriorityHalflife in configuration"; + } else if (consensus && + (halflife_ms = networkstatus_get_param( + consensus, "CircPriorityHalflifeMsec", -1) >= 0)) { + halflife = ((double)halflife_ms)/1000.0; + source = "CircPriorityHalflifeMsec in consensus"; + } else { + halflife = EWMA_DEFAULT_HALFLIFE; + source = "Default value"; + } + + if (halflife <= EPSILON) { + /* The cell EWMA algorithm is disabled. */ + ewma_scale_factor = 0.1; + ewma_enabled = 0; + log_info(LD_OR, + "Disabled cell_ewma algorithm because of value in %s", + source); + } else { + /* convert halflife into halflife-per-tick. */ + halflife /= EWMA_TICK_LEN; + /* compute per-tick scale factor. */ + ewma_scale_factor = exp( LOG_ONEHALF / halflife ); + ewma_enabled = 1; + log_info(LD_OR, + "Enabled cell_ewma algorithm because of value in %s; " + "scale factor is %lf per %d seconds", + source, ewma_scale_factor, EWMA_TICK_LEN); + } +} + +/** Return the multiplier necessary to convert the value of a cell sent in + * 'from_tick' to one sent in 'to_tick'. */ +static INLINE double +get_scale_factor(unsigned from_tick, unsigned to_tick) +{ + /* This math can wrap around, but that's okay: unsigned overflow is + well-defined */ + int diff = (int)(to_tick - from_tick); + return pow(ewma_scale_factor, diff); +} + +/** Adjust the cell count of <b>ewma</b> so that it is scaled with respect to + * <b>cur_tick</b> */ +static void +scale_single_cell_ewma(cell_ewma_t *ewma, unsigned cur_tick) +{ + double factor = get_scale_factor(ewma->last_adjusted_tick, cur_tick); + ewma->cell_count *= factor; + ewma->last_adjusted_tick = cur_tick; +} + +/** Adjust the cell count of every active circuit on <b>conn</b> so + * that they are scaled with respect to <b>cur_tick</b> */ +static void +scale_active_circuits(or_connection_t *conn, unsigned cur_tick) +{ + + double factor = get_scale_factor( + conn->active_circuit_pqueue_last_recalibrated, + cur_tick); + /** Ordinarily it isn't okay to change the value of an element in a heap, + * but it's okay here, since we are preserving the order. */ + SMARTLIST_FOREACH(conn->active_circuit_pqueue, cell_ewma_t *, e, { + tor_assert(e->last_adjusted_tick == + conn->active_circuit_pqueue_last_recalibrated); + e->cell_count *= factor; + e->last_adjusted_tick = cur_tick; + }); + conn->active_circuit_pqueue_last_recalibrated = cur_tick; +} + +/** Rescale <b>ewma</b> to the same scale as <b>conn</b>, and add it to + * <b>conn</b>'s priority queue of active circuits */ +static void +add_cell_ewma_to_conn(or_connection_t *conn, cell_ewma_t *ewma) +{ + tor_assert(ewma->heap_index == -1); + scale_single_cell_ewma(ewma, + conn->active_circuit_pqueue_last_recalibrated); + + smartlist_pqueue_add(conn->active_circuit_pqueue, + compare_cell_ewma_counts, + STRUCT_OFFSET(cell_ewma_t, heap_index), + ewma); +} + +/** Remove <b>ewma</b> from <b>conn</b>'s priority queue of active circuits */ +static void +remove_cell_ewma_from_conn(or_connection_t *conn, cell_ewma_t *ewma) +{ + tor_assert(ewma->heap_index != -1); + smartlist_pqueue_remove(conn->active_circuit_pqueue, + compare_cell_ewma_counts, + STRUCT_OFFSET(cell_ewma_t, heap_index), + ewma); +} + +/** Remove and return the first cell_ewma_t from conn's priority queue of + * active circuits. Requires that the priority queue is nonempty. */ +static cell_ewma_t * +pop_first_cell_ewma_from_conn(or_connection_t *conn) +{ + return smartlist_pqueue_pop(conn->active_circuit_pqueue, + compare_cell_ewma_counts, + STRUCT_OFFSET(cell_ewma_t, heap_index)); +} + /** Add <b>circ</b> to the list of circuits with pending cells on - * <b>conn</b>. No effect if <b>circ</b> is already unlinked. */ + * <b>conn</b>. No effect if <b>circ</b> is already linked. */ void make_circuit_active_on_conn(circuit_t *circ, or_connection_t *conn) { @@ -1679,6 +1983,8 @@ make_circuit_active_on_conn(circuit_t *circ, or_connection_t *conn) return; } + assert_active_circuits_ok_paranoid(conn); + if (! conn->active_circuits) { conn->active_circuits = circ; *prevp = *nextp = circ; @@ -1690,10 +1996,19 @@ make_circuit_active_on_conn(circuit_t *circ, or_connection_t *conn) *prev_circ_on_conn_p(head, conn) = circ; *prevp = old_tail; } + + if (circ->n_conn == conn) { + add_cell_ewma_to_conn(conn, &circ->n_cell_ewma); + } else { + or_circuit_t *orcirc = TO_OR_CIRCUIT(circ); + tor_assert(conn == orcirc->p_conn); + add_cell_ewma_to_conn(conn, &orcirc->p_cell_ewma); + } + assert_active_circuits_ok_paranoid(conn); } -/** Remove <b>circ</b> to the list of circuits with pending cells on +/** Remove <b>circ</b> from the list of circuits with pending cells on * <b>conn</b>. No effect if <b>circ</b> is already unlinked. */ void make_circuit_inactive_on_conn(circuit_t *circ, or_connection_t *conn) @@ -1707,6 +2022,8 @@ make_circuit_inactive_on_conn(circuit_t *circ, or_connection_t *conn) return; } + assert_active_circuits_ok_paranoid(conn); + tor_assert(next && prev); tor_assert(*prev_circ_on_conn_p(next, conn) == circ); tor_assert(*next_circ_on_conn_p(prev, conn) == circ); @@ -1720,6 +2037,15 @@ make_circuit_inactive_on_conn(circuit_t *circ, or_connection_t *conn) conn->active_circuits = next; } *prevp = *nextp = NULL; + + if (circ->n_conn == conn) { + remove_cell_ewma_from_conn(conn, &circ->n_cell_ewma); + } else { + or_circuit_t *orcirc = TO_OR_CIRCUIT(circ); + tor_assert(conn == orcirc->p_conn); + remove_cell_ewma_from_conn(conn, &orcirc->p_cell_ewma); + } + assert_active_circuits_ok_paranoid(conn); } @@ -1739,6 +2065,10 @@ connection_or_unlink_all_active_circs(or_connection_t *orconn) cur = next; } while (cur != head); orconn->active_circuits = NULL; + + SMARTLIST_FOREACH(orconn->active_circuit_pqueue, cell_ewma_t *, e, + e->heap_index = -1); + smartlist_clear(orconn->active_circuit_pqueue); } /** Block (if <b>block</b> is true) or unblock (if <b>block</b> is false) @@ -1781,7 +2111,7 @@ set_streams_blocked_on_circ(circuit_t *circ, or_connection_t *orconn, } /** Pull as many cells as possible (but no more than <b>max</b>) from the - * queue of the first active circuit on <b>conn</b>, and write then to + * queue of the first active circuit on <b>conn</b>, and write them to * <b>conn</b>->outbuf. Return the number of cells written. Advance * the active circuit pointer to the next active circuit in the ring. */ int @@ -1792,9 +2122,35 @@ connection_or_flush_from_first_active_circuit(or_connection_t *conn, int max, cell_queue_t *queue; circuit_t *circ; int streams_blocked; + + /* The current (hi-res) time */ + struct timeval now_hires; + + /* The EWMA cell counter for the circuit we're flushing. */ + cell_ewma_t *cell_ewma = NULL; + double ewma_increment = -1; + circ = conn->active_circuits; if (!circ) return 0; assert_active_circuits_ok_paranoid(conn); + + /* See if we're doing the ewma circuit selection algorithm. */ + if (ewma_enabled) { + unsigned tick; + double fractional_tick; + tor_gettimeofday_cached(&now_hires); + tick = cell_ewma_tick_from_timeval(&now_hires, &fractional_tick); + + if (tick != conn->active_circuit_pqueue_last_recalibrated) { + scale_active_circuits(conn, tick); + } + + ewma_increment = pow(ewma_scale_factor, -fractional_tick); + + cell_ewma = smartlist_get(conn->active_circuit_pqueue, 0); + circ = cell_ewma_to_circuit(cell_ewma); + } + if (circ->n_conn == conn) { queue = &circ->n_conn_cells; streams_blocked = circ->streams_blocked_on_n_conn; @@ -1808,10 +2164,58 @@ connection_or_flush_from_first_active_circuit(or_connection_t *conn, int max, packed_cell_t *cell = cell_queue_pop(queue); tor_assert(*next_circ_on_conn_p(circ,conn)); + /* Calculate the exact time that this cell has spent in the queue. */ + if (get_options()->CellStatistics && !CIRCUIT_IS_ORIGIN(circ)) { + struct timeval now; + uint32_t flushed; + uint32_t cell_waiting_time; + insertion_time_queue_t *it_queue = queue->insertion_times; + tor_gettimeofday_cached(&now); + flushed = (uint32_t)((now.tv_sec % SECONDS_IN_A_DAY) * 100L + + (uint32_t)now.tv_usec / (uint32_t)10000L); + if (!it_queue || !it_queue->first) { + log_warn(LD_BUG, "Cannot determine insertion time of cell."); + } else { + or_circuit_t *orcirc = TO_OR_CIRCUIT(circ); + insertion_time_elem_t *elem = it_queue->first; + cell_waiting_time = + (uint32_t)((flushed * 10L + SECONDS_IN_A_DAY * 1000L - + elem->insertion_time * 10L) % + (SECONDS_IN_A_DAY * 1000L)); +#undef SECONDS_IN_A_DAY + elem->counter--; + if (elem->counter < 1) { + it_queue->first = elem->next; + if (elem == it_queue->last) + it_queue->last = NULL; + mp_pool_release(elem); + } + orcirc->total_cell_waiting_time += cell_waiting_time; + orcirc->processed_cells++; + } + } + + /* If we just flushed our queue and this circuit is used for a + * tunneled directory request, possibly advance its state. */ + if (queue->n == 0 && TO_CONN(conn)->dirreq_id) + geoip_change_dirreq_state(TO_CONN(conn)->dirreq_id, + DIRREQ_TUNNELED, + DIRREQ_CIRC_QUEUE_FLUSHED); + connection_write_to_buf(cell->body, CELL_NETWORK_SIZE, TO_CONN(conn)); - packed_cell_free(cell); + packed_cell_free_unchecked(cell); ++n_flushed; + if (cell_ewma) { + cell_ewma_t *tmp; + cell_ewma->cell_count += ewma_increment; + /* We pop and re-add the cell_ewma_t here, not above, since we need to + * re-add it immediately to keep the priority queue consistent with + * the linked-list implementation */ + tmp = pop_first_cell_ewma_from_conn(conn); + tor_assert(tmp == cell_ewma); + add_cell_ewma_to_conn(conn, cell_ewma); + } if (circ != conn->active_circuits) { /* If this happens, the current circuit just got made inactive by * a call in connection_write_to_buf(). That's nothing to worry about: @@ -1831,7 +2235,7 @@ connection_or_flush_from_first_active_circuit(or_connection_t *conn, int max, if (streams_blocked && queue->n <= CELL_QUEUE_LOWWATER_SIZE) set_streams_blocked_on_circ(circ, conn, 0); /* unblock streams */ - /* Did we just ran out of cells on this queue? */ + /* Did we just run out of cells on this circuit's queue? */ if (queue->n == 0) { log_debug(LD_GENERAL, "Made a circuit inactive."); make_circuit_inactive_on_conn(circ, conn); @@ -1954,16 +2358,31 @@ assert_active_circuits_ok(or_connection_t *orconn) { circuit_t *head = orconn->active_circuits; circuit_t *cur = head; + int n = 0; if (! head) return; do { circuit_t *next = *next_circ_on_conn_p(cur, orconn); circuit_t *prev = *prev_circ_on_conn_p(cur, orconn); + cell_ewma_t *ewma; tor_assert(next); tor_assert(prev); tor_assert(*next_circ_on_conn_p(prev, orconn) == cur); tor_assert(*prev_circ_on_conn_p(next, orconn) == cur); + if (orconn == cur->n_conn) { + ewma = &cur->n_cell_ewma; + tor_assert(!ewma->is_for_p_conn); + } else { + ewma = &TO_OR_CIRCUIT(cur)->p_cell_ewma; + tor_assert(ewma->is_for_p_conn); + } + tor_assert(ewma->heap_index != -1); + tor_assert(ewma == smartlist_get(orconn->active_circuit_pqueue, + ewma->heap_index)); + n++; cur = next; } while (cur != head); + + tor_assert(n == smartlist_len(orconn->active_circuit_pqueue)); } diff --git a/src/or/rendclient.c b/src/or/rendclient.c index 47a8818a50..cce8437472 100644 --- a/src/or/rendclient.c +++ b/src/or/rendclient.c @@ -63,7 +63,7 @@ rend_client_send_introduction(origin_circuit_t *introcirc, rend_cache_entry_t *entry; crypt_path_t *cpath; off_t dh_offset; - crypto_pk_env_t *intro_key; /* either Bob's public key or an intro key. */ + crypto_pk_env_t *intro_key = NULL; tor_assert(introcirc->_base.purpose == CIRCUIT_PURPOSE_C_INTRODUCING); tor_assert(rendcirc->_base.purpose == CIRCUIT_PURPOSE_C_REND_READY); @@ -76,48 +76,26 @@ rend_client_send_introduction(origin_circuit_t *introcirc, &entry) < 1) { log_warn(LD_REND, "query %s didn't have valid rend desc in cache. Failing.", - escaped_safe_str(introcirc->rend_data->onion_address)); + escaped_safe_str_client(introcirc->rend_data->onion_address)); goto err; } - /* first 20 bytes of payload are the hash of Bob's pk */ - if (entry->parsed->version == 0) { /* un-versioned descriptor */ - intro_key = entry->parsed->pk; - } else { /* versioned descriptor */ - intro_key = NULL; - SMARTLIST_FOREACH(entry->parsed->intro_nodes, rend_intro_point_t *, - intro, { - if (!memcmp(introcirc->build_state->chosen_exit->identity_digest, - intro->extend_info->identity_digest, DIGEST_LEN)) { - intro_key = intro->intro_key; - break; - } - }); - if (!intro_key) { - /** XXX This case probably means that the intro point vanished while - * we were building a circuit to it. In the future, we should find - * out how that happened and whether we should kill the circuits to - * removed intro points immediately. See task 1073. */ - int num_intro_points = smartlist_len(entry->parsed->intro_nodes); - if (rend_cache_lookup_entry(introcirc->rend_data->onion_address, - 0, &entry) > 0) { - log_info(LD_REND, "We have both a v0 and a v2 rend desc for this " - "service. The v2 desc doesn't contain the introduction " - "point (and key) to send an INTRODUCE1/2 cell to this " - "introduction point. Assuming the introduction point " - "is for v0 rend clients and using the service key " - "from the v0 desc instead. (This is probably a bug, " - "because we shouldn't even have both a v0 and a v2 " - "descriptor for the same service.)"); - /* See flyspray task 1024. */ - intro_key = entry->parsed->pk; - } else { - log_info(LD_REND, "Internal error: could not find intro key; we " - "only have a v2 rend desc with %d intro points.", - num_intro_points); - goto err; - } + /* first 20 bytes of payload are the hash of the intro key */ + intro_key = NULL; + SMARTLIST_FOREACH(entry->parsed->intro_nodes, rend_intro_point_t *, + intro, { + if (!memcmp(introcirc->build_state->chosen_exit->identity_digest, + intro->extend_info->identity_digest, DIGEST_LEN)) { + intro_key = intro->intro_key; + break; } + }); + if (!intro_key) { + log_info(LD_REND, "Our introduction point knowledge changed in " + "mid-connect! Could not find intro key; we only have a " + "v2 rend desc with %d intro points. Giving up.", + smartlist_len(entry->parsed->intro_nodes)); + goto err; } if (crypto_pk_get_digest(intro_key, payload)<0) { log_warn(LD_BUG, "Internal error: couldn't hash public key."); @@ -291,7 +269,7 @@ rend_client_introduction_acked(origin_circuit_t *circ, extend_info = rend_client_get_random_intro(circ->rend_data); if (!extend_info) { log_warn(LD_REND, "No introduction points left for %s. Closing.", - escaped_safe_str(circ->rend_data->onion_address)); + escaped_safe_str_client(circ->rend_data->onion_address)); circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_INTERNAL); return -1; } @@ -299,7 +277,7 @@ rend_client_introduction_acked(origin_circuit_t *circ, log_info(LD_REND, "Got nack for %s from %s. Re-extending circ %d, " "this time to %s.", - escaped_safe_str(circ->rend_data->onion_address), + escaped_safe_str_client(circ->rend_data->onion_address), circ->build_state->chosen_exit->nickname, circ->_base.n_circ_id, extend_info->nickname); result = circuit_extend_to_new_exit(circ, extend_info); @@ -307,7 +285,7 @@ rend_client_introduction_acked(origin_circuit_t *circ, log_info(LD_REND, "Got nack for %s from %s. Building a new introduction " "circuit, this time to %s.", - escaped_safe_str(circ->rend_data->onion_address), + escaped_safe_str_client(circ->rend_data->onion_address), circ->build_state->chosen_exit->nickname, extend_info->nickname); circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_FINISHED); @@ -467,45 +445,21 @@ directory_get_from_hs_dir(const char *desc_id, const rend_data_t *rend_query) rend_query->onion_address, desc_id_base32, rend_query->auth_type, (rend_query->auth_type == REND_NO_AUTH ? "[none]" : - escaped_safe_str(descriptor_cookie_base64)), + escaped_safe_str_client(descriptor_cookie_base64)), hs_dir->nickname, hs_dir->dir_port); return 1; } -/** If we are not currently fetching a rendezvous service descriptor - * for the service ID <b>query</b>, start a directory connection to fetch a - * new one. - */ -void -rend_client_refetch_renddesc(const char *query) -{ - if (!get_options()->FetchHidServDescriptors) - return; - log_info(LD_REND, "Fetching rendezvous descriptor for service %s", - escaped_safe_str(query)); - if (connection_get_by_type_state_rendquery(CONN_TYPE_DIR, 0, query, 0)) { - log_info(LD_REND,"Would fetch a new renddesc here (for %s), but one is " - "already in progress.", escaped_safe_str(query)); - } else { - /* not one already; initiate a dir rend desc lookup */ - directory_get_from_dirserver(DIR_PURPOSE_FETCH_RENDDESC, - ROUTER_PURPOSE_GENERAL, query, - PDS_RETRY_IF_NO_SERVERS); - } -} - -/** Start a connection to a hidden service directory to fetch a v2 - * rendezvous service descriptor for the base32-encoded service ID - * <b>query</b>. - */ +/** Unless we already have a descriptor for <b>rend_query</b> with at least + * one (possibly) working introduction point in it, start a connection to a + * hidden service directory to fetch a v2 rendezvous service descriptor. */ void rend_client_refetch_v2_renddesc(const rend_data_t *rend_query) { char descriptor_id[DIGEST_LEN]; int replicas_left_to_try[REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS]; - int i, tries_left, r; + int i, tries_left; rend_cache_entry_t *e = NULL; - time_t now = time(NULL); tor_assert(rend_query); /* Are we configured to fetch descriptors? */ if (!get_options()->FetchHidServDescriptors) { @@ -514,15 +468,13 @@ rend_client_refetch_v2_renddesc(const rend_data_t *rend_query) return; } /* Before fetching, check if we already have the descriptor here. */ - r = rend_cache_lookup_entry(rend_query->onion_address, -1, &e); - if (r > 0 && now - e->received < NUM_SECONDS_BEFORE_HS_REFETCH) { + if (rend_cache_lookup_entry(rend_query->onion_address, -1, &e) > 0) { log_info(LD_REND, "We would fetch a v2 rendezvous descriptor, but we " - "already have a fresh copy of that descriptor here. " - "Not fetching."); + "already have that descriptor here. Not fetching."); return; } log_debug(LD_REND, "Fetching v2 rendezvous descriptor for service %s", - safe_str(rend_query->onion_address)); + safe_str_client(rend_query->onion_address)); /* Randomly iterate over the replicas until a descriptor can be fetched * from one of the consecutive nodes, or no options are left. */ tries_left = REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS; @@ -548,8 +500,8 @@ rend_client_refetch_v2_renddesc(const rend_data_t *rend_query) log_info(LD_REND, "Could not pick one of the responsible hidden " "service directories to fetch descriptors, because " "we already tried them all unsuccessfully."); - /* Close pending connections (unless a v0 request is still going on). */ - rend_client_desc_trynow(rend_query->onion_address, 2); + /* Close pending connections. */ + rend_client_desc_trynow(rend_query->onion_address); return; } @@ -570,18 +522,13 @@ rend_client_remove_intro_point(extend_info_t *failed_intro, r = rend_cache_lookup_entry(rend_query->onion_address, -1, &ent); if (r<0) { log_warn(LD_BUG, "Malformed service ID %s.", - escaped_safe_str(rend_query->onion_address)); + escaped_safe_str_client(rend_query->onion_address)); return -1; } if (r==0) { log_info(LD_REND, "Unknown service %s. Re-fetching descriptor.", - escaped_safe_str(rend_query->onion_address)); - /* Fetch both, v0 and v2 rend descriptors in parallel. Use whichever - * arrives first. Exception: When using client authorization, only - * fetch v2 descriptors.*/ + escaped_safe_str_client(rend_query->onion_address)); rend_client_refetch_v2_renddesc(rend_query); - if (rend_query->auth_type == REND_NO_AUTH) - rend_client_refetch_renddesc(rend_query->onion_address); return 0; } @@ -598,18 +545,13 @@ rend_client_remove_intro_point(extend_info_t *failed_intro, if (smartlist_len(ent->parsed->intro_nodes) == 0) { log_info(LD_REND, "No more intro points remain for %s. Re-fetching descriptor.", - escaped_safe_str(rend_query->onion_address)); - /* Fetch both, v0 and v2 rend descriptors in parallel. Use whichever - * arrives first. Exception: When using client authorization, only - * fetch v2 descriptors.*/ + escaped_safe_str_client(rend_query->onion_address)); rend_client_refetch_v2_renddesc(rend_query); - if (rend_query->auth_type == REND_NO_AUTH) - rend_client_refetch_renddesc(rend_query->onion_address); /* move all pending streams back to renddesc_wait */ while ((conn = connection_get_by_type_state_rendquery(CONN_TYPE_AP, AP_CONN_STATE_CIRCUIT_WAIT, - rend_query->onion_address, -1))) { + rend_query->onion_address))) { conn->state = AP_CONN_STATE_RENDDESC_WAIT; } @@ -617,7 +559,7 @@ rend_client_remove_intro_point(extend_info_t *failed_intro, } log_info(LD_REND,"%d options left for %s.", smartlist_len(ent->parsed->intro_nodes), - escaped_safe_str(rend_query->onion_address)); + escaped_safe_str_client(rend_query->onion_address)); return 1; } @@ -679,8 +621,9 @@ rend_client_receive_rendezvous(origin_circuit_t *circ, const char *request, tor_assert(circ->build_state->pending_final_cpath); hop = circ->build_state->pending_final_cpath; tor_assert(hop->dh_handshake_state); - if (crypto_dh_compute_secret(hop->dh_handshake_state, request, DH_KEY_LEN, - keys, DIGEST_LEN+CPATH_KEY_MATERIAL_LEN)<0) { + if (crypto_dh_compute_secret(LOG_PROTOCOL_WARN, hop->dh_handshake_state, + request, DH_KEY_LEN, keys, + DIGEST_LEN+CPATH_KEY_MATERIAL_LEN)<0) { log_warn(LD_GENERAL, "Couldn't complete DH handshake."); goto err; } @@ -718,24 +661,18 @@ rend_client_receive_rendezvous(origin_circuit_t *circ, const char *request, return -1; } -/** Find all the apconns in state AP_CONN_STATE_RENDDESC_WAIT that - * are waiting on query. If there's a working cache entry here - * with at least one intro point, move them to the next state. If - * <b>rend_version</b> is non-negative, fail connections that have - * requested <b>query</b> unless there are still descriptor fetch - * requests in progress for other descriptor versions than - * <b>rend_version</b>. - */ +/** Find all the apconns in state AP_CONN_STATE_RENDDESC_WAIT that are + * waiting on <b>query</b>. If there's a working cache entry here with at + * least one intro point, move them to the next state. */ void -rend_client_desc_trynow(const char *query, int rend_version) +rend_client_desc_trynow(const char *query) { edge_connection_t *conn; rend_cache_entry_t *entry; time_t now = time(NULL); smartlist_t *conns = get_connection_array(); - SMARTLIST_FOREACH(conns, connection_t *, _conn, - { + SMARTLIST_FOREACH_BEGIN(conns, connection_t *, _conn) { if (_conn->type != CONN_TYPE_AP || _conn->state != AP_CONN_STATE_RENDDESC_WAIT || _conn->marked_for_close) @@ -767,17 +704,12 @@ rend_client_desc_trynow(const char *query, int rend_version) connection_mark_unattached_ap(conn, END_STREAM_REASON_CANT_ATTACH); } } else { /* 404, or fetch didn't get that far */ - /* Unless there are requests for another descriptor version pending, - * close the connection. */ - if (rend_version >= 0 && - !connection_get_by_type_state_rendquery(CONN_TYPE_DIR, 0, query, - rend_version == 0 ? 2 : 0)) { - log_notice(LD_REND,"Closing stream for '%s.onion': hidden service is " - "unavailable (try again later).", safe_str(query)); - connection_mark_unattached_ap(conn, END_STREAM_REASON_RESOLVEFAILED); - } + log_notice(LD_REND,"Closing stream for '%s.onion': hidden service is " + "unavailable (try again later).", + safe_str_client(query)); + connection_mark_unattached_ap(conn, END_STREAM_REASON_RESOLVEFAILED); } - }); + } SMARTLIST_FOREACH_END(_conn); } /** Return a newly allocated extend_info_t* for a randomly chosen introduction @@ -795,7 +727,7 @@ rend_client_get_random_intro(const rend_data_t *rend_query) if (rend_cache_lookup_entry(rend_query->onion_address, -1, &entry) < 1) { log_warn(LD_REND, "Query '%s' didn't have valid rend desc in cache. Failing.", - safe_str(rend_query->onion_address)); + safe_str_client(rend_query->onion_address)); return NULL; } @@ -945,8 +877,7 @@ rend_parse_service_authorization(or_options_t *options, int validate_only) err: res = -1; done: - if (auth) - rend_service_authorization_free(auth); + rend_service_authorization_free(auth); SMARTLIST_FOREACH(sl, char *, c, tor_free(c);); smartlist_free(sl); if (!validate_only && res == 0) { diff --git a/src/or/rendcommon.c b/src/or/rendcommon.c index d21eb42efe..e4dc5b3d3c 100644 --- a/src/or/rendcommon.c +++ b/src/or/rendcommon.c @@ -22,6 +22,8 @@ rend_cmp_service_ids(const char *one, const char *two) void rend_service_descriptor_free(rend_service_descriptor_t *desc) { + if (!desc) + return; if (desc->pk) crypto_free_pk_env(desc->pk); if (desc->intro_nodes) { @@ -125,7 +127,8 @@ rend_compute_v2_desc_id(char *desc_id_out, const char *service_id, if (!service_id || strlen(service_id) != REND_SERVICE_ID_LEN_BASE32) { log_warn(LD_REND, "Could not compute v2 descriptor ID: " - "Illegal service ID: %s", safe_str(service_id)); + "Illegal service ID: %s", + safe_str(service_id)); return -1; } if (replica >= REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS) { @@ -138,7 +141,7 @@ rend_compute_v2_desc_id(char *desc_id_out, const char *service_id, service_id, REND_SERVICE_ID_LEN_BASE32) < 0) { log_warn(LD_REND, "Could not compute v2 descriptor ID: " "Illegal characters in service ID: %s", - safe_str(service_id)); + safe_str_client(service_id)); return -1; } /* Calculate current time-period. */ @@ -403,8 +406,7 @@ rend_desc_v2_is_parsable(rend_encoded_v2_service_descriptor_t *desc) &test_intro_size, &test_encoded_size, &test_next, desc->desc_str); - if (test_parsed) - rend_service_descriptor_free(test_parsed); + rend_service_descriptor_free(test_parsed); tor_free(test_intro_content); return (res >= 0); } @@ -414,6 +416,8 @@ void rend_encoded_v2_service_descriptor_free( rend_encoded_v2_service_descriptor_t *desc) { + if (!desc) + return; tor_free(desc->desc_str); tor_free(desc); } @@ -422,10 +426,11 @@ rend_encoded_v2_service_descriptor_free( void rend_intro_point_free(rend_intro_point_t *intro) { - if (intro->extend_info) - extend_info_free(intro->extend_info); - if (intro->intro_key) - crypto_free_pk_env(intro->intro_key); + if (!intro) + return; + + extend_info_free(intro->extend_info); + crypto_free_pk_env(intro->intro_key); tor_free(intro); } @@ -618,7 +623,8 @@ rend_encode_v2_descriptors(smartlist_t *descs_out, } if (router_append_dirobj_signature(desc_str + written, desc_len - written, - desc_digest, service_key) < 0) { + desc_digest, DIGEST_LEN, + service_key) < 0) { log_warn(LD_BUG, "Couldn't sign desc."); rend_encoded_v2_service_descriptor_free(enc); goto err; @@ -655,61 +661,6 @@ rend_encode_v2_descriptors(smartlist_t *descs_out, return seconds_valid; } -/** Encode a service descriptor for <b>desc</b>, and sign it with - * <b>key</b>. Store the descriptor in *<b>str_out</b>, and set - * *<b>len_out</b> to its length. - */ -int -rend_encode_service_descriptor(rend_service_descriptor_t *desc, - crypto_pk_env_t *key, - char **str_out, size_t *len_out) -{ - char *cp; - char *end; - int i, r; - size_t asn1len; - size_t buflen = - PK_BYTES*2*(smartlist_len(desc->intro_nodes)+2);/*Too long, but ok*/ - cp = *str_out = tor_malloc(buflen); - end = cp + PK_BYTES*2*(smartlist_len(desc->intro_nodes)+1); - r = crypto_pk_asn1_encode(desc->pk, cp+2, end-(cp+2)); - if (r < 0) { - tor_free(*str_out); - return -1; - } - asn1len = r; - set_uint16(cp, htons((uint16_t)asn1len)); - cp += 2+asn1len; - set_uint32(cp, htonl((uint32_t)desc->timestamp)); - cp += 4; - set_uint16(cp, htons((uint16_t)smartlist_len(desc->intro_nodes))); - cp += 2; - for (i=0; i < smartlist_len(desc->intro_nodes); ++i) { - rend_intro_point_t *intro = smartlist_get(desc->intro_nodes, i); - char ipoint[HEX_DIGEST_LEN+2]; - const size_t ipoint_len = HEX_DIGEST_LEN+1; - ipoint[0] = '$'; - base16_encode(ipoint+1, HEX_DIGEST_LEN+1, - intro->extend_info->identity_digest, - DIGEST_LEN); - tor_assert(strlen(ipoint) == ipoint_len); - /* Assert that appending ipoint and its NUL won't over overrun the - * buffer. */ - tor_assert(cp + ipoint_len+1 < *str_out + buflen); - memcpy(cp, ipoint, ipoint_len+1); - cp += ipoint_len+1; - } - note_crypto_pk_op(REND_SERVER); - r = crypto_pk_private_sign_digest(key, cp, *str_out, cp-*str_out); - if (r<0) { - tor_free(*str_out); - return -1; - } - cp += r; - *len_out = (size_t)(cp-*str_out); - return 0; -} - /** Parse a service descriptor at <b>str</b> (<b>len</b> bytes). On * success, return a newly alloced service_descriptor_t. On failure, * return NULL. @@ -826,22 +777,27 @@ rend_cache_init(void) /** Helper: free storage held by a single service descriptor cache entry. */ static void -_rend_cache_entry_free(void *p) +rend_cache_entry_free(rend_cache_entry_t *e) { - rend_cache_entry_t *e = p; + if (!e) + return; rend_service_descriptor_free(e->parsed); tor_free(e->desc); tor_free(e); } +static void +_rend_cache_entry_free(void *p) +{ + rend_cache_entry_free(p); +} + /** Free all storage held by the service descriptor cache. */ void rend_cache_free_all(void) { - if (rend_cache) - strmap_free(rend_cache, _rend_cache_entry_free); - if (rend_cache_v2_dir) - digestmap_free(rend_cache_v2_dir, _rend_cache_entry_free); + strmap_free(rend_cache, _rend_cache_entry_free); + digestmap_free(rend_cache_v2_dir, _rend_cache_entry_free); rend_cache = NULL; rend_cache_v2_dir = NULL; } @@ -862,7 +818,7 @@ rend_cache_clean(void) ent = (rend_cache_entry_t*)val; if (ent->parsed->timestamp < cutoff) { iter = strmap_iter_next_rmv(rend_cache, iter); - _rend_cache_entry_free(ent); + rend_cache_entry_free(ent); } else { iter = strmap_iter_next(rend_cache, iter); } @@ -888,9 +844,9 @@ rend_cache_clean_v2_descs_as_dir(void) char key_base32[REND_DESC_ID_V2_LEN_BASE32 + 1]; base32_encode(key_base32, sizeof(key_base32), key, DIGEST_LEN); log_info(LD_REND, "Removing descriptor with ID '%s' from cache", - safe_str(key_base32)); + safe_str_client(key_base32)); iter = digestmap_iter_next_rmv(rend_cache_v2_dir, iter); - _rend_cache_entry_free(ent); + rend_cache_entry_free(ent); } else { iter = digestmap_iter_next(rend_cache_v2_dir, iter); } @@ -966,6 +922,11 @@ rend_cache_lookup_entry(const char *query, int version, rend_cache_entry_t **e) } if (!*e) return 0; + tor_assert((*e)->parsed && (*e)->parsed->intro_nodes); + /* XXX022 hack for now, to return "not found" if there are no intro + * points remaining. See bug 997. */ + if (smartlist_len((*e)->parsed->intro_nodes) == 0) + return 0; return 1; } @@ -1044,7 +1005,6 @@ rend_cache_store(const char *desc, size_t desc_len, int published) char query[REND_SERVICE_ID_LEN_BASE32+1]; char key[REND_SERVICE_ID_LEN_BASE32+2]; /* 0<query>\0 */ time_t now; - or_options_t *options = get_options(); tor_assert(rend_cache); parsed = rend_parse_service_descriptor(desc,desc_len); if (!parsed) { @@ -1059,13 +1019,15 @@ rend_cache_store(const char *desc, size_t desc_len, int published) now = time(NULL); if (parsed->timestamp < now-REND_CACHE_MAX_AGE-REND_CACHE_MAX_SKEW) { log_fn(LOG_PROTOCOL_WARN, LD_REND, - "Service descriptor %s is too old.", safe_str(query)); + "Service descriptor %s is too old.", + safe_str_client(query)); rend_service_descriptor_free(parsed); return -2; } if (parsed->timestamp > now+REND_CACHE_MAX_SKEW) { log_fn(LOG_PROTOCOL_WARN, LD_REND, - "Service descriptor %s is too far in the future.", safe_str(query)); + "Service descriptor %s is too far in the future.", + safe_str_client(query)); rend_service_descriptor_free(parsed); return -2; } @@ -1073,25 +1035,22 @@ rend_cache_store(const char *desc, size_t desc_len, int published) tor_snprintf(key, sizeof(key), "2%s", query); if (!published && strmap_get_lc(rend_cache, key)) { log_info(LD_REND, "We already have a v2 descriptor for service %s.", - safe_str(query)); + safe_str_client(query)); rend_service_descriptor_free(parsed); return -1; } - /* report novel publication to statistics */ - if (published && options->HSAuthorityRecordStats) { - hs_usage_note_publish_total(query, now); - } tor_snprintf(key, sizeof(key), "0%s", query); e = (rend_cache_entry_t*) strmap_get_lc(rend_cache, key); if (e && e->parsed->timestamp > parsed->timestamp) { log_info(LD_REND,"We already have a newer service descriptor %s with the " - "same ID and version.", safe_str(query)); + "same ID and version.", + safe_str_client(query)); rend_service_descriptor_free(parsed); return 0; } if (e && e->len == desc_len && !memcmp(desc,e->desc,desc_len)) { log_info(LD_REND,"We already have this service descriptor %s.", - safe_str(query)); + safe_str_client(query)); e->received = time(NULL); rend_service_descriptor_free(parsed); return 0; @@ -1099,10 +1058,6 @@ rend_cache_store(const char *desc, size_t desc_len, int published) if (!e) { e = tor_malloc_zero(sizeof(rend_cache_entry_t)); strmap_set_lc(rend_cache, key, e); - /* report novel publication to statistics */ - if (published && options->HSAuthorityRecordStats) { - hs_usage_note_publish_novel(query, now); - } } else { rend_service_descriptor_free(e->parsed); tor_free(e->desc); @@ -1114,7 +1069,7 @@ rend_cache_store(const char *desc, size_t desc_len, int published) memcpy(e->desc, desc, desc_len); log_debug(LD_REND,"Successfully stored rend desc '%s', len %d.", - safe_str(query), (int)desc_len); + safe_str_client(query), (int)desc_len); return 1; } @@ -1165,7 +1120,7 @@ rend_cache_store_v2_desc_as_dir(const char *desc) if (!hid_serv_responsible_for_desc_id(desc_id)) { log_info(LD_REND, "Service descriptor with desc ID %s is not in " "interval that we are responsible for.", - safe_str(desc_id_base32)); + safe_str_client(desc_id_base32)); goto skip; } /* Is descriptor too old? */ @@ -1294,7 +1249,8 @@ rend_cache_store_v2_desc_as_client(const char *desc, /* Decode/decrypt introduction points. */ if (intro_content) { if (rend_query->auth_type != REND_NO_AUTH && - rend_query->descriptor_cookie) { + !tor_mem_is_zero(rend_query->descriptor_cookie, + sizeof(rend_query->descriptor_cookie))) { char *ipos_decrypted = NULL; size_t ipos_decrypted_size; if (rend_decrypt_introduction_points(&ipos_decrypted, @@ -1329,14 +1285,14 @@ rend_cache_store_v2_desc_as_client(const char *desc, /* Is descriptor too old? */ if (parsed->timestamp < now - REND_CACHE_MAX_AGE-REND_CACHE_MAX_SKEW) { log_warn(LD_REND, "Service descriptor with service ID %s is too old.", - safe_str(service_id)); + safe_str_client(service_id)); retval = -2; goto err; } /* Is descriptor too far in the future? */ if (parsed->timestamp > now + REND_CACHE_MAX_SKEW) { log_warn(LD_REND, "Service descriptor with service ID %s is too far in " - "the future.", safe_str(service_id)); + "the future.", safe_str_client(service_id)); retval = -2; goto err; } @@ -1344,7 +1300,7 @@ rend_cache_store_v2_desc_as_client(const char *desc, tor_snprintf(key, sizeof(key), "0%s", service_id); if (strmap_get_lc(rend_cache, key)) { log_info(LD_REND, "We already have a v0 descriptor for service ID %s.", - safe_str(service_id)); + safe_str_client(service_id)); retval = -1; goto err; } @@ -1354,14 +1310,14 @@ rend_cache_store_v2_desc_as_client(const char *desc, if (e && e->parsed->timestamp > parsed->timestamp) { log_info(LD_REND, "We already have a newer service descriptor for " "service ID %s with the same desc ID and version.", - safe_str(service_id)); + safe_str_client(service_id)); retval = 0; goto err; } /* Do we already have this descriptor? */ if (e && !strcmp(desc, e->desc)) { log_info(LD_REND,"We already have this service descriptor %s.", - safe_str(service_id)); + safe_str_client(service_id)); e->received = time(NULL); retval = 0; goto err; @@ -1379,12 +1335,11 @@ rend_cache_store_v2_desc_as_client(const char *desc, strlcpy(e->desc, desc, encoded_size + 1); e->len = encoded_size; log_debug(LD_REND,"Successfully stored rend desc '%s', len %d.", - safe_str(service_id), (int)encoded_size); + safe_str_client(service_id), (int)encoded_size); return 1; err: - if (parsed) - rend_service_descriptor_free(parsed); + rend_service_descriptor_free(parsed); tor_free(intro_content); return retval; } diff --git a/src/or/rendservice.c b/src/or/rendservice.c index d2868b738d..9fcf248c37 100644 --- a/src/or/rendservice.c +++ b/src/or/rendservice.c @@ -10,8 +10,7 @@ #include "or.h" static origin_circuit_t *find_intro_circuit(rend_intro_point_t *intro, - const char *pk_digest, - int desc_version); + const char *pk_digest); /** Represents the mapping from a virtual port of a rendezvous service to * a real port on some IP. @@ -42,8 +41,6 @@ typedef struct rend_service_t { /* Fields specified in config file */ char *directory; /**< where in the filesystem it stores it */ smartlist_t *ports; /**< List of rend_service_port_config_t */ - int descriptor_version; /**< Rendezvous descriptor version that will be - * published. */ rend_auth_type_t auth_type; /**< Client authorization type or 0 if no client * authorization is performed. */ smartlist_t *clients; /**< List of rend_authorized_client_t's of @@ -58,7 +55,7 @@ typedef struct rend_service_t { * or are trying to establish. */ time_t intro_period_started; /**< Start of the current period to build * introduction points. */ - int n_intro_circuits_launched; /**< count of intro circuits we have + int n_intro_circuits_launched; /**< Count of intro circuits we have * established in this period. */ rend_service_descriptor_t *desc; /**< Current hidden service descriptor. */ time_t desc_is_dirty; /**< Time at which changes to the hidden service @@ -90,7 +87,8 @@ num_rend_services(void) static void rend_authorized_client_free(rend_authorized_client_t *client) { - if (!client) return; + if (!client) + return; if (client->client_key) crypto_free_pk_env(client->client_key); tor_free(client->client_name); @@ -109,7 +107,9 @@ rend_authorized_client_strmap_item_free(void *authorized_client) static void rend_service_free(rend_service_t *service) { - if (!service) return; + if (!service) + return; + tor_free(service->directory); SMARTLIST_FOREACH(service->ports, void*, p, tor_free(p)); smartlist_free(service->ports); @@ -120,15 +120,14 @@ rend_service_free(rend_service_t *service) rend_intro_point_free(intro);); smartlist_free(service->intro_nodes); } - if (service->desc) - rend_service_descriptor_free(service->desc); + + rend_service_descriptor_free(service->desc); if (service->clients) { SMARTLIST_FOREACH(service->clients, rend_authorized_client_t *, c, rend_authorized_client_free(c);); smartlist_free(service->clients); } - if (service->accepted_intros) - digestmap_free(service->accepted_intros, _tor_free); + digestmap_free(service->accepted_intros, _tor_free); tor_free(service); } @@ -137,9 +136,9 @@ rend_service_free(rend_service_t *service) void rend_service_free_all(void) { - if (!rend_service_list) { + if (!rend_service_list) return; - } + SMARTLIST_FOREACH(rend_service_list, rend_service_t*, ptr, rend_service_free(ptr)); smartlist_free(rend_service_list); @@ -156,36 +155,6 @@ rend_add_service(rend_service_t *service) service->intro_nodes = smartlist_create(); - /* If the service is configured to publish unversioned (v0) and versioned - * descriptors (v2 or higher), split it up into two separate services - * (unless it is configured to perform client authorization). */ - if (service->descriptor_version == -1) { - if (service->auth_type == REND_NO_AUTH) { - rend_service_t *v0_service = tor_malloc_zero(sizeof(rend_service_t)); - v0_service->directory = tor_strdup(service->directory); - v0_service->ports = smartlist_create(); - SMARTLIST_FOREACH(service->ports, rend_service_port_config_t *, p, { - rend_service_port_config_t *copy = - tor_malloc_zero(sizeof(rend_service_port_config_t)); - memcpy(copy, p, sizeof(rend_service_port_config_t)); - smartlist_add(v0_service->ports, copy); - }); - v0_service->intro_period_started = service->intro_period_started; - v0_service->descriptor_version = 0; /* Unversioned descriptor. */ - v0_service->auth_type = REND_NO_AUTH; - rend_add_service(v0_service); - } - - service->descriptor_version = 2; /* Versioned descriptor. */ - } - - if (service->auth_type != REND_NO_AUTH && !service->descriptor_version) { - log_warn(LD_CONFIG, "Hidden service with client authorization and " - "version 0 descriptors configured; ignoring."); - rend_service_free(service); - return; - } - if (service->auth_type != REND_NO_AUTH && smartlist_len(service->clients) == 0) { log_warn(LD_CONFIG, "Hidden service with client authorization but no " @@ -297,7 +266,7 @@ rend_config_services(or_options_t *options, int validate_only) for (line = options->RendConfigLines; line; line = line->next) { if (!strcasecmp(line->key, "HiddenServiceDir")) { - if (service) { + if (service) { /* register the one we just finished parsing */ if (validate_only) rend_service_free(service); else @@ -307,7 +276,6 @@ rend_config_services(or_options_t *options, int validate_only) service->directory = tor_strdup(line->value); service->ports = smartlist_create(); service->intro_period_started = time(NULL); - service->descriptor_version = -1; /**< All descriptor versions. */ continue; } if (!service) { @@ -433,35 +401,13 @@ rend_config_services(or_options_t *options, int validate_only) return -1; } } else { - smartlist_t *versions; - char *version_str; - int i, version, ver_ok=1, versions_bitmask = 0; tor_assert(!strcasecmp(line->key, "HiddenServiceVersion")); - versions = smartlist_create(); - smartlist_split_string(versions, line->value, ",", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - for (i = 0; i < smartlist_len(versions); i++) { - version_str = smartlist_get(versions, i); - if (strlen(version_str) != 1 || strspn(version_str, "02") != 1) { - log_warn(LD_CONFIG, - "HiddenServiceVersion can only be 0 and/or 2."); - SMARTLIST_FOREACH(versions, char *, cp, tor_free(cp)); - smartlist_free(versions); - rend_service_free(service); - return -1; - } - version = (int)tor_parse_long(version_str, 10, 0, INT_MAX, &ver_ok, - NULL); - if (!ver_ok) - continue; - versions_bitmask |= 1 << version; + if (strcmp(line->value, "2")) { + log_warn(LD_CONFIG, + "The only supported HiddenServiceVersion is 2."); + rend_service_free(service); + return -1; } - /* If exactly one version is set, change descriptor_version to that - * value; otherwise leave it at -1. */ - if (versions_bitmask == 1 << 0) service->descriptor_version = 0; - if (versions_bitmask == 1 << 2) service->descriptor_version = 2; - SMARTLIST_FOREACH(versions, char *, cp, tor_free(cp)); - smartlist_free(versions); } } if (service) { @@ -483,8 +429,7 @@ rend_config_services(or_options_t *options, int validate_only) * probably ok? */ SMARTLIST_FOREACH(rend_service_list, rend_service_t *, new, { SMARTLIST_FOREACH(old_service_list, rend_service_t *, old, { - if (!strcmp(old->directory, new->directory) && - old->descriptor_version == new->descriptor_version) { + if (!strcmp(old->directory, new->directory)) { smartlist_add_all(new->intro_nodes, old->intro_nodes); smartlist_clear(old->intro_nodes); smartlist_add(surviving_services, old); @@ -507,18 +452,16 @@ rend_config_services(or_options_t *options, int validate_only) tor_assert(oc->rend_data); SMARTLIST_FOREACH(surviving_services, rend_service_t *, ptr, { if (!memcmp(ptr->pk_digest, oc->rend_data->rend_pk_digest, - DIGEST_LEN) && - ptr->descriptor_version == oc->rend_data->rend_desc_version) { + DIGEST_LEN)) { keep_it = 1; break; } }); if (keep_it) continue; - log_info(LD_REND, "Closing intro point %s for service %s version %d.", - safe_str(oc->build_state->chosen_exit->nickname), - oc->rend_data->onion_address, - oc->rend_data->rend_desc_version); + log_info(LD_REND, "Closing intro point %s for service %s.", + safe_str_client(oc->build_state->chosen_exit->nickname), + oc->rend_data->onion_address); circuit_mark_for_close(circ, END_CIRC_REASON_FINISHED); /* XXXX Is there another reason we should use here? */ } @@ -541,14 +484,13 @@ rend_service_update_descriptor(rend_service_t *service) rend_service_descriptor_t *d; origin_circuit_t *circ; int i; - if (service->desc) { - rend_service_descriptor_free(service->desc); - service->desc = NULL; - } + + rend_service_descriptor_free(service->desc); + service->desc = NULL; + d = service->desc = tor_malloc_zero(sizeof(rend_service_descriptor_t)); d->pk = crypto_pk_dup_key(service->private_key); d->timestamp = time(NULL); - d->version = service->descriptor_version; d->intro_nodes = smartlist_create(); /* Support intro protocols 2 and 3. */ d->protocols = (1 << 2) + (1 << 3); @@ -556,7 +498,7 @@ rend_service_update_descriptor(rend_service_t *service) for (i = 0; i < smartlist_len(service->intro_nodes); ++i) { rend_intro_point_t *intro_svc = smartlist_get(service->intro_nodes, i); rend_intro_point_t *intro_desc; - circ = find_intro_circuit(intro_svc, service->pk_digest, d->version); + circ = find_intro_circuit(intro_svc, service->pk_digest); if (!circ || circ->_base.purpose != CIRCUIT_PURPOSE_S_INTRO) continue; @@ -797,17 +739,15 @@ rend_service_load_keys(void) return r; } -/** Return the service whose public key has a digest of <b>digest</b> and - * which publishes the given descriptor <b>version</b>. Return NULL if no - * such service exists. +/** Return the service whose public key has a digest of <b>digest</b>, or + * NULL if no such service exists. */ static rend_service_t * -rend_service_get_by_pk_digest_and_version(const char* digest, - uint8_t version) +rend_service_get_by_pk_digest(const char* digest) { SMARTLIST_FOREACH(rend_service_list, rend_service_t*, s, - if (!memcmp(s->pk_digest,digest,DIGEST_LEN) && - s->descriptor_version == version) return s); + if (!memcmp(s->pk_digest,digest,DIGEST_LEN)) + return s); return NULL; } @@ -944,21 +884,16 @@ rend_service_introduce(origin_circuit_t *circuit, const char *request, } /* look up service depending on circuit. */ - service = rend_service_get_by_pk_digest_and_version( - circuit->rend_data->rend_pk_digest, - circuit->rend_data->rend_desc_version); + service = rend_service_get_by_pk_digest( + circuit->rend_data->rend_pk_digest); if (!service) { log_warn(LD_REND, "Got an INTRODUCE2 cell for an unrecognized service %s.", escaped(serviceid)); return -1; } - /* if descriptor version is 2, use intro key instead of service key. */ - if (circuit->rend_data->rend_desc_version == 0) { - intro_key = service->private_key; - } else { - intro_key = circuit->intro_key; - } + /* use intro key instead of service key. */ + intro_key = circuit->intro_key; /* first DIGEST_LEN bytes of request is intro or service pk digest */ crypto_pk_get_digest(intro_key, intro_key_digest); @@ -988,7 +923,7 @@ rend_service_introduce(origin_circuit_t *circuit, const char *request, len = r; if (*buf == 3) { /* Version 3 INTRODUCE2 cell. */ - time_t ts = 0, now = time(NULL); + time_t ts = 0; v3_shift = 1; auth_type = buf[1]; switch (auth_type) { @@ -1082,7 +1017,7 @@ rend_service_introduce(origin_circuit_t *circuit, const char *request, router = router_get_by_nickname(rp_nickname, 0); if (!router) { log_info(LD_REND, "Couldn't find router %s named in introduce2 cell.", - escaped_safe_str(rp_nickname)); + escaped_safe_str_client(rp_nickname)); /* XXXX Add a no-such-router reason? */ reason = END_CIRC_REASON_TORPROTOCOL; goto err; @@ -1157,7 +1092,8 @@ rend_service_introduce(origin_circuit_t *circuit, const char *request, reason = END_CIRC_REASON_INTERNAL; goto err; } - if (crypto_dh_compute_secret(dh, ptr+REND_COOKIE_LEN, DH_KEY_LEN, keys, + if (crypto_dh_compute_secret(LOG_PROTOCOL_WARN, dh, ptr+REND_COOKIE_LEN, + DH_KEY_LEN, keys, DIGEST_LEN+CPATH_KEY_MATERIAL_LEN)<0) { log_warn(LD_BUG, "Internal error: couldn't complete DH handshake"); reason = END_CIRC_REASON_INTERNAL; @@ -1167,7 +1103,7 @@ rend_service_introduce(origin_circuit_t *circuit, const char *request, circ_needs_uptime = rend_service_requires_uptime(service); /* help predict this next time */ - rep_hist_note_used_internal(time(NULL), circ_needs_uptime, 1); + rep_hist_note_used_internal(now, circ_needs_uptime, 1); /* Launch a circuit to alice's chosen rendezvous point. */ @@ -1183,14 +1119,16 @@ rend_service_introduce(origin_circuit_t *circuit, const char *request, if (!launched) { /* give up */ log_warn(LD_REND, "Giving up launching first hop of circuit to rendezvous " "point %s for service %s.", - escaped_safe_str(extend_info->nickname), serviceid); + escaped_safe_str_client(extend_info->nickname), + serviceid); reason = END_CIRC_REASON_CONNECTFAILED; goto err; } log_info(LD_REND, "Accepted intro; launching circuit to %s " "(cookie %s) for service %s.", - escaped_safe_str(extend_info->nickname), hexcookie, serviceid); + escaped_safe_str_client(extend_info->nickname), + hexcookie, serviceid); tor_assert(launched->build_state); /* Fill in the circuit's state. */ launched->rend_data = tor_malloc_zero(sizeof(rend_data_t)); @@ -1200,11 +1138,10 @@ rend_service_introduce(origin_circuit_t *circuit, const char *request, memcpy(launched->rend_data->rend_cookie, r_cookie, REND_COOKIE_LEN); strlcpy(launched->rend_data->onion_address, service->service_id, sizeof(launched->rend_data->onion_address)); - launched->rend_data->rend_desc_version = service->descriptor_version; launched->build_state->pending_final_cpath = cpath = tor_malloc_zero(sizeof(crypt_path_t)); cpath->magic = CRYPT_PATH_MAGIC; - launched->build_state->expiry_time = time(NULL) + MAX_REND_TIMEOUT; + launched->build_state->expiry_time = now + MAX_REND_TIMEOUT; cpath->dh_handshake_state = dh; dh = NULL; @@ -1286,7 +1223,7 @@ rend_service_launch_establish_intro(rend_service_t *service, log_info(LD_REND, "Launching circuit to introduction point %s for service %s", - escaped_safe_str(intro->extend_info->nickname), + escaped_safe_str_client(intro->extend_info->nickname), service->service_id); rep_hist_note_used_internal(time(NULL), 1, 0); @@ -1299,7 +1236,7 @@ rend_service_launch_establish_intro(rend_service_t *service, if (!launched) { log_info(LD_REND, "Can't launch circuit to establish introduction at %s.", - escaped_safe_str(intro->extend_info->nickname)); + escaped_safe_str_client(intro->extend_info->nickname)); return -1; } @@ -1322,18 +1259,16 @@ rend_service_launch_establish_intro(rend_service_t *service, strlcpy(launched->rend_data->onion_address, service->service_id, sizeof(launched->rend_data->onion_address)); memcpy(launched->rend_data->rend_pk_digest, service->pk_digest, DIGEST_LEN); - launched->rend_data->rend_desc_version = service->descriptor_version; - if (service->descriptor_version == 2) - launched->intro_key = crypto_pk_dup_key(intro->intro_key); + launched->intro_key = crypto_pk_dup_key(intro->intro_key); if (launched->_base.state == CIRCUIT_STATE_OPEN) rend_service_intro_has_opened(launched); return 0; } /** Return the number of introduction points that are or have been - * established for the given service address and rendezvous version. */ + * established for the given service address in <b>query</b>. */ static int -count_established_intro_points(const char *query, int rend_version) +count_established_intro_points(const char *query) { int num_ipos = 0; circuit_t *circ; @@ -1344,7 +1279,6 @@ count_established_intro_points(const char *query, int rend_version) circ->purpose == CIRCUIT_PURPOSE_S_INTRO)) { origin_circuit_t *oc = TO_ORIGIN_CIRCUIT(circ); if (oc->rend_data && - oc->rend_data->rend_desc_version == rend_version && !rend_cmp_service_ids(query, oc->rend_data->onion_address)) num_ipos++; } @@ -1374,9 +1308,8 @@ rend_service_intro_has_opened(origin_circuit_t *circuit) base32_encode(serviceid, REND_SERVICE_ID_LEN_BASE32+1, circuit->rend_data->rend_pk_digest, REND_SERVICE_ID_LEN); - service = rend_service_get_by_pk_digest_and_version( - circuit->rend_data->rend_pk_digest, - circuit->rend_data->rend_desc_version); + service = rend_service_get_by_pk_digest( + circuit->rend_data->rend_pk_digest); if (!service) { log_warn(LD_REND, "Unrecognized service ID %s on introduction circuit %d.", serviceid, circuit->_base.n_circ_id); @@ -1386,8 +1319,7 @@ rend_service_intro_has_opened(origin_circuit_t *circuit) /* If we already have enough introduction circuits for this service, * redefine this one as a general circuit. */ - if (count_established_intro_points(serviceid, - circuit->rend_data->rend_desc_version) > NUM_INTRO_POINTS) { + if (count_established_intro_points(serviceid) > NUM_INTRO_POINTS) { log_info(LD_CIRC|LD_REND, "We have just finished an introduction " "circuit, but we already have enough. Redefining purpose to " "general."); @@ -1400,13 +1332,8 @@ rend_service_intro_has_opened(origin_circuit_t *circuit) "Established circuit %d as introduction point for service %s", circuit->_base.n_circ_id, serviceid); - /* If the introduction point will not be used in an unversioned - * descriptor, use the intro key instead of the service key in - * ESTABLISH_INTRO. */ - if (service->descriptor_version == 0) - intro_key = service->private_key; - else - intro_key = circuit->intro_key; + /* Use the intro key instead of the service key in ESTABLISH_INTRO. */ + intro_key = circuit->intro_key; /* Build the payload for a RELAY_ESTABLISH_INTRO cell. */ r = crypto_pk_asn1_encode(intro_key, buf+2, RELAY_PAYLOAD_SIZE-2); @@ -1465,9 +1392,8 @@ rend_service_intro_established(origin_circuit_t *circuit, const char *request, goto err; } tor_assert(circuit->rend_data); - service = rend_service_get_by_pk_digest_and_version( - circuit->rend_data->rend_pk_digest, - circuit->rend_data->rend_desc_version); + service = rend_service_get_by_pk_digest( + circuit->rend_data->rend_pk_digest); if (!service) { log_warn(LD_REND, "Unknown service on introduction circuit %d.", circuit->_base.n_circ_id); @@ -1517,9 +1443,8 @@ rend_service_rendezvous_has_opened(origin_circuit_t *circuit) "cookie %s for service %s", circuit->_base.n_circ_id, hexcookie, serviceid); - service = rend_service_get_by_pk_digest_and_version( - circuit->rend_data->rend_pk_digest, - circuit->rend_data->rend_desc_version); + service = rend_service_get_by_pk_digest( + circuit->rend_data->rend_pk_digest); if (!service) { log_warn(LD_GENERAL, "Internal error: unrecognized service ID on " "introduction circuit."); @@ -1575,13 +1500,12 @@ rend_service_rendezvous_has_opened(origin_circuit_t *circuit) */ /** Return the (possibly non-open) introduction circuit ending at - * <b>intro</b> for the service whose public key is <b>pk_digest</b> and - * which publishes descriptor of version <b>desc_version</b>. Return - * NULL if no such service is found. + * <b>intro</b> for the service whose public key is <b>pk_digest</b>. + * (<b>desc_version</b> is ignored). Return NULL if no such service is + * found. */ static origin_circuit_t * -find_intro_circuit(rend_intro_point_t *intro, const char *pk_digest, - int desc_version) +find_intro_circuit(rend_intro_point_t *intro, const char *pk_digest) { origin_circuit_t *circ = NULL; @@ -1590,8 +1514,7 @@ find_intro_circuit(rend_intro_point_t *intro, const char *pk_digest, CIRCUIT_PURPOSE_S_INTRO))) { if (!memcmp(circ->build_state->chosen_exit->identity_digest, intro->extend_info->identity_digest, DIGEST_LEN) && - circ->rend_data && - circ->rend_data->rend_desc_version == desc_version) { + circ->rend_data) { return circ; } } @@ -1601,8 +1524,7 @@ find_intro_circuit(rend_intro_point_t *intro, const char *pk_digest, CIRCUIT_PURPOSE_S_ESTABLISH_INTRO))) { if (!memcmp(circ->build_state->chosen_exit->identity_digest, intro->extend_info->identity_digest, DIGEST_LEN) && - circ->rend_data && - circ->rend_data->rend_desc_version == desc_version) { + circ->rend_data) { return circ; } } @@ -1635,6 +1557,7 @@ directory_post_to_hs_dir(rend_service_descriptor_t *renddesc, } for (j = 0; j < smartlist_len(responsible_dirs); j++) { char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1]; + char *hs_dir_ip; hs_dir = smartlist_get(responsible_dirs, j); if (smartlist_digest_isin(renddesc->successful_uploads, hs_dir->identity_digest)) @@ -1656,15 +1579,18 @@ directory_post_to_hs_dir(rend_service_descriptor_t *renddesc, strlen(desc->desc_str), 0); base32_encode(desc_id_base32, sizeof(desc_id_base32), desc->desc_id, DIGEST_LEN); + hs_dir_ip = tor_dup_ip(hs_dir->addr); log_info(LD_REND, "Sending publish request for v2 descriptor for " "service '%s' with descriptor ID '%s' with validity " "of %d seconds to hidden service directory '%s' on " - "port %d.", - safe_str(service_id), - safe_str(desc_id_base32), + "%s:%d.", + safe_str_client(service_id), + safe_str_client(desc_id_base32), seconds_valid, hs_dir->nickname, - hs_dir->dir_port); + hs_dir_ip, + hs_dir->or_port); + tor_free(hs_dir_ip); /* Remember successful upload to this router for next time. */ if (!smartlist_digest_isin(successful_uploads, hs_dir->identity_digest)) smartlist_add(successful_uploads, hs_dir->identity_digest); @@ -1694,9 +1620,8 @@ directory_post_to_hs_dir(rend_service_descriptor_t *renddesc, smartlist_free(successful_uploads); } -/** Encode and sign up-to-date v0 and/or v2 service descriptors for - * <b>service</b>, and upload it/them to all the dirservers/to the - * responsible hidden service directories. +/** Encode and sign an up-to-date service descriptor for <b>service</b>, + * and upload it/them to the responsible hidden service directories. */ static void upload_service_descriptor(rend_service_t *service) @@ -1708,35 +1633,8 @@ upload_service_descriptor(rend_service_t *service) rendpostperiod = get_options()->RendPostPeriod; - /* Upload unversioned (v0) descriptor? */ - if (service->descriptor_version == 0 && - get_options()->PublishHidServDescriptors) { - char *desc; - size_t desc_len; - /* Encode the descriptor. */ - if (rend_encode_service_descriptor(service->desc, - service->private_key, - &desc, &desc_len)<0) { - log_warn(LD_BUG, "Internal error: couldn't encode service descriptor; " - "not uploading."); - return; - } - - /* Post it to the dirservers */ - rend_get_service_id(service->desc->pk, serviceid); - log_info(LD_REND, "Sending publish request for hidden service %s", - serviceid); - directory_post_to_dirservers(DIR_PURPOSE_UPLOAD_RENDDESC, - ROUTER_PURPOSE_GENERAL, - HIDSERV_AUTHORITY, desc, desc_len, 0); - tor_free(desc); - service->next_upload_time = now + rendpostperiod; - uploaded = 1; - } - - /* Upload v2 descriptor? */ - if (service->descriptor_version == 2 && - get_options()->PublishHidServDescriptors) { + /* Upload descriptor? */ + if (get_options()->PublishHidServDescriptors) { networkstatus_t *c = networkstatus_get_latest_consensus(); if (c && smartlist_len(c->routerstatus_list) > 0) { int seconds_valid, i, j, num_descs; @@ -1875,8 +1773,7 @@ rend_services_introduce(void) for (j=0; j < smartlist_len(service->intro_nodes); ++j) { intro = smartlist_get(service->intro_nodes, j); router = router_get_by_digest(intro->extend_info->identity_digest); - if (!router || !find_intro_circuit(intro, service->pk_digest, - service->descriptor_version)) { + if (!router || !find_intro_circuit(intro, service->pk_digest)) { log_info(LD_REND,"Giving up on %s as intro point for %s.", intro->extend_info->nickname, service->service_id); if (service->desc) { @@ -1928,7 +1825,7 @@ rend_services_introduce(void) router_crn_flags_t flags = CRN_NEED_UPTIME; if (get_options()->_AllowInvalid & ALLOW_INVALID_INTRODUCTION) flags |= CRN_ALLOW_INVALID; - router = router_choose_random_node(NULL, intro_routers, + router = router_choose_random_node(intro_routers, options->ExcludeNodes, flags); if (!router) { log_warn(LD_REND, @@ -1940,10 +1837,8 @@ rend_services_introduce(void) smartlist_add(intro_routers, router); intro = tor_malloc_zero(sizeof(rend_intro_point_t)); intro->extend_info = extend_info_from_router(router); - if (service->descriptor_version == 2) { - intro->intro_key = crypto_new_pk_env(); - tor_assert(!crypto_pk_generate_key(intro->intro_key)); - } + intro->intro_key = crypto_new_pk_env(); + tor_assert(!crypto_pk_generate_key(intro->intro_key)); smartlist_add(service->intro_nodes, intro); log_info(LD_REND, "Picked router %s as an intro point for %s.", router->nickname, service->service_id); @@ -2036,8 +1931,7 @@ rend_consider_descriptor_republication(void) for (i=0; i < smartlist_len(rend_service_list); ++i) { service = smartlist_get(rend_service_list, i); - if (service->descriptor_version && service->desc && - !service->desc->all_uploads_performed) { + if (service->desc && !service->desc->all_uploads_performed) { /* If we failed in uploading a descriptor last time, try again *without* * updating the descriptor's contents. */ upload_service_descriptor(service); @@ -2063,10 +1957,9 @@ rend_service_dump_stats(int severity) service->directory); for (j=0; j < smartlist_len(service->intro_nodes); ++j) { intro = smartlist_get(service->intro_nodes, j); - safe_name = safe_str(intro->extend_info->nickname); + safe_name = safe_str_client(intro->extend_info->nickname); - circ = find_intro_circuit(intro, service->pk_digest, - service->descriptor_version); + circ = find_intro_circuit(intro, service->pk_digest); if (!circ) { log(severity, LD_GENERAL, " Intro point %d at %s: no circuit", j, safe_name); @@ -2097,9 +1990,8 @@ rend_service_set_connection_addr_port(edge_connection_t *conn, log_debug(LD_REND,"beginning to hunt for addr/port"); base32_encode(serviceid, REND_SERVICE_ID_LEN_BASE32+1, circ->rend_data->rend_pk_digest, REND_SERVICE_ID_LEN); - service = rend_service_get_by_pk_digest_and_version( - circ->rend_data->rend_pk_digest, - circ->rend_data->rend_desc_version); + service = rend_service_get_by_pk_digest( + circ->rend_data->rend_pk_digest); if (!service) { log_warn(LD_REND, "Couldn't find any service associated with pk %s on " "rendezvous circuit %d; closing.", diff --git a/src/or/rephist.c b/src/or/rephist.c index 13fdb58b5e..e606db3b7b 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -14,7 +14,6 @@ static void bw_arrays_init(void); static void predicted_ports_init(void); -static void hs_usage_init(void); /** Total number of bytes currently allocated in fields used by rephist.c. */ uint64_t rephist_total_alloc=0; @@ -185,7 +184,6 @@ rep_hist_init(void) history_map = digestmap_new(); bw_arrays_init(); predicted_ports_init(); - hs_usage_init(); } /** Helper: note that we are no longer connected to the router with history @@ -1320,6 +1318,202 @@ rep_hist_note_bytes_read(size_t num_bytes, time_t when) add_obs(read_array, when, num_bytes); } +/* Some constants */ +/** To what multiple should byte numbers be rounded up? */ +#define EXIT_STATS_ROUND_UP_BYTES 1024 +/** To what multiple should stream counts be rounded up? */ +#define EXIT_STATS_ROUND_UP_STREAMS 4 +/** Number of TCP ports */ +#define EXIT_STATS_NUM_PORTS 65536 +/** Reciprocal of threshold (= 0.01%) of total bytes that a port needs to + * see in order to be included in exit stats. */ +#define EXIT_STATS_THRESHOLD_RECIPROCAL 10000 + +/* The following data structures are arrays and no fancy smartlists or maps, + * so that all write operations can be done in constant time. This comes at + * the price of some memory (1.25 MB) and linear complexity when writing + * stats for measuring relays. */ +/** Number of bytes read in current period by exit port */ +static uint64_t *exit_bytes_read = NULL; +/** Number of bytes written in current period by exit port */ +static uint64_t *exit_bytes_written = NULL; +/** Number of streams opened in current period by exit port */ +static uint32_t *exit_streams = NULL; + +/** When does the current exit stats period end? */ +static time_t start_of_exit_stats_interval; + +/** Initialize exit port stats. */ +void +rep_hist_exit_stats_init(time_t now) +{ + start_of_exit_stats_interval = now; + exit_bytes_read = tor_malloc_zero(EXIT_STATS_NUM_PORTS * + sizeof(uint64_t)); + exit_bytes_written = tor_malloc_zero(EXIT_STATS_NUM_PORTS * + sizeof(uint64_t)); + exit_streams = tor_malloc_zero(EXIT_STATS_NUM_PORTS * + sizeof(uint32_t)); +} + +/** Write exit stats to $DATADIR/stats/exit-stats and reset counters. */ +void +rep_hist_exit_stats_write(time_t now) +{ + char t[ISO_TIME_LEN+1]; + int r, i, comma; + uint64_t *b, total_bytes, threshold_bytes, other_bytes; + uint32_t other_streams; + + char *statsdir = NULL, *filename = NULL; + open_file_t *open_file = NULL; + FILE *out = NULL; + + if (!exit_streams) + return; /* Not initialized */ + + statsdir = get_datadir_fname("stats"); + if (check_private_dir(statsdir, CPD_CREATE) < 0) + goto done; + filename = get_datadir_fname2("stats", "exit-stats"); + format_iso_time(t, now); + log_info(LD_HIST, "Writing exit port statistics to disk for period " + "ending at %s.", t); + + if (!open_file) { + out = start_writing_to_stdio_file(filename, OPEN_FLAGS_APPEND, + 0600, &open_file); + if (!out) { + log_warn(LD_HIST, "Couldn't open '%s'.", filename); + goto done; + } + } + + /* written yyyy-mm-dd HH:MM:SS (n s) */ + if (fprintf(out, "exit-stats-end %s (%d s)\n", t, + (unsigned) (now - start_of_exit_stats_interval)) < 0) + goto done; + + /* Count the total number of bytes, so that we can attribute all + * observations below a threshold of 1 / EXIT_STATS_THRESHOLD_RECIPROCAL + * of all bytes to a special port 'other'. */ + total_bytes = 0; + for (i = 1; i < EXIT_STATS_NUM_PORTS; i++) { + total_bytes += exit_bytes_read[i]; + total_bytes += exit_bytes_written[i]; + } + threshold_bytes = total_bytes / EXIT_STATS_THRESHOLD_RECIPROCAL; + + /* exit-kibibytes-(read|written) port=kibibytes,.. */ + for (r = 0; r < 2; r++) { + b = r ? exit_bytes_read : exit_bytes_written; + tor_assert(b); + if (fprintf(out, "%s ", + r ? "exit-kibibytes-read" + : "exit-kibibytes-written") < 0) + goto done; + + comma = 0; + other_bytes = 0; + for (i = 1; i < EXIT_STATS_NUM_PORTS; i++) { + if (b[i] > 0) { + if (exit_bytes_read[i] + exit_bytes_written[i] > threshold_bytes) { + uint64_t num = round_uint64_to_next_multiple_of(b[i], + EXIT_STATS_ROUND_UP_BYTES); + num /= 1024; + if (fprintf(out, "%s%d="U64_FORMAT, + comma++ ? "," : "", i, + U64_PRINTF_ARG(num)) < 0) + goto done; + } else + other_bytes += b[i]; + } + } + other_bytes = round_uint64_to_next_multiple_of(other_bytes, + EXIT_STATS_ROUND_UP_BYTES); + other_bytes /= 1024; + if (fprintf(out, "%sother="U64_FORMAT"\n", + comma ? "," : "", U64_PRINTF_ARG(other_bytes))<0) + goto done; + } + /* exit-streams-opened port=num,.. */ + if (fprintf(out, "exit-streams-opened ") < 0) + goto done; + comma = 0; + other_streams = 0; + for (i = 1; i < EXIT_STATS_NUM_PORTS; i++) { + if (exit_streams[i] > 0) { + if (exit_bytes_read[i] + exit_bytes_written[i] > threshold_bytes) { + uint32_t num = round_uint32_to_next_multiple_of(exit_streams[i], + EXIT_STATS_ROUND_UP_STREAMS); + if (fprintf(out, "%s%d=%u", + comma++ ? "," : "", i, num)<0) + goto done; + } else + other_streams += exit_streams[i]; + } + } + other_streams = round_uint32_to_next_multiple_of(other_streams, + EXIT_STATS_ROUND_UP_STREAMS); + if (fprintf(out, "%sother=%u\n", + comma ? "," : "", other_streams)<0) + goto done; + /* Reset counters */ + memset(exit_bytes_read, 0, EXIT_STATS_NUM_PORTS * sizeof(uint64_t)); + memset(exit_bytes_written, 0, EXIT_STATS_NUM_PORTS * sizeof(uint64_t)); + memset(exit_streams, 0, EXIT_STATS_NUM_PORTS * sizeof(uint32_t)); + start_of_exit_stats_interval = now; + + if (open_file) + finish_writing_to_file(open_file); + open_file = NULL; + done: + if (open_file) + abort_writing_to_file(open_file); + tor_free(filename); + tor_free(statsdir); +} + +/** Note that we wrote <b>num_bytes</b> to an exit connection to + * <b>port</b>. */ +void +rep_hist_note_exit_bytes_written(uint16_t port, size_t num_bytes) +{ + if (!get_options()->ExitPortStatistics) + return; + if (!exit_bytes_written) + return; /* Not initialized */ + exit_bytes_written[port] += num_bytes; + log_debug(LD_HIST, "Written %lu bytes to exit connection to port %d.", + (unsigned long)num_bytes, port); +} + +/** Note that we read <b>num_bytes</b> from an exit connection to + * <b>port</b>. */ +void +rep_hist_note_exit_bytes_read(uint16_t port, size_t num_bytes) +{ + if (!get_options()->ExitPortStatistics) + return; + if (!exit_bytes_read) + return; /* Not initialized */ + exit_bytes_read[port] += num_bytes; + log_debug(LD_HIST, "Read %lu bytes from exit connection to port %d.", + (unsigned long)num_bytes, port); +} + +/** Note that we opened an exit stream to <b>port</b>. */ +void +rep_hist_note_exit_stream_opened(uint16_t port) +{ + if (!get_options()->ExitPortStatistics) + return; + if (!exit_streams) + return; /* Not initialized */ + exit_streams[port]++; + log_debug(LD_HIST, "Opened exit stream to port %d", port); +} + /** Helper: Return the largest value in b->maxima. (This is equal to the * most bandwidth used in any NUM_SECS_ROLLING_MEASURE period for the last * NUM_SECS_BW_SUM_IS_VALID seconds.) @@ -1419,7 +1613,7 @@ rep_hist_get_bandwidth_lines(int for_extrainfo) size_t len; /* opt (read|write)-history yyyy-mm-dd HH:MM:SS (n s) n,n,n,n,n... */ - len = (60+20*NUM_TOTALS)*2; + len = (60+21*NUM_TOTALS)*2; buf = tor_malloc_zero(len); cp = buf; for (r=0;r<2;++r) { @@ -1854,555 +2048,197 @@ rep_hist_free_all(void) tor_free(read_array); tor_free(write_array); tor_free(last_stability_doc); + tor_free(exit_bytes_read); + tor_free(exit_bytes_written); + tor_free(exit_streams); built_last_stability_doc_at = 0; predicted_ports_free(); } -/****************** hidden service usage statistics ******************/ - -/** How large are the intervals for which we track and report hidden service - * use? */ -#define NUM_SECS_HS_USAGE_SUM_INTERVAL (15*60) -/** How far in the past do we remember and publish hidden service use? */ -#define NUM_SECS_HS_USAGE_SUM_IS_VALID (24*60*60) -/** How many hidden service usage intervals do we remember? (derived) */ -#define NUM_TOTALS_HS_USAGE (NUM_SECS_HS_USAGE_SUM_IS_VALID/ \ - NUM_SECS_HS_USAGE_SUM_INTERVAL) - -/** List element containing a service id and the count. */ -typedef struct hs_usage_list_elem_t { - /** Service id of this elem. */ - char service_id[REND_SERVICE_ID_LEN_BASE32+1]; - /** Number of occurrences for the given service id. */ - uint32_t count; - /* Pointer to next list elem */ - struct hs_usage_list_elem_t *next; -} hs_usage_list_elem_t; - -/** Ordered list that stores service ids and the number of observations. It is - * ordered by the number of occurrences in descending order. Its purpose is to - * calculate the frequency distribution when the period is over. */ -typedef struct hs_usage_list_t { - /* Pointer to the first element in the list. */ - hs_usage_list_elem_t *start; - /* Number of total occurrences for all list elements. */ - uint32_t total_count; - /* Number of service ids, i.e. number of list elements. */ - uint32_t total_service_ids; -} hs_usage_list_t; - -/** Tracks service-related observations in the current period and their - * history. */ -typedef struct hs_usage_service_related_observation_t { - /** Ordered list that stores service ids and the number of observations in - * the current period. It is ordered by the number of occurrences in - * descending order. Its purpose is to calculate the frequency distribution - * when the period is over. */ - hs_usage_list_t *list; - /** Circular arrays that store the history of observations. totals stores all - * observations, twenty (ten, five) the number of observations related to a - * service id being accounted for the top 20 (10, 5) percent of all - * observations. */ - uint32_t totals[NUM_TOTALS_HS_USAGE]; - uint32_t five[NUM_TOTALS_HS_USAGE]; - uint32_t ten[NUM_TOTALS_HS_USAGE]; - uint32_t twenty[NUM_TOTALS_HS_USAGE]; -} hs_usage_service_related_observation_t; - -/** Tracks the history of general period-related observations, i.e. those that - * cannot be related to a specific service id. */ -typedef struct hs_usage_general_period_related_observations_t { - /** Circular array that stores the history of observations. */ - uint32_t totals[NUM_TOTALS_HS_USAGE]; -} hs_usage_general_period_related_observations_t; - -/** Keeps information about the current observation period and its relation to - * the histories of observations. */ -typedef struct hs_usage_current_observation_period_t { - /** Where do we write the next history entry? */ - int next_idx; - /** How many values in history have been set ever? (upper bound!) */ - int num_set; - /** When did this period begin? */ - time_t start_of_current_period; - /** When does the next period begin? */ - time_t start_of_next_period; -} hs_usage_current_observation_period_t; - -/** Usage statistics for the current observation period. */ -static hs_usage_current_observation_period_t *current_period = NULL; - -/** Total number of descriptor publish requests in the current observation - * period. */ -static hs_usage_service_related_observation_t *publish_total = NULL; - -/** Number of descriptor publish requests for services that have not been - * seen before in the current observation period. */ -static hs_usage_service_related_observation_t *publish_novel = NULL; - -/** Total number of descriptor fetch requests in the current observation - * period. */ -static hs_usage_service_related_observation_t *fetch_total = NULL; - -/** Number of successful descriptor fetch requests in the current - * observation period. */ -static hs_usage_service_related_observation_t *fetch_successful = NULL; - -/** Number of descriptors stored in the current observation period. */ -static hs_usage_general_period_related_observations_t *descs = NULL; - -/** Creates an empty ordered list element. */ -static hs_usage_list_elem_t * -hs_usage_list_elem_new(void) -{ - hs_usage_list_elem_t *e; - e = tor_malloc_zero(sizeof(hs_usage_list_elem_t)); - rephist_total_alloc += sizeof(hs_usage_list_elem_t); - e->count = 1; - e->next = NULL; - return e; -} - -/** Creates an empty ordered list. */ -static hs_usage_list_t * -hs_usage_list_new(void) -{ - hs_usage_list_t *l; - l = tor_malloc_zero(sizeof(hs_usage_list_t)); - rephist_total_alloc += sizeof(hs_usage_list_t); - l->start = NULL; - l->total_count = 0; - l->total_service_ids = 0; - return l; -} +/*** cell statistics ***/ -/** Creates an empty structure for storing service-related observations. */ -static hs_usage_service_related_observation_t * -hs_usage_service_related_observation_new(void) -{ - hs_usage_service_related_observation_t *h; - h = tor_malloc_zero(sizeof(hs_usage_service_related_observation_t)); - rephist_total_alloc += sizeof(hs_usage_service_related_observation_t); - h->list = hs_usage_list_new(); - return h; -} +/** Start of the current buffer stats interval. */ +static time_t start_of_buffer_stats_interval; -/** Creates an empty structure for storing general period-related - * observations. */ -static hs_usage_general_period_related_observations_t * -hs_usage_general_period_related_observations_new(void) +/** Initialize buffer stats. */ +void +rep_hist_buffer_stats_init(time_t now) { - hs_usage_general_period_related_observations_t *p; - p = tor_malloc_zero(sizeof(hs_usage_general_period_related_observations_t)); - rephist_total_alloc+= sizeof(hs_usage_general_period_related_observations_t); - return p; + start_of_buffer_stats_interval = now; } -/** Creates an empty structure for storing period-specific information. */ -static hs_usage_current_observation_period_t * -hs_usage_current_observation_period_new(void) -{ - hs_usage_current_observation_period_t *c; - time_t now; - c = tor_malloc_zero(sizeof(hs_usage_current_observation_period_t)); - rephist_total_alloc += sizeof(hs_usage_current_observation_period_t); - now = time(NULL); - c->start_of_current_period = now; - c->start_of_next_period = now + NUM_SECS_HS_USAGE_SUM_INTERVAL; - return c; -} +typedef struct circ_buffer_stats_t { + uint32_t processed_cells; + double mean_num_cells_in_queue; + double mean_time_cells_in_queue; + uint32_t local_circ_id; +} circ_buffer_stats_t; -/** Initializes the structures for collecting hidden service usage data. */ -static void -hs_usage_init(void) -{ - current_period = hs_usage_current_observation_period_new(); - publish_total = hs_usage_service_related_observation_new(); - publish_novel = hs_usage_service_related_observation_new(); - fetch_total = hs_usage_service_related_observation_new(); - fetch_successful = hs_usage_service_related_observation_new(); - descs = hs_usage_general_period_related_observations_new(); -} +/** Holds stats. */ +smartlist_t *circuits_for_buffer_stats = NULL; -/** Clears the given ordered list by resetting its attributes and releasing - * the memory allocated by its elements. */ -static void -hs_usage_list_clear(hs_usage_list_t *lst) -{ - /* walk through elements and free memory */ - hs_usage_list_elem_t *current = lst->start; - hs_usage_list_elem_t *tmp; - while (current != NULL) { - tmp = current->next; - rephist_total_alloc -= sizeof(hs_usage_list_elem_t); - tor_free(current); - current = tmp; - } - /* reset attributes */ - lst->start = NULL; - lst->total_count = 0; - lst->total_service_ids = 0; - return; -} - -/** Frees the memory used by the given list. */ -static void -hs_usage_list_free(hs_usage_list_t *lst) +/** Remember cell statistics for circuit <b>circ</b> at time + * <b>end_of_interval</b> and reset cell counters in case the circuit + * remains open in the next measurement interval. */ +void +rep_hist_buffer_stats_add_circ(circuit_t *circ, time_t end_of_interval) { - if (!lst) + circ_buffer_stats_t *stat; + time_t start_of_interval; + int interval_length; + or_circuit_t *orcirc; + if (CIRCUIT_IS_ORIGIN(circ)) return; - hs_usage_list_clear(lst); - rephist_total_alloc -= sizeof(hs_usage_list_t); - tor_free(lst); -} - -/** Frees the memory used by the given service-related observations. */ -static void -hs_usage_service_related_observation_free( - hs_usage_service_related_observation_t *s) -{ - if (!s) + orcirc = TO_OR_CIRCUIT(circ); + if (!orcirc->processed_cells) return; - hs_usage_list_free(s->list); - rephist_total_alloc -= sizeof(hs_usage_service_related_observation_t); - tor_free(s); -} - -/** Frees the memory used by the given period-specific observations. */ -static void -hs_usage_general_period_related_observations_free( - hs_usage_general_period_related_observations_t *s) -{ - rephist_total_alloc-=sizeof(hs_usage_general_period_related_observations_t); - tor_free(s); -} - -/** Frees the memory used by period-specific information. */ -static void -hs_usage_current_observation_period_free( - hs_usage_current_observation_period_t *s) + if (!circuits_for_buffer_stats) + circuits_for_buffer_stats = smartlist_create(); + start_of_interval = circ->timestamp_created > + start_of_buffer_stats_interval ? + circ->timestamp_created : + start_of_buffer_stats_interval; + interval_length = (int) (end_of_interval - start_of_interval); + stat = tor_malloc_zero(sizeof(circ_buffer_stats_t)); + stat->processed_cells = orcirc->processed_cells; + /* 1000.0 for s -> ms; 2.0 because of app-ward and exit-ward queues */ + stat->mean_num_cells_in_queue = interval_length == 0 ? 0.0 : + (double) orcirc->total_cell_waiting_time / + (double) interval_length / 1000.0 / 2.0; + stat->mean_time_cells_in_queue = + (double) orcirc->total_cell_waiting_time / + (double) orcirc->processed_cells; + smartlist_add(circuits_for_buffer_stats, stat); + orcirc->total_cell_waiting_time = 0; + orcirc->processed_cells = 0; +} + +/** Sorting helper: return -1, 1, or 0 based on comparison of two + * circ_buffer_stats_t */ +static int +_buffer_stats_compare_entries(const void **_a, const void **_b) { - rephist_total_alloc -= sizeof(hs_usage_current_observation_period_t); - tor_free(s); + const circ_buffer_stats_t *a = *_a, *b = *_b; + if (a->processed_cells < b->processed_cells) + return 1; + else if (a->processed_cells > b->processed_cells) + return -1; + else + return 0; } -/** Frees all memory that was used for collecting hidden service usage data. */ +/** Write buffer statistics to $DATADIR/stats/buffer-stats. */ void -hs_usage_free_all(void) +rep_hist_buffer_stats_write(time_t now) { - hs_usage_general_period_related_observations_free(descs); - descs = NULL; - hs_usage_service_related_observation_free(fetch_successful); - hs_usage_service_related_observation_free(fetch_total); - hs_usage_service_related_observation_free(publish_novel); - hs_usage_service_related_observation_free(publish_total); - fetch_successful = fetch_total = publish_novel = publish_total = NULL; - hs_usage_current_observation_period_free(current_period); - current_period = NULL; -} - -/** Inserts a new occurrence for the given service id to the given ordered - * list. */ -static void -hs_usage_insert_value(hs_usage_list_t *lst, const char *service_id) -{ - /* search if there is already an elem with same service_id in list */ - hs_usage_list_elem_t *current = lst->start; - hs_usage_list_elem_t *previous = NULL; - while (current != NULL && strcasecmp(current->service_id,service_id)) { - previous = current; - current = current->next; - } - /* found an element with same service_id? */ - if (current == NULL) { - /* not found! append to end (which could also be the end of a zero-length - * list), don't need to sort (1 is smallest value). */ - /* create elem */ - hs_usage_list_elem_t *e = hs_usage_list_elem_new(); - /* update list attributes (one new elem, one new occurrence) */ - lst->total_count++; - lst->total_service_ids++; - /* copy service id to elem */ - strlcpy(e->service_id,service_id,sizeof(e->service_id)); - /* let either l->start or previously last elem point to new elem */ - if (lst->start == NULL) { - /* this is the first elem */ - lst->start = e; - } else { - /* there were elems in the list before */ - previous->next = e; - } - } else { - /* found! add occurrence to elem and consider resorting */ - /* update list attributes (no new elem, but one new occurrence) */ - lst->total_count++; - /* add occurrence to elem */ - current->count++; - /* is it another than the first list elem? and has previous elem fewer - * count than current? then we need to resort */ - if (previous != NULL && previous->count < current->count) { - /* yes! we need to resort */ - /* remove current elem first */ - previous->next = current->next; - /* can we prepend elem to all other elements? */ - if (lst->start->count <= current->count) { - /* yes! prepend elem */ - current->next = lst->start; - lst->start = current; - } else { - /* no! walk through list a second time and insert at correct place */ - hs_usage_list_elem_t *insert_current = lst->start->next; - hs_usage_list_elem_t *insert_previous = lst->start; - while (insert_current != NULL && - insert_current->count > current->count) { - insert_previous = insert_current; - insert_current = insert_current->next; - } - /* insert here */ - current->next = insert_current; - insert_previous->next = current; - } - } + char *statsdir = NULL, *filename = NULL; + char written[ISO_TIME_LEN+1]; + open_file_t *open_file = NULL; + FILE *out; +#define SHARES 10 + int processed_cells[SHARES], circs_in_share[SHARES], + number_of_circuits, i; + double queued_cells[SHARES], time_in_queue[SHARES]; + smartlist_t *str_build = smartlist_create(); + char *str = NULL; + char buf[32]; + circuit_t *circ; + /* add current circuits to stats */ + for (circ = _circuit_get_global_list(); circ; circ = circ->next) + rep_hist_buffer_stats_add_circ(circ, now); + /* calculate deciles */ + memset(processed_cells, 0, SHARES * sizeof(int)); + memset(circs_in_share, 0, SHARES * sizeof(int)); + memset(queued_cells, 0, SHARES * sizeof(double)); + memset(time_in_queue, 0, SHARES * sizeof(double)); + if (!circuits_for_buffer_stats) + circuits_for_buffer_stats = smartlist_create(); + smartlist_sort(circuits_for_buffer_stats, + _buffer_stats_compare_entries); + number_of_circuits = smartlist_len(circuits_for_buffer_stats); + if (number_of_circuits < 1) { + log_info(LD_HIST, "Attempt to write cell statistics to disk failed. " + "We haven't seen a single circuit to report about."); + goto done; } -} - -/** Writes the current service-related observations to the history array and - * clears the observations of the current period. */ -static void -hs_usage_write_service_related_observations_to_history( - hs_usage_current_observation_period_t *p, - hs_usage_service_related_observation_t *h) -{ - /* walk through the first 20 % of list elements and calculate frequency - * distributions */ - /* maximum indices for the three frequencies */ - int five_percent_idx = h->list->total_service_ids/20; - int ten_percent_idx = h->list->total_service_ids/10; - int twenty_percent_idx = h->list->total_service_ids/5; - /* temp values */ - uint32_t five_percent = 0; - uint32_t ten_percent = 0; - uint32_t twenty_percent = 0; - /* walk through list */ - hs_usage_list_elem_t *current = h->list->start; - int i=0; - while (current != NULL && i <= twenty_percent_idx) { - twenty_percent += current->count; - if (i <= ten_percent_idx) - ten_percent += current->count; - if (i <= five_percent_idx) - five_percent += current->count; - current = current->next; - i++; + i = 0; + SMARTLIST_FOREACH_BEGIN(circuits_for_buffer_stats, + circ_buffer_stats_t *, stat) + { + int share = i++ * SHARES / number_of_circuits; + processed_cells[share] += stat->processed_cells; + queued_cells[share] += stat->mean_num_cells_in_queue; + time_in_queue[share] += stat->mean_time_cells_in_queue; + circs_in_share[share]++; } - /* copy frequencies */ - h->twenty[p->next_idx] = twenty_percent; - h->ten[p->next_idx] = ten_percent; - h->five[p->next_idx] = five_percent; - /* copy total number of observations */ - h->totals[p->next_idx] = h->list->total_count; - /* free memory of old list */ - hs_usage_list_clear(h->list); -} - -/** Advances to next observation period. */ -static void -hs_usage_advance_current_observation_period(void) -{ - /* aggregate observations to history, including frequency distribution - * arrays */ - hs_usage_write_service_related_observations_to_history( - current_period, publish_total); - hs_usage_write_service_related_observations_to_history( - current_period, publish_novel); - hs_usage_write_service_related_observations_to_history( - current_period, fetch_total); - hs_usage_write_service_related_observations_to_history( - current_period, fetch_successful); - /* write current number of descriptors to descs history */ - descs->totals[current_period->next_idx] = rend_cache_size(); - /* advance to next period */ - current_period->next_idx++; - if (current_period->next_idx == NUM_TOTALS_HS_USAGE) - current_period->next_idx = 0; - if (current_period->num_set < NUM_TOTALS_HS_USAGE) - ++current_period->num_set; - current_period->start_of_current_period=current_period->start_of_next_period; - current_period->start_of_next_period += NUM_SECS_HS_USAGE_SUM_INTERVAL; -} - -/** Checks if the current period is up to date, and if not, advances it. */ -static void -hs_usage_check_if_current_period_is_up_to_date(time_t now) -{ - while (now > current_period->start_of_next_period) { - hs_usage_advance_current_observation_period(); + SMARTLIST_FOREACH_END(stat); + /* clear buffer stats history */ + SMARTLIST_FOREACH(circuits_for_buffer_stats, circ_buffer_stats_t *, + stat, tor_free(stat)); + smartlist_clear(circuits_for_buffer_stats); + /* write to file */ + statsdir = get_datadir_fname("stats"); + if (check_private_dir(statsdir, CPD_CREATE) < 0) + goto done; + filename = get_datadir_fname2("stats", "buffer-stats"); + out = start_writing_to_stdio_file(filename, OPEN_FLAGS_APPEND, + 0600, &open_file); + if (!out) + goto done; + format_iso_time(written, now); + if (fprintf(out, "cell-stats-end %s (%d s)\n", written, + (unsigned) (now - start_of_buffer_stats_interval)) < 0) + goto done; + for (i = 0; i < SHARES; i++) { + tor_snprintf(buf, sizeof(buf), "%d", !circs_in_share[i] ? 0 : + processed_cells[i] / circs_in_share[i]); + smartlist_add(str_build, tor_strdup(buf)); } -} - -/** Adds a service-related observation, maybe after advancing to next - * observation period. */ -static void -hs_usage_add_service_related_observation( - hs_usage_service_related_observation_t *h, - time_t now, - const char *service_id) -{ - if (now < current_period->start_of_current_period) { - /* don't record old data */ - return; - } - /* check if we are up-to-date */ - hs_usage_check_if_current_period_is_up_to_date(now); - /* add observation */ - hs_usage_insert_value(h->list, service_id); -} - -/** Adds the observation of storing a rendezvous service descriptor to our - * cache in our role as HS authoritative directory. */ -void -hs_usage_note_publish_total(const char *service_id, time_t now) -{ - hs_usage_add_service_related_observation(publish_total, now, service_id); -} - -/** Adds the observation of storing a novel rendezvous service descriptor to - * our cache in our role as HS authoritative directory. */ -void -hs_usage_note_publish_novel(const char *service_id, time_t now) -{ - hs_usage_add_service_related_observation(publish_novel, now, service_id); -} - -/** Adds the observation of being requested for a rendezvous service descriptor - * in our role as HS authoritative directory. */ -void -hs_usage_note_fetch_total(const char *service_id, time_t now) -{ - hs_usage_add_service_related_observation(fetch_total, now, service_id); -} - -/** Adds the observation of being requested for a rendezvous service descriptor - * in our role as HS authoritative directory and being able to answer that - * request successfully. */ -void -hs_usage_note_fetch_successful(const char *service_id, time_t now) -{ - hs_usage_add_service_related_observation(fetch_successful, now, service_id); -} - -/** Writes the given circular array to a string. */ -static size_t -hs_usage_format_history(char *buf, size_t len, uint32_t *data) -{ - char *cp = buf; /* pointer where we are in the buffer */ - int i, n; - if (current_period->num_set <= current_period->next_idx) { - i = 0; /* not been through circular array */ - } else { - i = current_period->next_idx; + str = smartlist_join_strings(str_build, ",", 0, NULL); + if (fprintf(out, "cell-processed-cells %s\n", str) < 0) + goto done; + tor_free(str); + SMARTLIST_FOREACH(str_build, char *, c, tor_free(c)); + smartlist_clear(str_build); + for (i = 0; i < SHARES; i++) { + tor_snprintf(buf, sizeof(buf), "%.2f", circs_in_share[i] == 0 ? 0.0 : + queued_cells[i] / (double) circs_in_share[i]); + smartlist_add(str_build, tor_strdup(buf)); } - for (n = 0; n < current_period->num_set; ++n,++i) { - if (i >= NUM_TOTALS_HS_USAGE) - i -= NUM_TOTALS_HS_USAGE; - tor_assert(i < NUM_TOTALS_HS_USAGE); - if (n == (current_period->num_set-1)) - tor_snprintf(cp, len-(cp-buf), "%d", data[i]); - else - tor_snprintf(cp, len-(cp-buf), "%d,", data[i]); - cp += strlen(cp); + str = smartlist_join_strings(str_build, ",", 0, NULL); + if (fprintf(out, "cell-queued-cells %s\n", str) < 0) + goto done; + tor_free(str); + SMARTLIST_FOREACH(str_build, char *, c, tor_free(c)); + smartlist_clear(str_build); + for (i = 0; i < SHARES; i++) { + tor_snprintf(buf, sizeof(buf), "%.0f", circs_in_share[i] == 0 ? 0.0 : + time_in_queue[i] / (double) circs_in_share[i]); + smartlist_add(str_build, tor_strdup(buf)); } - return cp-buf; -} - -/** Writes the complete usage history as hidden service authoritative directory - * to a string. */ -static char * -hs_usage_format_statistics(void) -{ - char *buf, *cp, *s = NULL; - char t[ISO_TIME_LEN+1]; - int r; - uint32_t *data = NULL; - size_t len; - len = (70+20*NUM_TOTALS_HS_USAGE)*11; - buf = tor_malloc_zero(len); - cp = buf; - for (r = 0; r < 11; ++r) { - switch (r) { - case 0: - s = (char*) "publish-total-history"; - data = publish_total->totals; - break; - case 1: - s = (char*) "publish-novel-history"; - data = publish_novel->totals; - break; - case 2: - s = (char*) "publish-top-5-percent-history"; - data = publish_total->five; - break; - case 3: - s = (char*) "publish-top-10-percent-history"; - data = publish_total->ten; - break; - case 4: - s = (char*) "publish-top-20-percent-history"; - data = publish_total->twenty; - break; - case 5: - s = (char*) "fetch-total-history"; - data = fetch_total->totals; - break; - case 6: - s = (char*) "fetch-successful-history"; - data = fetch_successful->totals; - break; - case 7: - s = (char*) "fetch-top-5-percent-history"; - data = fetch_total->five; - break; - case 8: - s = (char*) "fetch-top-10-percent-history"; - data = fetch_total->ten; - break; - case 9: - s = (char*) "fetch-top-20-percent-history"; - data = fetch_total->twenty; - break; - case 10: - s = (char*) "desc-total-history"; - data = descs->totals; - break; - } - format_iso_time(t, current_period->start_of_current_period); - tor_snprintf(cp, len-(cp-buf), "%s %s (%d s) ", s, t, - NUM_SECS_HS_USAGE_SUM_INTERVAL); - cp += strlen(cp); - cp += hs_usage_format_history(cp, len-(cp-buf), data); - strlcat(cp, "\n", len-(cp-buf)); - ++cp; + str = smartlist_join_strings(str_build, ",", 0, NULL); + if (fprintf(out, "cell-time-in-queue %s\n", str) < 0) + goto done; + tor_free(str); + SMARTLIST_FOREACH(str_build, char *, c, tor_free(c)); + smartlist_free(str_build); + str_build = NULL; + if (fprintf(out, "cell-circuits-per-decile %d\n", + (number_of_circuits + SHARES - 1) / SHARES) < 0) + goto done; + finish_writing_to_file(open_file); + open_file = NULL; + start_of_buffer_stats_interval = now; + done: + if (open_file) + abort_writing_to_file(open_file); + tor_free(filename); + tor_free(statsdir); + if (str_build) { + SMARTLIST_FOREACH(str_build, char *, c, tor_free(c)); + smartlist_free(str_build); } - return buf; -} - -/** Write current statistics about hidden service usage to file. */ -void -hs_usage_write_statistics_to_file(time_t now) -{ - char *buf; - size_t len; - char *fname; - or_options_t *options = get_options(); - /* check if we are up-to-date */ - hs_usage_check_if_current_period_is_up_to_date(now); - buf = hs_usage_format_statistics(); - len = strlen(options->DataDirectory) + 16; - fname = tor_malloc(len); - tor_snprintf(fname, len, "%s"PATH_SEPARATOR"hsusage", - options->DataDirectory); - write_str_to_file(fname,buf,0); - tor_free(buf); - tor_free(fname); + tor_free(str); +#undef SHARES } diff --git a/src/or/router.c b/src/or/router.c index 97f411dcd6..827df0302c 100644 --- a/src/or/router.c +++ b/src/or/router.c @@ -61,8 +61,7 @@ static void set_onion_key(crypto_pk_env_t *k) { tor_mutex_acquire(key_lock); - if (onionkey) - crypto_free_pk_env(onionkey); + crypto_free_pk_env(onionkey); onionkey = k; onionkey_set_at = time(NULL); tor_mutex_release(key_lock); @@ -111,8 +110,7 @@ get_onion_key_set_at(void) void set_identity_key(crypto_pk_env_t *k) { - if (identitykey) - crypto_free_pk_env(identitykey); + crypto_free_pk_env(identitykey); identitykey = k; crypto_pk_get_digest(identitykey, identitykey_digest); } @@ -201,8 +199,7 @@ rotate_onion_key(void) } log_info(LD_GENERAL, "Rotating onion key"); tor_mutex_acquire(key_lock); - if (lastonionkey) - crypto_free_pk_env(lastonionkey); + crypto_free_pk_env(lastonionkey); lastonionkey = onionkey; onionkey = prkey; now = time(NULL); @@ -331,10 +328,9 @@ load_authority_keyset(int legacy, crypto_pk_env_t **key_out, goto done; } - if (*key_out) - crypto_free_pk_env(*key_out); - if (*cert_out) - authority_cert_free(*cert_out); + crypto_free_pk_env(*key_out); + authority_cert_free(*cert_out); + *key_out = signing_key; *cert_out = parsed; r = 0; @@ -344,10 +340,8 @@ load_authority_keyset(int legacy, crypto_pk_env_t **key_out, done: tor_free(fname); tor_free(cert); - if (signing_key) - crypto_free_pk_env(signing_key); - if (parsed) - authority_cert_free(parsed); + crypto_free_pk_env(signing_key); + authority_cert_free(parsed); return r; } @@ -442,7 +436,9 @@ init_keys(void) key_lock = tor_mutex_new(); /* There are a couple of paths that put us here before */ - if (crypto_global_init(get_options()->HardwareAccel)) { + if (crypto_global_init(get_options()->HardwareAccel, + get_options()->AccelName, + get_options()->AccelDir)) { log_err(LD_BUG, "Unable to initialize OpenSSL. Exiting."); return -1; } @@ -1267,6 +1263,7 @@ router_rebuild_descriptor(int force) uint32_t addr; char platform[256]; int hibernating = we_are_hibernating(); + size_t ei_size; or_options_t *options = get_options(); if (desc_clean_since && !force) @@ -1309,7 +1306,7 @@ router_rebuild_descriptor(int force) policies_parse_exit_policy(options->ExitPolicy, &ri->exit_policy, options->ExitPolicyRejectPrivate, - ri->address); + ri->address, !options->BridgeRelay); if (desc_routerinfo) { /* inherit values */ ri->is_valid = desc_routerinfo->is_valid; @@ -1380,9 +1377,10 @@ router_rebuild_descriptor(int force) ei->cache_info.published_on = ri->cache_info.published_on; memcpy(ei->cache_info.identity_digest, ri->cache_info.identity_digest, DIGEST_LEN); - ei->cache_info.signed_descriptor_body = tor_malloc(8192); - if (extrainfo_dump_to_string(ei->cache_info.signed_descriptor_body, 8192, - ei, get_identity_key()) < 0) { + ei_size = options->ExtraInfoStatistics ? MAX_EXTRAINFO_UPLOAD_SIZE : 8192; + ei->cache_info.signed_descriptor_body = tor_malloc(ei_size); + if (extrainfo_dump_to_string(ei->cache_info.signed_descriptor_body, + ei_size, ei, get_identity_key()) < 0) { log_warn(LD_BUG, "Couldn't generate extra-info descriptor."); routerinfo_free(ri); extrainfo_free(ei); @@ -1424,11 +1422,9 @@ router_rebuild_descriptor(int force) tor_assert(! routerinfo_incompatible_with_extrainfo(ri, ei, NULL, NULL)); - if (desc_routerinfo) - routerinfo_free(desc_routerinfo); + routerinfo_free(desc_routerinfo); desc_routerinfo = ri; - if (desc_extrainfo) - extrainfo_free(desc_extrainfo); + extrainfo_free(desc_extrainfo); desc_extrainfo = ei; desc_clean_since = time(NULL); @@ -1609,8 +1605,6 @@ router_guess_address_from_dir_headers(uint32_t *guess) return -1; } -extern const char tor_svn_revision[]; /* from tor_main.c */ - /** Set <b>platform</b> (max length <b>len</b>) to a NUL-terminated short * string describing the version of Tor and the operating system we're * currently running on. @@ -1669,7 +1663,7 @@ router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router, return -1; } - /* PEM-encode the identity key key */ + /* PEM-encode the identity key */ if (crypto_pk_write_public_key_to_string(router->identity_pkey, &identity_pkey,&identity_pkeylen)<0) { log_warn(LD_BUG,"write identity_pkey to string failed!"); @@ -1790,7 +1784,7 @@ router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router, note_crypto_pk_op(SIGN_RTR); if (router_append_dirobj_signature(s+written,maxlen-written, - digest,ident_key)<0) { + digest,DIGEST_LEN,ident_key)<0) { log_warn(LD_BUG, "Couldn't sign router descriptor"); return -1; } @@ -1825,6 +1819,57 @@ router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router, return (int)written+1; } +/** Load the contents of <b>filename</b>, find the last line starting with + * <b>end_line</b>, ensure that its timestamp is not more than 25 hours in + * the past or more than 1 hour in the future with respect to <b>now</b>, + * and write the file contents starting with that line to *<b>out</b>. + * Return 1 for success, 0 if the file does not exist, or -1 if the file + * does not contain a line matching these criteria or other failure. */ +static int +load_stats_file(const char *filename, const char *end_line, time_t now, + char **out) +{ + int r = -1; + char *fname = get_datadir_fname(filename); + char *contents, *start = NULL, *tmp, timestr[ISO_TIME_LEN+1]; + time_t written; + switch (file_status(fname)) { + case FN_FILE: + /* X022 Find an alternative to reading the whole file to memory. */ + if ((contents = read_file_to_str(fname, 0, NULL))) { + tmp = strstr(contents, end_line); + /* Find last block starting with end_line */ + while (tmp) { + start = tmp; + tmp = strstr(tmp + 1, end_line); + } + if (!start) + goto notfound; + if (strlen(start) < strlen(end_line) + 1 + sizeof(timestr)) + goto notfound; + strlcpy(timestr, start + 1 + strlen(end_line), sizeof(timestr)); + if (parse_iso_time(timestr, &written) < 0) + goto notfound; + if (written < now - (25*60*60) || written > now + (1*60*60)) + goto notfound; + *out = tor_strdup(start); + r = 1; + } + notfound: + tor_free(contents); + break; + case FN_NOENT: + r = 0; + break; + case FN_ERROR: + case FN_DIR: + default: + break; + } + tor_free(fname); + return r; +} + /** Write the contents of <b>extrainfo</b> to the <b>maxlen</b>-byte string * <b>s</b>, signing them with <b>ident_key</b>. Return 0 on success, * negative on failure. */ @@ -1839,6 +1884,8 @@ extrainfo_dump_to_string(char *s, size_t maxlen, extrainfo_t *extrainfo, char *bandwidth_usage; int result; size_t len; + static int write_stats_to_extrainfo = 1; + time_t now = time(NULL); base16_encode(identity, sizeof(identity), extrainfo->cache_info.identity_digest, DIGEST_LEN); @@ -1850,23 +1897,75 @@ extrainfo_dump_to_string(char *s, size_t maxlen, extrainfo_t *extrainfo, "published %s\n%s", extrainfo->nickname, identity, published, bandwidth_usage); + + if (options->ExtraInfoStatistics && write_stats_to_extrainfo) { + char *contents = NULL; + log_info(LD_GENERAL, "Adding stats to extra-info descriptor."); + if (options->DirReqStatistics && + load_stats_file("stats"PATH_SEPARATOR"dirreq-stats", + "dirreq-stats-end", now, &contents) > 0) { + size_t pos = strlen(s); + if (strlcpy(s + pos, contents, maxlen - strlen(s)) != + strlen(contents)) { + log_warn(LD_DIR, "Could not write dirreq-stats to extra-info " + "descriptor."); + s[pos] = '\0'; + } + tor_free(contents); + } + if (options->EntryStatistics && + load_stats_file("stats"PATH_SEPARATOR"entry-stats", + "entry-stats-end", now, &contents) > 0) { + size_t pos = strlen(s); + if (strlcpy(s + pos, contents, maxlen - strlen(s)) != + strlen(contents)) { + log_warn(LD_DIR, "Could not write entry-stats to extra-info " + "descriptor."); + s[pos] = '\0'; + } + tor_free(contents); + } + if (options->CellStatistics && + load_stats_file("stats"PATH_SEPARATOR"buffer-stats", + "cell-stats-end", now, &contents) > 0) { + size_t pos = strlen(s); + if (strlcpy(s + pos, contents, maxlen - strlen(s)) != + strlen(contents)) { + log_warn(LD_DIR, "Could not write buffer-stats to extra-info " + "descriptor."); + s[pos] = '\0'; + } + tor_free(contents); + } + if (options->ExitPortStatistics && + load_stats_file("stats"PATH_SEPARATOR"exit-stats", + "exit-stats-end", now, &contents) > 0) { + size_t pos = strlen(s); + if (strlcpy(s + pos, contents, maxlen - strlen(s)) != + strlen(contents)) { + log_warn(LD_DIR, "Could not write exit-stats to extra-info " + "descriptor."); + s[pos] = '\0'; + } + tor_free(contents); + } + } + tor_free(bandwidth_usage); if (result<0) return -1; if (should_record_bridge_info(options)) { - char *geoip_summary = extrainfo_get_client_geoip_summary(time(NULL)); - if (geoip_summary) { - char geoip_start[ISO_TIME_LEN+1]; - format_iso_time(geoip_start, geoip_get_history_start()); - result = tor_snprintf(s+strlen(s), maxlen-strlen(s), - "geoip-start-time %s\n" - "geoip-client-origins %s\n", - geoip_start, geoip_summary); - control_event_clients_seen(geoip_start, geoip_summary); - tor_free(geoip_summary); - if (result<0) - return -1; + char *bridge_stats = geoip_get_bridge_stats_extrainfo(now); + if (bridge_stats) { + size_t pos = strlen(s); + if (strlcpy(s + pos, bridge_stats, maxlen - strlen(s)) != + strlen(bridge_stats)) { + log_warn(LD_DIR, "Could not write bridge-stats to extra-info " + "descriptor."); + s[pos] = '\0'; + } + tor_free(bridge_stats); } } @@ -1875,10 +1974,10 @@ extrainfo_dump_to_string(char *s, size_t maxlen, extrainfo_t *extrainfo, len += strlen(s+len); if (router_get_extrainfo_hash(s, digest)<0) return -1; - if (router_append_dirobj_signature(s+len, maxlen-len, digest, ident_key)<0) + if (router_append_dirobj_signature(s+len, maxlen-len, digest, DIGEST_LEN, + ident_key)<0) return -1; -#ifdef DEBUG_ROUTER_DUMP_ROUTER_TO_STRING { char *cp, *s_dup; extrainfo_t *ei_tmp; @@ -1894,31 +1993,26 @@ extrainfo_dump_to_string(char *s, size_t maxlen, extrainfo_t *extrainfo, tor_free(s_dup); extrainfo_free(ei_tmp); } -#endif - - return (int)strlen(s)+1; -} -/** Wrapper function for geoip_get_client_history(). It first discards - * any items in the client history that are too old -- it dumps anything - * more than 48 hours old, but it only considers whether to dump at most - * once per 48 hours, so we aren't too precise to an observer (see also - * r14780). - */ -char * -extrainfo_get_client_geoip_summary(time_t now) -{ - static time_t last_purged_at = 0; - int geoip_purge_interval = 48*60*60; -#ifdef ENABLE_GEOIP_STATS - if (get_options()->DirRecordUsageByCountry) - geoip_purge_interval = get_options()->DirRecordUsageRetainIPs; -#endif - if (now > last_purged_at+geoip_purge_interval) { - geoip_remove_old_clients(now-geoip_purge_interval); - last_purged_at = now; + if (options->ExtraInfoStatistics && write_stats_to_extrainfo) { + char *cp, *s_dup; + extrainfo_t *ei_tmp; + cp = s_dup = tor_strdup(s); + ei_tmp = extrainfo_parse_entry_from_string(cp, NULL, 1, NULL); + if (!ei_tmp) { + log_warn(LD_GENERAL, + "We just generated an extra-info descriptor with " + "statistics that we can't parse. Not adding statistics to " + "this or any future extra-info descriptors. Descriptor " + "was:\n%s", s); + write_stats_to_extrainfo = 0; + extrainfo_dump_to_string(s, maxlen, extrainfo, ident_key); + } + tor_free(s_dup); + extrainfo_free(ei_tmp); } - return geoip_get_client_history(now, GEOIP_CLIENT_CONNECT); + + return (int)strlen(s)+1; } /** Return true iff <b>s</b> is a legally valid server nickname. */ @@ -2045,26 +2139,16 @@ router_purpose_from_string(const char *s) void router_free_all(void) { - if (onionkey) - crypto_free_pk_env(onionkey); - if (lastonionkey) - crypto_free_pk_env(lastonionkey); - if (identitykey) - crypto_free_pk_env(identitykey); - if (key_lock) - tor_mutex_free(key_lock); - if (desc_routerinfo) - routerinfo_free(desc_routerinfo); - if (desc_extrainfo) - extrainfo_free(desc_extrainfo); - if (authority_signing_key) - crypto_free_pk_env(authority_signing_key); - if (authority_key_certificate) - authority_cert_free(authority_key_certificate); - if (legacy_signing_key) - crypto_free_pk_env(legacy_signing_key); - if (legacy_key_certificate) - authority_cert_free(legacy_key_certificate); + crypto_free_pk_env(onionkey); + crypto_free_pk_env(lastonionkey); + crypto_free_pk_env(identitykey); + tor_mutex_free(key_lock); + routerinfo_free(desc_routerinfo); + extrainfo_free(desc_extrainfo); + crypto_free_pk_env(authority_signing_key); + authority_cert_free(authority_key_certificate); + crypto_free_pk_env(legacy_signing_key); + authority_cert_free(legacy_key_certificate); if (warned_nonexistent_family) { SMARTLIST_FOREACH(warned_nonexistent_family, char *, cp, tor_free(cp)); diff --git a/src/or/routerlist.c b/src/or/routerlist.c index 9e91fe825b..1b9df56031 100644 --- a/src/or/routerlist.c +++ b/src/or/routerlist.c @@ -156,21 +156,24 @@ already_have_cert(authority_cert_t *cert) /** Load a bunch of new key certificates from the string <b>contents</b>. If * <b>from_store</b> is true, the certificates are from the cache, and we - * don't need to flush them to disk. If <b>from_store</b> is false, we need - * to flush any changed certificates to disk. Return 0 on success, -1 on - * failure. */ + * don't need to flush them to disk. If <b>flush</b> is true, we need + * to flush any changed certificates to disk now. Return 0 on success, -1 + * if any certs fail to parse. */ int trusted_dirs_load_certs_from_string(const char *contents, int from_store, int flush) { trusted_dir_server_t *ds; const char *s, *eos; + int failure_code = 0; for (s = contents; *s; s = eos) { authority_cert_t *cert = authority_cert_parse_from_string(s, &eos); cert_list_t *cl; - if (!cert) + if (!cert) { + failure_code = -1; break; + } ds = trusteddirserver_get_by_v3_auth_digest( cert->cache_info.identity_digest); log_debug(LD_DIR, "Parsed certificate for %s", @@ -224,7 +227,7 @@ trusted_dirs_load_certs_from_string(const char *contents, int from_store, ds->dir_port != cert->dir_port)) { char *a = tor_dup_ip(cert->addr); log_notice(LD_DIR, "Updating address for directory authority %s " - "from %s:%d to %s:%d based on in certificate.", + "from %s:%d to %s:%d based on certificate.", ds->nickname, ds->address, (int)ds->dir_port, a, cert->dir_port); tor_free(a); @@ -241,8 +244,11 @@ trusted_dirs_load_certs_from_string(const char *contents, int from_store, if (flush) trusted_dirs_flush_certs_to_disk(); + /* call this even if failure_code is <0, since some certs might have + * succeeded. */ networkstatus_note_certs_arrived(); - return 0; + + return failure_code; } /** Save all v3 key certificates to the cached-certs file. */ @@ -442,17 +448,18 @@ authority_certs_fetch_missing(networkstatus_t *status, time_t now) list_pending_downloads(pending, DIR_PURPOSE_FETCH_CERTIFICATE, "fp/"); if (status) { - SMARTLIST_FOREACH(status->voters, networkstatus_voter_info_t *, voter, - { - if (tor_digest_is_zero(voter->signing_key_digest)) - continue; /* This authority never signed this consensus, so don't - * go looking for a cert with key digest 0000000000. */ - if (!cache && - !trusteddirserver_get_by_v3_auth_digest(voter->identity_digest)) - continue; /* We are not a cache, and we don't know this authority.*/ - cl = get_cert_list(voter->identity_digest); + SMARTLIST_FOREACH_BEGIN(status->voters, networkstatus_voter_info_t *, + voter) { + if (!smartlist_len(voter->sigs)) + continue; /* This authority never signed this consensus, so don't + * go looking for a cert with key digest 0000000000. */ + if (!cache && + !trusteddirserver_get_by_v3_auth_digest(voter->identity_digest)) + continue; /* We are not a cache, and we don't know this authority.*/ + cl = get_cert_list(voter->identity_digest); + SMARTLIST_FOREACH_BEGIN(voter->sigs, document_signature_t *, sig) { cert = authority_cert_get_by_digests(voter->identity_digest, - voter->signing_key_digest); + sig->signing_key_digest); if (cert) { if (now < cert->expires) download_status_reset(&cl->dl_status); @@ -463,37 +470,36 @@ authority_certs_fetch_missing(networkstatus_t *status, time_t now) !digestmap_get(pending, voter->identity_digest)) { log_notice(LD_DIR, "We're missing a certificate from authority " "with signing key %s: launching request.", - hex_str(voter->signing_key_digest, DIGEST_LEN)); - smartlist_add(missing_digests, voter->identity_digest); + hex_str(sig->signing_key_digest, DIGEST_LEN)); + smartlist_add(missing_digests, sig->identity_digest); } - }); + } SMARTLIST_FOREACH_END(sig); + } SMARTLIST_FOREACH_END(voter); } - SMARTLIST_FOREACH(trusted_dir_servers, trusted_dir_server_t *, ds, - { - int found = 0; - if (!(ds->type & V3_AUTHORITY)) - continue; - if (smartlist_digest_isin(missing_digests, ds->v3_identity_digest)) - continue; - cl = get_cert_list(ds->v3_identity_digest); - SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert, - { - if (!ftime_definitely_after(now, cert->expires)) { - /* It's not expired, and we weren't looking for something to - * verify a consensus with. Call it done. */ - download_status_reset(&cl->dl_status); - found = 1; - break; - } - }); - if (!found && - download_status_is_ready(&cl->dl_status, now,MAX_CERT_DL_FAILURES) && - !digestmap_get(pending, ds->v3_identity_digest)) { - log_notice(LD_DIR, "No current certificate known for authority %s; " - "launching request.", ds->nickname); - smartlist_add(missing_digests, ds->v3_identity_digest); + SMARTLIST_FOREACH_BEGIN(trusted_dir_servers, trusted_dir_server_t *, ds) { + int found = 0; + if (!(ds->type & V3_AUTHORITY)) + continue; + if (smartlist_digest_isin(missing_digests, ds->v3_identity_digest)) + continue; + cl = get_cert_list(ds->v3_identity_digest); + SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert, { + if (!ftime_definitely_after(now, cert->expires)) { + /* It's not expired, and we weren't looking for something to + * verify a consensus with. Call it done. */ + download_status_reset(&cl->dl_status); + found = 1; + break; } }); + if (!found && + download_status_is_ready(&cl->dl_status, now,MAX_CERT_DL_FAILURES) && + !digestmap_get(pending, ds->v3_identity_digest)) { + log_notice(LD_DIR, "No current certificate known for authority %s; " + "launching request.", ds->nickname); + smartlist_add(missing_digests, ds->v3_identity_digest); + } + } SMARTLIST_FOREACH_END(ds); if (!smartlist_len(missing_digests)) { goto done; @@ -751,8 +757,7 @@ router_rebuild_store(int flags, desc_store_t *store) store->journal_len = 0; store->bytes_dropped = 0; done: - if (signed_descriptors) - smartlist_free(signed_descriptors); + smartlist_free(signed_descriptors); tor_free(fname); tor_free(fname_tmp); if (chunk_list) { @@ -1693,12 +1698,12 @@ smartlist_choose_by_bandwidth(smartlist_t *sl, bandwidth_weight_rule_t rule, * For detailed derivation of this formula, see * http://archives.seul.org/or/dev/Jul-2007/msg00056.html */ - if (rule == WEIGHT_FOR_EXIT) + if (rule == WEIGHT_FOR_EXIT || !total_exit_bw) exit_weight = 1.0; else exit_weight = 1.0 - all_bw/(3.0*exit_bw); - if (rule == WEIGHT_FOR_GUARD) + if (rule == WEIGHT_FOR_GUARD || !total_guard_bw) guard_weight = 1.0; else guard_weight = 1.0 - all_bw/(3.0*guard_bw); @@ -1747,6 +1752,8 @@ smartlist_choose_by_bandwidth(smartlist_t *sl, bandwidth_weight_rule_t rule, /* Almost done: choose a random value from the bandwidth weights. */ rand_bw = crypto_rand_uint64(total_bw); + rand_bw++; /* crypto_rand_uint64() counts from 0, and we need to count + * from 1 below. See bug 1203 for details. */ /* Last, count through sl until we get to the element we picked */ tmp = 0; @@ -1803,13 +1810,10 @@ routerstatus_sl_choose_by_bandwidth(smartlist_t *sl) return smartlist_choose_by_bandwidth(sl, NO_WEIGHTING, 1); } -/** Return a random running router from the routerlist. If any node - * named in <b>preferred</b> is available, pick one of those. Never +/** Return a random running router from the routerlist. Never * pick a node whose routerinfo is in * <b>excludedsmartlist</b>, or whose routerinfo matches <b>excludedset</b>, - * even if they are the only nodes - * available. If <b>CRN_STRICT_PREFERRED</b> is set in flags, never pick - * any node besides those in <b>preferred</b>. + * even if they are the only nodes available. * If <b>CRN_NEED_UPTIME</b> is set in flags and any router has more than * a minimum uptime, return one of those. * If <b>CRN_NEED_CAPACITY</b> is set in flags, weight your choice by the @@ -1822,8 +1826,7 @@ routerstatus_sl_choose_by_bandwidth(smartlist_t *sl) * node (that is, possibly discounting exit nodes). */ routerinfo_t * -router_choose_random_node(const char *preferred, - smartlist_t *excludedsmartlist, +router_choose_random_node(smartlist_t *excludedsmartlist, routerset_t *excludedset, router_crn_flags_t flags) { @@ -1831,10 +1834,10 @@ router_choose_random_node(const char *preferred, const int need_capacity = (flags & CRN_NEED_CAPACITY) != 0; const int need_guard = (flags & CRN_NEED_GUARD) != 0; const int allow_invalid = (flags & CRN_ALLOW_INVALID) != 0; - const int strict = (flags & CRN_STRICT_PREFERRED) != 0; const int weight_for_exit = (flags & CRN_WEIGHT_AS_EXIT) != 0; - smartlist_t *sl, *excludednodes; + smartlist_t *sl=smartlist_create(), + *excludednodes=smartlist_create(); routerinfo_t *choice = NULL, *r; bandwidth_weight_rule_t rule; @@ -1842,8 +1845,6 @@ router_choose_random_node(const char *preferred, rule = weight_for_exit ? WEIGHT_FOR_EXIT : (need_guard ? WEIGHT_FOR_GUARD : NO_WEIGHTING); - excludednodes = smartlist_create(); - /* Exclude relays that allow single hop exit circuits, if the user * wants to (such relays might be risky) */ if (get_options()->ExcludeSingleHopRelays) { @@ -1859,60 +1860,37 @@ router_choose_random_node(const char *preferred, routerlist_add_family(excludednodes, r); } - /* Try the preferred nodes first. Ignore need_uptime and need_capacity - * and need_guard, since the user explicitly asked for these nodes. */ - if (preferred) { - sl = smartlist_create(); - add_nickname_list_to_smartlist(sl,preferred,1); - smartlist_subtract(sl,excludednodes); - if (excludedsmartlist) - smartlist_subtract(sl,excludedsmartlist); - if (excludedset) - routerset_subtract_routers(sl,excludedset); + router_add_running_routers_to_smartlist(sl, allow_invalid, + need_uptime, need_capacity, + need_guard); + smartlist_subtract(sl,excludednodes); + if (excludedsmartlist) + smartlist_subtract(sl,excludedsmartlist); + if (excludedset) + routerset_subtract_routers(sl,excludedset); + + if (need_capacity || need_guard) + choice = routerlist_sl_choose_by_bandwidth(sl, rule); + else choice = smartlist_choose(sl); - smartlist_free(sl); - } - if (!choice && !strict) { - /* Then give up on our preferred choices: any node - * will do that has the required attributes. */ - sl = smartlist_create(); - router_add_running_routers_to_smartlist(sl, allow_invalid, - need_uptime, need_capacity, - need_guard); - smartlist_subtract(sl,excludednodes); - if (excludedsmartlist) - smartlist_subtract(sl,excludedsmartlist); - if (excludedset) - routerset_subtract_routers(sl,excludedset); - - if (need_capacity || need_guard) - choice = routerlist_sl_choose_by_bandwidth(sl, rule); - else - choice = smartlist_choose(sl); - - smartlist_free(sl); - if (!choice && (need_uptime || need_capacity || need_guard)) { - /* try once more -- recurse but with fewer restrictions. */ - log_info(LD_CIRC, - "We couldn't find any live%s%s%s routers; falling back " - "to list of all routers.", - need_capacity?", fast":"", - need_uptime?", stable":"", - need_guard?", guard":""); - flags &= ~ (CRN_NEED_UPTIME|CRN_NEED_CAPACITY|CRN_NEED_GUARD); - choice = router_choose_random_node( - NULL, excludedsmartlist, excludedset, flags); - } + + smartlist_free(sl); + if (!choice && (need_uptime || need_capacity || need_guard)) { + /* try once more -- recurse but with fewer restrictions. */ + log_info(LD_CIRC, + "We couldn't find any live%s%s%s routers; falling back " + "to list of all routers.", + need_capacity?", fast":"", + need_uptime?", stable":"", + need_guard?", guard":""); + flags &= ~ (CRN_NEED_UPTIME|CRN_NEED_CAPACITY|CRN_NEED_GUARD); + choice = router_choose_random_node( + excludedsmartlist, excludedset, flags); } smartlist_free(excludednodes); if (!choice) { - if (strict) { - log_warn(LD_CIRC, "All preferred nodes were down when trying to choose " - "node, and the Strict[...]Nodes option is set. Failing."); - } else { - log_warn(LD_CIRC, - "No available nodes when trying to choose node. Failing."); - } + log_warn(LD_CIRC, + "No available nodes when trying to choose node. Failing."); } return choice; } @@ -2372,6 +2350,9 @@ extrainfo_free(extrainfo_t *extrainfo) static void signed_descriptor_free(signed_descriptor_t *sd) { + if (!sd) + return; + tor_free(sd->signed_descriptor_body); /* XXXX remove this once more bugs go away. */ @@ -2403,7 +2384,8 @@ _extrainfo_free(void *e) void routerlist_free(routerlist_t *rl) { - tor_assert(rl); + if (!rl) + return; rimap_free(rl->identity_map, NULL); sdmap_free(rl->desc_digest_map, NULL); sdmap_free(rl->desc_by_eid_map, NULL); @@ -2442,46 +2424,6 @@ dump_routerlist_mem_usage(int severity) "In %d old descriptors: "U64_FORMAT" bytes.", smartlist_len(routerlist->routers), U64_PRINTF_ARG(livedescs), smartlist_len(routerlist->old_routers), U64_PRINTF_ARG(olddescs)); - -#if 0 - { - const smartlist_t *networkstatus_v2_list = networkstatus_get_v2_list(); - networkstatus_t *consensus = networkstatus_get_latest_consensus(); - log(severity, LD_DIR, "Now let's look through old_descriptors!"); - SMARTLIST_FOREACH(routerlist->old_routers, signed_descriptor_t *, sd, { - int in_v2 = 0; - int in_v3 = 0; - char published[ISO_TIME_LEN+1]; - char last_valid_until[ISO_TIME_LEN+1]; - char last_served_at[ISO_TIME_LEN+1]; - char id[HEX_DIGEST_LEN+1]; - routerstatus_t *rs; - format_iso_time(published, sd->published_on); - format_iso_time(last_valid_until, sd->last_listed_as_valid_until); - format_iso_time(last_served_at, sd->last_served_at); - base16_encode(id, sizeof(id), sd->identity_digest, DIGEST_LEN); - SMARTLIST_FOREACH(networkstatus_v2_list, networkstatus_v2_t *, ns, - { - rs = networkstatus_v2_find_entry(ns, sd->identity_digest); - if (rs && !memcmp(rs->descriptor_digest, - sd->signed_descriptor_digest, DIGEST_LEN)) { - in_v2 = 1; break; - } - }); - if (consensus) { - rs = networkstatus_vote_find_entry(consensus, sd->identity_digest); - if (rs && !memcmp(rs->descriptor_digest, - sd->signed_descriptor_digest, DIGEST_LEN)) - in_v3 = 1; - } - log(severity, LD_DIR, - "Old descriptor for %s (published %s) %sin v2 ns, %sin v3 " - "consensus. Last valid until %s; last served at %s.", - id, published, in_v2 ? "" : "not ", in_v3 ? "" : "not ", - last_valid_until, last_served_at); - }); - } -#endif } /** Debugging helper: If <b>idx</b> is nonnegative, assert that <b>ri</b> is @@ -2851,8 +2793,7 @@ routerlist_reparse_old(routerlist_t *rl, signed_descriptor_t *sd) void routerlist_free_all(void) { - if (routerlist) - routerlist_free(routerlist); + routerlist_free(routerlist); routerlist = NULL; if (warned_nicknames) { SMARTLIST_FOREACH(warned_nicknames, char *, cp, tor_free(cp)); @@ -3741,12 +3682,8 @@ add_trusted_dir_server(const char *nickname, const char *address, if (ent->or_port) ent->fake_status.version_supports_begindir = 1; -/* XX021 - wait until authorities are upgraded */ -#if 0 + ent->fake_status.version_supports_conditional_consensus = 1; -#else - ent->fake_status.version_supports_conditional_consensus = 0; -#endif smartlist_add(trusted_dir_servers, ent); router_dir_info_changed(); @@ -3761,10 +3698,8 @@ authority_cert_free(authority_cert_t *cert) return; tor_free(cert->cache_info.signed_descriptor_body); - if (cert->signing_key) - crypto_free_pk_env(cert->signing_key); - if (cert->identity_key) - crypto_free_pk_env(cert->identity_key); + crypto_free_pk_env(cert->signing_key); + crypto_free_pk_env(cert->identity_key); tor_free(cert); } @@ -3773,6 +3708,9 @@ authority_cert_free(authority_cert_t *cert) static void trusted_dir_server_free(trusted_dir_server_t *ds) { + if (!ds) + return; + tor_free(ds->nickname); tor_free(ds->description); tor_free(ds->address); @@ -3826,7 +3764,7 @@ list_pending_downloads(digestmap_t *result, const char *resource = TO_DIR_CONN(conn)->requested_resource; if (!strcmpstart(resource, prefix)) dir_split_resource_into_fingerprints(resource + p_len, - tmp, NULL, 1, 0); + tmp, NULL, DSR_HEX); } }); SMARTLIST_FOREACH(tmp, char *, d, @@ -3928,7 +3866,7 @@ client_would_use_router(routerstatus_t *rs, time_t now, or_options_t *options) * this number per server. */ #define MIN_DL_PER_REQUEST 4 /** To prevent a single screwy cache from confusing us by selective reply, - * try to split our requests into at least this this many requests. */ + * try to split our requests into at least this many requests. */ #define MIN_REQUESTS 3 /** If we want fewer than this many descriptors, wait until we * want more, or until MAX_CLIENT_INTERVAL_WITHOUT_REQUEST has @@ -4816,8 +4754,8 @@ esc_router_info(routerinfo_t *router) static char *info=NULL; char *esc_contact, *esc_platform; size_t len; - if (info) - tor_free(info); + tor_free(info); + if (!router) return NULL; /* we're exiting; just free the memory we use */ @@ -4952,9 +4890,8 @@ void routerset_refresh_countries(routerset_t *target) { int cc; - if (target->countries) { - bitarray_free(target->countries); - } + bitarray_free(target->countries); + if (!geoip_is_loaded()) { target->countries = NULL; target->n_countries = 0; @@ -5028,7 +4965,9 @@ routerset_parse(routerset_t *target, const char *s, const char *description) return r; } -/** DOCDOC */ +/** Called when we change a node set, or when we reload the geoip list: + * recompute all country info in all configuration node sets and in the + * routerlist. */ void refresh_all_country_info(void) { @@ -5196,9 +5135,13 @@ routerset_get_all_routers(smartlist_t *out, const routerset_t *routerset, } } -/** Add to <b>target</b> every routerinfo_t from <b>source</b> that is in - * <b>include</b>, but not excluded in a more specific fashion by - * <b>exclude</b>. If <b>running_only</b>, only include running routers. +/** Add to <b>target</b> every routerinfo_t from <b>source</b> except: + * + * 1) Don't add it if <b>include</b> is non-empty and the relay isn't in + * <b>include</b>; and + * 2) Don't add it if <b>exclude</b> is non-empty and the relay is + * excluded in a more specific fashion by <b>exclude</b>. + * 3) If <b>running_only</b>, don't add non-running routers. */ void routersets_get_disjunction(smartlist_t *target, @@ -5268,35 +5211,15 @@ routerset_equal(const routerset_t *old, const routerset_t *new) }); return 1; - -#if 0 - /* XXXX: This won't work if the names/digests are identical but in a - different order. Checking for exact equality would be heavy going, - is it worth it? -RH*/ - /* This code is totally bogus; sizeof doesn't work even remotely like this - * code seems to think. Let's revert to a string-based comparison for - * now. -NM*/ - if (sizeof(old->names) != sizeof(new->names)) - return 0; - - if (memcmp(old->names,new->names,sizeof(new->names))) - return 0; - if (sizeof(old->digests) != sizeof(new->digests)) - return 0; - if (memcmp(old->digests,new->digests,sizeof(new->digests))) - return 0; - if (sizeof(old->countries) != sizeof(new->countries)) - return 0; - if (memcmp(old->countries,new->countries,sizeof(new->countries))) - return 0; - return 1; -#endif } /** Free all storage held in <b>routerset</b>. */ void routerset_free(routerset_t *routerset) { + if (!routerset) + return; + SMARTLIST_FOREACH(routerset->list, char *, cp, tor_free(cp)); smartlist_free(routerset->list); SMARTLIST_FOREACH(routerset->policies, addr_policy_t *, p, @@ -5307,8 +5230,7 @@ routerset_free(routerset_t *routerset) strmap_free(routerset->names, NULL); digestmap_free(routerset->digests, NULL); - if (routerset->countries) - bitarray_free(routerset->countries); + bitarray_free(routerset->countries); tor_free(routerset); } diff --git a/src/or/routerparse.c b/src/or/routerparse.c index 4e1d0cd592..13dc10c0ea 100644 --- a/src/or/routerparse.c +++ b/src/or/routerparse.c @@ -55,6 +55,7 @@ typedef enum { K_S, K_V, K_W, + K_M, K_EVENTDNS, K_EXTRA_INFO, K_EXTRA_INFO_DIGEST, @@ -62,6 +63,31 @@ typedef enum { K_HIDDEN_SERVICE_DIR, K_ALLOW_SINGLE_HOP_EXITS, + K_DIRREQ_END, + K_DIRREQ_V2_IPS, + K_DIRREQ_V3_IPS, + K_DIRREQ_V2_REQS, + K_DIRREQ_V3_REQS, + K_DIRREQ_V2_SHARE, + K_DIRREQ_V3_SHARE, + K_DIRREQ_V2_RESP, + K_DIRREQ_V3_RESP, + K_DIRREQ_V2_DIR, + K_DIRREQ_V3_DIR, + K_DIRREQ_V2_TUN, + K_DIRREQ_V3_TUN, + K_ENTRY_END, + K_ENTRY_IPS, + K_CELL_END, + K_CELL_PROCESSED, + K_CELL_QUEUED, + K_CELL_TIME, + K_CELL_CIRCS, + K_EXIT_END, + K_EXIT_WRITTEN, + K_EXIT_READ, + K_EXIT_OPENED, + K_DIR_KEY_CERTIFICATE_VERSION, K_DIR_IDENTITY_KEY, K_DIR_KEY_PUBLISHED, @@ -80,11 +106,14 @@ typedef enum { K_PARAMS, K_VOTE_DIGEST, K_CONSENSUS_DIGEST, + K_ADDITIONAL_DIGEST, + K_ADDITIONAL_SIGNATURE, K_CONSENSUS_METHODS, K_CONSENSUS_METHOD, K_LEGACY_DIR_KEY, A_PURPOSE, + A_LAST_LISTED, _A_UNKNOWN, R_RENDEZVOUS_SERVICE_DESCRIPTOR, @@ -122,7 +151,7 @@ typedef enum { * type. * * This structure is only allocated in memareas; do not allocate it on - * the heap, or token_free() won't work. + * the heap, or token_clear() won't work. */ typedef struct directory_token_t { directory_keyword tp; /**< Type of the token. */ @@ -258,6 +287,31 @@ static token_rule_t extrainfo_token_table[] = { T0N("opt", K_OPT, CONCAT_ARGS, OBJ_OK ), T01("read-history", K_READ_HISTORY, ARGS, NO_OBJ ), T01("write-history", K_WRITE_HISTORY, ARGS, NO_OBJ ), + T01("dirreq-stats-end", K_DIRREQ_END, ARGS, NO_OBJ ), + T01("dirreq-v2-ips", K_DIRREQ_V2_IPS, ARGS, NO_OBJ ), + T01("dirreq-v3-ips", K_DIRREQ_V3_IPS, ARGS, NO_OBJ ), + T01("dirreq-v2-reqs", K_DIRREQ_V2_REQS, ARGS, NO_OBJ ), + T01("dirreq-v3-reqs", K_DIRREQ_V3_REQS, ARGS, NO_OBJ ), + T01("dirreq-v2-share", K_DIRREQ_V2_SHARE, ARGS, NO_OBJ ), + T01("dirreq-v3-share", K_DIRREQ_V3_SHARE, ARGS, NO_OBJ ), + T01("dirreq-v2-resp", K_DIRREQ_V2_RESP, ARGS, NO_OBJ ), + T01("dirreq-v3-resp", K_DIRREQ_V3_RESP, ARGS, NO_OBJ ), + T01("dirreq-v2-direct-dl", K_DIRREQ_V2_DIR, ARGS, NO_OBJ ), + T01("dirreq-v3-direct-dl", K_DIRREQ_V3_DIR, ARGS, NO_OBJ ), + T01("dirreq-v2-tunneled-dl", K_DIRREQ_V2_TUN, ARGS, NO_OBJ ), + T01("dirreq-v3-tunneled-dl", K_DIRREQ_V3_TUN, ARGS, NO_OBJ ), + T01("entry-stats-end", K_ENTRY_END, ARGS, NO_OBJ ), + T01("entry-ips", K_ENTRY_IPS, ARGS, NO_OBJ ), + T01("cell-stats-end", K_CELL_END, ARGS, NO_OBJ ), + T01("cell-processed-cells", K_CELL_PROCESSED, ARGS, NO_OBJ ), + T01("cell-queued-cells", K_CELL_QUEUED, ARGS, NO_OBJ ), + T01("cell-time-in-queue", K_CELL_TIME, ARGS, NO_OBJ ), + T01("cell-circuits-per-decile", K_CELL_CIRCS, ARGS, NO_OBJ ), + T01("exit-stats-end", K_EXIT_END, ARGS, NO_OBJ ), + T01("exit-kibibytes-written", K_EXIT_WRITTEN, ARGS, NO_OBJ ), + T01("exit-kibibytes-read", K_EXIT_READ, ARGS, NO_OBJ ), + T01("exit-streams-opened", K_EXIT_OPENED, ARGS, NO_OBJ ), + T1_START( "extra-info", K_EXTRA_INFO, GE(2), NO_OBJ ), END_OF_TABLE @@ -267,10 +321,11 @@ static token_rule_t extrainfo_token_table[] = { * documents. */ static token_rule_t rtrstatus_token_table[] = { T01("p", K_P, CONCAT_ARGS, NO_OBJ ), - T1( "r", K_R, GE(8), NO_OBJ ), + T1( "r", K_R, GE(7), NO_OBJ ), T1( "s", K_S, ARGS, NO_OBJ ), T01("v", K_V, CONCAT_ARGS, NO_OBJ ), T01("w", K_W, ARGS, NO_OBJ ), + T0N("m", K_M, CONCAT_ARGS, NO_OBJ ), T0N("opt", K_OPT, CONCAT_ARGS, OBJ_OK ), END_OF_TABLE }; @@ -437,10 +492,20 @@ static token_rule_t networkstatus_vote_footer_token_table[] = { /** List of tokens allowable in detached networkstatus signature documents. */ static token_rule_t networkstatus_detached_signature_token_table[] = { T1_START("consensus-digest", K_CONSENSUS_DIGEST, GE(1), NO_OBJ ), + T("additional-digest", K_ADDITIONAL_DIGEST,GE(3), NO_OBJ ), T1("valid-after", K_VALID_AFTER, CONCAT_ARGS, NO_OBJ ), T1("fresh-until", K_FRESH_UNTIL, CONCAT_ARGS, NO_OBJ ), T1("valid-until", K_VALID_UNTIL, CONCAT_ARGS, NO_OBJ ), - T1N("directory-signature", K_DIRECTORY_SIGNATURE, GE(2), NEED_OBJ ), + T("additional-signature", K_ADDITIONAL_SIGNATURE, GE(4), NEED_OBJ ), + T1N("directory-signature", K_DIRECTORY_SIGNATURE, GE(2), NEED_OBJ ), + END_OF_TABLE +}; + +static token_rule_t microdesc_token_table[] = { + T1_START("onion-key", K_ONION_KEY, NO_ARGS, NEED_KEY_1024), + T01("family", K_FAMILY, ARGS, NO_OBJ ), + T01("p", K_P, CONCAT_ARGS, NO_OBJ ), + A01("@last-listed", A_LAST_LISTED, CONCAT_ARGS, NO_OBJ ), END_OF_TABLE }; @@ -453,8 +518,12 @@ static addr_policy_t *router_parse_addr_policy_private(directory_token_t *tok); static int router_get_hash_impl(const char *s, char *digest, const char *start_str, const char *end_str, - char end_char); -static void token_free(directory_token_t *tok); + char end_char, + digest_algorithm_t alg); +static int router_get_hashes_impl(const char *s, digests_t *digests, + const char *start_str, const char *end_str, + char end_char); +static void token_clear(directory_token_t *tok); static smartlist_t *find_all_exitpolicy(smartlist_t *s); static directory_token_t *_find_by_keyword(smartlist_t *s, directory_keyword keyword, @@ -478,6 +547,7 @@ static directory_token_t *get_next_token(memarea_t *area, #define CST_CHECK_AUTHORITY (1<<0) #define CST_NO_CHECK_OBJTYPE (1<<1) static int check_signature_token(const char *digest, + ssize_t digest_len, directory_token_t *tok, crypto_pk_env_t *pkey, int flags, @@ -498,6 +568,34 @@ static int tor_version_same_series(tor_version_t *a, tor_version_t *b); #define DUMP_AREA(a,name) STMT_NIL #endif +/** Last time we dumped a descriptor to disk. */ +static time_t last_desc_dumped = 0; + +/** For debugging purposes, dump unparseable descriptor *<b>desc</b> of + * type *<b>type</b> to file $DATADIR/unparseable-desc. Do not write more + * than one descriptor to disk per minute. If there is already such a + * file in the data directory, overwrite it. */ +static void +dump_desc(const char *desc, const char *type) +{ + time_t now = time(NULL); + tor_assert(desc); + tor_assert(type); + if (!last_desc_dumped || last_desc_dumped + 60 < now) { + char *debugfile = get_datadir_fname("unparseable-desc"); + size_t filelen = 50 + strlen(type) + strlen(desc); + char *content = tor_malloc_zero(filelen); + tor_snprintf(content, filelen, "Unable to parse descriptor of type " + "%s:\n%s", type, desc); + write_str_to_file(debugfile, content, 0); + log_info(LD_DIR, "Unable to parse descriptor of type %s. See file " + "unparseable-desc in data directory for details.", type); + tor_free(content); + tor_free(debugfile); + last_desc_dumped = now; + } +} + /** Set <b>digest</b> to the SHA-1 digest of the hash of the directory in * <b>s</b>. Return 0 on success, -1 on failure. */ @@ -505,7 +603,8 @@ int router_get_dir_hash(const char *s, char *digest) { return router_get_hash_impl(s,digest, - "signed-directory","\ndirectory-signature",'\n'); + "signed-directory","\ndirectory-signature",'\n', + DIGEST_SHA1); } /** Set <b>digest</b> to the SHA-1 digest of the hash of the first router in @@ -515,7 +614,8 @@ int router_get_router_hash(const char *s, char *digest) { return router_get_hash_impl(s,digest, - "router ","\nrouter-signature", '\n'); + "router ","\nrouter-signature", '\n', + DIGEST_SHA1); } /** Set <b>digest</b> to the SHA-1 digest of the hash of the running-routers @@ -525,7 +625,8 @@ int router_get_runningrouters_hash(const char *s, char *digest) { return router_get_hash_impl(s,digest, - "network-status","\ndirectory-signature", '\n'); + "network-status","\ndirectory-signature", '\n', + DIGEST_SHA1); } /** Set <b>digest</b> to the SHA-1 digest of the hash of the network-status @@ -535,17 +636,31 @@ router_get_networkstatus_v2_hash(const char *s, char *digest) { return router_get_hash_impl(s,digest, "network-status-version","\ndirectory-signature", - '\n'); + '\n', + DIGEST_SHA1); +} + +/** Set <b>digests</b> to all the digests of the consensus document in + * <b>s</b> */ +int +router_get_networkstatus_v3_hashes(const char *s, digests_t *digests) +{ + return router_get_hashes_impl(s,digests, + "network-status-version", + "\ndirectory-signature", + ' '); } /** Set <b>digest</b> to the SHA-1 digest of the hash of the network-status * string in <b>s</b>. Return 0 on success, -1 on failure. */ int -router_get_networkstatus_v3_hash(const char *s, char *digest) +router_get_networkstatus_v3_hash(const char *s, char *digest, + digest_algorithm_t alg) { return router_get_hash_impl(s,digest, - "network-status-version","\ndirectory-signature", - ' '); + "network-status-version", + "\ndirectory-signature", + ' ', alg); } /** Set <b>digest</b> to the SHA-1 digest of the hash of the extrainfo @@ -553,7 +668,8 @@ router_get_networkstatus_v3_hash(const char *s, char *digest) int router_get_extrainfo_hash(const char *s, char *digest) { - return router_get_hash_impl(s,digest,"extra-info","\nrouter-signature",'\n'); + return router_get_hash_impl(s,digest,"extra-info","\nrouter-signature",'\n', + DIGEST_SHA1); } /** Helper: used to generate signatures for routers, directories and @@ -565,14 +681,15 @@ router_get_extrainfo_hash(const char *s, char *digest) */ int router_append_dirobj_signature(char *buf, size_t buf_len, const char *digest, - crypto_pk_env_t *private_key) + size_t digest_len, crypto_pk_env_t *private_key) { char *signature; size_t i; + int siglen; signature = tor_malloc(crypto_pk_keysize(private_key)); - if (crypto_pk_private_sign(private_key, signature, digest, DIGEST_LEN) < 0) { - + siglen = crypto_pk_private_sign(private_key, signature, digest, digest_len); + if (siglen < 0) { log_warn(LD_BUG,"Couldn't sign digest."); goto err; } @@ -580,7 +697,7 @@ router_append_dirobj_signature(char *buf, size_t buf_len, const char *digest, goto truncated; i = strlen(buf); - if (base64_encode(buf+i, buf_len-i, signature, 128) < 0) { + if (base64_encode(buf+i, buf_len-i, signature, siglen) < 0) { log_warn(LD_BUG,"couldn't base64-encode signature"); goto err; } @@ -687,7 +804,7 @@ router_parse_directory(const char *str) char digest[DIGEST_LEN]; time_t published_on; int r; - const char *end, *cp; + const char *end, *cp, *str_dup = str; smartlist_t *tokens = NULL; crypto_pk_env_t *declared_key = NULL; memarea_t *area = memarea_new(); @@ -723,11 +840,11 @@ router_parse_directory(const char *str) } declared_key = find_dir_signing_key(str, str+strlen(str)); note_crypto_pk_op(VERIFY_DIR); - if (check_signature_token(digest, tok, declared_key, + if (check_signature_token(digest, DIGEST_LEN, tok, declared_key, CST_CHECK_AUTHORITY, "directory")<0) goto err; - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_clear(tokens); memarea_clear(area); @@ -760,11 +877,12 @@ router_parse_directory(const char *str) r = 0; goto done; err: + dump_desc(str_dup, "v1 directory"); r = -1; done: if (declared_key) crypto_free_pk_env(declared_key); if (tokens) { - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); } if (area) { @@ -786,7 +904,7 @@ router_parse_runningrouters(const char *str) int r = -1; crypto_pk_env_t *declared_key = NULL; smartlist_t *tokens = NULL; - const char *eos = str + strlen(str); + const char *eos = str + strlen(str), *str_dup = str; memarea_t *area = NULL; if (router_get_runningrouters_hash(str, digest)) { @@ -815,7 +933,7 @@ router_parse_runningrouters(const char *str) } declared_key = find_dir_signing_key(str, eos); note_crypto_pk_op(VERIFY_DIR); - if (check_signature_token(digest, tok, declared_key, + if (check_signature_token(digest, DIGEST_LEN, tok, declared_key, CST_CHECK_AUTHORITY, "running-routers") < 0) goto err; @@ -827,9 +945,10 @@ router_parse_runningrouters(const char *str) r = 0; err: + dump_desc(str_dup, "v1 running-routers"); if (declared_key) crypto_free_pk_env(declared_key); if (tokens) { - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); } if (area) { @@ -879,7 +998,7 @@ find_dir_signing_key(const char *str, const char *eos) } done: - if (tok) token_free(tok); + if (tok) token_clear(tok); if (area) { DUMP_AREA(area, "dir-signing-key token"); memarea_drop_all(area); @@ -915,6 +1034,7 @@ dir_signing_key_is_trusted(crypto_pk_env_t *key) */ static int check_signature_token(const char *digest, + ssize_t digest_len, directory_token_t *tok, crypto_pk_env_t *pkey, int flags, @@ -945,14 +1065,14 @@ check_signature_token(const char *digest, signed_digest = tor_malloc(tok->object_size); if (crypto_pk_public_checksig(pkey, signed_digest, tok->object_body, tok->object_size) - != DIGEST_LEN) { + < digest_len) { log_warn(LD_DIR, "Error reading %s: invalid signature.", doctype); tor_free(signed_digest); return -1; } // log_debug(LD_DIR,"Signed %s hash starts %s", doctype, // hex_str(signed_digest,4)); - if (memcmp(digest, signed_digest, DIGEST_LEN)) { + if (memcmp(digest, signed_digest, digest_len)) { log_warn(LD_DIR, "Error reading %s: signature does not match.", doctype); tor_free(signed_digest); return -1; @@ -1136,7 +1256,7 @@ router_parse_entry_from_string(const char *s, const char *end, smartlist_t *tokens = NULL, *exit_policy_tokens = NULL; directory_token_t *tok; struct in_addr in; - const char *start_of_annotations, *cp; + const char *start_of_annotations, *cp, *s_dup = s; size_t prepend_len = prepend_annotations ? strlen(prepend_annotations) : 0; int ok = 1; memarea_t *area = NULL; @@ -1408,7 +1528,7 @@ router_parse_entry_from_string(const char *s, const char *end, verified_digests = digestmap_new(); digestmap_set(verified_digests, signed_digest, (void*)(uintptr_t)1); #endif - if (check_signature_token(digest, tok, router->identity_pkey, 0, + if (check_signature_token(digest, DIGEST_LEN, tok, router->identity_pkey, 0, "router descriptor") < 0) goto err; @@ -1426,16 +1546,15 @@ router_parse_entry_from_string(const char *s, const char *end, goto done; err: + dump_desc(s_dup, "router descriptor"); routerinfo_free(router); router = NULL; done: if (tokens) { - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); } - if (exit_policy_tokens) { - smartlist_free(exit_policy_tokens); - } + smartlist_free(exit_policy_tokens); if (area) { DUMP_AREA(area, "routerinfo"); memarea_drop_all(area); @@ -1460,6 +1579,7 @@ extrainfo_parse_entry_from_string(const char *s, const char *end, crypto_pk_env_t *key = NULL; routerinfo_t *router = NULL; memarea_t *area = NULL; + const char *s_dup = s; if (!end) { end = s + strlen(s); @@ -1534,7 +1654,8 @@ extrainfo_parse_entry_from_string(const char *s, const char *end, if (key) { note_crypto_pk_op(VERIFY_RTR); - if (check_signature_token(digest, tok, key, 0, "extra-info") < 0) + if (check_signature_token(digest, DIGEST_LEN, tok, key, 0, + "extra-info") < 0) goto err; if (router) @@ -1548,12 +1669,12 @@ extrainfo_parse_entry_from_string(const char *s, const char *end, goto done; err: - if (extrainfo) - extrainfo_free(extrainfo); + dump_desc(s_dup, "extra-info descriptor"); + extrainfo_free(extrainfo); extrainfo = NULL; done: if (tokens) { - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); } if (area) { @@ -1577,6 +1698,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) size_t len; int found; memarea_t *area = NULL; + const char *s_dup = s; s = eat_whitespace(s); eos = strstr(s, "\ndir-key-certification"); @@ -1601,7 +1723,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) goto err; } if (router_get_hash_impl(s, digest, "dir-key-certificate-version", - "\ndir-key-certification", '\n') < 0) + "\ndir-key-certification", '\n', DIGEST_SHA1) < 0) goto err; tok = smartlist_get(tokens, 0); if (tok->tp != K_DIR_KEY_CERTIFICATE_VERSION || strcmp(tok->args[0], "3")) { @@ -1694,7 +1816,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) } } if (!found) { - if (check_signature_token(digest, tok, cert->identity_key, 0, + if (check_signature_token(digest, DIGEST_LEN, tok, cert->identity_key, 0, "key certificate")) { goto err; } @@ -1703,6 +1825,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) /* XXXX Once all authorities generate cross-certified certificates, * make this field mandatory. */ if (check_signature_token(cert->cache_info.identity_digest, + DIGEST_LEN, tok, cert->signing_key, CST_NO_CHECK_OBJTYPE, @@ -1722,7 +1845,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) if (end_of_string) { *end_of_string = eat_whitespace(eos); } - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); if (area) { DUMP_AREA(area, "authority cert"); @@ -1730,8 +1853,9 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) } return cert; err: + dump_desc(s_dup, "authority cert"); authority_cert_free(cert); - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); if (area) { DUMP_AREA(area, "authority cert"); @@ -1772,22 +1896,29 @@ find_start_of_next_routerstatus(const char *s) * If <b>consensus_method</b> is nonzero, this routerstatus is part of a * consensus, and we should parse it according to the method used to * make that consensus. + * + * Parse according to the syntax used by the consensus flavor <b>flav</b>. **/ static routerstatus_t * routerstatus_parse_entry_from_string(memarea_t *area, const char **s, smartlist_t *tokens, networkstatus_t *vote, vote_routerstatus_t *vote_rs, - int consensus_method) + int consensus_method, + consensus_flavor_t flav) { - const char *eos; + const char *eos, *s_dup = *s; routerstatus_t *rs = NULL; directory_token_t *tok; char timebuf[ISO_TIME_LEN+1]; struct in_addr in; + int offset = 0; tor_assert(tokens); tor_assert(bool_eq(vote, vote_rs)); + if (!consensus_method) + flav = FLAV_NS; + eos = find_start_of_next_routerstatus(*s); if (tokenize_string(area,*s, eos, tokens, rtrstatus_token_table,0)) { @@ -1799,7 +1930,15 @@ routerstatus_parse_entry_from_string(memarea_t *area, goto err; } tok = find_by_keyword(tokens, K_R); - tor_assert(tok->n_args >= 8); + tor_assert(tok->n_args >= 7); + if (flav == FLAV_NS) { + if (tok->n_args < 8) { + log_warn(LD_DIR, "Too few arguments to r"); + goto err; + } + } else { + offset = -1; + } if (vote_rs) { rs = &vote_rs->status; } else { @@ -1820,29 +1959,34 @@ routerstatus_parse_entry_from_string(memarea_t *area, goto err; } - if (digest_from_base64(rs->descriptor_digest, tok->args[2])) { - log_warn(LD_DIR, "Error decoding descriptor digest %s", - escaped(tok->args[2])); - goto err; + if (flav == FLAV_NS) { + if (digest_from_base64(rs->descriptor_digest, tok->args[2])) { + log_warn(LD_DIR, "Error decoding descriptor digest %s", + escaped(tok->args[2])); + goto err; + } } if (tor_snprintf(timebuf, sizeof(timebuf), "%s %s", - tok->args[3], tok->args[4]) < 0 || + tok->args[3+offset], tok->args[4+offset]) < 0 || parse_iso_time(timebuf, &rs->published_on)<0) { - log_warn(LD_DIR, "Error parsing time '%s %s'", - tok->args[3], tok->args[4]); + log_warn(LD_DIR, "Error parsing time '%s %s' [%d %d]", + tok->args[3+offset], tok->args[4+offset], + offset, (int)flav); goto err; } - if (tor_inet_aton(tok->args[5], &in) == 0) { + if (tor_inet_aton(tok->args[5+offset], &in) == 0) { log_warn(LD_DIR, "Error parsing router address in network-status %s", - escaped(tok->args[5])); + escaped(tok->args[5+offset])); goto err; } rs->addr = ntohl(in.s_addr); - rs->or_port =(uint16_t) tor_parse_long(tok->args[6],10,0,65535,NULL,NULL); - rs->dir_port = (uint16_t) tor_parse_long(tok->args[7],10,0,65535,NULL,NULL); + rs->or_port = (uint16_t) tor_parse_long(tok->args[6+offset], + 10,0,65535,NULL,NULL); + rs->dir_port = (uint16_t) tor_parse_long(tok->args[7+offset], + 10,0,65535,NULL,NULL); tok = find_opt_by_keyword(tokens, K_S); if (tok && vote) { @@ -1928,6 +2072,17 @@ routerstatus_parse_entry_from_string(memarea_t *area, goto err; } rs->has_bandwidth = 1; + } else if (!strcmpstart(tok->args[i], "Measured=")) { + int ok; + rs->measured_bw = + (uint32_t)tor_parse_ulong(strchr(tok->args[i], '=')+1, + 10, 0, UINT32_MAX, &ok, NULL); + if (!ok) { + log_warn(LD_DIR, "Invalid Measured Bandwidth %s", + escaped(tok->args[i])); + goto err; + } + rs->has_measured_bw = 1; } } } @@ -1949,16 +2104,29 @@ routerstatus_parse_entry_from_string(memarea_t *area, rs->has_exitsummary = 1; } + if (vote_rs) { + SMARTLIST_FOREACH_BEGIN(tokens, directory_token_t *, t) { + if (t->tp == K_M && t->n_args) { + vote_microdesc_hash_t *line = + tor_malloc(sizeof(vote_microdesc_hash_t)); + line->next = vote_rs->microdesc; + line->microdesc_hash_line = tor_strdup(t->args[0]); + vote_rs->microdesc = line; + } + } SMARTLIST_FOREACH_END(t); + } + if (!strcasecmp(rs->nickname, UNNAMED_ROUTER_NICKNAME)) rs->is_named = 0; goto done; err: + dump_desc(s_dup, "routerstatus entry"); if (rs && !vote_rs) routerstatus_free(rs); rs = NULL; done: - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_clear(tokens); if (area) { DUMP_AREA(area, "routerstatus entry"); @@ -1970,8 +2138,8 @@ routerstatus_parse_entry_from_string(memarea_t *area, } /** Helper to sort a smartlist of pointers to routerstatus_t */ -static int -_compare_routerstatus_entries(const void **_a, const void **_b) +int +compare_routerstatus_entries(const void **_a, const void **_b) { const routerstatus_t *a = *_a, *b = *_b; return memcmp(a->identity_digest, b->identity_digest, DIGEST_LEN); @@ -1995,7 +2163,7 @@ _free_duplicate_routerstatus_entry(void *e) networkstatus_v2_t * networkstatus_v2_parse_from_string(const char *s) { - const char *eos; + const char *eos, *s_dup = s; smartlist_t *tokens = smartlist_create(); smartlist_t *footer_tokens = smartlist_create(); networkstatus_v2_t *ns = NULL; @@ -2109,17 +2277,17 @@ networkstatus_v2_parse_from_string(const char *s) ns->entries = smartlist_create(); s = eos; - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_clear(tokens); memarea_clear(area); while (!strcmpstart(s, "r ")) { routerstatus_t *rs; if ((rs = routerstatus_parse_entry_from_string(area, &s, tokens, - NULL, NULL, 0))) + NULL, NULL, 0, 0))) smartlist_add(ns->entries, rs); } - smartlist_sort(ns->entries, _compare_routerstatus_entries); - smartlist_uniq(ns->entries, _compare_routerstatus_entries, + smartlist_sort(ns->entries, compare_routerstatus_entries); + smartlist_uniq(ns->entries, compare_routerstatus_entries, _free_duplicate_routerstatus_entry); if (tokenize_string(area,s, NULL, footer_tokens, dir_footer_token_table,0)) { @@ -2138,19 +2306,19 @@ networkstatus_v2_parse_from_string(const char *s) } note_crypto_pk_op(VERIFY_DIR); - if (check_signature_token(ns_digest, tok, ns->signing_key, 0, + if (check_signature_token(ns_digest, DIGEST_LEN, tok, ns->signing_key, 0, "network-status") < 0) goto err; goto done; err: - if (ns) - networkstatus_v2_free(ns); + dump_desc(s_dup, "v2 networkstatus"); + networkstatus_v2_free(ns); ns = NULL; done: - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); - SMARTLIST_FOREACH(footer_tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(footer_tokens, directory_token_t *, t, token_clear(t)); smartlist_free(footer_tokens); if (area) { DUMP_AREA(area, "v2 networkstatus"); @@ -2169,19 +2337,21 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, smartlist_t *rs_tokens = NULL, *footer_tokens = NULL; networkstatus_voter_info_t *voter = NULL; networkstatus_t *ns = NULL; - char ns_digest[DIGEST_LEN]; - const char *cert, *end_of_header, *end_of_footer; + digests_t ns_digests; + const char *cert, *end_of_header, *end_of_footer, *s_dup = s; directory_token_t *tok; int ok; struct in_addr in; int i, inorder, n_signatures = 0; memarea_t *area = NULL, *rs_area = NULL; + consensus_flavor_t flav = FLAV_NS; + tor_assert(s); if (eos_out) *eos_out = NULL; - if (router_get_networkstatus_v3_hash(s, ns_digest)) { + if (router_get_networkstatus_v3_hashes(s, &ns_digests)) { log_warn(LD_DIR, "Unable to compute digest of network-status"); goto err; } @@ -2197,7 +2367,23 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, } ns = tor_malloc_zero(sizeof(networkstatus_t)); - memcpy(ns->networkstatus_digest, ns_digest, DIGEST_LEN); + memcpy(&ns->digests, &ns_digests, sizeof(ns_digests)); + + tok = find_by_keyword(tokens, K_NETWORK_STATUS_VERSION); + tor_assert(tok); + if (tok->n_args > 1) { + int flavor = networkstatus_parse_flavor_name(tok->args[1]); + if (flavor < 0) { + log_warn(LD_DIR, "Can't parse document with unknown flavor %s", + escaped(tok->args[2])); + goto err; + } + ns->flavor = flav = flavor; + } + if (flav != FLAV_NS && ns_type != NS_TYPE_CONSENSUS) { + log_warn(LD_DIR, "Flavor found on non-consenus networkstatus."); + goto err; + } if (ns_type != NS_TYPE_CONSENSUS) { const char *end_of_cert = NULL; @@ -2351,8 +2537,9 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, if (voter) smartlist_add(ns->voters, voter); voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t)); + voter->sigs = smartlist_create(); if (ns->type != NS_TYPE_CONSENSUS) - memcpy(voter->vote_digest, ns_digest, DIGEST_LEN); + memcpy(voter->vote_digest, ns_digests.d[DIGEST_SHA1], DIGEST_LEN); voter->nickname = tor_strdup(tok->args[0]); if (strlen(tok->args[1]) != HEX_DIGEST_LEN || @@ -2444,7 +2631,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, if (ns->type != NS_TYPE_CONSENSUS) { vote_routerstatus_t *rs = tor_malloc_zero(sizeof(vote_routerstatus_t)); if (routerstatus_parse_entry_from_string(rs_area, &s, rs_tokens, ns, - rs, 0)) + rs, 0, 0)) smartlist_add(ns->routerstatus_list, rs); else { tor_free(rs->version); @@ -2454,7 +2641,8 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, routerstatus_t *rs; if ((rs = routerstatus_parse_entry_from_string(rs_area, &s, rs_tokens, NULL, NULL, - ns->consensus_method))) + ns->consensus_method, + flav))) smartlist_add(ns->routerstatus_list, rs); } } @@ -2487,14 +2675,33 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, goto err; } - SMARTLIST_FOREACH(footer_tokens, directory_token_t *, _tok, - { + SMARTLIST_FOREACH_BEGIN(footer_tokens, directory_token_t *, _tok) { char declared_identity[DIGEST_LEN]; networkstatus_voter_info_t *v; + document_signature_t *sig; + const char *id_hexdigest = NULL; + const char *sk_hexdigest = NULL; + digest_algorithm_t alg = DIGEST_SHA1; tok = _tok; if (tok->tp != K_DIRECTORY_SIGNATURE) continue; tor_assert(tok->n_args >= 2); + if (tok->n_args == 2) { + id_hexdigest = tok->args[0]; + sk_hexdigest = tok->args[1]; + } else { + const char *algname = tok->args[0]; + int a; + id_hexdigest = tok->args[1]; + sk_hexdigest = tok->args[2]; + a = crypto_digest_algorithm_parse_name(algname); + if (a<0) { + log_warn(LD_DIR, "Unknown digest algorithm %s; skipping", + escaped(algname)); + continue; + } + alg = a; + } if (!tok->object_type || strcmp(tok->object_type, "SIGNATURE") || @@ -2503,11 +2710,11 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, goto err; } - if (strlen(tok->args[0]) != HEX_DIGEST_LEN || + if (strlen(id_hexdigest) != HEX_DIGEST_LEN || base16_decode(declared_identity, sizeof(declared_identity), - tok->args[0], HEX_DIGEST_LEN) < 0) { + id_hexdigest, HEX_DIGEST_LEN) < 0) { log_warn(LD_DIR, "Error decoding declared identity %s in " - "network-status vote.", escaped(tok->args[0])); + "network-status vote.", escaped(id_hexdigest)); goto err; } if (!(v = networkstatus_get_voter_by_id(ns, declared_identity))) { @@ -2515,11 +2722,15 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, "any declared directory source."); goto err; } - if (strlen(tok->args[1]) != HEX_DIGEST_LEN || - base16_decode(v->signing_key_digest, sizeof(v->signing_key_digest), - tok->args[1], HEX_DIGEST_LEN) < 0) { - log_warn(LD_DIR, "Error decoding declared digest %s in " - "network-status vote.", escaped(tok->args[1])); + sig = tor_malloc_zero(sizeof(document_signature_t)); + memcpy(sig->identity_digest, v->identity_digest, DIGEST_LEN); + sig->alg = alg; + if (strlen(sk_hexdigest) != HEX_DIGEST_LEN || + base16_decode(sig->signing_key_digest, sizeof(sig->signing_key_digest), + sk_hexdigest, HEX_DIGEST_LEN) < 0) { + log_warn(LD_DIR, "Error decoding declared signing key digest %s in " + "network-status vote.", escaped(sk_hexdigest)); + tor_free(sig); goto err; } @@ -2528,35 +2739,49 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, DIGEST_LEN)) { log_warn(LD_DIR, "Digest mismatch between declared and actual on " "network-status vote."); + tor_free(sig); goto err; } } + if (voter_get_sig_by_algorithm(v, sig->alg)) { + /* We already parsed a vote with this algorithm from this voter. Use the + first one. */ + log_fn(LOG_PROTOCOL_WARN, LD_DIR, "We received a networkstatus " + "that contains two votes from the same voter with the same " + "algorithm. Ignoring the second vote."); + tor_free(sig); + continue; + } + if (ns->type != NS_TYPE_CONSENSUS) { - if (check_signature_token(ns_digest, tok, ns->cert->signing_key, 0, - "network-status vote")) + if (check_signature_token(ns_digests.d[DIGEST_SHA1], DIGEST_LEN, + tok, ns->cert->signing_key, 0, + "network-status vote")) { + tor_free(sig); goto err; - v->good_signature = 1; + } + sig->good_signature = 1; } else { - if (tok->object_size >= INT_MAX) + if (tok->object_size >= INT_MAX) { + tor_free(sig); goto err; - /* We already parsed a vote from this voter. Use the first one. */ - if (v->signature) { - log_fn(LOG_PROTOCOL_WARN, LD_DIR, "We received a networkstatus " - "that contains two votes from the same voter. Ignoring " - "the second vote."); - continue; } - - v->signature = tor_memdup(tok->object_body, tok->object_size); - v->signature_len = (int) tok->object_size; + sig->signature = tor_memdup(tok->object_body, tok->object_size); + sig->signature_len = (int) tok->object_size; } + smartlist_add(v->sigs, sig); + ++n_signatures; - }); + } SMARTLIST_FOREACH_END(_tok); if (! n_signatures) { log_warn(LD_DIR, "No signatures on networkstatus vote."); goto err; + } else if (ns->type == NS_TYPE_VOTE && n_signatures != 1) { + log_warn(LD_DIR, "Received more than one signature on a " + "network-status vote."); + goto err; } if (eos_out) @@ -2564,27 +2789,31 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, goto done; err: - if (ns) - networkstatus_vote_free(ns); + dump_desc(s_dup, "v3 networkstatus"); + networkstatus_vote_free(ns); ns = NULL; done: if (tokens) { - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); } if (voter) { + if (voter->sigs) { + SMARTLIST_FOREACH(voter->sigs, document_signature_t *, sig, + document_signature_free(sig)); + smartlist_free(voter->sigs); + } tor_free(voter->nickname); tor_free(voter->address); tor_free(voter->contact); - tor_free(voter->signature); tor_free(voter); } if (rs_tokens) { - SMARTLIST_FOREACH(rs_tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(rs_tokens, directory_token_t *, t, token_clear(t)); smartlist_free(rs_tokens); } if (footer_tokens) { - SMARTLIST_FOREACH(footer_tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(footer_tokens, directory_token_t *, t, token_clear(t)); smartlist_free(footer_tokens); } if (area) { @@ -2597,6 +2826,35 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, return ns; } +/** Return the digests_t that holds the digests of the + * <b>flavor_name</b>-flavored networkstatus according to the detached + * signatures document <b>sigs</b>, allocating a new digests_t as neeeded. */ +static digests_t * +detached_get_digests(ns_detached_signatures_t *sigs, const char *flavor_name) +{ + digests_t *d = strmap_get(sigs->digests, flavor_name); + if (!d) { + d = tor_malloc_zero(sizeof(digests_t)); + strmap_set(sigs->digests, flavor_name, d); + } + return d; +} + +/** Return the list of signatures of the <b>flavor_name</b>-flavored + * networkstatus according to the detached signatures document <b>sigs</b>, + * allocating a new digests_t as neeeded. */ +static smartlist_t * +detached_get_signatures(ns_detached_signatures_t *sigs, + const char *flavor_name) +{ + smartlist_t *sl = strmap_get(sigs->signatures, flavor_name); + if (!sl) { + sl = smartlist_create(); + strmap_set(sigs->signatures, flavor_name, sl); + } + return sl; +} + /** Parse a detached v3 networkstatus signature document between <b>s</b> and * <b>eos</b> and return the result. Return -1 on failure. */ ns_detached_signatures_t * @@ -2606,10 +2864,13 @@ networkstatus_parse_detached_signatures(const char *s, const char *eos) * networkstatus_parse_vote_from_string(). */ directory_token_t *tok; memarea_t *area = NULL; + digests_t *digests; smartlist_t *tokens = smartlist_create(); ns_detached_signatures_t *sigs = tor_malloc_zero(sizeof(ns_detached_signatures_t)); + sigs->digests = strmap_new(); + sigs->signatures = strmap_new(); if (!eos) eos = s + strlen(s); @@ -2621,18 +2882,57 @@ networkstatus_parse_detached_signatures(const char *s, const char *eos) goto err; } - tok = find_by_keyword(tokens, K_CONSENSUS_DIGEST); - if (strlen(tok->args[0]) != HEX_DIGEST_LEN) { - log_warn(LD_DIR, "Wrong length on consensus-digest in detached " - "networkstatus signatures"); - goto err; - } - if (base16_decode(sigs->networkstatus_digest, DIGEST_LEN, - tok->args[0], strlen(tok->args[0])) < 0) { - log_warn(LD_DIR, "Bad encoding on on consensus-digest in detached " - "networkstatus signatures"); - goto err; - } + /* Grab all the digest-like tokens. */ + SMARTLIST_FOREACH_BEGIN(tokens, directory_token_t *, _tok) { + const char *algname; + digest_algorithm_t alg; + const char *flavor; + const char *hexdigest; + size_t expected_length; + + tok = _tok; + + if (tok->tp == K_CONSENSUS_DIGEST) { + algname = "sha1"; + alg = DIGEST_SHA1; + flavor = "ns"; + hexdigest = tok->args[0]; + } else if (tok->tp == K_ADDITIONAL_DIGEST) { + int a = crypto_digest_algorithm_parse_name(tok->args[1]); + if (a<0) { + log_warn(LD_DIR, "Unrecognized algorithm name %s", tok->args[0]); + continue; + } + alg = (digest_algorithm_t) a; + flavor = tok->args[0]; + algname = tok->args[1]; + hexdigest = tok->args[2]; + } else { + continue; + } + + expected_length = + (alg == DIGEST_SHA1) ? HEX_DIGEST_LEN : HEX_DIGEST256_LEN; + + if (strlen(hexdigest) != expected_length) { + log_warn(LD_DIR, "Wrong length on consensus-digest in detached " + "networkstatus signatures"); + goto err; + } + digests = detached_get_digests(sigs, flavor); + tor_assert(digests); + if (!tor_mem_is_zero(digests->d[alg], DIGEST256_LEN)) { + log_warn(LD_DIR, "Multiple digests for %s with %s on detached " + "signatures document", flavor, algname); + continue; + } + if (base16_decode(digests->d[alg], DIGEST256_LEN, + hexdigest, strlen(hexdigest)) < 0) { + log_warn(LD_DIR, "Bad encoding on consensus-digest in detached " + "networkstatus signatures"); + goto err; + } + } SMARTLIST_FOREACH_END(_tok); tok = find_by_keyword(tokens, K_VALID_AFTER); if (parse_iso_time(tok->args[0], &sigs->valid_after)) { @@ -2652,57 +2952,102 @@ networkstatus_parse_detached_signatures(const char *s, const char *eos) goto err; } - sigs->signatures = smartlist_create(); - SMARTLIST_FOREACH(tokens, directory_token_t *, _tok, - { - char id_digest[DIGEST_LEN]; - char sk_digest[DIGEST_LEN]; - networkstatus_voter_info_t *voter; + SMARTLIST_FOREACH_BEGIN(tokens, directory_token_t *, _tok) { + const char *id_hexdigest; + const char *sk_hexdigest; + const char *algname; + const char *flavor; + digest_algorithm_t alg; + + char id_digest[DIGEST_LEN]; + char sk_digest[DIGEST_LEN]; + smartlist_t *siglist; + document_signature_t *sig; + int is_duplicate; - tok = _tok; - if (tok->tp != K_DIRECTORY_SIGNATURE) - continue; + tok = _tok; + if (tok->tp == K_DIRECTORY_SIGNATURE) { tor_assert(tok->n_args >= 2); + flavor = "ns"; + algname = "sha1"; + id_hexdigest = tok->args[0]; + sk_hexdigest = tok->args[1]; + } else if (tok->tp == K_ADDITIONAL_SIGNATURE) { + tor_assert(tok->n_args >= 4); + flavor = tok->args[0]; + algname = tok->args[1]; + id_hexdigest = tok->args[2]; + sk_hexdigest = tok->args[3]; + } else { + continue; + } - if (!tok->object_type || - strcmp(tok->object_type, "SIGNATURE") || - tok->object_size < 128 || tok->object_size > 512) { - log_warn(LD_DIR, "Bad object type or length on directory-signature"); - goto err; + { + int a = crypto_digest_algorithm_parse_name(algname); + if (a<0) { + log_warn(LD_DIR, "Unrecognized algorithm name %s", algname); + continue; } + alg = (digest_algorithm_t) a; + } - if (strlen(tok->args[0]) != HEX_DIGEST_LEN || - base16_decode(id_digest, sizeof(id_digest), - tok->args[0], HEX_DIGEST_LEN) < 0) { - log_warn(LD_DIR, "Error decoding declared identity %s in " - "network-status vote.", escaped(tok->args[0])); - goto err; - } - if (strlen(tok->args[1]) != HEX_DIGEST_LEN || - base16_decode(sk_digest, sizeof(sk_digest), - tok->args[1], HEX_DIGEST_LEN) < 0) { - log_warn(LD_DIR, "Error decoding declared digest %s in " - "network-status vote.", escaped(tok->args[1])); - goto err; - } + if (!tok->object_type || + strcmp(tok->object_type, "SIGNATURE") || + tok->object_size < 128 || tok->object_size > 512) { + log_warn(LD_DIR, "Bad object type or length on directory-signature"); + goto err; + } - voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t)); - memcpy(voter->identity_digest, id_digest, DIGEST_LEN); - memcpy(voter->signing_key_digest, sk_digest, DIGEST_LEN); - if (tok->object_size >= INT_MAX) - goto err; - voter->signature = tor_memdup(tok->object_body, tok->object_size); - voter->signature_len = (int) tok->object_size; + if (strlen(id_hexdigest) != HEX_DIGEST_LEN || + base16_decode(id_digest, sizeof(id_digest), + id_hexdigest, HEX_DIGEST_LEN) < 0) { + log_warn(LD_DIR, "Error decoding declared identity %s in " + "network-status vote.", escaped(id_hexdigest)); + goto err; + } + if (strlen(sk_hexdigest) != HEX_DIGEST_LEN || + base16_decode(sk_digest, sizeof(sk_digest), + sk_hexdigest, HEX_DIGEST_LEN) < 0) { + log_warn(LD_DIR, "Error decoding declared signing key digest %s in " + "network-status vote.", escaped(sk_hexdigest)); + goto err; + } - smartlist_add(sigs->signatures, voter); + siglist = detached_get_signatures(sigs, flavor); + is_duplicate = 0; + SMARTLIST_FOREACH(siglist, document_signature_t *, s, { + if (s->alg == alg && + !memcmp(id_digest, s->identity_digest, DIGEST_LEN) && + !memcmp(sk_digest, s->signing_key_digest, DIGEST_LEN)) { + is_duplicate = 1; + } }); + if (is_duplicate) { + log_warn(LD_DIR, "Two signatures with identical keys and algorithm " + "found."); + continue; + } + + sig = tor_malloc_zero(sizeof(document_signature_t)); + sig->alg = alg; + memcpy(sig->identity_digest, id_digest, DIGEST_LEN); + memcpy(sig->signing_key_digest, sk_digest, DIGEST_LEN); + if (tok->object_size >= INT_MAX) { + tor_free(sig); + goto err; + } + sig->signature = tor_memdup(tok->object_body, tok->object_size); + sig->signature_len = (int) tok->object_size; + + smartlist_add(siglist, sig); + } SMARTLIST_FOREACH_END(_tok); goto done; err: ns_detached_signatures_free(sigs); sigs = NULL; done: - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); if (area) { DUMP_AREA(area, "detached signatures"); @@ -2758,7 +3103,7 @@ router_parse_addr_policy_item_from_string(const char *s, int assume_action) err: r = NULL; done: - token_free(tok); + token_clear(tok); if (area) { DUMP_AREA(area, "policy item"); memarea_drop_all(area); @@ -2881,9 +3226,8 @@ assert_addr_policy_ok(smartlist_t *lst) /** Free all resources allocated for <b>tok</b> */ static void -token_free(directory_token_t *tok) +token_clear(directory_token_t *tok) { - tor_assert(tok); if (tok->key) crypto_free_pk_env(tok->key); } @@ -2895,7 +3239,7 @@ token_free(directory_token_t *tok) #define RET_ERR(msg) \ STMT_BEGIN \ - if (tok) token_free(tok); \ + if (tok) token_clear(tok); \ tok = ALLOC_ZERO(sizeof(directory_token_t)); \ tok->tp = _ERR; \ tok->error = STRDUP(msg); \ @@ -3173,7 +3517,7 @@ tokenize_string(memarea_t *area, tok = get_next_token(area, s, end, table); if (tok->tp == _ERR) { log_warn(LD_DIR, "parse error: %s", tok->error); - token_free(tok); + token_clear(tok); return -1; } ++counts[tok->tp]; @@ -3287,17 +3631,11 @@ find_all_exitpolicy(smartlist_t *s) return out; } -/** Compute the SHA-1 digest of the substring of <b>s</b> taken from the first - * occurrence of <b>start_str</b> through the first instance of c after the - * first subsequent occurrence of <b>end_str</b>; store the 20-byte result in - * <b>digest</b>; return 0 on success. - * - * If no such substring exists, return -1. - */ static int -router_get_hash_impl(const char *s, char *digest, - const char *start_str, - const char *end_str, char end_c) +router_get_hash_impl_helper(const char *s, + const char *start_str, + const char *end_str, char end_c, + const char **start_out, const char **end_out) { char *start, *end; start = strstr(s, start_str); @@ -3323,14 +3661,212 @@ router_get_hash_impl(const char *s, char *digest, } ++end; - if (crypto_digest(digest, start, end-start)) { - log_warn(LD_BUG,"couldn't compute digest"); + *start_out = start; + *end_out = end; + return 0; +} + +/** Compute the digest of the substring of <b>s</b> taken from the first + * occurrence of <b>start_str</b> through the first instance of c after the + * first subsequent occurrence of <b>end_str</b>; store the 20-byte result in + * <b>digest</b>; return 0 on success. + * + * If no such substring exists, return -1. + */ +static int +router_get_hash_impl(const char *s, char *digest, + const char *start_str, + const char *end_str, char end_c, + digest_algorithm_t alg) +{ + const char *start=NULL, *end=NULL; + if (router_get_hash_impl_helper(s,start_str,end_str,end_c,&start,&end)<0) + return -1; + + if (alg == DIGEST_SHA1) { + if (crypto_digest(digest, start, end-start)) { + log_warn(LD_BUG,"couldn't compute digest"); + return -1; + } + } else { + if (crypto_digest256(digest, start, end-start, alg)) { + log_warn(LD_BUG,"couldn't compute digest"); + return -1; + } + } + + return 0; +} + +/** As router_get_hash_impl, but compute all hashes. */ +static int +router_get_hashes_impl(const char *s, digests_t *digests, + const char *start_str, + const char *end_str, char end_c) +{ + const char *start=NULL, *end=NULL; + if (router_get_hash_impl_helper(s,start_str,end_str,end_c,&start,&end)<0) + return -1; + + if (crypto_digest_all(digests, start, end-start)) { + log_warn(LD_BUG,"couldn't compute digests"); return -1; } return 0; } +/** Assuming that s starts with a microdesc, return the start of the + * *NEXT* one. Return NULL on "not found." */ +static const char * +find_start_of_next_microdesc(const char *s, const char *eos) +{ + int started_with_annotations; + s = eat_whitespace_eos(s, eos); + if (!s) + return NULL; + +#define CHECK_LENGTH() STMT_BEGIN \ + if (s+32 > eos) \ + return NULL; \ + STMT_END + +#define NEXT_LINE() STMT_BEGIN \ + s = memchr(s, '\n', eos-s); \ + if (!s || s+1 >= eos) \ + return NULL; \ + s++; \ + STMT_END + + CHECK_LENGTH(); + + started_with_annotations = (*s == '@'); + + if (started_with_annotations) { + /* Start by advancing to the first non-annotation line. */ + while (*s == '@') + NEXT_LINE(); + } + CHECK_LENGTH(); + + /* Now we should be pointed at an onion-key line. If we are, then skip + * it. */ + if (!strcmpstart(s, "onion-key")) + NEXT_LINE(); + + /* Okay, now we're pointed at the first line of the microdescriptor which is + not an annotation or onion-key. The next line that _is_ an annotation or + onion-key is the start of the next microdescriptor. */ + while (s+32 < eos) { + if (*s == '@' || !strcmpstart(s, "onion-key")) + return s; + NEXT_LINE(); + } + return NULL; + +#undef CHECK_LENGTH +#undef NEXT_LINE +} + +/** Parse as many microdescriptors as are found from the string starting at + * <b>s</b> and ending at <b>eos</b>. If allow_annotations is set, read any + * annotations we recognize and ignore ones we don't. If <b>copy_body</b> is + * true, then strdup the bodies of the microdescriptors. Return all newly + * parsed microdescriptors in a newly allocated smartlist_t. */ +smartlist_t * +microdescs_parse_from_string(const char *s, const char *eos, + int allow_annotations, int copy_body) +{ + smartlist_t *tokens; + smartlist_t *result; + microdesc_t *md = NULL; + memarea_t *area; + const char *start = s; + const char *start_of_next_microdesc; + int flags = allow_annotations ? TS_ANNOTATIONS_OK : 0; + + directory_token_t *tok; + + if (!eos) + eos = s + strlen(s); + + s = eat_whitespace_eos(s, eos); + area = memarea_new(); + result = smartlist_create(); + tokens = smartlist_create(); + + while (s < eos) { + start_of_next_microdesc = find_start_of_next_microdesc(s, eos); + if (!start_of_next_microdesc) + start_of_next_microdesc = eos; + + if (tokenize_string(area, s, start_of_next_microdesc, tokens, + microdesc_token_table, flags)) { + log_warn(LD_DIR, "Unparseable microdescriptor"); + goto next; + } + + md = tor_malloc_zero(sizeof(microdesc_t)); + { + const char *cp = tor_memstr(s, start_of_next_microdesc-s, + "onion-key"); + tor_assert(cp); + + md->bodylen = start_of_next_microdesc - cp; + if (copy_body) + md->body = tor_strndup(cp, md->bodylen); + else + md->body = (char*)cp; + md->off = cp - start; + } + + if ((tok = find_opt_by_keyword(tokens, A_LAST_LISTED))) { + if (parse_iso_time(tok->args[0], &md->last_listed)) { + log_warn(LD_DIR, "Bad last-listed time in microdescriptor"); + goto next; + } + } + + tok = find_by_keyword(tokens, K_ONION_KEY); + md->onion_pkey = tok->key; + tok->key = NULL; + + if ((tok = find_opt_by_keyword(tokens, K_FAMILY))) { + int i; + md->family = smartlist_create(); + for (i=0;i<tok->n_args;++i) { + if (!is_legal_nickname_or_hexdigest(tok->args[i])) { + log_warn(LD_DIR, "Illegal nickname %s in family line", + escaped(tok->args[i])); + goto next; + } + smartlist_add(md->family, tor_strdup(tok->args[i])); + } + } + + if ((tok = find_opt_by_keyword(tokens, K_P))) { + md->exitsummary = tor_strdup(tok->args[0]); + } + + crypto_digest256(md->digest, md->body, md->bodylen, DIGEST_SHA256); + + smartlist_add(result, md); + + md = NULL; + next: + microdesc_free(md); + + memarea_clear(area); + smartlist_clear(tokens); + s = start_of_next_microdesc; + } + + memarea_drop_all(area); + smartlist_free(tokens); + + return result; +} + /** Parse the Tor version of the platform string <b>platform</b>, * and compare it to the version in <b>cutoff</b>. Return 1 if * the router is at least as new as the cutoff, else return 0. @@ -3355,7 +3891,7 @@ tor_version_as_new_as(const char *platform, const char *cutoff) if (!*start) return 0; s = (char *)find_whitespace(start); /* also finds '\0', which is fine */ s2 = (char*)eat_whitespace(s); - if (!strcmpstart(s2, "(r")) + if (!strcmpstart(s2, "(r") || !strcmpstart(s2, "(git-")) s = (char*)find_whitespace(s2); if ((size_t)(s-start+1) >= sizeof(tmp)) /* too big, no */ @@ -3451,6 +3987,23 @@ tor_version_parse(const char *s, tor_version_t *out) if (!strcmpstart(cp, "(r")) { cp += 2; out->svn_revision = (int) strtol(cp,&eos,10); + } else if (!strcmpstart(cp, "(git-")) { + char *close_paren = strchr(cp, ')'); + int hexlen; + char digest[DIGEST_LEN]; + if (! close_paren) + return -1; + cp += 5; + if (close_paren-cp > HEX_DIGEST_LEN) + return -1; + hexlen = (int)(close_paren-cp); + memset(digest, 0, sizeof(digest)); + if ( hexlen == 0 || (hexlen % 2) == 1) + return -1; + if (base16_decode(digest, hexlen/2, cp, hexlen)) + return -1; + memcpy(out->git_tag, digest, hexlen/2); + out->git_tag_len = hexlen/2; } return 0; @@ -3476,8 +4029,14 @@ tor_version_compare(tor_version_t *a, tor_version_t *b) return i; else if ((i = strcmp(a->status_tag, b->status_tag))) return i; + else if ((i = a->svn_revision - b->svn_revision)) + return i; + else if ((i = a->git_tag_len - b->git_tag_len)) + return i; + else if (a->git_tag_len) + return memcmp(a->git_tag, b->git_tag, a->git_tag_len); else - return a->svn_revision - b->svn_revision; + return 0; } /** Return true iff versions <b>a</b> and <b>b</b> belong to the same series. @@ -3566,7 +4125,7 @@ rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out, /* Compute descriptor hash for later validation. */ if (router_get_hash_impl(desc, desc_hash, "rendezvous-service-descriptor ", - "\nsignature", '\n') < 0) { + "\nsignature", '\n', DIGEST_SHA1) < 0) { log_warn(LD_REND, "Couldn't compute descriptor hash."); goto err; } @@ -3685,7 +4244,7 @@ rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out, /* Parse and verify signature. */ tok = find_by_keyword(tokens, R_SIGNATURE); note_crypto_pk_op(VERIFY_RTR); - if (check_signature_token(desc_hash, tok, result->pk, 0, + if (check_signature_token(desc_hash, DIGEST_LEN, tok, result->pk, 0, "v2 rendezvous service descriptor") < 0) goto err; /* Verify that descriptor ID belongs to public key and secret ID part. */ @@ -3699,12 +4258,11 @@ rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out, } goto done; err: - if (result) - rend_service_descriptor_free(result); + rend_service_descriptor_free(result); result = NULL; done: if (tokens) { - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); } if (area) @@ -3862,7 +4420,7 @@ rend_parse_introduction_points(rend_service_descriptor_t *parsed, eos = eos+1; tor_assert(eos <= intro_points_encoded+intro_points_encoded_size); /* Free tokens and clear token list. */ - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_clear(tokens); memarea_clear(area); /* Tokenize string. */ @@ -3935,7 +4493,7 @@ rend_parse_introduction_points(rend_service_descriptor_t *parsed, done: /* Free tokens and clear token list. */ - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); if (area) memarea_drop_all(area); @@ -3974,7 +4532,7 @@ rend_parse_client_keys(strmap_t *parsed_clients, const char *ckstr) else eos = eos + 1; /* Free tokens and clear token list. */ - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_clear(tokens); memarea_clear(area); /* Tokenize string. */ @@ -4046,7 +4604,7 @@ rend_parse_client_keys(strmap_t *parsed_clients, const char *ckstr) result = -1; done: /* Free tokens and clear token list. */ - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); if (area) memarea_drop_all(area); diff --git a/src/or/test.c b/src/or/test.c deleted file mode 100644 index e06dd5951f..0000000000 --- a/src/or/test.c +++ /dev/null @@ -1,4900 +0,0 @@ -/* Copyright (c) 2001-2004, Roger Dingledine. - * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2009, The Tor Project, Inc. */ -/* See LICENSE for licensing information */ - -/* Ordinarily defined in tor_main.c; this bit is just here to provide one - * since we're not linking to tor_main.c */ -const char tor_svn_revision[] = ""; - -/** - * \file test.c - * \brief Unit tests for many pieces of the lower level Tor modules. - **/ - -#include "orconfig.h" - -#include <stdio.h> -#ifdef HAVE_FCNTL_H -#include <fcntl.h> -#endif - -#ifdef MS_WINDOWS -/* For mkdir() */ -#include <direct.h> -#else -#include <dirent.h> -#endif - -/* These macros pull in declarations for some functions and structures that - * are typically file-private. */ -#define BUFFERS_PRIVATE -#define CONFIG_PRIVATE -#define CONTROL_PRIVATE -#define CRYPTO_PRIVATE -#define DIRSERV_PRIVATE -#define DIRVOTE_PRIVATE -#define GEOIP_PRIVATE -#define MEMPOOL_PRIVATE -#define ROUTER_PRIVATE - -#include "or.h" -#include "test.h" -#include "torgzip.h" -#include "mempool.h" -#include "memarea.h" - -#ifdef USE_DMALLOC -#include <dmalloc.h> -#include <openssl/crypto.h> -#endif - -/** Set to true if any unit test has failed. Mostly, this is set by the macros - * in test.h */ -int have_failed = 0; - -/** Temporary directory (set up by setup_directory) under which we store all - * our files during testing. */ -static char temp_dir[256]; - -/** Select and create the temporary directory we'll use to run our unit tests. - * Store it in <b>temp_dir</b>. Exit immediately if we can't create it. - * idempotent. */ -static void -setup_directory(void) -{ - static int is_setup = 0; - int r; - if (is_setup) return; - -#ifdef MS_WINDOWS - // XXXX - tor_snprintf(temp_dir, sizeof(temp_dir), - "c:\\windows\\temp\\tor_test_%d", (int)getpid()); - r = mkdir(temp_dir); -#else - tor_snprintf(temp_dir, sizeof(temp_dir), "/tmp/tor_test_%d", (int) getpid()); - r = mkdir(temp_dir, 0700); -#endif - if (r) { - fprintf(stderr, "Can't create directory %s:", temp_dir); - perror(""); - exit(1); - } - is_setup = 1; -} - -/** Return a filename relative to our testing temporary directory */ -static const char * -get_fname(const char *name) -{ - static char buf[1024]; - setup_directory(); - tor_snprintf(buf,sizeof(buf),"%s/%s",temp_dir,name); - return buf; -} - -/** Remove all files stored under the temporary directory, and the directory - * itself. */ -static void -remove_directory(void) -{ - smartlist_t *elements = tor_listdir(temp_dir); - if (elements) { - SMARTLIST_FOREACH(elements, const char *, cp, - { - size_t len = strlen(cp)+strlen(temp_dir)+16; - char *tmp = tor_malloc(len); - tor_snprintf(tmp, len, "%s"PATH_SEPARATOR"%s", temp_dir, cp); - unlink(tmp); - tor_free(tmp); - }); - SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp)); - smartlist_free(elements); - } - rmdir(temp_dir); -} - -/** Define this if unit tests spend too much time generating public keys*/ -#undef CACHE_GENERATED_KEYS - -static crypto_pk_env_t *pregen_keys[5] = {NULL, NULL, NULL, NULL, NULL}; -#define N_PREGEN_KEYS ((int)(sizeof(pregen_keys)/sizeof(pregen_keys[0]))) - -/** Generate and return a new keypair for use in unit tests. If we're using - * the key cache optimization, we might reuse keys: we only guarantee that - * keys made with distinct values for <b>idx</b> are different. The value of - * <b>idx</b> must be at least 0, and less than N_PREGEN_KEYS. */ -static crypto_pk_env_t * -pk_generate(int idx) -{ -#ifdef CACHE_GENERATED_KEYS - tor_assert(idx < N_PREGEN_KEYS); - if (! pregen_keys[idx]) { - pregen_keys[idx] = crypto_new_pk_env(); - tor_assert(!crypto_pk_generate_key(pregen_keys[idx])); - } - return crypto_pk_dup_key(pregen_keys[idx]); -#else - crypto_pk_env_t *result; - (void) idx; - result = crypto_new_pk_env(); - tor_assert(!crypto_pk_generate_key(result)); - return result; -#endif -} - -/** Free all storage used for the cached key optimization. */ -static void -free_pregenerated_keys(void) -{ - unsigned idx; - for (idx = 0; idx < N_PREGEN_KEYS; ++idx) { - if (pregen_keys[idx]) { - crypto_free_pk_env(pregen_keys[idx]); - pregen_keys[idx] = NULL; - } - } -} - -/** Run unit tests for buffers.c */ -static void -test_buffers(void) -{ - char str[256]; - char str2[256]; - - buf_t *buf = NULL, *buf2 = NULL; - const char *cp; - - int j; - size_t r; - - /**** - * buf_new - ****/ - if (!(buf = buf_new())) - test_fail(); - - //test_eq(buf_capacity(buf), 4096); - test_eq(buf_datalen(buf), 0); - - /**** - * General pointer frobbing - */ - for (j=0;j<256;++j) { - str[j] = (char)j; - } - write_to_buf(str, 256, buf); - write_to_buf(str, 256, buf); - test_eq(buf_datalen(buf), 512); - fetch_from_buf(str2, 200, buf); - test_memeq(str, str2, 200); - test_eq(buf_datalen(buf), 312); - memset(str2, 0, sizeof(str2)); - - fetch_from_buf(str2, 256, buf); - test_memeq(str+200, str2, 56); - test_memeq(str, str2+56, 200); - test_eq(buf_datalen(buf), 56); - memset(str2, 0, sizeof(str2)); - /* Okay, now we should be 512 bytes into the 4096-byte buffer. If we add - * another 3584 bytes, we hit the end. */ - for (j=0;j<15;++j) { - write_to_buf(str, 256, buf); - } - assert_buf_ok(buf); - test_eq(buf_datalen(buf), 3896); - fetch_from_buf(str2, 56, buf); - test_eq(buf_datalen(buf), 3840); - test_memeq(str+200, str2, 56); - for (j=0;j<15;++j) { - memset(str2, 0, sizeof(str2)); - fetch_from_buf(str2, 256, buf); - test_memeq(str, str2, 256); - } - test_eq(buf_datalen(buf), 0); - buf_free(buf); - buf = NULL; - - /* Okay, now make sure growing can work. */ - buf = buf_new_with_capacity(16); - //test_eq(buf_capacity(buf), 16); - write_to_buf(str+1, 255, buf); - //test_eq(buf_capacity(buf), 256); - fetch_from_buf(str2, 254, buf); - test_memeq(str+1, str2, 254); - //test_eq(buf_capacity(buf), 256); - assert_buf_ok(buf); - write_to_buf(str, 32, buf); - //test_eq(buf_capacity(buf), 256); - assert_buf_ok(buf); - write_to_buf(str, 256, buf); - assert_buf_ok(buf); - //test_eq(buf_capacity(buf), 512); - test_eq(buf_datalen(buf), 33+256); - fetch_from_buf(str2, 33, buf); - test_eq(*str2, str[255]); - - test_memeq(str2+1, str, 32); - //test_eq(buf_capacity(buf), 512); - test_eq(buf_datalen(buf), 256); - fetch_from_buf(str2, 256, buf); - test_memeq(str, str2, 256); - - /* now try shrinking: case 1. */ - buf_free(buf); - buf = buf_new_with_capacity(33668); - for (j=0;j<67;++j) { - write_to_buf(str,255, buf); - } - //test_eq(buf_capacity(buf), 33668); - test_eq(buf_datalen(buf), 17085); - for (j=0; j < 40; ++j) { - fetch_from_buf(str2, 255,buf); - test_memeq(str2, str, 255); - } - - /* now try shrinking: case 2. */ - buf_free(buf); - buf = buf_new_with_capacity(33668); - for (j=0;j<67;++j) { - write_to_buf(str,255, buf); - } - for (j=0; j < 20; ++j) { - fetch_from_buf(str2, 255,buf); - test_memeq(str2, str, 255); - } - for (j=0;j<80;++j) { - write_to_buf(str,255, buf); - } - //test_eq(buf_capacity(buf),33668); - for (j=0; j < 120; ++j) { - fetch_from_buf(str2, 255,buf); - test_memeq(str2, str, 255); - } - - /* Move from buf to buf. */ - buf_free(buf); - buf = buf_new_with_capacity(4096); - buf2 = buf_new_with_capacity(4096); - for (j=0;j<100;++j) - write_to_buf(str, 255, buf); - test_eq(buf_datalen(buf), 25500); - for (j=0;j<100;++j) { - r = 10; - move_buf_to_buf(buf2, buf, &r); - test_eq(r, 0); - } - test_eq(buf_datalen(buf), 24500); - test_eq(buf_datalen(buf2), 1000); - for (j=0;j<3;++j) { - fetch_from_buf(str2, 255, buf2); - test_memeq(str2, str, 255); - } - r = 8192; /*big move*/ - move_buf_to_buf(buf2, buf, &r); - test_eq(r, 0); - r = 30000; /* incomplete move */ - move_buf_to_buf(buf2, buf, &r); - test_eq(r, 13692); - for (j=0;j<97;++j) { - fetch_from_buf(str2, 255, buf2); - test_memeq(str2, str, 255); - } - buf_free(buf); - buf_free(buf2); - buf = buf2 = NULL; - - buf = buf_new_with_capacity(5); - cp = "Testing. This is a moderately long Testing string."; - for (j = 0; cp[j]; j++) - write_to_buf(cp+j, 1, buf); - test_eq(0, buf_find_string_offset(buf, "Testing", 7)); - test_eq(1, buf_find_string_offset(buf, "esting", 6)); - test_eq(1, buf_find_string_offset(buf, "est", 3)); - test_eq(39, buf_find_string_offset(buf, "ing str", 7)); - test_eq(35, buf_find_string_offset(buf, "Testing str", 11)); - test_eq(32, buf_find_string_offset(buf, "ng ", 3)); - test_eq(43, buf_find_string_offset(buf, "string.", 7)); - test_eq(-1, buf_find_string_offset(buf, "shrdlu", 6)); - test_eq(-1, buf_find_string_offset(buf, "Testing thing", 13)); - test_eq(-1, buf_find_string_offset(buf, "ngx", 3)); - buf_free(buf); - buf = NULL; - -#if 0 - { - int s; - int eof; - int i; - buf_t *buf2; - /**** - * read_to_buf - ****/ - s = open(get_fname("data"), O_WRONLY|O_CREAT|O_TRUNC, 0600); - write(s, str, 256); - close(s); - - s = open(get_fname("data"), O_RDONLY, 0); - eof = 0; - errno = 0; /* XXXX */ - i = read_to_buf(s, 10, buf, &eof); - printf("%s\n", strerror(errno)); - test_eq(i, 10); - test_eq(eof, 0); - //test_eq(buf_capacity(buf), 4096); - test_eq(buf_datalen(buf), 10); - - test_memeq(str, (char*)_buf_peek_raw_buffer(buf), 10); - - /* Test reading 0 bytes. */ - i = read_to_buf(s, 0, buf, &eof); - //test_eq(buf_capacity(buf), 512*1024); - test_eq(buf_datalen(buf), 10); - test_eq(eof, 0); - test_eq(i, 0); - - /* Now test when buffer is filled exactly. */ - buf2 = buf_new_with_capacity(6); - i = read_to_buf(s, 6, buf2, &eof); - //test_eq(buf_capacity(buf2), 6); - test_eq(buf_datalen(buf2), 6); - test_eq(eof, 0); - test_eq(i, 6); - test_memeq(str+10, (char*)_buf_peek_raw_buffer(buf2), 6); - buf_free(buf2); - buf2 = NULL; - - /* Now test when buffer is filled with more data to read. */ - buf2 = buf_new_with_capacity(32); - i = read_to_buf(s, 128, buf2, &eof); - //test_eq(buf_capacity(buf2), 128); - test_eq(buf_datalen(buf2), 32); - test_eq(eof, 0); - test_eq(i, 32); - buf_free(buf2); - buf2 = NULL; - - /* Now read to eof. */ - test_assert(buf_capacity(buf) > 256); - i = read_to_buf(s, 1024, buf, &eof); - test_eq(i, (256-32-10-6)); - test_eq(buf_capacity(buf), MAX_BUF_SIZE); - test_eq(buf_datalen(buf), 256-6-32); - test_memeq(str, (char*)_buf_peek_raw_buffer(buf), 10); /* XXX Check rest. */ - test_eq(eof, 0); - - i = read_to_buf(s, 1024, buf, &eof); - test_eq(i, 0); - test_eq(buf_capacity(buf), MAX_BUF_SIZE); - test_eq(buf_datalen(buf), 256-6-32); - test_eq(eof, 1); - } -#endif - - done: - if (buf) - buf_free(buf); - if (buf2) - buf_free(buf2); -} - -/** Run unit tests for Diffie-Hellman functionality. */ -static void -test_crypto_dh(void) -{ - crypto_dh_env_t *dh1 = crypto_dh_new(); - crypto_dh_env_t *dh2 = crypto_dh_new(); - char p1[DH_BYTES]; - char p2[DH_BYTES]; - char s1[DH_BYTES]; - char s2[DH_BYTES]; - ssize_t s1len, s2len; - - test_eq(crypto_dh_get_bytes(dh1), DH_BYTES); - test_eq(crypto_dh_get_bytes(dh2), DH_BYTES); - - memset(p1, 0, DH_BYTES); - memset(p2, 0, DH_BYTES); - test_memeq(p1, p2, DH_BYTES); - test_assert(! crypto_dh_get_public(dh1, p1, DH_BYTES)); - test_memneq(p1, p2, DH_BYTES); - test_assert(! crypto_dh_get_public(dh2, p2, DH_BYTES)); - test_memneq(p1, p2, DH_BYTES); - - memset(s1, 0, DH_BYTES); - memset(s2, 0xFF, DH_BYTES); - s1len = crypto_dh_compute_secret(dh1, p2, DH_BYTES, s1, 50); - s2len = crypto_dh_compute_secret(dh2, p1, DH_BYTES, s2, 50); - test_assert(s1len > 0); - test_eq(s1len, s2len); - test_memeq(s1, s2, s1len); - - { - /* XXXX Now fabricate some bad values and make sure they get caught, - * Check 0, 1, N-1, >= N, etc. - */ - } - - done: - crypto_dh_free(dh1); - crypto_dh_free(dh2); -} - -/** Run unit tests for our random number generation function and its wrappers. - */ -static void -test_crypto_rng(void) -{ - int i, j, allok; - char data1[100], data2[100]; - - /* Try out RNG. */ - test_assert(! crypto_seed_rng(0)); - crypto_rand(data1, 100); - crypto_rand(data2, 100); - test_memneq(data1,data2,100); - allok = 1; - for (i = 0; i < 100; ++i) { - uint64_t big; - char *host; - j = crypto_rand_int(100); - if (i < 0 || i >= 100) - allok = 0; - big = crypto_rand_uint64(U64_LITERAL(1)<<40); - if (big >= (U64_LITERAL(1)<<40)) - allok = 0; - big = crypto_rand_uint64(U64_LITERAL(5)); - if (big >= 5) - allok = 0; - host = crypto_random_hostname(3,8,"www.",".onion"); - if (strcmpstart(host,"www.") || - strcmpend(host,".onion") || - strlen(host) < 13 || - strlen(host) > 18) - allok = 0; - tor_free(host); - } - test_assert(allok); - done: - ; -} - -/** Run unit tests for our AES functionality */ -static void -test_crypto_aes(void) -{ - char *data1 = NULL, *data2 = NULL, *data3 = NULL; - crypto_cipher_env_t *env1 = NULL, *env2 = NULL; - int i, j; - - data1 = tor_malloc(1024); - data2 = tor_malloc(1024); - data3 = tor_malloc(1024); - - /* Now, test encryption and decryption with stream cipher. */ - data1[0]='\0'; - for (i = 1023; i>0; i -= 35) - strncat(data1, "Now is the time for all good onions", i); - - memset(data2, 0, 1024); - memset(data3, 0, 1024); - env1 = crypto_new_cipher_env(); - test_neq(env1, 0); - env2 = crypto_new_cipher_env(); - test_neq(env2, 0); - j = crypto_cipher_generate_key(env1); - crypto_cipher_set_key(env2, crypto_cipher_get_key(env1)); - crypto_cipher_encrypt_init_cipher(env1); - crypto_cipher_decrypt_init_cipher(env2); - - /* Try encrypting 512 chars. */ - crypto_cipher_encrypt(env1, data2, data1, 512); - crypto_cipher_decrypt(env2, data3, data2, 512); - test_memeq(data1, data3, 512); - test_memneq(data1, data2, 512); - - /* Now encrypt 1 at a time, and get 1 at a time. */ - for (j = 512; j < 560; ++j) { - crypto_cipher_encrypt(env1, data2+j, data1+j, 1); - } - for (j = 512; j < 560; ++j) { - crypto_cipher_decrypt(env2, data3+j, data2+j, 1); - } - test_memeq(data1, data3, 560); - /* Now encrypt 3 at a time, and get 5 at a time. */ - for (j = 560; j < 1024-5; j += 3) { - crypto_cipher_encrypt(env1, data2+j, data1+j, 3); - } - for (j = 560; j < 1024-5; j += 5) { - crypto_cipher_decrypt(env2, data3+j, data2+j, 5); - } - test_memeq(data1, data3, 1024-5); - /* Now make sure that when we encrypt with different chunk sizes, we get - the same results. */ - crypto_free_cipher_env(env2); - env2 = NULL; - - memset(data3, 0, 1024); - env2 = crypto_new_cipher_env(); - test_neq(env2, 0); - crypto_cipher_set_key(env2, crypto_cipher_get_key(env1)); - crypto_cipher_encrypt_init_cipher(env2); - for (j = 0; j < 1024-16; j += 17) { - crypto_cipher_encrypt(env2, data3+j, data1+j, 17); - } - for (j= 0; j < 1024-16; ++j) { - if (data2[j] != data3[j]) { - printf("%d: %d\t%d\n", j, (int) data2[j], (int) data3[j]); - } - } - test_memeq(data2, data3, 1024-16); - crypto_free_cipher_env(env1); - env1 = NULL; - crypto_free_cipher_env(env2); - env2 = NULL; - - /* NIST test vector for aes. */ - env1 = crypto_new_cipher_env(); /* IV starts at 0 */ - crypto_cipher_set_key(env1, "\x80\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00"); - crypto_cipher_encrypt_init_cipher(env1); - crypto_cipher_encrypt(env1, data1, - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00", 16); - test_memeq_hex(data1, "0EDD33D3C621E546455BD8BA1418BEC8"); - - /* Now test rollover. All these values are originally from a python - * script. */ - crypto_cipher_set_iv(env1, "\x00\x00\x00\x00\x00\x00\x00\x00" - "\xff\xff\xff\xff\xff\xff\xff\xff"); - memset(data2, 0, 1024); - crypto_cipher_encrypt(env1, data1, data2, 32); - test_memeq_hex(data1, "335fe6da56f843199066c14a00a40231" - "cdd0b917dbc7186908a6bfb5ffd574d3"); - - crypto_cipher_set_iv(env1, "\x00\x00\x00\x00\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff"); - memset(data2, 0, 1024); - crypto_cipher_encrypt(env1, data1, data2, 32); - test_memeq_hex(data1, "e627c6423fa2d77832a02b2794094b73" - "3e63c721df790d2c6469cc1953a3ffac"); - - crypto_cipher_set_iv(env1, "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff"); - memset(data2, 0, 1024); - crypto_cipher_encrypt(env1, data1, data2, 32); - test_memeq_hex(data1, "2aed2bff0de54f9328efd070bf48f70a" - "0EDD33D3C621E546455BD8BA1418BEC8"); - - /* Now check rollover on inplace cipher. */ - crypto_cipher_set_iv(env1, "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff"); - crypto_cipher_crypt_inplace(env1, data2, 64); - test_memeq_hex(data2, "2aed2bff0de54f9328efd070bf48f70a" - "0EDD33D3C621E546455BD8BA1418BEC8" - "93e2c5243d6839eac58503919192f7ae" - "1908e67cafa08d508816659c2e693191"); - crypto_cipher_set_iv(env1, "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff"); - crypto_cipher_crypt_inplace(env1, data2, 64); - test_assert(tor_mem_is_zero(data2, 64)); - - done: - if (env1) - crypto_free_cipher_env(env1); - if (env2) - crypto_free_cipher_env(env2); - tor_free(data1); - tor_free(data2); - tor_free(data3); -} - -/** Run unit tests for our SHA-1 functionality */ -static void -test_crypto_sha(void) -{ - crypto_digest_env_t *d1 = NULL, *d2 = NULL; - int i; - char key[80]; - char digest[20]; - char data[50]; - char d_out1[DIGEST_LEN], d_out2[DIGEST_LEN]; - - /* Test SHA-1 with a test vector from the specification. */ - i = crypto_digest(data, "abc", 3); - test_memeq_hex(data, "A9993E364706816ABA3E25717850C26C9CD0D89D"); - - /* Test HMAC-SHA-1 with test cases from RFC2202. */ - - /* Case 1. */ - memset(key, 0x0b, 20); - crypto_hmac_sha1(digest, key, 20, "Hi There", 8); - test_streq(hex_str(digest, 20), - "B617318655057264E28BC0B6FB378C8EF146BE00"); - /* Case 2. */ - crypto_hmac_sha1(digest, "Jefe", 4, "what do ya want for nothing?", 28); - test_streq(hex_str(digest, 20), - "EFFCDF6AE5EB2FA2D27416D5F184DF9C259A7C79"); - - /* Case 4. */ - base16_decode(key, 25, - "0102030405060708090a0b0c0d0e0f10111213141516171819", 50); - memset(data, 0xcd, 50); - crypto_hmac_sha1(digest, key, 25, data, 50); - test_streq(hex_str(digest, 20), - "4C9007F4026250C6BC8414F9BF50C86C2D7235DA"); - - /* Case . */ - memset(key, 0xaa, 80); - crypto_hmac_sha1(digest, key, 80, - "Test Using Larger Than Block-Size Key - Hash Key First", - 54); - test_streq(hex_str(digest, 20), - "AA4AE5E15272D00E95705637CE8A3B55ED402112"); - - /* Incremental digest code. */ - d1 = crypto_new_digest_env(); - test_assert(d1); - crypto_digest_add_bytes(d1, "abcdef", 6); - d2 = crypto_digest_dup(d1); - test_assert(d2); - crypto_digest_add_bytes(d2, "ghijkl", 6); - crypto_digest_get_digest(d2, d_out1, sizeof(d_out1)); - crypto_digest(d_out2, "abcdefghijkl", 12); - test_memeq(d_out1, d_out2, DIGEST_LEN); - crypto_digest_assign(d2, d1); - crypto_digest_add_bytes(d2, "mno", 3); - crypto_digest_get_digest(d2, d_out1, sizeof(d_out1)); - crypto_digest(d_out2, "abcdefmno", 9); - test_memeq(d_out1, d_out2, DIGEST_LEN); - crypto_digest_get_digest(d1, d_out1, sizeof(d_out1)); - crypto_digest(d_out2, "abcdef", 6); - test_memeq(d_out1, d_out2, DIGEST_LEN); - - done: - if (d1) - crypto_free_digest_env(d1); - if (d2) - crypto_free_digest_env(d2); -} - -/** Run unit tests for our public key crypto functions */ -static void -test_crypto_pk(void) -{ - crypto_pk_env_t *pk1 = NULL, *pk2 = NULL; - char *encoded = NULL; - char data1[1024], data2[1024], data3[1024]; - size_t size; - int i, j, p, len; - - /* Public-key ciphers */ - pk1 = pk_generate(0); - pk2 = crypto_new_pk_env(); - test_assert(pk1 && pk2); - test_assert(! crypto_pk_write_public_key_to_string(pk1, &encoded, &size)); - test_assert(! crypto_pk_read_public_key_from_string(pk2, encoded, size)); - test_eq(0, crypto_pk_cmp_keys(pk1, pk2)); - - test_eq(128, crypto_pk_keysize(pk1)); - test_eq(128, crypto_pk_keysize(pk2)); - - test_eq(128, crypto_pk_public_encrypt(pk2, data1, "Hello whirled.", 15, - PK_PKCS1_OAEP_PADDING)); - test_eq(128, crypto_pk_public_encrypt(pk1, data2, "Hello whirled.", 15, - PK_PKCS1_OAEP_PADDING)); - /* oaep padding should make encryption not match */ - test_memneq(data1, data2, 128); - test_eq(15, crypto_pk_private_decrypt(pk1, data3, data1, 128, - PK_PKCS1_OAEP_PADDING,1)); - test_streq(data3, "Hello whirled."); - memset(data3, 0, 1024); - test_eq(15, crypto_pk_private_decrypt(pk1, data3, data2, 128, - PK_PKCS1_OAEP_PADDING,1)); - test_streq(data3, "Hello whirled."); - /* Can't decrypt with public key. */ - test_eq(-1, crypto_pk_private_decrypt(pk2, data3, data2, 128, - PK_PKCS1_OAEP_PADDING,1)); - /* Try again with bad padding */ - memcpy(data2+1, "XYZZY", 5); /* This has fails ~ once-in-2^40 */ - test_eq(-1, crypto_pk_private_decrypt(pk1, data3, data2, 128, - PK_PKCS1_OAEP_PADDING,1)); - - /* File operations: save and load private key */ - test_assert(! crypto_pk_write_private_key_to_filename(pk1, - get_fname("pkey1"))); - /* failing case for read: can't read. */ - test_assert(crypto_pk_read_private_key_from_filename(pk2, - get_fname("xyzzy")) < 0); - write_str_to_file(get_fname("xyzzy"), "foobar", 6); - /* Failing case for read: no key. */ - test_assert(crypto_pk_read_private_key_from_filename(pk2, - get_fname("xyzzy")) < 0); - test_assert(! crypto_pk_read_private_key_from_filename(pk2, - get_fname("pkey1"))); - test_eq(15, crypto_pk_private_decrypt(pk2, data3, data1, 128, - PK_PKCS1_OAEP_PADDING,1)); - - /* Now try signing. */ - strlcpy(data1, "Ossifrage", 1024); - test_eq(128, crypto_pk_private_sign(pk1, data2, data1, 10)); - test_eq(10, crypto_pk_public_checksig(pk1, data3, data2, 128)); - test_streq(data3, "Ossifrage"); - /* Try signing digests. */ - test_eq(128, crypto_pk_private_sign_digest(pk1, data2, data1, 10)); - test_eq(20, crypto_pk_public_checksig(pk1, data3, data2, 128)); - test_eq(0, crypto_pk_public_checksig_digest(pk1, data1, 10, data2, 128)); - test_eq(-1, crypto_pk_public_checksig_digest(pk1, data1, 11, data2, 128)); - /*XXXX test failed signing*/ - - /* Try encoding */ - crypto_free_pk_env(pk2); - pk2 = NULL; - i = crypto_pk_asn1_encode(pk1, data1, 1024); - test_assert(i>0); - pk2 = crypto_pk_asn1_decode(data1, i); - test_assert(crypto_pk_cmp_keys(pk1,pk2) == 0); - - /* Try with hybrid encryption wrappers. */ - crypto_rand(data1, 1024); - for (i = 0; i < 3; ++i) { - for (j = 85; j < 140; ++j) { - memset(data2,0,1024); - memset(data3,0,1024); - if (i == 0 && j < 129) - continue; - p = (i==0)?PK_NO_PADDING: - (i==1)?PK_PKCS1_PADDING:PK_PKCS1_OAEP_PADDING; - len = crypto_pk_public_hybrid_encrypt(pk1,data2,data1,j,p,0); - test_assert(len>=0); - len = crypto_pk_private_hybrid_decrypt(pk1,data3,data2,len,p,1); - test_eq(len,j); - test_memeq(data1,data3,j); - } - } - - /* Try copy_full */ - crypto_free_pk_env(pk2); - pk2 = crypto_pk_copy_full(pk1); - test_assert(pk2 != NULL); - test_neq_ptr(pk1, pk2); - test_assert(crypto_pk_cmp_keys(pk1,pk2) == 0); - - done: - if (pk1) - crypto_free_pk_env(pk1); - if (pk2) - crypto_free_pk_env(pk2); - tor_free(encoded); -} - -/** Run unit tests for misc crypto functionality. */ -static void -test_crypto(void) -{ - char *data1 = NULL, *data2 = NULL, *data3 = NULL; - int i, j, idx; - - data1 = tor_malloc(1024); - data2 = tor_malloc(1024); - data3 = tor_malloc(1024); - test_assert(data1 && data2 && data3); - - /* Base64 tests */ - memset(data1, 6, 1024); - for (idx = 0; idx < 10; ++idx) { - i = base64_encode(data2, 1024, data1, idx); - test_assert(i >= 0); - j = base64_decode(data3, 1024, data2, i); - test_eq(j,idx); - test_memeq(data3, data1, idx); - } - - strlcpy(data1, "Test string that contains 35 chars.", 1024); - strlcat(data1, " 2nd string that contains 35 chars.", 1024); - - i = base64_encode(data2, 1024, data1, 71); - j = base64_decode(data3, 1024, data2, i); - test_eq(j, 71); - test_streq(data3, data1); - test_assert(data2[i] == '\0'); - - crypto_rand(data1, DIGEST_LEN); - memset(data2, 100, 1024); - digest_to_base64(data2, data1); - test_eq(BASE64_DIGEST_LEN, strlen(data2)); - test_eq(100, data2[BASE64_DIGEST_LEN+2]); - memset(data3, 99, 1024); - test_eq(digest_from_base64(data3, data2), 0); - test_memeq(data1, data3, DIGEST_LEN); - test_eq(99, data3[DIGEST_LEN+1]); - - test_assert(digest_from_base64(data3, "###") < 0); - - /* Base32 tests */ - strlcpy(data1, "5chrs", 1024); - /* bit pattern is: [35 63 68 72 73] -> - * [00110101 01100011 01101000 01110010 01110011] - * By 5s: [00110 10101 10001 10110 10000 11100 10011 10011] - */ - base32_encode(data2, 9, data1, 5); - test_streq(data2, "gvrwq4tt"); - - strlcpy(data1, "\xFF\xF5\x6D\x44\xAE\x0D\x5C\xC9\x62\xC4", 1024); - base32_encode(data2, 30, data1, 10); - test_streq(data2, "772w2rfobvomsywe"); - - /* Base16 tests */ - strlcpy(data1, "6chrs\xff", 1024); - base16_encode(data2, 13, data1, 6); - test_streq(data2, "3663687273FF"); - - strlcpy(data1, "f0d678affc000100", 1024); - i = base16_decode(data2, 8, data1, 16); - test_eq(i,0); - test_memeq(data2, "\xf0\xd6\x78\xaf\xfc\x00\x01\x00",8); - - /* now try some failing base16 decodes */ - test_eq(-1, base16_decode(data2, 8, data1, 15)); /* odd input len */ - test_eq(-1, base16_decode(data2, 7, data1, 16)); /* dest too short */ - strlcpy(data1, "f0dz!8affc000100", 1024); - test_eq(-1, base16_decode(data2, 8, data1, 16)); - - tor_free(data1); - tor_free(data2); - tor_free(data3); - - /* Add spaces to fingerprint */ - { - data1 = tor_strdup("ABCD1234ABCD56780000ABCD1234ABCD56780000"); - test_eq(strlen(data1), 40); - data2 = tor_malloc(FINGERPRINT_LEN+1); - add_spaces_to_fp(data2, FINGERPRINT_LEN+1, data1); - test_streq(data2, "ABCD 1234 ABCD 5678 0000 ABCD 1234 ABCD 5678 0000"); - tor_free(data1); - tor_free(data2); - } - - /* Check fingerprint */ - { - test_assert(crypto_pk_check_fingerprint_syntax( - "ABCD 1234 ABCD 5678 0000 ABCD 1234 ABCD 5678 0000")); - test_assert(!crypto_pk_check_fingerprint_syntax( - "ABCD 1234 ABCD 5678 0000 ABCD 1234 ABCD 5678 000")); - test_assert(!crypto_pk_check_fingerprint_syntax( - "ABCD 1234 ABCD 5678 0000 ABCD 1234 ABCD 5678 00000")); - test_assert(!crypto_pk_check_fingerprint_syntax( - "ABCD 1234 ABCD 5678 0000 ABCD1234 ABCD 5678 0000")); - test_assert(!crypto_pk_check_fingerprint_syntax( - "ABCD 1234 ABCD 5678 0000 ABCD1234 ABCD 5678 00000")); - test_assert(!crypto_pk_check_fingerprint_syntax( - "ACD 1234 ABCD 5678 0000 ABCD 1234 ABCD 5678 00000")); - } - - done: - tor_free(data1); - tor_free(data2); - tor_free(data3); -} - -/** Run unit tests for our secret-to-key passphrase hashing functionality. */ -static void -test_crypto_s2k(void) -{ - char buf[29]; - char buf2[29]; - char *buf3 = NULL; - int i; - - memset(buf, 0, sizeof(buf)); - memset(buf2, 0, sizeof(buf2)); - buf3 = tor_malloc(65536); - memset(buf3, 0, 65536); - - secret_to_key(buf+9, 20, "", 0, buf); - crypto_digest(buf2+9, buf3, 1024); - test_memeq(buf, buf2, 29); - - memcpy(buf,"vrbacrda",8); - memcpy(buf2,"vrbacrda",8); - buf[8] = 96; - buf2[8] = 96; - secret_to_key(buf+9, 20, "12345678", 8, buf); - for (i = 0; i < 65536; i += 16) { - memcpy(buf3+i, "vrbacrda12345678", 16); - } - crypto_digest(buf2+9, buf3, 65536); - test_memeq(buf, buf2, 29); - - done: - tor_free(buf3); -} - -/** Helper: return a tristate based on comparing the strings in *<b>a</b> and - * *<b>b</b>. */ -static int -_compare_strs(const void **a, const void **b) -{ - const char *s1 = *a, *s2 = *b; - return strcmp(s1, s2); -} - -/** Helper: return a tristate based on comparing the strings in *<b>a</b> and - * *<b>b</b>, excluding a's first character, and ignoring case. */ -static int -_compare_without_first_ch(const void *a, const void **b) -{ - const char *s1 = a, *s2 = *b; - return strcasecmp(s1+1, s2); -} - -/** Test basic utility functionality. */ -static void -test_util(void) -{ - struct timeval start, end; - struct tm a_time; - char timestr[RFC1123_TIME_LEN+1]; - char buf[1024]; - time_t t_res; - int i; - uint32_t u32; - uint16_t u16; - char *cp, *k, *v; - const char *str; - - start.tv_sec = 5; - start.tv_usec = 5000; - - end.tv_sec = 5; - end.tv_usec = 5000; - - test_eq(0L, tv_udiff(&start, &end)); - - end.tv_usec = 7000; - - test_eq(2000L, tv_udiff(&start, &end)); - - end.tv_sec = 6; - - test_eq(1002000L, tv_udiff(&start, &end)); - - end.tv_usec = 0; - - test_eq(995000L, tv_udiff(&start, &end)); - - end.tv_sec = 4; - - test_eq(-1005000L, tv_udiff(&start, &end)); - - end.tv_usec = 999990; - start.tv_sec = 1; - start.tv_usec = 500; - - /* The test values here are confirmed to be correct on a platform - * with a working timegm. */ - a_time.tm_year = 2003-1900; - a_time.tm_mon = 7; - a_time.tm_mday = 30; - a_time.tm_hour = 6; - a_time.tm_min = 14; - a_time.tm_sec = 55; - test_eq((time_t) 1062224095UL, tor_timegm(&a_time)); - a_time.tm_year = 2004-1900; /* Try a leap year, after feb. */ - test_eq((time_t) 1093846495UL, tor_timegm(&a_time)); - a_time.tm_mon = 1; /* Try a leap year, in feb. */ - a_time.tm_mday = 10; - test_eq((time_t) 1076393695UL, tor_timegm(&a_time)); - - format_rfc1123_time(timestr, 0); - test_streq("Thu, 01 Jan 1970 00:00:00 GMT", timestr); - format_rfc1123_time(timestr, (time_t)1091580502UL); - test_streq("Wed, 04 Aug 2004 00:48:22 GMT", timestr); - - t_res = 0; - i = parse_rfc1123_time(timestr, &t_res); - test_eq(i,0); - test_eq(t_res, (time_t)1091580502UL); - test_eq(-1, parse_rfc1123_time("Wed, zz Aug 2004 99-99x99 GMT", &t_res)); - tor_gettimeofday(&start); - - /* Tests for corner cases of strl operations */ - test_eq(5, strlcpy(buf, "Hello", 0)); - strlcpy(buf, "Hello", sizeof(buf)); - test_eq(10, strlcat(buf, "Hello", 5)); - - /* Test tor_strstrip() */ - strlcpy(buf, "Testing 1 2 3", sizeof(buf)); - tor_strstrip(buf, ",!"); - test_streq(buf, "Testing 1 2 3"); - strlcpy(buf, "!Testing 1 2 3?", sizeof(buf)); - tor_strstrip(buf, "!? "); - test_streq(buf, "Testing123"); - - /* Test parse_addr_port */ - cp = NULL; u32 = 3; u16 = 3; - test_assert(!parse_addr_port(LOG_WARN, "1.2.3.4", &cp, &u32, &u16)); - test_streq(cp, "1.2.3.4"); - test_eq(u32, 0x01020304u); - test_eq(u16, 0); - tor_free(cp); - test_assert(!parse_addr_port(LOG_WARN, "4.3.2.1:99", &cp, &u32, &u16)); - test_streq(cp, "4.3.2.1"); - test_eq(u32, 0x04030201u); - test_eq(u16, 99); - tor_free(cp); - test_assert(!parse_addr_port(LOG_WARN, "nonexistent.address:4040", - &cp, NULL, &u16)); - test_streq(cp, "nonexistent.address"); - test_eq(u16, 4040); - tor_free(cp); - test_assert(!parse_addr_port(LOG_WARN, "localhost:9999", &cp, &u32, &u16)); - test_streq(cp, "localhost"); - test_eq(u32, 0x7f000001u); - test_eq(u16, 9999); - tor_free(cp); - u32 = 3; - test_assert(!parse_addr_port(LOG_WARN, "localhost", NULL, &u32, &u16)); - test_eq(cp, NULL); - test_eq(u32, 0x7f000001u); - test_eq(u16, 0); - tor_free(cp); - test_eq(0, addr_mask_get_bits(0x0u)); - test_eq(32, addr_mask_get_bits(0xFFFFFFFFu)); - test_eq(16, addr_mask_get_bits(0xFFFF0000u)); - test_eq(31, addr_mask_get_bits(0xFFFFFFFEu)); - test_eq(1, addr_mask_get_bits(0x80000000u)); - - /* Test tor_parse_long. */ - test_eq(10L, tor_parse_long("10",10,0,100,NULL,NULL)); - test_eq(0L, tor_parse_long("10",10,50,100,NULL,NULL)); - test_eq(-50L, tor_parse_long("-50",10,-100,100,NULL,NULL)); - - /* Test tor_parse_ulong */ - test_eq(10UL, tor_parse_ulong("10",10,0,100,NULL,NULL)); - test_eq(0UL, tor_parse_ulong("10",10,50,100,NULL,NULL)); - - /* Test tor_parse_uint64. */ - test_assert(U64_LITERAL(10) == tor_parse_uint64("10 x",10,0,100, &i, &cp)); - test_assert(i == 1); - test_streq(cp, " x"); - test_assert(U64_LITERAL(12345678901) == - tor_parse_uint64("12345678901",10,0,UINT64_MAX, &i, &cp)); - test_assert(i == 1); - test_streq(cp, ""); - test_assert(U64_LITERAL(0) == - tor_parse_uint64("12345678901",10,500,INT32_MAX, &i, &cp)); - test_assert(i == 0); - - /* Test failing snprintf cases */ - test_eq(-1, tor_snprintf(buf, 0, "Foo")); - test_eq(-1, tor_snprintf(buf, 2, "Foo")); - - /* Test printf with uint64 */ - tor_snprintf(buf, sizeof(buf), "x!"U64_FORMAT"!x", - U64_PRINTF_ARG(U64_LITERAL(12345678901))); - test_streq(buf, "x!12345678901!x"); - - /* Test parse_config_line_from_str */ - strlcpy(buf, "k v\n" " key value with spaces \n" "keykey val\n" - "k2\n" - "k3 \n" "\n" " \n" "#comment\n" - "k4#a\n" "k5#abc\n" "k6 val #with comment\n" - "kseven \"a quoted 'string\"\n" - "k8 \"a \\x71uoted\\n\\\"str\\\\ing\\t\\001\\01\\1\\\"\"\n" - , sizeof(buf)); - str = buf; - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "k"); - test_streq(v, "v"); - tor_free(k); tor_free(v); - test_assert(!strcmpstart(str, "key value with")); - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "key"); - test_streq(v, "value with spaces"); - tor_free(k); tor_free(v); - test_assert(!strcmpstart(str, "keykey")); - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "keykey"); - test_streq(v, "val"); - tor_free(k); tor_free(v); - test_assert(!strcmpstart(str, "k2\n")); - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "k2"); - test_streq(v, ""); - tor_free(k); tor_free(v); - test_assert(!strcmpstart(str, "k3 \n")); - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "k3"); - test_streq(v, ""); - tor_free(k); tor_free(v); - test_assert(!strcmpstart(str, "#comment")); - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "k4"); - test_streq(v, ""); - tor_free(k); tor_free(v); - test_assert(!strcmpstart(str, "k5#abc")); - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "k5"); - test_streq(v, ""); - tor_free(k); tor_free(v); - test_assert(!strcmpstart(str, "k6")); - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "k6"); - test_streq(v, "val"); - tor_free(k); tor_free(v); - test_assert(!strcmpstart(str, "kseven")); - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "kseven"); - test_streq(v, "a quoted \'string"); - tor_free(k); tor_free(v); - test_assert(!strcmpstart(str, "k8 ")); - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "k8"); - test_streq(v, "a quoted\n\"str\\ing\t\x01\x01\x01\""); - tor_free(k); tor_free(v); - test_streq(str, ""); - - /* Test for strcmpstart and strcmpend. */ - test_assert(strcmpstart("abcdef", "abcdef")==0); - test_assert(strcmpstart("abcdef", "abc")==0); - test_assert(strcmpstart("abcdef", "abd")<0); - test_assert(strcmpstart("abcdef", "abb")>0); - test_assert(strcmpstart("ab", "abb")<0); - - test_assert(strcmpend("abcdef", "abcdef")==0); - test_assert(strcmpend("abcdef", "def")==0); - test_assert(strcmpend("abcdef", "deg")<0); - test_assert(strcmpend("abcdef", "dee")>0); - test_assert(strcmpend("ab", "abb")<0); - - test_assert(strcasecmpend("AbcDEF", "abcdef")==0); - test_assert(strcasecmpend("abcdef", "dEF")==0); - test_assert(strcasecmpend("abcDEf", "deg")<0); - test_assert(strcasecmpend("abcdef", "DEE")>0); - test_assert(strcasecmpend("ab", "abB")<0); - - /* Test mem_is_zero */ - memset(buf,0,128); - buf[128] = 'x'; - test_assert(tor_digest_is_zero(buf)); - test_assert(tor_mem_is_zero(buf, 10)); - test_assert(tor_mem_is_zero(buf, 20)); - test_assert(tor_mem_is_zero(buf, 128)); - test_assert(!tor_mem_is_zero(buf, 129)); - buf[60] = (char)255; - test_assert(!tor_mem_is_zero(buf, 128)); - buf[0] = (char)1; - test_assert(!tor_mem_is_zero(buf, 10)); - - /* Test inet_ntop */ - { - char tmpbuf[TOR_ADDR_BUF_LEN]; - const char *ip = "176.192.208.224"; - struct in_addr in; - tor_inet_pton(AF_INET, ip, &in); - tor_inet_ntop(AF_INET, &in, tmpbuf, sizeof(tmpbuf)); - test_streq(tmpbuf, ip); - } - - /* Test 'escaped' */ - test_streq("\"\"", escaped("")); - test_streq("\"abcd\"", escaped("abcd")); - test_streq("\"\\\\\\n\\r\\t\\\"\\'\"", escaped("\\\n\r\t\"\'")); - test_streq("\"z\\001abc\\277d\"", escaped("z\001abc\277d")); - test_assert(NULL == escaped(NULL)); - - /* Test strndup and memdup */ - { - const char *s = "abcdefghijklmnopqrstuvwxyz"; - cp = tor_strndup(s, 30); - test_streq(cp, s); /* same string, */ - test_neq(cp, s); /* but different pointers. */ - tor_free(cp); - - cp = tor_strndup(s, 5); - test_streq(cp, "abcde"); - tor_free(cp); - - s = "a\0b\0c\0d\0e\0"; - cp = tor_memdup(s,10); - test_memeq(cp, s, 10); /* same ram, */ - test_neq(cp, s); /* but different pointers. */ - tor_free(cp); - } - - /* Test str-foo functions */ - cp = tor_strdup("abcdef"); - test_assert(tor_strisnonupper(cp)); - cp[3] = 'D'; - test_assert(!tor_strisnonupper(cp)); - tor_strupper(cp); - test_streq(cp, "ABCDEF"); - test_assert(tor_strisprint(cp)); - cp[3] = 3; - test_assert(!tor_strisprint(cp)); - tor_free(cp); - - /* Test eat_whitespace. */ - { - const char *s = " \n a"; - test_eq_ptr(eat_whitespace(s), s+4); - s = "abcd"; - test_eq_ptr(eat_whitespace(s), s); - s = "#xyz\nab"; - test_eq_ptr(eat_whitespace(s), s+5); - } - - /* Test memmem and memstr */ - { - const char *haystack = "abcde"; - tor_assert(!tor_memmem(haystack, 5, "ef", 2)); - test_eq_ptr(tor_memmem(haystack, 5, "cd", 2), haystack + 2); - test_eq_ptr(tor_memmem(haystack, 5, "cde", 3), haystack + 2); - haystack = "ababcad"; - test_eq_ptr(tor_memmem(haystack, 7, "abc", 3), haystack + 2); - test_eq_ptr(tor_memstr(haystack, 7, "abc"), haystack + 2); - test_assert(!tor_memstr(haystack, 7, "fe")); - test_assert(!tor_memstr(haystack, 7, "longerthantheoriginal")); - } - - /* Test wrap_string */ - { - smartlist_t *sl = smartlist_create(); - wrap_string(sl, "This is a test of string wrapping functionality: woot.", - 10, "", ""); - cp = smartlist_join_strings(sl, "", 0, NULL); - test_streq(cp, - "This is a\ntest of\nstring\nwrapping\nfunctional\nity: woot.\n"); - tor_free(cp); - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_clear(sl); - - wrap_string(sl, "This is a test of string wrapping functionality: woot.", - 16, "### ", "# "); - cp = smartlist_join_strings(sl, "", 0, NULL); - test_streq(cp, - "### This is a\n# test of string\n# wrapping\n# functionality:\n" - "# woot.\n"); - - tor_free(cp); - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_free(sl); - } - - /* now make sure time works. */ - tor_gettimeofday(&end); - /* We might've timewarped a little. */ - test_assert(tv_udiff(&start, &end) >= -5000); - - /* Test tor_log2(). */ - test_eq(tor_log2(64), 6); - test_eq(tor_log2(65), 6); - test_eq(tor_log2(63), 5); - test_eq(tor_log2(1), 0); - test_eq(tor_log2(2), 1); - test_eq(tor_log2(3), 1); - test_eq(tor_log2(4), 2); - test_eq(tor_log2(5), 2); - test_eq(tor_log2(U64_LITERAL(40000000000000000)), 55); - test_eq(tor_log2(UINT64_MAX), 63); - - /* Test round_to_power_of_2 */ - test_eq(round_to_power_of_2(120), 128); - test_eq(round_to_power_of_2(128), 128); - test_eq(round_to_power_of_2(130), 128); - test_eq(round_to_power_of_2(U64_LITERAL(40000000000000000)), - U64_LITERAL(1)<<55); - test_eq(round_to_power_of_2(0), 2); - - done: - ; -} - -/** Helper: assert that IPv6 addresses <b>a</b> and <b>b</b> are the same. On - * failure, reports an error, describing the addresses as <b>e1</b> and - * <b>e2</b>, and reporting the line number as <b>line</b>. */ -static void -_test_eq_ip6(struct in6_addr *a, struct in6_addr *b, const char *e1, - const char *e2, int line) -{ - int i; - int ok = 1; - for (i = 0; i < 16; ++i) { - if (a->s6_addr[i] != b->s6_addr[i]) { - ok = 0; - break; - } - } - if (ok) { - printf("."); fflush(stdout); - } else { - char buf1[128], *cp1; - char buf2[128], *cp2; - have_failed = 1; - cp1 = buf1; cp2 = buf2; - for (i=0; i<16; ++i) { - tor_snprintf(cp1, sizeof(buf1)-(cp1-buf1), "%02x", a->s6_addr[i]); - tor_snprintf(cp2, sizeof(buf2)-(cp2-buf2), "%02x", b->s6_addr[i]); - cp1 += 2; cp2 += 2; - if ((i%2)==1 && i != 15) { - *cp1++ = ':'; - *cp2++ = ':'; - } - } - *cp1 = *cp2 = '\0'; - printf("Line %d: assertion failed: (%s == %s)\n" - " %s != %s\n", line, e1, e2, buf1, buf2); - fflush(stdout); - } -} - -/** Helper: Assert that two strings both decode as IPv6 addresses with - * tor_inet_pton(), and both decode to the same address. */ -#define test_pton6_same(a,b) STMT_BEGIN \ - test_eq(tor_inet_pton(AF_INET6, a, &a1), 1); \ - test_eq(tor_inet_pton(AF_INET6, b, &a2), 1); \ - _test_eq_ip6(&a1,&a2,#a,#b,__LINE__); \ - STMT_END - -/** Helper: Assert that <b>a</b> is recognized as a bad IPv6 address by - * tor_inet_pton(). */ -#define test_pton6_bad(a) \ - test_eq(0, tor_inet_pton(AF_INET6, a, &a1)) - -/** Helper: assert that <b>a</b>, when parsed by tor_inet_pton() and displayed - * with tor_inet_ntop(), yields <b>b</b>. Also assert that <b>b</b> parses to - * the same value as <b>a</b>. */ -#define test_ntop6_reduces(a,b) STMT_BEGIN \ - test_eq(tor_inet_pton(AF_INET6, a, &a1), 1); \ - test_streq(tor_inet_ntop(AF_INET6, &a1, buf, sizeof(buf)), b); \ - test_eq(tor_inet_pton(AF_INET6, b, &a2), 1); \ - _test_eq_ip6(&a1, &a2, a, b, __LINE__); \ - STMT_END - -/** Helper: assert that <b>a</b> parses by tor_inet_pton() into a address that - * passes tor_addr_is_internal() with <b>for_listening</b>. */ -#define test_internal_ip(a,for_listening) STMT_BEGIN \ - test_eq(tor_inet_pton(AF_INET6, a, &t1.addr.in6_addr), 1); \ - t1.family = AF_INET6; \ - if (!tor_addr_is_internal(&t1, for_listening)) \ - test_fail_msg( a "was not internal."); \ - STMT_END - -/** Helper: assert that <b>a</b> parses by tor_inet_pton() into a address that - * does not pass tor_addr_is_internal() with <b>for_listening</b>. */ -#define test_external_ip(a,for_listening) STMT_BEGIN \ - test_eq(tor_inet_pton(AF_INET6, a, &t1.addr.in6_addr), 1); \ - t1.family = AF_INET6; \ - if (tor_addr_is_internal(&t1, for_listening)) \ - test_fail_msg(a "was not external."); \ - STMT_END - -/** Helper: Assert that <b>a</b> and <b>b</b>, when parsed by - * tor_inet_pton(), give addresses that compare in the order defined by - * <b>op</b> with tor_addr_compare(). */ -#define test_addr_compare(a, op, b) STMT_BEGIN \ - test_eq(tor_inet_pton(AF_INET6, a, &t1.addr.in6_addr), 1); \ - test_eq(tor_inet_pton(AF_INET6, b, &t2.addr.in6_addr), 1); \ - t1.family = t2.family = AF_INET6; \ - r = tor_addr_compare(&t1,&t2,CMP_SEMANTIC); \ - if (!(r op 0)) \ - test_fail_msg("failed: tor_addr_compare("a","b") "#op" 0"); \ - STMT_END - -/** Helper: Assert that <b>a</b> and <b>b</b>, when parsed by - * tor_inet_pton(), give addresses that compare in the order defined by - * <b>op</b> with tor_addr_compare_masked() with <b>m</b> masked. */ -#define test_addr_compare_masked(a, op, b, m) STMT_BEGIN \ - test_eq(tor_inet_pton(AF_INET6, a, &t1.addr.in6_addr), 1); \ - test_eq(tor_inet_pton(AF_INET6, b, &t2.addr.in6_addr), 1); \ - t1.family = t2.family = AF_INET6; \ - r = tor_addr_compare_masked(&t1,&t2,m,CMP_SEMANTIC); \ - if (!(r op 0)) \ - test_fail_msg("failed: tor_addr_compare_masked("a","b","#m") "#op" 0"); \ - STMT_END - -/** Helper: assert that <b>xx</b> is parseable as a masked IPv6 address with - * ports by tor_parse_mask_addr_ports(), with family <b>f</b>, IP address - * as 4 32-bit words <b>ip1...ip4</b>, mask bits as <b>mm</b>, and port range - * as <b>pt1..pt2</b>. */ -#define test_addr_mask_ports_parse(xx, f, ip1, ip2, ip3, ip4, mm, pt1, pt2) \ - STMT_BEGIN \ - test_eq(tor_addr_parse_mask_ports(xx, &t1, &mask, &port1, &port2), f); \ - p1=tor_inet_ntop(AF_INET6, &t1.addr.in6_addr, bug, sizeof(bug)); \ - test_eq(htonl(ip1), tor_addr_to_in6_addr32(&t1)[0]); \ - test_eq(htonl(ip2), tor_addr_to_in6_addr32(&t1)[1]); \ - test_eq(htonl(ip3), tor_addr_to_in6_addr32(&t1)[2]); \ - test_eq(htonl(ip4), tor_addr_to_in6_addr32(&t1)[3]); \ - test_eq(mask, mm); \ - test_eq(port1, pt1); \ - test_eq(port2, pt2); \ - STMT_END - -/** Run unit tests for IPv6 encoding/decoding/manipulation functions. */ -static void -test_util_ip6_helpers(void) -{ - char buf[TOR_ADDR_BUF_LEN], bug[TOR_ADDR_BUF_LEN]; - struct in6_addr a1, a2; - tor_addr_t t1, t2; - int r, i; - uint16_t port1, port2; - maskbits_t mask; - const char *p1; - struct sockaddr_storage sa_storage; - struct sockaddr_in *sin; - struct sockaddr_in6 *sin6; - - // struct in_addr b1, b2; - /* Test tor_inet_ntop and tor_inet_pton: IPv6 */ - - /* ==== Converting to and from sockaddr_t. */ - sin = (struct sockaddr_in *)&sa_storage; - sin->sin_family = AF_INET; - sin->sin_port = 9090; - sin->sin_addr.s_addr = htonl(0x7f7f0102); /*127.127.1.2*/ - tor_addr_from_sockaddr(&t1, (struct sockaddr *)sin, NULL); - test_eq(tor_addr_family(&t1), AF_INET); - test_eq(tor_addr_to_ipv4h(&t1), 0x7f7f0102); - - memset(&sa_storage, 0, sizeof(sa_storage)); - test_eq(sizeof(struct sockaddr_in), - tor_addr_to_sockaddr(&t1, 1234, (struct sockaddr *)&sa_storage, - sizeof(sa_storage))); - test_eq(1234, ntohs(sin->sin_port)); - test_eq(0x7f7f0102, ntohl(sin->sin_addr.s_addr)); - - memset(&sa_storage, 0, sizeof(sa_storage)); - sin6 = (struct sockaddr_in6 *)&sa_storage; - sin6->sin6_family = AF_INET6; - sin6->sin6_port = htons(7070); - sin6->sin6_addr.s6_addr[0] = 128; - tor_addr_from_sockaddr(&t1, (struct sockaddr *)sin6, NULL); - test_eq(tor_addr_family(&t1), AF_INET6); - p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 0); - test_streq(p1, "8000::"); - - memset(&sa_storage, 0, sizeof(sa_storage)); - test_eq(sizeof(struct sockaddr_in6), - tor_addr_to_sockaddr(&t1, 9999, (struct sockaddr *)&sa_storage, - sizeof(sa_storage))); - test_eq(AF_INET6, sin6->sin6_family); - test_eq(9999, ntohs(sin6->sin6_port)); - test_eq(0x80000000, ntohl(S6_ADDR32(sin6->sin6_addr)[0])); - - /* ==== tor_addr_lookup: static cases. (Can't test dns without knowing we - * have a good resolver. */ - test_eq(0, tor_addr_lookup("127.128.129.130", AF_UNSPEC, &t1)); - test_eq(AF_INET, tor_addr_family(&t1)); - test_eq(tor_addr_to_ipv4h(&t1), 0x7f808182); - - test_eq(0, tor_addr_lookup("9000::5", AF_UNSPEC, &t1)); - test_eq(AF_INET6, tor_addr_family(&t1)); - test_eq(0x90, tor_addr_to_in6_addr8(&t1)[0]); - test_assert(tor_mem_is_zero((char*)tor_addr_to_in6_addr8(&t1)+1, 14)); - test_eq(0x05, tor_addr_to_in6_addr8(&t1)[15]); - - /* === Test pton: valid af_inet6 */ - /* Simple, valid parsing. */ - r = tor_inet_pton(AF_INET6, - "0102:0304:0506:0708:090A:0B0C:0D0E:0F10", &a1); - test_assert(r==1); - for (i=0;i<16;++i) { test_eq(i+1, (int)a1.s6_addr[i]); } - /* ipv4 ending. */ - test_pton6_same("0102:0304:0506:0708:090A:0B0C:0D0E:0F10", - "0102:0304:0506:0708:090A:0B0C:13.14.15.16"); - /* shortened words. */ - test_pton6_same("0001:0099:BEEF:0000:0123:FFFF:0001:0001", - "1:99:BEEF:0:0123:FFFF:1:1"); - /* zeros at the beginning */ - test_pton6_same("0000:0000:0000:0000:0009:C0A8:0001:0001", - "::9:c0a8:1:1"); - test_pton6_same("0000:0000:0000:0000:0009:C0A8:0001:0001", - "::9:c0a8:0.1.0.1"); - /* zeros in the middle. */ - test_pton6_same("fe80:0000:0000:0000:0202:1111:0001:0001", - "fe80::202:1111:1:1"); - /* zeros at the end. */ - test_pton6_same("1000:0001:0000:0007:0000:0000:0000:0000", - "1000:1:0:7::"); - - /* === Test ntop: af_inet6 */ - test_ntop6_reduces("0:0:0:0:0:0:0:0", "::"); - - test_ntop6_reduces("0001:0099:BEEF:0006:0123:FFFF:0001:0001", - "1:99:beef:6:123:ffff:1:1"); - - //test_ntop6_reduces("0:0:0:0:0:0:c0a8:0101", "::192.168.1.1"); - test_ntop6_reduces("0:0:0:0:0:ffff:c0a8:0101", "::ffff:192.168.1.1"); - test_ntop6_reduces("002:0:0000:0:3::4", "2::3:0:0:4"); - test_ntop6_reduces("0:0::1:0:3", "::1:0:3"); - test_ntop6_reduces("008:0::0", "8::"); - test_ntop6_reduces("0:0:0:0:0:ffff::1", "::ffff:0.0.0.1"); - test_ntop6_reduces("abcd:0:0:0:0:0:7f00::", "abcd::7f00:0"); - test_ntop6_reduces("0000:0000:0000:0000:0009:C0A8:0001:0001", - "::9:c0a8:1:1"); - test_ntop6_reduces("fe80:0000:0000:0000:0202:1111:0001:0001", - "fe80::202:1111:1:1"); - test_ntop6_reduces("1000:0001:0000:0007:0000:0000:0000:0000", - "1000:1:0:7::"); - - /* === Test pton: invalid in6. */ - test_pton6_bad("foobar."); - test_pton6_bad("55555::"); - test_pton6_bad("9:-60::"); - test_pton6_bad("1:2:33333:4:0002:3::"); - //test_pton6_bad("1:2:3333:4:00002:3::");// BAD, but glibc doesn't say so. - test_pton6_bad("1:2:3333:4:fish:3::"); - test_pton6_bad("1:2:3:4:5:6:7:8:9"); - test_pton6_bad("1:2:3:4:5:6:7"); - test_pton6_bad("1:2:3:4:5:6:1.2.3.4.5"); - test_pton6_bad("1:2:3:4:5:6:1.2.3"); - test_pton6_bad("::1.2.3"); - test_pton6_bad("::1.2.3.4.5"); - test_pton6_bad("99"); - test_pton6_bad(""); - test_pton6_bad("1::2::3:4"); - test_pton6_bad("a:::b:c"); - test_pton6_bad(":::a:b:c"); - test_pton6_bad("a:b:c:::"); - - /* test internal checking */ - test_external_ip("fbff:ffff::2:7", 0); - test_internal_ip("fc01::2:7", 0); - test_internal_ip("fdff:ffff::f:f", 0); - test_external_ip("fe00::3:f", 0); - - test_external_ip("fe7f:ffff::2:7", 0); - test_internal_ip("fe80::2:7", 0); - test_internal_ip("febf:ffff::f:f", 0); - - test_internal_ip("fec0::2:7:7", 0); - test_internal_ip("feff:ffff::e:7:7", 0); - test_external_ip("ff00::e:7:7", 0); - - test_internal_ip("::", 0); - test_internal_ip("::1", 0); - test_internal_ip("::1", 1); - test_internal_ip("::", 0); - test_external_ip("::", 1); - test_external_ip("::2", 0); - test_external_ip("2001::", 0); - test_external_ip("ffff::", 0); - - test_external_ip("::ffff:0.0.0.0", 1); - test_internal_ip("::ffff:0.0.0.0", 0); - test_internal_ip("::ffff:0.255.255.255", 0); - test_external_ip("::ffff:1.0.0.0", 0); - - test_external_ip("::ffff:9.255.255.255", 0); - test_internal_ip("::ffff:10.0.0.0", 0); - test_internal_ip("::ffff:10.255.255.255", 0); - test_external_ip("::ffff:11.0.0.0", 0); - - test_external_ip("::ffff:126.255.255.255", 0); - test_internal_ip("::ffff:127.0.0.0", 0); - test_internal_ip("::ffff:127.255.255.255", 0); - test_external_ip("::ffff:128.0.0.0", 0); - - test_external_ip("::ffff:172.15.255.255", 0); - test_internal_ip("::ffff:172.16.0.0", 0); - test_internal_ip("::ffff:172.31.255.255", 0); - test_external_ip("::ffff:172.32.0.0", 0); - - test_external_ip("::ffff:192.167.255.255", 0); - test_internal_ip("::ffff:192.168.0.0", 0); - test_internal_ip("::ffff:192.168.255.255", 0); - test_external_ip("::ffff:192.169.0.0", 0); - - test_external_ip("::ffff:169.253.255.255", 0); - test_internal_ip("::ffff:169.254.0.0", 0); - test_internal_ip("::ffff:169.254.255.255", 0); - test_external_ip("::ffff:169.255.0.0", 0); - test_assert(is_internal_IP(0x7f000001, 0)); - - /* tor_addr_compare(tor_addr_t x2) */ - test_addr_compare("ffff::", ==, "ffff::0"); - test_addr_compare("0::3:2:1", <, "0::ffff:0.3.2.1"); - test_addr_compare("0::2:2:1", <, "0::ffff:0.3.2.1"); - test_addr_compare("0::ffff:0.3.2.1", >, "0::0:0:0"); - test_addr_compare("0::ffff:5.2.2.1", <, "::ffff:6.0.0.0"); /* XXXX wrong. */ - tor_addr_parse_mask_ports("[::ffff:2.3.4.5]", &t1, NULL, NULL, NULL); - tor_addr_parse_mask_ports("2.3.4.5", &t2, NULL, NULL, NULL); - test_assert(tor_addr_compare(&t1, &t2, CMP_SEMANTIC) == 0); - tor_addr_parse_mask_ports("[::ffff:2.3.4.4]", &t1, NULL, NULL, NULL); - tor_addr_parse_mask_ports("2.3.4.5", &t2, NULL, NULL, NULL); - test_assert(tor_addr_compare(&t1, &t2, CMP_SEMANTIC) < 0); - - /* test compare_masked */ - test_addr_compare_masked("ffff::", ==, "ffff::0", 128); - test_addr_compare_masked("ffff::", ==, "ffff::0", 64); - test_addr_compare_masked("0::2:2:1", <, "0::8000:2:1", 81); - test_addr_compare_masked("0::2:2:1", ==, "0::8000:2:1", 80); - - /* Test decorated addr_to_string. */ - test_eq(AF_INET6, tor_addr_from_str(&t1, "[123:45:6789::5005:11]")); - p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 1); - test_streq(p1, "[123:45:6789::5005:11]"); - test_eq(AF_INET, tor_addr_from_str(&t1, "18.0.0.1")); - p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 1); - test_streq(p1, "18.0.0.1"); - - /* Test tor_addr_parse_reverse_lookup_name */ - i = tor_addr_parse_reverse_lookup_name(&t1, "Foobar.baz", AF_UNSPEC, 0); - test_eq(0, i); - i = tor_addr_parse_reverse_lookup_name(&t1, "Foobar.baz", AF_UNSPEC, 1); - test_eq(0, i); - i = tor_addr_parse_reverse_lookup_name(&t1, "1.0.168.192.in-addr.arpa", - AF_UNSPEC, 1); - test_eq(1, i); - test_eq(tor_addr_family(&t1), AF_INET); - p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 1); - test_streq(p1, "192.168.0.1"); - i = tor_addr_parse_reverse_lookup_name(&t1, "192.168.0.99", AF_UNSPEC, 0); - test_eq(0, i); - i = tor_addr_parse_reverse_lookup_name(&t1, "192.168.0.99", AF_UNSPEC, 1); - test_eq(1, i); - p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 1); - test_streq(p1, "192.168.0.99"); - memset(&t1, 0, sizeof(t1)); - i = tor_addr_parse_reverse_lookup_name(&t1, - "0.1.2.3.4.5.6.7.8.9.a.b.c.d.e.f." - "f.e.e.b.1.e.b.e.e.f.f.e.e.e.d.9." - "ip6.ARPA", - AF_UNSPEC, 0); - test_eq(1, i); - p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 1); - test_streq(p1, "[9dee:effe:ebe1:beef:fedc:ba98:7654:3210]"); - /* Failing cases. */ - i = tor_addr_parse_reverse_lookup_name(&t1, - "6.7.8.9.a.b.c.d.e.f." - "f.e.e.b.1.e.b.e.e.f.f.e.e.e.d.9." - "ip6.ARPA", - AF_UNSPEC, 0); - test_eq(i, -1); - i = tor_addr_parse_reverse_lookup_name(&t1, - "6.7.8.9.a.b.c.d.e.f.a.b.c.d.e.f.0." - "f.e.e.b.1.e.b.e.e.f.f.e.e.e.d.9." - "ip6.ARPA", - AF_UNSPEC, 0); - test_eq(i, -1); - i = tor_addr_parse_reverse_lookup_name(&t1, - "6.7.8.9.a.b.c.d.e.f.X.0.0.0.0.9." - "f.e.e.b.1.e.b.e.e.f.f.e.e.e.d.9." - "ip6.ARPA", - AF_UNSPEC, 0); - test_eq(i, -1); - i = tor_addr_parse_reverse_lookup_name(&t1, "32.1.1.in-addr.arpa", - AF_UNSPEC, 0); - test_eq(i, -1); - i = tor_addr_parse_reverse_lookup_name(&t1, ".in-addr.arpa", - AF_UNSPEC, 0); - test_eq(i, -1); - i = tor_addr_parse_reverse_lookup_name(&t1, "1.2.3.4.5.in-addr.arpa", - AF_UNSPEC, 0); - test_eq(i, -1); - i = tor_addr_parse_reverse_lookup_name(&t1, "1.2.3.4.5.in-addr.arpa", - AF_INET6, 0); - test_eq(i, -1); - i = tor_addr_parse_reverse_lookup_name(&t1, - "6.7.8.9.a.b.c.d.e.f.a.b.c.d.e.0." - "f.e.e.b.1.e.b.e.e.f.f.e.e.e.d.9." - "ip6.ARPA", - AF_INET, 0); - test_eq(i, -1); - - /* test tor_addr_parse_mask_ports */ - test_addr_mask_ports_parse("[::f]/17:47-95", AF_INET6, - 0, 0, 0, 0x0000000f, 17, 47, 95); - //test_addr_parse("[::fefe:4.1.1.7/120]:999-1000"); - //test_addr_parse_check("::fefe:401:107", 120, 999, 1000); - test_addr_mask_ports_parse("[::ffff:4.1.1.7]/120:443", AF_INET6, - 0, 0, 0x0000ffff, 0x04010107, 120, 443, 443); - test_addr_mask_ports_parse("[abcd:2::44a:0]:2-65000", AF_INET6, - 0xabcd0002, 0, 0, 0x044a0000, 128, 2, 65000); - - r=tor_addr_parse_mask_ports("[fefef::]/112", &t1, NULL, NULL, NULL); - test_assert(r == -1); - r=tor_addr_parse_mask_ports("efef::/112", &t1, NULL, NULL, NULL); - test_assert(r == -1); - r=tor_addr_parse_mask_ports("[f:f:f:f:f:f:f:f::]", &t1, NULL, NULL, NULL); - test_assert(r == -1); - r=tor_addr_parse_mask_ports("[::f:f:f:f:f:f:f:f]", &t1, NULL, NULL, NULL); - test_assert(r == -1); - r=tor_addr_parse_mask_ports("[f:f:f:f:f:f:f:f:f]", &t1, NULL, NULL, NULL); - test_assert(r == -1); - /* Test for V4-mapped address with mask < 96. (arguably not valid) */ - r=tor_addr_parse_mask_ports("[::ffff:1.1.2.2/33]", &t1, &mask, NULL, NULL); - test_assert(r == -1); - r=tor_addr_parse_mask_ports("1.1.2.2/33", &t1, &mask, NULL, NULL); - test_assert(r == -1); - r=tor_addr_parse_mask_ports("1.1.2.2/31", &t1, &mask, NULL, NULL); - test_assert(r == AF_INET); - r=tor_addr_parse_mask_ports("[efef::]/112", &t1, &mask, &port1, &port2); - test_assert(r == AF_INET6); - test_assert(port1 == 1); - test_assert(port2 == 65535); - - /* make sure inet address lengths >= max */ - test_assert(INET_NTOA_BUF_LEN >= sizeof("255.255.255.255")); - test_assert(TOR_ADDR_BUF_LEN >= - sizeof("ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255")); - - test_assert(sizeof(tor_addr_t) >= sizeof(struct in6_addr)); - - /* get interface addresses */ - r = get_interface_address6(LOG_DEBUG, AF_INET, &t1); - i = get_interface_address6(LOG_DEBUG, AF_INET6, &t2); -#if 0 - tor_inet_ntop(AF_INET, &t1.sa.sin_addr, buf, sizeof(buf)); - printf("\nv4 address: %s (family=%i)", buf, IN_FAMILY(&t1)); - tor_inet_ntop(AF_INET6, &t2.sa6.sin6_addr, buf, sizeof(buf)); - printf("\nv6 address: %s (family=%i)", buf, IN_FAMILY(&t2)); -#endif - - done: - ; -} - -/** Run unit tests for basic dynamic-sized array functionality. */ -static void -test_util_smartlist_basic(void) -{ - smartlist_t *sl; - - /* XXXX test sort_digests, uniq_strings, uniq_digests */ - - /* Test smartlist add, del_keeporder, insert, get. */ - sl = smartlist_create(); - smartlist_add(sl, (void*)1); - smartlist_add(sl, (void*)2); - smartlist_add(sl, (void*)3); - smartlist_add(sl, (void*)4); - smartlist_del_keeporder(sl, 1); - smartlist_insert(sl, 1, (void*)22); - smartlist_insert(sl, 0, (void*)0); - smartlist_insert(sl, 5, (void*)555); - test_eq_ptr((void*)0, smartlist_get(sl,0)); - test_eq_ptr((void*)1, smartlist_get(sl,1)); - test_eq_ptr((void*)22, smartlist_get(sl,2)); - test_eq_ptr((void*)3, smartlist_get(sl,3)); - test_eq_ptr((void*)4, smartlist_get(sl,4)); - test_eq_ptr((void*)555, smartlist_get(sl,5)); - /* Try deleting in the middle. */ - smartlist_del(sl, 1); - test_eq_ptr((void*)555, smartlist_get(sl, 1)); - /* Try deleting at the end. */ - smartlist_del(sl, 4); - test_eq(4, smartlist_len(sl)); - - /* test isin. */ - test_assert(smartlist_isin(sl, (void*)3)); - test_assert(!smartlist_isin(sl, (void*)99)); - - done: - smartlist_free(sl); -} - -/** Run unit tests for smartlist-of-strings functionality. */ -static void -test_util_smartlist_strings(void) -{ - smartlist_t *sl = smartlist_create(); - char *cp=NULL, *cp_alloc=NULL; - size_t sz; - - /* Test split and join */ - test_eq(0, smartlist_len(sl)); - smartlist_split_string(sl, "abc", ":", 0, 0); - test_eq(1, smartlist_len(sl)); - test_streq("abc", smartlist_get(sl, 0)); - smartlist_split_string(sl, "a::bc::", "::", 0, 0); - test_eq(4, smartlist_len(sl)); - test_streq("a", smartlist_get(sl, 1)); - test_streq("bc", smartlist_get(sl, 2)); - test_streq("", smartlist_get(sl, 3)); - cp_alloc = smartlist_join_strings(sl, "", 0, NULL); - test_streq(cp_alloc, "abcabc"); - tor_free(cp_alloc); - cp_alloc = smartlist_join_strings(sl, "!", 0, NULL); - test_streq(cp_alloc, "abc!a!bc!"); - tor_free(cp_alloc); - cp_alloc = smartlist_join_strings(sl, "XY", 0, NULL); - test_streq(cp_alloc, "abcXYaXYbcXY"); - tor_free(cp_alloc); - cp_alloc = smartlist_join_strings(sl, "XY", 1, NULL); - test_streq(cp_alloc, "abcXYaXYbcXYXY"); - tor_free(cp_alloc); - cp_alloc = smartlist_join_strings(sl, "", 1, NULL); - test_streq(cp_alloc, "abcabc"); - tor_free(cp_alloc); - - smartlist_split_string(sl, "/def/ /ghijk", "/", 0, 0); - test_eq(8, smartlist_len(sl)); - test_streq("", smartlist_get(sl, 4)); - test_streq("def", smartlist_get(sl, 5)); - test_streq(" ", smartlist_get(sl, 6)); - test_streq("ghijk", smartlist_get(sl, 7)); - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_clear(sl); - - smartlist_split_string(sl, "a,bbd,cdef", ",", SPLIT_SKIP_SPACE, 0); - test_eq(3, smartlist_len(sl)); - test_streq("a", smartlist_get(sl,0)); - test_streq("bbd", smartlist_get(sl,1)); - test_streq("cdef", smartlist_get(sl,2)); - smartlist_split_string(sl, " z <> zhasd <> <> bnud<> ", "<>", - SPLIT_SKIP_SPACE, 0); - test_eq(8, smartlist_len(sl)); - test_streq("z", smartlist_get(sl,3)); - test_streq("zhasd", smartlist_get(sl,4)); - test_streq("", smartlist_get(sl,5)); - test_streq("bnud", smartlist_get(sl,6)); - test_streq("", smartlist_get(sl,7)); - - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_clear(sl); - - smartlist_split_string(sl, " ab\tc \td ef ", NULL, - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - test_eq(4, smartlist_len(sl)); - test_streq("ab", smartlist_get(sl,0)); - test_streq("c", smartlist_get(sl,1)); - test_streq("d", smartlist_get(sl,2)); - test_streq("ef", smartlist_get(sl,3)); - smartlist_split_string(sl, "ghi\tj", NULL, - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - test_eq(6, smartlist_len(sl)); - test_streq("ghi", smartlist_get(sl,4)); - test_streq("j", smartlist_get(sl,5)); - - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_clear(sl); - - cp_alloc = smartlist_join_strings(sl, "XY", 0, NULL); - test_streq(cp_alloc, ""); - tor_free(cp_alloc); - cp_alloc = smartlist_join_strings(sl, "XY", 1, NULL); - test_streq(cp_alloc, "XY"); - tor_free(cp_alloc); - - smartlist_split_string(sl, " z <> zhasd <> <> bnud<> ", "<>", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - test_eq(3, smartlist_len(sl)); - test_streq("z", smartlist_get(sl, 0)); - test_streq("zhasd", smartlist_get(sl, 1)); - test_streq("bnud", smartlist_get(sl, 2)); - smartlist_split_string(sl, " z <> zhasd <> <> bnud<> ", "<>", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 2); - test_eq(5, smartlist_len(sl)); - test_streq("z", smartlist_get(sl, 3)); - test_streq("zhasd <> <> bnud<>", smartlist_get(sl, 4)); - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_clear(sl); - - smartlist_split_string(sl, "abcd\n", "\n", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - test_eq(1, smartlist_len(sl)); - test_streq("abcd", smartlist_get(sl, 0)); - smartlist_split_string(sl, "efgh", "\n", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - test_eq(2, smartlist_len(sl)); - test_streq("efgh", smartlist_get(sl, 1)); - - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_clear(sl); - - /* Test swapping, shuffling, and sorting. */ - smartlist_split_string(sl, "the,onion,router,by,arma,and,nickm", ",", 0, 0); - test_eq(7, smartlist_len(sl)); - smartlist_sort(sl, _compare_strs); - cp_alloc = smartlist_join_strings(sl, ",", 0, NULL); - test_streq(cp_alloc,"and,arma,by,nickm,onion,router,the"); - tor_free(cp_alloc); - smartlist_swap(sl, 1, 5); - cp_alloc = smartlist_join_strings(sl, ",", 0, NULL); - test_streq(cp_alloc,"and,router,by,nickm,onion,arma,the"); - tor_free(cp_alloc); - smartlist_shuffle(sl); - test_eq(7, smartlist_len(sl)); - test_assert(smartlist_string_isin(sl, "and")); - test_assert(smartlist_string_isin(sl, "router")); - test_assert(smartlist_string_isin(sl, "by")); - test_assert(smartlist_string_isin(sl, "nickm")); - test_assert(smartlist_string_isin(sl, "onion")); - test_assert(smartlist_string_isin(sl, "arma")); - test_assert(smartlist_string_isin(sl, "the")); - - /* Test bsearch. */ - smartlist_sort(sl, _compare_strs); - test_streq("nickm", smartlist_bsearch(sl, "zNicKM", - _compare_without_first_ch)); - test_streq("and", smartlist_bsearch(sl, " AND", _compare_without_first_ch)); - test_eq_ptr(NULL, smartlist_bsearch(sl, " ANz", _compare_without_first_ch)); - - /* Test bsearch_idx */ - { - int f; - test_eq(0, smartlist_bsearch_idx(sl," aaa",_compare_without_first_ch,&f)); - test_eq(f, 0); - test_eq(0, smartlist_bsearch_idx(sl," and",_compare_without_first_ch,&f)); - test_eq(f, 1); - test_eq(1, smartlist_bsearch_idx(sl," arm",_compare_without_first_ch,&f)); - test_eq(f, 0); - test_eq(1, smartlist_bsearch_idx(sl," arma",_compare_without_first_ch,&f)); - test_eq(f, 1); - test_eq(2, smartlist_bsearch_idx(sl," armb",_compare_without_first_ch,&f)); - test_eq(f, 0); - test_eq(7, smartlist_bsearch_idx(sl," zzzz",_compare_without_first_ch,&f)); - test_eq(f, 0); - } - - /* Test reverse() and pop_last() */ - smartlist_reverse(sl); - cp_alloc = smartlist_join_strings(sl, ",", 0, NULL); - test_streq(cp_alloc,"the,router,onion,nickm,by,arma,and"); - tor_free(cp_alloc); - cp_alloc = smartlist_pop_last(sl); - test_streq(cp_alloc, "and"); - tor_free(cp_alloc); - test_eq(smartlist_len(sl), 6); - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_clear(sl); - cp_alloc = smartlist_pop_last(sl); - test_eq(cp_alloc, NULL); - - /* Test uniq() */ - smartlist_split_string(sl, - "50,noon,radar,a,man,a,plan,a,canal,panama,radar,noon,50", - ",", 0, 0); - smartlist_sort(sl, _compare_strs); - smartlist_uniq(sl, _compare_strs, _tor_free); - cp_alloc = smartlist_join_strings(sl, ",", 0, NULL); - test_streq(cp_alloc, "50,a,canal,man,noon,panama,plan,radar"); - tor_free(cp_alloc); - - /* Test string_isin and isin_case and num_isin */ - test_assert(smartlist_string_isin(sl, "noon")); - test_assert(!smartlist_string_isin(sl, "noonoon")); - test_assert(smartlist_string_isin_case(sl, "nOOn")); - test_assert(!smartlist_string_isin_case(sl, "nooNooN")); - test_assert(smartlist_string_num_isin(sl, 50)); - test_assert(!smartlist_string_num_isin(sl, 60)); - - /* Test smartlist_choose */ - { - int i; - int allsame = 1; - int allin = 1; - void *first = smartlist_choose(sl); - test_assert(smartlist_isin(sl, first)); - for (i = 0; i < 100; ++i) { - void *second = smartlist_choose(sl); - if (second != first) - allsame = 0; - if (!smartlist_isin(sl, second)) - allin = 0; - } - test_assert(!allsame); - test_assert(allin); - } - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_clear(sl); - - /* Test string_remove and remove and join_strings2 */ - smartlist_split_string(sl, - "Some say the Earth will end in ice and some in fire", - " ", 0, 0); - cp = smartlist_get(sl, 4); - test_streq(cp, "will"); - smartlist_add(sl, cp); - smartlist_remove(sl, cp); - tor_free(cp); - cp_alloc = smartlist_join_strings(sl, ",", 0, NULL); - test_streq(cp_alloc, "Some,say,the,Earth,fire,end,in,ice,and,some,in"); - tor_free(cp_alloc); - smartlist_string_remove(sl, "in"); - cp_alloc = smartlist_join_strings2(sl, "+XX", 1, 0, &sz); - test_streq(cp_alloc, "Some+say+the+Earth+fire+end+some+ice+and"); - test_eq((int)sz, 40); - - done: - - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_free(sl); - tor_free(cp_alloc); -} - -/** Run unit tests for smartlist set manipulation functions. */ -static void -test_util_smartlist_overlap(void) -{ - smartlist_t *sl = smartlist_create(); - smartlist_t *ints = smartlist_create(); - smartlist_t *odds = smartlist_create(); - smartlist_t *evens = smartlist_create(); - smartlist_t *primes = smartlist_create(); - int i; - for (i=1; i < 10; i += 2) - smartlist_add(odds, (void*)(uintptr_t)i); - for (i=0; i < 10; i += 2) - smartlist_add(evens, (void*)(uintptr_t)i); - - /* add_all */ - smartlist_add_all(ints, odds); - smartlist_add_all(ints, evens); - test_eq(smartlist_len(ints), 10); - - smartlist_add(primes, (void*)2); - smartlist_add(primes, (void*)3); - smartlist_add(primes, (void*)5); - smartlist_add(primes, (void*)7); - - /* overlap */ - test_assert(smartlist_overlap(ints, odds)); - test_assert(smartlist_overlap(odds, primes)); - test_assert(smartlist_overlap(evens, primes)); - test_assert(!smartlist_overlap(odds, evens)); - - /* intersect */ - smartlist_add_all(sl, odds); - smartlist_intersect(sl, primes); - test_eq(smartlist_len(sl), 3); - test_assert(smartlist_isin(sl, (void*)3)); - test_assert(smartlist_isin(sl, (void*)5)); - test_assert(smartlist_isin(sl, (void*)7)); - - /* subtract */ - smartlist_add_all(sl, primes); - smartlist_subtract(sl, odds); - test_eq(smartlist_len(sl), 1); - test_assert(smartlist_isin(sl, (void*)2)); - - done: - smartlist_free(odds); - smartlist_free(evens); - smartlist_free(ints); - smartlist_free(primes); - smartlist_free(sl); -} - -/** Run unit tests for smartlist-of-digests functions. */ -static void -test_util_smartlist_digests(void) -{ - smartlist_t *sl = smartlist_create(); - - /* digest_isin. */ - smartlist_add(sl, tor_memdup("AAAAAAAAAAAAAAAAAAAA", DIGEST_LEN)); - smartlist_add(sl, tor_memdup("\00090AAB2AAAAaasdAAAAA", DIGEST_LEN)); - smartlist_add(sl, tor_memdup("\00090AAB2AAAAaasdAAAAA", DIGEST_LEN)); - test_eq(0, smartlist_digest_isin(NULL, "AAAAAAAAAAAAAAAAAAAA")); - test_assert(smartlist_digest_isin(sl, "AAAAAAAAAAAAAAAAAAAA")); - test_assert(smartlist_digest_isin(sl, "\00090AAB2AAAAaasdAAAAA")); - test_eq(0, smartlist_digest_isin(sl, "\00090AAB2AAABaasdAAAAA")); - - /* sort digests */ - smartlist_sort_digests(sl); - test_memeq(smartlist_get(sl, 0), "\00090AAB2AAAAaasdAAAAA", DIGEST_LEN); - test_memeq(smartlist_get(sl, 1), "\00090AAB2AAAAaasdAAAAA", DIGEST_LEN); - test_memeq(smartlist_get(sl, 2), "AAAAAAAAAAAAAAAAAAAA", DIGEST_LEN); - test_eq(3, smartlist_len(sl)); - - /* uniq_digests */ - smartlist_uniq_digests(sl); - test_eq(2, smartlist_len(sl)); - test_memeq(smartlist_get(sl, 0), "\00090AAB2AAAAaasdAAAAA", DIGEST_LEN); - test_memeq(smartlist_get(sl, 1), "AAAAAAAAAAAAAAAAAAAA", DIGEST_LEN); - - done: - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_free(sl); -} - -/** Run unit tests for concatenate-a-smartlist-of-strings functions. */ -static void -test_util_smartlist_join(void) -{ - smartlist_t *sl = smartlist_create(); - smartlist_t *sl2 = smartlist_create(), *sl3 = smartlist_create(), - *sl4 = smartlist_create(); - char *joined=NULL; - /* unique, sorted. */ - smartlist_split_string(sl, - "Abashments Ambush Anchorman Bacon Banks Borscht " - "Bunks Inhumane Insurance Knish Know Manners " - "Maraschinos Stamina Sunbonnets Unicorns Wombats", - " ", 0, 0); - /* non-unique, sorted. */ - smartlist_split_string(sl2, - "Ambush Anchorman Anchorman Anemias Anemias Bacon " - "Crossbowmen Inhumane Insurance Knish Know Manners " - "Manners Maraschinos Wombats Wombats Work", - " ", 0, 0); - SMARTLIST_FOREACH_JOIN(sl, char *, cp1, - sl2, char *, cp2, - strcmp(cp1,cp2), - smartlist_add(sl3, cp2)) { - test_streq(cp1, cp2); - smartlist_add(sl4, cp1); - } SMARTLIST_FOREACH_JOIN_END(cp1, cp2); - - SMARTLIST_FOREACH(sl3, const char *, cp, - test_assert(smartlist_isin(sl2, cp) && - !smartlist_string_isin(sl, cp))); - SMARTLIST_FOREACH(sl4, const char *, cp, - test_assert(smartlist_isin(sl, cp) && - smartlist_string_isin(sl2, cp))); - joined = smartlist_join_strings(sl3, ",", 0, NULL); - test_streq(joined, "Anemias,Anemias,Crossbowmen,Work"); - tor_free(joined); - joined = smartlist_join_strings(sl4, ",", 0, NULL); - test_streq(joined, "Ambush,Anchorman,Anchorman,Bacon,Inhumane,Insurance," - "Knish,Know,Manners,Manners,Maraschinos,Wombats,Wombats"); - tor_free(joined); - - done: - smartlist_free(sl4); - smartlist_free(sl3); - SMARTLIST_FOREACH(sl2, char *, cp, tor_free(cp)); - smartlist_free(sl2); - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_free(sl); - tor_free(joined); -} - -/** Run unit tests for bitarray code */ -static void -test_util_bitarray(void) -{ - bitarray_t *ba = NULL; - int i, j, ok=1; - - ba = bitarray_init_zero(1); - test_assert(ba); - test_assert(! bitarray_is_set(ba, 0)); - bitarray_set(ba, 0); - test_assert(bitarray_is_set(ba, 0)); - bitarray_clear(ba, 0); - test_assert(! bitarray_is_set(ba, 0)); - bitarray_free(ba); - - ba = bitarray_init_zero(1023); - for (i = 1; i < 64; ) { - for (j = 0; j < 1023; ++j) { - if (j % i) - bitarray_set(ba, j); - else - bitarray_clear(ba, j); - } - for (j = 0; j < 1023; ++j) { - if (!bool_eq(bitarray_is_set(ba, j), j%i)) - ok = 0; - } - test_assert(ok); - if (i < 7) - ++i; - else if (i == 28) - i = 32; - else - i += 7; - } - - done: - if (ba) - bitarray_free(ba); -} - -/** Run unit tests for digest set code (implemented as a hashtable or as a - * bloom filter) */ -static void -test_util_digestset(void) -{ - smartlist_t *included = smartlist_create(); - char d[DIGEST_LEN]; - int i; - int ok = 1; - int false_positives = 0; - digestset_t *set = NULL; - - for (i = 0; i < 1000; ++i) { - crypto_rand(d, DIGEST_LEN); - smartlist_add(included, tor_memdup(d, DIGEST_LEN)); - } - set = digestset_new(1000); - SMARTLIST_FOREACH(included, const char *, cp, - if (digestset_isin(set, cp)) - ok = 0); - test_assert(ok); - SMARTLIST_FOREACH(included, const char *, cp, - digestset_add(set, cp)); - SMARTLIST_FOREACH(included, const char *, cp, - if (!digestset_isin(set, cp)) - ok = 0); - test_assert(ok); - for (i = 0; i < 1000; ++i) { - crypto_rand(d, DIGEST_LEN); - if (digestset_isin(set, d)) - ++false_positives; - } - test_assert(false_positives < 50); /* Should be far lower. */ - - done: - if (set) - digestset_free(set); - SMARTLIST_FOREACH(included, char *, cp, tor_free(cp)); - smartlist_free(included); -} - -/** mutex for thread test to stop the threads hitting data at the same time. */ -static tor_mutex_t *_thread_test_mutex = NULL; -/** mutexes for the thread test to make sure that the threads have to - * interleave somewhat. */ -static tor_mutex_t *_thread_test_start1 = NULL, - *_thread_test_start2 = NULL; -/** Shared strmap for the thread test. */ -static strmap_t *_thread_test_strmap = NULL; -/** The name of thread1 for the thread test */ -static char *_thread1_name = NULL; -/** The name of thread2 for the thread test */ -static char *_thread2_name = NULL; - -static void _thread_test_func(void* _s) ATTR_NORETURN; - -/** How many iterations have the threads in the unit test run? */ -static int t1_count = 0, t2_count = 0; - -/** Helper function for threading unit tests: This function runs in a - * subthread. It grabs its own mutex (start1 or start2) to make sure that it - * should start, then it repeatedly alters _test_thread_strmap protected by - * _thread_test_mutex. */ -static void -_thread_test_func(void* _s) -{ - char *s = _s; - int i, *count; - tor_mutex_t *m; - char buf[64]; - char **cp; - if (!strcmp(s, "thread 1")) { - m = _thread_test_start1; - cp = &_thread1_name; - count = &t1_count; - } else { - m = _thread_test_start2; - cp = &_thread2_name; - count = &t2_count; - } - tor_mutex_acquire(m); - - tor_snprintf(buf, sizeof(buf), "%lu", tor_get_thread_id()); - *cp = tor_strdup(buf); - - for (i=0; i<10000; ++i) { - tor_mutex_acquire(_thread_test_mutex); - strmap_set(_thread_test_strmap, "last to run", *cp); - ++*count; - tor_mutex_release(_thread_test_mutex); - } - tor_mutex_acquire(_thread_test_mutex); - strmap_set(_thread_test_strmap, s, *cp); - tor_mutex_release(_thread_test_mutex); - - tor_mutex_release(m); - - spawn_exit(); -} - -/** Run unit tests for threading logic. */ -static void -test_util_threads(void) -{ - char *s1 = NULL, *s2 = NULL; - int done = 0, timedout = 0; - time_t started; -#ifndef TOR_IS_MULTITHREADED - /* Skip this test if we aren't threading. We should be threading most - * everywhere by now. */ - if (1) - return; -#endif - _thread_test_mutex = tor_mutex_new(); - _thread_test_start1 = tor_mutex_new(); - _thread_test_start2 = tor_mutex_new(); - _thread_test_strmap = strmap_new(); - s1 = tor_strdup("thread 1"); - s2 = tor_strdup("thread 2"); - tor_mutex_acquire(_thread_test_start1); - tor_mutex_acquire(_thread_test_start2); - spawn_func(_thread_test_func, s1); - spawn_func(_thread_test_func, s2); - tor_mutex_release(_thread_test_start2); - tor_mutex_release(_thread_test_start1); - started = time(NULL); - while (!done) { - tor_mutex_acquire(_thread_test_mutex); - strmap_assert_ok(_thread_test_strmap); - if (strmap_get(_thread_test_strmap, "thread 1") && - strmap_get(_thread_test_strmap, "thread 2")) { - done = 1; - } else if (time(NULL) > started + 25) { - timedout = done = 1; - } - tor_mutex_release(_thread_test_mutex); - } - tor_mutex_free(_thread_test_mutex); - - tor_mutex_acquire(_thread_test_start1); - tor_mutex_release(_thread_test_start1); - tor_mutex_acquire(_thread_test_start2); - tor_mutex_release(_thread_test_start2); - - if (timedout) { - printf("\nTimed out: %d %d", t1_count, t2_count); - test_assert(strmap_get(_thread_test_strmap, "thread 1")); - test_assert(strmap_get(_thread_test_strmap, "thread 2")); - test_assert(!timedout); - } - - /* different thread IDs. */ - test_assert(strcmp(strmap_get(_thread_test_strmap, "thread 1"), - strmap_get(_thread_test_strmap, "thread 2"))); - test_assert(!strcmp(strmap_get(_thread_test_strmap, "thread 1"), - strmap_get(_thread_test_strmap, "last to run")) || - !strcmp(strmap_get(_thread_test_strmap, "thread 2"), - strmap_get(_thread_test_strmap, "last to run"))); - - done: - tor_free(s1); - tor_free(s2); - tor_free(_thread1_name); - tor_free(_thread2_name); - if (_thread_test_strmap) - strmap_free(_thread_test_strmap, NULL); - if (_thread_test_start1) - tor_mutex_free(_thread_test_start1); - if (_thread_test_start2) - tor_mutex_free(_thread_test_start2); -} - -/** Helper: return a tristate based on comparing two strings. */ -static int -_compare_strings_for_pqueue(const void *s1, const void *s2) -{ - return strcmp((const char*)s1, (const char*)s2); -} - -/** Run unit tests for heap-based priority queue functions. */ -static void -test_util_pqueue(void) -{ - smartlist_t *sl = smartlist_create(); - int (*cmp)(const void *, const void*); -#define OK() smartlist_pqueue_assert_ok(sl, cmp) - - cmp = _compare_strings_for_pqueue; - - smartlist_pqueue_add(sl, cmp, (char*)"cows"); - smartlist_pqueue_add(sl, cmp, (char*)"zebras"); - smartlist_pqueue_add(sl, cmp, (char*)"fish"); - smartlist_pqueue_add(sl, cmp, (char*)"frogs"); - smartlist_pqueue_add(sl, cmp, (char*)"apples"); - smartlist_pqueue_add(sl, cmp, (char*)"squid"); - smartlist_pqueue_add(sl, cmp, (char*)"daschunds"); - smartlist_pqueue_add(sl, cmp, (char*)"eggplants"); - smartlist_pqueue_add(sl, cmp, (char*)"weissbier"); - smartlist_pqueue_add(sl, cmp, (char*)"lobsters"); - smartlist_pqueue_add(sl, cmp, (char*)"roquefort"); - - OK(); - - test_eq(smartlist_len(sl), 11); - test_streq(smartlist_get(sl, 0), "apples"); - test_streq(smartlist_pqueue_pop(sl, cmp), "apples"); - test_eq(smartlist_len(sl), 10); - OK(); - test_streq(smartlist_pqueue_pop(sl, cmp), "cows"); - test_streq(smartlist_pqueue_pop(sl, cmp), "daschunds"); - smartlist_pqueue_add(sl, cmp, (char*)"chinchillas"); - OK(); - smartlist_pqueue_add(sl, cmp, (char*)"fireflies"); - OK(); - test_streq(smartlist_pqueue_pop(sl, cmp), "chinchillas"); - test_streq(smartlist_pqueue_pop(sl, cmp), "eggplants"); - test_streq(smartlist_pqueue_pop(sl, cmp), "fireflies"); - OK(); - test_streq(smartlist_pqueue_pop(sl, cmp), "fish"); - test_streq(smartlist_pqueue_pop(sl, cmp), "frogs"); - test_streq(smartlist_pqueue_pop(sl, cmp), "lobsters"); - test_streq(smartlist_pqueue_pop(sl, cmp), "roquefort"); - OK(); - test_eq(smartlist_len(sl), 3); - test_streq(smartlist_pqueue_pop(sl, cmp), "squid"); - test_streq(smartlist_pqueue_pop(sl, cmp), "weissbier"); - test_streq(smartlist_pqueue_pop(sl, cmp), "zebras"); - test_eq(smartlist_len(sl), 0); - OK(); -#undef OK - - done: - - smartlist_free(sl); -} - -/** Run unit tests for compression functions */ -static void -test_util_gzip(void) -{ - char *buf1=NULL, *buf2=NULL, *buf3=NULL, *cp1, *cp2; - const char *ccp2; - size_t len1, len2; - tor_zlib_state_t *state = NULL; - - buf1 = tor_strdup("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ"); - test_assert(detect_compression_method(buf1, strlen(buf1)) == UNKNOWN_METHOD); - if (is_gzip_supported()) { - test_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1, - GZIP_METHOD)); - test_assert(buf2); - test_assert(!memcmp(buf2, "\037\213", 2)); /* Gzip magic. */ - test_assert(detect_compression_method(buf2, len1) == GZIP_METHOD); - - test_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1, - GZIP_METHOD, 1, LOG_INFO)); - test_assert(buf3); - test_streq(buf1,buf3); - - tor_free(buf2); - tor_free(buf3); - } - - test_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1, - ZLIB_METHOD)); - test_assert(buf2); - test_assert(!memcmp(buf2, "\x78\xDA", 2)); /* deflate magic. */ - test_assert(detect_compression_method(buf2, len1) == ZLIB_METHOD); - - test_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1, - ZLIB_METHOD, 1, LOG_INFO)); - test_assert(buf3); - test_streq(buf1,buf3); - - /* Check whether we can uncompress concatenated, compressed strings. */ - tor_free(buf3); - buf2 = tor_realloc(buf2, len1*2); - memcpy(buf2+len1, buf2, len1); - test_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1*2, - ZLIB_METHOD, 1, LOG_INFO)); - test_eq(len2, (strlen(buf1)+1)*2); - test_memeq(buf3, - "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ\0" - "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ\0", - (strlen(buf1)+1)*2); - - tor_free(buf1); - tor_free(buf2); - tor_free(buf3); - - /* Check whether we can uncompress partial strings. */ - buf1 = - tor_strdup("String with low redundancy that won't be compressed much."); - test_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1, - ZLIB_METHOD)); - tor_assert(len1>16); - /* when we allow an incomplete string, we should succeed.*/ - tor_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1-16, - ZLIB_METHOD, 0, LOG_INFO)); - buf3[len2]='\0'; - tor_assert(len2 > 5); - tor_assert(!strcmpstart(buf1, buf3)); - - /* when we demand a complete string, this must fail. */ - tor_free(buf3); - tor_assert(tor_gzip_uncompress(&buf3, &len2, buf2, len1-16, - ZLIB_METHOD, 1, LOG_INFO)); - tor_assert(!buf3); - - /* Now, try streaming compression. */ - tor_free(buf1); - tor_free(buf2); - tor_free(buf3); - state = tor_zlib_new(1, ZLIB_METHOD); - tor_assert(state); - cp1 = buf1 = tor_malloc(1024); - len1 = 1024; - ccp2 = "ABCDEFGHIJABCDEFGHIJ"; - len2 = 21; - test_assert(tor_zlib_process(state, &cp1, &len1, &ccp2, &len2, 0) - == TOR_ZLIB_OK); - test_eq(len2, 0); /* Make sure we compressed it all. */ - test_assert(cp1 > buf1); - - len2 = 0; - cp2 = cp1; - test_assert(tor_zlib_process(state, &cp1, &len1, &ccp2, &len2, 1) - == TOR_ZLIB_DONE); - test_eq(len2, 0); - test_assert(cp1 > cp2); /* Make sure we really added something. */ - - tor_assert(!tor_gzip_uncompress(&buf3, &len2, buf1, 1024-len1, - ZLIB_METHOD, 1, LOG_WARN)); - test_streq(buf3, "ABCDEFGHIJABCDEFGHIJ"); /*Make sure it compressed right.*/ - - done: - if (state) - tor_zlib_free(state); - tor_free(buf2); - tor_free(buf3); - tor_free(buf1); -} - -/** Run unit tests for string-to-void* map functions */ -static void -test_util_strmap(void) -{ - strmap_t *map; - strmap_iter_t *iter; - const char *k; - void *v; - char *visited = NULL; - smartlist_t *found_keys = NULL; - - map = strmap_new(); - test_assert(map); - test_eq(strmap_size(map), 0); - test_assert(strmap_isempty(map)); - v = strmap_set(map, "K1", (void*)99); - test_eq(v, NULL); - test_assert(!strmap_isempty(map)); - v = strmap_set(map, "K2", (void*)101); - test_eq(v, NULL); - v = strmap_set(map, "K1", (void*)100); - test_eq(v, (void*)99); - test_eq_ptr(strmap_get(map,"K1"), (void*)100); - test_eq_ptr(strmap_get(map,"K2"), (void*)101); - test_eq_ptr(strmap_get(map,"K-not-there"), NULL); - strmap_assert_ok(map); - - v = strmap_remove(map,"K2"); - strmap_assert_ok(map); - test_eq_ptr(v, (void*)101); - test_eq_ptr(strmap_get(map,"K2"), NULL); - test_eq_ptr(strmap_remove(map,"K2"), NULL); - - strmap_set(map, "K2", (void*)101); - strmap_set(map, "K3", (void*)102); - strmap_set(map, "K4", (void*)103); - test_eq(strmap_size(map), 4); - strmap_assert_ok(map); - strmap_set(map, "K5", (void*)104); - strmap_set(map, "K6", (void*)105); - strmap_assert_ok(map); - - /* Test iterator. */ - iter = strmap_iter_init(map); - found_keys = smartlist_create(); - while (!strmap_iter_done(iter)) { - strmap_iter_get(iter,&k,&v); - smartlist_add(found_keys, tor_strdup(k)); - test_eq_ptr(v, strmap_get(map, k)); - - if (!strcmp(k, "K2")) { - iter = strmap_iter_next_rmv(map,iter); - } else { - iter = strmap_iter_next(map,iter); - } - } - - /* Make sure we removed K2, but not the others. */ - test_eq_ptr(strmap_get(map, "K2"), NULL); - test_eq_ptr(strmap_get(map, "K5"), (void*)104); - /* Make sure we visited everyone once */ - smartlist_sort_strings(found_keys); - visited = smartlist_join_strings(found_keys, ":", 0, NULL); - test_streq(visited, "K1:K2:K3:K4:K5:K6"); - - strmap_assert_ok(map); - /* Clean up after ourselves. */ - strmap_free(map, NULL); - map = NULL; - - /* Now try some lc functions. */ - map = strmap_new(); - strmap_set_lc(map,"Ab.C", (void*)1); - test_eq_ptr(strmap_get(map,"ab.c"), (void*)1); - strmap_assert_ok(map); - test_eq_ptr(strmap_get_lc(map,"AB.C"), (void*)1); - test_eq_ptr(strmap_get(map,"AB.C"), NULL); - test_eq_ptr(strmap_remove_lc(map,"aB.C"), (void*)1); - strmap_assert_ok(map); - test_eq_ptr(strmap_get_lc(map,"AB.C"), NULL); - - done: - if (map) - strmap_free(map,NULL); - if (found_keys) { - SMARTLIST_FOREACH(found_keys, char *, cp, tor_free(cp)); - smartlist_free(found_keys); - } - tor_free(visited); -} - -/** Run unit tests for mmap() wrapper functionality. */ -static void -test_util_mmap(void) -{ - char *fname1 = tor_strdup(get_fname("mapped_1")); - char *fname2 = tor_strdup(get_fname("mapped_2")); - char *fname3 = tor_strdup(get_fname("mapped_3")); - const size_t buflen = 17000; - char *buf = tor_malloc(17000); - tor_mmap_t *mapping = NULL; - - crypto_rand(buf, buflen); - - mapping = tor_mmap_file(fname1); - test_assert(! mapping); - - write_str_to_file(fname1, "Short file.", 1); - write_bytes_to_file(fname2, buf, buflen, 1); - write_bytes_to_file(fname3, buf, 16384, 1); - - mapping = tor_mmap_file(fname1); - test_assert(mapping); - test_eq(mapping->size, strlen("Short file.")); - test_streq(mapping->data, "Short file."); -#ifdef MS_WINDOWS - tor_munmap_file(mapping); - mapping = NULL; - test_assert(unlink(fname1) == 0); -#else - /* make sure we can unlink. */ - test_assert(unlink(fname1) == 0); - test_streq(mapping->data, "Short file."); - tor_munmap_file(mapping); - mapping = NULL; -#endif - - /* Now a zero-length file. */ - write_str_to_file(fname1, "", 1); - mapping = tor_mmap_file(fname1); - test_eq(mapping, NULL); - test_eq(ERANGE, errno); - unlink(fname1); - - /* Make sure that we fail to map a no-longer-existent file. */ - mapping = tor_mmap_file(fname1); - test_assert(mapping == NULL); - - /* Now try a big file that stretches across a few pages and isn't aligned */ - mapping = tor_mmap_file(fname2); - test_assert(mapping); - test_eq(mapping->size, buflen); - test_memeq(mapping->data, buf, buflen); - tor_munmap_file(mapping); - mapping = NULL; - - /* Now try a big aligned file. */ - mapping = tor_mmap_file(fname3); - test_assert(mapping); - test_eq(mapping->size, 16384); - test_memeq(mapping->data, buf, 16384); - tor_munmap_file(mapping); - mapping = NULL; - - done: - unlink(fname1); - unlink(fname2); - unlink(fname3); - - tor_free(fname1); - tor_free(fname2); - tor_free(fname3); - tor_free(buf); - - if (mapping) - tor_munmap_file(mapping); -} - -/** Run unit tests for escaping/unescaping data for use by controllers. */ -static void -test_util_control_formats(void) -{ - char *out = NULL; - const char *inp = - "..This is a test\r\nof the emergency \nbroadcast\r\n..system.\r\nZ.\r\n"; - size_t sz; - - sz = read_escaped_data(inp, strlen(inp), &out); - test_streq(out, - ".This is a test\nof the emergency \nbroadcast\n.system.\nZ.\n"); - test_eq(sz, strlen(out)); - - done: - tor_free(out); -} - -static void -test_util_sscanf(void) -{ - unsigned u1, u2, u3; - char s1[10], s2[10], s3[10], ch; - int r; - - r = tor_sscanf("hello world", "hello world"); /* String match: success */ - test_eq(r, 0); - r = tor_sscanf("hello world 3", "hello worlb %u", &u1); /* String fail */ - test_eq(r, 0); - r = tor_sscanf("12345", "%u", &u1); /* Simple number */ - test_eq(r, 1); - test_eq(u1, 12345u); - r = tor_sscanf("", "%u", &u1); /* absent number */ - test_eq(r, 0); - r = tor_sscanf("A", "%u", &u1); /* bogus number */ - test_eq(r, 0); - r = tor_sscanf("4294967295", "%u", &u1); /* UINT32_MAX should work. */ - test_eq(r, 1); - test_eq(u1, 4294967295u); - r = tor_sscanf("4294967296", "%u", &u1); /* Always say -1 at 32 bits. */ - test_eq(r, 0); - r = tor_sscanf("123456", "%2u%u", &u1, &u2); /* Width */ - test_eq(r, 2); - test_eq(u1, 12u); - test_eq(u2, 3456u); - r = tor_sscanf("!12:3:456", "!%2u:%2u:%3u", &u1, &u2, &u3); /* separators */ - test_eq(r, 3); - test_eq(u1, 12u); - test_eq(u2, 3u); - test_eq(u3, 456u); - r = tor_sscanf("12:3:045", "%2u:%2u:%3u", &u1, &u2, &u3); /* 0s */ - test_eq(r, 3); - test_eq(u1, 12u); - test_eq(u2, 3u); - test_eq(u3, 45u); - /* %u does not match space.*/ - r = tor_sscanf("12:3: 45", "%2u:%2u:%3u", &u1, &u2, &u3); - test_eq(r, 2); - /* %u does not match negative numbers. */ - r = tor_sscanf("12:3:-4", "%2u:%2u:%3u", &u1, &u2, &u3); - test_eq(r, 2); - /* Arbitrary amounts of 0-padding are okay */ - r = tor_sscanf("12:03:000000000000000099", "%2u:%2u:%u", &u1, &u2, &u3); - test_eq(r, 3); - test_eq(u1, 12u); - test_eq(u2, 3u); - test_eq(u3, 99u); - - r = tor_sscanf("99% fresh", "%3u%% fresh", &u1); /* percents are scannable.*/ - test_eq(r, 1); - test_eq(u1, 99); - - r = tor_sscanf("hello", "%s", s1); /* %s needs a number. */ - test_eq(r, -1); - - r = tor_sscanf("hello", "%3s%7s", s1, s2); /* %s matches characters. */ - test_eq(r, 2); - test_streq(s1, "hel"); - test_streq(s2, "lo"); - r = tor_sscanf("WD40", "%2s%u", s3, &u1); /* %s%u */ - test_eq(r, 2); - test_streq(s3, "WD"); - test_eq(u1, 40); - r = tor_sscanf("76trombones", "%6u%9s", &u1, s1); /* %u%s */ - test_eq(r, 2); - test_eq(u1, 76); - test_streq(s1, "trombones"); - r = tor_sscanf("hello world", "%9s %9s", s1, s2); /* %s doesn't eat space. */ - test_eq(r, 2); - test_streq(s1, "hello"); - test_streq(s2, "world"); - r = tor_sscanf("hi", "%9s%9s%3s", s1, s2, s3); /* %s can be empty. */ - test_eq(r, 3); - test_streq(s1, "hi"); - test_streq(s2, ""); - test_streq(s3, ""); - - r = tor_sscanf("1.2.3", "%u.%u.%u%c", &u1, &u2, &u3, &ch); - test_eq(r, 3); - r = tor_sscanf("1.2.3 foobar", "%u.%u.%u%c", &u1, &u2, &u3, &ch); - test_eq(r, 4); - - done: - ; -} - -/** Run unit tests for the onion handshake code. */ -static void -test_onion_handshake(void) -{ - /* client-side */ - crypto_dh_env_t *c_dh = NULL; - char c_buf[ONIONSKIN_CHALLENGE_LEN]; - char c_keys[40]; - - /* server-side */ - char s_buf[ONIONSKIN_REPLY_LEN]; - char s_keys[40]; - - /* shared */ - crypto_pk_env_t *pk = NULL; - - pk = pk_generate(0); - - /* client handshake 1. */ - memset(c_buf, 0, ONIONSKIN_CHALLENGE_LEN); - test_assert(! onion_skin_create(pk, &c_dh, c_buf)); - - /* server handshake */ - memset(s_buf, 0, ONIONSKIN_REPLY_LEN); - memset(s_keys, 0, 40); - test_assert(! onion_skin_server_handshake(c_buf, pk, NULL, - s_buf, s_keys, 40)); - - /* client handshake 2 */ - memset(c_keys, 0, 40); - test_assert(! onion_skin_client_handshake(c_dh, s_buf, c_keys, 40)); - - if (memcmp(c_keys, s_keys, 40)) { - puts("Aiiiie"); - exit(1); - } - test_memeq(c_keys, s_keys, 40); - memset(s_buf, 0, 40); - test_memneq(c_keys, s_buf, 40); - - done: - if (c_dh) - crypto_dh_free(c_dh); - if (pk) - crypto_free_pk_env(pk); -} - -/** Run unit tests for router descriptor generation logic. */ -static void -test_dir_format(void) -{ - char buf[8192], buf2[8192]; - char platform[256]; - char fingerprint[FINGERPRINT_LEN+1]; - char *pk1_str = NULL, *pk2_str = NULL, *pk3_str = NULL, *cp; - size_t pk1_str_len, pk2_str_len, pk3_str_len; - routerinfo_t *r1=NULL, *r2=NULL; - crypto_pk_env_t *pk1 = NULL, *pk2 = NULL, *pk3 = NULL; - routerinfo_t *rp1 = NULL; - addr_policy_t *ex1, *ex2; - routerlist_t *dir1 = NULL, *dir2 = NULL; - tor_version_t ver1; - - pk1 = pk_generate(0); - pk2 = pk_generate(1); - pk3 = pk_generate(2); - - test_assert( is_legal_nickname("a")); - test_assert(!is_legal_nickname("")); - test_assert(!is_legal_nickname("abcdefghijklmnopqrst")); /* 20 chars */ - test_assert(!is_legal_nickname("hyphen-")); /* bad char */ - test_assert( is_legal_nickname("abcdefghijklmnopqrs")); /* 19 chars */ - test_assert(!is_legal_nickname("$AAAAAAAA01234AAAAAAAAAAAAAAAAAAAAAAAAAAA")); - /* valid */ - test_assert( is_legal_nickname_or_hexdigest( - "$AAAAAAAA01234AAAAAAAAAAAAAAAAAAAAAAAAAAA")); - test_assert( is_legal_nickname_or_hexdigest( - "$AAAAAAAA01234AAAAAAAAAAAAAAAAAAAAAAAAAAA=fred")); - test_assert( is_legal_nickname_or_hexdigest( - "$AAAAAAAA01234AAAAAAAAAAAAAAAAAAAAAAAAAAA~fred")); - /* too short */ - test_assert(!is_legal_nickname_or_hexdigest( - "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")); - /* illegal char */ - test_assert(!is_legal_nickname_or_hexdigest( - "$AAAAAAzAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")); - /* hex part too long */ - test_assert(!is_legal_nickname_or_hexdigest( - "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")); - test_assert(!is_legal_nickname_or_hexdigest( - "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=fred")); - /* Bad nickname */ - test_assert(!is_legal_nickname_or_hexdigest( - "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=")); - test_assert(!is_legal_nickname_or_hexdigest( - "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~")); - test_assert(!is_legal_nickname_or_hexdigest( - "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~hyphen-")); - test_assert(!is_legal_nickname_or_hexdigest( - "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~" - "abcdefghijklmnoppqrst")); - /* Bad extra char. */ - test_assert(!is_legal_nickname_or_hexdigest( - "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA!")); - test_assert(is_legal_nickname_or_hexdigest("xyzzy")); - test_assert(is_legal_nickname_or_hexdigest("abcdefghijklmnopqrs")); - test_assert(!is_legal_nickname_or_hexdigest("abcdefghijklmnopqrst")); - - get_platform_str(platform, sizeof(platform)); - r1 = tor_malloc_zero(sizeof(routerinfo_t)); - r1->address = tor_strdup("18.244.0.1"); - r1->addr = 0xc0a80001u; /* 192.168.0.1 */ - r1->cache_info.published_on = 0; - r1->or_port = 9000; - r1->dir_port = 9003; - r1->onion_pkey = crypto_pk_dup_key(pk1); - r1->identity_pkey = crypto_pk_dup_key(pk2); - r1->bandwidthrate = 1000; - r1->bandwidthburst = 5000; - r1->bandwidthcapacity = 10000; - r1->exit_policy = NULL; - r1->nickname = tor_strdup("Magri"); - r1->platform = tor_strdup(platform); - - ex1 = tor_malloc_zero(sizeof(addr_policy_t)); - ex2 = tor_malloc_zero(sizeof(addr_policy_t)); - ex1->policy_type = ADDR_POLICY_ACCEPT; - tor_addr_from_ipv4h(&ex1->addr, 0); - ex1->maskbits = 0; - ex1->prt_min = ex1->prt_max = 80; - ex2->policy_type = ADDR_POLICY_REJECT; - tor_addr_from_ipv4h(&ex2->addr, 18<<24); - ex2->maskbits = 8; - ex2->prt_min = ex2->prt_max = 24; - r2 = tor_malloc_zero(sizeof(routerinfo_t)); - r2->address = tor_strdup("1.1.1.1"); - r2->addr = 0x0a030201u; /* 10.3.2.1 */ - r2->platform = tor_strdup(platform); - r2->cache_info.published_on = 5; - r2->or_port = 9005; - r2->dir_port = 0; - r2->onion_pkey = crypto_pk_dup_key(pk2); - r2->identity_pkey = crypto_pk_dup_key(pk1); - r2->bandwidthrate = r2->bandwidthburst = r2->bandwidthcapacity = 3000; - r2->exit_policy = smartlist_create(); - smartlist_add(r2->exit_policy, ex2); - smartlist_add(r2->exit_policy, ex1); - r2->nickname = tor_strdup("Fred"); - - test_assert(!crypto_pk_write_public_key_to_string(pk1, &pk1_str, - &pk1_str_len)); - test_assert(!crypto_pk_write_public_key_to_string(pk2 , &pk2_str, - &pk2_str_len)); - test_assert(!crypto_pk_write_public_key_to_string(pk3 , &pk3_str, - &pk3_str_len)); - - memset(buf, 0, 2048); - test_assert(router_dump_router_to_string(buf, 2048, r1, pk2)>0); - - strlcpy(buf2, "router Magri 18.244.0.1 9000 0 9003\n" - "platform Tor "VERSION" on ", sizeof(buf2)); - strlcat(buf2, get_uname(), sizeof(buf2)); - strlcat(buf2, "\n" - "opt protocols Link 1 2 Circuit 1\n" - "published 1970-01-01 00:00:00\n" - "opt fingerprint ", sizeof(buf2)); - test_assert(!crypto_pk_get_fingerprint(pk2, fingerprint, 1)); - strlcat(buf2, fingerprint, sizeof(buf2)); - strlcat(buf2, "\nuptime 0\n" - /* XXX the "0" above is hard-coded, but even if we made it reflect - * uptime, that still wouldn't make it right, because the two - * descriptors might be made on different seconds... hm. */ - "bandwidth 1000 5000 10000\n" - "opt extra-info-digest 0000000000000000000000000000000000000000\n" - "onion-key\n", sizeof(buf2)); - strlcat(buf2, pk1_str, sizeof(buf2)); - strlcat(buf2, "signing-key\n", sizeof(buf2)); - strlcat(buf2, pk2_str, sizeof(buf2)); - strlcat(buf2, "opt hidden-service-dir\n", sizeof(buf2)); - strlcat(buf2, "reject *:*\nrouter-signature\n", sizeof(buf2)); - buf[strlen(buf2)] = '\0'; /* Don't compare the sig; it's never the same - * twice */ - - test_streq(buf, buf2); - - test_assert(router_dump_router_to_string(buf, 2048, r1, pk2)>0); - cp = buf; - rp1 = router_parse_entry_from_string((const char*)cp,NULL,1,0,NULL); - test_assert(rp1); - test_streq(rp1->address, r1->address); - test_eq(rp1->or_port, r1->or_port); - //test_eq(rp1->dir_port, r1->dir_port); - test_eq(rp1->bandwidthrate, r1->bandwidthrate); - test_eq(rp1->bandwidthburst, r1->bandwidthburst); - test_eq(rp1->bandwidthcapacity, r1->bandwidthcapacity); - test_assert(crypto_pk_cmp_keys(rp1->onion_pkey, pk1) == 0); - test_assert(crypto_pk_cmp_keys(rp1->identity_pkey, pk2) == 0); - //test_assert(rp1->exit_policy == NULL); - -#if 0 - /* XXX Once we have exit policies, test this again. XXX */ - strlcpy(buf2, "router tor.tor.tor 9005 0 0 3000\n", sizeof(buf2)); - strlcat(buf2, pk2_str, sizeof(buf2)); - strlcat(buf2, "signing-key\n", sizeof(buf2)); - strlcat(buf2, pk1_str, sizeof(buf2)); - strlcat(buf2, "accept *:80\nreject 18.*:24\n\n", sizeof(buf2)); - test_assert(router_dump_router_to_string(buf, 2048, &r2, pk2)>0); - test_streq(buf, buf2); - - cp = buf; - rp2 = router_parse_entry_from_string(&cp,1); - test_assert(rp2); - test_streq(rp2->address, r2.address); - test_eq(rp2->or_port, r2.or_port); - test_eq(rp2->dir_port, r2.dir_port); - test_eq(rp2->bandwidth, r2.bandwidth); - test_assert(crypto_pk_cmp_keys(rp2->onion_pkey, pk2) == 0); - test_assert(crypto_pk_cmp_keys(rp2->identity_pkey, pk1) == 0); - test_eq(rp2->exit_policy->policy_type, EXIT_POLICY_ACCEPT); - test_streq(rp2->exit_policy->string, "accept *:80"); - test_streq(rp2->exit_policy->address, "*"); - test_streq(rp2->exit_policy->port, "80"); - test_eq(rp2->exit_policy->next->policy_type, EXIT_POLICY_REJECT); - test_streq(rp2->exit_policy->next->string, "reject 18.*:24"); - test_streq(rp2->exit_policy->next->address, "18.*"); - test_streq(rp2->exit_policy->next->port, "24"); - test_assert(rp2->exit_policy->next->next == NULL); - - /* Okay, now for the directories. */ - { - fingerprint_list = smartlist_create(); - crypto_pk_get_fingerprint(pk2, buf, 1); - add_fingerprint_to_dir("Magri", buf, fingerprint_list); - crypto_pk_get_fingerprint(pk1, buf, 1); - add_fingerprint_to_dir("Fred", buf, fingerprint_list); - } - - { - char d[DIGEST_LEN]; - const char *m; - /* XXXX NM re-enable. */ - /* Make sure routers aren't too far in the past any more. */ - r1->cache_info.published_on = time(NULL); - r2->cache_info.published_on = time(NULL)-3*60*60; - test_assert(router_dump_router_to_string(buf, 2048, r1, pk2)>0); - test_eq(dirserv_add_descriptor(buf,&m,""), ROUTER_ADDED_NOTIFY_GENERATOR); - test_assert(router_dump_router_to_string(buf, 2048, r2, pk1)>0); - test_eq(dirserv_add_descriptor(buf,&m,""), ROUTER_ADDED_NOTIFY_GENERATOR); - get_options()->Nickname = tor_strdup("DirServer"); - test_assert(!dirserv_dump_directory_to_string(&cp,pk3, 0)); - crypto_pk_get_digest(pk3, d); - test_assert(!router_parse_directory(cp)); - test_eq(2, smartlist_len(dir1->routers)); - tor_free(cp); - } -#endif - dirserv_free_fingerprint_list(); - - /* Try out version parsing functionality */ - test_eq(0, tor_version_parse("0.3.4pre2-cvs", &ver1)); - test_eq(0, ver1.major); - test_eq(3, ver1.minor); - test_eq(4, ver1.micro); - test_eq(VER_PRE, ver1.status); - test_eq(2, ver1.patchlevel); - test_eq(0, tor_version_parse("0.3.4rc1", &ver1)); - test_eq(0, ver1.major); - test_eq(3, ver1.minor); - test_eq(4, ver1.micro); - test_eq(VER_RC, ver1.status); - test_eq(1, ver1.patchlevel); - test_eq(0, tor_version_parse("1.3.4", &ver1)); - test_eq(1, ver1.major); - test_eq(3, ver1.minor); - test_eq(4, ver1.micro); - test_eq(VER_RELEASE, ver1.status); - test_eq(0, ver1.patchlevel); - test_eq(0, tor_version_parse("1.3.4.999", &ver1)); - test_eq(1, ver1.major); - test_eq(3, ver1.minor); - test_eq(4, ver1.micro); - test_eq(VER_RELEASE, ver1.status); - test_eq(999, ver1.patchlevel); - test_eq(0, tor_version_parse("0.1.2.4-alpha", &ver1)); - test_eq(0, ver1.major); - test_eq(1, ver1.minor); - test_eq(2, ver1.micro); - test_eq(4, ver1.patchlevel); - test_eq(VER_RELEASE, ver1.status); - test_streq("alpha", ver1.status_tag); - test_eq(0, tor_version_parse("0.1.2.4", &ver1)); - test_eq(0, ver1.major); - test_eq(1, ver1.minor); - test_eq(2, ver1.micro); - test_eq(4, ver1.patchlevel); - test_eq(VER_RELEASE, ver1.status); - test_streq("", ver1.status_tag); - -#define test_eq_vs(vs1, vs2) test_eq_type(version_status_t, "%d", (vs1), (vs2)) -#define test_v_i_o(val, ver, lst) \ - test_eq_vs(val, tor_version_is_obsolete(ver, lst)) - - /* make sure tor_version_is_obsolete() works */ - test_v_i_o(VS_OLD, "0.0.1", "Tor 0.0.2"); - test_v_i_o(VS_OLD, "0.0.1", "0.0.2, Tor 0.0.3"); - test_v_i_o(VS_OLD, "0.0.1", "0.0.2,Tor 0.0.3"); - test_v_i_o(VS_OLD, "0.0.1","0.0.3,BetterTor 0.0.1"); - test_v_i_o(VS_RECOMMENDED, "0.0.2", "Tor 0.0.2,Tor 0.0.3"); - test_v_i_o(VS_NEW_IN_SERIES, "0.0.2", "Tor 0.0.2pre1,Tor 0.0.3"); - test_v_i_o(VS_OLD, "0.0.2", "Tor 0.0.2.1,Tor 0.0.3"); - test_v_i_o(VS_NEW, "0.1.0", "Tor 0.0.2,Tor 0.0.3"); - test_v_i_o(VS_RECOMMENDED, "0.0.7rc2", "0.0.7,Tor 0.0.7rc2,Tor 0.0.8"); - test_v_i_o(VS_OLD, "0.0.5.0", "0.0.5.1-cvs"); - test_v_i_o(VS_NEW_IN_SERIES, "0.0.5.1-cvs", "0.0.5, 0.0.6"); - /* Not on list, but newer than any in same series. */ - test_v_i_o(VS_NEW_IN_SERIES, "0.1.0.3", - "Tor 0.1.0.2,Tor 0.0.9.5,Tor 0.1.1.0"); - /* Series newer than any on list. */ - test_v_i_o(VS_NEW, "0.1.2.3", "Tor 0.1.0.2,Tor 0.0.9.5,Tor 0.1.1.0"); - /* Series older than any on list. */ - test_v_i_o(VS_OLD, "0.0.1.3", "Tor 0.1.0.2,Tor 0.0.9.5,Tor 0.1.1.0"); - /* Not on list, not newer than any on same series. */ - test_v_i_o(VS_UNRECOMMENDED, "0.1.0.1", - "Tor 0.1.0.2,Tor 0.0.9.5,Tor 0.1.1.0"); - /* On list, not newer than any on same series. */ - test_v_i_o(VS_UNRECOMMENDED, - "0.1.0.1", "Tor 0.1.0.2,Tor 0.0.9.5,Tor 0.1.1.0"); - test_eq(0, tor_version_as_new_as("Tor 0.0.5", "0.0.9pre1-cvs")); - test_eq(1, tor_version_as_new_as( - "Tor 0.0.8 on Darwin 64-121-192-100.c3-0." - "sfpo-ubr1.sfrn-sfpo.ca.cable.rcn.com Power Macintosh", - "0.0.8rc2")); - test_eq(0, tor_version_as_new_as( - "Tor 0.0.8 on Darwin 64-121-192-100.c3-0." - "sfpo-ubr1.sfrn-sfpo.ca.cable.rcn.com Power Macintosh", "0.0.8.2")); - - /* Now try svn revisions. */ - test_eq(1, tor_version_as_new_as("Tor 0.2.1.0-dev (r100)", - "Tor 0.2.1.0-dev (r99)")); - test_eq(1, tor_version_as_new_as("Tor 0.2.1.0-dev (r100) on Banana Jr", - "Tor 0.2.1.0-dev (r99) on Hal 9000")); - test_eq(1, tor_version_as_new_as("Tor 0.2.1.0-dev (r100)", - "Tor 0.2.1.0-dev on Colossus")); - test_eq(0, tor_version_as_new_as("Tor 0.2.1.0-dev (r99)", - "Tor 0.2.1.0-dev (r100)")); - test_eq(0, tor_version_as_new_as("Tor 0.2.1.0-dev (r99) on MCP", - "Tor 0.2.1.0-dev (r100) on AM")); - test_eq(0, tor_version_as_new_as("Tor 0.2.1.0-dev", - "Tor 0.2.1.0-dev (r99)")); - test_eq(1, tor_version_as_new_as("Tor 0.2.1.1", - "Tor 0.2.1.0-dev (r99)")); - done: - if (r1) - routerinfo_free(r1); - if (r2) - routerinfo_free(r2); - - tor_free(pk1_str); - tor_free(pk2_str); - tor_free(pk3_str); - if (pk1) crypto_free_pk_env(pk1); - if (pk2) crypto_free_pk_env(pk2); - if (pk3) crypto_free_pk_env(pk3); - if (rp1) routerinfo_free(rp1); - tor_free(dir1); /* XXXX And more !*/ - tor_free(dir2); /* And more !*/ -} - -/** Run unit tests for misc directory functions. */ -static void -test_dirutil(void) -{ - smartlist_t *sl = smartlist_create(); - fp_pair_t *pair; - - dir_split_resource_into_fingerprint_pairs( - /* Two pairs, out of order, with one duplicate. */ - "73656372657420646174612E0000000000FFFFFF-" - "557365204145532d32353620696e73746561642e+" - "73656372657420646174612E0000000000FFFFFF-" - "557365204145532d32353620696e73746561642e+" - "48657861646563696d616c2069736e277420736f-" - "676f6f6420666f7220686964696e6720796f7572.z", sl); - - test_eq(smartlist_len(sl), 2); - pair = smartlist_get(sl, 0); - test_memeq(pair->first, "Hexadecimal isn't so", DIGEST_LEN); - test_memeq(pair->second, "good for hiding your", DIGEST_LEN); - pair = smartlist_get(sl, 1); - test_memeq(pair->first, "secret data.\0\0\0\0\0\xff\xff\xff", DIGEST_LEN); - test_memeq(pair->second, "Use AES-256 instead.", DIGEST_LEN); - - done: - SMARTLIST_FOREACH(sl, fp_pair_t *, pair, tor_free(pair)); - smartlist_free(sl); -} - -extern const char AUTHORITY_CERT_1[]; -extern const char AUTHORITY_SIGNKEY_1[]; -extern const char AUTHORITY_CERT_2[]; -extern const char AUTHORITY_SIGNKEY_2[]; -extern const char AUTHORITY_CERT_3[]; -extern const char AUTHORITY_SIGNKEY_3[]; - -/** Helper: Test that two networkstatus_voter_info_t do in fact represent the - * same voting authority, and that they do in fact have all the same - * information. */ -static void -test_same_voter(networkstatus_voter_info_t *v1, - networkstatus_voter_info_t *v2) -{ - test_streq(v1->nickname, v2->nickname); - test_memeq(v1->identity_digest, v2->identity_digest, DIGEST_LEN); - test_streq(v1->address, v2->address); - test_eq(v1->addr, v2->addr); - test_eq(v1->dir_port, v2->dir_port); - test_eq(v1->or_port, v2->or_port); - test_streq(v1->contact, v2->contact); - test_memeq(v1->vote_digest, v2->vote_digest, DIGEST_LEN); - done: - ; -} - -/** Run unit tests for getting the median of a list. */ -static void -test_util_order_functions(void) -{ - int lst[25], n = 0; - // int a=12,b=24,c=25,d=60,e=77; - -#define median() median_int(lst, n) - - lst[n++] = 12; - test_eq(12, median()); /* 12 */ - lst[n++] = 77; - //smartlist_shuffle(sl); - test_eq(12, median()); /* 12, 77 */ - lst[n++] = 77; - //smartlist_shuffle(sl); - test_eq(77, median()); /* 12, 77, 77 */ - lst[n++] = 24; - test_eq(24, median()); /* 12,24,77,77 */ - lst[n++] = 60; - lst[n++] = 12; - lst[n++] = 25; - //smartlist_shuffle(sl); - test_eq(25, median()); /* 12,12,24,25,60,77,77 */ -#undef median - - done: - ; -} - -/** Helper: Make a new routerinfo containing the right information for a - * given vote_routerstatus_t. */ -static routerinfo_t * -generate_ri_from_rs(const vote_routerstatus_t *vrs) -{ - routerinfo_t *r; - const routerstatus_t *rs = &vrs->status; - static time_t published = 0; - - r = tor_malloc_zero(sizeof(routerinfo_t)); - memcpy(r->cache_info.identity_digest, rs->identity_digest, DIGEST_LEN); - memcpy(r->cache_info.signed_descriptor_digest, rs->descriptor_digest, - DIGEST_LEN); - r->cache_info.do_not_cache = 1; - r->cache_info.routerlist_index = -1; - r->cache_info.signed_descriptor_body = - tor_strdup("123456789012345678901234567890123"); - r->cache_info.signed_descriptor_len = - strlen(r->cache_info.signed_descriptor_body); - r->exit_policy = smartlist_create(); - r->cache_info.published_on = ++published + time(NULL); - return r; -} - -/** Run unit tests for generating and parsing V3 consensus networkstatus - * documents. */ -static void -test_v3_networkstatus(void) -{ - authority_cert_t *cert1=NULL, *cert2=NULL, *cert3=NULL; - crypto_pk_env_t *sign_skey_1=NULL, *sign_skey_2=NULL, *sign_skey_3=NULL; - crypto_pk_env_t *sign_skey_leg1=NULL; - const char *msg=NULL; - - time_t now = time(NULL); - networkstatus_voter_info_t *voter; - networkstatus_t *vote=NULL, *v1=NULL, *v2=NULL, *v3=NULL, *con=NULL; - vote_routerstatus_t *vrs; - routerstatus_t *rs; - char *v1_text=NULL, *v2_text=NULL, *v3_text=NULL, *consensus_text=NULL, *cp; - smartlist_t *votes = smartlist_create(); - - /* For generating the two other consensuses. */ - char *detached_text1=NULL, *detached_text2=NULL; - char *consensus_text2=NULL, *consensus_text3=NULL; - networkstatus_t *con2=NULL, *con3=NULL; - ns_detached_signatures_t *dsig1=NULL, *dsig2=NULL; - - /* Parse certificates and keys. */ - cert1 = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); - test_assert(cert1); - test_assert(cert1->is_cross_certified); - cert2 = authority_cert_parse_from_string(AUTHORITY_CERT_2, NULL); - test_assert(cert2); - cert3 = authority_cert_parse_from_string(AUTHORITY_CERT_3, NULL); - test_assert(cert3); - sign_skey_1 = crypto_new_pk_env(); - sign_skey_2 = crypto_new_pk_env(); - sign_skey_3 = crypto_new_pk_env(); - sign_skey_leg1 = pk_generate(4); - - test_assert(!crypto_pk_read_private_key_from_string(sign_skey_1, - AUTHORITY_SIGNKEY_1)); - test_assert(!crypto_pk_read_private_key_from_string(sign_skey_2, - AUTHORITY_SIGNKEY_2)); - test_assert(!crypto_pk_read_private_key_from_string(sign_skey_3, - AUTHORITY_SIGNKEY_3)); - - test_assert(!crypto_pk_cmp_keys(sign_skey_1, cert1->signing_key)); - test_assert(!crypto_pk_cmp_keys(sign_skey_2, cert2->signing_key)); - - /* - * Set up a vote; generate it; try to parse it. - */ - vote = tor_malloc_zero(sizeof(networkstatus_t)); - vote->type = NS_TYPE_VOTE; - vote->published = now; - vote->valid_after = now+1000; - vote->fresh_until = now+2000; - vote->valid_until = now+3000; - vote->vote_seconds = 100; - vote->dist_seconds = 200; - vote->supported_methods = smartlist_create(); - smartlist_split_string(vote->supported_methods, "1 2 3", NULL, 0, -1); - vote->client_versions = tor_strdup("0.1.2.14,0.1.2.15"); - vote->server_versions = tor_strdup("0.1.2.14,0.1.2.15,0.1.2.16"); - vote->known_flags = smartlist_create(); - smartlist_split_string(vote->known_flags, - "Authority Exit Fast Guard Running Stable V2Dir Valid", - 0, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - vote->voters = smartlist_create(); - voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t)); - voter->nickname = tor_strdup("Voter1"); - voter->address = tor_strdup("1.2.3.4"); - voter->addr = 0x01020304; - voter->dir_port = 80; - voter->or_port = 9000; - voter->contact = tor_strdup("voter@example.com"); - crypto_pk_get_digest(cert1->identity_key, voter->identity_digest); - smartlist_add(vote->voters, voter); - vote->cert = authority_cert_dup(cert1); - vote->routerstatus_list = smartlist_create(); - /* add the first routerstatus. */ - vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); - rs = &vrs->status; - vrs->version = tor_strdup("0.1.2.14"); - rs->published_on = now-1500; - strlcpy(rs->nickname, "router2", sizeof(rs->nickname)); - memset(rs->identity_digest, 3, DIGEST_LEN); - memset(rs->descriptor_digest, 78, DIGEST_LEN); - rs->addr = 0x99008801; - rs->or_port = 443; - rs->dir_port = 8000; - /* all flags but running cleared */ - rs->is_running = 1; - smartlist_add(vote->routerstatus_list, vrs); - test_assert(router_add_to_routerlist(generate_ri_from_rs(vrs), &msg,0,0)>=0); - - /* add the second routerstatus. */ - vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); - rs = &vrs->status; - vrs->version = tor_strdup("0.2.0.5"); - rs->published_on = now-1000; - strlcpy(rs->nickname, "router1", sizeof(rs->nickname)); - memset(rs->identity_digest, 5, DIGEST_LEN); - memset(rs->descriptor_digest, 77, DIGEST_LEN); - rs->addr = 0x99009901; - rs->or_port = 443; - rs->dir_port = 0; - rs->is_exit = rs->is_stable = rs->is_fast = rs->is_running = - rs->is_valid = rs->is_v2_dir = rs->is_possible_guard = 1; - smartlist_add(vote->routerstatus_list, vrs); - test_assert(router_add_to_routerlist(generate_ri_from_rs(vrs), &msg,0,0)>=0); - - /* add the third routerstatus. */ - vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); - rs = &vrs->status; - vrs->version = tor_strdup("0.1.0.3"); - rs->published_on = now-1000; - strlcpy(rs->nickname, "router3", sizeof(rs->nickname)); - memset(rs->identity_digest, 33, DIGEST_LEN); - memset(rs->descriptor_digest, 79, DIGEST_LEN); - rs->addr = 0xAA009901; - rs->or_port = 400; - rs->dir_port = 9999; - rs->is_authority = rs->is_exit = rs->is_stable = rs->is_fast = - rs->is_running = rs->is_valid = rs->is_v2_dir = rs->is_possible_guard = 1; - smartlist_add(vote->routerstatus_list, vrs); - test_assert(router_add_to_routerlist(generate_ri_from_rs(vrs), &msg,0,0)>=0); - - /* add a fourth routerstatus that is not running. */ - vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); - rs = &vrs->status; - vrs->version = tor_strdup("0.1.6.3"); - rs->published_on = now-1000; - strlcpy(rs->nickname, "router4", sizeof(rs->nickname)); - memset(rs->identity_digest, 34, DIGEST_LEN); - memset(rs->descriptor_digest, 48, DIGEST_LEN); - rs->addr = 0xC0000203; - rs->or_port = 500; - rs->dir_port = 1999; - /* Running flag (and others) cleared */ - smartlist_add(vote->routerstatus_list, vrs); - test_assert(router_add_to_routerlist(generate_ri_from_rs(vrs), &msg,0,0)>=0); - - /* dump the vote and try to parse it. */ - v1_text = format_networkstatus_vote(sign_skey_1, vote); - test_assert(v1_text); - v1 = networkstatus_parse_vote_from_string(v1_text, NULL, NS_TYPE_VOTE); - test_assert(v1); - - /* Make sure the parsed thing was right. */ - test_eq(v1->type, NS_TYPE_VOTE); - test_eq(v1->published, vote->published); - test_eq(v1->valid_after, vote->valid_after); - test_eq(v1->fresh_until, vote->fresh_until); - test_eq(v1->valid_until, vote->valid_until); - test_eq(v1->vote_seconds, vote->vote_seconds); - test_eq(v1->dist_seconds, vote->dist_seconds); - test_streq(v1->client_versions, vote->client_versions); - test_streq(v1->server_versions, vote->server_versions); - test_assert(v1->voters && smartlist_len(v1->voters)); - voter = smartlist_get(v1->voters, 0); - test_streq(voter->nickname, "Voter1"); - test_streq(voter->address, "1.2.3.4"); - test_eq(voter->addr, 0x01020304); - test_eq(voter->dir_port, 80); - test_eq(voter->or_port, 9000); - test_streq(voter->contact, "voter@example.com"); - test_assert(v1->cert); - test_assert(!crypto_pk_cmp_keys(sign_skey_1, v1->cert->signing_key)); - cp = smartlist_join_strings(v1->known_flags, ":", 0, NULL); - test_streq(cp, "Authority:Exit:Fast:Guard:Running:Stable:V2Dir:Valid"); - tor_free(cp); - test_eq(smartlist_len(v1->routerstatus_list), 4); - /* Check the first routerstatus. */ - vrs = smartlist_get(v1->routerstatus_list, 0); - rs = &vrs->status; - test_streq(vrs->version, "0.1.2.14"); - test_eq(rs->published_on, now-1500); - test_streq(rs->nickname, "router2"); - test_memeq(rs->identity_digest, - "\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3", - DIGEST_LEN); - test_memeq(rs->descriptor_digest, "NNNNNNNNNNNNNNNNNNNN", DIGEST_LEN); - test_eq(rs->addr, 0x99008801); - test_eq(rs->or_port, 443); - test_eq(rs->dir_port, 8000); - test_eq(vrs->flags, U64_LITERAL(16)); // no flags except "running" - /* Check the second routerstatus. */ - vrs = smartlist_get(v1->routerstatus_list, 1); - rs = &vrs->status; - test_streq(vrs->version, "0.2.0.5"); - test_eq(rs->published_on, now-1000); - test_streq(rs->nickname, "router1"); - test_memeq(rs->identity_digest, - "\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5", - DIGEST_LEN); - test_memeq(rs->descriptor_digest, "MMMMMMMMMMMMMMMMMMMM", DIGEST_LEN); - test_eq(rs->addr, 0x99009901); - test_eq(rs->or_port, 443); - test_eq(rs->dir_port, 0); - test_eq(vrs->flags, U64_LITERAL(254)); // all flags except "authority." - - /* Generate second vote. It disagrees on some of the times, - * and doesn't list versions, and knows some crazy flags */ - vote->published = now+1; - vote->fresh_until = now+3005; - vote->dist_seconds = 300; - authority_cert_free(vote->cert); - vote->cert = authority_cert_dup(cert2); - tor_free(vote->client_versions); - tor_free(vote->server_versions); - voter = smartlist_get(vote->voters, 0); - tor_free(voter->nickname); - tor_free(voter->address); - voter->nickname = tor_strdup("Voter2"); - voter->address = tor_strdup("2.3.4.5"); - voter->addr = 0x02030405; - crypto_pk_get_digest(cert2->identity_key, voter->identity_digest); - smartlist_add(vote->known_flags, tor_strdup("MadeOfCheese")); - smartlist_add(vote->known_flags, tor_strdup("MadeOfTin")); - smartlist_sort_strings(vote->known_flags); - vrs = smartlist_get(vote->routerstatus_list, 2); - smartlist_del_keeporder(vote->routerstatus_list, 2); - tor_free(vrs->version); - tor_free(vrs); - vrs = smartlist_get(vote->routerstatus_list, 0); - vrs->status.is_fast = 1; - /* generate and parse. */ - v2_text = format_networkstatus_vote(sign_skey_2, vote); - test_assert(v2_text); - v2 = networkstatus_parse_vote_from_string(v2_text, NULL, NS_TYPE_VOTE); - test_assert(v2); - /* Check that flags come out right.*/ - cp = smartlist_join_strings(v2->known_flags, ":", 0, NULL); - test_streq(cp, "Authority:Exit:Fast:Guard:MadeOfCheese:MadeOfTin:" - "Running:Stable:V2Dir:Valid"); - tor_free(cp); - vrs = smartlist_get(v2->routerstatus_list, 1); - /* 1023 - authority(1) - madeofcheese(16) - madeoftin(32) */ - test_eq(vrs->flags, U64_LITERAL(974)); - - /* Generate the third vote. */ - vote->published = now; - vote->fresh_until = now+2003; - vote->dist_seconds = 250; - authority_cert_free(vote->cert); - vote->cert = authority_cert_dup(cert3); - smartlist_add(vote->supported_methods, tor_strdup("4")); - vote->client_versions = tor_strdup("0.1.2.14,0.1.2.17"); - vote->server_versions = tor_strdup("0.1.2.10,0.1.2.15,0.1.2.16"); - voter = smartlist_get(vote->voters, 0); - tor_free(voter->nickname); - tor_free(voter->address); - voter->nickname = tor_strdup("Voter3"); - voter->address = tor_strdup("3.4.5.6"); - voter->addr = 0x03040506; - crypto_pk_get_digest(cert3->identity_key, voter->identity_digest); - /* This one has a legacy id. */ - memset(voter->legacy_id_digest, (int)'A', DIGEST_LEN); - vrs = smartlist_get(vote->routerstatus_list, 0); - smartlist_del_keeporder(vote->routerstatus_list, 0); - tor_free(vrs->version); - tor_free(vrs); - vrs = smartlist_get(vote->routerstatus_list, 0); - memset(vrs->status.descriptor_digest, (int)'Z', DIGEST_LEN); - test_assert(router_add_to_routerlist(generate_ri_from_rs(vrs), &msg,0,0)>=0); - - v3_text = format_networkstatus_vote(sign_skey_3, vote); - test_assert(v3_text); - - v3 = networkstatus_parse_vote_from_string(v3_text, NULL, NS_TYPE_VOTE); - test_assert(v3); - - /* Compute a consensus as voter 3. */ - smartlist_add(votes, v3); - smartlist_add(votes, v1); - smartlist_add(votes, v2); - consensus_text = networkstatus_compute_consensus(votes, 3, - cert3->identity_key, - sign_skey_3, - "AAAAAAAAAAAAAAAAAAAA", - sign_skey_leg1); - test_assert(consensus_text); - con = networkstatus_parse_vote_from_string(consensus_text, NULL, - NS_TYPE_CONSENSUS); - test_assert(con); - //log_notice(LD_GENERAL, "<<%s>>\n<<%s>>\n<<%s>>\n", - // v1_text, v2_text, v3_text); - - /* Check consensus contents. */ - test_assert(con->type == NS_TYPE_CONSENSUS); - test_eq(con->published, 0); /* this field only appears in votes. */ - test_eq(con->valid_after, now+1000); - test_eq(con->fresh_until, now+2003); /* median */ - test_eq(con->valid_until, now+3000); - test_eq(con->vote_seconds, 100); - test_eq(con->dist_seconds, 250); /* median */ - test_streq(con->client_versions, "0.1.2.14"); - test_streq(con->server_versions, "0.1.2.15,0.1.2.16"); - cp = smartlist_join_strings(v2->known_flags, ":", 0, NULL); - test_streq(cp, "Authority:Exit:Fast:Guard:MadeOfCheese:MadeOfTin:" - "Running:Stable:V2Dir:Valid"); - tor_free(cp); - test_eq(4, smartlist_len(con->voters)); /*3 voters, 1 legacy key.*/ - /* The voter id digests should be in this order. */ - test_assert(memcmp(cert2->cache_info.identity_digest, - cert1->cache_info.identity_digest,DIGEST_LEN)<0); - test_assert(memcmp(cert1->cache_info.identity_digest, - cert3->cache_info.identity_digest,DIGEST_LEN)<0); - test_same_voter(smartlist_get(con->voters, 1), - smartlist_get(v2->voters, 0)); - test_same_voter(smartlist_get(con->voters, 2), - smartlist_get(v1->voters, 0)); - test_same_voter(smartlist_get(con->voters, 3), - smartlist_get(v3->voters, 0)); - - test_assert(!con->cert); - test_eq(2, smartlist_len(con->routerstatus_list)); - /* There should be two listed routers: one with identity 3, one with - * identity 5. */ - /* This one showed up in 2 digests. */ - rs = smartlist_get(con->routerstatus_list, 0); - test_memeq(rs->identity_digest, - "\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3", - DIGEST_LEN); - test_memeq(rs->descriptor_digest, "NNNNNNNNNNNNNNNNNNNN", DIGEST_LEN); - test_assert(!rs->is_authority); - test_assert(!rs->is_exit); - test_assert(!rs->is_fast); - test_assert(!rs->is_possible_guard); - test_assert(!rs->is_stable); - test_assert(rs->is_running); /* If it wasn't running it wouldn't be here */ - test_assert(!rs->is_v2_dir); - test_assert(!rs->is_valid); - test_assert(!rs->is_named); - /* XXXX check version */ - - rs = smartlist_get(con->routerstatus_list, 1); - /* This one showed up in 3 digests. Twice with ID 'M', once with 'Z'. */ - test_memeq(rs->identity_digest, - "\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5", - DIGEST_LEN); - test_streq(rs->nickname, "router1"); - test_memeq(rs->descriptor_digest, "MMMMMMMMMMMMMMMMMMMM", DIGEST_LEN); - test_eq(rs->published_on, now-1000); - test_eq(rs->addr, 0x99009901); - test_eq(rs->or_port, 443); - test_eq(rs->dir_port, 0); - test_assert(!rs->is_authority); - test_assert(rs->is_exit); - test_assert(rs->is_fast); - test_assert(rs->is_possible_guard); - test_assert(rs->is_stable); - test_assert(rs->is_running); - test_assert(rs->is_v2_dir); - test_assert(rs->is_valid); - test_assert(!rs->is_named); - /* XXXX check version */ - // x231 - // x213 - - /* Check signatures. the first voter is a pseudo-entry with a legacy key. - * The second one hasn't signed. The fourth one has signed: validate it. */ - voter = smartlist_get(con->voters, 1); - test_assert(!voter->signature); - test_assert(!voter->good_signature); - test_assert(!voter->bad_signature); - - voter = smartlist_get(con->voters, 3); - test_assert(voter->signature); - test_assert(!voter->good_signature); - test_assert(!voter->bad_signature); - test_assert(!networkstatus_check_voter_signature(con, - smartlist_get(con->voters, 3), - cert3)); - test_assert(voter->signature); - test_assert(voter->good_signature); - test_assert(!voter->bad_signature); - - { - const char *msg=NULL; - /* Compute the other two signed consensuses. */ - smartlist_shuffle(votes); - consensus_text2 = networkstatus_compute_consensus(votes, 3, - cert2->identity_key, - sign_skey_2, NULL,NULL); - smartlist_shuffle(votes); - consensus_text3 = networkstatus_compute_consensus(votes, 3, - cert1->identity_key, - sign_skey_1, NULL,NULL); - test_assert(consensus_text2); - test_assert(consensus_text3); - con2 = networkstatus_parse_vote_from_string(consensus_text2, NULL, - NS_TYPE_CONSENSUS); - con3 = networkstatus_parse_vote_from_string(consensus_text3, NULL, - NS_TYPE_CONSENSUS); - test_assert(con2); - test_assert(con3); - - /* All three should have the same digest. */ - test_memeq(con->networkstatus_digest, con2->networkstatus_digest, - DIGEST_LEN); - test_memeq(con->networkstatus_digest, con3->networkstatus_digest, - DIGEST_LEN); - - /* Extract a detached signature from con3. */ - detached_text1 = networkstatus_get_detached_signatures(con3); - tor_assert(detached_text1); - /* Try to parse it. */ - dsig1 = networkstatus_parse_detached_signatures(detached_text1, NULL); - tor_assert(dsig1); - - /* Are parsed values as expected? */ - test_eq(dsig1->valid_after, con3->valid_after); - test_eq(dsig1->fresh_until, con3->fresh_until); - test_eq(dsig1->valid_until, con3->valid_until); - test_memeq(dsig1->networkstatus_digest, con3->networkstatus_digest, - DIGEST_LEN); - test_eq(1, smartlist_len(dsig1->signatures)); - voter = smartlist_get(dsig1->signatures, 0); - test_memeq(voter->identity_digest, cert1->cache_info.identity_digest, - DIGEST_LEN); - - /* Try adding it to con2. */ - detached_text2 = networkstatus_get_detached_signatures(con2); - test_eq(1, networkstatus_add_detached_signatures(con2, dsig1, &msg)); - tor_free(detached_text2); - detached_text2 = networkstatus_get_detached_signatures(con2); - //printf("\n<%s>\n", detached_text2); - dsig2 = networkstatus_parse_detached_signatures(detached_text2, NULL); - test_assert(dsig2); - /* - printf("\n"); - SMARTLIST_FOREACH(dsig2->signatures, networkstatus_voter_info_t *, vi, { - char hd[64]; - base16_encode(hd, sizeof(hd), vi->identity_digest, DIGEST_LEN); - printf("%s\n", hd); - }); - */ - test_eq(2, smartlist_len(dsig2->signatures)); - - /* Try adding to con2 twice; verify that nothing changes. */ - test_eq(0, networkstatus_add_detached_signatures(con2, dsig1, &msg)); - - /* Add to con. */ - test_eq(2, networkstatus_add_detached_signatures(con, dsig2, &msg)); - /* Check signatures */ - test_assert(!networkstatus_check_voter_signature(con, - smartlist_get(con->voters, 1), - cert2)); - test_assert(!networkstatus_check_voter_signature(con, - smartlist_get(con->voters, 2), - cert1)); - - } - - done: - smartlist_free(votes); - tor_free(v1_text); - tor_free(v2_text); - tor_free(v3_text); - tor_free(consensus_text); - - if (vote) - networkstatus_vote_free(vote); - if (v1) - networkstatus_vote_free(v1); - if (v2) - networkstatus_vote_free(v2); - if (v3) - networkstatus_vote_free(v3); - if (con) - networkstatus_vote_free(con); - if (sign_skey_1) - crypto_free_pk_env(sign_skey_1); - if (sign_skey_2) - crypto_free_pk_env(sign_skey_2); - if (sign_skey_3) - crypto_free_pk_env(sign_skey_3); - if (sign_skey_leg1) - crypto_free_pk_env(sign_skey_leg1); - if (cert1) - authority_cert_free(cert1); - if (cert2) - authority_cert_free(cert2); - if (cert3) - authority_cert_free(cert3); - - tor_free(consensus_text2); - tor_free(consensus_text3); - tor_free(detached_text1); - tor_free(detached_text2); - if (con2) - networkstatus_vote_free(con2); - if (con3) - networkstatus_vote_free(con3); - if (dsig1) - ns_detached_signatures_free(dsig1); - if (dsig2) - ns_detached_signatures_free(dsig2); -} - -/** Helper: Parse the exit policy string in <b>policy_str</b>, and make sure - * that policies_summarize() produces the string <b>expected_summary</b> from - * it. */ -static void -test_policy_summary_helper(const char *policy_str, - const char *expected_summary) -{ - config_line_t line; - smartlist_t *policy = smartlist_create(); - char *summary = NULL; - int r; - - line.key = (char*)"foo"; - line.value = (char *)policy_str; - line.next = NULL; - - r = policies_parse_exit_policy(&line, &policy, 0, NULL); - test_eq(r, 0); - summary = policy_summarize(policy); - - test_assert(summary != NULL); - test_streq(summary, expected_summary); - - done: - tor_free(summary); - if (policy) - addr_policy_list_free(policy); -} - -/** Run unit tests for generating summary lines of exit policies */ -static void -test_policies(void) -{ - int i; - smartlist_t *policy = NULL, *policy2 = NULL; - addr_policy_t *p; - tor_addr_t tar; - config_line_t line; - smartlist_t *sm = NULL; - char *policy_str = NULL; - - policy = smartlist_create(); - - p = router_parse_addr_policy_item_from_string("reject 192.168.0.0/16:*",-1); - test_assert(p != NULL); - test_eq(ADDR_POLICY_REJECT, p->policy_type); - tor_addr_from_ipv4h(&tar, 0xc0a80000u); - test_eq(0, tor_addr_compare(&p->addr, &tar, CMP_EXACT)); - test_eq(16, p->maskbits); - test_eq(1, p->prt_min); - test_eq(65535, p->prt_max); - - smartlist_add(policy, p); - - test_assert(ADDR_POLICY_ACCEPTED == - compare_addr_to_addr_policy(0x01020304u, 2, policy)); - test_assert(ADDR_POLICY_PROBABLY_ACCEPTED == - compare_addr_to_addr_policy(0, 2, policy)); - test_assert(ADDR_POLICY_REJECTED == - compare_addr_to_addr_policy(0xc0a80102, 2, policy)); - - policy2 = NULL; - test_assert(0 == policies_parse_exit_policy(NULL, &policy2, 1, NULL)); - test_assert(policy2); - - test_assert(!exit_policy_is_general_exit(policy)); - test_assert(exit_policy_is_general_exit(policy2)); - test_assert(!exit_policy_is_general_exit(NULL)); - - test_assert(cmp_addr_policies(policy, policy2)); - test_assert(cmp_addr_policies(policy, NULL)); - test_assert(!cmp_addr_policies(policy2, policy2)); - test_assert(!cmp_addr_policies(NULL, NULL)); - - test_assert(!policy_is_reject_star(policy2)); - test_assert(policy_is_reject_star(policy)); - test_assert(policy_is_reject_star(NULL)); - - addr_policy_list_free(policy); - policy = NULL; - - /* make sure compacting logic works. */ - policy = NULL; - line.key = (char*)"foo"; - line.value = (char*)"accept *:80,reject private:*,reject *:*"; - line.next = NULL; - test_assert(0 == policies_parse_exit_policy(&line, &policy, 0, NULL)); - test_assert(policy); - //test_streq(policy->string, "accept *:80"); - //test_streq(policy->next->string, "reject *:*"); - test_eq(smartlist_len(policy), 2); - - /* test policy summaries */ - /* check if we properly ignore private IP addresses */ - test_policy_summary_helper("reject 192.168.0.0/16:*," - "reject 0.0.0.0/8:*," - "reject 10.0.0.0/8:*," - "accept *:10-30," - "accept *:90," - "reject *:*", - "accept 10-30,90"); - /* check all accept policies, and proper counting of rejects */ - test_policy_summary_helper("reject 11.0.0.0/9:80," - "reject 12.0.0.0/9:80," - "reject 13.0.0.0/9:80," - "reject 14.0.0.0/9:80," - "accept *:*", "accept 1-65535"); - test_policy_summary_helper("reject 11.0.0.0/9:80," - "reject 12.0.0.0/9:80," - "reject 13.0.0.0/9:80," - "reject 14.0.0.0/9:80," - "reject 15.0.0.0:81," - "accept *:*", "accept 1-65535"); - test_policy_summary_helper("reject 11.0.0.0/9:80," - "reject 12.0.0.0/9:80," - "reject 13.0.0.0/9:80," - "reject 14.0.0.0/9:80," - "reject 15.0.0.0:80," - "accept *:*", - "reject 80"); - /* no exits */ - test_policy_summary_helper("accept 11.0.0.0/9:80," - "reject *:*", - "reject 1-65535"); - /* port merging */ - test_policy_summary_helper("accept *:80," - "accept *:81," - "accept *:100-110," - "accept *:111," - "reject *:*", - "accept 80-81,100-111"); - /* border ports */ - test_policy_summary_helper("accept *:1," - "accept *:3," - "accept *:65535," - "reject *:*", - "accept 1,3,65535"); - /* holes */ - test_policy_summary_helper("accept *:1," - "accept *:3," - "accept *:5," - "accept *:7," - "reject *:*", - "accept 1,3,5,7"); - test_policy_summary_helper("reject *:1," - "reject *:3," - "reject *:5," - "reject *:7," - "accept *:*", - "reject 1,3,5,7"); - - /* truncation ports */ - sm = smartlist_create(); - for (i=1; i<2000; i+=2) { - char buf[POLICY_BUF_LEN]; - tor_snprintf(buf, sizeof(buf), "reject *:%d", i); - smartlist_add(sm, tor_strdup(buf)); - } - smartlist_add(sm, tor_strdup("accept *:*")); - policy_str = smartlist_join_strings(sm, ",", 0, NULL); - test_policy_summary_helper( policy_str, - "accept 2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44," - "46,48,50,52,54,56,58,60,62,64,66,68,70,72,74,76,78,80,82,84,86,88,90," - "92,94,96,98,100,102,104,106,108,110,112,114,116,118,120,122,124,126,128," - "130,132,134,136,138,140,142,144,146,148,150,152,154,156,158,160,162,164," - "166,168,170,172,174,176,178,180,182,184,186,188,190,192,194,196,198,200," - "202,204,206,208,210,212,214,216,218,220,222,224,226,228,230,232,234,236," - "238,240,242,244,246,248,250,252,254,256,258,260,262,264,266,268,270,272," - "274,276,278,280,282,284,286,288,290,292,294,296,298,300,302,304,306,308," - "310,312,314,316,318,320,322,324,326,328,330,332,334,336,338,340,342,344," - "346,348,350,352,354,356,358,360,362,364,366,368,370,372,374,376,378,380," - "382,384,386,388,390,392,394,396,398,400,402,404,406,408,410,412,414,416," - "418,420,422,424,426,428,430,432,434,436,438,440,442,444,446,448,450,452," - "454,456,458,460,462,464,466,468,470,472,474,476,478,480,482,484,486,488," - "490,492,494,496,498,500,502,504,506,508,510,512,514,516,518,520,522"); - - done: - if (policy) - addr_policy_list_free(policy); - if (policy2) - addr_policy_list_free(policy2); - tor_free(policy_str); - if (sm) { - SMARTLIST_FOREACH(sm, char *, s, tor_free(s)); - smartlist_free(sm); - } -} - -/** Run unit tests for basic rendezvous functions. */ -static void -test_rend_fns(void) -{ - char address1[] = "fooaddress.onion"; - char address2[] = "aaaaaaaaaaaaaaaa.onion"; - char address3[] = "fooaddress.exit"; - char address4[] = "www.torproject.org"; - rend_service_descriptor_t *d1 = - tor_malloc_zero(sizeof(rend_service_descriptor_t)); - rend_service_descriptor_t *d2 = NULL; - char *encoded = NULL; - size_t len; - time_t now; - int i; - crypto_pk_env_t *pk1 = pk_generate(0), *pk2 = pk_generate(1); - - /* Test unversioned (v0) descriptor */ - d1->pk = crypto_pk_dup_key(pk1); - now = time(NULL); - d1->timestamp = now; - d1->version = 0; - d1->intro_nodes = smartlist_create(); - for (i = 0; i < 3; i++) { - rend_intro_point_t *intro = tor_malloc_zero(sizeof(rend_intro_point_t)); - intro->extend_info = tor_malloc_zero(sizeof(extend_info_t)); - crypto_rand(intro->extend_info->identity_digest, DIGEST_LEN); - intro->extend_info->nickname[0] = '$'; - base16_encode(intro->extend_info->nickname+1, HEX_DIGEST_LEN+1, - intro->extend_info->identity_digest, DIGEST_LEN); - smartlist_add(d1->intro_nodes, intro); - } - test_assert(! rend_encode_service_descriptor(d1, pk1, &encoded, &len)); - d2 = rend_parse_service_descriptor(encoded, len); - test_assert(d2); - - test_assert(!crypto_pk_cmp_keys(d1->pk, d2->pk)); - test_eq(d2->timestamp, now); - test_eq(d2->version, 0); - test_eq(d2->protocols, 1<<2); - test_eq(smartlist_len(d2->intro_nodes), 3); - for (i = 0; i < 3; i++) { - rend_intro_point_t *intro1 = smartlist_get(d1->intro_nodes, i); - rend_intro_point_t *intro2 = smartlist_get(d2->intro_nodes, i); - test_streq(intro1->extend_info->nickname, - intro2->extend_info->nickname); - } - - test_assert(BAD_HOSTNAME == parse_extended_hostname(address1)); - test_assert(ONION_HOSTNAME == parse_extended_hostname(address2)); - test_assert(EXIT_HOSTNAME == parse_extended_hostname(address3)); - test_assert(NORMAL_HOSTNAME == parse_extended_hostname(address4)); - - crypto_free_pk_env(pk1); - crypto_free_pk_env(pk2); - pk1 = pk2 = NULL; - rend_service_descriptor_free(d1); - rend_service_descriptor_free(d2); - d1 = d2 = NULL; - - done: - if (pk1) - crypto_free_pk_env(pk1); - if (pk2) - crypto_free_pk_env(pk2); - if (d1) - rend_service_descriptor_free(d1); - if (d2) - rend_service_descriptor_free(d2); - tor_free(encoded); -} - -/** Run AES performance benchmarks. */ -static void -bench_aes(void) -{ - int len, i; - char *b1, *b2; - crypto_cipher_env_t *c; - struct timeval start, end; - const int iters = 100000; - uint64_t nsec; - c = crypto_new_cipher_env(); - crypto_cipher_generate_key(c); - crypto_cipher_encrypt_init_cipher(c); - for (len = 1; len <= 8192; len *= 2) { - b1 = tor_malloc_zero(len); - b2 = tor_malloc_zero(len); - tor_gettimeofday(&start); - for (i = 0; i < iters; ++i) { - crypto_cipher_encrypt(c, b1, b2, len); - } - tor_gettimeofday(&end); - tor_free(b1); - tor_free(b2); - nsec = (uint64_t) tv_udiff(&start,&end); - nsec *= 1000; - nsec /= (iters*len); - printf("%d bytes: "U64_FORMAT" nsec per byte\n", len, - U64_PRINTF_ARG(nsec)); - } - crypto_free_cipher_env(c); -} - -/** Run digestmap_t performance benchmarks. */ -static void -bench_dmap(void) -{ - smartlist_t *sl = smartlist_create(); - smartlist_t *sl2 = smartlist_create(); - struct timeval start, end, pt2, pt3, pt4; - const int iters = 10000; - const int elts = 4000; - const int fpostests = 1000000; - char d[20]; - int i,n=0, fp = 0; - digestmap_t *dm = digestmap_new(); - digestset_t *ds = digestset_new(elts); - - for (i = 0; i < elts; ++i) { - crypto_rand(d, 20); - smartlist_add(sl, tor_memdup(d, 20)); - } - for (i = 0; i < elts; ++i) { - crypto_rand(d, 20); - smartlist_add(sl2, tor_memdup(d, 20)); - } - printf("nbits=%d\n", ds->mask+1); - - tor_gettimeofday(&start); - for (i = 0; i < iters; ++i) { - SMARTLIST_FOREACH(sl, const char *, cp, digestmap_set(dm, cp, (void*)1)); - } - tor_gettimeofday(&pt2); - for (i = 0; i < iters; ++i) { - SMARTLIST_FOREACH(sl, const char *, cp, digestmap_get(dm, cp)); - SMARTLIST_FOREACH(sl2, const char *, cp, digestmap_get(dm, cp)); - } - tor_gettimeofday(&pt3); - for (i = 0; i < iters; ++i) { - SMARTLIST_FOREACH(sl, const char *, cp, digestset_add(ds, cp)); - } - tor_gettimeofday(&pt4); - for (i = 0; i < iters; ++i) { - SMARTLIST_FOREACH(sl, const char *, cp, n += digestset_isin(ds, cp)); - SMARTLIST_FOREACH(sl2, const char *, cp, n += digestset_isin(ds, cp)); - } - tor_gettimeofday(&end); - - for (i = 0; i < fpostests; ++i) { - crypto_rand(d, 20); - if (digestset_isin(ds, d)) ++fp; - } - - printf("%ld\n",(unsigned long)tv_udiff(&start, &pt2)); - printf("%ld\n",(unsigned long)tv_udiff(&pt2, &pt3)); - printf("%ld\n",(unsigned long)tv_udiff(&pt3, &pt4)); - printf("%ld\n",(unsigned long)tv_udiff(&pt4, &end)); - printf("-- %d\n", n); - printf("++ %f\n", fp/(double)fpostests); - digestmap_free(dm, NULL); - digestset_free(ds); - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - SMARTLIST_FOREACH(sl2, char *, cp, tor_free(cp)); - smartlist_free(sl); - smartlist_free(sl2); -} - -/** Run unittests for memory pool allocator */ -static void -test_util_mempool(void) -{ - mp_pool_t *pool = NULL; - smartlist_t *allocated = NULL; - int i; - - pool = mp_pool_new(1, 100); - test_assert(pool); - test_assert(pool->new_chunk_capacity >= 100); - test_assert(pool->item_alloc_size >= sizeof(void*)+1); - mp_pool_destroy(pool); - pool = NULL; - - pool = mp_pool_new(241, 2500); - test_assert(pool); - test_assert(pool->new_chunk_capacity >= 10); - test_assert(pool->item_alloc_size >= sizeof(void*)+241); - test_eq(pool->item_alloc_size & 0x03, 0); - test_assert(pool->new_chunk_capacity < 60); - - allocated = smartlist_create(); - for (i = 0; i < 20000; ++i) { - if (smartlist_len(allocated) < 20 || crypto_rand_int(2)) { - void *m = mp_pool_get(pool); - memset(m, 0x09, 241); - smartlist_add(allocated, m); - //printf("%d: %p\n", i, m); - //mp_pool_assert_ok(pool); - } else { - int idx = crypto_rand_int(smartlist_len(allocated)); - void *m = smartlist_get(allocated, idx); - //printf("%d: free %p\n", i, m); - smartlist_del(allocated, idx); - mp_pool_release(m); - //mp_pool_assert_ok(pool); - } - if (crypto_rand_int(777)==0) - mp_pool_clean(pool, 1, 1); - - if (i % 777) - mp_pool_assert_ok(pool); - } - - done: - if (allocated) { - SMARTLIST_FOREACH(allocated, void *, m, mp_pool_release(m)); - mp_pool_assert_ok(pool); - mp_pool_clean(pool, 0, 0); - mp_pool_assert_ok(pool); - smartlist_free(allocated); - } - - if (pool) - mp_pool_destroy(pool); -} - -/** Run unittests for memory area allocator */ -static void -test_util_memarea(void) -{ - memarea_t *area = memarea_new(); - char *p1, *p2, *p3, *p1_orig; - void *malloced_ptr = NULL; - int i; - - test_assert(area); - - p1_orig = p1 = memarea_alloc(area,64); - p2 = memarea_alloc_zero(area,52); - p3 = memarea_alloc(area,11); - - test_assert(memarea_owns_ptr(area, p1)); - test_assert(memarea_owns_ptr(area, p2)); - test_assert(memarea_owns_ptr(area, p3)); - /* Make sure we left enough space. */ - test_assert(p1+64 <= p2); - test_assert(p2+52 <= p3); - /* Make sure we aligned. */ - test_eq(((uintptr_t)p1) % sizeof(void*), 0); - test_eq(((uintptr_t)p2) % sizeof(void*), 0); - test_eq(((uintptr_t)p3) % sizeof(void*), 0); - test_assert(!memarea_owns_ptr(area, p3+8192)); - test_assert(!memarea_owns_ptr(area, p3+30)); - test_assert(tor_mem_is_zero(p2, 52)); - /* Make sure we don't overalign. */ - p1 = memarea_alloc(area, 1); - p2 = memarea_alloc(area, 1); - test_eq(p1+sizeof(void*), p2); - { - malloced_ptr = tor_malloc(64); - test_assert(!memarea_owns_ptr(area, malloced_ptr)); - tor_free(malloced_ptr); - } - - /* memarea_memdup */ - { - malloced_ptr = tor_malloc(64); - crypto_rand((char*)malloced_ptr, 64); - p1 = memarea_memdup(area, malloced_ptr, 64); - test_assert(p1 != malloced_ptr); - test_memeq(p1, malloced_ptr, 64); - tor_free(malloced_ptr); - } - - /* memarea_strdup. */ - p1 = memarea_strdup(area,""); - p2 = memarea_strdup(area, "abcd"); - test_assert(p1); - test_assert(p2); - test_streq(p1, ""); - test_streq(p2, "abcd"); - - /* memarea_strndup. */ - { - const char *s = "Ad ogni porta batte la morte e grida: il nome!"; - /* (From Turandot, act 3.) */ - size_t len = strlen(s); - p1 = memarea_strndup(area, s, 1000); - p2 = memarea_strndup(area, s, 10); - test_streq(p1, s); - test_assert(p2 >= p1 + len + 1); - test_memeq(s, p2, 10); - test_eq(p2[10], '\0'); - p3 = memarea_strndup(area, s, len); - test_streq(p3, s); - p3 = memarea_strndup(area, s, len-1); - test_memeq(s, p3, len-1); - test_eq(p3[len-1], '\0'); - } - - memarea_clear(area); - p1 = memarea_alloc(area, 1); - test_eq(p1, p1_orig); - memarea_clear(area); - - /* Check for running over an area's size. */ - for (i = 0; i < 512; ++i) { - p1 = memarea_alloc(area, crypto_rand_int(5)+1); - test_assert(memarea_owns_ptr(area, p1)); - } - memarea_assert_ok(area); - /* Make sure we can allocate a too-big object. */ - p1 = memarea_alloc_zero(area, 9000); - p2 = memarea_alloc_zero(area, 16); - test_assert(memarea_owns_ptr(area, p1)); - test_assert(memarea_owns_ptr(area, p2)); - - done: - memarea_drop_all(area); - tor_free(malloced_ptr); -} - -/** Run unit tests for utility functions to get file names relative to - * the data directory. */ -static void -test_util_datadir(void) -{ - char buf[1024]; - char *f = NULL; - - f = get_datadir_fname(NULL); - test_streq(f, temp_dir); - tor_free(f); - f = get_datadir_fname("state"); - tor_snprintf(buf, sizeof(buf), "%s"PATH_SEPARATOR"state", temp_dir); - test_streq(f, buf); - tor_free(f); - f = get_datadir_fname2("cache", "thingy"); - tor_snprintf(buf, sizeof(buf), - "%s"PATH_SEPARATOR"cache"PATH_SEPARATOR"thingy", temp_dir); - test_streq(f, buf); - tor_free(f); - f = get_datadir_fname2_suffix("cache", "thingy", ".foo"); - tor_snprintf(buf, sizeof(buf), - "%s"PATH_SEPARATOR"cache"PATH_SEPARATOR"thingy.foo", temp_dir); - test_streq(f, buf); - tor_free(f); - f = get_datadir_fname_suffix("cache", ".foo"); - tor_snprintf(buf, sizeof(buf), "%s"PATH_SEPARATOR"cache.foo", - temp_dir); - test_streq(f, buf); - - done: - tor_free(f); -} - -/** Test AES-CTR encryption and decryption with IV. */ -static void -test_crypto_aes_iv(void) -{ - crypto_cipher_env_t *cipher; - char *plain, *encrypted1, *encrypted2, *decrypted1, *decrypted2; - char plain_1[1], plain_15[15], plain_16[16], plain_17[17]; - char key1[16], key2[16]; - ssize_t encrypted_size, decrypted_size; - - plain = tor_malloc(4095); - encrypted1 = tor_malloc(4095 + 1 + 16); - encrypted2 = tor_malloc(4095 + 1 + 16); - decrypted1 = tor_malloc(4095 + 1); - decrypted2 = tor_malloc(4095 + 1); - - crypto_rand(plain, 4095); - crypto_rand(key1, 16); - crypto_rand(key2, 16); - crypto_rand(plain_1, 1); - crypto_rand(plain_15, 15); - crypto_rand(plain_16, 16); - crypto_rand(plain_17, 17); - key1[0] = key2[0] + 128; /* Make sure that contents are different. */ - /* Encrypt and decrypt with the same key. */ - cipher = crypto_create_init_cipher(key1, 1); - encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted1, 16 + 4095, - plain, 4095); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(encrypted_size, 16 + 4095); - tor_assert(encrypted_size > 0); /* This is obviously true, since 4111 is - * greater than 0, but its truth is not - * obvious to all analysis tools. */ - cipher = crypto_create_init_cipher(key1, 0); - decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 4095, - encrypted1, encrypted_size); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(decrypted_size, 4095); - tor_assert(decrypted_size > 0); - test_memeq(plain, decrypted1, 4095); - /* Encrypt a second time (with a new random initialization vector). */ - cipher = crypto_create_init_cipher(key1, 1); - encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted2, 16 + 4095, - plain, 4095); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(encrypted_size, 16 + 4095); - tor_assert(encrypted_size > 0); - cipher = crypto_create_init_cipher(key1, 0); - decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted2, 4095, - encrypted2, encrypted_size); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(decrypted_size, 4095); - tor_assert(decrypted_size > 0); - test_memeq(plain, decrypted2, 4095); - test_memneq(encrypted1, encrypted2, encrypted_size); - /* Decrypt with the wrong key. */ - cipher = crypto_create_init_cipher(key2, 0); - decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted2, 4095, - encrypted1, encrypted_size); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_memneq(plain, decrypted2, encrypted_size); - /* Alter the initialization vector. */ - encrypted1[0] += 42; - cipher = crypto_create_init_cipher(key1, 0); - decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 4095, - encrypted1, encrypted_size); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_memneq(plain, decrypted2, 4095); - /* Special length case: 1. */ - cipher = crypto_create_init_cipher(key1, 1); - encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted1, 16 + 1, - plain_1, 1); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(encrypted_size, 16 + 1); - tor_assert(encrypted_size > 0); - cipher = crypto_create_init_cipher(key1, 0); - decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 1, - encrypted1, encrypted_size); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(decrypted_size, 1); - tor_assert(decrypted_size > 0); - test_memeq(plain_1, decrypted1, 1); - /* Special length case: 15. */ - cipher = crypto_create_init_cipher(key1, 1); - encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted1, 16 + 15, - plain_15, 15); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(encrypted_size, 16 + 15); - tor_assert(encrypted_size > 0); - cipher = crypto_create_init_cipher(key1, 0); - decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 15, - encrypted1, encrypted_size); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(decrypted_size, 15); - tor_assert(decrypted_size > 0); - test_memeq(plain_15, decrypted1, 15); - /* Special length case: 16. */ - cipher = crypto_create_init_cipher(key1, 1); - encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted1, 16 + 16, - plain_16, 16); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(encrypted_size, 16 + 16); - tor_assert(encrypted_size > 0); - cipher = crypto_create_init_cipher(key1, 0); - decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 16, - encrypted1, encrypted_size); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(decrypted_size, 16); - tor_assert(decrypted_size > 0); - test_memeq(plain_16, decrypted1, 16); - /* Special length case: 17. */ - cipher = crypto_create_init_cipher(key1, 1); - encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted1, 16 + 17, - plain_17, 17); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(encrypted_size, 16 + 17); - tor_assert(encrypted_size > 0); - cipher = crypto_create_init_cipher(key1, 0); - decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 17, - encrypted1, encrypted_size); - test_eq(decrypted_size, 17); - tor_assert(decrypted_size > 0); - test_memeq(plain_17, decrypted1, 17); - - done: - /* Free memory. */ - tor_free(plain); - tor_free(encrypted1); - tor_free(encrypted2); - tor_free(decrypted1); - tor_free(decrypted2); - if (cipher) - crypto_free_cipher_env(cipher); -} - -/** Test base32 decoding. */ -static void -test_crypto_base32_decode(void) -{ - char plain[60], encoded[96 + 1], decoded[60]; - int res; - crypto_rand(plain, 60); - /* Encode and decode a random string. */ - base32_encode(encoded, 96 + 1, plain, 60); - res = base32_decode(decoded, 60, encoded, 96); - test_eq(res, 0); - test_memeq(plain, decoded, 60); - /* Encode, uppercase, and decode a random string. */ - base32_encode(encoded, 96 + 1, plain, 60); - tor_strupper(encoded); - res = base32_decode(decoded, 60, encoded, 96); - test_eq(res, 0); - test_memeq(plain, decoded, 60); - /* Change encoded string and decode. */ - if (encoded[0] == 'A' || encoded[0] == 'a') - encoded[0] = 'B'; - else - encoded[0] = 'A'; - res = base32_decode(decoded, 60, encoded, 96); - test_eq(res, 0); - test_memneq(plain, decoded, 60); - /* Bad encodings. */ - encoded[0] = '!'; - res = base32_decode(decoded, 60, encoded, 96); - test_assert(res < 0); - - done: - ; -} - -/** Test encoding and parsing of v2 rendezvous service descriptors. */ -static void -test_rend_fns_v2(void) -{ - rend_service_descriptor_t *generated = NULL, *parsed = NULL; - char service_id[DIGEST_LEN]; - char service_id_base32[REND_SERVICE_ID_LEN_BASE32+1]; - const char *next_desc; - smartlist_t *descs = smartlist_create(); - char computed_desc_id[DIGEST_LEN]; - char parsed_desc_id[DIGEST_LEN]; - crypto_pk_env_t *pk1 = NULL, *pk2 = NULL; - time_t now; - char *intro_points_encrypted = NULL; - size_t intro_points_size; - size_t encoded_size; - int i; - pk1 = pk_generate(0); - pk2 = pk_generate(1); - generated = tor_malloc_zero(sizeof(rend_service_descriptor_t)); - generated->pk = crypto_pk_dup_key(pk1); - crypto_pk_get_digest(generated->pk, service_id); - base32_encode(service_id_base32, REND_SERVICE_ID_LEN_BASE32+1, - service_id, REND_SERVICE_ID_LEN); - now = time(NULL); - generated->timestamp = now; - generated->version = 2; - generated->protocols = 42; - generated->intro_nodes = smartlist_create(); - - for (i = 0; i < 3; i++) { - rend_intro_point_t *intro = tor_malloc_zero(sizeof(rend_intro_point_t)); - crypto_pk_env_t *okey = pk_generate(2 + i); - intro->extend_info = tor_malloc_zero(sizeof(extend_info_t)); - intro->extend_info->onion_key = okey; - crypto_pk_get_digest(intro->extend_info->onion_key, - intro->extend_info->identity_digest); - //crypto_rand(info->identity_digest, DIGEST_LEN); /* Would this work? */ - intro->extend_info->nickname[0] = '$'; - base16_encode(intro->extend_info->nickname + 1, - sizeof(intro->extend_info->nickname) - 1, - intro->extend_info->identity_digest, DIGEST_LEN); - /* Does not cover all IP addresses. */ - tor_addr_from_ipv4h(&intro->extend_info->addr, crypto_rand_int(65536)); - intro->extend_info->port = crypto_rand_int(65536); - intro->intro_key = crypto_pk_dup_key(pk2); - smartlist_add(generated->intro_nodes, intro); - } - test_assert(rend_encode_v2_descriptors(descs, generated, now, 0, - REND_NO_AUTH, NULL, NULL) > 0); - test_assert(rend_compute_v2_desc_id(computed_desc_id, service_id_base32, - NULL, now, 0) == 0); - test_memeq(((rend_encoded_v2_service_descriptor_t *) - smartlist_get(descs, 0))->desc_id, computed_desc_id, DIGEST_LEN); - test_assert(rend_parse_v2_service_descriptor(&parsed, parsed_desc_id, - &intro_points_encrypted, - &intro_points_size, - &encoded_size, - &next_desc, - ((rend_encoded_v2_service_descriptor_t *) - smartlist_get(descs, 0))->desc_str) == 0); - test_assert(parsed); - test_memeq(((rend_encoded_v2_service_descriptor_t *) - smartlist_get(descs, 0))->desc_id, parsed_desc_id, DIGEST_LEN); - test_eq(rend_parse_introduction_points(parsed, intro_points_encrypted, - intro_points_size), 3); - test_assert(!crypto_pk_cmp_keys(generated->pk, parsed->pk)); - test_eq(parsed->timestamp, now); - test_eq(parsed->version, 2); - test_eq(parsed->protocols, 42); - test_eq(smartlist_len(parsed->intro_nodes), 3); - for (i = 0; i < smartlist_len(parsed->intro_nodes); i++) { - rend_intro_point_t *par_intro = smartlist_get(parsed->intro_nodes, i), - *gen_intro = smartlist_get(generated->intro_nodes, i); - extend_info_t *par_info = par_intro->extend_info; - extend_info_t *gen_info = gen_intro->extend_info; - test_assert(!crypto_pk_cmp_keys(gen_info->onion_key, par_info->onion_key)); - test_memeq(gen_info->identity_digest, par_info->identity_digest, - DIGEST_LEN); - test_streq(gen_info->nickname, par_info->nickname); - test_assert(tor_addr_eq(&gen_info->addr, &par_info->addr)); - test_eq(gen_info->port, par_info->port); - } - - rend_service_descriptor_free(parsed); - rend_service_descriptor_free(generated); - parsed = generated = NULL; - - done: - if (descs) { - for (i = 0; i < smartlist_len(descs); i++) - rend_encoded_v2_service_descriptor_free(smartlist_get(descs, i)); - smartlist_free(descs); - } - if (parsed) - rend_service_descriptor_free(parsed); - if (generated) - rend_service_descriptor_free(generated); - if (pk1) - crypto_free_pk_env(pk1); - if (pk2) - crypto_free_pk_env(pk2); - tor_free(intro_points_encrypted); -} - -/** Run unit tests for GeoIP code. */ -static void -test_geoip(void) -{ - int i, j; - time_t now = time(NULL); - char *s = NULL; - - /* Populate the DB a bit. Add these in order, since we can't do the final - * 'sort' step. These aren't very good IP addresses, but they're perfectly - * fine uint32_t values. */ - test_eq(0, geoip_parse_entry("10,50,AB")); - test_eq(0, geoip_parse_entry("52,90,XY")); - test_eq(0, geoip_parse_entry("95,100,AB")); - test_eq(0, geoip_parse_entry("\"105\",\"140\",\"ZZ\"")); - test_eq(0, geoip_parse_entry("\"150\",\"190\",\"XY\"")); - test_eq(0, geoip_parse_entry("\"200\",\"250\",\"AB\"")); - - /* We should have 3 countries: ab, xy, zz. */ - test_eq(3, geoip_get_n_countries()); - /* Make sure that country ID actually works. */ -#define NAMEFOR(x) geoip_get_country_name(geoip_get_country_by_ip(x)) - test_streq("ab", NAMEFOR(32)); - test_streq("??", NAMEFOR(5)); - test_streq("??", NAMEFOR(51)); - test_streq("xy", NAMEFOR(150)); - test_streq("xy", NAMEFOR(190)); - test_streq("??", NAMEFOR(2000)); -#undef NAMEFOR - - get_options()->BridgeRelay = 1; - get_options()->BridgeRecordUsageByCountry = 1; - /* Put 9 observations in AB... */ - for (i=32; i < 40; ++i) - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, i, now-7200); - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, 225, now-7200); - /* and 3 observations in XY, several times. */ - for (j=0; j < 10; ++j) - for (i=52; i < 55; ++i) - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, i, now-3600); - /* and 17 observations in ZZ... */ - for (i=110; i < 127; ++i) - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, i, now); - s = geoip_get_client_history(now+5*24*60*60, GEOIP_CLIENT_CONNECT); - test_assert(s); - test_streq("zz=24,ab=16,xy=8", s); - tor_free(s); - - /* Now clear out all the AB observations. */ - geoip_remove_old_clients(now-6000); - s = geoip_get_client_history(now+5*24*60*60, GEOIP_CLIENT_CONNECT); - test_assert(s); - test_streq("zz=24,xy=8", s); - - done: - tor_free(s); -} - -/** For test_array. Declare an CLI-invocable off-by-default function in the - * unit tests, with function name and user-visible name <b>x</b>*/ -#define DISABLED(x) { #x, x, 0, 0, 0 } -/** For test_array. Declare an CLI-invocable unit test function, with function - * name test_<b>x</b>(), and user-visible name <b>x</b> */ -#define ENT(x) { #x, test_ ## x, 0, 0, 1 } -/** For test_array. Declare an CLI-invocable unit test function, with function - * name test_<b>x</b>_<b>y</b>(), and user-visible name - * <b>x</b>/<b>y</b>. This function will be treated as a subentry of <b>x</b>, - * so that invoking <b>x</b> from the CLI invokes this test too. */ -#define SUBENT(x,y) { #x "/" #y, test_ ## x ## _ ## y, 1, 0, 1 } - -/** An array of functions and information for all the unit tests we can run. */ -static struct { - const char *test_name; /**< How does the user refer to this test from the - * command line? */ - void (*test_fn)(void); /**< What function is called to run this test? */ - int is_subent; /**< Is this a subentry of a bigger set of related tests? */ - int selected; /**< Are we planning to run this one? */ - int is_default; /**< If the user doesn't say what tests they want, do they - * get this function by default? */ -} test_array[] = { - ENT(buffers), - ENT(crypto), - SUBENT(crypto, rng), - SUBENT(crypto, aes), - SUBENT(crypto, sha), - SUBENT(crypto, pk), - SUBENT(crypto, dh), - SUBENT(crypto, s2k), - SUBENT(crypto, aes_iv), - SUBENT(crypto, base32_decode), - ENT(util), - SUBENT(util, ip6_helpers), - SUBENT(util, gzip), - SUBENT(util, datadir), - SUBENT(util, smartlist_basic), - SUBENT(util, smartlist_strings), - SUBENT(util, smartlist_overlap), - SUBENT(util, smartlist_digests), - SUBENT(util, smartlist_join), - SUBENT(util, bitarray), - SUBENT(util, digestset), - SUBENT(util, mempool), - SUBENT(util, memarea), - SUBENT(util, strmap), - SUBENT(util, control_formats), - SUBENT(util, pqueue), - SUBENT(util, mmap), - SUBENT(util, threads), - SUBENT(util, order_functions), - SUBENT(util, sscanf), - ENT(onion_handshake), - ENT(dir_format), - ENT(dirutil), - ENT(v3_networkstatus), - ENT(policies), - ENT(rend_fns), - SUBENT(rend_fns, v2), - ENT(geoip), - - DISABLED(bench_aes), - DISABLED(bench_dmap), - { NULL, NULL, 0, 0, 0 }, -}; - -static void syntax(void) ATTR_NORETURN; - -/** Print a syntax usage message, and exit.*/ -static void -syntax(void) -{ - int i; - printf("Syntax:\n" - " test [-v|--verbose] [--warn|--notice|--info|--debug]\n" - " [testname...]\n" - "Recognized tests are:\n"); - for (i = 0; test_array[i].test_name; ++i) { - printf(" %s\n", test_array[i].test_name); - } - - exit(0); -} - -/** Main entry point for unit test code: parse the command line, and run - * some unit tests. */ -int -main(int c, char**v) -{ - or_options_t *options; - char *errmsg = NULL; - int i; - int verbose = 0, any_selected = 0; - int loglevel = LOG_ERR; - -#ifdef USE_DMALLOC - { - int r = CRYPTO_set_mem_ex_functions(_tor_malloc, _tor_realloc, _tor_free); - tor_assert(r); - } -#endif - - update_approx_time(time(NULL)); - options = options_new(); - tor_threads_init(); - init_logging(); - - for (i = 1; i < c; ++i) { - if (!strcmp(v[i], "-v") || !strcmp(v[i], "--verbose")) - verbose++; - else if (!strcmp(v[i], "--warn")) - loglevel = LOG_WARN; - else if (!strcmp(v[i], "--notice")) - loglevel = LOG_NOTICE; - else if (!strcmp(v[i], "--info")) - loglevel = LOG_INFO; - else if (!strcmp(v[i], "--debug")) - loglevel = LOG_DEBUG; - else if (!strcmp(v[i], "--help") || !strcmp(v[i], "-h") || v[i][0] == '-') - syntax(); - else { - int j, found=0; - for (j = 0; test_array[j].test_name; ++j) { - if (!strcmp(v[i], test_array[j].test_name) || - (test_array[j].is_subent && - !strcmpstart(test_array[j].test_name, v[i]) && - test_array[j].test_name[strlen(v[i])] == '/') || - (v[i][0] == '=' && !strcmp(v[i]+1, test_array[j].test_name))) { - test_array[j].selected = 1; - any_selected = 1; - found = 1; - } - } - if (!found) { - printf("Unknown test: %s\n", v[i]); - syntax(); - } - } - } - - if (!any_selected) { - for (i = 0; test_array[i].test_name; ++i) { - test_array[i].selected = test_array[i].is_default; - } - } - - { - log_severity_list_t s; - memset(&s, 0, sizeof(s)); - set_log_severity_config(loglevel, LOG_ERR, &s); - add_stream_log(&s, "", fileno(stdout)); - } - - options->command = CMD_RUN_UNITTESTS; - crypto_global_init(0); - rep_hist_init(); - network_init(); - setup_directory(); - options_init(options); - options->DataDirectory = tor_strdup(temp_dir); - if (set_options(options, &errmsg) < 0) { - printf("Failed to set initial options: %s\n", errmsg); - tor_free(errmsg); - return 1; - } - - crypto_seed_rng(1); - - atexit(remove_directory); - - printf("Running Tor unit tests on %s\n", get_uname()); - - for (i = 0; test_array[i].test_name; ++i) { - if (!test_array[i].selected) - continue; - if (!test_array[i].is_subent) { - printf("\n============================== %s\n",test_array[i].test_name); - } else if (test_array[i].is_subent && verbose) { - printf("\n%s", test_array[i].test_name); - } - test_array[i].test_fn(); - } - puts(""); - - free_pregenerated_keys(); -#ifdef USE_DMALLOC - tor_free_all(0); - dmalloc_log_unfreed(); -#endif - - if (have_failed) - return 1; - else - return 0; -} - diff --git a/src/or/tor_main.c b/src/or/tor_main.c index 137da02c41..4a6be7cddd 100644 --- a/src/or/tor_main.c +++ b/src/or/tor_main.c @@ -7,7 +7,7 @@ * built from. This string is generated by a bit of shell kludging int * src/or/Makefile.am, and is usually right. */ -const char tor_svn_revision[] = +const char tor_git_revision[] = #ifndef _MSC_VER #include "micro-revision.i" #endif diff --git a/src/test/Makefile.am b/src/test/Makefile.am new file mode 100644 index 0000000000..7df13da51e --- /dev/null +++ b/src/test/Makefile.am @@ -0,0 +1,30 @@ +TESTS = test + +noinst_PROGRAMS = test + +AM_CPPFLAGS = -DSHARE_DATADIR="\"$(datadir)\"" \ + -DLOCALSTATEDIR="\"$(localstatedir)\"" \ + -DBINDIR="\"$(bindir)\"" \ + -I"$(top_srcdir)/src/or" + +# -L flags need to go in LDFLAGS. -l flags need to go in LDADD. +# This seems to matter nowhere but on windows, but I assure you that it +# matters a lot there, and is quite hard to debug if you forget to do it. + +test_SOURCES = \ + test_data.c \ + test.c \ + test_addr.c \ + test_crypto.c \ + test_dir.c \ + test_containers.c \ + test_util.c \ + tinytest.c + +test_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ \ + @TOR_LDFLAGS_libevent@ +test_LDADD = ../or/libtor.a ../common/libor.a ../common/libor-crypto.a \ + ../common/libor-event.a \ + -lz -lm @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ + +noinst_HEADERS = tinytest.h tinytest_macros.h test.h diff --git a/src/test/test.c b/src/test/test.c new file mode 100644 index 0000000000..d85f1f0f87 --- /dev/null +++ b/src/test/test.c @@ -0,0 +1,1212 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2009, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/* Ordinarily defined in tor_main.c; this bit is just here to provide one + * since we're not linking to tor_main.c */ +const char tor_git_revision[] = ""; + +/** + * \file test.c + * \brief Unit tests for many pieces of the lower level Tor modules. + **/ + +#include "orconfig.h" + +#include <stdio.h> +#ifdef HAVE_FCNTL_H +#include <fcntl.h> +#endif + +#ifdef MS_WINDOWS +/* For mkdir() */ +#include <direct.h> +#else +#include <dirent.h> +#endif + +/* These macros pull in declarations for some functions and structures that + * are typically file-private. */ +#define BUFFERS_PRIVATE +#define CONFIG_PRIVATE +#define GEOIP_PRIVATE +#define ROUTER_PRIVATE +#define CIRCUIT_PRIVATE + +/* + * Linux doesn't provide lround in math.h by default, but mac os does... + * It's best just to leave math.h out of the picture entirely. + */ +//#include <math.h> +long int lround(double x); +double fabs(double x); + +#include "or.h" +#include "test.h" +#include "torgzip.h" +#include "mempool.h" +#include "memarea.h" + +#ifdef USE_DMALLOC +#include <dmalloc.h> +#include <openssl/crypto.h> +#endif + +/** Set to true if any unit test has failed. Mostly, this is set by the macros + * in test.h */ +int have_failed = 0; + +/** Temporary directory (set up by setup_directory) under which we store all + * our files during testing. */ +static char temp_dir[256]; + +/** Select and create the temporary directory we'll use to run our unit tests. + * Store it in <b>temp_dir</b>. Exit immediately if we can't create it. + * idempotent. */ +static void +setup_directory(void) +{ + static int is_setup = 0; + int r; + if (is_setup) return; + +#ifdef MS_WINDOWS + // XXXX + tor_snprintf(temp_dir, sizeof(temp_dir), + "c:\\windows\\temp\\tor_test_%d", (int)getpid()); + r = mkdir(temp_dir); +#else + tor_snprintf(temp_dir, sizeof(temp_dir), "/tmp/tor_test_%d", (int) getpid()); + r = mkdir(temp_dir, 0700); +#endif + if (r) { + fprintf(stderr, "Can't create directory %s:", temp_dir); + perror(""); + exit(1); + } + is_setup = 1; +} + +/** Return a filename relative to our testing temporary directory */ +const char * +get_fname(const char *name) +{ + static char buf[1024]; + setup_directory(); + tor_snprintf(buf,sizeof(buf),"%s/%s",temp_dir,name); + return buf; +} + +/** Remove all files stored under the temporary directory, and the directory + * itself. */ +static void +remove_directory(void) +{ + smartlist_t *elements = tor_listdir(temp_dir); + if (elements) { + SMARTLIST_FOREACH(elements, const char *, cp, + { + size_t len = strlen(cp)+strlen(temp_dir)+16; + char *tmp = tor_malloc(len); + tor_snprintf(tmp, len, "%s"PATH_SEPARATOR"%s", temp_dir, cp); + unlink(tmp); + tor_free(tmp); + }); + SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp)); + smartlist_free(elements); + } + rmdir(temp_dir); +} + +/** Define this if unit tests spend too much time generating public keys*/ +#undef CACHE_GENERATED_KEYS + +static crypto_pk_env_t *pregen_keys[5] = {NULL, NULL, NULL, NULL, NULL}; +#define N_PREGEN_KEYS ((int)(sizeof(pregen_keys)/sizeof(pregen_keys[0]))) + +/** Generate and return a new keypair for use in unit tests. If we're using + * the key cache optimization, we might reuse keys: we only guarantee that + * keys made with distinct values for <b>idx</b> are different. The value of + * <b>idx</b> must be at least 0, and less than N_PREGEN_KEYS. */ +crypto_pk_env_t * +pk_generate(int idx) +{ +#ifdef CACHE_GENERATED_KEYS + tor_assert(idx < N_PREGEN_KEYS); + if (! pregen_keys[idx]) { + pregen_keys[idx] = crypto_new_pk_env(); + tor_assert(!crypto_pk_generate_key(pregen_keys[idx])); + } + return crypto_pk_dup_key(pregen_keys[idx]); +#else + crypto_pk_env_t *result; + (void) idx; + result = crypto_new_pk_env(); + tor_assert(!crypto_pk_generate_key(result)); + return result; +#endif +} + +/** Free all storage used for the cached key optimization. */ +static void +free_pregenerated_keys(void) +{ + unsigned idx; + for (idx = 0; idx < N_PREGEN_KEYS; ++idx) { + if (pregen_keys[idx]) { + crypto_free_pk_env(pregen_keys[idx]); + pregen_keys[idx] = NULL; + } + } +} + +/** Run unit tests for buffers.c */ +static void +test_buffers(void) +{ + char str[256]; + char str2[256]; + + buf_t *buf = NULL, *buf2 = NULL; + const char *cp; + + int j; + size_t r; + + /**** + * buf_new + ****/ + if (!(buf = buf_new())) + test_fail(); + + //test_eq(buf_capacity(buf), 4096); + test_eq(buf_datalen(buf), 0); + + /**** + * General pointer frobbing + */ + for (j=0;j<256;++j) { + str[j] = (char)j; + } + write_to_buf(str, 256, buf); + write_to_buf(str, 256, buf); + test_eq(buf_datalen(buf), 512); + fetch_from_buf(str2, 200, buf); + test_memeq(str, str2, 200); + test_eq(buf_datalen(buf), 312); + memset(str2, 0, sizeof(str2)); + + fetch_from_buf(str2, 256, buf); + test_memeq(str+200, str2, 56); + test_memeq(str, str2+56, 200); + test_eq(buf_datalen(buf), 56); + memset(str2, 0, sizeof(str2)); + /* Okay, now we should be 512 bytes into the 4096-byte buffer. If we add + * another 3584 bytes, we hit the end. */ + for (j=0;j<15;++j) { + write_to_buf(str, 256, buf); + } + assert_buf_ok(buf); + test_eq(buf_datalen(buf), 3896); + fetch_from_buf(str2, 56, buf); + test_eq(buf_datalen(buf), 3840); + test_memeq(str+200, str2, 56); + for (j=0;j<15;++j) { + memset(str2, 0, sizeof(str2)); + fetch_from_buf(str2, 256, buf); + test_memeq(str, str2, 256); + } + test_eq(buf_datalen(buf), 0); + buf_free(buf); + buf = NULL; + + /* Okay, now make sure growing can work. */ + buf = buf_new_with_capacity(16); + //test_eq(buf_capacity(buf), 16); + write_to_buf(str+1, 255, buf); + //test_eq(buf_capacity(buf), 256); + fetch_from_buf(str2, 254, buf); + test_memeq(str+1, str2, 254); + //test_eq(buf_capacity(buf), 256); + assert_buf_ok(buf); + write_to_buf(str, 32, buf); + //test_eq(buf_capacity(buf), 256); + assert_buf_ok(buf); + write_to_buf(str, 256, buf); + assert_buf_ok(buf); + //test_eq(buf_capacity(buf), 512); + test_eq(buf_datalen(buf), 33+256); + fetch_from_buf(str2, 33, buf); + test_eq(*str2, str[255]); + + test_memeq(str2+1, str, 32); + //test_eq(buf_capacity(buf), 512); + test_eq(buf_datalen(buf), 256); + fetch_from_buf(str2, 256, buf); + test_memeq(str, str2, 256); + + /* now try shrinking: case 1. */ + buf_free(buf); + buf = buf_new_with_capacity(33668); + for (j=0;j<67;++j) { + write_to_buf(str,255, buf); + } + //test_eq(buf_capacity(buf), 33668); + test_eq(buf_datalen(buf), 17085); + for (j=0; j < 40; ++j) { + fetch_from_buf(str2, 255,buf); + test_memeq(str2, str, 255); + } + + /* now try shrinking: case 2. */ + buf_free(buf); + buf = buf_new_with_capacity(33668); + for (j=0;j<67;++j) { + write_to_buf(str,255, buf); + } + for (j=0; j < 20; ++j) { + fetch_from_buf(str2, 255,buf); + test_memeq(str2, str, 255); + } + for (j=0;j<80;++j) { + write_to_buf(str,255, buf); + } + //test_eq(buf_capacity(buf),33668); + for (j=0; j < 120; ++j) { + fetch_from_buf(str2, 255,buf); + test_memeq(str2, str, 255); + } + + /* Move from buf to buf. */ + buf_free(buf); + buf = buf_new_with_capacity(4096); + buf2 = buf_new_with_capacity(4096); + for (j=0;j<100;++j) + write_to_buf(str, 255, buf); + test_eq(buf_datalen(buf), 25500); + for (j=0;j<100;++j) { + r = 10; + move_buf_to_buf(buf2, buf, &r); + test_eq(r, 0); + } + test_eq(buf_datalen(buf), 24500); + test_eq(buf_datalen(buf2), 1000); + for (j=0;j<3;++j) { + fetch_from_buf(str2, 255, buf2); + test_memeq(str2, str, 255); + } + r = 8192; /*big move*/ + move_buf_to_buf(buf2, buf, &r); + test_eq(r, 0); + r = 30000; /* incomplete move */ + move_buf_to_buf(buf2, buf, &r); + test_eq(r, 13692); + for (j=0;j<97;++j) { + fetch_from_buf(str2, 255, buf2); + test_memeq(str2, str, 255); + } + buf_free(buf); + buf_free(buf2); + buf = buf2 = NULL; + + buf = buf_new_with_capacity(5); + cp = "Testing. This is a moderately long Testing string."; + for (j = 0; cp[j]; j++) + write_to_buf(cp+j, 1, buf); + test_eq(0, buf_find_string_offset(buf, "Testing", 7)); + test_eq(1, buf_find_string_offset(buf, "esting", 6)); + test_eq(1, buf_find_string_offset(buf, "est", 3)); + test_eq(39, buf_find_string_offset(buf, "ing str", 7)); + test_eq(35, buf_find_string_offset(buf, "Testing str", 11)); + test_eq(32, buf_find_string_offset(buf, "ng ", 3)); + test_eq(43, buf_find_string_offset(buf, "string.", 7)); + test_eq(-1, buf_find_string_offset(buf, "shrdlu", 6)); + test_eq(-1, buf_find_string_offset(buf, "Testing thing", 13)); + test_eq(-1, buf_find_string_offset(buf, "ngx", 3)); + buf_free(buf); + buf = NULL; + +#if 0 + { + int s; + int eof; + int i; + buf_t *buf2; + /**** + * read_to_buf + ****/ + s = open(get_fname("data"), O_WRONLY|O_CREAT|O_TRUNC, 0600); + write(s, str, 256); + close(s); + + s = open(get_fname("data"), O_RDONLY, 0); + eof = 0; + errno = 0; /* XXXX */ + i = read_to_buf(s, 10, buf, &eof); + printf("%s\n", strerror(errno)); + test_eq(i, 10); + test_eq(eof, 0); + //test_eq(buf_capacity(buf), 4096); + test_eq(buf_datalen(buf), 10); + + test_memeq(str, (char*)_buf_peek_raw_buffer(buf), 10); + + /* Test reading 0 bytes. */ + i = read_to_buf(s, 0, buf, &eof); + //test_eq(buf_capacity(buf), 512*1024); + test_eq(buf_datalen(buf), 10); + test_eq(eof, 0); + test_eq(i, 0); + + /* Now test when buffer is filled exactly. */ + buf2 = buf_new_with_capacity(6); + i = read_to_buf(s, 6, buf2, &eof); + //test_eq(buf_capacity(buf2), 6); + test_eq(buf_datalen(buf2), 6); + test_eq(eof, 0); + test_eq(i, 6); + test_memeq(str+10, (char*)_buf_peek_raw_buffer(buf2), 6); + buf_free(buf2); + buf2 = NULL; + + /* Now test when buffer is filled with more data to read. */ + buf2 = buf_new_with_capacity(32); + i = read_to_buf(s, 128, buf2, &eof); + //test_eq(buf_capacity(buf2), 128); + test_eq(buf_datalen(buf2), 32); + test_eq(eof, 0); + test_eq(i, 32); + buf_free(buf2); + buf2 = NULL; + + /* Now read to eof. */ + test_assert(buf_capacity(buf) > 256); + i = read_to_buf(s, 1024, buf, &eof); + test_eq(i, (256-32-10-6)); + test_eq(buf_capacity(buf), MAX_BUF_SIZE); + test_eq(buf_datalen(buf), 256-6-32); + test_memeq(str, (char*)_buf_peek_raw_buffer(buf), 10); /* XXX Check rest. */ + test_eq(eof, 0); + + i = read_to_buf(s, 1024, buf, &eof); + test_eq(i, 0); + test_eq(buf_capacity(buf), MAX_BUF_SIZE); + test_eq(buf_datalen(buf), 256-6-32); + test_eq(eof, 1); + } +#endif + + done: + if (buf) + buf_free(buf); + if (buf2) + buf_free(buf2); +} + +/** Run unit tests for the onion handshake code. */ +static void +test_onion_handshake(void) +{ + /* client-side */ + crypto_dh_env_t *c_dh = NULL; + char c_buf[ONIONSKIN_CHALLENGE_LEN]; + char c_keys[40]; + + /* server-side */ + char s_buf[ONIONSKIN_REPLY_LEN]; + char s_keys[40]; + + /* shared */ + crypto_pk_env_t *pk = NULL; + + pk = pk_generate(0); + + /* client handshake 1. */ + memset(c_buf, 0, ONIONSKIN_CHALLENGE_LEN); + test_assert(! onion_skin_create(pk, &c_dh, c_buf)); + + /* server handshake */ + memset(s_buf, 0, ONIONSKIN_REPLY_LEN); + memset(s_keys, 0, 40); + test_assert(! onion_skin_server_handshake(c_buf, pk, NULL, + s_buf, s_keys, 40)); + + /* client handshake 2 */ + memset(c_keys, 0, 40); + test_assert(! onion_skin_client_handshake(c_dh, s_buf, c_keys, 40)); + + if (memcmp(c_keys, s_keys, 40)) { + puts("Aiiiie"); + exit(1); + } + test_memeq(c_keys, s_keys, 40); + memset(s_buf, 0, 40); + test_memneq(c_keys, s_buf, 40); + + done: + if (c_dh) + crypto_dh_free(c_dh); + if (pk) + crypto_free_pk_env(pk); +} + +static void +test_circuit_timeout(void) +{ + /* Plan: + * 1. Generate 1000 samples + * 2. Estimate parameters + * 3. If difference, repeat + * 4. Save state + * 5. load state + * 6. Estimate parameters + * 7. compare differences + */ + circuit_build_times_t initial; + circuit_build_times_t estimate; + circuit_build_times_t final; + double timeout1, timeout2; + or_state_t state; + char *msg; + int i, runs; + circuit_build_times_init(&initial); + circuit_build_times_init(&estimate); + circuit_build_times_init(&final); + + memset(&state, 0, sizeof(or_state_t)); + + circuitbuild_running_unit_tests(); +#define timeout0 (build_time_t)(30*1000.0) + initial.Xm = 750; + circuit_build_times_initial_alpha(&initial, BUILDTIMEOUT_QUANTILE_CUTOFF, + timeout0); + do { + int n = 0; + for (i=0; i < MIN_CIRCUITS_TO_OBSERVE; i++) { + if (circuit_build_times_add_time(&estimate, + circuit_build_times_generate_sample(&initial, 0, 1)) == 0) { + n++; + } + } + circuit_build_times_update_alpha(&estimate); + timeout1 = circuit_build_times_calculate_timeout(&estimate, + BUILDTIMEOUT_QUANTILE_CUTOFF); + circuit_build_times_set_timeout(&estimate); + log_warn(LD_CIRC, "Timeout is %lf, Xm is %d", timeout1, estimate.Xm); + /* XXX: 5% distribution error may not be the right metric */ + } while (fabs(circuit_build_times_cdf(&initial, timeout0) - + circuit_build_times_cdf(&initial, timeout1)) > 0.05 + /* 5% error */ + && estimate.total_build_times < NCIRCUITS_TO_OBSERVE); + + test_assert(estimate.total_build_times < NCIRCUITS_TO_OBSERVE); + + circuit_build_times_update_state(&estimate, &state); + test_assert(circuit_build_times_parse_state(&final, &state, &msg) == 0); + + circuit_build_times_update_alpha(&final); + timeout2 = circuit_build_times_calculate_timeout(&final, + BUILDTIMEOUT_QUANTILE_CUTOFF); + + circuit_build_times_set_timeout(&final); + log_warn(LD_CIRC, "Timeout is %lf, Xm is %d", timeout2, final.Xm); + + test_assert(fabs(circuit_build_times_cdf(&initial, timeout0) - + circuit_build_times_cdf(&initial, timeout2)) < 0.05); + + for (runs = 0; runs < 50; runs++) { + int build_times_idx = 0; + int total_build_times = 0; + + final.timeout_ms = BUILD_TIMEOUT_INITIAL_VALUE; + estimate.timeout_ms = BUILD_TIMEOUT_INITIAL_VALUE; + + for (i = 0; i < RECENT_CIRCUITS*2; i++) { + circuit_build_times_network_circ_success(&estimate); + circuit_build_times_add_time(&estimate, + circuit_build_times_generate_sample(&estimate, 0, + BUILDTIMEOUT_QUANTILE_CUTOFF)); + estimate.have_computed_timeout = 1; + circuit_build_times_network_circ_success(&estimate); + circuit_build_times_add_time(&final, + circuit_build_times_generate_sample(&final, 0, + BUILDTIMEOUT_QUANTILE_CUTOFF)); + final.have_computed_timeout = 1; + } + + test_assert(!circuit_build_times_network_check_changed(&estimate)); + test_assert(!circuit_build_times_network_check_changed(&final)); + + /* Reset liveness to be non-live */ + final.liveness.network_last_live = 0; + estimate.liveness.network_last_live = 0; + + build_times_idx = estimate.build_times_idx; + total_build_times = estimate.total_build_times; + for (i = 0; i < NETWORK_NONLIVE_TIMEOUT_COUNT; i++) { + test_assert(circuit_build_times_network_check_live(&estimate)); + test_assert(circuit_build_times_network_check_live(&final)); + + if (circuit_build_times_add_timeout(&estimate, 0, + (time_t)(approx_time()-estimate.timeout_ms/1000.0-1))) + estimate.have_computed_timeout = 1; + if (circuit_build_times_add_timeout(&final, 0, + (time_t)(approx_time()-final.timeout_ms/1000.0-1))) + final.have_computed_timeout = 1; + } + + test_assert(!circuit_build_times_network_check_live(&estimate)); + test_assert(!circuit_build_times_network_check_live(&final)); + + for ( ; i < NETWORK_NONLIVE_DISCARD_COUNT; i++) { + if (circuit_build_times_add_timeout(&estimate, 0, + (time_t)(approx_time()-estimate.timeout_ms/1000.0-1))) + estimate.have_computed_timeout = 1; + + if (i < NETWORK_NONLIVE_DISCARD_COUNT-1) { + if (circuit_build_times_add_timeout(&final, 0, + (time_t)(approx_time()-final.timeout_ms/1000.0-1))) + final.have_computed_timeout = 1; + } + } + + test_assert(!circuit_build_times_network_check_live(&estimate)); + test_assert(!circuit_build_times_network_check_live(&final)); + + log_info(LD_CIRC, "idx: %d %d, tot: %d %d", + build_times_idx, estimate.build_times_idx, + total_build_times, estimate.total_build_times); + + /* Check rollback index. Should match top of loop. */ + test_assert(build_times_idx == estimate.build_times_idx); + test_assert(total_build_times == estimate.total_build_times); + + /* Now simulate that the network has become live and we need + * a change */ + circuit_build_times_network_is_live(&estimate); + circuit_build_times_network_is_live(&final); + + for (i = 0; i < MAX_RECENT_TIMEOUT_COUNT; i++) { + if (circuit_build_times_add_timeout(&estimate, 1, approx_time()-1)) + estimate.have_computed_timeout = 1; + + if (i < MAX_RECENT_TIMEOUT_COUNT-1) { + if (circuit_build_times_add_timeout(&final, 1, approx_time()-1)) + final.have_computed_timeout = 1; + } + } + + test_assert(estimate.liveness.after_firsthop_idx == 0); + test_assert(final.liveness.after_firsthop_idx == + MAX_RECENT_TIMEOUT_COUNT-1); + + test_assert(circuit_build_times_network_check_live(&estimate)); + test_assert(circuit_build_times_network_check_live(&final)); + + if (circuit_build_times_add_timeout(&final, 1, approx_time()-1)) + final.have_computed_timeout = 1; + + } + +done: + return; +} + +/** Helper: Parse the exit policy string in <b>policy_str</b>, and make sure + * that policies_summarize() produces the string <b>expected_summary</b> from + * it. */ +static void +test_policy_summary_helper(const char *policy_str, + const char *expected_summary) +{ + config_line_t line; + smartlist_t *policy = smartlist_create(); + char *summary = NULL; + int r; + + line.key = (char*)"foo"; + line.value = (char *)policy_str; + line.next = NULL; + + r = policies_parse_exit_policy(&line, &policy, 0, NULL, 1); + test_eq(r, 0); + summary = policy_summarize(policy); + + test_assert(summary != NULL); + test_streq(summary, expected_summary); + + done: + tor_free(summary); + if (policy) + addr_policy_list_free(policy); +} + +/** Run unit tests for generating summary lines of exit policies */ +static void +test_policies(void) +{ + int i; + smartlist_t *policy = NULL, *policy2 = NULL; + addr_policy_t *p; + tor_addr_t tar; + config_line_t line; + smartlist_t *sm = NULL; + char *policy_str = NULL; + + policy = smartlist_create(); + + p = router_parse_addr_policy_item_from_string("reject 192.168.0.0/16:*",-1); + test_assert(p != NULL); + test_eq(ADDR_POLICY_REJECT, p->policy_type); + tor_addr_from_ipv4h(&tar, 0xc0a80000u); + test_eq(0, tor_addr_compare(&p->addr, &tar, CMP_EXACT)); + test_eq(16, p->maskbits); + test_eq(1, p->prt_min); + test_eq(65535, p->prt_max); + + smartlist_add(policy, p); + + test_assert(ADDR_POLICY_ACCEPTED == + compare_addr_to_addr_policy(0x01020304u, 2, policy)); + test_assert(ADDR_POLICY_PROBABLY_ACCEPTED == + compare_addr_to_addr_policy(0, 2, policy)); + test_assert(ADDR_POLICY_REJECTED == + compare_addr_to_addr_policy(0xc0a80102, 2, policy)); + + policy2 = NULL; + test_assert(0 == policies_parse_exit_policy(NULL, &policy2, 1, NULL, 1)); + test_assert(policy2); + + test_assert(!exit_policy_is_general_exit(policy)); + test_assert(exit_policy_is_general_exit(policy2)); + test_assert(!exit_policy_is_general_exit(NULL)); + + test_assert(cmp_addr_policies(policy, policy2)); + test_assert(cmp_addr_policies(policy, NULL)); + test_assert(!cmp_addr_policies(policy2, policy2)); + test_assert(!cmp_addr_policies(NULL, NULL)); + + test_assert(!policy_is_reject_star(policy2)); + test_assert(policy_is_reject_star(policy)); + test_assert(policy_is_reject_star(NULL)); + + addr_policy_list_free(policy); + policy = NULL; + + /* make sure compacting logic works. */ + policy = NULL; + line.key = (char*)"foo"; + line.value = (char*)"accept *:80,reject private:*,reject *:*"; + line.next = NULL; + test_assert(0 == policies_parse_exit_policy(&line, &policy, 0, NULL, 1)); + test_assert(policy); + //test_streq(policy->string, "accept *:80"); + //test_streq(policy->next->string, "reject *:*"); + test_eq(smartlist_len(policy), 2); + + /* test policy summaries */ + /* check if we properly ignore private IP addresses */ + test_policy_summary_helper("reject 192.168.0.0/16:*," + "reject 0.0.0.0/8:*," + "reject 10.0.0.0/8:*," + "accept *:10-30," + "accept *:90," + "reject *:*", + "accept 10-30,90"); + /* check all accept policies, and proper counting of rejects */ + test_policy_summary_helper("reject 11.0.0.0/9:80," + "reject 12.0.0.0/9:80," + "reject 13.0.0.0/9:80," + "reject 14.0.0.0/9:80," + "accept *:*", "accept 1-65535"); + test_policy_summary_helper("reject 11.0.0.0/9:80," + "reject 12.0.0.0/9:80," + "reject 13.0.0.0/9:80," + "reject 14.0.0.0/9:80," + "reject 15.0.0.0:81," + "accept *:*", "accept 1-65535"); + test_policy_summary_helper("reject 11.0.0.0/9:80," + "reject 12.0.0.0/9:80," + "reject 13.0.0.0/9:80," + "reject 14.0.0.0/9:80," + "reject 15.0.0.0:80," + "accept *:*", + "reject 80"); + /* no exits */ + test_policy_summary_helper("accept 11.0.0.0/9:80," + "reject *:*", + "reject 1-65535"); + /* port merging */ + test_policy_summary_helper("accept *:80," + "accept *:81," + "accept *:100-110," + "accept *:111," + "reject *:*", + "accept 80-81,100-111"); + /* border ports */ + test_policy_summary_helper("accept *:1," + "accept *:3," + "accept *:65535," + "reject *:*", + "accept 1,3,65535"); + /* holes */ + test_policy_summary_helper("accept *:1," + "accept *:3," + "accept *:5," + "accept *:7," + "reject *:*", + "accept 1,3,5,7"); + test_policy_summary_helper("reject *:1," + "reject *:3," + "reject *:5," + "reject *:7," + "accept *:*", + "reject 1,3,5,7"); + + /* truncation ports */ + sm = smartlist_create(); + for (i=1; i<2000; i+=2) { + char buf[POLICY_BUF_LEN]; + tor_snprintf(buf, sizeof(buf), "reject *:%d", i); + smartlist_add(sm, tor_strdup(buf)); + } + smartlist_add(sm, tor_strdup("accept *:*")); + policy_str = smartlist_join_strings(sm, ",", 0, NULL); + test_policy_summary_helper( policy_str, + "accept 2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44," + "46,48,50,52,54,56,58,60,62,64,66,68,70,72,74,76,78,80,82,84,86,88,90," + "92,94,96,98,100,102,104,106,108,110,112,114,116,118,120,122,124,126,128," + "130,132,134,136,138,140,142,144,146,148,150,152,154,156,158,160,162,164," + "166,168,170,172,174,176,178,180,182,184,186,188,190,192,194,196,198,200," + "202,204,206,208,210,212,214,216,218,220,222,224,226,228,230,232,234,236," + "238,240,242,244,246,248,250,252,254,256,258,260,262,264,266,268,270,272," + "274,276,278,280,282,284,286,288,290,292,294,296,298,300,302,304,306,308," + "310,312,314,316,318,320,322,324,326,328,330,332,334,336,338,340,342,344," + "346,348,350,352,354,356,358,360,362,364,366,368,370,372,374,376,378,380," + "382,384,386,388,390,392,394,396,398,400,402,404,406,408,410,412,414,416," + "418,420,422,424,426,428,430,432,434,436,438,440,442,444,446,448,450,452," + "454,456,458,460,462,464,466,468,470,472,474,476,478,480,482,484,486,488," + "490,492,494,496,498,500,502,504,506,508,510,512,514,516,518,520,522"); + + done: + if (policy) + addr_policy_list_free(policy); + if (policy2) + addr_policy_list_free(policy2); + tor_free(policy_str); + if (sm) { + SMARTLIST_FOREACH(sm, char *, s, tor_free(s)); + smartlist_free(sm); + } +} + +/** Run AES performance benchmarks. */ +static void +bench_aes(void) +{ + int len, i; + char *b1, *b2; + crypto_cipher_env_t *c; + struct timeval start, end; + const int iters = 100000; + uint64_t nsec; + c = crypto_new_cipher_env(); + crypto_cipher_generate_key(c); + crypto_cipher_encrypt_init_cipher(c); + for (len = 1; len <= 8192; len *= 2) { + b1 = tor_malloc_zero(len); + b2 = tor_malloc_zero(len); + tor_gettimeofday(&start); + for (i = 0; i < iters; ++i) { + crypto_cipher_encrypt(c, b1, b2, len); + } + tor_gettimeofday(&end); + tor_free(b1); + tor_free(b2); + nsec = (uint64_t) tv_udiff(&start,&end); + nsec *= 1000; + nsec /= (iters*len); + printf("%d bytes: "U64_FORMAT" nsec per byte\n", len, + U64_PRINTF_ARG(nsec)); + } + crypto_free_cipher_env(c); +} + +/** Run digestmap_t performance benchmarks. */ +static void +bench_dmap(void) +{ + smartlist_t *sl = smartlist_create(); + smartlist_t *sl2 = smartlist_create(); + struct timeval start, end, pt2, pt3, pt4; + const int iters = 10000; + const int elts = 4000; + const int fpostests = 1000000; + char d[20]; + int i,n=0, fp = 0; + digestmap_t *dm = digestmap_new(); + digestset_t *ds = digestset_new(elts); + + for (i = 0; i < elts; ++i) { + crypto_rand(d, 20); + smartlist_add(sl, tor_memdup(d, 20)); + } + for (i = 0; i < elts; ++i) { + crypto_rand(d, 20); + smartlist_add(sl2, tor_memdup(d, 20)); + } + printf("nbits=%d\n", ds->mask+1); + + tor_gettimeofday(&start); + for (i = 0; i < iters; ++i) { + SMARTLIST_FOREACH(sl, const char *, cp, digestmap_set(dm, cp, (void*)1)); + } + tor_gettimeofday(&pt2); + for (i = 0; i < iters; ++i) { + SMARTLIST_FOREACH(sl, const char *, cp, digestmap_get(dm, cp)); + SMARTLIST_FOREACH(sl2, const char *, cp, digestmap_get(dm, cp)); + } + tor_gettimeofday(&pt3); + for (i = 0; i < iters; ++i) { + SMARTLIST_FOREACH(sl, const char *, cp, digestset_add(ds, cp)); + } + tor_gettimeofday(&pt4); + for (i = 0; i < iters; ++i) { + SMARTLIST_FOREACH(sl, const char *, cp, n += digestset_isin(ds, cp)); + SMARTLIST_FOREACH(sl2, const char *, cp, n += digestset_isin(ds, cp)); + } + tor_gettimeofday(&end); + + for (i = 0; i < fpostests; ++i) { + crypto_rand(d, 20); + if (digestset_isin(ds, d)) ++fp; + } + + printf("%ld\n",(unsigned long)tv_udiff(&start, &pt2)); + printf("%ld\n",(unsigned long)tv_udiff(&pt2, &pt3)); + printf("%ld\n",(unsigned long)tv_udiff(&pt3, &pt4)); + printf("%ld\n",(unsigned long)tv_udiff(&pt4, &end)); + printf("-- %d\n", n); + printf("++ %f\n", fp/(double)fpostests); + digestmap_free(dm, NULL); + digestset_free(ds); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + SMARTLIST_FOREACH(sl2, char *, cp, tor_free(cp)); + smartlist_free(sl); + smartlist_free(sl2); +} + +/** Test encoding and parsing of rendezvous service descriptors. */ +static void +test_rend_fns(void) +{ + rend_service_descriptor_t *generated = NULL, *parsed = NULL; + char service_id[DIGEST_LEN]; + char service_id_base32[REND_SERVICE_ID_LEN_BASE32+1]; + const char *next_desc; + smartlist_t *descs = smartlist_create(); + char computed_desc_id[DIGEST_LEN]; + char parsed_desc_id[DIGEST_LEN]; + crypto_pk_env_t *pk1 = NULL, *pk2 = NULL; + time_t now; + char *intro_points_encrypted = NULL; + size_t intro_points_size; + size_t encoded_size; + int i; + char address1[] = "fooaddress.onion"; + char address2[] = "aaaaaaaaaaaaaaaa.onion"; + char address3[] = "fooaddress.exit"; + char address4[] = "www.torproject.org"; + + test_assert(BAD_HOSTNAME == parse_extended_hostname(address1, 1)); + test_assert(ONION_HOSTNAME == parse_extended_hostname(address2, 1)); + test_assert(EXIT_HOSTNAME == parse_extended_hostname(address3, 1)); + test_assert(NORMAL_HOSTNAME == parse_extended_hostname(address4, 1)); + + pk1 = pk_generate(0); + pk2 = pk_generate(1); + generated = tor_malloc_zero(sizeof(rend_service_descriptor_t)); + generated->pk = crypto_pk_dup_key(pk1); + crypto_pk_get_digest(generated->pk, service_id); + base32_encode(service_id_base32, REND_SERVICE_ID_LEN_BASE32+1, + service_id, REND_SERVICE_ID_LEN); + now = time(NULL); + generated->timestamp = now; + generated->version = 2; + generated->protocols = 42; + generated->intro_nodes = smartlist_create(); + + for (i = 0; i < 3; i++) { + rend_intro_point_t *intro = tor_malloc_zero(sizeof(rend_intro_point_t)); + crypto_pk_env_t *okey = pk_generate(2 + i); + intro->extend_info = tor_malloc_zero(sizeof(extend_info_t)); + intro->extend_info->onion_key = okey; + crypto_pk_get_digest(intro->extend_info->onion_key, + intro->extend_info->identity_digest); + //crypto_rand(info->identity_digest, DIGEST_LEN); /* Would this work? */ + intro->extend_info->nickname[0] = '$'; + base16_encode(intro->extend_info->nickname + 1, + sizeof(intro->extend_info->nickname) - 1, + intro->extend_info->identity_digest, DIGEST_LEN); + /* Does not cover all IP addresses. */ + tor_addr_from_ipv4h(&intro->extend_info->addr, crypto_rand_int(65536)); + intro->extend_info->port = crypto_rand_int(65536); + intro->intro_key = crypto_pk_dup_key(pk2); + smartlist_add(generated->intro_nodes, intro); + } + test_assert(rend_encode_v2_descriptors(descs, generated, now, 0, + REND_NO_AUTH, NULL, NULL) > 0); + test_assert(rend_compute_v2_desc_id(computed_desc_id, service_id_base32, + NULL, now, 0) == 0); + test_memeq(((rend_encoded_v2_service_descriptor_t *) + smartlist_get(descs, 0))->desc_id, computed_desc_id, DIGEST_LEN); + test_assert(rend_parse_v2_service_descriptor(&parsed, parsed_desc_id, + &intro_points_encrypted, + &intro_points_size, + &encoded_size, + &next_desc, + ((rend_encoded_v2_service_descriptor_t *) + smartlist_get(descs, 0))->desc_str) == 0); + test_assert(parsed); + test_memeq(((rend_encoded_v2_service_descriptor_t *) + smartlist_get(descs, 0))->desc_id, parsed_desc_id, DIGEST_LEN); + test_eq(rend_parse_introduction_points(parsed, intro_points_encrypted, + intro_points_size), 3); + test_assert(!crypto_pk_cmp_keys(generated->pk, parsed->pk)); + test_eq(parsed->timestamp, now); + test_eq(parsed->version, 2); + test_eq(parsed->protocols, 42); + test_eq(smartlist_len(parsed->intro_nodes), 3); + for (i = 0; i < smartlist_len(parsed->intro_nodes); i++) { + rend_intro_point_t *par_intro = smartlist_get(parsed->intro_nodes, i), + *gen_intro = smartlist_get(generated->intro_nodes, i); + extend_info_t *par_info = par_intro->extend_info; + extend_info_t *gen_info = gen_intro->extend_info; + test_assert(!crypto_pk_cmp_keys(gen_info->onion_key, par_info->onion_key)); + test_memeq(gen_info->identity_digest, par_info->identity_digest, + DIGEST_LEN); + test_streq(gen_info->nickname, par_info->nickname); + test_assert(tor_addr_eq(&gen_info->addr, &par_info->addr)); + test_eq(gen_info->port, par_info->port); + } + + rend_service_descriptor_free(parsed); + rend_service_descriptor_free(generated); + parsed = generated = NULL; + + done: + if (descs) { + for (i = 0; i < smartlist_len(descs); i++) + rend_encoded_v2_service_descriptor_free(smartlist_get(descs, i)); + smartlist_free(descs); + } + if (parsed) + rend_service_descriptor_free(parsed); + if (generated) + rend_service_descriptor_free(generated); + if (pk1) + crypto_free_pk_env(pk1); + if (pk2) + crypto_free_pk_env(pk2); + tor_free(intro_points_encrypted); +} + +/** Run unit tests for GeoIP code. */ +static void +test_geoip(void) +{ + int i, j; + time_t now = time(NULL); + char *s = NULL; + + /* Populate the DB a bit. Add these in order, since we can't do the final + * 'sort' step. These aren't very good IP addresses, but they're perfectly + * fine uint32_t values. */ + test_eq(0, geoip_parse_entry("10,50,AB")); + test_eq(0, geoip_parse_entry("52,90,XY")); + test_eq(0, geoip_parse_entry("95,100,AB")); + test_eq(0, geoip_parse_entry("\"105\",\"140\",\"ZZ\"")); + test_eq(0, geoip_parse_entry("\"150\",\"190\",\"XY\"")); + test_eq(0, geoip_parse_entry("\"200\",\"250\",\"AB\"")); + + /* We should have 3 countries: ab, xy, zz. */ + test_eq(3, geoip_get_n_countries()); + /* Make sure that country ID actually works. */ +#define NAMEFOR(x) geoip_get_country_name(geoip_get_country_by_ip(x)) + test_streq("ab", NAMEFOR(32)); + test_streq("??", NAMEFOR(5)); + test_streq("??", NAMEFOR(51)); + test_streq("xy", NAMEFOR(150)); + test_streq("xy", NAMEFOR(190)); + test_streq("??", NAMEFOR(2000)); +#undef NAMEFOR + + get_options()->BridgeRelay = 1; + get_options()->BridgeRecordUsageByCountry = 1; + /* Put 9 observations in AB... */ + for (i=32; i < 40; ++i) + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, i, now-7200); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, 225, now-7200); + /* and 3 observations in XY, several times. */ + for (j=0; j < 10; ++j) + for (i=52; i < 55; ++i) + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, i, now-3600); + /* and 17 observations in ZZ... */ + for (i=110; i < 127; ++i) + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, i, now); + s = geoip_get_client_history_bridge(now+5*24*60*60, + GEOIP_CLIENT_CONNECT); + test_assert(s); + test_streq("zz=24,ab=16,xy=8", s); + tor_free(s); + + /* Now clear out all the AB observations. */ + geoip_remove_old_clients(now-6000); + s = geoip_get_client_history_bridge(now+5*24*60*60, + GEOIP_CLIENT_CONNECT); + test_assert(s); + test_streq("zz=24,xy=8", s); + + done: + tor_free(s); +} + +static void * +legacy_test_setup(const struct testcase_t *testcase) +{ + return testcase->setup_data; +} + +void +legacy_test_helper(void *data) +{ + void (*fn)(void) = data; + fn(); +} + +static int +legacy_test_cleanup(const struct testcase_t *testcase, void *ptr) +{ + (void)ptr; + (void)testcase; + return 1; +} + +const struct testcase_setup_t legacy_setup = { + legacy_test_setup, legacy_test_cleanup +}; + +#define ENT(name) \ + { #name, legacy_test_helper, 0, &legacy_setup, test_ ## name } +#define SUBENT(group, name) \ + { #group "_" #name, legacy_test_helper, 0, &legacy_setup, \ + test_ ## group ## _ ## name } +#define DISABLED(name) \ + { #name, legacy_test_helper, TT_SKIP, &legacy_setup, name } + +static struct testcase_t test_array[] = { + ENT(buffers), + ENT(onion_handshake), + ENT(circuit_timeout), + ENT(policies), + ENT(rend_fns), + ENT(geoip), + + DISABLED(bench_aes), + DISABLED(bench_dmap), + END_OF_TESTCASES +}; + +extern struct testcase_t addr_tests[]; +extern struct testcase_t crypto_tests[]; +extern struct testcase_t container_tests[]; +extern struct testcase_t util_tests[]; +extern struct testcase_t dir_tests[]; + +static struct testgroup_t testgroups[] = { + { "", test_array }, + { "addr/", addr_tests }, + { "crypto/", crypto_tests }, + { "container/", container_tests }, + { "util/", util_tests }, + { "dir/", dir_tests }, + END_OF_GROUPS +}; + +/** Main entry point for unit test code: parse the command line, and run + * some unit tests. */ +int +main(int c, const char **v) +{ + or_options_t *options; + char *errmsg = NULL; + int i, i_out; + int loglevel = LOG_ERR; + +#ifdef USE_DMALLOC + { + int r = CRYPTO_set_mem_ex_functions(_tor_malloc, _tor_realloc, _tor_free); + tor_assert(r); + } +#endif + + update_approx_time(time(NULL)); + options = options_new(); + tor_threads_init(); + init_logging(); + + for (i_out = i = 1; i < c; ++i) { + if (!strcmp(v[i], "--warn")) { + loglevel = LOG_WARN; + } else if (!strcmp(v[i], "--notice")) { + loglevel = LOG_NOTICE; + } else if (!strcmp(v[i], "--info")) { + loglevel = LOG_INFO; + } else if (!strcmp(v[i], "--debug")) { + loglevel = LOG_DEBUG; + } else { + v[i_out++] = v[i]; + } + } + c = i_out; + + { + log_severity_list_t s; + memset(&s, 0, sizeof(s)); + set_log_severity_config(loglevel, LOG_ERR, &s); + add_stream_log(&s, "", fileno(stdout)); + } + + options->command = CMD_RUN_UNITTESTS; + crypto_global_init(0, NULL, NULL); + rep_hist_init(); + network_init(); + setup_directory(); + options_init(options); + options->DataDirectory = tor_strdup(temp_dir); + options->EntryStatistics = 1; + if (set_options(options, &errmsg) < 0) { + printf("Failed to set initial options: %s\n", errmsg); + tor_free(errmsg); + return 1; + } + + crypto_seed_rng(1); + + atexit(remove_directory); + + have_failed = (tinytest_main(c, v, testgroups) != 0); + + free_pregenerated_keys(); +#ifdef USE_DMALLOC + tor_free_all(0); + dmalloc_log_unfreed(); +#endif + + if (have_failed) + return 1; + else + return 0; +} + diff --git a/src/test/test.h b/src/test/test.h new file mode 100644 index 0000000000..ed0eb316ad --- /dev/null +++ b/src/test/test.h @@ -0,0 +1,75 @@ +/* Copyright (c) 2001-2003, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2009, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef _TOR_TEST_H +#define _TOR_TEST_H + +/** + * \file test.h + * \brief Macros and functions used by unit tests. + */ + +#include "compat.h" +#include "tinytest.h" +#define TT_EXIT_TEST_FUNCTION STMT_BEGIN goto done; STMT_END +#include "tinytest_macros.h" + +#ifdef __GNUC__ +#define PRETTY_FUNCTION __PRETTY_FUNCTION__ +#else +#define PRETTY_FUNCTION "" +#endif + +#define test_fail_msg(msg) TT_DIE((msg)) + +#define test_fail() test_fail_msg("Assertion failed.") + +#define test_assert(expr) tt_assert(expr) + +#define test_eq(expr1, expr2) tt_int_op((expr1), ==, (expr2)) +#define test_eq_ptr(expr1, expr2) tt_ptr_op((expr1), ==, (expr2)) +#define test_neq(expr1, expr2) tt_int_op((expr1), !=, (expr2)) +#define test_neq_ptr(expr1, expr2) tt_ptr_op((expr1), !=, (expr2)) +#define test_streq(expr1, expr2) tt_str_op((expr1), ==, (expr2)) +#define test_strneq(expr1, expr2) tt_str_op((expr1), !=, (expr2)) +#define test_streq(expr1, expr2) tt_str_op((expr1), ==, (expr2)) + +#define test_mem_op(expr1, op, expr2, len) \ + tt_assert_test_fmt_type(expr1,expr2,#expr1" "#op" "#expr2, \ + const char *, \ + (memcmp(_val1, _val2, len) op 0), \ + char *, "%s", \ + { size_t printlen = (len)*2+1; \ + _print = tor_malloc(printlen); \ + base16_encode(_print, printlen, _value, \ + (len)); }, \ + { tor_free(_print); } \ + ); + +#define test_memeq(expr1, expr2, len) test_mem_op((expr1), ==, (expr2), len) +#define test_memneq(expr1, expr2, len) test_mem_op((expr1), !=, (expr2), len) + +/* As test_mem_op, but decodes 'hex' before comparing. There must be a + * local char* variable called mem_op_hex_tmp for this to work. */ +#define test_mem_op_hex(expr1, op, hex) \ + STMT_BEGIN \ + size_t length = strlen(hex); \ + tor_free(mem_op_hex_tmp); \ + mem_op_hex_tmp = tor_malloc(length/2); \ + tor_assert((length&1)==0); \ + base16_decode(mem_op_hex_tmp, length/2, hex, length); \ + test_mem_op(expr1, op, mem_op_hex_tmp, length/2); \ + STMT_END + +#define test_memeq_hex(expr1, hex) test_mem_op_hex(expr1, ==, hex) + +const char *get_fname(const char *name); +crypto_pk_env_t *pk_generate(int idx); + +void legacy_test_helper(void *data); +extern const struct testcase_setup_t legacy_setup; + +#endif + diff --git a/src/test/test_addr.c b/src/test/test_addr.c new file mode 100644 index 0000000000..327fc651f6 --- /dev/null +++ b/src/test/test_addr.c @@ -0,0 +1,497 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2009, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#include "or.h" +#include "test.h" + +static void +test_addr_basic(void) +{ + uint32_t u32; + uint16_t u16; + char *cp; + + /* Test parse_addr_port */ + cp = NULL; u32 = 3; u16 = 3; + test_assert(!parse_addr_port(LOG_WARN, "1.2.3.4", &cp, &u32, &u16)); + test_streq(cp, "1.2.3.4"); + test_eq(u32, 0x01020304u); + test_eq(u16, 0); + tor_free(cp); + test_assert(!parse_addr_port(LOG_WARN, "4.3.2.1:99", &cp, &u32, &u16)); + test_streq(cp, "4.3.2.1"); + test_eq(u32, 0x04030201u); + test_eq(u16, 99); + tor_free(cp); + test_assert(!parse_addr_port(LOG_WARN, "nonexistent.address:4040", + &cp, NULL, &u16)); + test_streq(cp, "nonexistent.address"); + test_eq(u16, 4040); + tor_free(cp); + test_assert(!parse_addr_port(LOG_WARN, "localhost:9999", &cp, &u32, &u16)); + test_streq(cp, "localhost"); + test_eq(u32, 0x7f000001u); + test_eq(u16, 9999); + tor_free(cp); + u32 = 3; + test_assert(!parse_addr_port(LOG_WARN, "localhost", NULL, &u32, &u16)); + test_eq(cp, NULL); + test_eq(u32, 0x7f000001u); + test_eq(u16, 0); + tor_free(cp); + test_eq(0, addr_mask_get_bits(0x0u)); + test_eq(32, addr_mask_get_bits(0xFFFFFFFFu)); + test_eq(16, addr_mask_get_bits(0xFFFF0000u)); + test_eq(31, addr_mask_get_bits(0xFFFFFFFEu)); + test_eq(1, addr_mask_get_bits(0x80000000u)); + + /* Test inet_ntop */ + { + char tmpbuf[TOR_ADDR_BUF_LEN]; + const char *ip = "176.192.208.224"; + struct in_addr in; + tor_inet_pton(AF_INET, ip, &in); + tor_inet_ntop(AF_INET, &in, tmpbuf, sizeof(tmpbuf)); + test_streq(tmpbuf, ip); + } + + done: + ; +} + +#define _test_op_ip6(a,op,b,e1,e2) \ + STMT_BEGIN \ + tt_assert_test_fmt_type(a,b,e1" "#op" "e2,struct in6_addr*, \ + (memcmp(_val1->s6_addr, _val2->s6_addr, 16) op 0), \ + char *, "%s", \ + { int i; char *cp; \ + cp = _print = tor_malloc(64); \ + for (i=0;i<16;++i) { \ + tor_snprintf(cp, 3,"%02x", (unsigned)_value->s6_addr[i]);\ + cp += 2; \ + if (i != 15) *cp++ = ':'; \ + } \ + }, { tor_free(_print); } \ + ); \ + STMT_END + +/** Helper: Assert that two strings both decode as IPv6 addresses with + * tor_inet_pton(), and both decode to the same address. */ +#define test_pton6_same(a,b) STMT_BEGIN \ + test_eq(tor_inet_pton(AF_INET6, a, &a1), 1); \ + test_eq(tor_inet_pton(AF_INET6, b, &a2), 1); \ + _test_op_ip6(&a1,==,&a2,#a,#b); \ + STMT_END + +/** Helper: Assert that <b>a</b> is recognized as a bad IPv6 address by + * tor_inet_pton(). */ +#define test_pton6_bad(a) \ + test_eq(0, tor_inet_pton(AF_INET6, a, &a1)) + +/** Helper: assert that <b>a</b>, when parsed by tor_inet_pton() and displayed + * with tor_inet_ntop(), yields <b>b</b>. Also assert that <b>b</b> parses to + * the same value as <b>a</b>. */ +#define test_ntop6_reduces(a,b) STMT_BEGIN \ + test_eq(tor_inet_pton(AF_INET6, a, &a1), 1); \ + test_streq(tor_inet_ntop(AF_INET6, &a1, buf, sizeof(buf)), b); \ + test_eq(tor_inet_pton(AF_INET6, b, &a2), 1); \ + _test_op_ip6(&a1, ==, &a2, a, b); \ + STMT_END + +/** Helper: assert that <b>a</b> parses by tor_inet_pton() into a address that + * passes tor_addr_is_internal() with <b>for_listening</b>. */ +#define test_internal_ip(a,for_listening) STMT_BEGIN \ + test_eq(tor_inet_pton(AF_INET6, a, &t1.addr.in6_addr), 1); \ + t1.family = AF_INET6; \ + if (!tor_addr_is_internal(&t1, for_listening)) \ + test_fail_msg( a "was not internal."); \ + STMT_END + +/** Helper: assert that <b>a</b> parses by tor_inet_pton() into a address that + * does not pass tor_addr_is_internal() with <b>for_listening</b>. */ +#define test_external_ip(a,for_listening) STMT_BEGIN \ + test_eq(tor_inet_pton(AF_INET6, a, &t1.addr.in6_addr), 1); \ + t1.family = AF_INET6; \ + if (tor_addr_is_internal(&t1, for_listening)) \ + test_fail_msg(a "was not external."); \ + STMT_END + +/** Helper: Assert that <b>a</b> and <b>b</b>, when parsed by + * tor_inet_pton(), give addresses that compare in the order defined by + * <b>op</b> with tor_addr_compare(). */ +#define test_addr_compare(a, op, b) STMT_BEGIN \ + test_eq(tor_inet_pton(AF_INET6, a, &t1.addr.in6_addr), 1); \ + test_eq(tor_inet_pton(AF_INET6, b, &t2.addr.in6_addr), 1); \ + t1.family = t2.family = AF_INET6; \ + r = tor_addr_compare(&t1,&t2,CMP_SEMANTIC); \ + if (!(r op 0)) \ + test_fail_msg("failed: tor_addr_compare("a","b") "#op" 0"); \ + STMT_END + +/** Helper: Assert that <b>a</b> and <b>b</b>, when parsed by + * tor_inet_pton(), give addresses that compare in the order defined by + * <b>op</b> with tor_addr_compare_masked() with <b>m</b> masked. */ +#define test_addr_compare_masked(a, op, b, m) STMT_BEGIN \ + test_eq(tor_inet_pton(AF_INET6, a, &t1.addr.in6_addr), 1); \ + test_eq(tor_inet_pton(AF_INET6, b, &t2.addr.in6_addr), 1); \ + t1.family = t2.family = AF_INET6; \ + r = tor_addr_compare_masked(&t1,&t2,m,CMP_SEMANTIC); \ + if (!(r op 0)) \ + test_fail_msg("failed: tor_addr_compare_masked("a","b","#m") "#op" 0"); \ + STMT_END + +/** Helper: assert that <b>xx</b> is parseable as a masked IPv6 address with + * ports by tor_parse_mask_addr_ports(), with family <b>f</b>, IP address + * as 4 32-bit words <b>ip1...ip4</b>, mask bits as <b>mm</b>, and port range + * as <b>pt1..pt2</b>. */ +#define test_addr_mask_ports_parse(xx, f, ip1, ip2, ip3, ip4, mm, pt1, pt2) \ + STMT_BEGIN \ + test_eq(tor_addr_parse_mask_ports(xx, &t1, &mask, &port1, &port2), f); \ + p1=tor_inet_ntop(AF_INET6, &t1.addr.in6_addr, bug, sizeof(bug)); \ + test_eq(htonl(ip1), tor_addr_to_in6_addr32(&t1)[0]); \ + test_eq(htonl(ip2), tor_addr_to_in6_addr32(&t1)[1]); \ + test_eq(htonl(ip3), tor_addr_to_in6_addr32(&t1)[2]); \ + test_eq(htonl(ip4), tor_addr_to_in6_addr32(&t1)[3]); \ + test_eq(mask, mm); \ + test_eq(port1, pt1); \ + test_eq(port2, pt2); \ + STMT_END + +/** Run unit tests for IPv6 encoding/decoding/manipulation functions. */ +static void +test_addr_ip6_helpers(void) +{ + char buf[TOR_ADDR_BUF_LEN], bug[TOR_ADDR_BUF_LEN]; + struct in6_addr a1, a2; + tor_addr_t t1, t2; + int r, i; + uint16_t port1, port2; + maskbits_t mask; + const char *p1; + struct sockaddr_storage sa_storage; + struct sockaddr_in *sin; + struct sockaddr_in6 *sin6; + + // struct in_addr b1, b2; + /* Test tor_inet_ntop and tor_inet_pton: IPv6 */ + + /* ==== Converting to and from sockaddr_t. */ + sin = (struct sockaddr_in *)&sa_storage; + sin->sin_family = AF_INET; + sin->sin_port = 9090; + sin->sin_addr.s_addr = htonl(0x7f7f0102); /*127.127.1.2*/ + tor_addr_from_sockaddr(&t1, (struct sockaddr *)sin, NULL); + test_eq(tor_addr_family(&t1), AF_INET); + test_eq(tor_addr_to_ipv4h(&t1), 0x7f7f0102); + + memset(&sa_storage, 0, sizeof(sa_storage)); + test_eq(sizeof(struct sockaddr_in), + tor_addr_to_sockaddr(&t1, 1234, (struct sockaddr *)&sa_storage, + sizeof(sa_storage))); + test_eq(1234, ntohs(sin->sin_port)); + test_eq(0x7f7f0102, ntohl(sin->sin_addr.s_addr)); + + memset(&sa_storage, 0, sizeof(sa_storage)); + sin6 = (struct sockaddr_in6 *)&sa_storage; + sin6->sin6_family = AF_INET6; + sin6->sin6_port = htons(7070); + sin6->sin6_addr.s6_addr[0] = 128; + tor_addr_from_sockaddr(&t1, (struct sockaddr *)sin6, NULL); + test_eq(tor_addr_family(&t1), AF_INET6); + p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 0); + test_streq(p1, "8000::"); + + memset(&sa_storage, 0, sizeof(sa_storage)); + test_eq(sizeof(struct sockaddr_in6), + tor_addr_to_sockaddr(&t1, 9999, (struct sockaddr *)&sa_storage, + sizeof(sa_storage))); + test_eq(AF_INET6, sin6->sin6_family); + test_eq(9999, ntohs(sin6->sin6_port)); + test_eq(0x80000000, ntohl(S6_ADDR32(sin6->sin6_addr)[0])); + + /* ==== tor_addr_lookup: static cases. (Can't test dns without knowing we + * have a good resolver. */ + test_eq(0, tor_addr_lookup("127.128.129.130", AF_UNSPEC, &t1)); + test_eq(AF_INET, tor_addr_family(&t1)); + test_eq(tor_addr_to_ipv4h(&t1), 0x7f808182); + + test_eq(0, tor_addr_lookup("9000::5", AF_UNSPEC, &t1)); + test_eq(AF_INET6, tor_addr_family(&t1)); + test_eq(0x90, tor_addr_to_in6_addr8(&t1)[0]); + test_assert(tor_mem_is_zero((char*)tor_addr_to_in6_addr8(&t1)+1, 14)); + test_eq(0x05, tor_addr_to_in6_addr8(&t1)[15]); + + /* === Test pton: valid af_inet6 */ + /* Simple, valid parsing. */ + r = tor_inet_pton(AF_INET6, + "0102:0304:0506:0708:090A:0B0C:0D0E:0F10", &a1); + test_assert(r==1); + for (i=0;i<16;++i) { test_eq(i+1, (int)a1.s6_addr[i]); } + /* ipv4 ending. */ + test_pton6_same("0102:0304:0506:0708:090A:0B0C:0D0E:0F10", + "0102:0304:0506:0708:090A:0B0C:13.14.15.16"); + /* shortened words. */ + test_pton6_same("0001:0099:BEEF:0000:0123:FFFF:0001:0001", + "1:99:BEEF:0:0123:FFFF:1:1"); + /* zeros at the beginning */ + test_pton6_same("0000:0000:0000:0000:0009:C0A8:0001:0001", + "::9:c0a8:1:1"); + test_pton6_same("0000:0000:0000:0000:0009:C0A8:0001:0001", + "::9:c0a8:0.1.0.1"); + /* zeros in the middle. */ + test_pton6_same("fe80:0000:0000:0000:0202:1111:0001:0001", + "fe80::202:1111:1:1"); + /* zeros at the end. */ + test_pton6_same("1000:0001:0000:0007:0000:0000:0000:0000", + "1000:1:0:7::"); + + /* === Test ntop: af_inet6 */ + test_ntop6_reduces("0:0:0:0:0:0:0:0", "::"); + + test_ntop6_reduces("0001:0099:BEEF:0006:0123:FFFF:0001:0001", + "1:99:beef:6:123:ffff:1:1"); + + //test_ntop6_reduces("0:0:0:0:0:0:c0a8:0101", "::192.168.1.1"); + test_ntop6_reduces("0:0:0:0:0:ffff:c0a8:0101", "::ffff:192.168.1.1"); + test_ntop6_reduces("002:0:0000:0:3::4", "2::3:0:0:4"); + test_ntop6_reduces("0:0::1:0:3", "::1:0:3"); + test_ntop6_reduces("008:0::0", "8::"); + test_ntop6_reduces("0:0:0:0:0:ffff::1", "::ffff:0.0.0.1"); + test_ntop6_reduces("abcd:0:0:0:0:0:7f00::", "abcd::7f00:0"); + test_ntop6_reduces("0000:0000:0000:0000:0009:C0A8:0001:0001", + "::9:c0a8:1:1"); + test_ntop6_reduces("fe80:0000:0000:0000:0202:1111:0001:0001", + "fe80::202:1111:1:1"); + test_ntop6_reduces("1000:0001:0000:0007:0000:0000:0000:0000", + "1000:1:0:7::"); + + /* === Test pton: invalid in6. */ + test_pton6_bad("foobar."); + test_pton6_bad("55555::"); + test_pton6_bad("9:-60::"); + test_pton6_bad("1:2:33333:4:0002:3::"); + //test_pton6_bad("1:2:3333:4:00002:3::");// BAD, but glibc doesn't say so. + test_pton6_bad("1:2:3333:4:fish:3::"); + test_pton6_bad("1:2:3:4:5:6:7:8:9"); + test_pton6_bad("1:2:3:4:5:6:7"); + test_pton6_bad("1:2:3:4:5:6:1.2.3.4.5"); + test_pton6_bad("1:2:3:4:5:6:1.2.3"); + test_pton6_bad("::1.2.3"); + test_pton6_bad("::1.2.3.4.5"); + test_pton6_bad("99"); + test_pton6_bad(""); + test_pton6_bad("1::2::3:4"); + test_pton6_bad("a:::b:c"); + test_pton6_bad(":::a:b:c"); + test_pton6_bad("a:b:c:::"); + + /* test internal checking */ + test_external_ip("fbff:ffff::2:7", 0); + test_internal_ip("fc01::2:7", 0); + test_internal_ip("fdff:ffff::f:f", 0); + test_external_ip("fe00::3:f", 0); + + test_external_ip("fe7f:ffff::2:7", 0); + test_internal_ip("fe80::2:7", 0); + test_internal_ip("febf:ffff::f:f", 0); + + test_internal_ip("fec0::2:7:7", 0); + test_internal_ip("feff:ffff::e:7:7", 0); + test_external_ip("ff00::e:7:7", 0); + + test_internal_ip("::", 0); + test_internal_ip("::1", 0); + test_internal_ip("::1", 1); + test_internal_ip("::", 0); + test_external_ip("::", 1); + test_external_ip("::2", 0); + test_external_ip("2001::", 0); + test_external_ip("ffff::", 0); + + test_external_ip("::ffff:0.0.0.0", 1); + test_internal_ip("::ffff:0.0.0.0", 0); + test_internal_ip("::ffff:0.255.255.255", 0); + test_external_ip("::ffff:1.0.0.0", 0); + + test_external_ip("::ffff:9.255.255.255", 0); + test_internal_ip("::ffff:10.0.0.0", 0); + test_internal_ip("::ffff:10.255.255.255", 0); + test_external_ip("::ffff:11.0.0.0", 0); + + test_external_ip("::ffff:126.255.255.255", 0); + test_internal_ip("::ffff:127.0.0.0", 0); + test_internal_ip("::ffff:127.255.255.255", 0); + test_external_ip("::ffff:128.0.0.0", 0); + + test_external_ip("::ffff:172.15.255.255", 0); + test_internal_ip("::ffff:172.16.0.0", 0); + test_internal_ip("::ffff:172.31.255.255", 0); + test_external_ip("::ffff:172.32.0.0", 0); + + test_external_ip("::ffff:192.167.255.255", 0); + test_internal_ip("::ffff:192.168.0.0", 0); + test_internal_ip("::ffff:192.168.255.255", 0); + test_external_ip("::ffff:192.169.0.0", 0); + + test_external_ip("::ffff:169.253.255.255", 0); + test_internal_ip("::ffff:169.254.0.0", 0); + test_internal_ip("::ffff:169.254.255.255", 0); + test_external_ip("::ffff:169.255.0.0", 0); + test_assert(is_internal_IP(0x7f000001, 0)); + + /* tor_addr_compare(tor_addr_t x2) */ + test_addr_compare("ffff::", ==, "ffff::0"); + test_addr_compare("0::3:2:1", <, "0::ffff:0.3.2.1"); + test_addr_compare("0::2:2:1", <, "0::ffff:0.3.2.1"); + test_addr_compare("0::ffff:0.3.2.1", >, "0::0:0:0"); + test_addr_compare("0::ffff:5.2.2.1", <, "::ffff:6.0.0.0"); /* XXXX wrong. */ + tor_addr_parse_mask_ports("[::ffff:2.3.4.5]", &t1, NULL, NULL, NULL); + tor_addr_parse_mask_ports("2.3.4.5", &t2, NULL, NULL, NULL); + test_assert(tor_addr_compare(&t1, &t2, CMP_SEMANTIC) == 0); + tor_addr_parse_mask_ports("[::ffff:2.3.4.4]", &t1, NULL, NULL, NULL); + tor_addr_parse_mask_ports("2.3.4.5", &t2, NULL, NULL, NULL); + test_assert(tor_addr_compare(&t1, &t2, CMP_SEMANTIC) < 0); + + /* test compare_masked */ + test_addr_compare_masked("ffff::", ==, "ffff::0", 128); + test_addr_compare_masked("ffff::", ==, "ffff::0", 64); + test_addr_compare_masked("0::2:2:1", <, "0::8000:2:1", 81); + test_addr_compare_masked("0::2:2:1", ==, "0::8000:2:1", 80); + + /* Test decorated addr_to_string. */ + test_eq(AF_INET6, tor_addr_from_str(&t1, "[123:45:6789::5005:11]")); + p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 1); + test_streq(p1, "[123:45:6789::5005:11]"); + test_eq(AF_INET, tor_addr_from_str(&t1, "18.0.0.1")); + p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 1); + test_streq(p1, "18.0.0.1"); + + /* Test tor_addr_parse_reverse_lookup_name */ + i = tor_addr_parse_reverse_lookup_name(&t1, "Foobar.baz", AF_UNSPEC, 0); + test_eq(0, i); + i = tor_addr_parse_reverse_lookup_name(&t1, "Foobar.baz", AF_UNSPEC, 1); + test_eq(0, i); + i = tor_addr_parse_reverse_lookup_name(&t1, "1.0.168.192.in-addr.arpa", + AF_UNSPEC, 1); + test_eq(1, i); + test_eq(tor_addr_family(&t1), AF_INET); + p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 1); + test_streq(p1, "192.168.0.1"); + i = tor_addr_parse_reverse_lookup_name(&t1, "192.168.0.99", AF_UNSPEC, 0); + test_eq(0, i); + i = tor_addr_parse_reverse_lookup_name(&t1, "192.168.0.99", AF_UNSPEC, 1); + test_eq(1, i); + p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 1); + test_streq(p1, "192.168.0.99"); + memset(&t1, 0, sizeof(t1)); + i = tor_addr_parse_reverse_lookup_name(&t1, + "0.1.2.3.4.5.6.7.8.9.a.b.c.d.e.f." + "f.e.e.b.1.e.b.e.e.f.f.e.e.e.d.9." + "ip6.ARPA", + AF_UNSPEC, 0); + test_eq(1, i); + p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 1); + test_streq(p1, "[9dee:effe:ebe1:beef:fedc:ba98:7654:3210]"); + /* Failing cases. */ + i = tor_addr_parse_reverse_lookup_name(&t1, + "6.7.8.9.a.b.c.d.e.f." + "f.e.e.b.1.e.b.e.e.f.f.e.e.e.d.9." + "ip6.ARPA", + AF_UNSPEC, 0); + test_eq(i, -1); + i = tor_addr_parse_reverse_lookup_name(&t1, + "6.7.8.9.a.b.c.d.e.f.a.b.c.d.e.f.0." + "f.e.e.b.1.e.b.e.e.f.f.e.e.e.d.9." + "ip6.ARPA", + AF_UNSPEC, 0); + test_eq(i, -1); + i = tor_addr_parse_reverse_lookup_name(&t1, + "6.7.8.9.a.b.c.d.e.f.X.0.0.0.0.9." + "f.e.e.b.1.e.b.e.e.f.f.e.e.e.d.9." + "ip6.ARPA", + AF_UNSPEC, 0); + test_eq(i, -1); + i = tor_addr_parse_reverse_lookup_name(&t1, "32.1.1.in-addr.arpa", + AF_UNSPEC, 0); + test_eq(i, -1); + i = tor_addr_parse_reverse_lookup_name(&t1, ".in-addr.arpa", + AF_UNSPEC, 0); + test_eq(i, -1); + i = tor_addr_parse_reverse_lookup_name(&t1, "1.2.3.4.5.in-addr.arpa", + AF_UNSPEC, 0); + test_eq(i, -1); + i = tor_addr_parse_reverse_lookup_name(&t1, "1.2.3.4.5.in-addr.arpa", + AF_INET6, 0); + test_eq(i, -1); + i = tor_addr_parse_reverse_lookup_name(&t1, + "6.7.8.9.a.b.c.d.e.f.a.b.c.d.e.0." + "f.e.e.b.1.e.b.e.e.f.f.e.e.e.d.9." + "ip6.ARPA", + AF_INET, 0); + test_eq(i, -1); + + /* test tor_addr_parse_mask_ports */ + test_addr_mask_ports_parse("[::f]/17:47-95", AF_INET6, + 0, 0, 0, 0x0000000f, 17, 47, 95); + //test_addr_parse("[::fefe:4.1.1.7/120]:999-1000"); + //test_addr_parse_check("::fefe:401:107", 120, 999, 1000); + test_addr_mask_ports_parse("[::ffff:4.1.1.7]/120:443", AF_INET6, + 0, 0, 0x0000ffff, 0x04010107, 120, 443, 443); + test_addr_mask_ports_parse("[abcd:2::44a:0]:2-65000", AF_INET6, + 0xabcd0002, 0, 0, 0x044a0000, 128, 2, 65000); + + r=tor_addr_parse_mask_ports("[fefef::]/112", &t1, NULL, NULL, NULL); + test_assert(r == -1); + r=tor_addr_parse_mask_ports("efef::/112", &t1, NULL, NULL, NULL); + test_assert(r == -1); + r=tor_addr_parse_mask_ports("[f:f:f:f:f:f:f:f::]", &t1, NULL, NULL, NULL); + test_assert(r == -1); + r=tor_addr_parse_mask_ports("[::f:f:f:f:f:f:f:f]", &t1, NULL, NULL, NULL); + test_assert(r == -1); + r=tor_addr_parse_mask_ports("[f:f:f:f:f:f:f:f:f]", &t1, NULL, NULL, NULL); + test_assert(r == -1); + /* Test for V4-mapped address with mask < 96. (arguably not valid) */ + r=tor_addr_parse_mask_ports("[::ffff:1.1.2.2/33]", &t1, &mask, NULL, NULL); + test_assert(r == -1); + r=tor_addr_parse_mask_ports("1.1.2.2/33", &t1, &mask, NULL, NULL); + test_assert(r == -1); + r=tor_addr_parse_mask_ports("1.1.2.2/31", &t1, &mask, NULL, NULL); + test_assert(r == AF_INET); + r=tor_addr_parse_mask_ports("[efef::]/112", &t1, &mask, &port1, &port2); + test_assert(r == AF_INET6); + test_assert(port1 == 1); + test_assert(port2 == 65535); + + /* make sure inet address lengths >= max */ + test_assert(INET_NTOA_BUF_LEN >= sizeof("255.255.255.255")); + test_assert(TOR_ADDR_BUF_LEN >= + sizeof("ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255")); + + test_assert(sizeof(tor_addr_t) >= sizeof(struct in6_addr)); + + /* get interface addresses */ + r = get_interface_address6(LOG_DEBUG, AF_INET, &t1); + i = get_interface_address6(LOG_DEBUG, AF_INET6, &t2); +#if 0 + tor_inet_ntop(AF_INET, &t1.sa.sin_addr, buf, sizeof(buf)); + printf("\nv4 address: %s (family=%i)", buf, IN_FAMILY(&t1)); + tor_inet_ntop(AF_INET6, &t2.sa6.sin6_addr, buf, sizeof(buf)); + printf("\nv6 address: %s (family=%i)", buf, IN_FAMILY(&t2)); +#endif + + done: + ; +} + +#define ADDR_LEGACY(name) \ + { #name, legacy_test_helper, 0, &legacy_setup, test_addr_ ## name } + +struct testcase_t addr_tests[] = { + ADDR_LEGACY(basic), + ADDR_LEGACY(ip6_helpers), + END_OF_TESTCASES +}; + diff --git a/src/test/test_containers.c b/src/test/test_containers.c new file mode 100644 index 0000000000..efc879b8c2 --- /dev/null +++ b/src/test/test_containers.c @@ -0,0 +1,765 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2009, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#include "or.h" +#include "test.h" + +/** Helper: return a tristate based on comparing the strings in *<b>a</b> and + * *<b>b</b>. */ +static int +_compare_strs(const void **a, const void **b) +{ + const char *s1 = *a, *s2 = *b; + return strcmp(s1, s2); +} + +/** Helper: return a tristate based on comparing the strings in *<b>a</b> and + * *<b>b</b>, excluding a's first character, and ignoring case. */ +static int +_compare_without_first_ch(const void *a, const void **b) +{ + const char *s1 = a, *s2 = *b; + return strcasecmp(s1+1, s2); +} + +/** Run unit tests for basic dynamic-sized array functionality. */ +static void +test_container_smartlist_basic(void) +{ + smartlist_t *sl; + + /* XXXX test sort_digests, uniq_strings, uniq_digests */ + + /* Test smartlist add, del_keeporder, insert, get. */ + sl = smartlist_create(); + smartlist_add(sl, (void*)1); + smartlist_add(sl, (void*)2); + smartlist_add(sl, (void*)3); + smartlist_add(sl, (void*)4); + smartlist_del_keeporder(sl, 1); + smartlist_insert(sl, 1, (void*)22); + smartlist_insert(sl, 0, (void*)0); + smartlist_insert(sl, 5, (void*)555); + test_eq_ptr((void*)0, smartlist_get(sl,0)); + test_eq_ptr((void*)1, smartlist_get(sl,1)); + test_eq_ptr((void*)22, smartlist_get(sl,2)); + test_eq_ptr((void*)3, smartlist_get(sl,3)); + test_eq_ptr((void*)4, smartlist_get(sl,4)); + test_eq_ptr((void*)555, smartlist_get(sl,5)); + /* Try deleting in the middle. */ + smartlist_del(sl, 1); + test_eq_ptr((void*)555, smartlist_get(sl, 1)); + /* Try deleting at the end. */ + smartlist_del(sl, 4); + test_eq(4, smartlist_len(sl)); + + /* test isin. */ + test_assert(smartlist_isin(sl, (void*)3)); + test_assert(!smartlist_isin(sl, (void*)99)); + + done: + smartlist_free(sl); +} + +/** Run unit tests for smartlist-of-strings functionality. */ +static void +test_container_smartlist_strings(void) +{ + smartlist_t *sl = smartlist_create(); + char *cp=NULL, *cp_alloc=NULL; + size_t sz; + + /* Test split and join */ + test_eq(0, smartlist_len(sl)); + smartlist_split_string(sl, "abc", ":", 0, 0); + test_eq(1, smartlist_len(sl)); + test_streq("abc", smartlist_get(sl, 0)); + smartlist_split_string(sl, "a::bc::", "::", 0, 0); + test_eq(4, smartlist_len(sl)); + test_streq("a", smartlist_get(sl, 1)); + test_streq("bc", smartlist_get(sl, 2)); + test_streq("", smartlist_get(sl, 3)); + cp_alloc = smartlist_join_strings(sl, "", 0, NULL); + test_streq(cp_alloc, "abcabc"); + tor_free(cp_alloc); + cp_alloc = smartlist_join_strings(sl, "!", 0, NULL); + test_streq(cp_alloc, "abc!a!bc!"); + tor_free(cp_alloc); + cp_alloc = smartlist_join_strings(sl, "XY", 0, NULL); + test_streq(cp_alloc, "abcXYaXYbcXY"); + tor_free(cp_alloc); + cp_alloc = smartlist_join_strings(sl, "XY", 1, NULL); + test_streq(cp_alloc, "abcXYaXYbcXYXY"); + tor_free(cp_alloc); + cp_alloc = smartlist_join_strings(sl, "", 1, NULL); + test_streq(cp_alloc, "abcabc"); + tor_free(cp_alloc); + + smartlist_split_string(sl, "/def/ /ghijk", "/", 0, 0); + test_eq(8, smartlist_len(sl)); + test_streq("", smartlist_get(sl, 4)); + test_streq("def", smartlist_get(sl, 5)); + test_streq(" ", smartlist_get(sl, 6)); + test_streq("ghijk", smartlist_get(sl, 7)); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + smartlist_split_string(sl, "a,bbd,cdef", ",", SPLIT_SKIP_SPACE, 0); + test_eq(3, smartlist_len(sl)); + test_streq("a", smartlist_get(sl,0)); + test_streq("bbd", smartlist_get(sl,1)); + test_streq("cdef", smartlist_get(sl,2)); + smartlist_split_string(sl, " z <> zhasd <> <> bnud<> ", "<>", + SPLIT_SKIP_SPACE, 0); + test_eq(8, smartlist_len(sl)); + test_streq("z", smartlist_get(sl,3)); + test_streq("zhasd", smartlist_get(sl,4)); + test_streq("", smartlist_get(sl,5)); + test_streq("bnud", smartlist_get(sl,6)); + test_streq("", smartlist_get(sl,7)); + + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + smartlist_split_string(sl, " ab\tc \td ef ", NULL, + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + test_eq(4, smartlist_len(sl)); + test_streq("ab", smartlist_get(sl,0)); + test_streq("c", smartlist_get(sl,1)); + test_streq("d", smartlist_get(sl,2)); + test_streq("ef", smartlist_get(sl,3)); + smartlist_split_string(sl, "ghi\tj", NULL, + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + test_eq(6, smartlist_len(sl)); + test_streq("ghi", smartlist_get(sl,4)); + test_streq("j", smartlist_get(sl,5)); + + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + cp_alloc = smartlist_join_strings(sl, "XY", 0, NULL); + test_streq(cp_alloc, ""); + tor_free(cp_alloc); + cp_alloc = smartlist_join_strings(sl, "XY", 1, NULL); + test_streq(cp_alloc, "XY"); + tor_free(cp_alloc); + + smartlist_split_string(sl, " z <> zhasd <> <> bnud<> ", "<>", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + test_eq(3, smartlist_len(sl)); + test_streq("z", smartlist_get(sl, 0)); + test_streq("zhasd", smartlist_get(sl, 1)); + test_streq("bnud", smartlist_get(sl, 2)); + smartlist_split_string(sl, " z <> zhasd <> <> bnud<> ", "<>", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 2); + test_eq(5, smartlist_len(sl)); + test_streq("z", smartlist_get(sl, 3)); + test_streq("zhasd <> <> bnud<>", smartlist_get(sl, 4)); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + smartlist_split_string(sl, "abcd\n", "\n", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + test_eq(1, smartlist_len(sl)); + test_streq("abcd", smartlist_get(sl, 0)); + smartlist_split_string(sl, "efgh", "\n", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + test_eq(2, smartlist_len(sl)); + test_streq("efgh", smartlist_get(sl, 1)); + + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + /* Test swapping, shuffling, and sorting. */ + smartlist_split_string(sl, "the,onion,router,by,arma,and,nickm", ",", 0, 0); + test_eq(7, smartlist_len(sl)); + smartlist_sort(sl, _compare_strs); + cp_alloc = smartlist_join_strings(sl, ",", 0, NULL); + test_streq(cp_alloc,"and,arma,by,nickm,onion,router,the"); + tor_free(cp_alloc); + smartlist_swap(sl, 1, 5); + cp_alloc = smartlist_join_strings(sl, ",", 0, NULL); + test_streq(cp_alloc,"and,router,by,nickm,onion,arma,the"); + tor_free(cp_alloc); + smartlist_shuffle(sl); + test_eq(7, smartlist_len(sl)); + test_assert(smartlist_string_isin(sl, "and")); + test_assert(smartlist_string_isin(sl, "router")); + test_assert(smartlist_string_isin(sl, "by")); + test_assert(smartlist_string_isin(sl, "nickm")); + test_assert(smartlist_string_isin(sl, "onion")); + test_assert(smartlist_string_isin(sl, "arma")); + test_assert(smartlist_string_isin(sl, "the")); + + /* Test bsearch. */ + smartlist_sort(sl, _compare_strs); + test_streq("nickm", smartlist_bsearch(sl, "zNicKM", + _compare_without_first_ch)); + test_streq("and", smartlist_bsearch(sl, " AND", _compare_without_first_ch)); + test_eq_ptr(NULL, smartlist_bsearch(sl, " ANz", _compare_without_first_ch)); + + /* Test bsearch_idx */ + { + int f; + test_eq(0, smartlist_bsearch_idx(sl," aaa",_compare_without_first_ch,&f)); + test_eq(f, 0); + test_eq(0, smartlist_bsearch_idx(sl," and",_compare_without_first_ch,&f)); + test_eq(f, 1); + test_eq(1, smartlist_bsearch_idx(sl," arm",_compare_without_first_ch,&f)); + test_eq(f, 0); + test_eq(1, smartlist_bsearch_idx(sl," arma",_compare_without_first_ch,&f)); + test_eq(f, 1); + test_eq(2, smartlist_bsearch_idx(sl," armb",_compare_without_first_ch,&f)); + test_eq(f, 0); + test_eq(7, smartlist_bsearch_idx(sl," zzzz",_compare_without_first_ch,&f)); + test_eq(f, 0); + } + + /* Test reverse() and pop_last() */ + smartlist_reverse(sl); + cp_alloc = smartlist_join_strings(sl, ",", 0, NULL); + test_streq(cp_alloc,"the,router,onion,nickm,by,arma,and"); + tor_free(cp_alloc); + cp_alloc = smartlist_pop_last(sl); + test_streq(cp_alloc, "and"); + tor_free(cp_alloc); + test_eq(smartlist_len(sl), 6); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + cp_alloc = smartlist_pop_last(sl); + test_eq(cp_alloc, NULL); + + /* Test uniq() */ + smartlist_split_string(sl, + "50,noon,radar,a,man,a,plan,a,canal,panama,radar,noon,50", + ",", 0, 0); + smartlist_sort(sl, _compare_strs); + smartlist_uniq(sl, _compare_strs, _tor_free); + cp_alloc = smartlist_join_strings(sl, ",", 0, NULL); + test_streq(cp_alloc, "50,a,canal,man,noon,panama,plan,radar"); + tor_free(cp_alloc); + + /* Test string_isin and isin_case and num_isin */ + test_assert(smartlist_string_isin(sl, "noon")); + test_assert(!smartlist_string_isin(sl, "noonoon")); + test_assert(smartlist_string_isin_case(sl, "nOOn")); + test_assert(!smartlist_string_isin_case(sl, "nooNooN")); + test_assert(smartlist_string_num_isin(sl, 50)); + test_assert(!smartlist_string_num_isin(sl, 60)); + + /* Test smartlist_choose */ + { + int i; + int allsame = 1; + int allin = 1; + void *first = smartlist_choose(sl); + test_assert(smartlist_isin(sl, first)); + for (i = 0; i < 100; ++i) { + void *second = smartlist_choose(sl); + if (second != first) + allsame = 0; + if (!smartlist_isin(sl, second)) + allin = 0; + } + test_assert(!allsame); + test_assert(allin); + } + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + /* Test string_remove and remove and join_strings2 */ + smartlist_split_string(sl, + "Some say the Earth will end in ice and some in fire", + " ", 0, 0); + cp = smartlist_get(sl, 4); + test_streq(cp, "will"); + smartlist_add(sl, cp); + smartlist_remove(sl, cp); + tor_free(cp); + cp_alloc = smartlist_join_strings(sl, ",", 0, NULL); + test_streq(cp_alloc, "Some,say,the,Earth,fire,end,in,ice,and,some,in"); + tor_free(cp_alloc); + smartlist_string_remove(sl, "in"); + cp_alloc = smartlist_join_strings2(sl, "+XX", 1, 0, &sz); + test_streq(cp_alloc, "Some+say+the+Earth+fire+end+some+ice+and"); + test_eq((int)sz, 40); + + done: + + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_free(sl); + tor_free(cp_alloc); +} + +/** Run unit tests for smartlist set manipulation functions. */ +static void +test_container_smartlist_overlap(void) +{ + smartlist_t *sl = smartlist_create(); + smartlist_t *ints = smartlist_create(); + smartlist_t *odds = smartlist_create(); + smartlist_t *evens = smartlist_create(); + smartlist_t *primes = smartlist_create(); + int i; + for (i=1; i < 10; i += 2) + smartlist_add(odds, (void*)(uintptr_t)i); + for (i=0; i < 10; i += 2) + smartlist_add(evens, (void*)(uintptr_t)i); + + /* add_all */ + smartlist_add_all(ints, odds); + smartlist_add_all(ints, evens); + test_eq(smartlist_len(ints), 10); + + smartlist_add(primes, (void*)2); + smartlist_add(primes, (void*)3); + smartlist_add(primes, (void*)5); + smartlist_add(primes, (void*)7); + + /* overlap */ + test_assert(smartlist_overlap(ints, odds)); + test_assert(smartlist_overlap(odds, primes)); + test_assert(smartlist_overlap(evens, primes)); + test_assert(!smartlist_overlap(odds, evens)); + + /* intersect */ + smartlist_add_all(sl, odds); + smartlist_intersect(sl, primes); + test_eq(smartlist_len(sl), 3); + test_assert(smartlist_isin(sl, (void*)3)); + test_assert(smartlist_isin(sl, (void*)5)); + test_assert(smartlist_isin(sl, (void*)7)); + + /* subtract */ + smartlist_add_all(sl, primes); + smartlist_subtract(sl, odds); + test_eq(smartlist_len(sl), 1); + test_assert(smartlist_isin(sl, (void*)2)); + + done: + smartlist_free(odds); + smartlist_free(evens); + smartlist_free(ints); + smartlist_free(primes); + smartlist_free(sl); +} + +/** Run unit tests for smartlist-of-digests functions. */ +static void +test_container_smartlist_digests(void) +{ + smartlist_t *sl = smartlist_create(); + + /* digest_isin. */ + smartlist_add(sl, tor_memdup("AAAAAAAAAAAAAAAAAAAA", DIGEST_LEN)); + smartlist_add(sl, tor_memdup("\00090AAB2AAAAaasdAAAAA", DIGEST_LEN)); + smartlist_add(sl, tor_memdup("\00090AAB2AAAAaasdAAAAA", DIGEST_LEN)); + test_eq(0, smartlist_digest_isin(NULL, "AAAAAAAAAAAAAAAAAAAA")); + test_assert(smartlist_digest_isin(sl, "AAAAAAAAAAAAAAAAAAAA")); + test_assert(smartlist_digest_isin(sl, "\00090AAB2AAAAaasdAAAAA")); + test_eq(0, smartlist_digest_isin(sl, "\00090AAB2AAABaasdAAAAA")); + + /* sort digests */ + smartlist_sort_digests(sl); + test_memeq(smartlist_get(sl, 0), "\00090AAB2AAAAaasdAAAAA", DIGEST_LEN); + test_memeq(smartlist_get(sl, 1), "\00090AAB2AAAAaasdAAAAA", DIGEST_LEN); + test_memeq(smartlist_get(sl, 2), "AAAAAAAAAAAAAAAAAAAA", DIGEST_LEN); + test_eq(3, smartlist_len(sl)); + + /* uniq_digests */ + smartlist_uniq_digests(sl); + test_eq(2, smartlist_len(sl)); + test_memeq(smartlist_get(sl, 0), "\00090AAB2AAAAaasdAAAAA", DIGEST_LEN); + test_memeq(smartlist_get(sl, 1), "AAAAAAAAAAAAAAAAAAAA", DIGEST_LEN); + + done: + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_free(sl); +} + +/** Run unit tests for concatenate-a-smartlist-of-strings functions. */ +static void +test_container_smartlist_join(void) +{ + smartlist_t *sl = smartlist_create(); + smartlist_t *sl2 = smartlist_create(), *sl3 = smartlist_create(), + *sl4 = smartlist_create(); + char *joined=NULL; + /* unique, sorted. */ + smartlist_split_string(sl, + "Abashments Ambush Anchorman Bacon Banks Borscht " + "Bunks Inhumane Insurance Knish Know Manners " + "Maraschinos Stamina Sunbonnets Unicorns Wombats", + " ", 0, 0); + /* non-unique, sorted. */ + smartlist_split_string(sl2, + "Ambush Anchorman Anchorman Anemias Anemias Bacon " + "Crossbowmen Inhumane Insurance Knish Know Manners " + "Manners Maraschinos Wombats Wombats Work", + " ", 0, 0); + SMARTLIST_FOREACH_JOIN(sl, char *, cp1, + sl2, char *, cp2, + strcmp(cp1,cp2), + smartlist_add(sl3, cp2)) { + test_streq(cp1, cp2); + smartlist_add(sl4, cp1); + } SMARTLIST_FOREACH_JOIN_END(cp1, cp2); + + SMARTLIST_FOREACH(sl3, const char *, cp, + test_assert(smartlist_isin(sl2, cp) && + !smartlist_string_isin(sl, cp))); + SMARTLIST_FOREACH(sl4, const char *, cp, + test_assert(smartlist_isin(sl, cp) && + smartlist_string_isin(sl2, cp))); + joined = smartlist_join_strings(sl3, ",", 0, NULL); + test_streq(joined, "Anemias,Anemias,Crossbowmen,Work"); + tor_free(joined); + joined = smartlist_join_strings(sl4, ",", 0, NULL); + test_streq(joined, "Ambush,Anchorman,Anchorman,Bacon,Inhumane,Insurance," + "Knish,Know,Manners,Manners,Maraschinos,Wombats,Wombats"); + tor_free(joined); + + done: + smartlist_free(sl4); + smartlist_free(sl3); + SMARTLIST_FOREACH(sl2, char *, cp, tor_free(cp)); + smartlist_free(sl2); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_free(sl); + tor_free(joined); +} + +/** Run unit tests for bitarray code */ +static void +test_container_bitarray(void) +{ + bitarray_t *ba = NULL; + int i, j, ok=1; + + ba = bitarray_init_zero(1); + test_assert(ba); + test_assert(! bitarray_is_set(ba, 0)); + bitarray_set(ba, 0); + test_assert(bitarray_is_set(ba, 0)); + bitarray_clear(ba, 0); + test_assert(! bitarray_is_set(ba, 0)); + bitarray_free(ba); + + ba = bitarray_init_zero(1023); + for (i = 1; i < 64; ) { + for (j = 0; j < 1023; ++j) { + if (j % i) + bitarray_set(ba, j); + else + bitarray_clear(ba, j); + } + for (j = 0; j < 1023; ++j) { + if (!bool_eq(bitarray_is_set(ba, j), j%i)) + ok = 0; + } + test_assert(ok); + if (i < 7) + ++i; + else if (i == 28) + i = 32; + else + i += 7; + } + + done: + if (ba) + bitarray_free(ba); +} + +/** Run unit tests for digest set code (implemented as a hashtable or as a + * bloom filter) */ +static void +test_container_digestset(void) +{ + smartlist_t *included = smartlist_create(); + char d[DIGEST_LEN]; + int i; + int ok = 1; + int false_positives = 0; + digestset_t *set = NULL; + + for (i = 0; i < 1000; ++i) { + crypto_rand(d, DIGEST_LEN); + smartlist_add(included, tor_memdup(d, DIGEST_LEN)); + } + set = digestset_new(1000); + SMARTLIST_FOREACH(included, const char *, cp, + if (digestset_isin(set, cp)) + ok = 0); + test_assert(ok); + SMARTLIST_FOREACH(included, const char *, cp, + digestset_add(set, cp)); + SMARTLIST_FOREACH(included, const char *, cp, + if (!digestset_isin(set, cp)) + ok = 0); + test_assert(ok); + for (i = 0; i < 1000; ++i) { + crypto_rand(d, DIGEST_LEN); + if (digestset_isin(set, d)) + ++false_positives; + } + test_assert(false_positives < 50); /* Should be far lower. */ + + done: + if (set) + digestset_free(set); + SMARTLIST_FOREACH(included, char *, cp, tor_free(cp)); + smartlist_free(included); +} + +typedef struct pq_entry_t { + const char *val; + int idx; +} pq_entry_t; + +/** Helper: return a tristate based on comparing two pq_entry_t values. */ +static int +_compare_strings_for_pqueue(const void *p1, const void *p2) +{ + const pq_entry_t *e1=p1, *e2=p2; + return strcmp(e1->val, e2->val); +} + +/** Run unit tests for heap-based priority queue functions. */ +static void +test_container_pqueue(void) +{ + smartlist_t *sl = smartlist_create(); + int (*cmp)(const void *, const void*); + const int offset = STRUCT_OFFSET(pq_entry_t, idx); +#define ENTRY(s) pq_entry_t s = { #s, -1 } + ENTRY(cows); + ENTRY(zebras); + ENTRY(fish); + ENTRY(frogs); + ENTRY(apples); + ENTRY(squid); + ENTRY(daschunds); + ENTRY(eggplants); + ENTRY(weissbier); + ENTRY(lobsters); + ENTRY(roquefort); + ENTRY(chinchillas); + ENTRY(fireflies); + +#define OK() smartlist_pqueue_assert_ok(sl, cmp, offset) + + cmp = _compare_strings_for_pqueue; + smartlist_pqueue_add(sl, cmp, offset, &cows); + smartlist_pqueue_add(sl, cmp, offset, &zebras); + smartlist_pqueue_add(sl, cmp, offset, &fish); + smartlist_pqueue_add(sl, cmp, offset, &frogs); + smartlist_pqueue_add(sl, cmp, offset, &apples); + smartlist_pqueue_add(sl, cmp, offset, &squid); + smartlist_pqueue_add(sl, cmp, offset, &daschunds); + smartlist_pqueue_add(sl, cmp, offset, &eggplants); + smartlist_pqueue_add(sl, cmp, offset, &weissbier); + smartlist_pqueue_add(sl, cmp, offset, &lobsters); + smartlist_pqueue_add(sl, cmp, offset, &roquefort); + + OK(); + + test_eq(smartlist_len(sl), 11); + test_eq_ptr(smartlist_get(sl, 0), &apples); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &apples); + test_eq(smartlist_len(sl), 10); + OK(); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &cows); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &daschunds); + smartlist_pqueue_add(sl, cmp, offset, &chinchillas); + OK(); + smartlist_pqueue_add(sl, cmp, offset, &fireflies); + OK(); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &chinchillas); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &eggplants); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &fireflies); + OK(); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &fish); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &frogs); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &lobsters); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &roquefort); + OK(); + test_eq(smartlist_len(sl), 3); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &squid); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &weissbier); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &zebras); + test_eq(smartlist_len(sl), 0); + OK(); + + /* Now test remove. */ + smartlist_pqueue_add(sl, cmp, offset, &cows); + smartlist_pqueue_add(sl, cmp, offset, &fish); + smartlist_pqueue_add(sl, cmp, offset, &frogs); + smartlist_pqueue_add(sl, cmp, offset, &apples); + smartlist_pqueue_add(sl, cmp, offset, &squid); + smartlist_pqueue_add(sl, cmp, offset, &zebras); + test_eq(smartlist_len(sl), 6); + OK(); + smartlist_pqueue_remove(sl, cmp, offset, &zebras); + test_eq(smartlist_len(sl), 5); + OK(); + smartlist_pqueue_remove(sl, cmp, offset, &cows); + test_eq(smartlist_len(sl), 4); + OK(); + smartlist_pqueue_remove(sl, cmp, offset, &apples); + test_eq(smartlist_len(sl), 3); + OK(); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &fish); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &frogs); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &squid); + test_eq(smartlist_len(sl), 0); + OK(); + +#undef OK + + done: + + smartlist_free(sl); +} + +/** Run unit tests for string-to-void* map functions */ +static void +test_container_strmap(void) +{ + strmap_t *map; + strmap_iter_t *iter; + const char *k; + void *v; + char *visited = NULL; + smartlist_t *found_keys = NULL; + + map = strmap_new(); + test_assert(map); + test_eq(strmap_size(map), 0); + test_assert(strmap_isempty(map)); + v = strmap_set(map, "K1", (void*)99); + test_eq(v, NULL); + test_assert(!strmap_isempty(map)); + v = strmap_set(map, "K2", (void*)101); + test_eq(v, NULL); + v = strmap_set(map, "K1", (void*)100); + test_eq(v, (void*)99); + test_eq_ptr(strmap_get(map,"K1"), (void*)100); + test_eq_ptr(strmap_get(map,"K2"), (void*)101); + test_eq_ptr(strmap_get(map,"K-not-there"), NULL); + strmap_assert_ok(map); + + v = strmap_remove(map,"K2"); + strmap_assert_ok(map); + test_eq_ptr(v, (void*)101); + test_eq_ptr(strmap_get(map,"K2"), NULL); + test_eq_ptr(strmap_remove(map,"K2"), NULL); + + strmap_set(map, "K2", (void*)101); + strmap_set(map, "K3", (void*)102); + strmap_set(map, "K4", (void*)103); + test_eq(strmap_size(map), 4); + strmap_assert_ok(map); + strmap_set(map, "K5", (void*)104); + strmap_set(map, "K6", (void*)105); + strmap_assert_ok(map); + + /* Test iterator. */ + iter = strmap_iter_init(map); + found_keys = smartlist_create(); + while (!strmap_iter_done(iter)) { + strmap_iter_get(iter,&k,&v); + smartlist_add(found_keys, tor_strdup(k)); + test_eq_ptr(v, strmap_get(map, k)); + + if (!strcmp(k, "K2")) { + iter = strmap_iter_next_rmv(map,iter); + } else { + iter = strmap_iter_next(map,iter); + } + } + + /* Make sure we removed K2, but not the others. */ + test_eq_ptr(strmap_get(map, "K2"), NULL); + test_eq_ptr(strmap_get(map, "K5"), (void*)104); + /* Make sure we visited everyone once */ + smartlist_sort_strings(found_keys); + visited = smartlist_join_strings(found_keys, ":", 0, NULL); + test_streq(visited, "K1:K2:K3:K4:K5:K6"); + + strmap_assert_ok(map); + /* Clean up after ourselves. */ + strmap_free(map, NULL); + map = NULL; + + /* Now try some lc functions. */ + map = strmap_new(); + strmap_set_lc(map,"Ab.C", (void*)1); + test_eq_ptr(strmap_get(map,"ab.c"), (void*)1); + strmap_assert_ok(map); + test_eq_ptr(strmap_get_lc(map,"AB.C"), (void*)1); + test_eq_ptr(strmap_get(map,"AB.C"), NULL); + test_eq_ptr(strmap_remove_lc(map,"aB.C"), (void*)1); + strmap_assert_ok(map); + test_eq_ptr(strmap_get_lc(map,"AB.C"), NULL); + + done: + if (map) + strmap_free(map,NULL); + if (found_keys) { + SMARTLIST_FOREACH(found_keys, char *, cp, tor_free(cp)); + smartlist_free(found_keys); + } + tor_free(visited); +} + +/** Run unit tests for getting the median of a list. */ +static void +test_container_order_functions(void) +{ + int lst[25], n = 0; + // int a=12,b=24,c=25,d=60,e=77; + +#define median() median_int(lst, n) + + lst[n++] = 12; + test_eq(12, median()); /* 12 */ + lst[n++] = 77; + //smartlist_shuffle(sl); + test_eq(12, median()); /* 12, 77 */ + lst[n++] = 77; + //smartlist_shuffle(sl); + test_eq(77, median()); /* 12, 77, 77 */ + lst[n++] = 24; + test_eq(24, median()); /* 12,24,77,77 */ + lst[n++] = 60; + lst[n++] = 12; + lst[n++] = 25; + //smartlist_shuffle(sl); + test_eq(25, median()); /* 12,12,24,25,60,77,77 */ +#undef median + + done: + ; +} + +#define CONTAINER_LEGACY(name) \ + { #name, legacy_test_helper, 0, &legacy_setup, test_container_ ## name } + +struct testcase_t container_tests[] = { + CONTAINER_LEGACY(smartlist_basic), + CONTAINER_LEGACY(smartlist_strings), + CONTAINER_LEGACY(smartlist_overlap), + CONTAINER_LEGACY(smartlist_digests), + CONTAINER_LEGACY(smartlist_join), + CONTAINER_LEGACY(bitarray), + CONTAINER_LEGACY(digestset), + CONTAINER_LEGACY(strmap), + CONTAINER_LEGACY(pqueue), + CONTAINER_LEGACY(order_functions), + END_OF_TESTCASES +}; + diff --git a/src/test/test_crypto.c b/src/test/test_crypto.c new file mode 100644 index 0000000000..656c3c94fc --- /dev/null +++ b/src/test/test_crypto.c @@ -0,0 +1,784 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2009, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#define CRYPTO_PRIVATE +#include "or.h" +#include "test.h" + +/** Run unit tests for Diffie-Hellman functionality. */ +static void +test_crypto_dh(void) +{ + crypto_dh_env_t *dh1 = crypto_dh_new(); + crypto_dh_env_t *dh2 = crypto_dh_new(); + char p1[DH_BYTES]; + char p2[DH_BYTES]; + char s1[DH_BYTES]; + char s2[DH_BYTES]; + ssize_t s1len, s2len; + + test_eq(crypto_dh_get_bytes(dh1), DH_BYTES); + test_eq(crypto_dh_get_bytes(dh2), DH_BYTES); + + memset(p1, 0, DH_BYTES); + memset(p2, 0, DH_BYTES); + test_memeq(p1, p2, DH_BYTES); + test_assert(! crypto_dh_get_public(dh1, p1, DH_BYTES)); + test_memneq(p1, p2, DH_BYTES); + test_assert(! crypto_dh_get_public(dh2, p2, DH_BYTES)); + test_memneq(p1, p2, DH_BYTES); + + memset(s1, 0, DH_BYTES); + memset(s2, 0xFF, DH_BYTES); + s1len = crypto_dh_compute_secret(LOG_WARN, dh1, p2, DH_BYTES, s1, 50); + s2len = crypto_dh_compute_secret(LOG_WARN, dh2, p1, DH_BYTES, s2, 50); + test_assert(s1len > 0); + test_eq(s1len, s2len); + test_memeq(s1, s2, s1len); + + { + /* XXXX Now fabricate some bad values and make sure they get caught, + * Check 0, 1, N-1, >= N, etc. + */ + } + + done: + crypto_dh_free(dh1); + crypto_dh_free(dh2); +} + +/** Run unit tests for our random number generation function and its wrappers. + */ +static void +test_crypto_rng(void) +{ + int i, j, allok; + char data1[100], data2[100]; + + /* Try out RNG. */ + test_assert(! crypto_seed_rng(0)); + crypto_rand(data1, 100); + crypto_rand(data2, 100); + test_memneq(data1,data2,100); + allok = 1; + for (i = 0; i < 100; ++i) { + uint64_t big; + char *host; + j = crypto_rand_int(100); + if (i < 0 || i >= 100) + allok = 0; + big = crypto_rand_uint64(U64_LITERAL(1)<<40); + if (big >= (U64_LITERAL(1)<<40)) + allok = 0; + big = crypto_rand_uint64(U64_LITERAL(5)); + if (big >= 5) + allok = 0; + host = crypto_random_hostname(3,8,"www.",".onion"); + if (strcmpstart(host,"www.") || + strcmpend(host,".onion") || + strlen(host) < 13 || + strlen(host) > 18) + allok = 0; + tor_free(host); + } + test_assert(allok); + done: + ; +} + +/** Run unit tests for our AES functionality */ +static void +test_crypto_aes(void) +{ + char *data1 = NULL, *data2 = NULL, *data3 = NULL; + crypto_cipher_env_t *env1 = NULL, *env2 = NULL; + int i, j; + char *mem_op_hex_tmp=NULL; + + data1 = tor_malloc(1024); + data2 = tor_malloc(1024); + data3 = tor_malloc(1024); + + /* Now, test encryption and decryption with stream cipher. */ + data1[0]='\0'; + for (i = 1023; i>0; i -= 35) + strncat(data1, "Now is the time for all good onions", i); + + memset(data2, 0, 1024); + memset(data3, 0, 1024); + env1 = crypto_new_cipher_env(); + test_neq(env1, 0); + env2 = crypto_new_cipher_env(); + test_neq(env2, 0); + j = crypto_cipher_generate_key(env1); + crypto_cipher_set_key(env2, crypto_cipher_get_key(env1)); + crypto_cipher_encrypt_init_cipher(env1); + crypto_cipher_decrypt_init_cipher(env2); + + /* Try encrypting 512 chars. */ + crypto_cipher_encrypt(env1, data2, data1, 512); + crypto_cipher_decrypt(env2, data3, data2, 512); + test_memeq(data1, data3, 512); + test_memneq(data1, data2, 512); + + /* Now encrypt 1 at a time, and get 1 at a time. */ + for (j = 512; j < 560; ++j) { + crypto_cipher_encrypt(env1, data2+j, data1+j, 1); + } + for (j = 512; j < 560; ++j) { + crypto_cipher_decrypt(env2, data3+j, data2+j, 1); + } + test_memeq(data1, data3, 560); + /* Now encrypt 3 at a time, and get 5 at a time. */ + for (j = 560; j < 1024-5; j += 3) { + crypto_cipher_encrypt(env1, data2+j, data1+j, 3); + } + for (j = 560; j < 1024-5; j += 5) { + crypto_cipher_decrypt(env2, data3+j, data2+j, 5); + } + test_memeq(data1, data3, 1024-5); + /* Now make sure that when we encrypt with different chunk sizes, we get + the same results. */ + crypto_free_cipher_env(env2); + env2 = NULL; + + memset(data3, 0, 1024); + env2 = crypto_new_cipher_env(); + test_neq(env2, 0); + crypto_cipher_set_key(env2, crypto_cipher_get_key(env1)); + crypto_cipher_encrypt_init_cipher(env2); + for (j = 0; j < 1024-16; j += 17) { + crypto_cipher_encrypt(env2, data3+j, data1+j, 17); + } + for (j= 0; j < 1024-16; ++j) { + if (data2[j] != data3[j]) { + printf("%d: %d\t%d\n", j, (int) data2[j], (int) data3[j]); + } + } + test_memeq(data2, data3, 1024-16); + crypto_free_cipher_env(env1); + env1 = NULL; + crypto_free_cipher_env(env2); + env2 = NULL; + + /* NIST test vector for aes. */ + env1 = crypto_new_cipher_env(); /* IV starts at 0 */ + crypto_cipher_set_key(env1, "\x80\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00"); + crypto_cipher_encrypt_init_cipher(env1); + crypto_cipher_encrypt(env1, data1, + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00", 16); + test_memeq_hex(data1, "0EDD33D3C621E546455BD8BA1418BEC8"); + + /* Now test rollover. All these values are originally from a python + * script. */ + crypto_cipher_set_iv(env1, "\x00\x00\x00\x00\x00\x00\x00\x00" + "\xff\xff\xff\xff\xff\xff\xff\xff"); + memset(data2, 0, 1024); + crypto_cipher_encrypt(env1, data1, data2, 32); + test_memeq_hex(data1, "335fe6da56f843199066c14a00a40231" + "cdd0b917dbc7186908a6bfb5ffd574d3"); + + crypto_cipher_set_iv(env1, "\x00\x00\x00\x00\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff"); + memset(data2, 0, 1024); + crypto_cipher_encrypt(env1, data1, data2, 32); + test_memeq_hex(data1, "e627c6423fa2d77832a02b2794094b73" + "3e63c721df790d2c6469cc1953a3ffac"); + + crypto_cipher_set_iv(env1, "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff"); + memset(data2, 0, 1024); + crypto_cipher_encrypt(env1, data1, data2, 32); + test_memeq_hex(data1, "2aed2bff0de54f9328efd070bf48f70a" + "0EDD33D3C621E546455BD8BA1418BEC8"); + + /* Now check rollover on inplace cipher. */ + crypto_cipher_set_iv(env1, "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff"); + crypto_cipher_crypt_inplace(env1, data2, 64); + test_memeq_hex(data2, "2aed2bff0de54f9328efd070bf48f70a" + "0EDD33D3C621E546455BD8BA1418BEC8" + "93e2c5243d6839eac58503919192f7ae" + "1908e67cafa08d508816659c2e693191"); + crypto_cipher_set_iv(env1, "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff"); + crypto_cipher_crypt_inplace(env1, data2, 64); + test_assert(tor_mem_is_zero(data2, 64)); + + done: + tor_free(mem_op_hex_tmp); + if (env1) + crypto_free_cipher_env(env1); + if (env2) + crypto_free_cipher_env(env2); + tor_free(data1); + tor_free(data2); + tor_free(data3); +} + +/** Run unit tests for our SHA-1 functionality */ +static void +test_crypto_sha(void) +{ + crypto_digest_env_t *d1 = NULL, *d2 = NULL; + int i; + char key[80]; + char digest[32]; + char data[50]; + char d_out1[DIGEST_LEN], d_out2[DIGEST256_LEN]; + char *mem_op_hex_tmp=NULL; + + /* Test SHA-1 with a test vector from the specification. */ + i = crypto_digest(data, "abc", 3); + test_memeq_hex(data, "A9993E364706816ABA3E25717850C26C9CD0D89D"); + + /* Test SHA-256 with a test vector from the specification. */ + i = crypto_digest256(data, "abc", 3, DIGEST_SHA256); + test_memeq_hex(data, "BA7816BF8F01CFEA414140DE5DAE2223B00361A3" + "96177A9CB410FF61F20015AD"); + + /* Test HMAC-SHA-1 with test cases from RFC2202. */ + + /* Case 1. */ + memset(key, 0x0b, 20); + crypto_hmac_sha1(digest, key, 20, "Hi There", 8); + test_streq(hex_str(digest, 20), + "B617318655057264E28BC0B6FB378C8EF146BE00"); + /* Case 2. */ + crypto_hmac_sha1(digest, "Jefe", 4, "what do ya want for nothing?", 28); + test_streq(hex_str(digest, 20), + "EFFCDF6AE5EB2FA2D27416D5F184DF9C259A7C79"); + + /* Case 4. */ + base16_decode(key, 25, + "0102030405060708090a0b0c0d0e0f10111213141516171819", 50); + memset(data, 0xcd, 50); + crypto_hmac_sha1(digest, key, 25, data, 50); + test_streq(hex_str(digest, 20), + "4C9007F4026250C6BC8414F9BF50C86C2D7235DA"); + + /* Case 5. */ + memset(key, 0xaa, 80); + crypto_hmac_sha1(digest, key, 80, + "Test Using Larger Than Block-Size Key - Hash Key First", + 54); + test_streq(hex_str(digest, 20), + "AA4AE5E15272D00E95705637CE8A3B55ED402112"); + + /* Incremental digest code. */ + d1 = crypto_new_digest_env(); + test_assert(d1); + crypto_digest_add_bytes(d1, "abcdef", 6); + d2 = crypto_digest_dup(d1); + test_assert(d2); + crypto_digest_add_bytes(d2, "ghijkl", 6); + crypto_digest_get_digest(d2, d_out1, sizeof(d_out1)); + crypto_digest(d_out2, "abcdefghijkl", 12); + test_memeq(d_out1, d_out2, DIGEST_LEN); + crypto_digest_assign(d2, d1); + crypto_digest_add_bytes(d2, "mno", 3); + crypto_digest_get_digest(d2, d_out1, sizeof(d_out1)); + crypto_digest(d_out2, "abcdefmno", 9); + test_memeq(d_out1, d_out2, DIGEST_LEN); + crypto_digest_get_digest(d1, d_out1, sizeof(d_out1)); + crypto_digest(d_out2, "abcdef", 6); + test_memeq(d_out1, d_out2, DIGEST_LEN); + crypto_free_digest_env(d1); + crypto_free_digest_env(d2); + + /* Incremental digest code with sha256 */ + d1 = crypto_new_digest256_env(DIGEST_SHA256); + test_assert(d1); + crypto_digest_add_bytes(d1, "abcdef", 6); + d2 = crypto_digest_dup(d1); + test_assert(d2); + crypto_digest_add_bytes(d2, "ghijkl", 6); + crypto_digest_get_digest(d2, d_out1, sizeof(d_out1)); + crypto_digest256(d_out2, "abcdefghijkl", 12, DIGEST_SHA256); + test_memeq(d_out1, d_out2, DIGEST_LEN); + crypto_digest_assign(d2, d1); + crypto_digest_add_bytes(d2, "mno", 3); + crypto_digest_get_digest(d2, d_out1, sizeof(d_out1)); + crypto_digest256(d_out2, "abcdefmno", 9, DIGEST_SHA256); + test_memeq(d_out1, d_out2, DIGEST_LEN); + crypto_digest_get_digest(d1, d_out1, sizeof(d_out1)); + crypto_digest256(d_out2, "abcdef", 6, DIGEST_SHA256); + test_memeq(d_out1, d_out2, DIGEST_LEN); + + done: + if (d1) + crypto_free_digest_env(d1); + if (d2) + crypto_free_digest_env(d2); + tor_free(mem_op_hex_tmp); +} + +/** Run unit tests for our public key crypto functions */ +static void +test_crypto_pk(void) +{ + crypto_pk_env_t *pk1 = NULL, *pk2 = NULL; + char *encoded = NULL; + char data1[1024], data2[1024], data3[1024]; + size_t size; + int i, j, p, len; + + /* Public-key ciphers */ + pk1 = pk_generate(0); + pk2 = crypto_new_pk_env(); + test_assert(pk1 && pk2); + test_assert(! crypto_pk_write_public_key_to_string(pk1, &encoded, &size)); + test_assert(! crypto_pk_read_public_key_from_string(pk2, encoded, size)); + test_eq(0, crypto_pk_cmp_keys(pk1, pk2)); + + test_eq(128, crypto_pk_keysize(pk1)); + test_eq(128, crypto_pk_keysize(pk2)); + + test_eq(128, crypto_pk_public_encrypt(pk2, data1, "Hello whirled.", 15, + PK_PKCS1_OAEP_PADDING)); + test_eq(128, crypto_pk_public_encrypt(pk1, data2, "Hello whirled.", 15, + PK_PKCS1_OAEP_PADDING)); + /* oaep padding should make encryption not match */ + test_memneq(data1, data2, 128); + test_eq(15, crypto_pk_private_decrypt(pk1, data3, data1, 128, + PK_PKCS1_OAEP_PADDING,1)); + test_streq(data3, "Hello whirled."); + memset(data3, 0, 1024); + test_eq(15, crypto_pk_private_decrypt(pk1, data3, data2, 128, + PK_PKCS1_OAEP_PADDING,1)); + test_streq(data3, "Hello whirled."); + /* Can't decrypt with public key. */ + test_eq(-1, crypto_pk_private_decrypt(pk2, data3, data2, 128, + PK_PKCS1_OAEP_PADDING,1)); + /* Try again with bad padding */ + memcpy(data2+1, "XYZZY", 5); /* This has fails ~ once-in-2^40 */ + test_eq(-1, crypto_pk_private_decrypt(pk1, data3, data2, 128, + PK_PKCS1_OAEP_PADDING,1)); + + /* File operations: save and load private key */ + test_assert(! crypto_pk_write_private_key_to_filename(pk1, + get_fname("pkey1"))); + /* failing case for read: can't read. */ + test_assert(crypto_pk_read_private_key_from_filename(pk2, + get_fname("xyzzy")) < 0); + write_str_to_file(get_fname("xyzzy"), "foobar", 6); + /* Failing case for read: no key. */ + test_assert(crypto_pk_read_private_key_from_filename(pk2, + get_fname("xyzzy")) < 0); + test_assert(! crypto_pk_read_private_key_from_filename(pk2, + get_fname("pkey1"))); + test_eq(15, crypto_pk_private_decrypt(pk2, data3, data1, 128, + PK_PKCS1_OAEP_PADDING,1)); + + /* Now try signing. */ + strlcpy(data1, "Ossifrage", 1024); + test_eq(128, crypto_pk_private_sign(pk1, data2, data1, 10)); + test_eq(10, crypto_pk_public_checksig(pk1, data3, data2, 128)); + test_streq(data3, "Ossifrage"); + /* Try signing digests. */ + test_eq(128, crypto_pk_private_sign_digest(pk1, data2, data1, 10)); + test_eq(20, crypto_pk_public_checksig(pk1, data3, data2, 128)); + test_eq(0, crypto_pk_public_checksig_digest(pk1, data1, 10, data2, 128)); + test_eq(-1, crypto_pk_public_checksig_digest(pk1, data1, 11, data2, 128)); + /*XXXX test failed signing*/ + + /* Try encoding */ + crypto_free_pk_env(pk2); + pk2 = NULL; + i = crypto_pk_asn1_encode(pk1, data1, 1024); + test_assert(i>0); + pk2 = crypto_pk_asn1_decode(data1, i); + test_assert(crypto_pk_cmp_keys(pk1,pk2) == 0); + + /* Try with hybrid encryption wrappers. */ + crypto_rand(data1, 1024); + for (i = 0; i < 3; ++i) { + for (j = 85; j < 140; ++j) { + memset(data2,0,1024); + memset(data3,0,1024); + if (i == 0 && j < 129) + continue; + p = (i==0)?PK_NO_PADDING: + (i==1)?PK_PKCS1_PADDING:PK_PKCS1_OAEP_PADDING; + len = crypto_pk_public_hybrid_encrypt(pk1,data2,data1,j,p,0); + test_assert(len>=0); + len = crypto_pk_private_hybrid_decrypt(pk1,data3,data2,len,p,1); + test_eq(len,j); + test_memeq(data1,data3,j); + } + } + + /* Try copy_full */ + crypto_free_pk_env(pk2); + pk2 = crypto_pk_copy_full(pk1); + test_assert(pk2 != NULL); + test_neq_ptr(pk1, pk2); + test_assert(crypto_pk_cmp_keys(pk1,pk2) == 0); + + done: + if (pk1) + crypto_free_pk_env(pk1); + if (pk2) + crypto_free_pk_env(pk2); + tor_free(encoded); +} + +/** Run unit tests for misc crypto formatting functionality (base64, base32, + * fingerprints, etc) */ +static void +test_crypto_formats(void) +{ + char *data1 = NULL, *data2 = NULL, *data3 = NULL; + int i, j, idx; + + data1 = tor_malloc(1024); + data2 = tor_malloc(1024); + data3 = tor_malloc(1024); + test_assert(data1 && data2 && data3); + + /* Base64 tests */ + memset(data1, 6, 1024); + for (idx = 0; idx < 10; ++idx) { + i = base64_encode(data2, 1024, data1, idx); + test_assert(i >= 0); + j = base64_decode(data3, 1024, data2, i); + test_eq(j,idx); + test_memeq(data3, data1, idx); + } + + strlcpy(data1, "Test string that contains 35 chars.", 1024); + strlcat(data1, " 2nd string that contains 35 chars.", 1024); + + i = base64_encode(data2, 1024, data1, 71); + test_assert(i >= 0); + j = base64_decode(data3, 1024, data2, i); + test_eq(j, 71); + test_streq(data3, data1); + test_assert(data2[i] == '\0'); + + crypto_rand(data1, DIGEST_LEN); + memset(data2, 100, 1024); + digest_to_base64(data2, data1); + test_eq(BASE64_DIGEST_LEN, strlen(data2)); + test_eq(100, data2[BASE64_DIGEST_LEN+2]); + memset(data3, 99, 1024); + test_eq(digest_from_base64(data3, data2), 0); + test_memeq(data1, data3, DIGEST_LEN); + test_eq(99, data3[DIGEST_LEN+1]); + + test_assert(digest_from_base64(data3, "###") < 0); + + /* Encoding SHA256 */ + crypto_rand(data2, DIGEST256_LEN); + memset(data2, 100, 1024); + digest256_to_base64(data2, data1); + test_eq(BASE64_DIGEST256_LEN, strlen(data2)); + test_eq(100, data2[BASE64_DIGEST256_LEN+2]); + memset(data3, 99, 1024); + test_eq(digest256_from_base64(data3, data2), 0); + test_memeq(data1, data3, DIGEST256_LEN); + test_eq(99, data3[DIGEST256_LEN+1]); + + /* Base32 tests */ + strlcpy(data1, "5chrs", 1024); + /* bit pattern is: [35 63 68 72 73] -> + * [00110101 01100011 01101000 01110010 01110011] + * By 5s: [00110 10101 10001 10110 10000 11100 10011 10011] + */ + base32_encode(data2, 9, data1, 5); + test_streq(data2, "gvrwq4tt"); + + strlcpy(data1, "\xFF\xF5\x6D\x44\xAE\x0D\x5C\xC9\x62\xC4", 1024); + base32_encode(data2, 30, data1, 10); + test_streq(data2, "772w2rfobvomsywe"); + + /* Base16 tests */ + strlcpy(data1, "6chrs\xff", 1024); + base16_encode(data2, 13, data1, 6); + test_streq(data2, "3663687273FF"); + + strlcpy(data1, "f0d678affc000100", 1024); + i = base16_decode(data2, 8, data1, 16); + test_eq(i,0); + test_memeq(data2, "\xf0\xd6\x78\xaf\xfc\x00\x01\x00",8); + + /* now try some failing base16 decodes */ + test_eq(-1, base16_decode(data2, 8, data1, 15)); /* odd input len */ + test_eq(-1, base16_decode(data2, 7, data1, 16)); /* dest too short */ + strlcpy(data1, "f0dz!8affc000100", 1024); + test_eq(-1, base16_decode(data2, 8, data1, 16)); + + tor_free(data1); + tor_free(data2); + tor_free(data3); + + /* Add spaces to fingerprint */ + { + data1 = tor_strdup("ABCD1234ABCD56780000ABCD1234ABCD56780000"); + test_eq(strlen(data1), 40); + data2 = tor_malloc(FINGERPRINT_LEN+1); + add_spaces_to_fp(data2, FINGERPRINT_LEN+1, data1); + test_streq(data2, "ABCD 1234 ABCD 5678 0000 ABCD 1234 ABCD 5678 0000"); + tor_free(data1); + tor_free(data2); + } + + /* Check fingerprint */ + { + test_assert(crypto_pk_check_fingerprint_syntax( + "ABCD 1234 ABCD 5678 0000 ABCD 1234 ABCD 5678 0000")); + test_assert(!crypto_pk_check_fingerprint_syntax( + "ABCD 1234 ABCD 5678 0000 ABCD 1234 ABCD 5678 000")); + test_assert(!crypto_pk_check_fingerprint_syntax( + "ABCD 1234 ABCD 5678 0000 ABCD 1234 ABCD 5678 00000")); + test_assert(!crypto_pk_check_fingerprint_syntax( + "ABCD 1234 ABCD 5678 0000 ABCD1234 ABCD 5678 0000")); + test_assert(!crypto_pk_check_fingerprint_syntax( + "ABCD 1234 ABCD 5678 0000 ABCD1234 ABCD 5678 00000")); + test_assert(!crypto_pk_check_fingerprint_syntax( + "ACD 1234 ABCD 5678 0000 ABCD 1234 ABCD 5678 00000")); + } + + done: + tor_free(data1); + tor_free(data2); + tor_free(data3); +} + +/** Run unit tests for our secret-to-key passphrase hashing functionality. */ +static void +test_crypto_s2k(void) +{ + char buf[29]; + char buf2[29]; + char *buf3 = NULL; + int i; + + memset(buf, 0, sizeof(buf)); + memset(buf2, 0, sizeof(buf2)); + buf3 = tor_malloc(65536); + memset(buf3, 0, 65536); + + secret_to_key(buf+9, 20, "", 0, buf); + crypto_digest(buf2+9, buf3, 1024); + test_memeq(buf, buf2, 29); + + memcpy(buf,"vrbacrda",8); + memcpy(buf2,"vrbacrda",8); + buf[8] = 96; + buf2[8] = 96; + secret_to_key(buf+9, 20, "12345678", 8, buf); + for (i = 0; i < 65536; i += 16) { + memcpy(buf3+i, "vrbacrda12345678", 16); + } + crypto_digest(buf2+9, buf3, 65536); + test_memeq(buf, buf2, 29); + + done: + tor_free(buf3); +} + +/** Test AES-CTR encryption and decryption with IV. */ +static void +test_crypto_aes_iv(void) +{ + crypto_cipher_env_t *cipher; + char *plain, *encrypted1, *encrypted2, *decrypted1, *decrypted2; + char plain_1[1], plain_15[15], plain_16[16], plain_17[17]; + char key1[16], key2[16]; + ssize_t encrypted_size, decrypted_size; + + plain = tor_malloc(4095); + encrypted1 = tor_malloc(4095 + 1 + 16); + encrypted2 = tor_malloc(4095 + 1 + 16); + decrypted1 = tor_malloc(4095 + 1); + decrypted2 = tor_malloc(4095 + 1); + + crypto_rand(plain, 4095); + crypto_rand(key1, 16); + crypto_rand(key2, 16); + crypto_rand(plain_1, 1); + crypto_rand(plain_15, 15); + crypto_rand(plain_16, 16); + crypto_rand(plain_17, 17); + key1[0] = key2[0] + 128; /* Make sure that contents are different. */ + /* Encrypt and decrypt with the same key. */ + cipher = crypto_create_init_cipher(key1, 1); + encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted1, 16 + 4095, + plain, 4095); + crypto_free_cipher_env(cipher); + cipher = NULL; + test_eq(encrypted_size, 16 + 4095); + tor_assert(encrypted_size > 0); /* This is obviously true, since 4111 is + * greater than 0, but its truth is not + * obvious to all analysis tools. */ + cipher = crypto_create_init_cipher(key1, 0); + decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 4095, + encrypted1, encrypted_size); + crypto_free_cipher_env(cipher); + cipher = NULL; + test_eq(decrypted_size, 4095); + tor_assert(decrypted_size > 0); + test_memeq(plain, decrypted1, 4095); + /* Encrypt a second time (with a new random initialization vector). */ + cipher = crypto_create_init_cipher(key1, 1); + encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted2, 16 + 4095, + plain, 4095); + crypto_free_cipher_env(cipher); + cipher = NULL; + test_eq(encrypted_size, 16 + 4095); + tor_assert(encrypted_size > 0); + cipher = crypto_create_init_cipher(key1, 0); + decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted2, 4095, + encrypted2, encrypted_size); + crypto_free_cipher_env(cipher); + cipher = NULL; + test_eq(decrypted_size, 4095); + tor_assert(decrypted_size > 0); + test_memeq(plain, decrypted2, 4095); + test_memneq(encrypted1, encrypted2, encrypted_size); + /* Decrypt with the wrong key. */ + cipher = crypto_create_init_cipher(key2, 0); + decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted2, 4095, + encrypted1, encrypted_size); + crypto_free_cipher_env(cipher); + cipher = NULL; + test_memneq(plain, decrypted2, encrypted_size); + /* Alter the initialization vector. */ + encrypted1[0] += 42; + cipher = crypto_create_init_cipher(key1, 0); + decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 4095, + encrypted1, encrypted_size); + crypto_free_cipher_env(cipher); + cipher = NULL; + test_memneq(plain, decrypted2, 4095); + /* Special length case: 1. */ + cipher = crypto_create_init_cipher(key1, 1); + encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted1, 16 + 1, + plain_1, 1); + crypto_free_cipher_env(cipher); + cipher = NULL; + test_eq(encrypted_size, 16 + 1); + tor_assert(encrypted_size > 0); + cipher = crypto_create_init_cipher(key1, 0); + decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 1, + encrypted1, encrypted_size); + crypto_free_cipher_env(cipher); + cipher = NULL; + test_eq(decrypted_size, 1); + tor_assert(decrypted_size > 0); + test_memeq(plain_1, decrypted1, 1); + /* Special length case: 15. */ + cipher = crypto_create_init_cipher(key1, 1); + encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted1, 16 + 15, + plain_15, 15); + crypto_free_cipher_env(cipher); + cipher = NULL; + test_eq(encrypted_size, 16 + 15); + tor_assert(encrypted_size > 0); + cipher = crypto_create_init_cipher(key1, 0); + decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 15, + encrypted1, encrypted_size); + crypto_free_cipher_env(cipher); + cipher = NULL; + test_eq(decrypted_size, 15); + tor_assert(decrypted_size > 0); + test_memeq(plain_15, decrypted1, 15); + /* Special length case: 16. */ + cipher = crypto_create_init_cipher(key1, 1); + encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted1, 16 + 16, + plain_16, 16); + crypto_free_cipher_env(cipher); + cipher = NULL; + test_eq(encrypted_size, 16 + 16); + tor_assert(encrypted_size > 0); + cipher = crypto_create_init_cipher(key1, 0); + decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 16, + encrypted1, encrypted_size); + crypto_free_cipher_env(cipher); + cipher = NULL; + test_eq(decrypted_size, 16); + tor_assert(decrypted_size > 0); + test_memeq(plain_16, decrypted1, 16); + /* Special length case: 17. */ + cipher = crypto_create_init_cipher(key1, 1); + encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted1, 16 + 17, + plain_17, 17); + crypto_free_cipher_env(cipher); + cipher = NULL; + test_eq(encrypted_size, 16 + 17); + tor_assert(encrypted_size > 0); + cipher = crypto_create_init_cipher(key1, 0); + decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 17, + encrypted1, encrypted_size); + test_eq(decrypted_size, 17); + tor_assert(decrypted_size > 0); + test_memeq(plain_17, decrypted1, 17); + + done: + /* Free memory. */ + tor_free(plain); + tor_free(encrypted1); + tor_free(encrypted2); + tor_free(decrypted1); + tor_free(decrypted2); + if (cipher) + crypto_free_cipher_env(cipher); +} + +/** Test base32 decoding. */ +static void +test_crypto_base32_decode(void) +{ + char plain[60], encoded[96 + 1], decoded[60]; + int res; + crypto_rand(plain, 60); + /* Encode and decode a random string. */ + base32_encode(encoded, 96 + 1, plain, 60); + res = base32_decode(decoded, 60, encoded, 96); + test_eq(res, 0); + test_memeq(plain, decoded, 60); + /* Encode, uppercase, and decode a random string. */ + base32_encode(encoded, 96 + 1, plain, 60); + tor_strupper(encoded); + res = base32_decode(decoded, 60, encoded, 96); + test_eq(res, 0); + test_memeq(plain, decoded, 60); + /* Change encoded string and decode. */ + if (encoded[0] == 'A' || encoded[0] == 'a') + encoded[0] = 'B'; + else + encoded[0] = 'A'; + res = base32_decode(decoded, 60, encoded, 96); + test_eq(res, 0); + test_memneq(plain, decoded, 60); + /* Bad encodings. */ + encoded[0] = '!'; + res = base32_decode(decoded, 60, encoded, 96); + test_assert(res < 0); + + done: + ; +} + +#define CRYPTO_LEGACY(name) \ + { #name, legacy_test_helper, 0, &legacy_setup, test_crypto_ ## name } + +struct testcase_t crypto_tests[] = { + CRYPTO_LEGACY(formats), + CRYPTO_LEGACY(rng), + CRYPTO_LEGACY(aes), + CRYPTO_LEGACY(sha), + CRYPTO_LEGACY(pk), + CRYPTO_LEGACY(dh), + CRYPTO_LEGACY(s2k), + CRYPTO_LEGACY(aes_iv), + CRYPTO_LEGACY(base32_decode), + END_OF_TESTCASES +}; + diff --git a/src/or/test_data.c b/src/test/test_data.c index bd7d989856..bd7d989856 100644 --- a/src/or/test_data.c +++ b/src/test/test_data.c diff --git a/src/test/test_dir.c b/src/test/test_dir.c new file mode 100644 index 0000000000..b7ee403299 --- /dev/null +++ b/src/test/test_dir.c @@ -0,0 +1,1311 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2009, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#define DIRSERV_PRIVATE +#define DIRVOTE_PRIVATE +#define ROUTER_PRIVATE +#include "or.h" +#include "test.h" + +static void +test_dir_nicknames(void) +{ + test_assert( is_legal_nickname("a")); + test_assert(!is_legal_nickname("")); + test_assert(!is_legal_nickname("abcdefghijklmnopqrst")); /* 20 chars */ + test_assert(!is_legal_nickname("hyphen-")); /* bad char */ + test_assert( is_legal_nickname("abcdefghijklmnopqrs")); /* 19 chars */ + test_assert(!is_legal_nickname("$AAAAAAAA01234AAAAAAAAAAAAAAAAAAAAAAAAAAA")); + /* valid */ + test_assert( is_legal_nickname_or_hexdigest( + "$AAAAAAAA01234AAAAAAAAAAAAAAAAAAAAAAAAAAA")); + test_assert( is_legal_nickname_or_hexdigest( + "$AAAAAAAA01234AAAAAAAAAAAAAAAAAAAAAAAAAAA=fred")); + test_assert( is_legal_nickname_or_hexdigest( + "$AAAAAAAA01234AAAAAAAAAAAAAAAAAAAAAAAAAAA~fred")); + /* too short */ + test_assert(!is_legal_nickname_or_hexdigest( + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")); + /* illegal char */ + test_assert(!is_legal_nickname_or_hexdigest( + "$AAAAAAzAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")); + /* hex part too long */ + test_assert(!is_legal_nickname_or_hexdigest( + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")); + test_assert(!is_legal_nickname_or_hexdigest( + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=fred")); + /* Bad nickname */ + test_assert(!is_legal_nickname_or_hexdigest( + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=")); + test_assert(!is_legal_nickname_or_hexdigest( + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~")); + test_assert(!is_legal_nickname_or_hexdigest( + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~hyphen-")); + test_assert(!is_legal_nickname_or_hexdigest( + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~" + "abcdefghijklmnoppqrst")); + /* Bad extra char. */ + test_assert(!is_legal_nickname_or_hexdigest( + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA!")); + test_assert(is_legal_nickname_or_hexdigest("xyzzy")); + test_assert(is_legal_nickname_or_hexdigest("abcdefghijklmnopqrs")); + test_assert(!is_legal_nickname_or_hexdigest("abcdefghijklmnopqrst")); + done: + ; +} + +/** Run unit tests for router descriptor generation logic. */ +static void +test_dir_formats(void) +{ + char buf[8192], buf2[8192]; + char platform[256]; + char fingerprint[FINGERPRINT_LEN+1]; + char *pk1_str = NULL, *pk2_str = NULL, *pk3_str = NULL, *cp; + size_t pk1_str_len, pk2_str_len, pk3_str_len; + routerinfo_t *r1=NULL, *r2=NULL; + crypto_pk_env_t *pk1 = NULL, *pk2 = NULL, *pk3 = NULL; + routerinfo_t *rp1 = NULL; + addr_policy_t *ex1, *ex2; + routerlist_t *dir1 = NULL, *dir2 = NULL; + + pk1 = pk_generate(0); + pk2 = pk_generate(1); + pk3 = pk_generate(2); + + test_assert(pk1 && pk2 && pk3); + + get_platform_str(platform, sizeof(platform)); + r1 = tor_malloc_zero(sizeof(routerinfo_t)); + r1->address = tor_strdup("18.244.0.1"); + r1->addr = 0xc0a80001u; /* 192.168.0.1 */ + r1->cache_info.published_on = 0; + r1->or_port = 9000; + r1->dir_port = 9003; + r1->onion_pkey = crypto_pk_dup_key(pk1); + r1->identity_pkey = crypto_pk_dup_key(pk2); + r1->bandwidthrate = 1000; + r1->bandwidthburst = 5000; + r1->bandwidthcapacity = 10000; + r1->exit_policy = NULL; + r1->nickname = tor_strdup("Magri"); + r1->platform = tor_strdup(platform); + + ex1 = tor_malloc_zero(sizeof(addr_policy_t)); + ex2 = tor_malloc_zero(sizeof(addr_policy_t)); + ex1->policy_type = ADDR_POLICY_ACCEPT; + tor_addr_from_ipv4h(&ex1->addr, 0); + ex1->maskbits = 0; + ex1->prt_min = ex1->prt_max = 80; + ex2->policy_type = ADDR_POLICY_REJECT; + tor_addr_from_ipv4h(&ex2->addr, 18<<24); + ex2->maskbits = 8; + ex2->prt_min = ex2->prt_max = 24; + r2 = tor_malloc_zero(sizeof(routerinfo_t)); + r2->address = tor_strdup("1.1.1.1"); + r2->addr = 0x0a030201u; /* 10.3.2.1 */ + r2->platform = tor_strdup(platform); + r2->cache_info.published_on = 5; + r2->or_port = 9005; + r2->dir_port = 0; + r2->onion_pkey = crypto_pk_dup_key(pk2); + r2->identity_pkey = crypto_pk_dup_key(pk1); + r2->bandwidthrate = r2->bandwidthburst = r2->bandwidthcapacity = 3000; + r2->exit_policy = smartlist_create(); + smartlist_add(r2->exit_policy, ex2); + smartlist_add(r2->exit_policy, ex1); + r2->nickname = tor_strdup("Fred"); + + test_assert(!crypto_pk_write_public_key_to_string(pk1, &pk1_str, + &pk1_str_len)); + test_assert(!crypto_pk_write_public_key_to_string(pk2 , &pk2_str, + &pk2_str_len)); + test_assert(!crypto_pk_write_public_key_to_string(pk3 , &pk3_str, + &pk3_str_len)); + + memset(buf, 0, 2048); + test_assert(router_dump_router_to_string(buf, 2048, r1, pk2)>0); + + strlcpy(buf2, "router Magri 18.244.0.1 9000 0 9003\n" + "platform Tor "VERSION" on ", sizeof(buf2)); + strlcat(buf2, get_uname(), sizeof(buf2)); + strlcat(buf2, "\n" + "opt protocols Link 1 2 Circuit 1\n" + "published 1970-01-01 00:00:00\n" + "opt fingerprint ", sizeof(buf2)); + test_assert(!crypto_pk_get_fingerprint(pk2, fingerprint, 1)); + strlcat(buf2, fingerprint, sizeof(buf2)); + strlcat(buf2, "\nuptime 0\n" + /* XXX the "0" above is hard-coded, but even if we made it reflect + * uptime, that still wouldn't make it right, because the two + * descriptors might be made on different seconds... hm. */ + "bandwidth 1000 5000 10000\n" + "opt extra-info-digest 0000000000000000000000000000000000000000\n" + "onion-key\n", sizeof(buf2)); + strlcat(buf2, pk1_str, sizeof(buf2)); + strlcat(buf2, "signing-key\n", sizeof(buf2)); + strlcat(buf2, pk2_str, sizeof(buf2)); + strlcat(buf2, "opt hidden-service-dir\n", sizeof(buf2)); + strlcat(buf2, "reject *:*\nrouter-signature\n", sizeof(buf2)); + buf[strlen(buf2)] = '\0'; /* Don't compare the sig; it's never the same + * twice */ + + test_streq(buf, buf2); + + test_assert(router_dump_router_to_string(buf, 2048, r1, pk2)>0); + cp = buf; + rp1 = router_parse_entry_from_string((const char*)cp,NULL,1,0,NULL); + test_assert(rp1); + test_streq(rp1->address, r1->address); + test_eq(rp1->or_port, r1->or_port); + //test_eq(rp1->dir_port, r1->dir_port); + test_eq(rp1->bandwidthrate, r1->bandwidthrate); + test_eq(rp1->bandwidthburst, r1->bandwidthburst); + test_eq(rp1->bandwidthcapacity, r1->bandwidthcapacity); + test_assert(crypto_pk_cmp_keys(rp1->onion_pkey, pk1) == 0); + test_assert(crypto_pk_cmp_keys(rp1->identity_pkey, pk2) == 0); + //test_assert(rp1->exit_policy == NULL); + +#if 0 + /* XXX Once we have exit policies, test this again. XXX */ + strlcpy(buf2, "router tor.tor.tor 9005 0 0 3000\n", sizeof(buf2)); + strlcat(buf2, pk2_str, sizeof(buf2)); + strlcat(buf2, "signing-key\n", sizeof(buf2)); + strlcat(buf2, pk1_str, sizeof(buf2)); + strlcat(buf2, "accept *:80\nreject 18.*:24\n\n", sizeof(buf2)); + test_assert(router_dump_router_to_string(buf, 2048, &r2, pk2)>0); + test_streq(buf, buf2); + + cp = buf; + rp2 = router_parse_entry_from_string(&cp,1); + test_assert(rp2); + test_streq(rp2->address, r2.address); + test_eq(rp2->or_port, r2.or_port); + test_eq(rp2->dir_port, r2.dir_port); + test_eq(rp2->bandwidth, r2.bandwidth); + test_assert(crypto_pk_cmp_keys(rp2->onion_pkey, pk2) == 0); + test_assert(crypto_pk_cmp_keys(rp2->identity_pkey, pk1) == 0); + test_eq(rp2->exit_policy->policy_type, EXIT_POLICY_ACCEPT); + test_streq(rp2->exit_policy->string, "accept *:80"); + test_streq(rp2->exit_policy->address, "*"); + test_streq(rp2->exit_policy->port, "80"); + test_eq(rp2->exit_policy->next->policy_type, EXIT_POLICY_REJECT); + test_streq(rp2->exit_policy->next->string, "reject 18.*:24"); + test_streq(rp2->exit_policy->next->address, "18.*"); + test_streq(rp2->exit_policy->next->port, "24"); + test_assert(rp2->exit_policy->next->next == NULL); + + /* Okay, now for the directories. */ + { + fingerprint_list = smartlist_create(); + crypto_pk_get_fingerprint(pk2, buf, 1); + add_fingerprint_to_dir("Magri", buf, fingerprint_list); + crypto_pk_get_fingerprint(pk1, buf, 1); + add_fingerprint_to_dir("Fred", buf, fingerprint_list); + } + + { + char d[DIGEST_LEN]; + const char *m; + /* XXXX NM re-enable. */ + /* Make sure routers aren't too far in the past any more. */ + r1->cache_info.published_on = time(NULL); + r2->cache_info.published_on = time(NULL)-3*60*60; + test_assert(router_dump_router_to_string(buf, 2048, r1, pk2)>0); + test_eq(dirserv_add_descriptor(buf,&m,""), ROUTER_ADDED_NOTIFY_GENERATOR); + test_assert(router_dump_router_to_string(buf, 2048, r2, pk1)>0); + test_eq(dirserv_add_descriptor(buf,&m,""), ROUTER_ADDED_NOTIFY_GENERATOR); + get_options()->Nickname = tor_strdup("DirServer"); + test_assert(!dirserv_dump_directory_to_string(&cp,pk3, 0)); + crypto_pk_get_digest(pk3, d); + test_assert(!router_parse_directory(cp)); + test_eq(2, smartlist_len(dir1->routers)); + tor_free(cp); + } +#endif + dirserv_free_fingerprint_list(); + + done: + if (r1) + routerinfo_free(r1); + if (r2) + routerinfo_free(r2); + + tor_free(pk1_str); + tor_free(pk2_str); + tor_free(pk3_str); + if (pk1) crypto_free_pk_env(pk1); + if (pk2) crypto_free_pk_env(pk2); + if (pk3) crypto_free_pk_env(pk3); + if (rp1) routerinfo_free(rp1); + tor_free(dir1); /* XXXX And more !*/ + tor_free(dir2); /* And more !*/ +} + +static void +test_dir_versions(void) +{ + tor_version_t ver1; + + /* Try out version parsing functionality */ + test_eq(0, tor_version_parse("0.3.4pre2-cvs", &ver1)); + test_eq(0, ver1.major); + test_eq(3, ver1.minor); + test_eq(4, ver1.micro); + test_eq(VER_PRE, ver1.status); + test_eq(2, ver1.patchlevel); + test_eq(0, tor_version_parse("0.3.4rc1", &ver1)); + test_eq(0, ver1.major); + test_eq(3, ver1.minor); + test_eq(4, ver1.micro); + test_eq(VER_RC, ver1.status); + test_eq(1, ver1.patchlevel); + test_eq(0, tor_version_parse("1.3.4", &ver1)); + test_eq(1, ver1.major); + test_eq(3, ver1.minor); + test_eq(4, ver1.micro); + test_eq(VER_RELEASE, ver1.status); + test_eq(0, ver1.patchlevel); + test_eq(0, tor_version_parse("1.3.4.999", &ver1)); + test_eq(1, ver1.major); + test_eq(3, ver1.minor); + test_eq(4, ver1.micro); + test_eq(VER_RELEASE, ver1.status); + test_eq(999, ver1.patchlevel); + test_eq(0, tor_version_parse("0.1.2.4-alpha", &ver1)); + test_eq(0, ver1.major); + test_eq(1, ver1.minor); + test_eq(2, ver1.micro); + test_eq(4, ver1.patchlevel); + test_eq(VER_RELEASE, ver1.status); + test_streq("alpha", ver1.status_tag); + test_eq(0, tor_version_parse("0.1.2.4", &ver1)); + test_eq(0, ver1.major); + test_eq(1, ver1.minor); + test_eq(2, ver1.micro); + test_eq(4, ver1.patchlevel); + test_eq(VER_RELEASE, ver1.status); + test_streq("", ver1.status_tag); + +#define tt_versionstatus_op(vs1, op, vs2) \ + tt_assert_test_type(vs1,vs2,#vs1" "#op" "#vs2,version_status_t, \ + (_val1 op _val2),"%d") +#define test_v_i_o(val, ver, lst) \ + tt_versionstatus_op(val, ==, tor_version_is_obsolete(ver, lst)) + + /* make sure tor_version_is_obsolete() works */ + test_v_i_o(VS_OLD, "0.0.1", "Tor 0.0.2"); + test_v_i_o(VS_OLD, "0.0.1", "0.0.2, Tor 0.0.3"); + test_v_i_o(VS_OLD, "0.0.1", "0.0.2,Tor 0.0.3"); + test_v_i_o(VS_OLD, "0.0.1","0.0.3,BetterTor 0.0.1"); + test_v_i_o(VS_RECOMMENDED, "0.0.2", "Tor 0.0.2,Tor 0.0.3"); + test_v_i_o(VS_NEW_IN_SERIES, "0.0.2", "Tor 0.0.2pre1,Tor 0.0.3"); + test_v_i_o(VS_OLD, "0.0.2", "Tor 0.0.2.1,Tor 0.0.3"); + test_v_i_o(VS_NEW, "0.1.0", "Tor 0.0.2,Tor 0.0.3"); + test_v_i_o(VS_RECOMMENDED, "0.0.7rc2", "0.0.7,Tor 0.0.7rc2,Tor 0.0.8"); + test_v_i_o(VS_OLD, "0.0.5.0", "0.0.5.1-cvs"); + test_v_i_o(VS_NEW_IN_SERIES, "0.0.5.1-cvs", "0.0.5, 0.0.6"); + /* Not on list, but newer than any in same series. */ + test_v_i_o(VS_NEW_IN_SERIES, "0.1.0.3", + "Tor 0.1.0.2,Tor 0.0.9.5,Tor 0.1.1.0"); + /* Series newer than any on list. */ + test_v_i_o(VS_NEW, "0.1.2.3", "Tor 0.1.0.2,Tor 0.0.9.5,Tor 0.1.1.0"); + /* Series older than any on list. */ + test_v_i_o(VS_OLD, "0.0.1.3", "Tor 0.1.0.2,Tor 0.0.9.5,Tor 0.1.1.0"); + /* Not on list, not newer than any on same series. */ + test_v_i_o(VS_UNRECOMMENDED, "0.1.0.1", + "Tor 0.1.0.2,Tor 0.0.9.5,Tor 0.1.1.0"); + /* On list, not newer than any on same series. */ + test_v_i_o(VS_UNRECOMMENDED, + "0.1.0.1", "Tor 0.1.0.2,Tor 0.0.9.5,Tor 0.1.1.0"); + test_eq(0, tor_version_as_new_as("Tor 0.0.5", "0.0.9pre1-cvs")); + test_eq(1, tor_version_as_new_as( + "Tor 0.0.8 on Darwin 64-121-192-100.c3-0." + "sfpo-ubr1.sfrn-sfpo.ca.cable.rcn.com Power Macintosh", + "0.0.8rc2")); + test_eq(0, tor_version_as_new_as( + "Tor 0.0.8 on Darwin 64-121-192-100.c3-0." + "sfpo-ubr1.sfrn-sfpo.ca.cable.rcn.com Power Macintosh", "0.0.8.2")); + + /* Now try svn revisions. */ + test_eq(1, tor_version_as_new_as("Tor 0.2.1.0-dev (r100)", + "Tor 0.2.1.0-dev (r99)")); + test_eq(1, tor_version_as_new_as("Tor 0.2.1.0-dev (r100) on Banana Jr", + "Tor 0.2.1.0-dev (r99) on Hal 9000")); + test_eq(1, tor_version_as_new_as("Tor 0.2.1.0-dev (r100)", + "Tor 0.2.1.0-dev on Colossus")); + test_eq(0, tor_version_as_new_as("Tor 0.2.1.0-dev (r99)", + "Tor 0.2.1.0-dev (r100)")); + test_eq(0, tor_version_as_new_as("Tor 0.2.1.0-dev (r99) on MCP", + "Tor 0.2.1.0-dev (r100) on AM")); + test_eq(0, tor_version_as_new_as("Tor 0.2.1.0-dev", + "Tor 0.2.1.0-dev (r99)")); + test_eq(1, tor_version_as_new_as("Tor 0.2.1.1", + "Tor 0.2.1.0-dev (r99)")); + + /* Now try git revisions */ + test_eq(0, tor_version_parse("0.5.6.7 (git-ff00ff)", &ver1)); + test_eq(0, ver1.major); + test_eq(5, ver1.minor); + test_eq(6, ver1.micro); + test_eq(7, ver1.patchlevel); + test_eq(3, ver1.git_tag_len); + test_memeq(ver1.git_tag, "\xff\x00\xff", 3); + test_eq(-1, tor_version_parse("0.5.6.7 (git-ff00xx)", &ver1)); + test_eq(-1, tor_version_parse("0.5.6.7 (git-ff00fff)", &ver1)); + test_eq(0, tor_version_parse("0.5.6.7 (git ff00fff)", &ver1)); + done: + ; +} + +/** Run unit tests for directory fp_pair functions. */ +static void +test_dir_fp_pairs(void) +{ + smartlist_t *sl = smartlist_create(); + fp_pair_t *pair; + + dir_split_resource_into_fingerprint_pairs( + /* Two pairs, out of order, with one duplicate. */ + "73656372657420646174612E0000000000FFFFFF-" + "557365204145532d32353620696e73746561642e+" + "73656372657420646174612E0000000000FFFFFF-" + "557365204145532d32353620696e73746561642e+" + "48657861646563696d616c2069736e277420736f-" + "676f6f6420666f7220686964696e6720796f7572.z", sl); + + test_eq(smartlist_len(sl), 2); + pair = smartlist_get(sl, 0); + test_memeq(pair->first, "Hexadecimal isn't so", DIGEST_LEN); + test_memeq(pair->second, "good for hiding your", DIGEST_LEN); + pair = smartlist_get(sl, 1); + test_memeq(pair->first, "secret data.\0\0\0\0\0\xff\xff\xff", DIGEST_LEN); + test_memeq(pair->second, "Use AES-256 instead.", DIGEST_LEN); + + done: + SMARTLIST_FOREACH(sl, fp_pair_t *, pair, tor_free(pair)); + smartlist_free(sl); +} + +static void +test_dir_split_fps(void *testdata) +{ + smartlist_t *sl = smartlist_create(); + char *mem_op_hex_tmp = NULL; + (void)testdata; + + /* Some example hex fingerprints and their base64 equivalents */ +#define HEX1 "Fe0daff89127389bc67558691231234551193EEE" +#define HEX2 "Deadbeef99999991111119999911111111f00ba4" +#define HEX3 "b33ff00db33ff00db33ff00db33ff00db33ff00d" +#define HEX256_1 \ + "f3f3f3f3fbbbbf3f3f3f3fbbbf3f3f3f3fbbbbf3f3f3f3fbbbf3f3f3f3fbbbbf" +#define HEX256_2 \ + "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccCCc" +#define HEX256_3 \ + "0123456789ABCdef0123456789ABCdef0123456789ABCdef0123456789ABCdef" +#define B64_1 "/g2v+JEnOJvGdVhpEjEjRVEZPu4" +#define B64_2 "3q2+75mZmZERERmZmRERERHwC6Q" +#define B64_3 "sz/wDbM/8A2zP/ANsz/wDbM/8A0" +#define B64_256_1 "8/Pz8/u7vz8/Pz+7vz8/Pz+7u/Pz8/P7u/Pz8/P7u78" +#define B64_256_2 "zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMw" +#define B64_256_3 "ASNFZ4mrze8BI0VniavN7wEjRWeJq83vASNFZ4mrze8" + + /* no flags set */ + dir_split_resource_into_fingerprints("A+C+B", sl, NULL, 0); + tt_int_op(smartlist_len(sl), ==, 3); + tt_str_op(smartlist_get(sl, 0), ==, "A"); + tt_str_op(smartlist_get(sl, 1), ==, "C"); + tt_str_op(smartlist_get(sl, 2), ==, "B"); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + /* uniq strings. */ + dir_split_resource_into_fingerprints("A+C+B+A+B+B", sl, NULL, DSR_SORT_UNIQ); + tt_int_op(smartlist_len(sl), ==, 3); + tt_str_op(smartlist_get(sl, 0), ==, "A"); + tt_str_op(smartlist_get(sl, 1), ==, "B"); + tt_str_op(smartlist_get(sl, 2), ==, "C"); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + /* Decode hex. */ + dir_split_resource_into_fingerprints(HEX1"+"HEX2, sl, NULL, DSR_HEX); + tt_int_op(smartlist_len(sl), ==, 2); + test_mem_op_hex(smartlist_get(sl, 0), ==, HEX1); + test_mem_op_hex(smartlist_get(sl, 1), ==, HEX2); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + /* decode hex and drop weirdness. */ + dir_split_resource_into_fingerprints(HEX1"+bogus+"HEX2"+"HEX256_1, + sl, NULL, DSR_HEX); + tt_int_op(smartlist_len(sl), ==, 2); + test_mem_op_hex(smartlist_get(sl, 0), ==, HEX1); + test_mem_op_hex(smartlist_get(sl, 1), ==, HEX2); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + /* Decode long hex */ + dir_split_resource_into_fingerprints(HEX256_1"+"HEX256_2"+"HEX2"+"HEX256_3, + sl, NULL, DSR_HEX|DSR_DIGEST256); + tt_int_op(smartlist_len(sl), ==, 3); + test_mem_op_hex(smartlist_get(sl, 0), ==, HEX256_1); + test_mem_op_hex(smartlist_get(sl, 1), ==, HEX256_2); + test_mem_op_hex(smartlist_get(sl, 2), ==, HEX256_3); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + /* Decode hex and sort. */ + dir_split_resource_into_fingerprints(HEX1"+"HEX2"+"HEX3"+"HEX2, + sl, NULL, DSR_HEX|DSR_SORT_UNIQ); + tt_int_op(smartlist_len(sl), ==, 3); + test_mem_op_hex(smartlist_get(sl, 0), ==, HEX3); + test_mem_op_hex(smartlist_get(sl, 1), ==, HEX2); + test_mem_op_hex(smartlist_get(sl, 2), ==, HEX1); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + /* Decode long hex and sort */ + dir_split_resource_into_fingerprints(HEX256_1"+"HEX256_2"+"HEX256_3 + "+"HEX256_1, + sl, NULL, + DSR_HEX|DSR_DIGEST256|DSR_SORT_UNIQ); + tt_int_op(smartlist_len(sl), ==, 3); + test_mem_op_hex(smartlist_get(sl, 0), ==, HEX256_3); + test_mem_op_hex(smartlist_get(sl, 1), ==, HEX256_2); + test_mem_op_hex(smartlist_get(sl, 2), ==, HEX256_1); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + /* Decode base64 */ + dir_split_resource_into_fingerprints(B64_1"-"B64_2, sl, NULL, DSR_BASE64); + tt_int_op(smartlist_len(sl), ==, 2); + test_mem_op_hex(smartlist_get(sl, 0), ==, HEX1); + test_mem_op_hex(smartlist_get(sl, 1), ==, HEX2); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + /* Decode long base64 */ + dir_split_resource_into_fingerprints(B64_256_1"-"B64_256_2, + sl, NULL, DSR_BASE64|DSR_DIGEST256); + tt_int_op(smartlist_len(sl), ==, 2); + test_mem_op_hex(smartlist_get(sl, 0), ==, HEX256_1); + test_mem_op_hex(smartlist_get(sl, 1), ==, HEX256_2); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + dir_split_resource_into_fingerprints(B64_256_1, + sl, NULL, DSR_BASE64|DSR_DIGEST256); + tt_int_op(smartlist_len(sl), ==, 1); + test_mem_op_hex(smartlist_get(sl, 0), ==, HEX256_1); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + done: + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_free(sl); + tor_free(mem_op_hex_tmp); +} + +static void +test_dir_measured_bw(void) +{ + measured_bw_line_t mbwl; + int i; + const char *lines_pass[] = { + "node_id=$557365204145532d32353620696e73746561642e bw=1024\n", + "node_id=$557365204145532d32353620696e73746561642e\t bw=1024 \n", + " node_id=$557365204145532d32353620696e73746561642e bw=1024\n", + "\tnoise\tnode_id=$557365204145532d32353620696e73746561642e " + "bw=1024 junk=007\n", + "misc=junk node_id=$557365204145532d32353620696e73746561642e " + "bw=1024 junk=007\n", + "end" + }; + const char *lines_fail[] = { + /* Test possible python stupidity on input */ + "node_id=None bw=1024\n", + "node_id=$None bw=1024\n", + "node_id=$557365204145532d32353620696e73746561642e bw=None\n", + "node_id=$557365204145532d32353620696e73746561642e bw=1024.0\n", + "node_id=$557365204145532d32353620696e73746561642e bw=.1024\n", + "node_id=$557365204145532d32353620696e73746561642e bw=1.024\n", + "node_id=$557365204145532d32353620696e73746561642e bw=1024 bw=0\n", + "node_id=$557365204145532d32353620696e73746561642e bw=1024 bw=None\n", + "node_id=$557365204145532d32353620696e73746561642e bw=-1024\n", + /* Test incomplete writes due to race conditions, partial copies, etc */ + "node_i", + "node_i\n", + "node_id=", + "node_id=\n", + "node_id=$557365204145532d32353620696e73746561642e bw=", + "node_id=$557365204145532d32353620696e73746561642e bw=1024", + "node_id=$557365204145532d32353620696e73746561642e bw=\n", + "node_id=$557365204145532d32353620696e7374", + "node_id=$557365204145532d32353620696e7374\n", + "", + "\n", + " \n ", + " \n\n", + /* Test assorted noise */ + " node_id= ", + "node_id==$557365204145532d32353620696e73746561642e bw==1024\n", + "node_id=$55736520414552d32353620696e73746561642e bw=1024\n", + "node_id=557365204145532d32353620696e73746561642e bw=1024\n", + "node_id= $557365204145532d32353620696e73746561642e bw=0.23\n", + "end" + }; + + for (i = 0; strcmp(lines_fail[i], "end"); i++) { + //fprintf(stderr, "Testing: %s\n", lines_fail[i]); + test_assert(measured_bw_line_parse(&mbwl, lines_fail[i]) == -1); + } + + for (i = 0; strcmp(lines_pass[i], "end"); i++) { + //fprintf(stderr, "Testing: %s %d\n", lines_pass[i], TOR_ISSPACE('\n')); + test_assert(measured_bw_line_parse(&mbwl, lines_pass[i]) == 0); + test_assert(mbwl.bw == 1024); + test_assert(strcmp(mbwl.node_hex, + "557365204145532d32353620696e73746561642e") == 0); + } + +done: + return; +} + +static void +test_dir_param_voting(void) +{ + networkstatus_t vote1, vote2, vote3, vote4; + smartlist_t *votes = smartlist_create(); + char *res = NULL; + + /* dirvote_compute_params only looks at the net_params field of the votes, + so that's all we need to set. + */ + memset(&vote1, 0, sizeof(vote1)); + memset(&vote2, 0, sizeof(vote2)); + memset(&vote3, 0, sizeof(vote3)); + memset(&vote4, 0, sizeof(vote4)); + vote1.net_params = smartlist_create(); + vote2.net_params = smartlist_create(); + vote3.net_params = smartlist_create(); + vote4.net_params = smartlist_create(); + smartlist_split_string(vote1.net_params, + "ab=90 abcd=20 cw=50 x-yz=-99", NULL, 0, 0); + smartlist_split_string(vote2.net_params, + "ab=27 cw=5 x-yz=88", NULL, 0, 0); + smartlist_split_string(vote3.net_params, + "abcd=20 c=60 cw=500 x-yz=-9 zzzzz=101", NULL, 0, 0); + smartlist_split_string(vote4.net_params, + "ab=900 abcd=200 c=1 cw=51 x-yz=100", NULL, 0, 0); + test_eq(100, networkstatus_get_param(&vote4, "x-yz", 50)); + test_eq(222, networkstatus_get_param(&vote4, "foobar", 222)); + + smartlist_add(votes, &vote1); + smartlist_add(votes, &vote2); + smartlist_add(votes, &vote3); + smartlist_add(votes, &vote4); + + res = dirvote_compute_params(votes); + test_streq(res, + "ab=90 abcd=20 c=1 cw=50 x-yz=-9 zzzzz=101"); + + done: + tor_free(res); + SMARTLIST_FOREACH(vote1.net_params, char *, cp, tor_free(cp)); + SMARTLIST_FOREACH(vote2.net_params, char *, cp, tor_free(cp)); + SMARTLIST_FOREACH(vote3.net_params, char *, cp, tor_free(cp)); + SMARTLIST_FOREACH(vote4.net_params, char *, cp, tor_free(cp)); + smartlist_free(vote1.net_params); + smartlist_free(vote2.net_params); + smartlist_free(vote3.net_params); + smartlist_free(vote4.net_params); + smartlist_free(votes); + + return; +} + +extern const char AUTHORITY_CERT_1[]; +extern const char AUTHORITY_SIGNKEY_1[]; +extern const char AUTHORITY_CERT_2[]; +extern const char AUTHORITY_SIGNKEY_2[]; +extern const char AUTHORITY_CERT_3[]; +extern const char AUTHORITY_SIGNKEY_3[]; + +/** Helper: Test that two networkstatus_voter_info_t do in fact represent the + * same voting authority, and that they do in fact have all the same + * information. */ +static void +test_same_voter(networkstatus_voter_info_t *v1, + networkstatus_voter_info_t *v2) +{ + test_streq(v1->nickname, v2->nickname); + test_memeq(v1->identity_digest, v2->identity_digest, DIGEST_LEN); + test_streq(v1->address, v2->address); + test_eq(v1->addr, v2->addr); + test_eq(v1->dir_port, v2->dir_port); + test_eq(v1->or_port, v2->or_port); + test_streq(v1->contact, v2->contact); + test_memeq(v1->vote_digest, v2->vote_digest, DIGEST_LEN); + done: + ; +} + +/** Helper: Make a new routerinfo containing the right information for a + * given vote_routerstatus_t. */ +static routerinfo_t * +generate_ri_from_rs(const vote_routerstatus_t *vrs) +{ + routerinfo_t *r; + const routerstatus_t *rs = &vrs->status; + static time_t published = 0; + + r = tor_malloc_zero(sizeof(routerinfo_t)); + memcpy(r->cache_info.identity_digest, rs->identity_digest, DIGEST_LEN); + memcpy(r->cache_info.signed_descriptor_digest, rs->descriptor_digest, + DIGEST_LEN); + r->cache_info.do_not_cache = 1; + r->cache_info.routerlist_index = -1; + r->cache_info.signed_descriptor_body = + tor_strdup("123456789012345678901234567890123"); + r->cache_info.signed_descriptor_len = + strlen(r->cache_info.signed_descriptor_body); + r->exit_policy = smartlist_create(); + r->cache_info.published_on = ++published + time(NULL); + return r; +} + +/** Helper: get a detached signatures document for one or two + * consensuses. */ +static char * +get_detached_sigs(networkstatus_t *ns, networkstatus_t *ns2) +{ + char *r; + smartlist_t *sl; + tor_assert(ns && ns->flavor == FLAV_NS); + sl = smartlist_create(); + smartlist_add(sl,ns); + if (ns2) + smartlist_add(sl,ns2); + r = networkstatus_get_detached_signatures(sl); + smartlist_free(sl); + return r; +} + +/** Run unit tests for generating and parsing V3 consensus networkstatus + * documents. */ +static void +test_dir_v3_networkstatus(void) +{ + authority_cert_t *cert1=NULL, *cert2=NULL, *cert3=NULL; + crypto_pk_env_t *sign_skey_1=NULL, *sign_skey_2=NULL, *sign_skey_3=NULL; + crypto_pk_env_t *sign_skey_leg1=NULL; + const char *msg=NULL; + + time_t now = time(NULL); + networkstatus_voter_info_t *voter; + document_signature_t *sig; + networkstatus_t *vote=NULL, *v1=NULL, *v2=NULL, *v3=NULL, *con=NULL, + *con_md=NULL; + vote_routerstatus_t *vrs; + routerstatus_t *rs; + char *v1_text=NULL, *v2_text=NULL, *v3_text=NULL, *consensus_text=NULL, *cp; + smartlist_t *votes = smartlist_create(); + + /* For generating the two other consensuses. */ + char *detached_text1=NULL, *detached_text2=NULL; + char *consensus_text2=NULL, *consensus_text3=NULL; + char *consensus_text_md2=NULL, *consensus_text_md3=NULL; + char *consensus_text_md=NULL; + networkstatus_t *con2=NULL, *con_md2=NULL, *con3=NULL, *con_md3=NULL; + ns_detached_signatures_t *dsig1=NULL, *dsig2=NULL; + + /* Parse certificates and keys. */ + cert1 = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); + test_assert(cert1); + test_assert(cert1->is_cross_certified); + cert2 = authority_cert_parse_from_string(AUTHORITY_CERT_2, NULL); + test_assert(cert2); + cert3 = authority_cert_parse_from_string(AUTHORITY_CERT_3, NULL); + test_assert(cert3); + sign_skey_1 = crypto_new_pk_env(); + sign_skey_2 = crypto_new_pk_env(); + sign_skey_3 = crypto_new_pk_env(); + sign_skey_leg1 = pk_generate(4); + + test_assert(!crypto_pk_read_private_key_from_string(sign_skey_1, + AUTHORITY_SIGNKEY_1)); + test_assert(!crypto_pk_read_private_key_from_string(sign_skey_2, + AUTHORITY_SIGNKEY_2)); + test_assert(!crypto_pk_read_private_key_from_string(sign_skey_3, + AUTHORITY_SIGNKEY_3)); + + test_assert(!crypto_pk_cmp_keys(sign_skey_1, cert1->signing_key)); + test_assert(!crypto_pk_cmp_keys(sign_skey_2, cert2->signing_key)); + + /* + * Set up a vote; generate it; try to parse it. + */ + vote = tor_malloc_zero(sizeof(networkstatus_t)); + vote->type = NS_TYPE_VOTE; + vote->published = now; + vote->valid_after = now+1000; + vote->fresh_until = now+2000; + vote->valid_until = now+3000; + vote->vote_seconds = 100; + vote->dist_seconds = 200; + vote->supported_methods = smartlist_create(); + smartlist_split_string(vote->supported_methods, "1 2 3", NULL, 0, -1); + vote->client_versions = tor_strdup("0.1.2.14,0.1.2.15"); + vote->server_versions = tor_strdup("0.1.2.14,0.1.2.15,0.1.2.16"); + vote->known_flags = smartlist_create(); + smartlist_split_string(vote->known_flags, + "Authority Exit Fast Guard Running Stable V2Dir Valid", + 0, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + vote->voters = smartlist_create(); + voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t)); + voter->nickname = tor_strdup("Voter1"); + voter->address = tor_strdup("1.2.3.4"); + voter->addr = 0x01020304; + voter->dir_port = 80; + voter->or_port = 9000; + voter->contact = tor_strdup("voter@example.com"); + crypto_pk_get_digest(cert1->identity_key, voter->identity_digest); + smartlist_add(vote->voters, voter); + vote->cert = authority_cert_dup(cert1); + vote->net_params = smartlist_create(); + smartlist_split_string(vote->net_params, "circuitwindow=101 foo=990", + NULL, 0, 0); + vote->routerstatus_list = smartlist_create(); + /* add the first routerstatus. */ + vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); + rs = &vrs->status; + vrs->version = tor_strdup("0.1.2.14"); + rs->published_on = now-1500; + strlcpy(rs->nickname, "router2", sizeof(rs->nickname)); + memset(rs->identity_digest, 3, DIGEST_LEN); + memset(rs->descriptor_digest, 78, DIGEST_LEN); + rs->addr = 0x99008801; + rs->or_port = 443; + rs->dir_port = 8000; + /* all flags but running cleared */ + rs->is_running = 1; + smartlist_add(vote->routerstatus_list, vrs); + test_assert(router_add_to_routerlist(generate_ri_from_rs(vrs), &msg,0,0)>=0); + + /* add the second routerstatus. */ + vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); + rs = &vrs->status; + vrs->version = tor_strdup("0.2.0.5"); + rs->published_on = now-1000; + strlcpy(rs->nickname, "router1", sizeof(rs->nickname)); + memset(rs->identity_digest, 5, DIGEST_LEN); + memset(rs->descriptor_digest, 77, DIGEST_LEN); + rs->addr = 0x99009901; + rs->or_port = 443; + rs->dir_port = 0; + rs->is_exit = rs->is_stable = rs->is_fast = rs->is_running = + rs->is_valid = rs->is_v2_dir = rs->is_possible_guard = 1; + smartlist_add(vote->routerstatus_list, vrs); + test_assert(router_add_to_routerlist(generate_ri_from_rs(vrs), &msg,0,0)>=0); + + /* add the third routerstatus. */ + vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); + rs = &vrs->status; + vrs->version = tor_strdup("0.1.0.3"); + rs->published_on = now-1000; + strlcpy(rs->nickname, "router3", sizeof(rs->nickname)); + memset(rs->identity_digest, 33, DIGEST_LEN); + memset(rs->descriptor_digest, 79, DIGEST_LEN); + rs->addr = 0xAA009901; + rs->or_port = 400; + rs->dir_port = 9999; + rs->is_authority = rs->is_exit = rs->is_stable = rs->is_fast = + rs->is_running = rs->is_valid = rs->is_v2_dir = rs->is_possible_guard = 1; + smartlist_add(vote->routerstatus_list, vrs); + test_assert(router_add_to_routerlist(generate_ri_from_rs(vrs), &msg,0,0)>=0); + + /* add a fourth routerstatus that is not running. */ + vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); + rs = &vrs->status; + vrs->version = tor_strdup("0.1.6.3"); + rs->published_on = now-1000; + strlcpy(rs->nickname, "router4", sizeof(rs->nickname)); + memset(rs->identity_digest, 34, DIGEST_LEN); + memset(rs->descriptor_digest, 47, DIGEST_LEN); + rs->addr = 0xC0000203; + rs->or_port = 500; + rs->dir_port = 1999; + /* Running flag (and others) cleared */ + smartlist_add(vote->routerstatus_list, vrs); + test_assert(router_add_to_routerlist(generate_ri_from_rs(vrs), &msg,0,0)>=0); + + /* dump the vote and try to parse it. */ + v1_text = format_networkstatus_vote(sign_skey_1, vote); + test_assert(v1_text); + v1 = networkstatus_parse_vote_from_string(v1_text, NULL, NS_TYPE_VOTE); + test_assert(v1); + + /* Make sure the parsed thing was right. */ + test_eq(v1->type, NS_TYPE_VOTE); + test_eq(v1->published, vote->published); + test_eq(v1->valid_after, vote->valid_after); + test_eq(v1->fresh_until, vote->fresh_until); + test_eq(v1->valid_until, vote->valid_until); + test_eq(v1->vote_seconds, vote->vote_seconds); + test_eq(v1->dist_seconds, vote->dist_seconds); + test_streq(v1->client_versions, vote->client_versions); + test_streq(v1->server_versions, vote->server_versions); + test_assert(v1->voters && smartlist_len(v1->voters)); + voter = smartlist_get(v1->voters, 0); + test_streq(voter->nickname, "Voter1"); + test_streq(voter->address, "1.2.3.4"); + test_eq(voter->addr, 0x01020304); + test_eq(voter->dir_port, 80); + test_eq(voter->or_port, 9000); + test_streq(voter->contact, "voter@example.com"); + test_assert(v1->cert); + test_assert(!crypto_pk_cmp_keys(sign_skey_1, v1->cert->signing_key)); + cp = smartlist_join_strings(v1->known_flags, ":", 0, NULL); + test_streq(cp, "Authority:Exit:Fast:Guard:Running:Stable:V2Dir:Valid"); + tor_free(cp); + test_eq(smartlist_len(v1->routerstatus_list), 4); + /* Check the first routerstatus. */ + vrs = smartlist_get(v1->routerstatus_list, 0); + rs = &vrs->status; + test_streq(vrs->version, "0.1.2.14"); + test_eq(rs->published_on, now-1500); + test_streq(rs->nickname, "router2"); + test_memeq(rs->identity_digest, + "\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3", + DIGEST_LEN); + test_memeq(rs->descriptor_digest, "NNNNNNNNNNNNNNNNNNNN", DIGEST_LEN); + test_eq(rs->addr, 0x99008801); + test_eq(rs->or_port, 443); + test_eq(rs->dir_port, 8000); + test_eq(vrs->flags, U64_LITERAL(16)); // no flags except "running" + /* Check the second routerstatus. */ + vrs = smartlist_get(v1->routerstatus_list, 1); + rs = &vrs->status; + test_streq(vrs->version, "0.2.0.5"); + test_eq(rs->published_on, now-1000); + test_streq(rs->nickname, "router1"); + test_memeq(rs->identity_digest, + "\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5", + DIGEST_LEN); + test_memeq(rs->descriptor_digest, "MMMMMMMMMMMMMMMMMMMM", DIGEST_LEN); + test_eq(rs->addr, 0x99009901); + test_eq(rs->or_port, 443); + test_eq(rs->dir_port, 0); + test_eq(vrs->flags, U64_LITERAL(254)); // all flags except "authority." + + { + measured_bw_line_t mbw; + memset(mbw.node_id, 33, sizeof(mbw.node_id)); + mbw.bw = 1024; + test_assert(measured_bw_line_apply(&mbw, + v1->routerstatus_list) == 1); + vrs = smartlist_get(v1->routerstatus_list, 2); + test_assert(vrs->status.has_measured_bw && + vrs->status.measured_bw == 1024); + } + + /* Generate second vote. It disagrees on some of the times, + * and doesn't list versions, and knows some crazy flags */ + vote->published = now+1; + vote->fresh_until = now+3005; + vote->dist_seconds = 300; + authority_cert_free(vote->cert); + vote->cert = authority_cert_dup(cert2); + vote->net_params = smartlist_create(); + smartlist_split_string(vote->net_params, "bar=2000000000 circuitwindow=20", + NULL, 0, 0); + tor_free(vote->client_versions); + tor_free(vote->server_versions); + voter = smartlist_get(vote->voters, 0); + tor_free(voter->nickname); + tor_free(voter->address); + voter->nickname = tor_strdup("Voter2"); + voter->address = tor_strdup("2.3.4.5"); + voter->addr = 0x02030405; + crypto_pk_get_digest(cert2->identity_key, voter->identity_digest); + smartlist_add(vote->known_flags, tor_strdup("MadeOfCheese")); + smartlist_add(vote->known_flags, tor_strdup("MadeOfTin")); + smartlist_sort_strings(vote->known_flags); + vrs = smartlist_get(vote->routerstatus_list, 2); + smartlist_del_keeporder(vote->routerstatus_list, 2); + tor_free(vrs->version); + tor_free(vrs); + vrs = smartlist_get(vote->routerstatus_list, 0); + vrs->status.is_fast = 1; + /* generate and parse. */ + v2_text = format_networkstatus_vote(sign_skey_2, vote); + test_assert(v2_text); + v2 = networkstatus_parse_vote_from_string(v2_text, NULL, NS_TYPE_VOTE); + test_assert(v2); + /* Check that flags come out right.*/ + cp = smartlist_join_strings(v2->known_flags, ":", 0, NULL); + test_streq(cp, "Authority:Exit:Fast:Guard:MadeOfCheese:MadeOfTin:" + "Running:Stable:V2Dir:Valid"); + tor_free(cp); + vrs = smartlist_get(v2->routerstatus_list, 1); + /* 1023 - authority(1) - madeofcheese(16) - madeoftin(32) */ + test_eq(vrs->flags, U64_LITERAL(974)); + + /* Generate the third vote. */ + vote->published = now; + vote->fresh_until = now+2003; + vote->dist_seconds = 250; + authority_cert_free(vote->cert); + vote->cert = authority_cert_dup(cert3); + vote->net_params = smartlist_create(); + smartlist_split_string(vote->net_params, "circuitwindow=80 foo=660", + NULL, 0, 0); + smartlist_add(vote->supported_methods, tor_strdup("4")); + vote->client_versions = tor_strdup("0.1.2.14,0.1.2.17"); + vote->server_versions = tor_strdup("0.1.2.10,0.1.2.15,0.1.2.16"); + voter = smartlist_get(vote->voters, 0); + tor_free(voter->nickname); + tor_free(voter->address); + voter->nickname = tor_strdup("Voter3"); + voter->address = tor_strdup("3.4.5.6"); + voter->addr = 0x03040506; + crypto_pk_get_digest(cert3->identity_key, voter->identity_digest); + /* This one has a legacy id. */ + memset(voter->legacy_id_digest, (int)'A', DIGEST_LEN); + vrs = smartlist_get(vote->routerstatus_list, 0); + smartlist_del_keeporder(vote->routerstatus_list, 0); + tor_free(vrs->version); + tor_free(vrs); + vrs = smartlist_get(vote->routerstatus_list, 0); + memset(vrs->status.descriptor_digest, (int)'Z', DIGEST_LEN); + test_assert(router_add_to_routerlist(generate_ri_from_rs(vrs), &msg,0,0)>=0); + + v3_text = format_networkstatus_vote(sign_skey_3, vote); + test_assert(v3_text); + + v3 = networkstatus_parse_vote_from_string(v3_text, NULL, NS_TYPE_VOTE); + test_assert(v3); + + /* Compute a consensus as voter 3. */ + smartlist_add(votes, v3); + smartlist_add(votes, v1); + smartlist_add(votes, v2); + consensus_text = networkstatus_compute_consensus(votes, 3, + cert3->identity_key, + sign_skey_3, + "AAAAAAAAAAAAAAAAAAAA", + sign_skey_leg1, + FLAV_NS); + test_assert(consensus_text); + con = networkstatus_parse_vote_from_string(consensus_text, NULL, + NS_TYPE_CONSENSUS); + test_assert(con); + //log_notice(LD_GENERAL, "<<%s>>\n<<%s>>\n<<%s>>\n", + // v1_text, v2_text, v3_text); + consensus_text_md = networkstatus_compute_consensus(votes, 3, + cert3->identity_key, + sign_skey_3, + "AAAAAAAAAAAAAAAAAAAA", + sign_skey_leg1, + FLAV_MICRODESC); + test_assert(consensus_text_md); + con_md = networkstatus_parse_vote_from_string(consensus_text_md, NULL, + NS_TYPE_CONSENSUS); + test_assert(con_md); + test_eq(con_md->flavor, FLAV_MICRODESC); + + /* Check consensus contents. */ + test_assert(con->type == NS_TYPE_CONSENSUS); + test_eq(con->published, 0); /* this field only appears in votes. */ + test_eq(con->valid_after, now+1000); + test_eq(con->fresh_until, now+2003); /* median */ + test_eq(con->valid_until, now+3000); + test_eq(con->vote_seconds, 100); + test_eq(con->dist_seconds, 250); /* median */ + test_streq(con->client_versions, "0.1.2.14"); + test_streq(con->server_versions, "0.1.2.15,0.1.2.16"); + cp = smartlist_join_strings(v2->known_flags, ":", 0, NULL); + test_streq(cp, "Authority:Exit:Fast:Guard:MadeOfCheese:MadeOfTin:" + "Running:Stable:V2Dir:Valid"); + tor_free(cp); + cp = smartlist_join_strings(con->net_params, ":", 0, NULL); + test_streq(cp, "bar=2000000000:circuitwindow=80:foo=660"); + tor_free(cp); + + test_eq(4, smartlist_len(con->voters)); /*3 voters, 1 legacy key.*/ + /* The voter id digests should be in this order. */ + test_assert(memcmp(cert2->cache_info.identity_digest, + cert1->cache_info.identity_digest,DIGEST_LEN)<0); + test_assert(memcmp(cert1->cache_info.identity_digest, + cert3->cache_info.identity_digest,DIGEST_LEN)<0); + test_same_voter(smartlist_get(con->voters, 1), + smartlist_get(v2->voters, 0)); + test_same_voter(smartlist_get(con->voters, 2), + smartlist_get(v1->voters, 0)); + test_same_voter(smartlist_get(con->voters, 3), + smartlist_get(v3->voters, 0)); + + test_assert(!con->cert); + test_eq(2, smartlist_len(con->routerstatus_list)); + /* There should be two listed routers: one with identity 3, one with + * identity 5. */ + /* This one showed up in 2 digests. */ + rs = smartlist_get(con->routerstatus_list, 0); + test_memeq(rs->identity_digest, + "\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3", + DIGEST_LEN); + test_memeq(rs->descriptor_digest, "NNNNNNNNNNNNNNNNNNNN", DIGEST_LEN); + test_assert(!rs->is_authority); + test_assert(!rs->is_exit); + test_assert(!rs->is_fast); + test_assert(!rs->is_possible_guard); + test_assert(!rs->is_stable); + test_assert(rs->is_running); /* If it wasn't running it wouldn't be here */ + test_assert(!rs->is_v2_dir); + test_assert(!rs->is_valid); + test_assert(!rs->is_named); + /* XXXX check version */ + + rs = smartlist_get(con->routerstatus_list, 1); + /* This one showed up in 3 digests. Twice with ID 'M', once with 'Z'. */ + test_memeq(rs->identity_digest, + "\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5", + DIGEST_LEN); + test_streq(rs->nickname, "router1"); + test_memeq(rs->descriptor_digest, "MMMMMMMMMMMMMMMMMMMM", DIGEST_LEN); + test_eq(rs->published_on, now-1000); + test_eq(rs->addr, 0x99009901); + test_eq(rs->or_port, 443); + test_eq(rs->dir_port, 0); + test_assert(!rs->is_authority); + test_assert(rs->is_exit); + test_assert(rs->is_fast); + test_assert(rs->is_possible_guard); + test_assert(rs->is_stable); + test_assert(rs->is_running); + test_assert(rs->is_v2_dir); + test_assert(rs->is_valid); + test_assert(!rs->is_named); + /* XXXX check version */ + + /* Check signatures. the first voter is a pseudo-entry with a legacy key. + * The second one hasn't signed. The fourth one has signed: validate it. */ + voter = smartlist_get(con->voters, 1); + test_eq(smartlist_len(voter->sigs), 0); + + voter = smartlist_get(con->voters, 3); + test_eq(smartlist_len(voter->sigs), 1); + sig = smartlist_get(voter->sigs, 0); + test_assert(sig->signature); + test_assert(!sig->good_signature); + test_assert(!sig->bad_signature); + + test_assert(!networkstatus_check_document_signature(con, sig, cert3)); + test_assert(sig->signature); + test_assert(sig->good_signature); + test_assert(!sig->bad_signature); + + { + const char *msg=NULL; + /* Compute the other two signed consensuses. */ + smartlist_shuffle(votes); + consensus_text2 = networkstatus_compute_consensus(votes, 3, + cert2->identity_key, + sign_skey_2, NULL,NULL, + FLAV_NS); + consensus_text_md2 = networkstatus_compute_consensus(votes, 3, + cert2->identity_key, + sign_skey_2, NULL,NULL, + FLAV_MICRODESC); + smartlist_shuffle(votes); + consensus_text3 = networkstatus_compute_consensus(votes, 3, + cert1->identity_key, + sign_skey_1, NULL,NULL, + FLAV_NS); + consensus_text_md3 = networkstatus_compute_consensus(votes, 3, + cert1->identity_key, + sign_skey_1, NULL,NULL, + FLAV_MICRODESC); + test_assert(consensus_text2); + test_assert(consensus_text3); + test_assert(consensus_text_md2); + test_assert(consensus_text_md3); + con2 = networkstatus_parse_vote_from_string(consensus_text2, NULL, + NS_TYPE_CONSENSUS); + con3 = networkstatus_parse_vote_from_string(consensus_text3, NULL, + NS_TYPE_CONSENSUS); + con_md2 = networkstatus_parse_vote_from_string(consensus_text_md2, NULL, + NS_TYPE_CONSENSUS); + con_md3 = networkstatus_parse_vote_from_string(consensus_text_md3, NULL, + NS_TYPE_CONSENSUS); + test_assert(con2); + test_assert(con3); + test_assert(con_md2); + test_assert(con_md3); + + /* All three should have the same digest. */ + test_memeq(&con->digests, &con2->digests, sizeof(digests_t)); + test_memeq(&con->digests, &con3->digests, sizeof(digests_t)); + + test_memeq(&con_md->digests, &con_md2->digests, sizeof(digests_t)); + test_memeq(&con_md->digests, &con_md3->digests, sizeof(digests_t)); + + /* Extract a detached signature from con3. */ + detached_text1 = get_detached_sigs(con3, con_md3); + tor_assert(detached_text1); + /* Try to parse it. */ + dsig1 = networkstatus_parse_detached_signatures(detached_text1, NULL); + tor_assert(dsig1); + + /* Are parsed values as expected? */ + test_eq(dsig1->valid_after, con3->valid_after); + test_eq(dsig1->fresh_until, con3->fresh_until); + test_eq(dsig1->valid_until, con3->valid_until); + { + digests_t *dsig_digests = strmap_get(dsig1->digests, "ns"); + test_assert(dsig_digests); + test_memeq(dsig_digests->d[DIGEST_SHA1], con3->digests.d[DIGEST_SHA1], + DIGEST_LEN); + dsig_digests = strmap_get(dsig1->digests, "microdesc"); + test_assert(dsig_digests); + test_memeq(dsig_digests->d[DIGEST_SHA256], + con_md3->digests.d[DIGEST_SHA256], + DIGEST256_LEN); + } + { + smartlist_t *dsig_signatures = strmap_get(dsig1->signatures, "ns"); + test_assert(dsig_signatures); + test_eq(1, smartlist_len(dsig_signatures)); + sig = smartlist_get(dsig_signatures, 0); + test_memeq(sig->identity_digest, cert1->cache_info.identity_digest, + DIGEST_LEN); + test_eq(sig->alg, DIGEST_SHA1); + + dsig_signatures = strmap_get(dsig1->signatures, "microdesc"); + test_assert(dsig_signatures); + test_eq(1, smartlist_len(dsig_signatures)); + sig = smartlist_get(dsig_signatures, 0); + test_memeq(sig->identity_digest, cert1->cache_info.identity_digest, + DIGEST_LEN); + test_eq(sig->alg, DIGEST_SHA256); + } + + /* Try adding it to con2. */ + detached_text2 = get_detached_sigs(con2,con_md2); + test_eq(1, networkstatus_add_detached_signatures(con2, dsig1, &msg)); + tor_free(detached_text2); + test_eq(1, networkstatus_add_detached_signatures(con_md2, dsig1, &msg)); + tor_free(detached_text2); + detached_text2 = get_detached_sigs(con2,con_md2); + //printf("\n<%s>\n", detached_text2); + dsig2 = networkstatus_parse_detached_signatures(detached_text2, NULL); + test_assert(dsig2); + /* + printf("\n"); + SMARTLIST_FOREACH(dsig2->signatures, networkstatus_voter_info_t *, vi, { + char hd[64]; + base16_encode(hd, sizeof(hd), vi->identity_digest, DIGEST_LEN); + printf("%s\n", hd); + }); + */ + test_eq(2, + smartlist_len((smartlist_t*)strmap_get(dsig2->signatures, "ns"))); + test_eq(2, + smartlist_len((smartlist_t*)strmap_get(dsig2->signatures, + "microdesc"))); + + /* Try adding to con2 twice; verify that nothing changes. */ + test_eq(0, networkstatus_add_detached_signatures(con2, dsig1, &msg)); + + /* Add to con. */ + test_eq(2, networkstatus_add_detached_signatures(con, dsig2, &msg)); + /* Check signatures */ + voter = smartlist_get(con->voters, 1); + sig = smartlist_get(voter->sigs, 0); + test_assert(sig); + test_assert(!networkstatus_check_document_signature(con, sig, cert2)); + voter = smartlist_get(con->voters, 2); + sig = smartlist_get(voter->sigs, 0); + test_assert(sig); + test_assert(!networkstatus_check_document_signature(con, sig, cert1)); + } + + done: + smartlist_free(votes); + tor_free(v1_text); + tor_free(v2_text); + tor_free(v3_text); + tor_free(consensus_text); + tor_free(consensus_text_md); + + if (vote) + networkstatus_vote_free(vote); + if (v1) + networkstatus_vote_free(v1); + if (v2) + networkstatus_vote_free(v2); + if (v3) + networkstatus_vote_free(v3); + if (con) + networkstatus_vote_free(con); + if (con_md) + networkstatus_vote_free(con_md); + if (sign_skey_1) + crypto_free_pk_env(sign_skey_1); + if (sign_skey_2) + crypto_free_pk_env(sign_skey_2); + if (sign_skey_3) + crypto_free_pk_env(sign_skey_3); + if (sign_skey_leg1) + crypto_free_pk_env(sign_skey_leg1); + if (cert1) + authority_cert_free(cert1); + if (cert2) + authority_cert_free(cert2); + if (cert3) + authority_cert_free(cert3); + + tor_free(consensus_text2); + tor_free(consensus_text3); + tor_free(consensus_text_md2); + tor_free(consensus_text_md3); + tor_free(detached_text1); + tor_free(detached_text2); + if (con2) + networkstatus_vote_free(con2); + if (con3) + networkstatus_vote_free(con3); + if (con_md2) + networkstatus_vote_free(con_md2); + if (con_md3) + networkstatus_vote_free(con_md3); + if (dsig1) + ns_detached_signatures_free(dsig1); + if (dsig2) + ns_detached_signatures_free(dsig2); +} + +#define DIR_LEGACY(name) \ + { #name, legacy_test_helper, 0, &legacy_setup, test_dir_ ## name } + +#define DIR(name) \ + { #name, test_dir_##name, 0, NULL, NULL } + +struct testcase_t dir_tests[] = { + DIR_LEGACY(nicknames), + DIR_LEGACY(formats), + DIR_LEGACY(versions), + DIR_LEGACY(fp_pairs), + DIR(split_fps), + DIR_LEGACY(measured_bw), + DIR_LEGACY(param_voting), + DIR_LEGACY(v3_networkstatus), + END_OF_TESTCASES +}; + diff --git a/src/test/test_util.c b/src/test/test_util.c new file mode 100644 index 0000000000..ba0f8cdf2d --- /dev/null +++ b/src/test/test_util.c @@ -0,0 +1,1075 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2009, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#define CONTROL_PRIVATE +#define MEMPOOL_PRIVATE +#include "or.h" +#include "test.h" +#include "mempool.h" +#include "memarea.h" + +static void +test_util_time(void) +{ + struct timeval start, end; + struct tm a_time; + char timestr[RFC1123_TIME_LEN+1]; + time_t t_res; + int i; + + start.tv_sec = 5; + start.tv_usec = 5000; + + end.tv_sec = 5; + end.tv_usec = 5000; + + test_eq(0L, tv_udiff(&start, &end)); + + end.tv_usec = 7000; + + test_eq(2000L, tv_udiff(&start, &end)); + + end.tv_sec = 6; + + test_eq(1002000L, tv_udiff(&start, &end)); + + end.tv_usec = 0; + + test_eq(995000L, tv_udiff(&start, &end)); + + end.tv_sec = 4; + + test_eq(-1005000L, tv_udiff(&start, &end)); + + end.tv_usec = 999990; + start.tv_sec = 1; + start.tv_usec = 500; + + /* The test values here are confirmed to be correct on a platform + * with a working timegm. */ + a_time.tm_year = 2003-1900; + a_time.tm_mon = 7; + a_time.tm_mday = 30; + a_time.tm_hour = 6; + a_time.tm_min = 14; + a_time.tm_sec = 55; + test_eq((time_t) 1062224095UL, tor_timegm(&a_time)); + a_time.tm_year = 2004-1900; /* Try a leap year, after feb. */ + test_eq((time_t) 1093846495UL, tor_timegm(&a_time)); + a_time.tm_mon = 1; /* Try a leap year, in feb. */ + a_time.tm_mday = 10; + test_eq((time_t) 1076393695UL, tor_timegm(&a_time)); + + format_rfc1123_time(timestr, 0); + test_streq("Thu, 01 Jan 1970 00:00:00 GMT", timestr); + format_rfc1123_time(timestr, (time_t)1091580502UL); + test_streq("Wed, 04 Aug 2004 00:48:22 GMT", timestr); + + t_res = 0; + i = parse_rfc1123_time(timestr, &t_res); + test_eq(i,0); + test_eq(t_res, (time_t)1091580502UL); + test_eq(-1, parse_rfc1123_time("Wed, zz Aug 2004 99-99x99 GMT", &t_res)); + + tor_gettimeofday(&start); + /* now make sure time works. */ + tor_gettimeofday(&end); + /* We might've timewarped a little. */ + tt_int_op(tv_udiff(&start, &end), >=, -5000); + + done: + ; +} + +static void +test_util_config_line(void) +{ + char buf[1024]; + char *k=NULL, *v=NULL; + const char *str; + + /* Test parse_config_line_from_str */ + strlcpy(buf, "k v\n" " key value with spaces \n" "keykey val\n" + "k2\n" + "k3 \n" "\n" " \n" "#comment\n" + "k4#a\n" "k5#abc\n" "k6 val #with comment\n" + "kseven \"a quoted 'string\"\n" + "k8 \"a \\x71uoted\\n\\\"str\\\\ing\\t\\001\\01\\1\\\"\"\n" + , sizeof(buf)); + str = buf; + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "k"); + test_streq(v, "v"); + tor_free(k); tor_free(v); + test_assert(!strcmpstart(str, "key value with")); + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "key"); + test_streq(v, "value with spaces"); + tor_free(k); tor_free(v); + test_assert(!strcmpstart(str, "keykey")); + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "keykey"); + test_streq(v, "val"); + tor_free(k); tor_free(v); + test_assert(!strcmpstart(str, "k2\n")); + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "k2"); + test_streq(v, ""); + tor_free(k); tor_free(v); + test_assert(!strcmpstart(str, "k3 \n")); + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "k3"); + test_streq(v, ""); + tor_free(k); tor_free(v); + test_assert(!strcmpstart(str, "#comment")); + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "k4"); + test_streq(v, ""); + tor_free(k); tor_free(v); + test_assert(!strcmpstart(str, "k5#abc")); + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "k5"); + test_streq(v, ""); + tor_free(k); tor_free(v); + test_assert(!strcmpstart(str, "k6")); + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "k6"); + test_streq(v, "val"); + tor_free(k); tor_free(v); + test_assert(!strcmpstart(str, "kseven")); + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "kseven"); + test_streq(v, "a quoted \'string"); + tor_free(k); tor_free(v); + test_assert(!strcmpstart(str, "k8 ")); + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "k8"); + test_streq(v, "a quoted\n\"str\\ing\t\x01\x01\x01\""); + tor_free(k); tor_free(v); + test_streq(str, ""); + done: + tor_free(k); + tor_free(v); +} + +/** Test basic string functionality. */ +static void +test_util_strmisc(void) +{ + char buf[1024]; + int i; + char *cp; + + /* Tests for corner cases of strl operations */ + test_eq(5, strlcpy(buf, "Hello", 0)); + strlcpy(buf, "Hello", sizeof(buf)); + test_eq(10, strlcat(buf, "Hello", 5)); + + /* Test tor_strstrip() */ + strlcpy(buf, "Testing 1 2 3", sizeof(buf)); + tor_strstrip(buf, ",!"); + test_streq(buf, "Testing 1 2 3"); + strlcpy(buf, "!Testing 1 2 3?", sizeof(buf)); + tor_strstrip(buf, "!? "); + test_streq(buf, "Testing123"); + + /* Test tor_parse_long. */ + test_eq(10L, tor_parse_long("10",10,0,100,NULL,NULL)); + test_eq(0L, tor_parse_long("10",10,50,100,NULL,NULL)); + test_eq(-50L, tor_parse_long("-50",10,-100,100,NULL,NULL)); + + /* Test tor_parse_ulong */ + test_eq(10UL, tor_parse_ulong("10",10,0,100,NULL,NULL)); + test_eq(0UL, tor_parse_ulong("10",10,50,100,NULL,NULL)); + + /* Test tor_parse_uint64. */ + test_assert(U64_LITERAL(10) == tor_parse_uint64("10 x",10,0,100, &i, &cp)); + test_assert(i == 1); + test_streq(cp, " x"); + test_assert(U64_LITERAL(12345678901) == + tor_parse_uint64("12345678901",10,0,UINT64_MAX, &i, &cp)); + test_assert(i == 1); + test_streq(cp, ""); + test_assert(U64_LITERAL(0) == + tor_parse_uint64("12345678901",10,500,INT32_MAX, &i, &cp)); + test_assert(i == 0); + + { + /* Test tor_parse_double. */ + double d = tor_parse_double("10", 0, UINT64_MAX,&i,NULL); + test_assert(i == 1); + test_assert(DBL_TO_U64(d) == 10); + d = tor_parse_double("0", 0, UINT64_MAX,&i,NULL); + test_assert(i == 1); + test_assert(DBL_TO_U64(d) == 0); + d = tor_parse_double(" ", 0, UINT64_MAX,&i,NULL); + test_assert(i == 0); + d = tor_parse_double(".0a", 0, UINT64_MAX,&i,NULL); + test_assert(i == 0); + d = tor_parse_double(".0a", 0, UINT64_MAX,&i,&cp); + test_assert(i == 1); + d = tor_parse_double("-.0", 0, UINT64_MAX,&i,NULL); + test_assert(i == 1); + } + + /* Test failing snprintf cases */ + test_eq(-1, tor_snprintf(buf, 0, "Foo")); + test_eq(-1, tor_snprintf(buf, 2, "Foo")); + + /* Test printf with uint64 */ + tor_snprintf(buf, sizeof(buf), "x!"U64_FORMAT"!x", + U64_PRINTF_ARG(U64_LITERAL(12345678901))); + test_streq(buf, "x!12345678901!x"); + + /* Test for strcmpstart and strcmpend. */ + test_assert(strcmpstart("abcdef", "abcdef")==0); + test_assert(strcmpstart("abcdef", "abc")==0); + test_assert(strcmpstart("abcdef", "abd")<0); + test_assert(strcmpstart("abcdef", "abb")>0); + test_assert(strcmpstart("ab", "abb")<0); + + test_assert(strcmpend("abcdef", "abcdef")==0); + test_assert(strcmpend("abcdef", "def")==0); + test_assert(strcmpend("abcdef", "deg")<0); + test_assert(strcmpend("abcdef", "dee")>0); + test_assert(strcmpend("ab", "abb")<0); + + test_assert(strcasecmpend("AbcDEF", "abcdef")==0); + test_assert(strcasecmpend("abcdef", "dEF")==0); + test_assert(strcasecmpend("abcDEf", "deg")<0); + test_assert(strcasecmpend("abcdef", "DEE")>0); + test_assert(strcasecmpend("ab", "abB")<0); + + /* Test mem_is_zero */ + memset(buf,0,128); + buf[128] = 'x'; + test_assert(tor_digest_is_zero(buf)); + test_assert(tor_mem_is_zero(buf, 10)); + test_assert(tor_mem_is_zero(buf, 20)); + test_assert(tor_mem_is_zero(buf, 128)); + test_assert(!tor_mem_is_zero(buf, 129)); + buf[60] = (char)255; + test_assert(!tor_mem_is_zero(buf, 128)); + buf[0] = (char)1; + test_assert(!tor_mem_is_zero(buf, 10)); + + /* Test 'escaped' */ + test_streq("\"\"", escaped("")); + test_streq("\"abcd\"", escaped("abcd")); + test_streq("\"\\\\\\n\\r\\t\\\"\\'\"", escaped("\\\n\r\t\"\'")); + test_streq("\"z\\001abc\\277d\"", escaped("z\001abc\277d")); + test_assert(NULL == escaped(NULL)); + + /* Test strndup and memdup */ + { + const char *s = "abcdefghijklmnopqrstuvwxyz"; + cp = tor_strndup(s, 30); + test_streq(cp, s); /* same string, */ + test_neq(cp, s); /* but different pointers. */ + tor_free(cp); + + cp = tor_strndup(s, 5); + test_streq(cp, "abcde"); + tor_free(cp); + + s = "a\0b\0c\0d\0e\0"; + cp = tor_memdup(s,10); + test_memeq(cp, s, 10); /* same ram, */ + test_neq(cp, s); /* but different pointers. */ + tor_free(cp); + } + + /* Test str-foo functions */ + cp = tor_strdup("abcdef"); + test_assert(tor_strisnonupper(cp)); + cp[3] = 'D'; + test_assert(!tor_strisnonupper(cp)); + tor_strupper(cp); + test_streq(cp, "ABCDEF"); + test_assert(tor_strisprint(cp)); + cp[3] = 3; + test_assert(!tor_strisprint(cp)); + tor_free(cp); + + /* Test eat_whitespace. */ + { + const char *s = " \n a"; + test_eq_ptr(eat_whitespace(s), s+4); + s = "abcd"; + test_eq_ptr(eat_whitespace(s), s); + s = "#xyz\nab"; + test_eq_ptr(eat_whitespace(s), s+5); + } + + /* Test memmem and memstr */ + { + const char *haystack = "abcde"; + tor_assert(!tor_memmem(haystack, 5, "ef", 2)); + test_eq_ptr(tor_memmem(haystack, 5, "cd", 2), haystack + 2); + test_eq_ptr(tor_memmem(haystack, 5, "cde", 3), haystack + 2); + haystack = "ababcad"; + test_eq_ptr(tor_memmem(haystack, 7, "abc", 3), haystack + 2); + test_eq_ptr(tor_memstr(haystack, 7, "abc"), haystack + 2); + test_assert(!tor_memstr(haystack, 7, "fe")); + test_assert(!tor_memstr(haystack, 7, "longerthantheoriginal")); + } + + /* Test wrap_string */ + { + smartlist_t *sl = smartlist_create(); + wrap_string(sl, "This is a test of string wrapping functionality: woot.", + 10, "", ""); + cp = smartlist_join_strings(sl, "", 0, NULL); + test_streq(cp, + "This is a\ntest of\nstring\nwrapping\nfunctional\nity: woot.\n"); + tor_free(cp); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + wrap_string(sl, "This is a test of string wrapping functionality: woot.", + 16, "### ", "# "); + cp = smartlist_join_strings(sl, "", 0, NULL); + test_streq(cp, + "### This is a\n# test of string\n# wrapping\n# functionality:\n" + "# woot.\n"); + + tor_free(cp); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_free(sl); + } + done: + ; +} + +static void +test_util_pow2(void) +{ + /* Test tor_log2(). */ + test_eq(tor_log2(64), 6); + test_eq(tor_log2(65), 6); + test_eq(tor_log2(63), 5); + test_eq(tor_log2(1), 0); + test_eq(tor_log2(2), 1); + test_eq(tor_log2(3), 1); + test_eq(tor_log2(4), 2); + test_eq(tor_log2(5), 2); + test_eq(tor_log2(U64_LITERAL(40000000000000000)), 55); + test_eq(tor_log2(UINT64_MAX), 63); + + /* Test round_to_power_of_2 */ + test_eq(round_to_power_of_2(120), 128); + test_eq(round_to_power_of_2(128), 128); + test_eq(round_to_power_of_2(130), 128); + test_eq(round_to_power_of_2(U64_LITERAL(40000000000000000)), + U64_LITERAL(1)<<55); + test_eq(round_to_power_of_2(0), 2); + + done: + ; +} + +/** mutex for thread test to stop the threads hitting data at the same time. */ +static tor_mutex_t *_thread_test_mutex = NULL; +/** mutexes for the thread test to make sure that the threads have to + * interleave somewhat. */ +static tor_mutex_t *_thread_test_start1 = NULL, + *_thread_test_start2 = NULL; +/** Shared strmap for the thread test. */ +static strmap_t *_thread_test_strmap = NULL; +/** The name of thread1 for the thread test */ +static char *_thread1_name = NULL; +/** The name of thread2 for the thread test */ +static char *_thread2_name = NULL; + +static void _thread_test_func(void* _s) ATTR_NORETURN; + +/** How many iterations have the threads in the unit test run? */ +static int t1_count = 0, t2_count = 0; + +/** Helper function for threading unit tests: This function runs in a + * subthread. It grabs its own mutex (start1 or start2) to make sure that it + * should start, then it repeatedly alters _test_thread_strmap protected by + * _thread_test_mutex. */ +static void +_thread_test_func(void* _s) +{ + char *s = _s; + int i, *count; + tor_mutex_t *m; + char buf[64]; + char **cp; + if (!strcmp(s, "thread 1")) { + m = _thread_test_start1; + cp = &_thread1_name; + count = &t1_count; + } else { + m = _thread_test_start2; + cp = &_thread2_name; + count = &t2_count; + } + tor_mutex_acquire(m); + + tor_snprintf(buf, sizeof(buf), "%lu", tor_get_thread_id()); + *cp = tor_strdup(buf); + + for (i=0; i<10000; ++i) { + tor_mutex_acquire(_thread_test_mutex); + strmap_set(_thread_test_strmap, "last to run", *cp); + ++*count; + tor_mutex_release(_thread_test_mutex); + } + tor_mutex_acquire(_thread_test_mutex); + strmap_set(_thread_test_strmap, s, *cp); + tor_mutex_release(_thread_test_mutex); + + tor_mutex_release(m); + + spawn_exit(); +} + +/** Run unit tests for threading logic. */ +static void +test_util_threads(void) +{ + char *s1 = NULL, *s2 = NULL; + int done = 0, timedout = 0; + time_t started; +#ifndef TOR_IS_MULTITHREADED + /* Skip this test if we aren't threading. We should be threading most + * everywhere by now. */ + if (1) + return; +#endif + _thread_test_mutex = tor_mutex_new(); + _thread_test_start1 = tor_mutex_new(); + _thread_test_start2 = tor_mutex_new(); + _thread_test_strmap = strmap_new(); + s1 = tor_strdup("thread 1"); + s2 = tor_strdup("thread 2"); + tor_mutex_acquire(_thread_test_start1); + tor_mutex_acquire(_thread_test_start2); + spawn_func(_thread_test_func, s1); + spawn_func(_thread_test_func, s2); + tor_mutex_release(_thread_test_start2); + tor_mutex_release(_thread_test_start1); + started = time(NULL); + while (!done) { + tor_mutex_acquire(_thread_test_mutex); + strmap_assert_ok(_thread_test_strmap); + if (strmap_get(_thread_test_strmap, "thread 1") && + strmap_get(_thread_test_strmap, "thread 2")) { + done = 1; + } else if (time(NULL) > started + 25) { + timedout = done = 1; + } + tor_mutex_release(_thread_test_mutex); + } + tor_mutex_free(_thread_test_mutex); + + tor_mutex_acquire(_thread_test_start1); + tor_mutex_release(_thread_test_start1); + tor_mutex_acquire(_thread_test_start2); + tor_mutex_release(_thread_test_start2); + + if (timedout) { + printf("\nTimed out: %d %d", t1_count, t2_count); + test_assert(strmap_get(_thread_test_strmap, "thread 1")); + test_assert(strmap_get(_thread_test_strmap, "thread 2")); + test_assert(!timedout); + } + + /* different thread IDs. */ + test_assert(strcmp(strmap_get(_thread_test_strmap, "thread 1"), + strmap_get(_thread_test_strmap, "thread 2"))); + test_assert(!strcmp(strmap_get(_thread_test_strmap, "thread 1"), + strmap_get(_thread_test_strmap, "last to run")) || + !strcmp(strmap_get(_thread_test_strmap, "thread 2"), + strmap_get(_thread_test_strmap, "last to run"))); + + done: + tor_free(s1); + tor_free(s2); + tor_free(_thread1_name); + tor_free(_thread2_name); + if (_thread_test_strmap) + strmap_free(_thread_test_strmap, NULL); + if (_thread_test_start1) + tor_mutex_free(_thread_test_start1); + if (_thread_test_start2) + tor_mutex_free(_thread_test_start2); +} + +/** Run unit tests for compression functions */ +static void +test_util_gzip(void) +{ + char *buf1=NULL, *buf2=NULL, *buf3=NULL, *cp1, *cp2; + const char *ccp2; + size_t len1, len2; + tor_zlib_state_t *state = NULL; + + buf1 = tor_strdup("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ"); + test_assert(detect_compression_method(buf1, strlen(buf1)) == UNKNOWN_METHOD); + if (is_gzip_supported()) { + test_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1, + GZIP_METHOD)); + test_assert(buf2); + test_assert(!memcmp(buf2, "\037\213", 2)); /* Gzip magic. */ + test_assert(detect_compression_method(buf2, len1) == GZIP_METHOD); + + test_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1, + GZIP_METHOD, 1, LOG_INFO)); + test_assert(buf3); + test_streq(buf1,buf3); + + tor_free(buf2); + tor_free(buf3); + } + + test_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1, + ZLIB_METHOD)); + test_assert(buf2); + test_assert(!memcmp(buf2, "\x78\xDA", 2)); /* deflate magic. */ + test_assert(detect_compression_method(buf2, len1) == ZLIB_METHOD); + + test_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1, + ZLIB_METHOD, 1, LOG_INFO)); + test_assert(buf3); + test_streq(buf1,buf3); + + /* Check whether we can uncompress concatenated, compressed strings. */ + tor_free(buf3); + buf2 = tor_realloc(buf2, len1*2); + memcpy(buf2+len1, buf2, len1); + test_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1*2, + ZLIB_METHOD, 1, LOG_INFO)); + test_eq(len2, (strlen(buf1)+1)*2); + test_memeq(buf3, + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ\0" + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ\0", + (strlen(buf1)+1)*2); + + tor_free(buf1); + tor_free(buf2); + tor_free(buf3); + + /* Check whether we can uncompress partial strings. */ + buf1 = + tor_strdup("String with low redundancy that won't be compressed much."); + test_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1, + ZLIB_METHOD)); + tor_assert(len1>16); + /* when we allow an incomplete string, we should succeed.*/ + tor_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1-16, + ZLIB_METHOD, 0, LOG_INFO)); + buf3[len2]='\0'; + tor_assert(len2 > 5); + tor_assert(!strcmpstart(buf1, buf3)); + + /* when we demand a complete string, this must fail. */ + tor_free(buf3); + tor_assert(tor_gzip_uncompress(&buf3, &len2, buf2, len1-16, + ZLIB_METHOD, 1, LOG_INFO)); + tor_assert(!buf3); + + /* Now, try streaming compression. */ + tor_free(buf1); + tor_free(buf2); + tor_free(buf3); + state = tor_zlib_new(1, ZLIB_METHOD); + tor_assert(state); + cp1 = buf1 = tor_malloc(1024); + len1 = 1024; + ccp2 = "ABCDEFGHIJABCDEFGHIJ"; + len2 = 21; + test_assert(tor_zlib_process(state, &cp1, &len1, &ccp2, &len2, 0) + == TOR_ZLIB_OK); + test_eq(len2, 0); /* Make sure we compressed it all. */ + test_assert(cp1 > buf1); + + len2 = 0; + cp2 = cp1; + test_assert(tor_zlib_process(state, &cp1, &len1, &ccp2, &len2, 1) + == TOR_ZLIB_DONE); + test_eq(len2, 0); + test_assert(cp1 > cp2); /* Make sure we really added something. */ + + tor_assert(!tor_gzip_uncompress(&buf3, &len2, buf1, 1024-len1, + ZLIB_METHOD, 1, LOG_WARN)); + test_streq(buf3, "ABCDEFGHIJABCDEFGHIJ"); /*Make sure it compressed right.*/ + + done: + if (state) + tor_zlib_free(state); + tor_free(buf2); + tor_free(buf3); + tor_free(buf1); +} + +/** Run unit tests for mmap() wrapper functionality. */ +static void +test_util_mmap(void) +{ + char *fname1 = tor_strdup(get_fname("mapped_1")); + char *fname2 = tor_strdup(get_fname("mapped_2")); + char *fname3 = tor_strdup(get_fname("mapped_3")); + const size_t buflen = 17000; + char *buf = tor_malloc(17000); + tor_mmap_t *mapping = NULL; + + crypto_rand(buf, buflen); + + mapping = tor_mmap_file(fname1); + test_assert(! mapping); + + write_str_to_file(fname1, "Short file.", 1); + write_bytes_to_file(fname2, buf, buflen, 1); + write_bytes_to_file(fname3, buf, 16384, 1); + + mapping = tor_mmap_file(fname1); + test_assert(mapping); + test_eq(mapping->size, strlen("Short file.")); + test_streq(mapping->data, "Short file."); +#ifdef MS_WINDOWS + tor_munmap_file(mapping); + mapping = NULL; + test_assert(unlink(fname1) == 0); +#else + /* make sure we can unlink. */ + test_assert(unlink(fname1) == 0); + test_streq(mapping->data, "Short file."); + tor_munmap_file(mapping); + mapping = NULL; +#endif + + /* Now a zero-length file. */ + write_str_to_file(fname1, "", 1); + mapping = tor_mmap_file(fname1); + test_eq(mapping, NULL); + test_eq(ERANGE, errno); + unlink(fname1); + + /* Make sure that we fail to map a no-longer-existent file. */ + mapping = tor_mmap_file(fname1); + test_assert(mapping == NULL); + + /* Now try a big file that stretches across a few pages and isn't aligned */ + mapping = tor_mmap_file(fname2); + test_assert(mapping); + test_eq(mapping->size, buflen); + test_memeq(mapping->data, buf, buflen); + tor_munmap_file(mapping); + mapping = NULL; + + /* Now try a big aligned file. */ + mapping = tor_mmap_file(fname3); + test_assert(mapping); + test_eq(mapping->size, 16384); + test_memeq(mapping->data, buf, 16384); + tor_munmap_file(mapping); + mapping = NULL; + + done: + unlink(fname1); + unlink(fname2); + unlink(fname3); + + tor_free(fname1); + tor_free(fname2); + tor_free(fname3); + tor_free(buf); + + if (mapping) + tor_munmap_file(mapping); +} + +/** Run unit tests for escaping/unescaping data for use by controllers. */ +static void +test_util_control_formats(void) +{ + char *out = NULL; + const char *inp = + "..This is a test\r\nof the emergency \nbroadcast\r\n..system.\r\nZ.\r\n"; + size_t sz; + + sz = read_escaped_data(inp, strlen(inp), &out); + test_streq(out, + ".This is a test\nof the emergency \nbroadcast\n.system.\nZ.\n"); + test_eq(sz, strlen(out)); + + done: + tor_free(out); +} + +static void +test_util_sscanf(void) +{ + unsigned u1, u2, u3; + char s1[10], s2[10], s3[10], ch; + int r; + + r = tor_sscanf("hello world", "hello world"); /* String match: success */ + test_eq(r, 0); + r = tor_sscanf("hello world 3", "hello worlb %u", &u1); /* String fail */ + test_eq(r, 0); + r = tor_sscanf("12345", "%u", &u1); /* Simple number */ + test_eq(r, 1); + test_eq(u1, 12345u); + r = tor_sscanf("", "%u", &u1); /* absent number */ + test_eq(r, 0); + r = tor_sscanf("A", "%u", &u1); /* bogus number */ + test_eq(r, 0); + r = tor_sscanf("4294967295", "%u", &u1); /* UINT32_MAX should work. */ + test_eq(r, 1); + test_eq(u1, 4294967295u); + r = tor_sscanf("4294967296", "%u", &u1); /* Always say -1 at 32 bits. */ + test_eq(r, 0); + r = tor_sscanf("123456", "%2u%u", &u1, &u2); /* Width */ + test_eq(r, 2); + test_eq(u1, 12u); + test_eq(u2, 3456u); + r = tor_sscanf("!12:3:456", "!%2u:%2u:%3u", &u1, &u2, &u3); /* separators */ + test_eq(r, 3); + test_eq(u1, 12u); + test_eq(u2, 3u); + test_eq(u3, 456u); + r = tor_sscanf("12:3:045", "%2u:%2u:%3u", &u1, &u2, &u3); /* 0s */ + test_eq(r, 3); + test_eq(u1, 12u); + test_eq(u2, 3u); + test_eq(u3, 45u); + /* %u does not match space.*/ + r = tor_sscanf("12:3: 45", "%2u:%2u:%3u", &u1, &u2, &u3); + test_eq(r, 2); + /* %u does not match negative numbers. */ + r = tor_sscanf("12:3:-4", "%2u:%2u:%3u", &u1, &u2, &u3); + test_eq(r, 2); + /* Arbitrary amounts of 0-padding are okay */ + r = tor_sscanf("12:03:000000000000000099", "%2u:%2u:%u", &u1, &u2, &u3); + test_eq(r, 3); + test_eq(u1, 12u); + test_eq(u2, 3u); + test_eq(u3, 99u); + + r = tor_sscanf("99% fresh", "%3u%% fresh", &u1); /* percents are scannable.*/ + test_eq(r, 1); + test_eq(u1, 99); + + r = tor_sscanf("hello", "%s", s1); /* %s needs a number. */ + test_eq(r, -1); + + r = tor_sscanf("hello", "%3s%7s", s1, s2); /* %s matches characters. */ + test_eq(r, 2); + test_streq(s1, "hel"); + test_streq(s2, "lo"); + r = tor_sscanf("WD40", "%2s%u", s3, &u1); /* %s%u */ + test_eq(r, 2); + test_streq(s3, "WD"); + test_eq(u1, 40); + r = tor_sscanf("76trombones", "%6u%9s", &u1, s1); /* %u%s */ + test_eq(r, 2); + test_eq(u1, 76); + test_streq(s1, "trombones"); + r = tor_sscanf("hello world", "%9s %9s", s1, s2); /* %s doesn't eat space. */ + test_eq(r, 2); + test_streq(s1, "hello"); + test_streq(s2, "world"); + r = tor_sscanf("hi", "%9s%9s%3s", s1, s2, s3); /* %s can be empty. */ + test_eq(r, 3); + test_streq(s1, "hi"); + test_streq(s2, ""); + test_streq(s3, ""); + + r = tor_sscanf("1.2.3", "%u.%u.%u%c", &u1, &u2, &u3, &ch); + test_eq(r, 3); + r = tor_sscanf("1.2.3 foobar", "%u.%u.%u%c", &u1, &u2, &u3, &ch); + test_eq(r, 4); + + done: + ; +} + +/** Run unittests for memory pool allocator */ +static void +test_util_mempool(void) +{ + mp_pool_t *pool = NULL; + smartlist_t *allocated = NULL; + int i; + + pool = mp_pool_new(1, 100); + test_assert(pool); + test_assert(pool->new_chunk_capacity >= 100); + test_assert(pool->item_alloc_size >= sizeof(void*)+1); + mp_pool_destroy(pool); + pool = NULL; + + pool = mp_pool_new(241, 2500); + test_assert(pool); + test_assert(pool->new_chunk_capacity >= 10); + test_assert(pool->item_alloc_size >= sizeof(void*)+241); + test_eq(pool->item_alloc_size & 0x03, 0); + test_assert(pool->new_chunk_capacity < 60); + + allocated = smartlist_create(); + for (i = 0; i < 20000; ++i) { + if (smartlist_len(allocated) < 20 || crypto_rand_int(2)) { + void *m = mp_pool_get(pool); + memset(m, 0x09, 241); + smartlist_add(allocated, m); + //printf("%d: %p\n", i, m); + //mp_pool_assert_ok(pool); + } else { + int idx = crypto_rand_int(smartlist_len(allocated)); + void *m = smartlist_get(allocated, idx); + //printf("%d: free %p\n", i, m); + smartlist_del(allocated, idx); + mp_pool_release(m); + //mp_pool_assert_ok(pool); + } + if (crypto_rand_int(777)==0) + mp_pool_clean(pool, 1, 1); + + if (i % 777) + mp_pool_assert_ok(pool); + } + + done: + if (allocated) { + SMARTLIST_FOREACH(allocated, void *, m, mp_pool_release(m)); + mp_pool_assert_ok(pool); + mp_pool_clean(pool, 0, 0); + mp_pool_assert_ok(pool); + smartlist_free(allocated); + } + + if (pool) + mp_pool_destroy(pool); +} + +/** Run unittests for memory area allocator */ +static void +test_util_memarea(void) +{ + memarea_t *area = memarea_new(); + char *p1, *p2, *p3, *p1_orig; + void *malloced_ptr = NULL; + int i; + + test_assert(area); + + p1_orig = p1 = memarea_alloc(area,64); + p2 = memarea_alloc_zero(area,52); + p3 = memarea_alloc(area,11); + + test_assert(memarea_owns_ptr(area, p1)); + test_assert(memarea_owns_ptr(area, p2)); + test_assert(memarea_owns_ptr(area, p3)); + /* Make sure we left enough space. */ + test_assert(p1+64 <= p2); + test_assert(p2+52 <= p3); + /* Make sure we aligned. */ + test_eq(((uintptr_t)p1) % sizeof(void*), 0); + test_eq(((uintptr_t)p2) % sizeof(void*), 0); + test_eq(((uintptr_t)p3) % sizeof(void*), 0); + test_assert(!memarea_owns_ptr(area, p3+8192)); + test_assert(!memarea_owns_ptr(area, p3+30)); + test_assert(tor_mem_is_zero(p2, 52)); + /* Make sure we don't overalign. */ + p1 = memarea_alloc(area, 1); + p2 = memarea_alloc(area, 1); + test_eq(p1+sizeof(void*), p2); + { + malloced_ptr = tor_malloc(64); + test_assert(!memarea_owns_ptr(area, malloced_ptr)); + tor_free(malloced_ptr); + } + + /* memarea_memdup */ + { + malloced_ptr = tor_malloc(64); + crypto_rand((char*)malloced_ptr, 64); + p1 = memarea_memdup(area, malloced_ptr, 64); + test_assert(p1 != malloced_ptr); + test_memeq(p1, malloced_ptr, 64); + tor_free(malloced_ptr); + } + + /* memarea_strdup. */ + p1 = memarea_strdup(area,""); + p2 = memarea_strdup(area, "abcd"); + test_assert(p1); + test_assert(p2); + test_streq(p1, ""); + test_streq(p2, "abcd"); + + /* memarea_strndup. */ + { + const char *s = "Ad ogni porta batte la morte e grida: il nome!"; + /* (From Turandot, act 3.) */ + size_t len = strlen(s); + p1 = memarea_strndup(area, s, 1000); + p2 = memarea_strndup(area, s, 10); + test_streq(p1, s); + test_assert(p2 >= p1 + len + 1); + test_memeq(s, p2, 10); + test_eq(p2[10], '\0'); + p3 = memarea_strndup(area, s, len); + test_streq(p3, s); + p3 = memarea_strndup(area, s, len-1); + test_memeq(s, p3, len-1); + test_eq(p3[len-1], '\0'); + } + + memarea_clear(area); + p1 = memarea_alloc(area, 1); + test_eq(p1, p1_orig); + memarea_clear(area); + + /* Check for running over an area's size. */ + for (i = 0; i < 512; ++i) { + p1 = memarea_alloc(area, crypto_rand_int(5)+1); + test_assert(memarea_owns_ptr(area, p1)); + } + memarea_assert_ok(area); + /* Make sure we can allocate a too-big object. */ + p1 = memarea_alloc_zero(area, 9000); + p2 = memarea_alloc_zero(area, 16); + test_assert(memarea_owns_ptr(area, p1)); + test_assert(memarea_owns_ptr(area, p2)); + + done: + memarea_drop_all(area); + tor_free(malloced_ptr); +} + +/** Run unit tests for utility functions to get file names relative to + * the data directory. */ +static void +test_util_datadir(void) +{ + char buf[1024]; + char *f = NULL; + char *temp_dir = NULL; + + temp_dir = get_datadir_fname(NULL); + f = get_datadir_fname("state"); + tor_snprintf(buf, sizeof(buf), "%s"PATH_SEPARATOR"state", temp_dir); + test_streq(f, buf); + tor_free(f); + f = get_datadir_fname2("cache", "thingy"); + tor_snprintf(buf, sizeof(buf), + "%s"PATH_SEPARATOR"cache"PATH_SEPARATOR"thingy", temp_dir); + test_streq(f, buf); + tor_free(f); + f = get_datadir_fname2_suffix("cache", "thingy", ".foo"); + tor_snprintf(buf, sizeof(buf), + "%s"PATH_SEPARATOR"cache"PATH_SEPARATOR"thingy.foo", temp_dir); + test_streq(f, buf); + tor_free(f); + f = get_datadir_fname_suffix("cache", ".foo"); + tor_snprintf(buf, sizeof(buf), "%s"PATH_SEPARATOR"cache.foo", + temp_dir); + test_streq(f, buf); + + done: + tor_free(f); + tor_free(temp_dir); +} + +static void +test_util_strtok(void) +{ + char buf[128]; + char buf2[128]; + char *cp1, *cp2; + strlcpy(buf, "Graved on the dark in gestures of descent", sizeof(buf)); + strlcpy(buf2, "they.seemed;their!own;most.perfect;monument", sizeof(buf2)); + /* -- "Year's End", Richard Wilbur */ + + test_streq("Graved", tor_strtok_r_impl(buf, " ", &cp1)); + test_streq("they", tor_strtok_r_impl(buf2, ".!..;!", &cp2)); +#define S1() tor_strtok_r_impl(NULL, " ", &cp1) +#define S2() tor_strtok_r_impl(NULL, ".!..;!", &cp2) + test_streq("on", S1()); + test_streq("the", S1()); + test_streq("dark", S1()); + test_streq("seemed", S2()); + test_streq("their", S2()); + test_streq("own", S2()); + test_streq("in", S1()); + test_streq("gestures", S1()); + test_streq("of", S1()); + test_streq("most", S2()); + test_streq("perfect", S2()); + test_streq("descent", S1()); + test_streq("monument", S2()); + test_assert(NULL == S1()); + test_assert(NULL == S2()); + done: + ; +} + +static void +test_util_find_str_at_start_of_line(void *ptr) +{ + const char *long_string = + "hello world. hello world. hello hello. howdy.\n" + "hello hello world\n"; + + (void)ptr; + + /* not-found case. */ + tt_assert(! find_str_at_start_of_line(long_string, "fred")); + + /* not-found case where haystack doesn't end with \n */ + tt_assert(! find_str_at_start_of_line("foobar\nbaz", "fred")); + + /* start-of-string case */ + tt_assert(long_string == + find_str_at_start_of_line(long_string, "hello world.")); + + /* start-of-line case */ + tt_assert(strchr(long_string,'\n')+1 == + find_str_at_start_of_line(long_string, "hello hello")); + done: + ; +} + +#define UTIL_LEGACY(name) \ + { #name, legacy_test_helper, 0, &legacy_setup, test_util_ ## name } + +#define UTIL_TEST(name, flags) \ + { #name, test_util_ ## name, flags, NULL, NULL } + +struct testcase_t util_tests[] = { + UTIL_LEGACY(time), + UTIL_LEGACY(config_line), + UTIL_LEGACY(strmisc), + UTIL_LEGACY(pow2), + UTIL_LEGACY(gzip), + UTIL_LEGACY(datadir), + UTIL_LEGACY(mempool), + UTIL_LEGACY(memarea), + UTIL_LEGACY(control_formats), + UTIL_LEGACY(mmap), + UTIL_LEGACY(threads), + UTIL_LEGACY(sscanf), + UTIL_LEGACY(strtok), + UTIL_TEST(find_str_at_start_of_line, 0), + END_OF_TESTCASES +}; + diff --git a/src/test/tinytest.c b/src/test/tinytest.c new file mode 100644 index 0000000000..b358bb3a4d --- /dev/null +++ b/src/test/tinytest.c @@ -0,0 +1,369 @@ +/* tinytest.c -- Copyright 2009 Nick Mathewson + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <assert.h> + +#ifdef WIN32 +#include <windows.h> +#else +#include <sys/types.h> +#include <sys/wait.h> +#include <unistd.h> +#endif + +#ifndef __GNUC__ +#define __attribute__(x) +#endif + +#include "tinytest.h" +#include "tinytest_macros.h" + +#define LONGEST_TEST_NAME 16384 + +static int in_tinytest_main = 0; /**< true if we're in tinytest_main().*/ +static int n_ok = 0; /**< Number of tests that have passed */ +static int n_bad = 0; /**< Number of tests that have failed. */ +static int n_skipped = 0; /**< Number of tests that have been skipped. */ + +static int opt_forked = 0; /**< True iff we're called from inside a win32 fork*/ +static int opt_nofork = 0; /**< Suppress calls to fork() for debugging. */ +static int opt_verbosity = 1; /**< -==quiet,0==terse,1==normal,2==verbose */ +const char *verbosity_flag = ""; + +enum outcome { SKIP=2, OK=1, FAIL=0 }; +static enum outcome cur_test_outcome = 0; +const char *cur_test_prefix = NULL; /**< prefix of the current test group */ +/** Name of the current test, if we haven't logged is yet. Used for --quiet */ +const char *cur_test_name = NULL; + +#ifdef WIN32 +/** Pointer to argv[0] for win32. */ +static const char *commandname = NULL; +#endif + +static void usage(struct testgroup_t *groups, int list_groups) + __attribute__((noreturn)); + +static enum outcome +_testcase_run_bare(const struct testcase_t *testcase) +{ + void *env = NULL; + int outcome; + if (testcase->setup) { + env = testcase->setup->setup_fn(testcase); + if (!env) + return FAIL; + else if (env == (void*)TT_SKIP) + return SKIP; + } + + cur_test_outcome = OK; + testcase->fn(env); + outcome = cur_test_outcome; + + if (testcase->setup) { + if (testcase->setup->cleanup_fn(testcase, env) == 0) + outcome = FAIL; + } + + return outcome; +} + +#define MAGIC_EXITCODE 42 + +static enum outcome +_testcase_run_forked(const struct testgroup_t *group, + const struct testcase_t *testcase) +{ +#ifdef WIN32 + /* Fork? On Win32? How primitive! We'll do what the smart kids do: + we'll invoke our own exe (whose name we recall from the command + line) with a command line that tells it to run just the test we + want, and this time without forking. + + (No, threads aren't an option. The whole point of forking is to + share no state between tests.) + */ + int ok; + char buffer[LONGEST_TEST_NAME+256]; + STARTUPINFO si; + PROCESS_INFORMATION info; + DWORD exitcode; + + if (!in_tinytest_main) { + printf("\nERROR. On Windows, _testcase_run_forked must be" + " called from within tinytest_main.\n"); + abort(); + } + if (opt_verbosity>0) + printf("[forking] "); + + snprintf(buffer, sizeof(buffer), "%s --RUNNING-FORKED %s %s%s", + commandname, verbosity_flag, group->prefix, testcase->name); + + memset(&si, 0, sizeof(si)); + memset(&info, 0, sizeof(info)); + si.cb = sizeof(si); + + ok = CreateProcess(commandname, buffer, NULL, NULL, 0, + 0, NULL, NULL, &si, &info); + if (!ok) { + printf("CreateProcess failed!\n"); + return 0; + } + WaitForSingleObject(info.hProcess, INFINITE); + GetExitCodeProcess(info.hProcess, &exitcode); + CloseHandle(info.hProcess); + CloseHandle(info.hThread); + if (exitcode == 0) + return OK; + else if (exitcode == MAGIC_EXITCODE) + return SKIP; + else + return FAIL; +#else + int outcome_pipe[2]; + pid_t pid; + (void)group; + + if (pipe(outcome_pipe)) + perror("opening pipe"); + + if (opt_verbosity>0) + printf("[forking] "); + pid = fork(); + if (!pid) { + /* child. */ + int test_r, write_r; + char b[1]; + close(outcome_pipe[0]); + test_r = _testcase_run_bare(testcase); + assert(0<=(int)test_r && (int)test_r<=2); + b[0] = "NYS"[test_r]; + write_r = (int)write(outcome_pipe[1], b, 1); + if (write_r != 1) { + perror("write outcome to pipe"); + exit(1); + } + exit(0); + } else { + /* parent */ + int status, r; + char b[1]; + /* Close this now, so that if the other side closes it, + * our read fails. */ + close(outcome_pipe[1]); + r = (int)read(outcome_pipe[0], b, 1); + if (r == 0) { + printf("[Lost connection!] "); + return 0; + } else if (r != 1) { + perror("read outcome from pipe"); + } + waitpid(pid, &status, 0); + close(outcome_pipe[0]); + return b[0]=='Y' ? OK : (b[0]=='S' ? SKIP : FAIL); + } +#endif +} + +int +testcase_run_one(const struct testgroup_t *group, + const struct testcase_t *testcase) +{ + enum outcome outcome; + + if (testcase->flags & TT_SKIP) { + if (opt_verbosity>0) + printf("%s%s: SKIPPED\n", + group->prefix, testcase->name); + ++n_skipped; + return SKIP; + } + + if (opt_verbosity>0 && !opt_forked) { + printf("%s%s: ", group->prefix, testcase->name); + } else { + if (opt_verbosity==0) printf("."); + cur_test_prefix = group->prefix; + cur_test_name = testcase->name; + } + + if ((testcase->flags & TT_FORK) && !(opt_forked||opt_nofork)) { + outcome = _testcase_run_forked(group, testcase); + } else { + outcome = _testcase_run_bare(testcase); + } + + if (outcome == OK) { + ++n_ok; + if (opt_verbosity>0 && !opt_forked) + puts(opt_verbosity==1?"OK":""); + } else if (outcome == SKIP) { + ++n_skipped; + if (opt_verbosity>0 && !opt_forked) + puts("SKIPPED"); + } else { + ++n_bad; + if (!opt_forked) + printf("\n [%s FAILED]\n", testcase->name); + } + + if (opt_forked) { + exit(outcome==OK ? 0 : (outcome==SKIP?MAGIC_EXITCODE : 1)); + } else { + return (int)outcome; + } +} + +int +_tinytest_set_flag(struct testgroup_t *groups, const char *arg, unsigned long flag) +{ + int i, j; + size_t length = LONGEST_TEST_NAME; + char fullname[LONGEST_TEST_NAME]; + int found=0; + if (strstr(arg, "..")) + length = strstr(arg,"..")-arg; + for (i=0; groups[i].prefix; ++i) { + for (j=0; groups[i].cases[j].name; ++j) { + snprintf(fullname, sizeof(fullname), "%s%s", + groups[i].prefix, groups[i].cases[j].name); + if (!flag) /* Hack! */ + printf(" %s\n", fullname); + if (!strncmp(fullname, arg, length)) { + groups[i].cases[j].flags |= flag; + ++found; + } + } + } + return found; +} + +static void +usage(struct testgroup_t *groups, int list_groups) +{ + puts("Options are: [--verbose|--quiet|--terse] [--no-fork]"); + puts(" Specify tests by name, or using a prefix ending with '..'"); + puts(" Use --list-tests for a list of tests."); + if (list_groups) { + puts("Known tests are:"); + _tinytest_set_flag(groups, "..", 0); + } + exit(0); +} + +int +tinytest_main(int c, const char **v, struct testgroup_t *groups) +{ + int i, j, n=0; + +#ifdef WIN32 + commandname = v[0]; +#endif + for (i=1; i<c; ++i) { + if (v[i][0] == '-') { + if (!strcmp(v[i], "--RUNNING-FORKED")) { + opt_forked = 1; + } else if (!strcmp(v[i], "--no-fork")) { + opt_nofork = 1; + } else if (!strcmp(v[i], "--quiet")) { + opt_verbosity = -1; + verbosity_flag = "--quiet"; + } else if (!strcmp(v[i], "--verbose")) { + opt_verbosity = 2; + verbosity_flag = "--verbose"; + } else if (!strcmp(v[i], "--terse")) { + opt_verbosity = 0; + verbosity_flag = "--terse"; + } else if (!strcmp(v[i], "--help")) { + usage(groups, 0); + } else if (!strcmp(v[i], "--list-tests")) { + usage(groups, 1); + } else { + printf("Unknown option %s. Try --help\n",v[i]); + return -1; + } + } else { + ++n; + if (!_tinytest_set_flag(groups, v[i], _TT_ENABLED)) { + printf("No such test as %s!\n", v[i]); + return -1; + } + } + } + if (!n) + _tinytest_set_flag(groups, "..", _TT_ENABLED); + + setvbuf(stdout, NULL, _IONBF, 0); + + ++in_tinytest_main; + for (i=0; groups[i].prefix; ++i) + for (j=0; groups[i].cases[j].name; ++j) + if (groups[i].cases[j].flags & _TT_ENABLED) + testcase_run_one(&groups[i], + &groups[i].cases[j]); + + --in_tinytest_main; + + if (opt_verbosity==0) + puts(""); + + if (n_bad) + printf("%d/%d TESTS FAILED. (%d skipped)\n", n_bad, + n_bad+n_ok,n_skipped); + else if (opt_verbosity >= 1) + printf("%d tests ok. (%d skipped)\n", n_ok, n_skipped); + + return (n_bad == 0) ? 0 : 1; +} + +int +_tinytest_get_verbosity(void) +{ + return opt_verbosity; +} + +void +_tinytest_set_test_failed(void) +{ + if (opt_verbosity <= 0 && cur_test_name) { + if (opt_verbosity==0) puts(""); + printf("%s%s: ", cur_test_prefix, cur_test_name); + cur_test_name = NULL; + } + cur_test_outcome = 0; +} + +void +_tinytest_set_test_skipped(void) +{ + if (cur_test_outcome==OK) + cur_test_outcome = SKIP; +} + diff --git a/src/test/tinytest.h b/src/test/tinytest.h new file mode 100644 index 0000000000..a0cb913138 --- /dev/null +++ b/src/test/tinytest.h @@ -0,0 +1,87 @@ +/* tinytest.h -- Copyright 2009 Nick Mathewson + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _TINYTEST_H +#define _TINYTEST_H + +/** Flag for a test that needs to run in a subprocess. */ +#define TT_FORK (1<<0) +/** Runtime flag for a test we've decided to skip. */ +#define TT_SKIP (1<<1) +/** Internal runtime flag for a test we've decided to run. */ +#define _TT_ENABLED (1<<2) +/** If you add your own flags, make them start at this point. */ +#define TT_FIRST_USER_FLAG (1<<3) + +typedef void (*testcase_fn)(void *); + +struct testcase_t; + +/** Functions to initialize/teardown a structure for a testcase. */ +struct testcase_setup_t { + /** Return a new structure for use by a given testcase. */ + void *(*setup_fn)(const struct testcase_t *); + /** Clean/free a structure from setup_fn. Return 1 if ok, 0 on err. */ + int (*cleanup_fn)(const struct testcase_t *, void *); +}; + +/** A single test-case that you can run. */ +struct testcase_t { + const char *name; /**< An identifier for this case. */ + testcase_fn fn; /**< The function to run to implement this case. */ + unsigned long flags; /**< Bitfield of TT_* flags. */ + const struct testcase_setup_t *setup; /**< Optional setup/cleanup fns*/ + void *setup_data; /**< Extra data usable by setup function */ +}; +#define END_OF_TESTCASES { NULL, NULL, 0, NULL, NULL } + +/** A group of tests that are selectable together. */ +struct testgroup_t { + const char *prefix; /**< Prefix to prepend to testnames. */ + struct testcase_t *cases; /** Array, ending with END_OF_TESTCASES */ +}; +#define END_OF_GROUPS { NULL, NULL} + +/** Implementation: called from a test to indicate failure, before logging. */ +void _tinytest_set_test_failed(void); +/** Implementation: called from a test to indicate that we're skipping. */ +void _tinytest_set_test_skipped(void); +/** Implementation: return 0 for quiet, 1 for normal, 2 for loud. */ +int _tinytest_get_verbosity(void); +/** Implementation: Set a flag on tests matching a name; returns number + * of tests that matched. */ +int _tinytest_set_flag(struct testgroup_t *, const char *, unsigned long); + +/** Set all tests in 'groups' matching the name 'named' to be skipped. */ +#define tinytest_skip(groups, named) \ + _tinytest_set_flag(groups, named, TT_SKIP) + +/** Run a single testcase in a single group. */ +int testcase_run_one(const struct testgroup_t *,const struct testcase_t *); +/** Run a set of testcases from an END_OF_GROUPS-terminated array of groups, + as selected from the command line. */ +int tinytest_main(int argc, const char **argv, struct testgroup_t *groups); + +#endif diff --git a/src/test/tinytest_demo.c b/src/test/tinytest_demo.c new file mode 100644 index 0000000000..0117176eb4 --- /dev/null +++ b/src/test/tinytest_demo.c @@ -0,0 +1,215 @@ +/* tinytest_demo.c -- Copyright 2009 Nick Mathewson + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + + +/* Welcome to the example file for tinytest! I'll show you how to set up + * some simple and not-so-simple testcases. */ + +/* Make sure you include these headers. */ +#include "tinytest.h" +#include "tinytest_macros.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <errno.h> + +/* ============================================================ */ + +/* First, let's see if strcmp is working. (All your test cases should be + * functions declared to take a single void * as) an argument. */ +void +test_strcmp(void *data) +{ + (void)data; /* This testcase takes no data. */ + + /* Let's make sure the empty string is equal to itself */ + if (strcmp("","")) { + /* This macro tells tinytest to stop the current test + * and go straight to the "end" label. */ + tt_abort_msg("The empty string was not equal to itself"); + } + + /* Pretty often, calling tt_abort_msg to indicate failure is more + heavy-weight than you want. Instead, just say: */ + tt_assert(strcmp("testcase", "testcase") == 0); + + /* Occasionally, you don't want to stop the current testcase just + because a single assertion has failed. In that case, use + tt_want: */ + tt_want(strcmp("tinytest", "testcase") > 0); + + /* You can use the tt_*_op family of macros to compare values and to + fail unless they have the relationship you want. They produce + more useful output than tt_assert, since they display the actual + values of the failing things. + + Fail unless strcmp("abc, "abc") == 0 */ + tt_int_op(strcmp("abc", "abc"), ==, 0); + + /* Fail unless strcmp("abc, "abcd") is less than 0 */ + tt_int_op(strcmp("abc", "abcd"), < , 0); + + /* Incidentally, there's a test_str_op that uses strcmp internally. */ + tt_str_op("abc", <, "abcd"); + + + /* Every test-case function needs to finish with an "end:" + label and (optionally) code to clean up local variables. */ + end: + ; +} + +/* ============================================================ */ + +/* Now let's mess with setup and teardown functions! These are handy if + you have a bunch of tests that all need a similar environment, and you + wnat to reconstruct that environment freshly for each one. */ + +/* First you declare a type to hold the environment info, and functions to + set it up and tear it down. */ +struct data_buffer { + /* We're just going to have couple of character buffer. Using + setup/teardown functions is probably overkill for this case. + + You could also do file descriptors, complicated handles, temporary + files, etc. */ + char buffer1[512]; + char buffer2[512]; +}; +/* The setup function needs to take a const struct testcase_t and return + void* */ +void * +setup_data_buffer(const struct testcase_t *testcase) +{ + struct data_buffer *db = malloc(sizeof(struct data_buffer)); + + /* If you had a complicated set of setup rules, you might behave + differently here depending on testcase->flags or + testcase->setup_data or even or testcase->name. */ + + /* Returning a NULL here would mean that we couldn't set up for this + test, so we don't need to test db for null. */ + return db; +} +/* The clean function deallocates storage carefully and returns true on + success. */ +int +clean_data_buffer(const struct testcase_t *testcase, void *ptr) +{ + struct data_buffer *db = ptr; + + if (db) { + free(db); + return 1; + } + return 0; +} +/* Finally, declare a testcase_setup_t with these functions. */ +struct testcase_setup_t data_buffer_setup = { + setup_data_buffer, clean_data_buffer +}; + + +/* Now let's write our test. */ +void +test_memcpy(void *ptr) +{ + /* This time, we use the argument. */ + struct data_buffer *db = ptr; + + /* We'll also introduce a local variable that might need cleaning up. */ + char *mem = NULL; + + /* Let's make sure that memcpy does what we'd like. */ + strcpy(db->buffer1, "String 0"); + memcpy(db->buffer2, db->buffer1, sizeof(db->buffer1)); + tt_str_op(db->buffer1, ==, db->buffer2); + + /* Now we've allocated memory that's referenced by a local variable. + The end block of the function will clean it up. */ + mem = strdup("Hello world."); + tt_assert(mem); + + /* Another rather trivial test. */ + tt_str_op(db->buffer1, !=, mem); + + end: + /* This time our end block has something to do. */ + if (mem) + free(mem); +} + +/* ============================================================ */ + +/* Now we need to make sure that our tests get invoked. First, you take + a bunch of related tests and put them into an array of struct testcase_t. +*/ + +struct testcase_t demo_tests[] = { + /* Here's a really simple test: it has a name you can refer to it + with, and a function to invoke it. */ + { "strcmp", test_strcmp, }, + + /* The second test has a flag, "TT_FORK", to make it run in a + subprocess, and a pointer to the testcase_setup_t that configures + its environment. */ + { "memcpy", test_memcpy, TT_FORK, &data_buffer_setup }, + + /* The array has to end with END_OF_TESTCASES. */ + END_OF_TESTCASES +}; + +/* Next, we make an array of testgroups. This is mandatory. Unlike more + heavy-duty testing frameworks, groups can't nest. */ +struct testgroup_t groups[] = { + + /* Every group has a 'prefix', and an array of tests. That's it. */ + { "demo/", demo_tests }, + + END_OF_GROUPS +}; + + +int +main(int c, const char **v) +{ + /* Finally, just call tinytest_main(). It lets you specify verbose + or quiet output with --verbose and --quiet. You can list + specific tests: + + tinytest-demo demo/memcpy + + or use a ..-wildcard to select multiple tests with a common + prefix: + + tinytest-demo demo/.. + + If you list no tests, you get them all by default, so that + "tinytest-demo" and "tinytest-demo .." mean the same thing. + + */ + return tinytest_main(c, v, groups); +} diff --git a/src/test/tinytest_macros.h b/src/test/tinytest_macros.h new file mode 100644 index 0000000000..48c1fbdfb8 --- /dev/null +++ b/src/test/tinytest_macros.h @@ -0,0 +1,167 @@ +/* tinytest_macros.h -- Copyright 2009 Nick Mathewson + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _TINYTEST_MACROS_H +#define _TINYTEST_MACROS_H + +/* Helpers for defining statement-like macros */ +#define TT_STMT_BEGIN do { +#define TT_STMT_END } while(0) + +/* Redefine this if your test functions want to abort with something besides + * "goto end;" */ +#ifndef TT_EXIT_TEST_FUNCTION +#define TT_EXIT_TEST_FUNCTION TT_STMT_BEGIN goto end; TT_STMT_END +#endif + +/* Redefine this if you want to note success/failure in some different way. */ +#ifndef TT_DECLARE +#define TT_DECLARE(prefix, args) \ + TT_STMT_BEGIN \ + printf("\n %s %s:%d: ",prefix,__FILE__,__LINE__); \ + printf args ; \ + TT_STMT_END +#endif + +/* Announce a failure. Args are parenthesized printf args. */ +#define TT_GRIPE(args) TT_DECLARE("FAIL", args) + +/* Announce a non-failure if we're verbose. */ +#define TT_BLATHER(args) \ + TT_STMT_BEGIN \ + if (_tinytest_get_verbosity()>1) TT_DECLARE(" OK", args); \ + TT_STMT_END + +#define TT_DIE(args) \ + TT_STMT_BEGIN \ + _tinytest_set_test_failed(); \ + TT_GRIPE(args); \ + TT_EXIT_TEST_FUNCTION; \ + TT_STMT_END + +#define TT_FAIL(args) \ + TT_STMT_BEGIN \ + _tinytest_set_test_failed(); \ + TT_GRIPE(args); \ + TT_STMT_END + +/* Fail and abort the current test for the reason in msg */ +#define tt_abort_printf(msg) TT_DIE(msg) +#define tt_abort_perror(op) TT_DIE(("%s: %s [%d]",(op),strerror(errno), errno)) +#define tt_abort_msg(msg) TT_DIE(("%s", msg)) +#define tt_abort() TT_DIE(("%s", "(Failed.)")) + +/* Fail but do not abort the current test for the reason in msg. */ +#define tt_fail_printf(msg) TT_FAIL(msg) +#define tt_fail_perror(op) TT_FAIL(("%s: %s [%d]",(op),strerror(errno), errno)) +#define tt_fail_msg(msg) TT_FAIL(("%s", msg)) +#define tt_fail() TT_FAIL(("%s", "(Failed.)")) + +/* End the current test, and indicate we are skipping it. */ +#define tt_skip() \ + TT_STMT_BEGIN \ + _tinytest_set_test_skipped(); \ + TT_EXIT_TEST_FUNCTION; \ + TT_STMT_END + +#define _tt_want(b, msg, fail) \ + TT_STMT_BEGIN \ + if (!(b)) { \ + _tinytest_set_test_failed(); \ + TT_GRIPE((msg)); \ + fail; \ + } else { \ + TT_BLATHER((msg)); \ + } \ + TT_STMT_END + +/* Assert b, but do not stop the test if b fails. Log msg on failure. */ +#define tt_want_msg(b, msg) \ + _tt_want(b, msg, ); + +/* Assert b and stop the test if b fails. Log msg on failure. */ +#define tt_assert_msg(b, msg) \ + _tt_want(b, msg, TT_EXIT_TEST_FUNCTION); + +/* Assert b, but do not stop the test if b fails. */ +#define tt_want(b) tt_want_msg( (b), "want("#b")") +/* Assert b, and stop the test if b fails. */ +#define tt_assert(b) tt_assert_msg((b), "assert("#b")") + +#define tt_assert_test_fmt_type(a,b,str_test,type,test,printf_type,printf_fmt, \ + setup_block,cleanup_block) \ + TT_STMT_BEGIN \ + type _val1 = (type)(a); \ + type _val2 = (type)(b); \ + int _tt_status = (test); \ + if (!_tt_status || _tinytest_get_verbosity()>1) { \ + printf_type _print; \ + printf_type _print1; \ + printf_type _print2; \ + type _value = _val1; \ + setup_block; \ + _print1 = _print; \ + _value = _val2; \ + setup_block; \ + _print2 = _print; \ + TT_DECLARE(_tt_status?" OK":"FAIL", \ + ("assert(%s): "printf_fmt" vs "printf_fmt, \ + str_test, _print1, _print2)); \ + _print = _print1; \ + cleanup_block; \ + _print = _print2; \ + cleanup_block; \ + if (!_tt_status) { \ + _tinytest_set_test_failed(); \ + TT_EXIT_TEST_FUNCTION; \ + } \ + } \ + TT_STMT_END + +#define tt_assert_test_type(a,b,str_test,type,test,fmt) \ + tt_assert_test_fmt_type(a,b,str_test,type,test,type,fmt, \ + {_print=_value;},{}) + +/* Helper: assert that a op b, when cast to type. Format the values with + * printf format fmt on failure. */ +#define tt_assert_op_type(a,op,b,type,fmt) \ + tt_assert_test_type(a,b,#a" "#op" "#b,type,(_val1 op _val2),fmt) + +#define tt_int_op(a,op,b) \ + tt_assert_test_type(a,b,#a" "#op" "#b,long,(_val1 op _val2),"%ld") + +#define tt_uint_op(a,op,b) \ + tt_assert_test_type(a,b,#a" "#op" "#b,unsigned long, \ + (_val1 op _val2),"%lu") + +#define tt_ptr_op(a,op,b) \ + tt_assert_test_type(a,b,#a" "#op" "#b,void*, \ + (_val1 op _val2),"%p") + +#define tt_str_op(a,op,b) \ + tt_assert_test_type(a,b,#a" "#op" "#b,const char *, \ + (strcmp(_val1,_val2) op 0),"<%s>") + +#endif diff --git a/src/tools/Makefile.am b/src/tools/Makefile.am index 41786e4378..be03f8d892 100644 --- a/src/tools/Makefile.am +++ b/src/tools/Makefile.am @@ -3,16 +3,16 @@ noinst_PROGRAMS = tor-checkkey tor_resolve_SOURCES = tor-resolve.c tor_resolve_LDFLAGS = @TOR_LDFLAGS_libevent@ -tor_resolve_LDADD = ../common/libor.a @TOR_LIBEVENT_LIBS@ @TOR_LIB_WS32@ +tor_resolve_LDADD = ../common/libor.a -lm @TOR_LIBEVENT_LIBS@ @TOR_LIB_WS32@ tor_gencert_SOURCES = tor-gencert.c tor_gencert_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ \ @TOR_LDFLAGS_libevent@ tor_gencert_LDADD = ../common/libor.a ../common/libor-crypto.a \ - -lz @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ + -lm -lz @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ tor_checkkey_SOURCES = tor-checkkey.c tor_checkkey_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ \ @TOR_LDFLAGS_libevent@ tor_checkkey_LDADD = ../common/libor.a ../common/libor-crypto.a \ - -lz @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ + -lm -lz @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ diff --git a/src/tools/tor-checkkey.c b/src/tools/tor-checkkey.c index b29b52d8db..739f7332df 100644 --- a/src/tools/tor-checkkey.c +++ b/src/tools/tor-checkkey.c @@ -7,7 +7,7 @@ #include <stdlib.h> #include "crypto.h" #include "log.h" -#include "util.h" +#include "../common/util.h" #include "compat.h" #include <openssl/bn.h> #include <openssl/rsa.h> @@ -29,7 +29,7 @@ int main(int c, char **v) return 1; } - if (crypto_global_init(0)) { + if (crypto_global_init(0, NULL, NULL)) { fprintf(stderr, "Couldn't initialize crypto library.\n"); return 1; } diff --git a/src/tools/tor-gencert.c b/src/tools/tor-gencert.c index c7d9282076..04d53be072 100644 --- a/src/tools/tor-gencert.c +++ b/src/tools/tor-gencert.c @@ -13,6 +13,7 @@ #include <openssl/evp.h> #include <openssl/pem.h> +#include <openssl/rsa.h> #include <openssl/objects.h> #include <openssl/obj_mac.h> #include <openssl/err.h> @@ -27,8 +28,8 @@ #define CRYPTO_PRIVATE #include "compat.h" -#include "util.h" -#include "log.h" +#include "../common/util.h" +#include "../common/log.h" #include "crypto.h" #include "address.h" @@ -218,6 +219,20 @@ parse_commandline(int argc, char **argv) return 0; } +static RSA * +generate_key(int bits) +{ + RSA *rsa = NULL; + crypto_pk_env_t *env = crypto_new_pk_env(); + if (crypto_pk_generate_key_with_bits(env,bits)<0) + goto done; + rsa = _crypto_pk_env_get_rsa(env); + rsa = RSAPrivateKey_dup(rsa); + done: + crypto_free_pk_env(env); + return rsa; +} + /** Try to read the identity key from <b>identity_key_file</b>. If no such * file exists and create_identity_key is set, make a new identity key and * store it. Return 0 on success, nonzero on failure. @@ -238,7 +253,7 @@ load_identity_key(void) } log_notice(LD_GENERAL, "Generating %d-bit RSA identity key.", IDENTITY_KEY_BITS); - if (!(key = RSA_generate_key(IDENTITY_KEY_BITS, 65537, NULL, NULL))) { + if (!(key = generate_key(IDENTITY_KEY_BITS))) { log_err(LD_GENERAL, "Couldn't generate identity key."); crypto_log_errors(LOG_ERR, "Generating identity key"); return 1; @@ -323,7 +338,7 @@ generate_signing_key(void) RSA *key; log_notice(LD_GENERAL, "Generating %d-bit RSA signing key.", SIGNING_KEY_BITS); - if (!(key = RSA_generate_key(SIGNING_KEY_BITS, 65537, NULL, NULL))) { + if (!(key = generate_key(SIGNING_KEY_BITS))) { log_err(LD_GENERAL, "Couldn't generate signing key."); crypto_log_errors(LOG_ERR, "Generating signing key"); return 1; @@ -496,7 +511,7 @@ main(int argc, char **argv) init_logging(); /* Don't bother using acceleration. */ - if (crypto_global_init(0)) { + if (crypto_global_init(0, NULL, NULL)) { fprintf(stderr, "Couldn't initialize crypto library.\n"); return 1; } diff --git a/src/tools/tor-resolve.c b/src/tools/tor-resolve.c index f12c3d8dd3..fe7f793dbb 100644 --- a/src/tools/tor-resolve.c +++ b/src/tools/tor-resolve.c @@ -6,9 +6,9 @@ #include "orconfig.h" #include "compat.h" -#include "util.h" +#include "../common/util.h" #include "address.h" -#include "log.h" +#include "../common/log.h" #include <stdio.h> #include <stdlib.h> diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h index dec1a9c459..67e21ae22b 100644 --- a/src/win32/orconfig.h +++ b/src/win32/orconfig.h @@ -226,6 +226,5 @@ #define USING_TWOS_COMPLEMENT /* Version number of package */ -#define VERSION "0.2.1.22" - +#define VERSION "0.2.2.7-alpha" diff --git a/tor.spec.in b/tor.spec.in index 54b9be092a..3d78e4d12c 100644 --- a/tor.spec.in +++ b/tor.spec.in @@ -15,13 +15,6 @@ %define toruser @TORUSER@ %define torgroup @TORGROUP@ -## Target a specific arch and OS -# -# default is i386 linux - -## Override any system rpm macros -# - ## Version song and dance # # This should be the Tor version number, as it appears on the tarball, @@ -54,14 +47,11 @@ %if %{is_fc} %define ostag %(sed -e 's/^.*release /fc/' -e 's/ .*$//' -e 's/\\./_/g' < /etc/fedora-release) -%else +%endif + %if %{is_rh} %define ostag %(sed -e 's/^.*release /rh/' -e 's/ .*$//' -e 's/\\./_/g' < /etc/redhat-release) %endif -%endif - -# These are probably wrong... just placeholders should we actually -# end up supporting these distributions %if %{is_mdk} %define ostag mdk @@ -290,6 +280,9 @@ exit 0 %changelog +* Fri May 01 2009 Andrew Lewman <andrew@torproject.org> +- clean up distro detection and remove dead comment blocks + * Sun Feb 22 2009 Andrew Lewman <andrew@torproject.org> - update the description, vendor, and packager |