diff options
65 files changed, 4445 insertions, 452 deletions
diff --git a/.gitignore b/.gitignore index b5d4442ca6..369d925a9d 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,7 @@ /aclocal.m4 /autom4te.cache /build-stamp +/compile /configure /Doxyfile /orconfig.h @@ -97,6 +98,11 @@ /doc/tor.html /doc/tor.html.in /doc/tor.1.xml +/doc/tor-fw-helper.1 +/doc/tor-fw-helper.1.in +/doc/tor-fw-helper.html +/doc/tor-fw-helper.html.in +/doc/tor-fw-helper.1.xml /doc/tor-gencert.1 /doc/tor-gencert.1.in /doc/tor-gencert.html @@ -161,6 +167,12 @@ /src/tools/Makefile /src/tools/Makefile.in +# /src/tools/tor-fw-helper/ +/src/tools/tor-fw-helper/tor-fw-helper +/src/tools/tor-fw-helper/tor-fw-helper.exe +/src/tools/tor-fw-helper/Makefile +/src/tools/tor-fw-helper/Makefile.in + # /src/win32/ /src/win32/Makefile /src/win32/Makefile.in diff --git a/Makefile.am b/Makefile.am index cb85d2559e..5caae4cdc6 100644 --- a/Makefile.am +++ b/Makefile.am @@ -3,8 +3,9 @@ # Copyright (c) 2007-2010, The Tor Project, Inc. # See LICENSE for licensing information -AUTOMAKE_OPTIONS = foreign - # else it keeps trying to put COPYING back in +# "foreign" means we don't follow GNU package layout standards +# 1.6 means we require automake vesion 1.6 +AUTOMAKE_OPTIONS = foreign 1.6 SUBDIRS = src doc contrib @@ -68,7 +69,8 @@ check-spaces: src/common/*.h \ src/common/[^asO]*.c src/common/address.c \ src/or/[^e]*.[ch] src/or/eventdns_tor.h \ - src/test/test*.[ch] src/tools/*.[ch] + src/test/test*.[ch] src/tools/*.[ch] \ + src/tools/tor-fw-helper/*.[ch] check-docs: ./contrib/checkOptionDocs.pl diff --git a/autogen.sh b/autogen.sh index eb9395c719..0592f16c2e 100755 --- a/autogen.sh +++ b/autogen.sh @@ -1,5 +1,9 @@ #!/bin/sh +if [ -x "`which autoreconf 2>/dev/null`" ] ; then + exec autoreconf -ivf +fi + set -e # Run this to generate all the initial makefiles, etc. diff --git a/changes/automake_required b/changes/automake_required new file mode 100644 index 0000000000..bb5035518f --- /dev/null +++ b/changes/automake_required @@ -0,0 +1,6 @@ + o Build changes + - Our build system requires automake 1.6 or later to create the + Makefile.in files. Previously, you could have used 1.4. + This only affects developers and people building Tor from git; + people who build Tor from the source distribution without changing + the Makefile.am files should be fine. diff --git a/changes/autoreconf b/changes/autoreconf new file mode 100644 index 0000000000..c08cc7a376 --- /dev/null +++ b/changes/autoreconf @@ -0,0 +1,5 @@ + o Minor build changes: + - Use autoreconf to launch autoconf, automake, etc from autogen.sh. + This is more robust against some of the failure modes associated + with running the autotools chain on its own. + diff --git a/changes/bufferevent-support b/changes/bufferevent-support new file mode 100644 index 0000000000..e39c36f13b --- /dev/null +++ b/changes/bufferevent-support @@ -0,0 +1,12 @@ + o Major features + - Tor can now optionally build with the "bufferevents" buffered IO + backend provided by Libevent, when building with Libevent 2.0.7-rc + or later. To use this feature, make sure you have the latest possible + version of Libevent, and run autoconf with the --enable-bufferevents + flag. Using this feature will make our networking code more flexible, + lets us stack layers on each other, and let us use more efficient + zero-copy transports where available. + - As an experimental feature, when using the "bufferevents" buffered + IO backend, Tor can try to use Windows's IOCP networking API. This + is off by default. To turn it on, add "DisableIOCP 0" to your torrc. + diff --git a/changes/cpudetect b/changes/cpudetect new file mode 100644 index 0000000000..eeaa492f58 --- /dev/null +++ b/changes/cpudetect @@ -0,0 +1,3 @@ + o Minor features + - If you set the NumCPUs option to 0, Tor will try to detect how many + CPUs you have. This is the new default behavior. diff --git a/changes/eventdns_routerinfo_key b/changes/eventdns_routerinfo_key new file mode 100644 index 0000000000..46dcb52dbb --- /dev/null +++ b/changes/eventdns_routerinfo_key @@ -0,0 +1,4 @@ + o Minor feature removals + - Removed old code to work around versions of Tor so old that they + used multiple forked processes to handle DNS requests. Such + versions of Tor are no longer in use as servers. diff --git a/changes/microdesc_dl b/changes/microdesc_dl new file mode 100644 index 0000000000..aca634c13f --- /dev/null +++ b/changes/microdesc_dl @@ -0,0 +1,4 @@ + o Major features: + - Caches now download and cache all the consensus flavors that + they know about. This allows them to assess which microdescriptors + they need to fetch. diff --git a/changes/tor-fw-helper b/changes/tor-fw-helper new file mode 100644 index 0000000000..f87d828b8f --- /dev/null +++ b/changes/tor-fw-helper @@ -0,0 +1,14 @@ + o Major features: + - Tor now has the ability to wrangle NAT devices like a good network cowbot + with the tor-fw-helper tool. The tor-fw-helper tool supports Apple's + NAT-PMP protocol and the UPnP standard for TCP port mapping. This + optional tool may be enabled at compile time by configuring with + '--enable-upnp' or '--enable-natpmp' or with both. This tool may be + called by hand or by Tor. By configuring the PortForwarding option, Tor + will launch the helper on a regular basis to ensure that the NAT mapping + is regularly updated. Additionally, a user may also specify an + alternative helper by using the PortForwardingHelper option. The helper + may be specified by name or with the full path to the helper. The default + helper is named 'tor-fw-helper' and any alternative helper must take the + tor-fw-helper-spec.txt into account. + diff --git a/configure.in b/configure.in index 306edc0c0f..18e312b7ae 100644 --- a/configure.in +++ b/configure.in @@ -4,7 +4,7 @@ dnl Copyright (c) 2007-2008, The Tor Project, Inc. dnl See LICENSE for licensing information AC_INIT -AM_INIT_AUTOMAKE(tor, 0.2.2.17-alpha-dev) +AM_INIT_AUTOMAKE(tor, 0.2.3.0-alpha-dev) AM_CONFIG_HEADER(orconfig.h) AC_CANONICAL_HOST @@ -59,6 +59,24 @@ AC_ARG_ENABLE(asciidoc, *) AC_MSG_ERROR(bad value for --disable-asciidoc) ;; esac], [asciidoc=true]) +# By default, we're not ready to ship a NAT-PMP aware Tor +AC_ARG_ENABLE(nat-pmp, + AS_HELP_STRING(--enable-nat-pmp, enable NAT-PMP support), + [case "${enableval}" in + yes) natpmp=true ;; + no) natpmp=false ;; + * ) AC_MSG_ERROR(bad value for --enable-nat-pmp) ;; + esac], [natpmp=false]) + +# By default, we're not ready to ship a UPnP aware Tor +AC_ARG_ENABLE(upnp, + AS_HELP_STRING(--enable-upnp, enable UPnP support), + [case "${enableval}" in + yes) upnp=true ;; + no) upnp=false ;; + * ) AC_MSG_ERROR(bad value for --enable-upnp) ;; + esac], [upnp=false]) + AC_ARG_ENABLE(threads, AS_HELP_STRING(--disable-threads, disable multi-threading support)) @@ -118,6 +136,9 @@ if test "$enable_local_appdata" = "yes"; then [Defined if we default to host local appdata paths on Windows]) fi +AC_ARG_ENABLE(bufferevents, + AS_HELP_STRING(--enable-bufferevents, use Libevent's buffered IO.)) + AC_PROG_CC AC_PROG_CPP AC_PROG_MAKE_SET @@ -132,6 +153,11 @@ AC_PATH_PROG([A2X], [a2x], none) AM_CONDITIONAL(USE_ASCIIDOC, test x$asciidoc = xtrue) +AM_CONDITIONAL(USE_FW_HELPER, test x$natpmp = xtrue || test x$upnp = xtrue) +AM_CONDITIONAL(NAT_PMP, test x$natpmp = xtrue) +AM_CONDITIONAL(MINIUPNPC, test x$upnp = xtrue) +AM_PROG_CC_C_O + AC_PATH_PROG([SHA1SUM], [sha1sum], none) AC_PATH_PROG([OPENSSL], [openssl], none) @@ -223,7 +249,7 @@ dnl ------------------------------------------------------------------- dnl Check for functions before libevent, since libevent-1.2 apparently dnl exports strlcpy without defining it in a header. -AC_CHECK_FUNCS(gettimeofday ftime socketpair uname inet_aton strptime getrlimit strlcat strlcpy strtoull getaddrinfo localtime_r gmtime_r memmem strtok_r writev readv flock prctl vasprintf) +AC_CHECK_FUNCS(gettimeofday ftime socketpair uname inet_aton strptime getrlimit strlcat strlcpy strtoull getaddrinfo localtime_r gmtime_r memmem strtok_r flock prctl vasprintf sysconf) using_custom_malloc=no if test x$enable_openbsd_malloc = xyes ; then @@ -302,7 +328,7 @@ AC_CHECK_MEMBERS([struct event.min_heap_idx], , , [#include <event.h> ]) -AC_CHECK_HEADERS(event2/event.h event2/dns.h) +AC_CHECK_HEADERS(event2/event.h event2/dns.h event2/bufferevent_ssl.h) LIBS="$save_LIBS" LDFLAGS="$save_LDFLAGS" @@ -322,6 +348,54 @@ else fi AC_SUBST(TOR_LIBEVENT_LIBS) +dnl This isn't the best test for Libevent 2.0.3-alpha. Once it's released, +dnl we can do much better. +if test "$enable_bufferevents" = "yes" ; then + if test "$ac_cv_header_event2_bufferevent_ssl_h" != "yes" ; then + AC_MSG_ERROR([You've asked for bufferevent support, but you're using a version of Libevent without SSL support. This won't work. We need Libevent 2.0.7-rc or later, and you don't seem to even have Libevent 2.0.3-alpha.]) + else + + CPPFLAGS="$CPPFLAGS $TOR_CPPFLAGS_libevent" + + # Check for the right version. First see if version detection works. + AC_MSG_CHECKING([whether we can detect the Libevent version]) + AC_COMPILE_IFELSE([ +#include <event2/event.h> +#if !defined(LIBEVENT_VERSION_NUMBER) || LIBEVENT_VERSION_NUMBER < 10 +#error +int x = y(zz); +#else +int x = 1; +#endif + ], [event_version_number_works=yes; AC_MSG_RESULT([yes]) ], + [event_version_number_works=no; AC_MSG_RESULT([no])]) + if test "$event_version_number_works" != 'yes'; then + AC_MSG_WARN([Version detection on Libevent seems broken. Your Libevent installation is probably screwed up or very old.]) + else + AC_MSG_CHECKING([whether Libevent is new enough for bufferevents]) + AC_COMPILE_IFELSE([ +#include <event2/event.h> +#if !defined(LIBEVENT_VERSION_NUMBER) || LIBEVENT_VERSION_NUMBER < 0x02000700 +#error +int x = y(zz); +#else +int x = 1; +#endif + ], [ AC_MSG_RESULT([yes]) ], + [ AC_MSG_RESULT([no]) + AC_MSG_ERROR([Libevent does not seem new enough to support bufferevents. We require 2.0.7-rc or later]) ] ) + fi + fi +fi + +LIBS="$save_LIBS" +LDFLAGS="$save_LDFLAGS" +CPPFLAGS="$save_CPPFLAGS" + +AM_CONDITIONAL(USE_BUFFEREVENTS, test "$enable_bufferevents" = "yes") +if test "$enable_bufferevents" = "yes"; then + AC_DEFINE(USE_BUFFEREVENTS, 1, [Defined if we're going to use Libevent's buffered IO API]) +fi dnl ------------------------------------------------------ dnl Where do you live, openssl? And how do we call you? @@ -389,11 +463,49 @@ AC_SUBST(TOR_ZLIB_LIBS) dnl Make sure to enable support for large off_t if available. + +dnl ------------------------------------------------------ +dnl Where do you live, libnatpmp? And how do we call you? +dnl There are no packages for Debian or Redhat as of this patch + +if test "$natpmp" = "true"; then + AC_DEFINE(NAT_PMP, 1, [Define to 1 if we are building with nat-pmp.]) + TOR_SEARCH_LIBRARY(libnatpmp, $trylibnatpmpdir, [-lnatpmp], + [#include <natpmp.h>], + [#include <natpmp.h>], + [ int r; + natpmp_t natpmp; + natpmpresp_t response; + r = initnatpmp(&natpmp);], + [printf("initnatpmp() returned %d (%s)\n", r, r?"FAILED":"SUCCESS"); + exit(0);], + [--with-libnatpmp-dir], + [/usr/lib/]) +fi + + +dnl ------------------------------------------------------ +dnl Where do you live, libminiupnpc? And how do we call you? +dnl There are no packages for Debian or Redhat as of this patch + +if test "$upnp" = "true"; then + AC_DEFINE(MINIUPNPC, 1, [Define to 1 if we are building with UPnP.]) + TOR_SEARCH_LIBRARY(libminiupnpc, $trylibminiupnpcdir, [-lminiupnpc], + [#include <miniupnpc/miniwget.h> + #include <miniupnpc/miniupnpc.h> + #include <miniupnpc/upnpcommands.h>], + [void upnpDiscover(int delay, const char * multicastif, + const char * minissdpdsock, int sameport);], + [upnpDiscover(1, 0, 0, 0); exit(0);], + [--with-libminiupnpc-dir], + [/usr/lib/]) +fi + AC_SYS_LARGEFILE AC_CHECK_HEADERS(unistd.h string.h signal.h sys/stat.h sys/types.h fcntl.h sys/fcntl.h sys/time.h errno.h assert.h time.h, , AC_MSG_WARN(Some headers were not found, compilation may fail. If compilation succeeds, please send your orconfig.h to the developers so we can fix this warning.)) -AC_CHECK_HEADERS(netdb.h sys/ioctl.h sys/socket.h arpa/inet.h netinet/in.h pwd.h grp.h sys/un.h sys/uio.h) +AC_CHECK_HEADERS(netdb.h sys/ioctl.h sys/socket.h arpa/inet.h netinet/in.h pwd.h grp.h sys/un.h) dnl These headers are not essential @@ -916,7 +1028,7 @@ fi CPPFLAGS="$CPPFLAGS $TOR_CPPFLAGS_libevent $TOR_CPPFLAGS_openssl $TOR_CPPFLAGS_zlib" -AC_CONFIG_FILES([Makefile tor.spec Doxyfile contrib/tor.sh contrib/torctl contrib/torify contrib/tor.logrotate contrib/Makefile contrib/osx/Makefile contrib/osx/TorBundleDesc.plist contrib/osx/TorBundleInfo.plist contrib/osx/TorDesc.plist contrib/osx/TorInfo.plist contrib/osx/TorStartupDesc.plist src/config/torrc.sample src/Makefile doc/Makefile doc/spec/Makefile src/config/Makefile src/common/Makefile src/or/Makefile src/test/Makefile src/win32/Makefile src/tools/Makefile contrib/suse/Makefile contrib/suse/tor.sh]) +AC_CONFIG_FILES([Makefile tor.spec Doxyfile contrib/tor.sh contrib/torctl contrib/torify contrib/tor.logrotate contrib/Makefile contrib/osx/Makefile contrib/osx/TorBundleDesc.plist contrib/osx/TorBundleInfo.plist contrib/osx/TorDesc.plist contrib/osx/TorInfo.plist contrib/osx/TorStartupDesc.plist src/config/torrc.sample src/Makefile doc/Makefile doc/spec/Makefile src/config/Makefile src/common/Makefile src/or/Makefile src/test/Makefile src/win32/Makefile src/tools/Makefile src/tools/tor-fw-helper/Makefile contrib/suse/Makefile contrib/suse/tor.sh]) AC_OUTPUT if test -x /usr/bin/perl && test -x ./contrib/updateVersions.pl ; then diff --git a/contrib/tor-mingw.nsi.in b/contrib/tor-mingw.nsi.in index 5485303bc5..1cecf34f8f 100644 --- a/contrib/tor-mingw.nsi.in +++ b/contrib/tor-mingw.nsi.in @@ -8,7 +8,7 @@ !include "LogicLib.nsh" !include "FileFunc.nsh" !insertmacro GetParameters -!define VERSION "0.2.2.17-alpha-dev" +!define VERSION "0.2.3.0-alpha-dev" !define INSTALLER "tor-${VERSION}-win32.exe" !define WEBSITE "https://www.torproject.org/" !define LICENSE "LICENSE" diff --git a/doc/Makefile.am b/doc/Makefile.am index 68747c8d2d..500a57c7b6 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -14,7 +14,11 @@ # just use the .1 and .html files. if USE_ASCIIDOC +if USE_FW_HELPER +asciidoc_files = tor tor-gencert tor-resolve torify tor-fw-helper +else asciidoc_files = tor tor-gencert tor-resolve torify +endif html_in = $(asciidoc_files:=.html.in) man_in = $(asciidoc_files:=.1.in) txt_in = $(asciidoc_files:=.1.txt) @@ -51,6 +55,7 @@ tor.html.in : tor.1.txt torify.html.in : torify.1.txt tor-gencert.html.in : tor-gencert.1.txt tor-resolve.html.in : tor-resolve.1.txt +tor-fw-helper.html.in : tor-fw-helper.1.txt # Generate the manpage from asciidoc, but don't do # machine-specific replacements yet @@ -61,6 +66,7 @@ tor.1.in : tor.1.txt torify.1.in : torify.1.txt tor-gencert.1.in : tor-gencert.1.txt tor-resolve.1.in : tor-resolve.1.txt +tor-fw-helper.1.in : tor-fw-helper.1.txt # use ../config.status to swap all machine-specific magic strings # in the asciidoc with their replacements. @@ -74,10 +80,12 @@ tor.1 : tor.1.in torify.1 : torify.1.in tor-gencert.1 : tor-gencert.1.in tor-resolve.1 : tor-resolve.1.in +tor-fw-helper.1 : tor-fw-helper.1.in tor.html : tor.html.in torify.html : torify.html.in tor-gencert.html : tor-gencert.html.in tor-resolve.html : tor-resolve.html.in +tor-fw-helper.html : tor-fw-helper.html.in CLEANFILES = $(asciidoc_product) config.log DISTCLEANFILES = $(html_in) $(man_in) diff --git a/doc/asciidoc-helper.sh b/doc/asciidoc-helper.sh index 00f8b8d07f..33e1360a71 100755 --- a/doc/asciidoc-helper.sh +++ b/doc/asciidoc-helper.sh @@ -17,6 +17,7 @@ output=$3 if [ "$1" = "html" ]; then input=${output%%.html.in}.1.txt base=${output%%.html.in} + if [ "$2" != none ]; then "$2" -d manpage -o $output $input; else @@ -32,7 +33,7 @@ if [ "$1" = "html" ]; then elif [ "$1" = "man" ]; then input=${output%%.1.in}.1.txt base=${output%%.1.in} - + if test "$2" = none; then echo "=================================="; echo; diff --git a/doc/spec/dir-spec.txt b/doc/spec/dir-spec.txt index 6e35deb00e..4a7a557b31 100644 --- a/doc/spec/dir-spec.txt +++ b/doc/spec/dir-spec.txt @@ -530,16 +530,8 @@ dns logic. Versions of Tor with this field set to false SHOULD NOT be used for reverse hostname lookups. - [All versions of Tor before 0.1.2.2-alpha should be assumed to have - this option set to 0 if it is not present. All Tor versions at - 0.1.2.2-alpha or later should be assumed to have this option set to - 1 if it is not present. Until 0.1.2.1-alpha-dev, this option was - not generated, even when the new DNS code was in use. Versions of Tor - before 0.1.2.1-alpha-dev did not parse this option, so it should be - marked "opt". The dnsworker logic has been removed, so this option - should not be used by new server code. However, it can still be - used, and should still be recognized by new code until Tor 0.1.2.x - is obsolete.] + [This option is obsolete. All Tor current servers should be presumed + to have the evdns backend.] "caches-extra-info" NL diff --git a/doc/spec/proposals/000-index.txt b/doc/spec/proposals/000-index.txt index f6f313e58d..8ea73540a3 100644 --- a/doc/spec/proposals/000-index.txt +++ b/doc/spec/proposals/000-index.txt @@ -94,6 +94,7 @@ Proposals by number: 172 GETINFO controller option for circuit information [ACCEPTED] 173 GETINFO Option Expansion [ACCEPTED] 174 Optimistic Data for Tor: Server Side [OPEN] +175 Automatically promoting Tor clients to nodes [DRAFT] Proposals by status: @@ -106,6 +107,7 @@ Proposals by status: 144 Increase the diversity of circuits by detecting nodes belonging the same provider 169 Eliminate TLS renegotiation for the Tor connection handshake [for 0.2.2] 170 Configuration options regarding circuit building + 175 Automatically promoting Tor clients to nodes [for ] NEEDS-REVISION: 131 Help users to verify they are using Tor OPEN: diff --git a/doc/spec/proposals/175-automatic-node-promotion.txt b/doc/spec/proposals/175-automatic-node-promotion.txt new file mode 100644 index 0000000000..c990b3f060 --- /dev/null +++ b/doc/spec/proposals/175-automatic-node-promotion.txt @@ -0,0 +1,238 @@ +Filename: 175-automatic-node-promotion.txt +Title: Automatically promoting Tor clients to nodes +Author: Steven Murdoch +Created: 12-Mar-2010 +Status: Draft + +1. Overview + + This proposal describes how Tor clients could determine when they + have sufficient bandwidth capacity and are sufficiently reliable to + become either bridges or Tor relays. When they meet this + criteria, they will automatically promote themselves, based on user + preferences. The proposal also defines the new controller messages + and options which will control this process. + + Note that for the moment, only transitions between client and + bridge are being considered. Transitions to public relay will + be considered at a future date, but will use the same + infrastructure for measuring capacity and reliability. + +2. Motivation and history + + Tor has a growing user-base and one of the major impediments to the + quality of service offered is the lack of network capacity. This is + particularly the case for bridges, because these are gradually + being blocked, and thus no longer of use to people within some + countries. By automatically promoting Tor clients to bridges, and + perhaps also to full public relays, this proposal aims to solve + these problems. + + Only Tor clients which are sufficiently useful should be promoted, + and the process of determining usefulness should be performed + without reporting the existence of the client to the central + authorities. The criteria used for determining usefulness will be + in terms of bandwidth capacity and uptime, but parameters should be + specified in the directory consensus. State stored at the client + should be in no more detail than necessary, to prevent sensitive + information being recorded. + +3. Design + +3.x Opt-in state model + + Tor can be in one of five node-promotion states: + + - off (O): Currently a client, and will stay as such + - auto (A): Currently a client, but will consider promotion + - bridge (B): Currently a bridge, and will stay as such + - auto-bridge (AB): Currently a bridge, but will consider promotion + - relay (R): Currently a public relay, and will stay as such + + The state can be fully controlled from the configuration file or + controller, but the normal state transitions are as follows: + + Any state -> off: User has opted out of node promotion + Off -> any state: Only permitted with user consent + + Auto -> auto-bridge: Tor has detected that it is sufficiently + reliable to be a *bridge* + Auto -> bridge: Tor has detected that it is sufficiently reliable + to be a *relay*, but the user has chosen to remain a *bridge* + Auto -> relay: Tor has detected that it is sufficiently reliable + to be *relay*, and will skip being a *bridge* + Auto-bridge -> relay: Tor has detected that it is sufficiently + reliable to be a *relay* + + Note that this model does not support automatic demotion. If this + is desirable, there should be some memory as to whether the + previous state was relay, bridge, or auto-bridge. Otherwise the + user may be prompted to become a relay, although he has opted to + only be a bridge. + +3.x User interaction policy + + There are a variety of options in how to involve the user into the + decision as to whether and when to perform node promotion. The + choice also may be different when Tor is running from Vidalia (and + thus can readily prompt the user for information), and standalone + (where Tor can only log messages, which may or may not be read). + + The option requiring minimal user interaction is to automatically + promote nodes according to reliability, and allow the user to opt + out, by changing settings in the configuration file or Vidalia user + interface. + + Alternatively, if a user interface is available, Tor could prompt + the user when it detects that a transition is available, and allow + the user to choose which of the available options to select. If + Vidalia is not available, it still may be possible to solicit an + email address on install, and contact the operator to ask whether + a transition to bridge or relay is permitted. + + Finally, Tor could by default not make any transition, and the user + would need to opt in by stating the maximum level (bridge or + relay) to which the node may automatically promote itself. + +3.x Performance monitoring model + + To prevent a large number of clients activating as relays, but + being too unreliable to be useful, clients should measure their + performance. If this performance meets a parameterized acceptance + criteria, a client should consider promotion. To measure + reliability, this proposal adopts a simple user model: + + - A user decides to use Tor at times which follow a Poisson + distribution + - At each time, the user will be happy if the bridge chosen has + adequate bandwidth and is reachable + - If the chosen bridge is down or slow too many times, the user + will consider Tor to be bad + + If we additionally assume that the recent history of relay + performance matches the current performance, we can measure + reliability by simulating this simple user. + + The following parameters are distributed to clients in the + directory consensus: + + - min_bandwidth: Minimum self-measured bandwidth for a node to be + considered useful, in bytes per second + - check_period: How long, in seconds, to wait between checking + reachability and bandwidth (on average) + - num_samples: Number of recent samples to keep + - num_useful: Minimum number of recent samples where the node was + reachable and had at least min_bandwidth capacity, for a client + to consider promoting to a bridge + + A different set of parameters may be used for considering when to + promote a bridge to a full relay, but this will be the subject of a + future revision of the proposal. + +3.x Performance monitoring algorithm + + The simulation described above can be implemented as follows: + + Every 60 seconds: + 1. Tor generates a random floating point number x in + the interval [0, 1). + 2. If x > (1 / (check_period / 60)) GOTO end; otherwise: + 3. Tor sets the value last_check to the current_time (in seconds) + 4. Tor measures reachability + 5. If the client is reachable, Tor measures its bandwidth + 6. If the client is reachable and the bandwidth is >= + min_bandwidth, the test has succeeded, otherwise it has failed. + 7. Tor adds the test result to the end of a ring-buffer containing + the last num_samples results: measurement_results + 8. Tor saves last_check and measurements_results to disk + 9. If the length of measurements_results == num_samples and + the number of successes >= num_useful, Tor should consider + promotion to a bridge + end. + + When Tor starts, it must fill in the samples for which it was not + running. This can only happen once the consensus has downloaded, + because the value of check_period is needed. + + 1. Tor generates a random number y from the Poisson distribution [1] + with lambda = (current_time - last_check) * (1 / check_period) + 2. Tor sets the value last_check to the current_time (in seconds) + 3. Add y test failures to the ring buffer measurements_results + 4. Tor saves last_check and measurements_results to disk + + In this way, a Tor client will measure its bandwidth and + reachability every check_period seconds, on average. Provided + check_period is sufficiently greater than a minute (say, at least an + hour), the times of check will follow a Poisson distribution. [2] + + While this does require that Tor does record the state of a client + over time, this does not leak much information. Only a binary + reachable/non-reachable is stored, and the timing of samples becomes + increasingly fuzzy as the data becomes less recent. + + On IP address changes, Tor should clear the ring-buffer, because + from the perspective of users with the old IP address, this node + might as well be a new one with no history. This policy may change + once we start allowing the bridge authority to hand out new IP + addresses given the fingerprint. + +3.x Bandwidth measurement + + Tor needs to measure its bandwidth to test the usefulness as a + bridge. A non-intrusive way to do this would be to passively measure + the peak data transfer rate since the last reachability test. Once + this exceeds min_bandwidth, Tor can set a flag that this node + currently has sufficient bandwidth to pass the bandwidth component + of the upcoming performance measurement. + + For the first version we may simply skip the bandwidth test, + because the existing reachability test sends 500 kB over several + circuits, and checks whether the node can transfer at least 50 + kB/s. This is probably good enough for a bridge, so this test + might be sufficient to record a success in the ring buffer. + +3.x New options + +3.x New controller message + +4. Migration plan + + We should start by setting a high bandwidth and uptime requirement + in the consensus, so as to avoid overloading the bridge authority + with too many bridges. Once we are confident our systems can scale, + the criteria can be gradually shifted down to gain more bridges. + +5. Related proposals + +6. Open questions: + + - What user interaction policy should we take? + + - When (if ever) should we turn a relay into an exit relay? + + - What should the rate limits be for auto-promoted bridges/relays? + Should we prompt the user for this? + + - Perhaps the bridge authority should tell potential bridges + whether to enable themselves, by taking into account whether + their IP address is blocked + + - How do we explain the possible risks of running a bridge/relay + * Use of bandwidth/congestion + * Publication of IP address + * Blocking from IRC (even for non-exit relays) + + - What feedback should we give to bridge relays, to encourage then + e.g. number of recent users (what about reserve bridges)? + + - Can clients back-off from doing these tests (yes, we should do + this) + +[1] For algorithms to generate random numbers from the Poisson + distribution, see: http://en.wikipedia.org/wiki/Poisson_distribution#Generating_Poisson-distributed_random_variables +[2] "The sample size n should be equal to or larger than 20 and the + probability of a single success, p, should be smaller than or equal to + .05. If n >= 100, the approximation is excellent if np is also <= 10." + http://www.itl.nist.gov/div898/handbook/pmc/section3/pmc331.htm (e-Handbook of Statistical Methods) + +% vim: spell ai et: diff --git a/doc/spec/tor-fw-helper-spec.txt b/doc/spec/tor-fw-helper-spec.txt new file mode 100644 index 0000000000..0068b26556 --- /dev/null +++ b/doc/spec/tor-fw-helper-spec.txt @@ -0,0 +1,57 @@ + + Tor's (little) Firewall Helper specification + Jacob Appelbaum + +0. Preface + + This document describes issues faced by Tor users who are behind NAT devices + and wish to share their resources with the rest of the Tor network. It also + explains a possible solution for some NAT devices. + +1. Overview + + Tor users often wish to relay traffic for the Tor network and their upstream + firewall thwarts their attempted generosity. Automatic port forwarding + configuration for many consumer NAT devices is often available with two common + protocols NAT-PMP[0] and UPnP[1]. + +2. Implementation + + tor-fw-helper is a program that implements basic port forwarding requests; it + may be used alone or called from Tor itself. + +2.1 Output format + + When tor-fw-helper has completed the requested action successfully, it will + report the following message to standard output: + + tor-fw-helper: SUCCESS + + If tor-fw-helper was unable to complete the requested action successfully, it + will report the following message to standard error: + + tor-fw-helper: FAILURE + + All informational messages are printed to standard output; all error messages + are printed to standard error. Messages other than SUCCESS and FAILURE + may be printed by any compliant tor-fw-helper. + +2.2 Output format stability + + The above SUCCESS and FAILURE messages are the only stable output formats + provided by this specification. tor-fw-helper-spec compliant implementations + must return SUCCESS or FAILURE as defined above. + +3. Security Concerns + + It is probably best to hand configure port forwarding and in the process, we + suggest disabling NAT-PMP and/or UPnP. This is of course absolutely confusing + to users and so we support automatic, non-authenticated NAT port mapping + protocols with compliant tor-fw-helper applications. + + NAT should not be considered a security boundary. NAT-PMP and UPnP are hacks + to deal with the shortcomings of user education about TCP/IP, IPv4 shortages, + and of course, NAT devices that suffer from horrible user interface design. + +[0] http://en.wikipedia.org/wiki/NAT_Port_Mapping_Protocol +[1] http://en.wikipedia.org/wiki/Universal_Plug_and_Play diff --git a/doc/tor-fw-helper.1.txt b/doc/tor-fw-helper.1.txt new file mode 100644 index 0000000000..49b0910380 --- /dev/null +++ b/doc/tor-fw-helper.1.txt @@ -0,0 +1,68 @@ +// Copyright (c) The Tor Project, Inc. +// See LICENSE for licensing information +// This is an asciidoc file used to generate the manpage/html reference. +// Learn asciidoc on http://www.methods.co.nz/asciidoc/userguide.html +tor-fw-helper(1) +================ +Jacob Appelbaum + +NAME +---- +tor-fw-helper - Manage upstream firewall/NAT devices + +SYNOPSIS +-------- +**tor-fw-helper** [-h|--help] [-T|--test] [-v|--verbose] [-g|--fetch-public-ip] + -i|--internal-or-port __TCP port__ [-e|--external-or-port _TCP port_] + [-d|--internal-dir-port _TCP port_] [-p|--external-dir-port _TCP port_] + +DESCRIPTION +----------- +**tor-fw-helper** currently supports Apple's NAT-PMP protocol and the UPnP +standard for TCP port mapping. It is written as the reference implementation of +tor-fw-helper-spec.txt and conforms to that loose plugin API. If your network +supports either NAT-PMP or UPnP, tor-fw-helper will attempt to automatically +map the required TCP ports for Tor's Or and Dir ports. + + +OPTIONS +------- +**-h** or **--help**:: + Display help text and exit. + +**-v**:: + Display verbose output. + +**-T** or **--test**:: + Display test information and print the test information in + tor-fw-helper.log + +**-g** or **--fetch-public-ip**:: + Fetch the the public ip address for each supported NAT helper method. + +**-i** or **--internal-or-port** __port__:: + Inform **tor-fw-helper** of your internal OR port. This is the only + required argument. + +**-e** or **--external-or-port** __port__:: + Inform **tor-fw-helper** of your external OR port. + +**-d** or **--internal-dir-port** __port__:: + Inform **tor-fw-helper** of your internal Dir port. + +**-p** or **--external-dir-port** __port__:: + Inform **tor-fw-helper** of your external Dir port. + +BUGS +---- +This probably doesn't run on Windows. That's not a big issue, since we don't +really want to deal with Windows before October 2010 anyway. + +SEE ALSO +-------- +**tor**(1) + + +See also the "tor-fw-helper-spec.txt" file, distributed with Tor. + +AUTHORS +------- + Jacob Appelbaum <jacob@torproject.org>, Steven J. Murdoch <Steven.Murdoch@cl.cam.ac.uk> diff --git a/doc/tor.1.txt b/doc/tor.1.txt index df2b17ed08..28f5a4288d 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -397,6 +397,11 @@ Other options can be specified either on the command-line (--option networkstatus. This is an advanced option; you generally shouldn't have to mess with it. (Default: not set.) +**DisableIOCP** **0**|**1**:: + If Tor was built to use the Libevent's "bufferevents" networking code + and you're running on Windows, setting this option to 1 will tell Libevent + not to use the Windows IOCP networking API. (Default: 1) + CLIENT OPTIONS -------------- @@ -855,7 +860,9 @@ is non-zero): characters inclusive, and must contain only the characters [a-zA-Z0-9]. **NumCPUs** __num__:: - How many processes to use at once for decrypting onionskins. (Default: 1) + How many processes to use at once for decrypting onionskins and other + parallelizable operations. If this is set to 0, Tor will try to detect + how many CPUs you have, defaulting to 1 if it can't tell. (Default: 0) **ORPort** __PORT__:: Advertise this port to listen for connections from Tor clients and servers. @@ -866,6 +873,18 @@ is non-zero): specified in ORPort. (Default: 0.0.0.0) This directive can be specified multiple times to bind to multiple addresses/ports. +**PortForwarding** **0**|**1**:: + Attempt to automatically forward the DirPort and ORPort on a NAT router + connecting this Tor server to the Internet. If set, Tor will try both + NAT-PMP (common on Apple routers) and UPnP (common on routers from other + manufacturers). (Default: 0) + +**PortForwardingHelper** __filename__|__pathname__:: + If PortForwarding is set, use this executable to configure the forwarding. + If set to a filename, the system path will be searched for the executable. + If set to a path, only the specified path will be executed. + (Default: tor-fw-helper) + **PublishServerDescriptor** **0**|**1**|**v1**|**v2**|**v3**|**bridge**|**hidserv**,**...**:: This option specifies which descriptors Tor will publish when acting as a relay or hidden service. You can diff --git a/src/common/compat.c b/src/common/compat.c index 20394b4c5d..b7f4f17cae 100644 --- a/src/common/compat.c +++ b/src/common/compat.c @@ -1898,6 +1898,52 @@ spawn_exit(void) #endif } +/** Implementation logic for compute_num_cpus(). */ +static int +compute_num_cpus_impl(void) +{ +#ifdef MS_WINDOWS + SYSTEM_INFO info; + memset(&info, 0, sizeof(info)); + GetSystemInfo(&info); + if (info.dwNumberOfProcessors >= 1 && info.dwNumberOfProcessors < INT_MAX) + return (int)info.dwNumberOfProcessors; + else + return -1; +#elif defined(HAVE_SYSCONF) && defined(_SC_NPROCESSORS_CONF) + long cpus = sysconf(_SC_NPROCESSORS_CONF); + if (cpus >= 1 && cpus < INT_MAX) + return (int)cpus; + else + return -1; +#else + return -1; +#endif +} + +#define MAX_DETECTABLE_CPUS 16 + +/** Return how many CPUs we are running with. We assume that nobody is + * using hot-swappable CPUs, so we don't recompute this after the first + * time. Return -1 if we don't know how to tell the number of CPUs on this + * system. + */ +int +compute_num_cpus(void) +{ + static int num_cpus = -2; + if (num_cpus == -2) { + num_cpus = compute_num_cpus_impl(); + tor_assert(num_cpus != -2); + if (num_cpus > MAX_DETECTABLE_CPUS) + log_notice(LD_GENERAL, "Wow! I detected that you have %d CPUs. I " + "will not autodetect any more than %d, though. If you " + "want to configure more, set NumCPUs in your torrc", + num_cpus, MAX_DETECTABLE_CPUS); + } + return num_cpus; +} + /** Set *timeval to the current time of day. On error, log and terminate. * (Same as gettimeofday(timeval,NULL), but never returns -1.) */ diff --git a/src/common/compat.h b/src/common/compat.h index 7d59501e2b..5b6cfc40b5 100644 --- a/src/common/compat.h +++ b/src/common/compat.h @@ -522,6 +522,8 @@ void spawn_exit(void) ATTR_NORETURN; #undef TOR_IS_MULTITHREADED #endif +int compute_num_cpus(void); + /* Because we use threads instead of processes on most platforms (Windows, * Linux, etc), we need locking for them. On platforms with poor thread * support or broken gethostbyname_r, these functions are no-ops. */ diff --git a/src/common/compat_libevent.c b/src/common/compat_libevent.c index 250fa2bdb7..d94f615f74 100644 --- a/src/common/compat_libevent.c +++ b/src/common/compat_libevent.c @@ -159,9 +159,11 @@ struct event_base *the_event_base = NULL; /** Initialize the Libevent library and set up the event base. */ void -tor_libevent_initialize(void) +tor_libevent_initialize(tor_libevent_cfg *torcfg) { tor_assert(the_event_base == NULL); + /* some paths below don't use torcfg, so avoid unused variable warnings */ + (void)torcfg; #ifdef __APPLE__ if (MACOSX_KQUEUE_IS_BROKEN || @@ -171,7 +173,21 @@ tor_libevent_initialize(void) #endif #ifdef HAVE_EVENT2_EVENT_H - the_event_base = event_base_new(); + { + struct event_config *cfg = event_config_new(); + +#if defined(MS_WINDOWS) && defined(USE_BUFFEREVENTS) + if (! torcfg->disable_iocp) + event_config_set_flag(cfg, EVENT_BASE_FLAG_STARTUP_IOCP); +#endif + +#if defined(LIBEVENT_VERSION_NUMBER) && LIBEVENT_VERSION_NUMBER >= V(2,0,7) + if (torcfg->num_cpus > 0) + event_config_set_num_cpus_hint(cfg, torcfg->num_cpus); +#endif + + the_event_base = event_base_new_with_config(cfg); + } #else the_event_base = event_init(); #endif @@ -551,3 +567,29 @@ periodic_timer_free(periodic_timer_t *timer) tor_free(timer); } +#ifdef USE_BUFFEREVENTS +static const struct timeval *one_tick = NULL; +/** + * Return a special timeout to be passed whenever libevent's O(1) timeout + * implementation should be used. Only use this when the timer is supposed + * to fire after 1 / TOR_LIBEVENT_TICKS_PER_SECOND seconds have passed. +*/ +const struct timeval * +tor_libevent_get_one_tick_timeout(void) +{ + if (PREDICT_UNLIKELY(one_tick == NULL)) { + struct event_base *base = tor_libevent_get_base(); + struct timeval tv; + if (TOR_LIBEVENT_TICKS_PER_SECOND == 1) { + tv.tv_sec = 1; + tv.tv_usec = 0; + } else { + tv.tv_sec = 0; + tv.tv_usec = 1000000 / TOR_LIBEVENT_TICKS_PER_SECOND; + } + one_tick = event_base_init_common_timeout(base, &tv); + } + return one_tick; +} +#endif + diff --git a/src/common/compat_libevent.h b/src/common/compat_libevent.h index fdf5e0a18f..ecf25806d5 100644 --- a/src/common/compat_libevent.h +++ b/src/common/compat_libevent.h @@ -8,6 +8,9 @@ struct event; struct event_base; +#ifdef USE_BUFFEREVENTS +struct bufferevent; +#endif #ifdef HAVE_EVENT2_EVENT_H #include <event2/util.h> @@ -53,7 +56,12 @@ struct timeval; int tor_event_base_loopexit(struct event_base *base, struct timeval *tv); #endif -void tor_libevent_initialize(void); +typedef struct tor_libevent_cfg { + int disable_iocp; + int num_cpus; +} tor_libevent_cfg; + +void tor_libevent_initialize(tor_libevent_cfg *cfg); struct event_base *tor_libevent_get_base(void); const char *tor_libevent_get_method(void); void tor_check_libevent_version(const char *m, int server, @@ -61,5 +69,10 @@ void tor_check_libevent_version(const char *m, int server, void tor_check_libevent_header_compatibility(void); const char *tor_libevent_get_version_str(void); +#ifdef USE_BUFFEREVENTS +#define TOR_LIBEVENT_TICKS_PER_SECOND 3 +const struct timeval *tor_libevent_get_one_tick_timeout(void); +#endif + #endif diff --git a/src/common/crypto.h b/src/common/crypto.h index a30e5bcbae..c433938d5b 100644 --- a/src/common/crypto.h +++ b/src/common/crypto.h @@ -238,7 +238,8 @@ void secret_to_key(char *key_out, size_t key_out_len, const char *secret, size_t secret_len, const char *s2k_specifier); #ifdef CRYPTO_PRIVATE -/* Prototypes for private functions only used by tortls.c and crypto.c */ +/* Prototypes for private functions only used by tortls.c, crypto.c, and the + * unit tests. */ struct rsa_st; struct evp_pkey_st; struct dh_st; diff --git a/src/common/tortls.c b/src/common/tortls.c index 99f3f4abbc..7568a6b47f 100644 --- a/src/common/tortls.c +++ b/src/common/tortls.c @@ -44,7 +44,14 @@ #error "We require OpenSSL >= 0.9.7" #endif +#ifdef USE_BUFFEREVENTS +#include <event2/bufferevent_ssl.h> +#include <event2/buffer.h> +#include "compat_libevent.h" +#endif + #define CRYPTO_PRIVATE /* to import prototypes from crypto.h */ +#define TORTLS_PRIVATE #include "crypto.h" #include "tortls.h" @@ -109,6 +116,7 @@ struct tor_tls_t { enum { TOR_TLS_ST_HANDSHAKE, TOR_TLS_ST_OPEN, TOR_TLS_ST_GOTCLOSE, TOR_TLS_ST_SENTCLOSE, TOR_TLS_ST_CLOSED, TOR_TLS_ST_RENEGOTIATE, + TOR_TLS_ST_BUFFEREVENT } state : 3; /**< The current SSL state, depending on which operations have * completed successfully. */ unsigned int isServer:1; /**< True iff this is a server-side connection */ @@ -189,7 +197,6 @@ static X509* tor_tls_create_certificate(crypto_pk_env_t *rsa, const char *cname, const char *cname_sign, unsigned int lifetime); -static void tor_tls_unblock_renegotiation(tor_tls_t *tls); /** Global tls context. We keep it here because nobody else needs to * touch it. */ @@ -1026,7 +1033,7 @@ tor_tls_set_renegotiate_callback(tor_tls_t *tls, /** If this version of openssl requires it, turn on renegotiation on * <b>tls</b>. */ -static void +void tor_tls_unblock_renegotiation(tor_tls_t *tls) { /* Yes, we know what we are doing here. No, we do not treat a renegotiation @@ -1194,56 +1201,86 @@ tor_tls_handshake(tor_tls_t *tls) } if (r == TOR_TLS_DONE) { tls->state = TOR_TLS_ST_OPEN; - if (tls->isServer) { - SSL_set_info_callback(tls->ssl, NULL); - SSL_set_verify(tls->ssl, SSL_VERIFY_PEER, always_accept_verify_cb); - /* There doesn't seem to be a clear OpenSSL API to clear mode flags. */ - tls->ssl->mode &= ~SSL_MODE_NO_AUTO_CHAIN; + return tor_tls_finish_handshake(tls); + } + return r; +} + +/** Perform the final part of the intial TLS handshake on <b>tls</b>. This + * should be called for the first handshake only: it determines whether the v1 + * or the v2 handshake was used, and adjusts things for the renegotiation + * handshake as appropriate. + * + * tor_tls_handshake() calls this on its own; you only need to call this if + * bufferevent is doing the handshake for you. + */ +int +tor_tls_finish_handshake(tor_tls_t *tls) +{ + int r = TOR_TLS_DONE; + if (tls->isServer) { + SSL_set_info_callback(tls->ssl, NULL); + SSL_set_verify(tls->ssl, SSL_VERIFY_PEER, always_accept_verify_cb); + /* There doesn't seem to be a clear OpenSSL API to clear mode flags. */ + tls->ssl->mode &= ~SSL_MODE_NO_AUTO_CHAIN; #ifdef V2_HANDSHAKE_SERVER - if (tor_tls_client_is_using_v2_ciphers(tls->ssl, ADDR(tls))) { - /* This check is redundant, but back when we did it in the callback, - * we might have not been able to look up the tor_tls_t if the code - * was buggy. Fixing that. */ - if (!tls->wasV2Handshake) { - log_warn(LD_BUG, "For some reason, wasV2Handshake didn't" - " get set. Fixing that."); - } - tls->wasV2Handshake = 1; - log_debug(LD_HANDSHAKE, - "Completed V2 TLS handshake with client; waiting " - "for renegotiation."); - } else { - tls->wasV2Handshake = 0; + if (tor_tls_client_is_using_v2_ciphers(tls->ssl, ADDR(tls))) { + /* This check is redundant, but back when we did it in the callback, + * we might have not been able to look up the tor_tls_t if the code + * was buggy. Fixing that. */ + if (!tls->wasV2Handshake) { + log_warn(LD_BUG, "For some reason, wasV2Handshake didn't" + " get set. Fixing that."); } -#endif + tls->wasV2Handshake = 1; + log_debug(LD_HANDSHAKE, "Completed V2 TLS handshake with client; waiting" + " for renegotiation."); } else { + tls->wasV2Handshake = 0; + } +#endif + } else { #ifdef V2_HANDSHAKE_CLIENT - /* If we got no ID cert, we're a v2 handshake. */ - X509 *cert = SSL_get_peer_certificate(tls->ssl); - STACK_OF(X509) *chain = SSL_get_peer_cert_chain(tls->ssl); - int n_certs = sk_X509_num(chain); - if (n_certs > 1 || (n_certs == 1 && cert != sk_X509_value(chain, 0))) { - log_debug(LD_HANDSHAKE, "Server sent back multiple certificates; it " - "looks like a v1 handshake on %p", tls); - tls->wasV2Handshake = 0; - } else { - log_debug(LD_HANDSHAKE, - "Server sent back a single certificate; looks like " - "a v2 handshake on %p.", tls); - tls->wasV2Handshake = 1; - } - if (cert) - X509_free(cert); + /* If we got no ID cert, we're a v2 handshake. */ + X509 *cert = SSL_get_peer_certificate(tls->ssl); + STACK_OF(X509) *chain = SSL_get_peer_cert_chain(tls->ssl); + int n_certs = sk_X509_num(chain); + if (n_certs > 1 || (n_certs == 1 && cert != sk_X509_value(chain, 0))) { + log_debug(LD_HANDSHAKE, "Server sent back multiple certificates; it " + "looks like a v1 handshake on %p", tls); + tls->wasV2Handshake = 0; + } else { + log_debug(LD_HANDSHAKE, + "Server sent back a single certificate; looks like " + "a v2 handshake on %p.", tls); + tls->wasV2Handshake = 1; + } + if (cert) + X509_free(cert); #endif - if (SSL_set_cipher_list(tls->ssl, SERVER_CIPHER_LIST) == 0) { - tls_log_errors(NULL, LOG_WARN, LD_HANDSHAKE, "re-setting ciphers"); - r = TOR_TLS_ERROR_MISC; - } + if (SSL_set_cipher_list(tls->ssl, SERVER_CIPHER_LIST) == 0) { + tls_log_errors(NULL, LOG_WARN, LD_HANDSHAKE, "re-setting ciphers"); + r = TOR_TLS_ERROR_MISC; } } return r; } +#ifdef USE_BUFFEREVENTS +/** Put <b>tls</b>, which must be a client connection, into renegotiation + * mode. */ +int +tor_tls_start_renegotiating(tor_tls_t *tls) +{ + int r = SSL_renegotiate(tls->ssl); + if (r <= 0) { + return tor_tls_get_error(tls, r, 0, "renegotiating", LOG_WARN, + LD_HANDSHAKE); + } + return 0; +} +#endif + /** Client only: Renegotiate a TLS session. When finished, returns * TOR_TLS_DONE. On failure, returns TOR_TLS_ERROR, TOR_TLS_WANTREAD, or * TOR_TLS_WANTWRITE. @@ -1460,6 +1497,8 @@ tor_tls_verify(int severity, tor_tls_t *tls, crypto_pk_env_t **identity_key) log_fn(severity,LD_PROTOCOL,"No distinct identity certificate found"); goto done; } + tls_log_errors(tls, severity, LD_HANDSHAKE, "before verifying certificate"); + if (!(id_pkey = X509_get_pubkey(id_cert)) || X509_verify(cert, id_pkey) <= 0) { log_fn(severity,LD_PROTOCOL,"X509_verify on cert and pkey returned <= 0"); @@ -1631,3 +1670,59 @@ tor_tls_get_buffer_sizes(tor_tls_t *tls, *wbuf_bytes = tls->ssl->s3->wbuf.left; } +#ifdef USE_BUFFEREVENTS +/** Construct and return an TLS-encrypting bufferevent to send data over + * <b>socket</b>, which must match the socket of the underlying bufferevent + * <b>bufev_in</b>. The TLS object <b>tls</b> is used for encryption. + * + * This function will either create a filtering bufferevent that wraps around + * <b>bufev_in</b>, or it will free bufev_in and return a new bufferevent that + * uses the <b>tls</b> to talk to the network directly. Do not use + * <b>bufev_in</b> after calling this function. + * + * The connection will start out doing a server handshake if <b>receiving</b> + * is strue, and a client handshake otherwise. + * + * Returns NULL on failure. + */ +struct bufferevent * +tor_tls_init_bufferevent(tor_tls_t *tls, struct bufferevent *bufev_in, + evutil_socket_t socket, int receiving) +{ + struct bufferevent *out; + const enum bufferevent_ssl_state state = receiving ? + BUFFEREVENT_SSL_ACCEPTING : BUFFEREVENT_SSL_CONNECTING; + +#if 0 + (void) socket; + out = bufferevent_openssl_filter_new(tor_libevent_get_base(), + bufev_in, + tls->ssl, + state, + BEV_OPT_DEFER_CALLBACKS); +#else + if (bufev_in) { + evutil_socket_t s = bufferevent_getfd(bufev_in); + tor_assert(s == -1 || s == socket); + tor_assert(evbuffer_get_length(bufferevent_get_input(bufev_in)) == 0); + tor_assert(evbuffer_get_length(bufferevent_get_output(bufev_in)) == 0); + tor_assert(BIO_number_read(SSL_get_rbio(tls->ssl)) == 0); + tor_assert(BIO_number_written(SSL_get_rbio(tls->ssl)) == 0); + bufferevent_free(bufev_in); + } + tls->state = TOR_TLS_ST_BUFFEREVENT; + + /* Current versions (as of 2.0.7-rc) of Libevent need to defer + * bufferevent_openssl callbacks, or else our callback functions will + * get called reentrantly, which is bad for us. + */ + out = bufferevent_openssl_socket_new(tor_libevent_get_base(), + socket, + tls->ssl, + state, + BEV_OPT_DEFER_CALLBACKS); +#endif + return out; +} +#endif + diff --git a/src/common/tortls.h b/src/common/tortls.h index e4b1ad65f2..f8603b529b 100644 --- a/src/common/tortls.h +++ b/src/common/tortls.h @@ -64,7 +64,9 @@ int tor_tls_check_lifetime(tor_tls_t *tls, int tolerance); int tor_tls_read(tor_tls_t *tls, char *cp, size_t len); int tor_tls_write(tor_tls_t *tls, const char *cp, size_t n); int tor_tls_handshake(tor_tls_t *tls); +int tor_tls_finish_handshake(tor_tls_t *tls); int tor_tls_renegotiate(tor_tls_t *tls); +void tor_tls_unblock_renegotiation(tor_tls_t *tls); void tor_tls_block_renegotiation(tor_tls_t *tls); int tor_tls_shutdown(tor_tls_t *tls); int tor_tls_get_pending_bytes(tor_tls_t *tls); @@ -85,5 +87,12 @@ int tor_tls_used_v1_handshake(tor_tls_t *tls); void _check_no_tls_errors(const char *fname, int line); +#ifdef USE_BUFFEREVENTS +int tor_tls_start_renegotiating(tor_tls_t *tls); +struct bufferevent *tor_tls_init_bufferevent(tor_tls_t *tls, + struct bufferevent *bufev_in, + evutil_socket_t socket, int receiving); +#endif + #endif diff --git a/src/common/util.c b/src/common/util.c index dafe12071f..b5a3ade2bd 100644 --- a/src/common/util.c +++ b/src/common/util.c @@ -14,6 +14,7 @@ #define _GNU_SOURCE #include "orconfig.h" +#define UTIL_PRIVATE #include "util.h" #include "torlog.h" #undef log @@ -2878,3 +2879,411 @@ load_windows_system_library(const TCHAR *library_name) } #endif +/** Format child_state and saved_errno as a hex string placed in hex_errno. + * Called between fork and _exit, so must be signal-handler safe */ +void +format_helper_exit_status(unsigned char child_state, int saved_errno, + char *hex_errno) +{ + /* Convert errno to be unsigned for hex conversion */ + unsigned int unsigned_errno; + char *cur; + + /* If errno is negative, negate it */ + if (saved_errno < 0) { + unsigned_errno = (unsigned int) -saved_errno; + } else { + unsigned_errno = (unsigned int) saved_errno; + } + + /* Convert errno to hex (start before \n) */ + cur = hex_errno + HEX_ERRNO_SIZE - 2; + + do { + *cur-- = "0123456789ABCDEF"[unsigned_errno % 16]; + unsigned_errno /= 16; + } while (unsigned_errno != 0 && cur >= hex_errno); + + /* Add on the minus side if errno was negative */ + if (saved_errno < 0) + *cur-- = '-'; + + /* Leave a gap */ + *cur-- = '/'; + + /* Convert child_state to hex */ + do { + *cur-- = "0123456789ABCDEF"[child_state % 16]; + child_state /= 16; + } while (child_state != 0 && cur >= hex_errno); +} + +/* Maximum number of file descriptors, if we cannot get it via sysconf() */ +#define DEFAULT_MAX_FD 256 + +#define CHILD_STATE_INIT 0 +#define CHILD_STATE_PIPE 1 +#define CHILD_STATE_MAXFD 2 +#define CHILD_STATE_FORK 3 +#define CHILD_STATE_DUPOUT 4 +#define CHILD_STATE_DUPERR 5 +#define CHILD_STATE_REDIRECT 6 +#define CHILD_STATE_CLOSEFD 7 +#define CHILD_STATE_EXEC 8 +#define CHILD_STATE_FAILEXEC 9 + +#define SPAWN_ERROR_MESSAGE "ERR: Failed to spawn background process - code " + +/** Start a program in the background. If <b>filename</b> contains a '/', then + * it will be treated as an absolute or relative path. Otherwise the system + * path will be searched for <b>filename</b>. Returns pid on success, otherwise + * returns -1. + * Some parts of this code are based on the POSIX subprocess module from Python + */ +static int +tor_spawn_background(const char *const filename, int *stdout_read, + int *stderr_read, const char **argv) +{ +#ifdef MS_WINDOWS + (void) filename; (void) stdout_read; (void) stderr_read; (void) argv; + log_warn(LD_BUG, "not yet implemented on Windows."); + return -1; +#else + pid_t pid; + int stdout_pipe[2]; + int stderr_pipe[2]; + int fd, retval; + ssize_t nbytes; + + const char *error_message = SPAWN_ERROR_MESSAGE; + size_t error_message_length; + + /* Represents where in the process of spawning the program is; + this is used for printing out the error message */ + unsigned char child_state = CHILD_STATE_INIT; + + char hex_errno[HEX_ERRNO_SIZE]; + + static int max_fd = -1; + + /* We do the strlen here because strlen() is not signal handler safe, + and we are not allowed to use unsafe functions between fork and exec */ + error_message_length = strlen(error_message); + + /* Fill hex_errno with spaces, and a trailing newline */ + memset(hex_errno, ' ', sizeof(hex_errno) - 1); + hex_errno[sizeof(hex_errno) - 1] = '\n'; + + child_state = CHILD_STATE_PIPE; + + /* Set up pipe for redirecting stdout and stderr of child */ + retval = pipe(stdout_pipe); + if (-1 == retval) { + log_err(LD_GENERAL, + "Failed to set up pipe for stdout communication with child process: %s", + strerror(errno)); + return -1; + } + + retval = pipe(stderr_pipe); + if (-1 == retval) { + log_err(LD_GENERAL, + "Failed to set up pipe for stderr communication with child process: %s", + strerror(errno)); + return -1; + } + + child_state = CHILD_STATE_MAXFD; + +#ifdef _SC_OPEN_MAX + if (-1 != max_fd) { + max_fd = (int) sysconf(_SC_OPEN_MAX); + if (max_fd == -1) + max_fd = DEFAULT_MAX_FD; + log_warn(LD_GENERAL, + "Cannot find maximum file descriptor, assuming %d", max_fd); + } +#else + max_fd = DEFAULT_MAX_FD; +#endif + + child_state = CHILD_STATE_FORK; + + pid = fork(); + if (0 == pid) { + /* In child */ + + child_state = CHILD_STATE_DUPOUT; + + /* Link child stdout to the write end of the pipe */ + retval = dup2(stdout_pipe[1], STDOUT_FILENO); + if (-1 == retval) + goto error; + + child_state = CHILD_STATE_DUPERR; + + /* Link child stderr to the write end of the pipe */ + retval = dup2(stderr_pipe[1], STDERR_FILENO); + if (-1 == retval) + goto error; + + child_state = CHILD_STATE_REDIRECT; + + /* Link stdin to /dev/null */ + fd = open("/dev/null", O_RDONLY); + if (fd != -1) + dup2(STDIN_FILENO, fd); + else + goto error; + + child_state = CHILD_STATE_CLOSEFD; + + /* Close all other fds, including the read end of the pipe */ + /* TODO: use closefrom if available */ + for (fd = STDERR_FILENO + 1; fd < max_fd; fd++) + close(fd); + + child_state = CHILD_STATE_EXEC; + + /* Call the requested program. We need the cast because + execvp doesn't define argv as const, even though it + does not modify the arguments */ + execvp(filename, (char *const *) argv); + + /* If we got here, the exec or open(/dev/null) failed */ + + child_state = CHILD_STATE_FAILEXEC; + + error: + /* TODO: are we leaking fds from the pipe? */ + + format_helper_exit_status(child_state, errno, hex_errno); + + /* Write the error message. GCC requires that we check the return + value, but there is nothing we can do if it fails */ + nbytes = write(STDOUT_FILENO, error_message, error_message_length); + nbytes = write(STDOUT_FILENO, hex_errno, sizeof(hex_errno)); + + _exit(255); + return -1; /* Never reached, but avoids compiler warning */ + } + + /* In parent */ + + if (-1 == pid) { + log_err(LD_GENERAL, "Failed to fork child process: %s", strerror(errno)); + close(stdout_pipe[0]); + close(stdout_pipe[1]); + close(stderr_pipe[0]); + close(stderr_pipe[1]); + return -1; + } + + /* Return read end of the pipes to caller, and close write end */ + *stdout_read = stdout_pipe[0]; + retval = close(stdout_pipe[1]); + + if (-1 == retval) { + log_err(LD_GENERAL, + "Failed to close write end of stdout pipe in parent process: %s", + strerror(errno)); + /* Do not return -1, because the child is running, so the parent + needs to know about the pid in order to reap it later */ + } + + *stderr_read = stderr_pipe[0]; + retval = close(stderr_pipe[1]); + + if (-1 == retval) { + log_err(LD_GENERAL, + "Failed to close write end of stderr pipe in parent process: %s", + strerror(errno)); + /* Do not return -1, because the child is running, so the parent + needs to know about the pid in order to reap it later */ + } + + return pid; +#endif +} + +/** Read from stream, and send lines to log at the specified log level. + * Returns 1 if stream is closed normally, -1 if there is a error reading, and + * 0 otherwise. Handles lines from tor-fw-helper and + * tor_spawn_background() specially. + */ +static int +log_from_pipe(FILE *stream, int severity, const char *executable, + int *child_status) +{ + char buf[256]; + + for (;;) { + char *retval; + retval = fgets(buf, sizeof(buf), stream); + + if (NULL == retval) { + if (feof(stream)) { + /* Program has closed stream (probably it exited) */ + /* TODO: check error */ + fclose(stream); + return 1; + } else { + if (EAGAIN == errno) { + /* Nothing more to read, try again next time */ + return 0; + } else { + /* There was a problem, abandon this child process */ + fclose(stream); + return -1; + } + } + } else { + /* We have some data, log it and keep asking for more */ + size_t len; + + len = strlen(buf); + if (buf[len - 1] == '\n') { + /* Remove the trailing newline */ + buf[len - 1] = '\0'; + } else { + /* No newline; check whether we overflowed the buffer */ + if (!feof(stream)) + log_err(LD_GENERAL, + "Line from port forwarding helper was truncated: %s", buf); + /* TODO: What to do with this error? */ + } + + /* Check if buf starts with SPAWN_ERROR_MESSAGE */ + if (strstr(buf, SPAWN_ERROR_MESSAGE) == buf) { + /* Parse error message */ + int retval, child_state, saved_errno; + retval = sscanf(buf, SPAWN_ERROR_MESSAGE "%d/%d", + &child_state, &saved_errno); + if (retval == 2) { + log_err(LD_GENERAL, + "Failed to start child process \"%s\" in state %d: %s", + executable, child_state, strerror(saved_errno)); + if (child_status) + *child_status = 1; + } else { + /* Failed to parse message from child process, log it as error */ + log_err(LD_GENERAL, + "Unexpected message from port forwarding helper \"%s\": %s", + executable, buf); + } + } else { + log_fn(severity, LD_GENERAL, "Port forwarding helper says: %s", buf); + } + } + } + + /* We should never get here */ + return -1; +} + +void +tor_check_port_forwarding(const char *filename, int dir_port, int or_port, + time_t now) +{ +#ifdef MS_WINDOWS + (void) filename; (void) dir_port; (void) or_port; (void) now; + (void) tor_spawn_background; + (void) log_from_pipe; + log_warn(LD_GENERAL, "Sorry, port forwarding is not yet supported " + "on windows."); +#else +/* When fw-helper succeeds, how long do we wait until running it again */ +#define TIME_TO_EXEC_FWHELPER_SUCCESS 300 +/* When fw-helper fails, how long do we wait until running it again */ +#define TIME_TO_EXEC_FWHELPER_FAIL 60 + + static int child_pid = -1; + static FILE *stdout_read = NULL; + static FILE *stderr_read = NULL; + static time_t time_to_run_helper = 0; + int stdout_status, stderr_status, retval; + const char *argv[10]; + char s_dirport[6], s_orport[6]; + + tor_assert(filename); + + /* Set up command line for tor-fw-helper */ + snprintf(s_dirport, sizeof s_dirport, "%d", dir_port); + snprintf(s_orport, sizeof s_orport, "%d", or_port); + + /* TODO: Allow different internal and external ports */ + argv[0] = filename; + argv[1] = "--internal-or-port"; + argv[2] = s_orport; + argv[3] = "--external-or-port"; + argv[4] = s_orport; + argv[5] = "--internal-dir-port"; + argv[6] = s_dirport; + argv[7] = "--external-dir-port"; + argv[8] = s_dirport; + argv[9] = NULL; + + /* Start the child, if it is not already running */ + if (-1 == child_pid && + time_to_run_helper < now) { + int fd_out=-1, fd_err=-1; + + /* Assume tor-fw-helper will succeed, start it later*/ + time_to_run_helper = now + TIME_TO_EXEC_FWHELPER_SUCCESS; + + child_pid = tor_spawn_background(filename, &fd_out, &fd_err, argv); + if (child_pid < 0) { + log_err(LD_GENERAL, "Failed to start port forwarding helper %s", + filename); + child_pid = -1; + return; + } + /* Set stdout/stderr pipes to be non-blocking */ + fcntl(fd_out, F_SETFL, O_NONBLOCK); + fcntl(fd_err, F_SETFL, O_NONBLOCK); + /* Open the buffered IO streams */ + stdout_read = fdopen(fd_out, "r"); + stderr_read = fdopen(fd_err, "r"); + + log_info(LD_GENERAL, + "Started port forwarding helper (%s) with pid %d", filename, child_pid); + } + + /* If child is running, read from its stdout and stderr) */ + if (child_pid > 0) { + /* Read from stdout/stderr and log result */ + retval = 0; + stdout_status = log_from_pipe(stdout_read, LOG_INFO, filename, &retval); + stderr_status = log_from_pipe(stderr_read, LOG_ERR, filename, &retval); + if (retval) { + /* There was a problem in the child process */ + time_to_run_helper = now + TIME_TO_EXEC_FWHELPER_FAIL; + } + + /* Combine the two statuses in order of severity */ + if (-1 == stdout_status || -1 == stderr_status) + /* There was a failure */ + retval = -1; + else if (1 == stdout_status || 1 == stderr_status) + /* stdout or stderr was closed */ + retval = 1; + else + /* Both are fine */ + retval = 0; + + /* If either pipe indicates a failure, act on it */ + if (0 != retval) { + if (1 == retval) { + log_info(LD_GENERAL, "Port forwarding helper terminated"); + } else { + log_err(LD_GENERAL, "Failed to read from port forwarding helper"); + } + + /* TODO: The child might not actually be finished (maybe it failed or + closed stdout/stderr), so maybe we shouldn't start another? */ + child_pid = -1; + } + } +#endif +} + diff --git a/src/common/util.h b/src/common/util.h index 833fd904c7..86555eeb19 100644 --- a/src/common/util.h +++ b/src/common/util.h @@ -340,10 +340,25 @@ void start_daemon(void); void finish_daemon(const char *desired_cwd); void write_pidfile(char *filename); +/* Port forwarding */ +void tor_check_port_forwarding(const char *filename, + int dir_port, int or_port, time_t now); + #ifdef MS_WINDOWS HANDLE load_windows_system_library(const TCHAR *library_name); #endif +#ifdef UTIL_PRIVATE +/* Prototypes for private functions only used by util.c (and unit tests) */ +void format_helper_exit_status(unsigned char child_state, + int saved_errno, char *hex_errno); + +/* Space for hex values of child state, a slash, saved_errno (with + leading minus) and newline (no null) */ +#define HEX_ERRNO_SIZE (sizeof(char) * 2 + 1 + \ + 1 + sizeof(int) * 2 + 1) +#endif + const char *libor_get_digests(void); #endif diff --git a/src/or/Makefile.am b/src/or/Makefile.am index a9ac3cdee1..9d8fb663cf 100644 --- a/src/or/Makefile.am +++ b/src/or/Makefile.am @@ -40,10 +40,17 @@ AM_CPPFLAGS = -DSHARE_DATADIR="\"$(datadir)\"" \ # This seems to matter nowhere but on windows, but I assure you that it # matters a lot there, and is quite hard to debug if you forget to do it. +if USE_BUFFEREVENTS +levent_openssl_lib = -levent_openssl +else +levent_openssl_lib = +endif + tor_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ @TOR_LDFLAGS_libevent@ tor_LDADD = ./libtor.a ../common/libor.a ../common/libor-crypto.a \ ../common/libor-event.a \ - @TOR_ZLIB_LIBS@ -lm @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ + @TOR_ZLIB_LIBS@ -lm @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ \ + @TOR_LIB_WS32@ @TOR_LIB_GDI@ $(levent_openssl_lib) noinst_HEADERS = buffers.h circuitbuild.h circuitlist.h circuituse.h \ command.h config.h connection_edge.h connection.h connection_or.h \ diff --git a/src/or/buffers.c b/src/or/buffers.c index 09ccb7cb0d..11c656f23a 100644 --- a/src/or/buffers.c +++ b/src/or/buffers.c @@ -23,9 +23,6 @@ #ifdef HAVE_UNISTD_H #include <unistd.h> #endif -#ifdef HAVE_SYS_UIO_H -#include <sys/uio.h> -#endif //#define PARANOIA @@ -56,6 +53,13 @@ * forever. */ +static int parse_socks(const char *data, size_t datalen, socks_request_t *req, + int log_sockstype, int safe_socks, ssize_t *drain_out, + size_t *want_length_out); +static int parse_socks_client(const uint8_t *data, size_t datalen, + int state, char **reason, + ssize_t *drain_out); + /* Chunk manipulation functions */ /** A single chunk on a buffer or in a freelist. */ @@ -544,6 +548,7 @@ buf_free(buf_t *buf) { if (!buf) return; + buf_clear(buf); buf->magic = 0xdeadbeef; tor_free(buf); @@ -575,10 +580,6 @@ buf_add_chunk_with_capacity(buf_t *buf, size_t capacity, int capped) return chunk; } -/** If we're using readv and writev, how many chunks are we willing to - * read/write at a time? */ -#define N_IOV 3 - /** Read up to <b>at_most</b> bytes from the socket <b>fd</b> into * <b>chunk</b> (which must be on <b>buf</b>). If we get an EOF, set * *<b>reached_eof</b> to 1. Return -1 on error, 0 on eof or blocking, @@ -588,25 +589,9 @@ read_to_chunk(buf_t *buf, chunk_t *chunk, int fd, size_t at_most, int *reached_eof, int *socket_error) { ssize_t read_result; -#if 0 && defined(HAVE_READV) && !defined(WIN32) - struct iovec iov[N_IOV]; - int i; - size_t remaining = at_most; - for (i=0; chunk && i < N_IOV && remaining; ++i) { - iov[i].iov_base = CHUNK_WRITE_PTR(chunk); - if (remaining > CHUNK_REMAINING_CAPACITY(chunk)) - iov[i].iov_len = CHUNK_REMAINING_CAPACITY(chunk); - else - iov[i].iov_len = remaining; - remaining -= iov[i].iov_len; - chunk = chunk->next; - } - read_result = readv(fd, iov, i); -#else if (at_most > CHUNK_REMAINING_CAPACITY(chunk)) at_most = CHUNK_REMAINING_CAPACITY(chunk); read_result = tor_socket_recv(fd, CHUNK_WRITE_PTR(chunk), at_most, 0); -#endif if (read_result < 0) { int e = tor_socket_errno(fd); @@ -625,14 +610,6 @@ read_to_chunk(buf_t *buf, chunk_t *chunk, int fd, size_t at_most, return 0; } else { /* actually got bytes. */ buf->datalen += read_result; -#if 0 && defined(HAVE_READV) && !defined(WIN32) - while ((size_t)read_result > CHUNK_REMAINING_CAPACITY(chunk)) { - chunk->datalen += CHUNK_REMAINING_CAPACITY(chunk); - read_result -= CHUNK_REMAINING_CAPACITY(chunk); - chunk = chunk->next; - tor_assert(chunk); - } -#endif chunk->datalen += read_result; log_debug(LD_NET,"Read %ld bytes. %d on inbuf.", (long)read_result, (int)buf->datalen); @@ -768,25 +745,10 @@ flush_chunk(int s, buf_t *buf, chunk_t *chunk, size_t sz, size_t *buf_flushlen) { ssize_t write_result; -#if 0 && defined(HAVE_WRITEV) && !defined(WIN32) - struct iovec iov[N_IOV]; - int i; - size_t remaining = sz; - for (i=0; chunk && i < N_IOV && remaining; ++i) { - iov[i].iov_base = chunk->data; - if (remaining > chunk->datalen) - iov[i].iov_len = chunk->datalen; - else - iov[i].iov_len = remaining; - remaining -= iov[i].iov_len; - chunk = chunk->next; - } - write_result = writev(s, iov, i); -#else + if (sz > chunk->datalen) sz = chunk->datalen; write_result = tor_socket_send(s, chunk->data, sz, 0); -#endif if (write_result < 0) { int e = tor_socket_errno(s); @@ -1053,6 +1015,103 @@ fetch_var_cell_from_buf(buf_t *buf, var_cell_t **out, int linkproto) return 1; } +#ifdef USE_BUFFEREVENTS +/** Try to read <b>n</b> bytes from <b>buf</b> at <b>pos</b> (which may be + * NULL for the start of the buffer), copying the data only if necessary. Set + * *<b>data</b> to a pointer to the desired bytes. Set <b>free_out</b> to 1 + * if we needed to malloc *<b>data</b> because the original bytes were + * noncontiguous; 0 otherwise. Return the number of bytes actually available + * at <b>data</b>. + */ +static ssize_t +inspect_evbuffer(struct evbuffer *buf, char **data, size_t n, int *free_out, + struct evbuffer_ptr *pos) +{ + int n_vecs, i; + + if (evbuffer_get_length(buf) < n) + n = evbuffer_get_length(buf); + if (n == 0) + return 0; + n_vecs = evbuffer_peek(buf, n, pos, NULL, 0); + tor_assert(n_vecs > 0); + if (n_vecs == 1) { + struct evbuffer_iovec v; + i = evbuffer_peek(buf, n, pos, &v, 1); + tor_assert(i == 1); + *data = v.iov_base; + *free_out = 0; + return v.iov_len; + } else { + struct evbuffer_iovec *vecs = + tor_malloc(sizeof(struct evbuffer_iovec)*n_vecs); + size_t copied = 0; + i = evbuffer_peek(buf, n, NULL, vecs, n_vecs); + tor_assert(i == n_vecs); + *data = tor_malloc(n); + for (i=0; i < n_vecs; ++i) { + size_t copy = n - copied; + if (copy > vecs[i].iov_len) + copy = vecs[i].iov_len; + tor_assert(copied+copy <= n); + memcpy(data+copied, vecs[i].iov_base, copy); + copied += copy; + } + *free_out = 1; + return copied; + } +} + +/** As fetch_var_cell_from_buf, buf works on an evbuffer. */ +int +fetch_var_cell_from_evbuffer(struct evbuffer *buf, var_cell_t **out, + int linkproto) +{ + char *hdr = NULL; + int free_hdr = 0; + size_t n; + size_t buf_len; + uint8_t command; + uint16_t cell_length; + var_cell_t *cell; + int result = 0; + if (linkproto == 1) + return 0; + + *out = NULL; + buf_len = evbuffer_get_length(buf); + if (buf_len < VAR_CELL_HEADER_SIZE) + return 0; + + n = inspect_evbuffer(buf, &hdr, VAR_CELL_HEADER_SIZE, &free_hdr, NULL); + tor_assert(n >= VAR_CELL_HEADER_SIZE); + + command = get_uint8(hdr+2); + if (!(CELL_COMMAND_IS_VAR_LENGTH(command))) { + goto done; + } + + cell_length = ntohs(get_uint16(hdr+3)); + if (buf_len < (size_t)(VAR_CELL_HEADER_SIZE+cell_length)) { + result = 1; /* Not all here yet. */ + goto done; + } + + cell = var_cell_new(cell_length); + cell->command = command; + cell->circ_id = ntohs(get_uint16(hdr)); + evbuffer_drain(buf, VAR_CELL_HEADER_SIZE); + evbuffer_remove(buf, cell->payload, cell_length); + *out = cell; + result = 1; + + done: + if (free_hdr && hdr) + tor_free(hdr); + return result; +} +#endif + /** Move up to *<b>buf_flushlen</b> bytes from <b>buf_in</b> to * <b>buf_out</b>, and modify *<b>buf_flushlen</b> appropriately. * Return the number of bytes actually copied. @@ -1296,6 +1355,94 @@ fetch_from_buf_http(buf_t *buf, return 1; } +#ifdef USE_BUFFEREVENTS +/** As fetch_from_buf_http, buf works on an evbuffer. */ +int +fetch_from_evbuffer_http(struct evbuffer *buf, + char **headers_out, size_t max_headerlen, + char **body_out, size_t *body_used, size_t max_bodylen, + int force_complete) +{ + struct evbuffer_ptr crlf, content_length; + size_t headerlen, bodylen, contentlen; + + /* Find the first \r\n\r\n in the buffer */ + crlf = evbuffer_search(buf, "\r\n\r\n", 4, NULL); + if (crlf.pos < 0) { + /* We didn't find one. */ + if (evbuffer_get_length(buf) > max_headerlen) + return -1; /* Headers too long. */ + return 0; /* Headers not here yet. */ + } else if (crlf.pos > (int)max_headerlen) { + return -1; /* Headers too long. */ + } + + headerlen = crlf.pos + 4; /* Skip over the \r\n\r\n */ + bodylen = evbuffer_get_length(buf) - headerlen; + if (bodylen > max_bodylen) + return -1; /* body too long */ + + /* Look for the first occurrence of CONTENT_LENGTH insize buf before the + * crlfcrlf */ + content_length = evbuffer_search_range(buf, CONTENT_LENGTH, + strlen(CONTENT_LENGTH), NULL, &crlf); + + if (content_length.pos >= 0) { + /* We found a content_length: parse it and figure out if the body is here + * yet. */ + struct evbuffer_ptr eol; + char *data = NULL; + int free_data = 0; + int n, i; + n = evbuffer_ptr_set(buf, &content_length, strlen(CONTENT_LENGTH), + EVBUFFER_PTR_ADD); + tor_assert(n == 0); + eol = evbuffer_search_eol(buf, &content_length, NULL, EVBUFFER_EOL_CRLF); + tor_assert(eol.pos > content_length.pos); + tor_assert(eol.pos <= crlf.pos); + inspect_evbuffer(buf, &data, eol.pos - content_length.pos, &free_data, + &content_length); + + i = atoi(data); + if (free_data) + tor_free(data); + if (i < 0) { + log_warn(LD_PROTOCOL, "Content-Length is less than zero; it looks like " + "someone is trying to crash us."); + return -1; + } + contentlen = i; + /* if content-length is malformed, then our body length is 0. fine. */ + log_debug(LD_HTTP,"Got a contentlen of %d.",(int)contentlen); + if (bodylen < contentlen) { + if (!force_complete) { + log_debug(LD_HTTP,"body not all here yet."); + return 0; /* not all there yet */ + } + } + if (bodylen > contentlen) { + bodylen = contentlen; + log_debug(LD_HTTP,"bodylen reduced to %d.",(int)bodylen); + } + } + + if (headers_out) { + *headers_out = tor_malloc(headerlen+1); + evbuffer_remove(buf, *headers_out, headerlen); + (*headers_out)[headerlen] = '\0'; + } + if (body_out) { + tor_assert(headers_out); + tor_assert(body_used); + *body_used = bodylen; + *body_out = tor_malloc(bodylen+1); + evbuffer_remove(buf, *body_out, bodylen); + (*body_out)[bodylen] = '\0'; + } + return 1; +} +#endif + /** There is a (possibly incomplete) socks handshake on <b>buf</b>, of one * of the forms * - socks4: "socksheader username\\0" @@ -1325,6 +1472,128 @@ int fetch_from_buf_socks(buf_t *buf, socks_request_t *req, int log_sockstype, int safe_socks) { + int res; + ssize_t n_drain; + size_t want_length = 128; + + if (buf->datalen < 2) /* version and another byte */ + return 0; + + do { + n_drain = 0; + buf_pullup(buf, want_length, 0); + tor_assert(buf->head && buf->head->datalen >= 2); + want_length = 0; + + res = parse_socks(buf->head->data, buf->head->datalen, req, log_sockstype, + safe_socks, &n_drain, &want_length); + + if (n_drain < 0) + buf_clear(buf); + else if (n_drain > 0) + buf_remove_from_front(buf, n_drain); + + } while (res == 0 && buf->head && + buf->datalen > buf->head->datalen && + want_length < buf->head->datalen); + + return res; +} + +#ifdef USE_BUFFEREVENTS +/* As fetch_from_buf_socks(), but targets an evbuffer instead. */ +int +fetch_from_evbuffer_socks(struct evbuffer *buf, socks_request_t *req, + int log_sockstype, int safe_socks) +{ + char *data; + ssize_t n_drain; + size_t datalen, buflen, want_length; + int res; + + buflen = evbuffer_get_length(buf); + if (buflen < 2) + return 0; + + { + /* See if we can find the socks request in the first chunk of the buffer. + */ + struct evbuffer_iovec v; + int i; + want_length = evbuffer_get_contiguous_space(buf); + n_drain = 0; + i = evbuffer_peek(buf, want_length, NULL, &v, 1); + tor_assert(i == 1); + data = v.iov_base; + datalen = v.iov_len; + + res = parse_socks(data, datalen, req, log_sockstype, + safe_socks, &n_drain, &want_length); + + if (n_drain < 0) + evbuffer_drain(buf, evbuffer_get_length(buf)); + else if (n_drain > 0) + evbuffer_drain(buf, n_drain); + + if (res) + return res; + } + + /* Okay, the first chunk of the buffer didn't have a complete socks request. + * That means that either we don't have a whole socks request at all, or + * it's gotten split up. We're going to try passing parse_socks() bigger + * and bigger chunks until either it says "Okay, I got it", or it says it + * will need more data than we currently have. */ + + /* Loop while we have more data that we haven't given parse_socks() yet. */ + while (evbuffer_get_length(buf) > datalen) { + int free_data = 0; + n_drain = 0; + data = NULL; + datalen = inspect_evbuffer(buf, &data, want_length, &free_data, NULL); + + res = parse_socks(data, datalen, req, log_sockstype, + safe_socks, &n_drain, &want_length); + + if (free_data) + tor_free(data); + + if (n_drain < 0) + evbuffer_drain(buf, evbuffer_get_length(buf)); + else if (n_drain > 0) + evbuffer_drain(buf, n_drain); + + if (res) /* If res is nonzero, parse_socks() made up its mind. */ + return res; + + /* If parse_socks says that we want less data than we actually tried to + give it, we've got some kind of weird situation; just exit the loop for + now. + */ + if (want_length <= datalen) + break; + /* Otherwise, it wants more data than we gave it. If we can provide more + * data than we gave it, we'll try to do so in the next iteration of the + * loop. If we can't, the while loop will exit. It's okay if it asked for + * more than we have total; maybe it doesn't really need so much. */ + } + + return res; +} +#endif + +/** Implementation helper to implement fetch_from_*_socks. Instead of looking + * at a buffer's contents, we look at the <b>datalen</b> bytes of data in + * <b>data</b>. Instead of removing data from the buffer, we set + * <b>drain_out</b> to the amount of data that should be removed (or -1 if the + * buffer should be cleared). Instead of pulling more data into the first + * chunk of the buffer, we set *<b>want_length_out</b> to the number of bytes + * we'd like to see in the input buffer, if they're available. */ +static int +parse_socks(const char *data, size_t datalen, socks_request_t *req, + int log_sockstype, int safe_socks, ssize_t *drain_out, + size_t *want_length_out) +{ unsigned int len; char tmpbuf[TOR_ADDR_BUF_LEN+1]; tor_addr_t destaddr; @@ -1338,25 +1607,20 @@ fetch_from_buf_socks(buf_t *buf, socks_request_t *req, * then log a warning to let him know that it might be unwise. */ static int have_warned_about_unsafe_socks = 0; - if (buf->datalen < 2) /* version and another byte */ - return 0; - - buf_pullup(buf, 128, 0); - tor_assert(buf->head && buf->head->datalen >= 2); - - socksver = *buf->head->data; + socksver = *data; switch (socksver) { /* which version of socks? */ case 5: /* socks5 */ if (req->socks_version != 5) { /* we need to negotiate a method */ - unsigned char nummethods = (unsigned char)*(buf->head->data+1); + unsigned char nummethods = (unsigned char)*(data+1); tor_assert(!req->socks_version); - if (buf->datalen < 2u+nummethods) + if (datalen < 2u+nummethods) { + *want_length_out = 2u+nummethods; return 0; - buf_pullup(buf, 2u+nummethods, 0); - if (!nummethods || !memchr(buf->head->data+2, 0, nummethods)) { + } + if (!nummethods || !memchr(data+2, 0, nummethods)) { log_warn(LD_APP, "socks5: offered methods don't include 'no auth'. " "Rejecting."); @@ -1367,7 +1631,7 @@ fetch_from_buf_socks(buf_t *buf, socks_request_t *req, } /* remove packet from buf. also remove any other extraneous * bytes, to support broken socks clients. */ - buf_clear(buf); + *drain_out = -1; req->replylen = 2; /* 2 bytes of response */ req->reply[0] = 5; /* socks5 reply */ @@ -1378,10 +1642,11 @@ fetch_from_buf_socks(buf_t *buf, socks_request_t *req, } /* we know the method; read in the request */ log_debug(LD_APP,"socks5: checking request"); - if (buf->datalen < 8) /* basic info plus >=2 for addr plus 2 for port */ + if (datalen < 8) {/* basic info plus >=2 for addr plus 2 for port */ + *want_length_out = 8; return 0; /* not yet */ - tor_assert(buf->head->datalen >= 8); - req->command = (unsigned char) *(buf->head->data+1); + } + req->command = (unsigned char) *(data+1); if (req->command != SOCKS_COMMAND_CONNECT && req->command != SOCKS_COMMAND_RESOLVE && req->command != SOCKS_COMMAND_RESOLVE_PTR) { @@ -1390,19 +1655,21 @@ fetch_from_buf_socks(buf_t *buf, socks_request_t *req, req->command); return -1; } - switch (*(buf->head->data+3)) { /* address type */ + switch (*(data+3)) { /* address type */ case 1: /* IPv4 address */ case 4: /* IPv6 address */ { - const int is_v6 = *(buf->head->data+3) == 4; + const int is_v6 = *(data+3) == 4; const unsigned addrlen = is_v6 ? 16 : 4; log_debug(LD_APP,"socks5: ipv4 address type"); - if (buf->datalen < 6+addrlen) /* ip/port there? */ + if (datalen < 6+addrlen) {/* ip/port there? */ + *want_length_out = 6+addrlen; return 0; /* not yet */ + } if (is_v6) - tor_addr_from_ipv6_bytes(&destaddr, buf->head->data+4); + tor_addr_from_ipv6_bytes(&destaddr, data+4); else - tor_addr_from_ipv4n(&destaddr, get_uint32(buf->head->data+4)); + tor_addr_from_ipv4n(&destaddr, get_uint32(data+4)); tor_addr_to_str(tmpbuf, &destaddr, sizeof(tmpbuf), 1); @@ -1414,8 +1681,8 @@ fetch_from_buf_socks(buf_t *buf, socks_request_t *req, return -1; } strlcpy(req->address,tmpbuf,sizeof(req->address)); - req->port = ntohs(get_uint16(buf->head->data+4+addrlen)); - buf_remove_from_front(buf, 6+addrlen); + req->port = ntohs(get_uint16(data+4+addrlen)); + *drain_out = 6+addrlen; if (req->command != SOCKS_COMMAND_RESOLVE_PTR && !addressmap_have_mapping(req->address,0) && !have_warned_about_unsafe_socks) { @@ -1446,21 +1713,21 @@ fetch_from_buf_socks(buf_t *buf, socks_request_t *req, "hostname type. Rejecting."); return -1; } - len = (unsigned char)*(buf->head->data+4); - if (buf->datalen < 7+len) /* addr/port there? */ + len = (unsigned char)*(data+4); + if (datalen < 7+len) { /* addr/port there? */ + *want_length_out = 7+len; return 0; /* not yet */ - buf_pullup(buf, 7+len, 0); - tor_assert(buf->head->datalen >= 7+len); + } if (len+1 > MAX_SOCKS_ADDR_LEN) { log_warn(LD_APP, "socks5 hostname is %d bytes, which doesn't fit in " "%d. Rejecting.", len+1,MAX_SOCKS_ADDR_LEN); return -1; } - memcpy(req->address,buf->head->data+5,len); + memcpy(req->address,data+5,len); req->address[len] = 0; - req->port = ntohs(get_uint16(buf->head->data+5+len)); - buf_remove_from_front(buf, 5+len+2); + req->port = ntohs(get_uint16(data+5+len)); + *drain_out = 5+len+2; if (!tor_strisprint(req->address) || strchr(req->address,'\"')) { log_warn(LD_PROTOCOL, "Your application (using socks5 to port %d) gave Tor " @@ -1476,7 +1743,7 @@ fetch_from_buf_socks(buf_t *buf, socks_request_t *req, return 1; default: /* unsupported */ log_warn(LD_APP,"socks5: unsupported address type %d. Rejecting.", - (int) *(buf->head->data+3)); + (int) *(data+3)); return -1; } tor_assert(0); @@ -1485,10 +1752,12 @@ fetch_from_buf_socks(buf_t *buf, socks_request_t *req, /* http://archive.socks.permeo.com/protocol/socks4a.protocol */ req->socks_version = 4; - if (buf->datalen < SOCKS4_NETWORK_LEN) /* basic info available? */ + if (datalen < SOCKS4_NETWORK_LEN) {/* basic info available? */ + *want_length_out = SOCKS4_NETWORK_LEN; return 0; /* not yet */ - buf_pullup(buf, 1280, 0); - req->command = (unsigned char) *(buf->head->data+1); + } + // buf_pullup(buf, 1280, 0); + req->command = (unsigned char) *(data+1); if (req->command != SOCKS_COMMAND_CONNECT && req->command != SOCKS_COMMAND_RESOLVE) { /* not a connect or resolve? we don't support it. (No resolve_ptr with @@ -1498,8 +1767,8 @@ fetch_from_buf_socks(buf_t *buf, socks_request_t *req, return -1; } - req->port = ntohs(get_uint16(buf->head->data+2)); - destip = ntohl(get_uint32(buf->head->data+4)); + req->port = ntohs(get_uint16(data+2)); + destip = ntohl(get_uint32(data+4)); if ((!req->port && req->command!=SOCKS_COMMAND_RESOLVE) || !destip) { log_warn(LD_APP,"socks4: Port or DestIP is zero. Rejecting."); return -1; @@ -1519,17 +1788,18 @@ fetch_from_buf_socks(buf_t *buf, socks_request_t *req, socks4_prot = socks4; } - next = memchr(buf->head->data+SOCKS4_NETWORK_LEN, 0, - buf->head->datalen-SOCKS4_NETWORK_LEN); + next = memchr(data+SOCKS4_NETWORK_LEN, 0, + datalen-SOCKS4_NETWORK_LEN); if (!next) { - if (buf->head->datalen >= 1024) { + if (datalen >= 1024) { log_debug(LD_APP, "Socks4 user name too long; rejecting."); return -1; } log_debug(LD_APP,"socks4: Username not here yet."); + *want_length_out = datalen+1024; /* ???? */ return 0; } - tor_assert(next < CHUNK_WRITE_PTR(buf->head)); + tor_assert(next < data+datalen); startaddr = NULL; if (socks4_prot != socks4a && @@ -1554,18 +1824,20 @@ fetch_from_buf_socks(buf_t *buf, socks_request_t *req, return -1; } if (socks4_prot == socks4a) { - if (next+1 == CHUNK_WRITE_PTR(buf->head)) { + if (next+1 == data+datalen) { log_debug(LD_APP,"socks4: No part of destaddr here yet."); + *want_length_out = datalen + 1024; /* More than we need, but safe */ return 0; } startaddr = next+1; - next = memchr(startaddr, 0, CHUNK_WRITE_PTR(buf->head)-startaddr); + next = memchr(startaddr, 0, data + datalen - startaddr); if (!next) { - if (buf->head->datalen >= 1024) { + if (datalen >= 1024) { log_debug(LD_APP,"socks4: Destaddr too long."); return -1; } log_debug(LD_APP,"socks4: Destaddr not all here yet."); + *want_length_out = datalen + 1024; return 0; } if (MAX_SOCKS_ADDR_LEN <= next-startaddr) { @@ -1591,7 +1863,7 @@ fetch_from_buf_socks(buf_t *buf, socks_request_t *req, return -1; } /* next points to the final \0 on inbuf */ - buf_remove_from_front(buf, next - buf->head->data + 1); + *drain_out = next - data + 1; return 1; case 'G': /* get */ @@ -1629,9 +1901,10 @@ fetch_from_buf_socks(buf_t *buf, socks_request_t *req, default: /* version is not socks4 or socks5 */ log_warn(LD_APP, "Socks version %d not recognized. (Tor is not an http proxy.)", - *(buf->head->data)); + *(data)); { - char *tmp = tor_strndup(buf->head->data, 8); /*XXXX what if longer?*/ + /* Tell the controller the first 8 bytes. */ + char *tmp = tor_strndup(data, datalen < 8 ? datalen : 8); control_event_client_status(LOG_WARN, "SOCKS_UNKNOWN_PROTOCOL DATA=\"%s\"", escaped(tmp)); @@ -1653,21 +1926,63 @@ fetch_from_buf_socks(buf_t *buf, socks_request_t *req, int fetch_from_buf_socks_client(buf_t *buf, int state, char **reason) { - unsigned char *data; - size_t addrlen; - + ssize_t drain = 0; + int r; if (buf->datalen < 2) return 0; buf_pullup(buf, 128, 0); tor_assert(buf->head && buf->head->datalen >= 2); - data = (unsigned char *) buf->head->data; + r = parse_socks_client((uint8_t*)buf->head->data, buf->head->datalen, + state, reason, &drain); + if (drain > 0) + buf_remove_from_front(buf, drain); + else if (drain < 0) + buf_clear(buf); + + return r; +} + +#ifdef USE_BUFFEREVENTS +/** As fetch_from_buf_socks_client, buf works on an evbuffer */ +int +fetch_from_evbuffer_socks_client(struct evbuffer *buf, int state, + char **reason) +{ + ssize_t drain = 0; + uint8_t *data; + size_t datalen; + int r; + + data = evbuffer_pullup(buf, 128); /* Make sure we have at least 128 + * contiguous bytes if possible. */ + datalen = evbuffer_get_contiguous_space(buf); + r = parse_socks_client(data, datalen, state, reason, &drain); + if (drain > 0) + evbuffer_drain(buf, drain); + else + evbuffer_drain(buf, evbuffer_get_length(buf)); + + return r; +} +#endif + +/** Implementation logic for fetch_from_*_socks_client. */ +static int +parse_socks_client(const uint8_t *data, size_t datalen, + int state, char **reason, + ssize_t *drain_out) +{ + unsigned int addrlen; + *drain_out = 0; + if (datalen < 2) + return 0; switch (state) { case PROXY_SOCKS4_WANT_CONNECT_OK: /* Wait for the complete response */ - if (buf->head->datalen < 8) + if (datalen < 8) return 0; if (data[1] != 0x5a) { @@ -1676,7 +1991,7 @@ fetch_from_buf_socks_client(buf_t *buf, int state, char **reason) } /* Success */ - buf_remove_from_front(buf, 8); + *drain_out = 8; return 1; case PROXY_SOCKS5_WANT_AUTH_METHOD_NONE: @@ -1688,7 +2003,7 @@ fetch_from_buf_socks_client(buf_t *buf, int state, char **reason) } log_info(LD_NET, "SOCKS 5 client: continuing without authentication"); - buf_clear(buf); + *drain_out = -1; return 1; case PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929: @@ -1698,11 +2013,11 @@ fetch_from_buf_socks_client(buf_t *buf, int state, char **reason) case 0x00: log_info(LD_NET, "SOCKS 5 client: we have auth details but server " "doesn't require authentication."); - buf_clear(buf); + *drain_out = -1; return 1; case 0x02: log_info(LD_NET, "SOCKS 5 client: need authentication."); - buf_clear(buf); + *drain_out = -1; return 2; /* fall through */ } @@ -1719,7 +2034,7 @@ fetch_from_buf_socks_client(buf_t *buf, int state, char **reason) } log_info(LD_NET, "SOCKS 5 client: authentication successful."); - buf_clear(buf); + *drain_out = -1; return 1; case PROXY_SOCKS5_WANT_CONNECT_OK: @@ -1728,7 +2043,7 @@ fetch_from_buf_socks_client(buf_t *buf, int state, char **reason) * the data used */ /* wait for address type field to arrive */ - if (buf->datalen < 4) + if (datalen < 4) return 0; switch (data[3]) { @@ -1739,7 +2054,7 @@ fetch_from_buf_socks_client(buf_t *buf, int state, char **reason) addrlen = 16; break; case 0x03: /* fqdn (can this happen here?) */ - if (buf->datalen < 5) + if (datalen < 5) return 0; addrlen = 1 + data[4]; break; @@ -1749,7 +2064,7 @@ fetch_from_buf_socks_client(buf_t *buf, int state, char **reason) } /* wait for address and port */ - if (buf->datalen < 6 + addrlen) + if (datalen < 6 + addrlen) return 0; if (data[1] != 0x00) { @@ -1757,7 +2072,7 @@ fetch_from_buf_socks_client(buf_t *buf, int state, char **reason) return -1; } - buf_remove_from_front(buf, 6 + addrlen); + *drain_out = 6 + addrlen; return 1; } @@ -1783,6 +2098,27 @@ peek_buf_has_control0_command(buf_t *buf) return 0; } +#ifdef USE_BUFFEREVENTS +int +peek_evbuffer_has_control0_command(struct evbuffer *buf) +{ + int result = 0; + if (evbuffer_get_length(buf) >= 4) { + int free_out = 0; + char *data = NULL; + size_t n = inspect_evbuffer(buf, &data, 4, &free_out, NULL); + uint16_t cmd; + tor_assert(n >= 4); + cmd = ntohs(get_uint16(data+2)); + if (cmd <= 0x14) + result = 1; + if (free_out) + tor_free(data); + } + return result; +} +#endif + /** Return the index within <b>buf</b> at which <b>ch</b> first appears, * or -1 if <b>ch</b> does not appear on buf. */ static off_t @@ -1800,12 +2136,12 @@ buf_find_offset_of_char(buf_t *buf, char ch) return -1; } -/** Try to read a single LF-terminated line from <b>buf</b>, and write it, - * NUL-terminated, into the *<b>data_len</b> byte buffer at <b>data_out</b>. - * Set *<b>data_len</b> to the number of bytes in the line, not counting the - * terminating NUL. Return 1 if we read a whole line, return 0 if we don't - * have a whole line yet, and return -1 if the line length exceeds - * *<b>data_len</b>. +/** Try to read a single LF-terminated line from <b>buf</b>, and write it + * (including the LF), NUL-terminated, into the *<b>data_len</b> byte buffer + * at <b>data_out</b>. Set *<b>data_len</b> to the number of bytes in the + * line, not counting the terminating NUL. Return 1 if we read a whole line, + * return 0 if we don't have a whole line yet, and return -1 if the line + * length exceeds *<b>data_len</b>. */ int fetch_from_buf_line(buf_t *buf, char *data_out, size_t *data_len) @@ -1879,6 +2215,61 @@ write_to_buf_zlib(buf_t *buf, tor_zlib_state_t *state, return 0; } +#ifdef USE_BUFFEREVENTS +int +write_to_evbuffer_zlib(struct evbuffer *buf, tor_zlib_state_t *state, + const char *data, size_t data_len, + int done) +{ + char *next; + size_t old_avail, avail; + int over = 0, n; + struct evbuffer_iovec vec[1]; + do { + int need_new_chunk = 0; + { + size_t cap = data_len / 4; + if (cap < 128) + cap = 128; + /* XXXX NM this strategy is fragmentation-prone. We should really have + * two iovecs, and write first into the one, and then into the + * second if the first gets full. */ + n = evbuffer_reserve_space(buf, cap, vec, 1); + tor_assert(n == 1); + } + + next = vec[0].iov_base; + avail = old_avail = vec[0].iov_len; + + switch (tor_zlib_process(state, &next, &avail, &data, &data_len, done)) { + case TOR_ZLIB_DONE: + over = 1; + break; + case TOR_ZLIB_ERR: + return -1; + case TOR_ZLIB_OK: + if (data_len == 0) + over = 1; + break; + case TOR_ZLIB_BUF_FULL: + if (avail) { + /* Zlib says we need more room (ZLIB_BUF_FULL). Start a new chunk + * automatically, whether were going to or not. */ + need_new_chunk = 1; + } + break; + } + + /* XXXX possible infinite loop on BUF_FULL. */ + vec[0].iov_len = old_avail - avail; + evbuffer_commit_space(buf, vec, 1); + + } while (!over); + check(); + return 0; +} +#endif + /** Log an error and exit if <b>buf</b> is corrupted. */ void diff --git a/src/or/buffers.h b/src/or/buffers.h index 8fd403d954..35c1dd2ea5 100644 --- a/src/or/buffers.h +++ b/src/or/buffers.h @@ -48,6 +48,23 @@ int fetch_from_buf_line(buf_t *buf, char *data_out, size_t *data_len); int peek_buf_has_control0_command(buf_t *buf); +#ifdef USE_BUFFEREVENTS +int fetch_var_cell_from_evbuffer(struct evbuffer *buf, var_cell_t **out, + int linkproto); +int fetch_from_evbuffer_socks(struct evbuffer *buf, socks_request_t *req, + int log_sockstype, int safe_socks); +int fetch_from_evbuffer_socks_client(struct evbuffer *buf, int state, + char **reason); +int fetch_from_evbuffer_http(struct evbuffer *buf, + char **headers_out, size_t max_headerlen, + char **body_out, size_t *body_used, size_t max_bodylen, + int force_complete); +int peek_evbuffer_has_control0_command(struct evbuffer *buf); +int write_to_evbuffer_zlib(struct evbuffer *buf, tor_zlib_state_t *state, + const char *data, size_t data_len, + int done); +#endif + void assert_buf_ok(buf_t *buf); #ifdef BUFFERS_PRIVATE diff --git a/src/or/config.c b/src/or/config.c index d66f9136b7..4f7077153d 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -224,6 +224,7 @@ static config_var_t _option_vars[] = { V(DirReqStatistics, BOOL, "0"), VAR("DirServer", LINELIST, DirServers, NULL), V(DisableAllSwap, BOOL, "0"), + V(DisableIOCP, BOOL, "1"), V(DNSPort, UINT, "0"), V(DNSListenAddress, LINELIST, NULL), V(DownloadExtraInfo, BOOL, "0"), @@ -306,7 +307,7 @@ static config_var_t _option_vars[] = { V(WarnUnsafeSocks, BOOL, "1"), V(NoPublish, BOOL, "0"), VAR("NodeFamily", LINELIST, NodeFamilies, NULL), - V(NumCpus, UINT, "1"), + V(NumCpus, UINT, "0"), V(NumEntryGuards, UINT, "3"), V(ORListenAddress, LINELIST, NULL), V(ORPort, UINT, "0"), @@ -316,6 +317,8 @@ static config_var_t _option_vars[] = { V(PerConnBWRate, MEMUNIT, "0"), V(PidFile, STRING, NULL), V(TestingTorNetwork, BOOL, "0"), + V(PortForwarding, BOOL, "0"), + V(PortForwardingHelper, FILENAME, "tor-fw-helper"), V(PreferTunneledDirConns, BOOL, "1"), V(ProtocolWarnings, BOOL, "0"), V(PublishServerDescriptor, CSV, "1"), @@ -554,7 +557,7 @@ static int is_listening_on_low_port(uint16_t port_option, static uint64_t config_parse_memunit(const char *s, int *ok); static int config_parse_interval(const char *s, int *ok); -static void init_libevent(void); +static void init_libevent(const or_options_t *options); static int opt_streq(const char *s1, const char *s2); /** Magic value for or_options_t. */ @@ -955,7 +958,7 @@ options_act_reversible(or_options_t *old_options, char **msg) /* Set up libevent. (We need to do this before we can register the * listeners as listeners.) */ if (running_tor && !libevent_initialized) { - init_libevent(); + init_libevent(options); libevent_initialized = 1; } @@ -1231,6 +1234,17 @@ options_act(or_options_t *old_options) if (accounting_is_enabled(options)) configure_accounting(time(NULL)); +#ifdef USE_BUFFEREVENTS + /* If we're using the bufferevents implementation and our rate limits + * changed, we need to tell the rate-limiting system about it. */ + if (!old_options || + old_options->BandwidthRate != options->BandwidthRate || + old_options->BandwidthBurst != options->BandwidthBurst || + old_options->RelayBandwidthRate != options->RelayBandwidthRate || + old_options->RelayBandwidthBurst != options->RelayBandwidthBurst) + connection_bucket_init(); +#endif + /* parse RefuseUnknownExits tristate */ if (!strcmp(options->RefuseUnknownExits, "0")) options->RefuseUnknownExits_ = 0; @@ -4880,13 +4894,29 @@ config_parse_interval(const char *s, int *ok) return (int)r; } +/** Return the number of cpus configured in <b>options</b>. If we are + * told to auto-detect the number of cpus, return the auto-detected number. */ +int +get_num_cpus(const or_options_t *options) +{ + if (options->NumCpus == 0) { + int n = compute_num_cpus(); + return (n >= 1) ? n : 1; + } else { + return options->NumCpus; + } +} + /** * Initialize the libevent library. */ static void -init_libevent(void) +init_libevent(const or_options_t *options) { const char *badness=NULL; + tor_libevent_cfg cfg; + + tor_assert(options); configure_libevent_logging(); /* If the kernel complains that some method (say, epoll) doesn't @@ -4896,7 +4926,11 @@ init_libevent(void) tor_check_libevent_header_compatibility(); - tor_libevent_initialize(); + memset(&cfg, 0, sizeof(cfg)); + cfg.disable_iocp = options->DisableIOCP; + cfg.num_cpus = get_num_cpus(options); + + tor_libevent_initialize(&cfg); suppress_libevent_log_msg(NULL); diff --git a/src/or/config.h b/src/or/config.h index 7a4ba5c60f..bd5827b4e8 100644 --- a/src/or/config.h +++ b/src/or/config.h @@ -57,6 +57,8 @@ char *options_get_datadir_fname2_suffix(or_options_t *options, #define get_datadir_fname_suffix(sub1, suffix) \ get_datadir_fname2_suffix((sub1), NULL, (suffix)) +int get_num_cpus(const or_options_t *options); + or_state_t *get_or_state(void); int or_state_save(time_t now); diff --git a/src/or/connection.c b/src/or/connection.c index 91ce74b5b0..d1b079a97e 100644 --- a/src/or/connection.c +++ b/src/or/connection.c @@ -36,6 +36,10 @@ #include "router.h" #include "routerparse.h" +#ifdef USE_BUFFEREVENTS +#include <event2/event.h> +#endif + static connection_t *connection_create_listener( struct sockaddr *listensockaddr, socklen_t listensocklen, int type, @@ -45,8 +49,10 @@ static void connection_init(time_t now, connection_t *conn, int type, static int connection_init_accepted_conn(connection_t *conn, uint8_t listener_type); static int connection_handle_listener_read(connection_t *conn, int new_type); +#ifndef USE_BUFFEREVENTS static int connection_bucket_should_increase(int bucket, or_connection_t *conn); +#endif static int connection_finished_flushing(connection_t *conn); static int connection_flushed_some(connection_t *conn); static int connection_finished_connecting(connection_t *conn); @@ -183,6 +189,26 @@ conn_state_to_string(int type, int state) return buf; } +#ifdef USE_BUFFEREVENTS +/** Return true iff the connection's type is one that can use a + bufferevent-based implementation. */ +int +connection_type_uses_bufferevent(connection_t *conn) +{ + switch (conn->type) { + case CONN_TYPE_AP: + case CONN_TYPE_EXIT: + case CONN_TYPE_DIR: + case CONN_TYPE_CONTROL: + case CONN_TYPE_OR: + case CONN_TYPE_CPUWORKER: + return 1; + default: + return 0; + } +} +#endif + /** Allocate and return a new dir_connection_t, initialized as by * connection_init(). */ dir_connection_t * @@ -308,10 +334,13 @@ connection_init(time_t now, connection_t *conn, int type, int socket_family) conn->type = type; conn->socket_family = socket_family; - if (!connection_is_listener(conn)) { /* listeners never use their buf */ +#ifndef USE_BUFFEREVENTS + if (!connection_is_listener(conn)) { + /* listeners never use their buf */ conn->inbuf = buf_new(); conn->outbuf = buf_new(); } +#endif conn->timestamp_created = now; conn->timestamp_lastread = now; @@ -377,7 +406,8 @@ _connection_free(connection_t *conn) "bytes on inbuf, %d on outbuf.", conn_type_to_string(conn->type), conn_state_to_string(conn->type, conn->state), - (int)buf_datalen(conn->inbuf), (int)buf_datalen(conn->outbuf)); + (int)connection_get_inbuf_len(conn), + (int)connection_get_outbuf_len(conn)); } if (!connection_is_listener(conn)) { @@ -424,6 +454,15 @@ _connection_free(connection_t *conn) tor_free(conn->read_event); /* Probably already freed by connection_free. */ tor_free(conn->write_event); /* Probably already freed by connection_free. */ + IF_HAS_BUFFEREVENT(conn, { + /* This was a workaround to handle bugs in some old versions of libevent + * where callbacks can occur after calling bufferevent_free(). Setting + * the callbacks to NULL prevented this. It shouldn't be necessary any + * more, but let's not tempt fate for now. */ + bufferevent_setcb(conn->bufev, NULL, NULL, NULL, NULL); + bufferevent_free(conn->bufev); + conn->bufev = NULL; + }); if (conn->type == CONN_TYPE_DIR) { dir_connection_t *dir_conn = TO_DIR_CONN(conn); @@ -450,6 +489,11 @@ _connection_free(connection_t *conn) log_warn(LD_BUG, "called on OR conn with non-zeroed identity_digest"); connection_or_remove_from_identity_map(TO_OR_CONN(conn)); } +#ifdef USE_BUFFEREVENTS + if (conn->type == CONN_TYPE_OR && TO_OR_CONN(conn)->bucket_cfg) { + ev_token_bucket_cfg_free(TO_OR_CONN(conn)->bucket_cfg); + } +#endif memset(mem, 0xCC, memlen); /* poison memory */ tor_free(mem); @@ -675,10 +719,9 @@ connection_close_immediate(connection_t *conn) conn->s = -1; if (conn->linked) conn->linked_conn_is_closed = 1; - if (!connection_is_listener(conn)) { + if (conn->outbuf) buf_clear(conn->outbuf); - conn->outbuf_flushlen = 0; - } + conn->outbuf_flushlen = 0; } /** Mark <b>conn</b> to be closed next time we loop through @@ -1319,7 +1362,7 @@ connection_connect(connection_t *conn, const char *address, escaped_safe_str_client(address), port, inprogress?"in progress":"established", s); conn->s = s; - if (connection_add(conn) < 0) /* no space, forget it */ + if (connection_add_connecting(conn) < 0) /* no space, forget it */ return -1; return inprogress ? 0 : 1; } @@ -1542,6 +1585,19 @@ connection_send_socks5_connect(connection_t *conn) conn->proxy_state = PROXY_SOCKS5_WANT_CONNECT_OK; } +/** DOCDOC */ +static int +connection_fetch_from_buf_socks_client(connection_t *conn, + int state, char **reason) +{ + IF_HAS_BUFFEREVENT(conn, { + struct evbuffer *input = bufferevent_get_input(conn->bufev); + return fetch_from_evbuffer_socks_client(input, state, reason); + }) ELSE_IF_NO_BUFFEREVENT { + return fetch_from_buf_socks_client(conn->inbuf, state, reason); + } +} + /** Call this from connection_*_process_inbuf() to advance the proxy * handshake. * @@ -1569,17 +1625,17 @@ connection_read_proxy_handshake(connection_t *conn) break; case PROXY_SOCKS4_WANT_CONNECT_OK: - ret = fetch_from_buf_socks_client(conn->inbuf, - conn->proxy_state, - &reason); + ret = connection_fetch_from_buf_socks_client(conn, + conn->proxy_state, + &reason); if (ret == 1) conn->proxy_state = PROXY_CONNECTED; break; case PROXY_SOCKS5_WANT_AUTH_METHOD_NONE: - ret = fetch_from_buf_socks_client(conn->inbuf, - conn->proxy_state, - &reason); + ret = connection_fetch_from_buf_socks_client(conn, + conn->proxy_state, + &reason); /* no auth needed, do connect */ if (ret == 1) { connection_send_socks5_connect(conn); @@ -1588,9 +1644,9 @@ connection_read_proxy_handshake(connection_t *conn) break; case PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929: - ret = fetch_from_buf_socks_client(conn->inbuf, - conn->proxy_state, - &reason); + ret = connection_fetch_from_buf_socks_client(conn, + conn->proxy_state, + &reason); /* send auth if needed, otherwise do connect */ if (ret == 1) { @@ -1625,9 +1681,9 @@ connection_read_proxy_handshake(connection_t *conn) break; case PROXY_SOCKS5_WANT_AUTH_RFC1929_OK: - ret = fetch_from_buf_socks_client(conn->inbuf, - conn->proxy_state, - &reason); + ret = connection_fetch_from_buf_socks_client(conn, + conn->proxy_state, + &reason); /* send the connect request */ if (ret == 1) { connection_send_socks5_connect(conn); @@ -1636,9 +1692,9 @@ connection_read_proxy_handshake(connection_t *conn) break; case PROXY_SOCKS5_WANT_CONNECT_OK: - ret = fetch_from_buf_socks_client(conn->inbuf, - conn->proxy_state, - &reason); + ret = connection_fetch_from_buf_socks_client(conn, + conn->proxy_state, + &reason); if (ret == 1) conn->proxy_state = PROXY_CONNECTED; break; @@ -1902,6 +1958,9 @@ connection_is_rate_limited(connection_t *conn) return 1; } +#ifdef USE_BUFFEREVENTS +static struct bufferevent_rate_limit_group *global_rate_limit = NULL; +#else extern int global_read_bucket, global_write_bucket; extern int global_relayed_read_bucket, global_relayed_write_bucket; @@ -1909,11 +1968,13 @@ extern int global_relayed_read_bucket, global_relayed_write_bucket; * we are likely to run dry again this second, so be stingy with the * tokens we just put in. */ static int write_buckets_empty_last_second = 0; +#endif /** How many seconds of no active local circuits will make the * connection revert to the "relayed" bandwidth class? */ #define CLIENT_IDLE_TIME_FOR_PRIORITY 30 +#ifndef USE_BUFFEREVENTS /** Return 1 if <b>conn</b> should use tokens from the "relayed" * bandwidth rates, else 0. Currently, only OR conns with bandwidth * class 1, and directory conns that are serving data out, count. @@ -2024,6 +2085,20 @@ connection_bucket_write_limit(connection_t *conn, time_t now) return connection_bucket_round_robin(base, priority, global_bucket, conn_bucket); } +#else +static ssize_t +connection_bucket_read_limit(connection_t *conn, time_t now) +{ + (void) now; + return bufferevent_get_max_to_read(conn->bufev); +} +ssize_t +connection_bucket_write_limit(connection_t *conn, time_t now) +{ + (void) now; + return bufferevent_get_max_to_write(conn->bufev); +} +#endif /** Return 1 if the global write buckets are low enough that we * shouldn't send <b>attempt</b> bytes of low-priority directory stuff @@ -2048,8 +2123,12 @@ connection_bucket_write_limit(connection_t *conn, time_t now) int global_write_bucket_low(connection_t *conn, size_t attempt, int priority) { +#ifdef USE_BUFFEREVENTS + ssize_t smaller_bucket = bufferevent_get_max_to_write(conn->bufev); +#else int smaller_bucket = global_write_bucket < global_relayed_write_bucket ? global_write_bucket : global_relayed_write_bucket; +#endif if (authdir_mode(get_options()) && priority>1) return 0; /* there's always room to answer v2 if we're an auth dir */ @@ -2059,8 +2138,10 @@ global_write_bucket_low(connection_t *conn, size_t attempt, int priority) if (smaller_bucket < (int)attempt) return 1; /* not enough space no matter the priority */ +#ifndef USE_BUFFEREVENTS if (write_buckets_empty_last_second) return 1; /* we're already hitting our limits, no more please */ +#endif if (priority == 1) { /* old-style v1 query */ /* Could we handle *two* of these requests within the next two seconds? */ @@ -2076,6 +2157,7 @@ global_write_bucket_low(connection_t *conn, size_t attempt, int priority) return 0; } +#ifndef USE_BUFFEREVENTS /** We just read <b>num_read</b> and wrote <b>num_written</b> bytes * onto <b>conn</b>. Decrement buckets appropriately. */ static void @@ -2319,6 +2401,88 @@ connection_bucket_should_increase(int bucket, or_connection_t *conn) return 1; } +#else + +static void +connection_buckets_decrement(connection_t *conn, time_t now, + size_t num_read, size_t num_written) +{ + (void) conn; + (void) now; + (void) num_read; + (void) num_written; + /* Libevent does this for us. */ +} +void +connection_bucket_refill(int seconds_elapsed, time_t now) +{ + (void) seconds_elapsed; + (void) now; + /* Libevent does this for us. */ +} +void +connection_bucket_init(void) +{ + or_options_t *options = get_options(); + const struct timeval *tick = tor_libevent_get_one_tick_timeout(); + struct ev_token_bucket_cfg *bucket_cfg; + + uint64_t rate, burst; + if (options->RelayBandwidthRate) { + rate = options->RelayBandwidthRate; + burst = options->RelayBandwidthBurst; + } else { + rate = options->BandwidthRate; + burst = options->BandwidthBurst; + } + + rate /= TOR_LIBEVENT_TICKS_PER_SECOND; + bucket_cfg = ev_token_bucket_cfg_new((uint32_t)rate, (uint32_t)burst, + (uint32_t)rate, (uint32_t)burst, + tick); + + if (!global_rate_limit) { + global_rate_limit = + bufferevent_rate_limit_group_new(tor_libevent_get_base(), bucket_cfg); + } else { + bufferevent_rate_limit_group_set_cfg(global_rate_limit, bucket_cfg); + } + ev_token_bucket_cfg_free(bucket_cfg); +} + +void +connection_get_rate_limit_totals(uint64_t *read_out, uint64_t *written_out) +{ + if (global_rate_limit == NULL) { + *read_out = *written_out = 0; + } else { + bufferevent_rate_limit_group_get_totals( + global_rate_limit, read_out, written_out); + } +} + +/** DOCDOC */ +void +connection_enable_rate_limiting(connection_t *conn) +{ + if (conn->bufev) { + if (!global_rate_limit) + connection_bucket_init(); + bufferevent_add_to_rate_limit_group(conn->bufev, global_rate_limit); + } +} + +static void +connection_consider_empty_write_buckets(connection_t *conn) +{ + (void) conn; +} +static void +connection_consider_empty_read_buckets(connection_t *conn) +{ + (void) conn; +} +#endif /** Read bytes from conn-\>s and process them. * @@ -2572,13 +2736,13 @@ connection_read_to_buf(connection_t *conn, int *max_to_read, int *socket_error) } if (n_read > 0) { /* change *max_to_read */ - /*XXXX021 check for overflow*/ + /*XXXX022 check for overflow*/ *max_to_read = (int)(at_most - n_read); } if (conn->type == CONN_TYPE_AP) { edge_connection_t *edge_conn = TO_EDGE_CONN(conn); - /*XXXX021 check for overflow*/ + /*XXXX022 check for overflow*/ edge_conn->n_read += (int)n_read; } @@ -2600,11 +2764,202 @@ connection_read_to_buf(connection_t *conn, int *max_to_read, int *socket_error) return 0; } +#ifdef USE_BUFFEREVENTS +/* XXXX These generic versions could be simplified by making them + type-specific */ + +/** Callback: Invoked whenever bytes are added to or drained from an input + * evbuffer. Used to track the number of bytes read. */ +static void +evbuffer_inbuf_callback(struct evbuffer *buf, + const struct evbuffer_cb_info *info, void *arg) +{ + connection_t *conn = arg; + (void) buf; + /* XXXX These need to get real counts on the non-nested TLS case. - NM */ + if (info->n_added) { + time_t now = approx_time(); + conn->timestamp_lastread = now; + connection_buckets_decrement(conn, now, info->n_added, 0); + connection_consider_empty_read_buckets(conn); + if (conn->type == CONN_TYPE_AP) { + edge_connection_t *edge_conn = TO_EDGE_CONN(conn); + /*XXXX022 check for overflow*/ + edge_conn->n_read += (int)info->n_added; + } + } +} + +/** Callback: Invoked whenever bytes are added to or drained from an output + * evbuffer. Used to track the number of bytes written. */ +static void +evbuffer_outbuf_callback(struct evbuffer *buf, + const struct evbuffer_cb_info *info, void *arg) +{ + connection_t *conn = arg; + (void)buf; + if (info->n_deleted) { + time_t now = approx_time(); + conn->timestamp_lastwritten = now; + connection_buckets_decrement(conn, now, 0, info->n_deleted); + connection_consider_empty_write_buckets(conn); + if (conn->type == CONN_TYPE_AP) { + edge_connection_t *edge_conn = TO_EDGE_CONN(conn); + /*XXXX022 check for overflow*/ + edge_conn->n_written += (int)info->n_deleted; + } + } +} + +/** Callback: invoked whenever a bufferevent has read data. */ +void +connection_handle_read_cb(struct bufferevent *bufev, void *arg) +{ + connection_t *conn = arg; + (void) bufev; + if (!conn->marked_for_close) + if (connection_process_inbuf(conn, 1)<0) /* XXXX Always 1? */ + connection_mark_for_close(conn); +} + +/** Callback: invoked whenever a bufferevent has written data. */ +void +connection_handle_write_cb(struct bufferevent *bufev, void *arg) +{ + connection_t *conn = arg; + struct evbuffer *output; + if (connection_flushed_some(conn)<0) { + connection_mark_for_close(conn); + return; + } + + output = bufferevent_get_output(bufev); + if (!evbuffer_get_length(output)) { + connection_finished_flushing(conn); + if (conn->marked_for_close && conn->hold_open_until_flushed) { + conn->hold_open_until_flushed = 0; + if (conn->linked) { + /* send eof */ + bufferevent_flush(conn->bufev, EV_WRITE, BEV_FINISHED); + } + } + } +} + +/** Callback: invoked whenever a bufferevent has had an event (like a + * connection, or an eof, or an error) occur. */ +void +connection_handle_event_cb(struct bufferevent *bufev, short event, void *arg) +{ + connection_t *conn = arg; + (void) bufev; + if (event & BEV_EVENT_CONNECTED) { + tor_assert(connection_state_is_connecting(conn)); + if (connection_finished_connecting(conn)<0) + return; + } + if (event & BEV_EVENT_EOF) { + if (!conn->marked_for_close) { + conn->inbuf_reached_eof = 1; + if (connection_reached_eof(conn)<0) + return; + } + } + if (event & BEV_EVENT_ERROR) { + int socket_error = evutil_socket_geterror(conn->s); + if (conn->type == CONN_TYPE_OR && + conn->state == OR_CONN_STATE_CONNECTING) { + connection_or_connect_failed(TO_OR_CONN(conn), + errno_to_orconn_end_reason(socket_error), + tor_socket_strerror(socket_error)); + } else if (CONN_IS_EDGE(conn)) { + edge_connection_t *edge_conn = TO_EDGE_CONN(conn); + if (!edge_conn->edge_has_sent_end) + connection_edge_end_errno(edge_conn); + if (edge_conn->socks_request) /* broken, don't send a socks reply back */ + edge_conn->socks_request->has_finished = 1; + } + connection_close_immediate(conn); /* Connection is dead. */ + if (!conn->marked_for_close) + connection_mark_for_close(conn); + } +} + +/** Set up the generic callbacks for the bufferevent on <b>conn</b>. */ +void +connection_configure_bufferevent_callbacks(connection_t *conn) +{ + struct bufferevent *bufev; + struct evbuffer *input, *output; + tor_assert(conn->bufev); + bufev = conn->bufev; + bufferevent_setcb(bufev, + connection_handle_read_cb, + connection_handle_write_cb, + connection_handle_event_cb, + conn); + + input = bufferevent_get_input(bufev); + output = bufferevent_get_output(bufev); + evbuffer_add_cb(input, evbuffer_inbuf_callback, conn); + evbuffer_add_cb(output, evbuffer_outbuf_callback, conn); +} +#endif + /** A pass-through to fetch_from_buf. */ int connection_fetch_from_buf(char *string, size_t len, connection_t *conn) { - return fetch_from_buf(string, len, conn->inbuf); + IF_HAS_BUFFEREVENT(conn, { + /* XXX overflow -seb */ + return (int)bufferevent_read(conn->bufev, string, len); + }) ELSE_IF_NO_BUFFEREVENT { + return fetch_from_buf(string, len, conn->inbuf); + } +} + +/** As fetch_from_buf_line(), but read from a connection's input buffer. */ +int +connection_fetch_from_buf_line(connection_t *conn, char *data, + size_t *data_len) +{ + IF_HAS_BUFFEREVENT(conn, { + int r; + size_t eol_len=0; + struct evbuffer *input = bufferevent_get_input(conn->bufev); + struct evbuffer_ptr ptr = + evbuffer_search_eol(input, NULL, &eol_len, EVBUFFER_EOL_LF); + if (ptr.pos == -1) + return 0; /* No EOL found. */ + if ((size_t)ptr.pos+eol_len >= *data_len) { + return -1; /* Too long */ + } + *data_len = ptr.pos+eol_len; + r = evbuffer_remove(input, data, ptr.pos+eol_len); + tor_assert(r >= 0); + data[ptr.pos+eol_len] = '\0'; + return 1; + }) ELSE_IF_NO_BUFFEREVENT { + return fetch_from_buf_line(conn->inbuf, data, data_len); + } +} + +/** As fetch_from_buf_http, but fetches from a conncetion's input buffer_t or + * its bufferevent as appropriate. */ +int +connection_fetch_from_buf_http(connection_t *conn, + char **headers_out, size_t max_headerlen, + char **body_out, size_t *body_used, + size_t max_bodylen, int force_complete) +{ + IF_HAS_BUFFEREVENT(conn, { + struct evbuffer *input = bufferevent_get_input(conn->bufev); + return fetch_from_evbuffer_http(input, headers_out, max_headerlen, + body_out, body_used, max_bodylen, force_complete); + }) ELSE_IF_NO_BUFFEREVENT { + return fetch_from_buf_http(conn->inbuf, headers_out, max_headerlen, + body_out, body_used, max_bodylen, force_complete); + } } /** Return conn-\>outbuf_flushlen: how many bytes conn wants to flush @@ -2725,6 +3080,7 @@ connection_handle_write_impl(connection_t *conn, int force) /* If we just flushed the last bytes, check if this tunneled dir * request is done. */ + /* XXXX move this to flushed_some or finished_flushing -NM */ if (buf_datalen(conn->outbuf) == 0 && conn->dirreq_id) geoip_change_dirreq_state(conn->dirreq_id, DIRREQ_TUNNELED, DIRREQ_OR_CONN_BUFFER_FLUSHED); @@ -2780,7 +3136,7 @@ connection_handle_write_impl(connection_t *conn, int force) if (conn->type == CONN_TYPE_AP) { edge_connection_t *edge_conn = TO_EDGE_CONN(conn); - /*XXXX021 check for overflow.*/ + /*XXXX022 check for overflow.*/ edge_conn->n_written += (int)n_written; } @@ -2853,6 +3209,22 @@ _connection_write_to_buf_impl(const char *string, size_t len, if (conn->marked_for_close && !conn->hold_open_until_flushed) return; + IF_HAS_BUFFEREVENT(conn, { + if (zlib) { + int done = zlib < 0; + r = write_to_evbuffer_zlib(bufferevent_get_output(conn->bufev), + TO_DIR_CONN(conn)->zlib_state, + string, len, done); + } else { + r = bufferevent_write(conn->bufev, string, len); + } + if (r < 0) { + /* XXXX mark for close? */ + log_warn(LD_NET, "bufferevent_write failed! That shouldn't happen."); + } + return; + }); + old_datalen = buf_datalen(conn->outbuf); if (zlib) { dir_connection_t *dir_conn = TO_DIR_CONN(conn); @@ -3025,6 +3397,32 @@ connection_get_by_type_state_rendquery(int type, int state, return NULL; } +/** Return a directory connection (if any one exists) that is fetching + * the item described by <b>state</b>/<b>resource</b> */ +dir_connection_t * +connection_dir_get_by_purpose_and_resource(int purpose, + const char *resource) +{ + smartlist_t *conns = get_connection_array(); + + SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) { + dir_connection_t *dirconn; + if (conn->type != CONN_TYPE_DIR || conn->marked_for_close || + conn->purpose != purpose) + continue; + dirconn = TO_DIR_CONN(conn); + if (dirconn->requested_resource == NULL) { + if (resource == NULL) + return dirconn; + } else if (resource) { + if (0 == strcmp(resource, dirconn->requested_resource)) + return dirconn; + } + } SMARTLIST_FOREACH_END(conn); + + return NULL; +} + /** Return an open, non-marked connection of a given type and purpose, or NULL * if no such connection exists. */ connection_t * @@ -3264,6 +3662,9 @@ connection_finished_flushing(connection_t *conn) // log_fn(LOG_DEBUG,"entered. Socket %u.", conn->s); + IF_HAS_NO_BUFFEREVENT(conn) + connection_stop_writing(conn); + switch (conn->type) { case CONN_TYPE_OR: return connection_or_finished_flushing(TO_OR_CONN(conn)); @@ -3389,6 +3790,16 @@ assert_connection_ok(connection_t *conn, time_t now) tor_assert(conn); tor_assert(conn->type >= _CONN_TYPE_MIN); tor_assert(conn->type <= _CONN_TYPE_MAX); + +#ifdef USE_BUFFEREVENTS + if (conn->bufev) { + tor_assert(conn->read_event == NULL); + tor_assert(conn->write_event == NULL); + tor_assert(conn->inbuf == NULL); + tor_assert(conn->outbuf == NULL); + } +#endif + switch (conn->type) { case CONN_TYPE_OR: tor_assert(conn->magic == OR_CONNECTION_MAGIC); @@ -3427,10 +3838,10 @@ assert_connection_ok(connection_t *conn, time_t now) * marked_for_close. */ /* buffers */ - if (!connection_is_listener(conn)) { + if (conn->inbuf) assert_buf_ok(conn->inbuf); + if (conn->outbuf) assert_buf_ok(conn->outbuf); - } if (conn->type == CONN_TYPE_OR) { or_connection_t *or_conn = TO_OR_CONN(conn); diff --git a/src/or/connection.h b/src/or/connection.h index f38927e788..004ede5d04 100644 --- a/src/or/connection.h +++ b/src/or/connection.h @@ -12,6 +12,9 @@ #ifndef _TOR_CONNECTION_H #define _TOR_CONNECTION_H +/* XXXX For buf_datalen in inline function */ +#include "buffers.h" + const char *conn_type_to_string(int type); const char *conn_state_to_string(int type, int state); @@ -31,6 +34,15 @@ void _connection_mark_for_close(connection_t *conn,int line, const char *file); #define connection_mark_for_close(c) \ _connection_mark_for_close((c), __LINE__, _SHORT_FILE_) +#define connection_mark_and_flush(c) \ + do { \ + connection_t *tmp_conn_ = (c); \ + _connection_mark_for_close(tmp_conn_, __LINE__, _SHORT_FILE_); \ + tmp_conn_->hold_open_until_flushed = 1; \ + IF_HAS_BUFFEREVENT(tmp_conn_, \ + connection_start_writing(tmp_conn_)); \ + } while (0) + void connection_expire_held_open(void); int connection_connect(connection_t *conn, const char *address, @@ -51,6 +63,12 @@ void connection_bucket_refill(int seconds_elapsed, time_t now); int connection_handle_read(connection_t *conn); int connection_fetch_from_buf(char *string, size_t len, connection_t *conn); +int connection_fetch_from_buf_line(connection_t *conn, char *data, + size_t *data_len); +int connection_fetch_from_buf_http(connection_t *conn, + char **headers_out, size_t max_headerlen, + char **body_out, size_t *body_used, + size_t max_bodylen, int force_complete); int connection_wants_to_flush(connection_t *conn); int connection_outbuf_too_full(connection_t *conn); @@ -73,6 +91,29 @@ connection_write_to_buf_zlib(const char *string, size_t len, _connection_write_to_buf_impl(string, len, TO_CONN(conn), done ? -1 : 1); } +static size_t connection_get_inbuf_len(connection_t *conn); +static size_t connection_get_outbuf_len(connection_t *conn); + +static INLINE size_t +connection_get_inbuf_len(connection_t *conn) +{ + IF_HAS_BUFFEREVENT(conn, { + return evbuffer_get_length(bufferevent_get_input(conn->bufev)); + }) ELSE_IF_NO_BUFFEREVENT { + return conn->inbuf ? buf_datalen(conn->inbuf) : 0; + } +} + +static INLINE size_t +connection_get_outbuf_len(connection_t *conn) +{ + IF_HAS_BUFFEREVENT(conn, { + return evbuffer_get_length(bufferevent_get_output(conn->bufev)); + }) ELSE_IF_NO_BUFFEREVENT { + return conn->outbuf ? buf_datalen(conn->outbuf) : 0; + } +} + connection_t *connection_get_by_global_id(uint64_t id); connection_t *connection_get_by_type(int type); @@ -83,6 +124,8 @@ connection_t *connection_get_by_type_addr_port_purpose(int type, connection_t *connection_get_by_type_state(int type, int state); connection_t *connection_get_by_type_state_rendquery(int type, int state, const char *rendquery); +dir_connection_t *connection_dir_get_by_purpose_and_resource( + int state, const char *resource); #define connection_speaks_cells(conn) ((conn)->type == CONN_TYPE_OR) int connection_is_listener(connection_t *conn); @@ -96,5 +139,19 @@ int connection_or_nonopen_was_started_here(or_connection_t *conn); void connection_dump_buffer_mem_stats(int severity); void remove_file_if_very_old(const char *fname, time_t now); +#ifdef USE_BUFFEREVENTS +int connection_type_uses_bufferevent(connection_t *conn); +void connection_configure_bufferevent_callbacks(connection_t *conn); +void connection_handle_read_cb(struct bufferevent *bufev, void *arg); +void connection_handle_write_cb(struct bufferevent *bufev, void *arg); +void connection_handle_event_cb(struct bufferevent *bufev, short event, + void *arg); +void connection_get_rate_limit_totals(uint64_t *read_out, + uint64_t *written_out); +void connection_enable_rate_limiting(connection_t *conn); +#else +#define connection_type_uses_bufferevent(c) (0) +#endif + #endif diff --git a/src/or/connection_edge.c b/src/or/connection_edge.c index da0fc1856c..9627631ad9 100644 --- a/src/or/connection_edge.c +++ b/src/or/connection_edge.c @@ -92,6 +92,7 @@ _connection_mark_unattached_ap(edge_connection_t *conn, int endreason, _connection_mark_for_close(TO_CONN(conn), line, file); conn->_base.hold_open_until_flushed = 1; + IF_HAS_BUFFEREVENT(TO_CONN(conn), connection_start_writing(TO_CONN(conn))); conn->end_reason = endreason; } @@ -100,7 +101,7 @@ _connection_mark_unattached_ap(edge_connection_t *conn, int endreason, int connection_edge_reached_eof(edge_connection_t *conn) { - if (buf_datalen(conn->_base.inbuf) && + if (connection_get_inbuf_len(TO_CONN(conn)) && connection_state_is_open(TO_CONN(conn))) { /* it still has stuff to process. don't let it die yet. */ return 0; @@ -191,8 +192,7 @@ connection_edge_destroy(circid_t circ_id, edge_connection_t *conn) conn->edge_has_sent_end = 1; conn->end_reason = END_STREAM_REASON_DESTROY; conn->end_reason |= END_STREAM_REASON_FLAG_ALREADY_SENT_CLOSED; - connection_mark_for_close(TO_CONN(conn)); - conn->_base.hold_open_until_flushed = 1; + connection_mark_and_flush(TO_CONN(conn)); } } conn->cpath_layer = NULL; @@ -319,7 +319,6 @@ connection_edge_finished_flushing(edge_connection_t *conn) switch (conn->_base.state) { case AP_CONN_STATE_OPEN: case EXIT_CONN_STATE_OPEN: - connection_stop_writing(TO_CONN(conn)); connection_edge_consider_sending_sendme(conn); return 0; case AP_CONN_STATE_SOCKS_WAIT: @@ -328,7 +327,7 @@ connection_edge_finished_flushing(edge_connection_t *conn) case AP_CONN_STATE_CIRCUIT_WAIT: case AP_CONN_STATE_CONNECT_WAIT: case AP_CONN_STATE_CONTROLLER_WAIT: - connection_stop_writing(TO_CONN(conn)); + case AP_CONN_STATE_RESOLVE_WAIT: return 0; default: log_warn(LD_BUG, "Called in unexpected state %d.",conn->_base.state); @@ -358,8 +357,9 @@ connection_edge_finished_connecting(edge_connection_t *edge_conn) rep_hist_note_exit_stream_opened(conn->port); conn->state = EXIT_CONN_STATE_OPEN; - connection_watch_events(conn, READ_EVENT); /* stop writing, keep reading */ - if (connection_wants_to_flush(conn)) /* in case there are any queued relay + IF_HAS_NO_BUFFEREVENT(conn) + connection_watch_events(conn, READ_EVENT); /* stop writing, keep reading */ + if (connection_get_outbuf_len(conn)) /* in case there are any queued relay * cells */ connection_start_writing(conn); /* deliver a 'connected' relay cell back through the circuit. */ @@ -1895,8 +1895,14 @@ connection_ap_handshake_process_socks(edge_connection_t *conn) log_debug(LD_APP,"entered."); - sockshere = fetch_from_buf_socks(conn->_base.inbuf, socks, - options->TestSocks, options->SafeSocks); + IF_HAS_BUFFEREVENT(TO_CONN(conn), { + struct evbuffer *input = bufferevent_get_input(conn->_base.bufev); + sockshere = fetch_from_evbuffer_socks(input, socks, + options->TestSocks, options->SafeSocks); + }) ELSE_IF_NO_BUFFEREVENT { + sockshere = fetch_from_buf_socks(conn->_base.inbuf, socks, + options->TestSocks, options->SafeSocks); + }; if (sockshere == 0) { if (socks->replylen) { connection_write_to_buf(socks->reply, socks->replylen, TO_CONN(conn)); @@ -1997,7 +2003,7 @@ connection_ap_process_natd(edge_connection_t *conn) /* look for LF-terminated "[DEST ip_addr port]" * where ip_addr is a dotted-quad and port is in string form */ - err = fetch_from_buf_line(conn->_base.inbuf, tmp_buf, &tlen); + err = connection_fetch_from_buf_line(TO_CONN(conn), tmp_buf, &tlen); if (err == 0) return 0; if (err < 0) { @@ -2104,8 +2110,10 @@ connection_ap_handshake_send_begin(edge_connection_t *ap_conn) ap_conn->socks_request->port); payload_len = (int)strlen(payload)+1; - log_debug(LD_APP, - "Sending relay cell to begin stream %d.", ap_conn->stream_id); + log_info(LD_APP, + "Sending relay cell %d to begin stream %d.", + (int)ap_conn->use_begindir, + ap_conn->stream_id); begin_type = ap_conn->use_begindir ? RELAY_COMMAND_BEGIN_DIR : RELAY_COMMAND_BEGIN; @@ -2213,9 +2221,11 @@ connection_ap_handshake_send_resolve(edge_connection_t *ap_conn) * and call connection_ap_handshake_attach_circuit(conn) on it. * * Return the other end of the linked connection pair, or -1 if error. + * DOCDOC partner. */ edge_connection_t * -connection_ap_make_link(char *address, uint16_t port, +connection_ap_make_link(connection_t *partner, + char *address, uint16_t port, const char *digest, int use_begindir, int want_onehop) { edge_connection_t *conn; @@ -2250,6 +2260,8 @@ connection_ap_make_link(char *address, uint16_t port, tor_addr_make_unspec(&conn->_base.addr); conn->_base.port = 0; + connection_link_connections(partner, TO_CONN(conn)); + if (connection_add(TO_CONN(conn)) < 0) { /* no space, forget it */ connection_free(TO_CONN(conn)); return NULL; @@ -2765,12 +2777,13 @@ connection_exit_connect(edge_connection_t *edge_conn) } conn->state = EXIT_CONN_STATE_OPEN; - if (connection_wants_to_flush(conn)) { + if (connection_get_outbuf_len(conn)) { /* in case there are any queued data cells */ log_warn(LD_BUG,"newly connected conn had data waiting!"); // connection_start_writing(conn); } - connection_watch_events(conn, READ_EVENT); + IF_HAS_NO_BUFFEREVENT(conn) + connection_watch_events(conn, READ_EVENT); /* also, deliver a 'connected' cell back through the circuit. */ if (connection_edge_is_rendezvous_stream(edge_conn)) { @@ -2924,11 +2937,6 @@ connection_ap_can_use_exit(edge_connection_t *conn, routerinfo_t *exit, * addresses with this port. Since the user didn't ask for * this node, err on the side of caution. */ } else if (SOCKS_COMMAND_IS_RESOLVE(conn->socks_request->command)) { - /* Can't support reverse lookups without eventdns. */ - if (conn->socks_request->command == SOCKS_COMMAND_RESOLVE_PTR && - exit->has_old_dnsworkers) - return 0; - /* Don't send DNS requests to non-exit servers by default. */ if (!conn->chosen_exit_name && policy_is_reject_star(exit->exit_policy)) return 0; diff --git a/src/or/connection_edge.h b/src/or/connection_edge.h index 762af5172e..0f7bf07809 100644 --- a/src/or/connection_edge.h +++ b/src/or/connection_edge.h @@ -29,7 +29,8 @@ int connection_edge_finished_connecting(edge_connection_t *conn); int connection_ap_handshake_send_begin(edge_connection_t *ap_conn); int connection_ap_handshake_send_resolve(edge_connection_t *ap_conn); -edge_connection_t *connection_ap_make_link(char *address, uint16_t port, +edge_connection_t *connection_ap_make_link(connection_t *partner, + char *address, uint16_t port, const char *digest, int use_begindir, int want_onehop); void connection_ap_handshake_socks_reply(edge_connection_t *conn, char *reply, diff --git a/src/or/connection_or.c b/src/or/connection_or.c index 09f310a3df..069c3e133d 100644 --- a/src/or/connection_or.c +++ b/src/or/connection_or.c @@ -28,6 +28,10 @@ #include "router.h" #include "routerlist.h" +#ifdef USE_BUFFEREVENTS +#include <event2/bufferevent_ssl.h> +#endif + static int connection_tls_finish_handshake(or_connection_t *conn); static int connection_or_process_cells_from_inbuf(or_connection_t *conn); static int connection_or_send_versions(or_connection_t *conn); @@ -37,6 +41,12 @@ static int connection_or_check_valid_tls_handshake(or_connection_t *conn, int started_here, char *digest_rcvd_out); +#ifdef USE_BUFFEREVENTS +static void connection_or_handle_event_cb(struct bufferevent *bufev, + short event, void *arg); +#include <event2/buffer.h>/*XXXX REMOVE */ +#endif + /**************************************************************/ /** Map from identity digest of connected OR or desired OR to a connection_t @@ -248,7 +258,7 @@ connection_or_process_inbuf(or_connection_t *conn) int connection_or_flushed_some(or_connection_t *conn) { - size_t datalen = buf_datalen(conn->_base.outbuf); + size_t datalen = connection_get_outbuf_len(TO_CONN(conn)); /* If we're under the low water mark, add cells until we're just over the * high water mark. */ if (datalen < OR_CONN_LOWWATER) { @@ -281,7 +291,6 @@ connection_or_finished_flushing(or_connection_t *conn) case OR_CONN_STATE_PROXY_HANDSHAKING: case OR_CONN_STATE_OPEN: case OR_CONN_STATE_OR_HANDSHAKING: - connection_stop_writing(TO_CONN(conn)); break; default: log_err(LD_BUG,"Called in unexpected state %d.", conn->_base.state); @@ -379,6 +388,22 @@ connection_or_update_token_buckets_helper(or_connection_t *conn, int reset, conn->bandwidthrate = rate; conn->bandwidthburst = burst; +#ifdef USE_BUFFEREVENTS + { + const struct timeval *tick = tor_libevent_get_one_tick_timeout(); + struct ev_token_bucket_cfg *cfg, *old_cfg; + int rate_per_tick = rate / TOR_LIBEVENT_TICKS_PER_SECOND; + cfg = ev_token_bucket_cfg_new(rate_per_tick, burst, rate_per_tick, + burst, tick); + old_cfg = conn->bucket_cfg; + if (conn->_base.bufev) + bufferevent_set_rate_limit(conn->_base.bufev, cfg); + if (old_cfg) + ev_token_bucket_cfg_free(old_cfg); + conn->bucket_cfg = cfg; + (void) reset; /* No way to do this with libevent yet. */ + } +#else if (reset) { /* set up the token buckets to be full */ conn->read_bucket = conn->write_bucket = burst; return; @@ -389,6 +414,7 @@ connection_or_update_token_buckets_helper(or_connection_t *conn, int reset, conn->read_bucket = burst; if (conn->write_bucket > burst) conn->write_bucket = burst; +#endif } /** Either our set of relays or our per-conn rate limits have changed. @@ -859,6 +885,7 @@ int connection_tls_start_handshake(or_connection_t *conn, int receiving) { conn->_base.state = OR_CONN_STATE_TLS_HANDSHAKING; + tor_assert(!conn->tls); conn->tls = tor_tls_new(conn->_base.s, receiving); tor_tls_set_logged_address(conn->tls, // XXX client and relay? escaped_safe_str(conn->_base.address)); @@ -866,12 +893,34 @@ connection_tls_start_handshake(or_connection_t *conn, int receiving) log_warn(LD_BUG,"tor_tls_new failed. Closing."); return -1; } +#ifdef USE_BUFFEREVENTS + if (connection_type_uses_bufferevent(TO_CONN(conn))) { + struct bufferevent *b = + tor_tls_init_bufferevent(conn->tls, conn->_base.bufev, conn->_base.s, + receiving); + if (!b) { + log_warn(LD_BUG,"tor_tls_init_bufferevent failed. Closing."); + return -1; + } + conn->_base.bufev = b; + if (conn->bucket_cfg) + bufferevent_set_rate_limit(conn->_base.bufev, conn->bucket_cfg); + connection_enable_rate_limiting(TO_CONN(conn)); + bufferevent_setcb(b, connection_handle_read_cb, + connection_handle_write_cb, + connection_or_handle_event_cb, + TO_CONN(conn)); + } +#endif connection_start_reading(TO_CONN(conn)); log_debug(LD_HANDSHAKE,"starting TLS handshake on fd %d", conn->_base.s); note_crypto_pk_op(receiving ? TLS_HANDSHAKE_S : TLS_HANDSHAKE_C); - if (connection_tls_continue_handshake(conn) < 0) { - return -1; + IF_HAS_BUFFEREVENT(TO_CONN(conn), { + /* ???? */; + }) ELSE_IF_NO_BUFFEREVENT { + if (connection_tls_continue_handshake(conn) < 0) + return -1; } return 0; } @@ -956,6 +1005,55 @@ connection_tls_continue_handshake(or_connection_t *conn) return 0; } +#ifdef USE_BUFFEREVENTS +static void +connection_or_handle_event_cb(struct bufferevent *bufev, short event, + void *arg) +{ + struct or_connection_t *conn = TO_OR_CONN(arg); + + /* XXXX cut-and-paste code; should become a function. */ + if (event & BEV_EVENT_CONNECTED) { + if (conn->_base.state == OR_CONN_STATE_TLS_HANDSHAKING) { + if (tor_tls_finish_handshake(conn->tls) < 0) { + log_warn(LD_OR, "Problem finishing handshake"); + connection_mark_for_close(TO_CONN(conn)); + return; + } + } + + if (! tor_tls_used_v1_handshake(conn->tls)) { + if (!tor_tls_is_server(conn->tls)) { + if (conn->_base.state == OR_CONN_STATE_TLS_HANDSHAKING) { + conn->_base.state = OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING; + tor_tls_unblock_renegotiation(conn->tls); + if (bufferevent_ssl_renegotiate(conn->_base.bufev)<0) { + log_warn(LD_OR, "Start_renegotiating went badly."); + connection_mark_for_close(TO_CONN(conn)); + } + tor_tls_unblock_renegotiation(conn->tls); + return; /* ???? */ + } + } else { + /* improved handshake, but not a client. */ + tor_tls_set_renegotiate_callback(conn->tls, + connection_or_tls_renegotiated_cb, + conn); + conn->_base.state = OR_CONN_STATE_TLS_SERVER_RENEGOTIATING; + /* return 0; */ + return; /* ???? */ + } + } + connection_watch_events(TO_CONN(conn), READ_EVENT|WRITE_EVENT); + if (connection_tls_finish_handshake(conn) < 0) + connection_mark_for_close(TO_CONN(conn)); /* ???? */ + return; + } + + connection_handle_event_cb(bufev, event, arg); +} +#endif + /** Return 1 if we initiated this connection, or 0 if it started * out as an incoming connection. */ @@ -1216,8 +1314,12 @@ connection_or_set_state_open(or_connection_t *conn) or_handshake_state_free(conn->handshake_state); conn->handshake_state = NULL; + IF_HAS_BUFFEREVENT(TO_CONN(conn), { + connection_watch_events(TO_CONN(conn), READ_EVENT|WRITE_EVENT); + }) ELSE_IF_NO_BUFFEREVENT { + connection_start_reading(TO_CONN(conn)); + } - connection_start_reading(TO_CONN(conn)); circuit_n_conn_done(conn, 1); /* send the pending creates, if any. */ return 0; @@ -1261,12 +1363,18 @@ connection_or_write_var_cell_to_buf(const var_cell_t *cell, conn->timestamp_last_added_nonpadding = approx_time(); } -/** See whether there's a variable-length cell waiting on <b>conn</b>'s +/** See whether there's a variable-length cell waiting on <b>or_conn</b>'s * inbuf. Return values as for fetch_var_cell_from_buf(). */ static int -connection_fetch_var_cell_from_buf(or_connection_t *conn, var_cell_t **out) +connection_fetch_var_cell_from_buf(or_connection_t *or_conn, var_cell_t **out) { - return fetch_var_cell_from_buf(conn->_base.inbuf, out, conn->link_proto); + connection_t *conn = TO_CONN(or_conn); + IF_HAS_BUFFEREVENT(conn, { + struct evbuffer *input = bufferevent_get_input(conn->bufev); + return fetch_var_cell_from_evbuffer(input, out, or_conn->link_proto); + }) ELSE_IF_NO_BUFFEREVENT { + return fetch_var_cell_from_buf(conn->inbuf, out, or_conn->link_proto); + } } /** Process cells from <b>conn</b>'s inbuf. @@ -1284,7 +1392,7 @@ connection_or_process_cells_from_inbuf(or_connection_t *conn) while (1) { log_debug(LD_OR, "%d: starting, inbuf_datalen %d (%d pending in tls object).", - conn->_base.s,(int)buf_datalen(conn->_base.inbuf), + conn->_base.s,(int)connection_get_inbuf_len(TO_CONN(conn)), tor_tls_get_pending_bytes(conn->tls)); if (connection_fetch_var_cell_from_buf(conn, &var_cell)) { if (!var_cell) @@ -1295,8 +1403,8 @@ connection_or_process_cells_from_inbuf(or_connection_t *conn) } else { char buf[CELL_NETWORK_SIZE]; cell_t cell; - if (buf_datalen(conn->_base.inbuf) < CELL_NETWORK_SIZE) /* whole response - available? */ + if (connection_get_inbuf_len(TO_CONN(conn)) + < CELL_NETWORK_SIZE) /* whole response available? */ return 0; /* not yet */ circuit_build_times_network_is_live(&circ_times); diff --git a/src/or/control.c b/src/or/control.c index 4d505a98fb..37ebfd88dc 100644 --- a/src/or/control.c +++ b/src/or/control.c @@ -2719,8 +2719,6 @@ int connection_control_finished_flushing(control_connection_t *conn) { tor_assert(conn); - - connection_stop_writing(TO_CONN(conn)); return 0; } @@ -2755,6 +2753,17 @@ is_valid_initial_command(control_connection_t *conn, const char *cmd) * interfaces is broken. */ #define MAX_COMMAND_LINE_LENGTH (1024*1024) +static int +peek_connection_has_control0_command(connection_t *conn) +{ + IF_HAS_BUFFEREVENT(conn, { + struct evbuffer *input = bufferevent_get_input(conn->bufev); + return peek_evbuffer_has_control0_command(input); + }) ELSE_IF_NO_BUFFEREVENT { + return peek_buf_has_control0_command(conn->inbuf); + } +} + /** Called when data has arrived on a v1 control connection: Try to fetch * commands from conn->inbuf, and execute them. */ @@ -2777,7 +2786,7 @@ connection_control_process_inbuf(control_connection_t *conn) } if (conn->_base.state == CONTROL_CONN_STATE_NEEDAUTH && - peek_buf_has_control0_command(conn->_base.inbuf)) { + peek_connection_has_control0_command(TO_CONN(conn))) { /* Detect v0 commands and send a "no more v0" message. */ size_t body_len; char buf[128]; @@ -2801,7 +2810,7 @@ connection_control_process_inbuf(control_connection_t *conn) /* First, fetch a line. */ do { data_len = conn->incoming_cmd_len - conn->incoming_cmd_cur_len; - r = fetch_from_buf_line(conn->_base.inbuf, + r = connection_fetch_from_buf_line(TO_CONN(conn), conn->incoming_cmd+conn->incoming_cmd_cur_len, &data_len); if (r == 0) diff --git a/src/or/cpuworker.c b/src/or/cpuworker.c index 6f943d78b8..cfe9f3af9c 100644 --- a/src/or/cpuworker.c +++ b/src/or/cpuworker.c @@ -62,7 +62,6 @@ connection_cpu_finished_flushing(connection_t *conn) { tor_assert(conn); tor_assert(conn->type == CONN_TYPE_CPUWORKER); - connection_stop_writing(conn); return 0; } @@ -141,13 +140,13 @@ connection_cpu_process_inbuf(connection_t *conn) tor_assert(conn); tor_assert(conn->type == CONN_TYPE_CPUWORKER); - if (!buf_datalen(conn->inbuf)) + if (!connection_get_inbuf_len(conn)) return 0; if (conn->state == CPUWORKER_STATE_BUSY_ONION) { - if (buf_datalen(conn->inbuf) < LEN_ONION_RESPONSE) /* answer available? */ + if (connection_get_inbuf_len(conn) < LEN_ONION_RESPONSE) return 0; /* not yet */ - tor_assert(buf_datalen(conn->inbuf) == LEN_ONION_RESPONSE); + tor_assert(connection_get_inbuf_len(conn) == LEN_ONION_RESPONSE); connection_fetch_from_buf(&success,1,conn); connection_fetch_from_buf(buf,LEN_ONION_RESPONSE-1,conn); @@ -367,7 +366,7 @@ spawn_cpuworker(void) static void spawn_enough_cpuworkers(void) { - int num_cpuworkers_needed = get_options()->NumCpus; + int num_cpuworkers_needed = get_num_cpus(get_options()); if (num_cpuworkers_needed < MIN_CPUWORKERS) num_cpuworkers_needed = MIN_CPUWORKERS; diff --git a/src/or/directory.c b/src/or/directory.c index b109cb53a7..657b210b5f 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -15,6 +15,7 @@ #include "dirvote.h" #include "geoip.h" #include "main.h" +#include "microdesc.h" #include "networkstatus.h" #include "policies.h" #include "rendclient.h" @@ -78,6 +79,8 @@ static void dir_routerdesc_download_failed(smartlist_t *failed, int router_purpose, int was_extrainfo, int was_descriptor_digests); +static void dir_microdesc_download_failed(smartlist_t *failed, + int status_code); static void note_client_request(int purpose, int compressed, size_t bytes); static int client_likes_consensus(networkstatus_t *v, const char *want_url); @@ -137,7 +140,8 @@ purpose_needs_anonymity(uint8_t dir_purpose, uint8_t router_purpose) dir_purpose == DIR_PURPOSE_FETCH_CONSENSUS || dir_purpose == DIR_PURPOSE_FETCH_CERTIFICATE || dir_purpose == DIR_PURPOSE_FETCH_SERVERDESC || - dir_purpose == DIR_PURPOSE_FETCH_EXTRAINFO) + dir_purpose == DIR_PURPOSE_FETCH_EXTRAINFO || + dir_purpose == DIR_PURPOSE_FETCH_MICRODESC) return 0; return 1; } @@ -201,6 +205,8 @@ dir_conn_purpose_to_string(int purpose) return "hidden-service v2 descriptor fetch"; case DIR_PURPOSE_UPLOAD_RENDDESC_V2: return "hidden-service v2 descriptor upload"; + case DIR_PURPOSE_FETCH_MICRODESC: + return "microdescriptor fetch"; } log_warn(LD_BUG, "Called with unknown purpose %d", purpose); @@ -355,15 +361,33 @@ directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose, case DIR_PURPOSE_FETCH_CERTIFICATE: type = V3_AUTHORITY; break; + case DIR_PURPOSE_FETCH_MICRODESC: + type = V3_AUTHORITY; + break; default: log_warn(LD_BUG, "Unexpected purpose %d", (int)dir_purpose); return; } if (dir_purpose == DIR_PURPOSE_FETCH_CONSENSUS) { - networkstatus_t *v = networkstatus_get_latest_consensus(); - if (v) - if_modified_since = v->valid_after + 180; + int flav = FLAV_NS; + networkstatus_t *v; + if (resource) + flav = networkstatus_parse_flavor_name(resource); + + if (flav != -1) { + /* IF we have a parsed consensus of this type, we can do an + * if-modified-time based on it. */ + v = networkstatus_get_latest_consensus_by_flavor(flav); + if (v) + if_modified_since = v->valid_after + 180; + } else { + /* Otherwise it might be a consensus we don't parse, but which we + * do cache. Look at the cached copy, perhaps. */ + cached_dir_t *cd = dirserv_get_consensus(resource ? resource : "ns"); + if (cd) + if_modified_since = cd->published + 180; + } } if (!options->FetchServerDescriptors && type != HIDSERV_AUTHORITY) @@ -395,7 +419,8 @@ directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose, if (prefer_authority || type == BRIDGE_AUTHORITY) { /* only ask authdirservers, and don't ask myself */ rs = router_pick_trusteddirserver(type, pds_flags); - if (rs == NULL && (pds_flags & PDS_NO_EXISTING_SERVERDESC_FETCH)) { + if (rs == NULL && (pds_flags & (PDS_NO_EXISTING_SERVERDESC_FETCH| + PDS_NO_EXISTING_MICRODESC_FETCH))) { /* We don't want to fetch from any authorities that we're currently * fetching server descriptors from, and we got no match. Did we * get no match because all the authorities have connections @@ -403,7 +428,8 @@ directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose, * return,) or because all the authorities are down or on fire or * unreachable or something (in which case we should go on with * our fallback code)? */ - pds_flags &= ~PDS_NO_EXISTING_SERVERDESC_FETCH; + pds_flags &= ~(PDS_NO_EXISTING_SERVERDESC_FETCH| + PDS_NO_EXISTING_MICRODESC_FETCH); rs = router_pick_trusteddirserver(type, pds_flags); if (rs) { log_debug(LD_DIR, "Deferring serverdesc fetch: all authorities " @@ -592,15 +618,19 @@ connection_dir_request_failed(dir_connection_t *conn) connection_dir_download_networkstatus_failed(conn, -1); } else if (conn->_base.purpose == DIR_PURPOSE_FETCH_SERVERDESC || conn->_base.purpose == DIR_PURPOSE_FETCH_EXTRAINFO) { - log_info(LD_DIR, "Giving up on directory server at '%s'; retrying", + log_info(LD_DIR, "Giving up on serverdesc/extrainfo fetch from " + "directory server at '%s'; retrying", conn->_base.address); if (conn->router_purpose == ROUTER_PURPOSE_BRIDGE) connection_dir_bridge_routerdesc_failed(conn); connection_dir_download_routerdesc_failed(conn); } else if (conn->_base.purpose == DIR_PURPOSE_FETCH_CONSENSUS) { - networkstatus_consensus_download_failed(0); + const char *flavname = + conn->requested_resource ? conn->requested_resource : "ns"; + networkstatus_consensus_download_failed(0, flavname); } else if (conn->_base.purpose == DIR_PURPOSE_FETCH_CERTIFICATE) { - log_info(LD_DIR, "Giving up on directory server at '%s'; retrying", + log_info(LD_DIR, "Giving up on certificate fetch from directory server " + "at '%s'; retrying", conn->_base.address); connection_dir_download_cert_failed(conn, 0); } else if (conn->_base.purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES) { @@ -609,6 +639,10 @@ connection_dir_request_failed(dir_connection_t *conn) } else if (conn->_base.purpose == DIR_PURPOSE_FETCH_STATUS_VOTE) { log_info(LD_DIR, "Giving up downloading votes from '%s'", conn->_base.address); + } else if (conn->_base.purpose == DIR_PURPOSE_FETCH_MICRODESC) { + log_info(LD_DIR, "Giving up on downloading microdescriptors from " + " directory server at '%s'; will retry", conn->_base.address); + connection_dir_download_routerdesc_failed(conn); } } @@ -679,7 +713,8 @@ connection_dir_download_routerdesc_failed(dir_connection_t *conn) /* No need to relaunch descriptor downloads here: we already do it * every 10 or 60 seconds (FOO_DESCRIPTOR_RETRY_INTERVAL) in main.c. */ tor_assert(conn->_base.purpose == DIR_PURPOSE_FETCH_SERVERDESC || - conn->_base.purpose == DIR_PURPOSE_FETCH_EXTRAINFO); + conn->_base.purpose == DIR_PURPOSE_FETCH_EXTRAINFO || + conn->_base.purpose == DIR_PURPOSE_FETCH_MICRODESC); (void) conn; } @@ -892,14 +927,14 @@ directory_initiate_command_rend(const char *address, const tor_addr_t *_addr, * hook up both sides */ linked_conn = - connection_ap_make_link(conn->_base.address, conn->_base.port, + connection_ap_make_link(TO_CONN(conn), + conn->_base.address, conn->_base.port, digest, use_begindir, conn->dirconn_direct); if (!linked_conn) { log_warn(LD_NET,"Making tunnel to dirserver failed."); connection_mark_for_close(TO_CONN(conn)); return; } - connection_link_connections(TO_CONN(conn), TO_CONN(linked_conn)); if (connection_add(TO_CONN(conn)) < 0) { log_warn(LD_NET,"Unable to add connection for link to dirserver."); @@ -912,8 +947,12 @@ directory_initiate_command_rend(const char *address, const tor_addr_t *_addr, payload, payload_len, supports_conditional_consensus, if_modified_since); + connection_watch_events(TO_CONN(conn), READ_EVENT|WRITE_EVENT); - connection_start_reading(TO_CONN(linked_conn)); + IF_HAS_BUFFEREVENT(TO_CONN(linked_conn), { + connection_watch_events(TO_CONN(linked_conn), READ_EVENT|WRITE_EVENT); + }) ELSE_IF_NO_BUFFEREVENT + connection_start_reading(TO_CONN(linked_conn)); } } @@ -951,12 +990,16 @@ _compare_strs(const void **a, const void **b) * This url depends on whether or not the server we go to * is sufficiently new to support conditional consensus downloading, * i.e. GET .../consensus/<b>fpr</b>+<b>fpr</b>+<b>fpr</b> + * + * If 'resource' is provided, it is the name of a consensus flavor to request. */ static char * -directory_get_consensus_url(int supports_conditional_consensus) +directory_get_consensus_url(int supports_conditional_consensus, + const char *resource) { - char *url; - size_t len; + char *url = NULL; + const char *hyphen = resource ? "-" : ""; + const char *flavor = resource ? resource : ""; if (supports_conditional_consensus) { char *authority_id_list; @@ -978,16 +1021,15 @@ directory_get_consensus_url(int supports_conditional_consensus) authority_id_list = smartlist_join_strings(authority_digests, "+", 0, NULL); - len = strlen(authority_id_list)+64; - url = tor_malloc(len); - tor_snprintf(url, len, "/tor/status-vote/current/consensus/%s.z", - authority_id_list); + tor_asprintf(&url, "/tor/status-vote/current/consensus%s%s/%s.z", + hyphen, flavor, authority_id_list); SMARTLIST_FOREACH(authority_digests, char *, cp, tor_free(cp)); smartlist_free(authority_digests); tor_free(authority_id_list); } else { - url = tor_strdup("/tor/status-vote/current/consensus.z"); + tor_asprintf(&url, "/tor/status-vote/current/consensus%s%s.z", + hyphen, flavor); } return url; } @@ -1068,10 +1110,11 @@ directory_send_command(dir_connection_t *conn, tor_snprintf(url, len, "/tor/status/%s", resource); break; case DIR_PURPOSE_FETCH_CONSENSUS: - tor_assert(!resource); + /* resource is optional. If present, it's a flavor name */ tor_assert(!payload); httpcommand = "GET"; - url = directory_get_consensus_url(supports_conditional_consensus); + url = directory_get_consensus_url(supports_conditional_consensus, + resource); log_info(LD_DIR, "Downloading consensus from %s using %s", hoststring, url); break; @@ -1111,6 +1154,11 @@ directory_send_command(dir_connection_t *conn, url = tor_malloc(len); tor_snprintf(url, len, "/tor/extra/%s", resource); break; + case DIR_PURPOSE_FETCH_MICRODESC: + tor_assert(resource); + httpcommand = "GET"; + tor_asprintf(&url, "/tor/micro/%s.z", resource); + break; case DIR_PURPOSE_UPLOAD_DIR: tor_assert(!resource); tor_assert(payload); @@ -1386,6 +1434,9 @@ body_is_plausible(const char *body, size_t len, int purpose) return 1; /* empty bodies don't need decompression */ if (len < 32) return 0; + if (purpose == DIR_PURPOSE_FETCH_MICRODESC) { + return (!strcmpstart(body,"onion-key")); + } if (purpose != DIR_PURPOSE_FETCH_RENDDESC) { if (!strcmpstart(body,"router") || !strcmpstart(body,"signed-directory") || @@ -1462,11 +1513,12 @@ connection_dir_client_reached_eof(dir_connection_t *conn) int plausible; int skewed=0; int allow_partial = (conn->_base.purpose == DIR_PURPOSE_FETCH_SERVERDESC || - conn->_base.purpose == DIR_PURPOSE_FETCH_EXTRAINFO); + conn->_base.purpose == DIR_PURPOSE_FETCH_EXTRAINFO || + conn->_base.purpose == DIR_PURPOSE_FETCH_MICRODESC); int was_compressed=0; time_t now = time(NULL); - switch (fetch_from_buf_http(conn->_base.inbuf, + switch (connection_fetch_from_buf_http(TO_CONN(conn), &headers, MAX_HEADERS_SIZE, &body, &body_len, MAX_DIR_DL_SIZE, allow_partial)) { @@ -1686,6 +1738,8 @@ connection_dir_client_reached_eof(dir_connection_t *conn) if (conn->_base.purpose == DIR_PURPOSE_FETCH_CONSENSUS) { int r; + const char *flavname = + conn->requested_resource ? conn->requested_resource : "ns"; if (status_code != 200) { int severity = (status_code == 304) ? LOG_INFO : LOG_WARN; log(severity, LD_DIR, @@ -1694,22 +1748,24 @@ connection_dir_client_reached_eof(dir_connection_t *conn) status_code, escaped(reason), conn->_base.address, conn->_base.port); tor_free(body); tor_free(headers); tor_free(reason); - networkstatus_consensus_download_failed(status_code); + networkstatus_consensus_download_failed(status_code, flavname); return -1; } log_info(LD_DIR,"Received consensus directory (size %d) from server " "'%s:%d'", (int)body_len, conn->_base.address, conn->_base.port); - if ((r=networkstatus_set_current_consensus(body, "ns", 0))<0) { + if ((r=networkstatus_set_current_consensus(body, flavname, 0))<0) { log_fn(r<-1?LOG_WARN:LOG_INFO, LD_DIR, - "Unable to load consensus directory downloaded from " + "Unable to load %s consensus directory downloaded from " "server '%s:%d'. I'll try again soon.", - conn->_base.address, conn->_base.port); + flavname, conn->_base.address, conn->_base.port); tor_free(body); tor_free(headers); tor_free(reason); - networkstatus_consensus_download_failed(0); + networkstatus_consensus_download_failed(0, flavname); return -1; } /* launches router downloads as needed */ routers_update_all_from_networkstatus(now, 3); + update_microdescs_from_networkstatus(now); + update_microdesc_downloads(now); directory_info_has_arrived(now, 0); log_info(LD_DIR, "Successfully loaded consensus."); } @@ -1859,6 +1915,42 @@ connection_dir_client_reached_eof(dir_connection_t *conn) if (directory_conn_is_self_reachability_test(conn)) router_dirport_found_reachable(); } + if (conn->_base.purpose == DIR_PURPOSE_FETCH_MICRODESC) { + smartlist_t *which = NULL; + log_info(LD_DIR,"Received answer to microdescriptor request (status %d, " + "size %d) from server '%s:%d'", + status_code, (int)body_len, conn->_base.address, + conn->_base.port); + tor_assert(conn->requested_resource && + !strcmpstart(conn->requested_resource, "d/")); + which = smartlist_create(); + dir_split_resource_into_fingerprints(conn->requested_resource+2, + which, NULL, + DSR_DIGEST256|DSR_BASE64); + if (status_code != 200) { + log_info(LD_DIR, "Received status code %d (%s) from server " + "'%s:%d' while fetching \"/tor/micro/%s\". I'll try again " + "soon.", + status_code, escaped(reason), conn->_base.address, + (int)conn->_base.port, conn->requested_resource); + dir_microdesc_download_failed(which, status_code); + SMARTLIST_FOREACH(which, char *, cp, tor_free(cp)); + smartlist_free(which); + tor_free(body); tor_free(headers); tor_free(reason); + return 0; + } else { + smartlist_t *mds; + mds = microdescs_add_to_cache(get_microdesc_cache(), + body, body+body_len, SAVED_NOWHERE, 0, + now, which); + if (smartlist_len(which)) { + /* Mark remaining ones as failed. */ + dir_microdesc_download_failed(which, status_code); + } + SMARTLIST_FOREACH(which, char *, cp, tor_free(cp)); + smartlist_free(which); + } + } if (conn->_base.purpose == DIR_PURPOSE_UPLOAD_DIR) { switch (status_code) { @@ -2134,7 +2226,7 @@ connection_dir_process_inbuf(dir_connection_t *conn) return 0; } - if (buf_datalen(conn->_base.inbuf) > MAX_DIRECTORY_OBJECT_SIZE) { + if (connection_get_inbuf_len(TO_CONN(conn)) > MAX_DIRECTORY_OBJECT_SIZE) { log_warn(LD_HTTP, "Too much data received from directory connection: " "denial of service attempt, or you need to upgrade?"); connection_mark_for_close(TO_CONN(conn)); @@ -3299,7 +3391,7 @@ directory_handle_command(dir_connection_t *conn) tor_assert(conn); tor_assert(conn->_base.type == CONN_TYPE_DIR); - switch (fetch_from_buf_http(conn->_base.inbuf, + switch (connection_fetch_from_buf_http(TO_CONN(conn), &headers, MAX_HEADERS_SIZE, &body, &body_len, MAX_DIR_UL_SIZE, 0)) { case -1: /* overflow */ @@ -3352,10 +3444,10 @@ connection_dir_finished_flushing(dir_connection_t *conn) DIRREQ_DIRECT, DIRREQ_FLUSHING_DIR_CONN_FINISHED); switch (conn->_base.state) { + case DIR_CONN_STATE_CONNECTING: case DIR_CONN_STATE_CLIENT_SENDING: log_debug(LD_DIR,"client finished sending command."); conn->_base.state = DIR_CONN_STATE_CLIENT_READING; - connection_stop_writing(TO_CONN(conn)); return 0; case DIR_CONN_STATE_SERVER_WRITING: log_debug(LD_DIRSERV,"Finished writing server response. Closing."); @@ -3585,6 +3677,36 @@ dir_routerdesc_download_failed(smartlist_t *failed, int status_code, * every 10 or 60 seconds (FOO_DESCRIPTOR_RETRY_INTERVAL) in main.c. */ } +/* DOCDOC NM */ +static void +dir_microdesc_download_failed(smartlist_t *failed, + int status_code) +{ + networkstatus_t *consensus + = networkstatus_get_latest_consensus_by_flavor(FLAV_MICRODESC); + routerstatus_t *rs; + download_status_t *dls; + time_t now = time(NULL); + int server = directory_fetches_from_authorities(get_options()); + + if (! consensus) + return; + SMARTLIST_FOREACH_BEGIN(failed, const char *, d) { + rs = router_get_consensus_status_by_descriptor_digest(consensus, d); + if (!rs) + continue; + dls = &rs->dl_status; + if (dls->n_download_failures >= MAX_MICRODESC_DOWNLOAD_FAILURES) + continue; + { + char buf[BASE64_DIGEST256_LEN+1]; + digest256_to_base64(buf, d); + download_status_increment_failure(dls, status_code, buf, + server, now); + } + } SMARTLIST_FOREACH_END(d); +} + /** Helper. Compare two fp_pair_t objects, and return -1, 0, or 1 as * appropriate. */ static int diff --git a/src/or/dirserv.c b/src/or/dirserv.c index 8ae03424a2..75e3e86109 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -1263,7 +1263,8 @@ static cached_dir_t cached_runningrouters; * cached_dir_t. */ static digestmap_t *cached_v2_networkstatus = NULL; -/** Map from flavor name to the v3 consensuses that we're currently serving. */ +/** Map from flavor name to the cached_dir_t for the v3 consensuses that we're + * currently serving. */ static strmap_t *cached_consensuses = NULL; /** Possibly replace the contents of <b>d</b> with the value of @@ -3395,7 +3396,7 @@ connection_dirserv_add_servers_to_outbuf(dir_connection_t *conn) time_t publish_cutoff = time(NULL)-ROUTER_MAX_AGE_TO_PUBLISH; while (smartlist_len(conn->fingerprint_stack) && - buf_datalen(conn->_base.outbuf) < DIRSERV_BUFFER_MIN) { + connection_get_outbuf_len(TO_CONN(conn)) < DIRSERV_BUFFER_MIN) { const char *body; char *fp = smartlist_pop_last(conn->fingerprint_stack); signed_descriptor_t *sd = NULL; @@ -3495,7 +3496,7 @@ connection_dirserv_add_dir_bytes_to_outbuf(dir_connection_t *conn) ssize_t bytes; int64_t remaining; - bytes = DIRSERV_BUFFER_MIN - buf_datalen(conn->_base.outbuf); + bytes = DIRSERV_BUFFER_MIN - connection_get_outbuf_len(TO_CONN(conn)); tor_assert(bytes > 0); tor_assert(conn->cached_dir); if (bytes < 8192) @@ -3534,7 +3535,7 @@ static int connection_dirserv_add_networkstatus_bytes_to_outbuf(dir_connection_t *conn) { - while (buf_datalen(conn->_base.outbuf) < DIRSERV_BUFFER_MIN) { + while (connection_get_outbuf_len(TO_CONN(conn)) < DIRSERV_BUFFER_MIN) { if (conn->cached_dir) { int uncompressing = (conn->zlib_state != NULL); int r = connection_dirserv_add_dir_bytes_to_outbuf(conn); @@ -3580,7 +3581,7 @@ connection_dirserv_flushed_some(dir_connection_t *conn) { tor_assert(conn->_base.state == DIR_CONN_STATE_SERVER_WRITING); - if (buf_datalen(conn->_base.outbuf) >= DIRSERV_BUFFER_MIN) + if (connection_get_outbuf_len(TO_CONN(conn)) >= DIRSERV_BUFFER_MIN) return 0; switch (conn->dir_spool_src) { diff --git a/src/or/main.c b/src/or/main.c index 582a1c287b..ddd5da3643 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -56,6 +56,10 @@ #include <event.h> #endif +#ifdef USE_BUFFEREVENTS +#include <event2/bufferevent.h> +#endif + void evdns_shutdown(int); /********* PROTOTYPES **********/ @@ -72,6 +76,7 @@ static int connection_should_read_from_linked_conn(connection_t *conn); /********* START VARIABLES **********/ +#ifndef USE_BUFFEREVENTS int global_read_bucket; /**< Max number of bytes I can read this second. */ int global_write_bucket; /**< Max number of bytes I can write this second. */ @@ -79,13 +84,17 @@ int global_write_bucket; /**< Max number of bytes I can write this second. */ int global_relayed_read_bucket; /** Max number of relayed (bandwidth class 1) bytes I can write this second. */ int global_relayed_write_bucket; - /** What was the read bucket before the last second_elapsed_callback() call? * (used to determine how many bytes we've read). */ static int stats_prev_global_read_bucket; /** What was the write bucket before the last second_elapsed_callback() call? * (used to determine how many bytes we've written). */ static int stats_prev_global_write_bucket; +#else +static uint64_t stats_prev_n_read = 0; +static uint64_t stats_prev_n_written = 0; +#endif + /* XXX we might want to keep stats about global_relayed_*_bucket too. Or not.*/ /** How many bytes have we read since we started the process? */ static uint64_t stats_n_bytes_read = 0; @@ -151,12 +160,38 @@ int can_complete_circuit=0; * ****************************************************************************/ +#ifdef USE_BUFFEREVENTS +static void +free_old_inbuf(connection_t *conn) +{ + if (! conn->inbuf) + return; + + tor_assert(conn->outbuf); + tor_assert(buf_datalen(conn->inbuf) == 0); + tor_assert(buf_datalen(conn->outbuf) == 0); + buf_free(conn->inbuf); + buf_free(conn->outbuf); + conn->inbuf = conn->outbuf = NULL; + + if (conn->read_event) { + event_del(conn->read_event); + tor_event_free(conn->read_event); + } + if (conn->write_event) { + event_del(conn->read_event); + tor_event_free(conn->write_event); + } + conn->read_event = conn->write_event = NULL; +} +#endif + /** Add <b>conn</b> to the array of connections that we can poll on. The * connection's socket must be set; the connection starts out * non-reading and non-writing. */ int -connection_add(connection_t *conn) +connection_add_impl(connection_t *conn, int is_connecting) { tor_assert(conn); tor_assert(conn->s >= 0 || @@ -168,11 +203,59 @@ connection_add(connection_t *conn) conn->conn_array_index = smartlist_len(connection_array); smartlist_add(connection_array, conn); - if (conn->s >= 0 || conn->linked) { +#ifdef USE_BUFFEREVENTS + if (connection_type_uses_bufferevent(conn)) { + if (conn->s >= 0 && !conn->linked) { + conn->bufev = bufferevent_socket_new( + tor_libevent_get_base(), + conn->s, + BEV_OPT_DEFER_CALLBACKS); + /* XXXX CHECK FOR NULL RETURN! */ + if (is_connecting) { + /* Put the bufferevent into a "connecting" state so that we'll get + * a "connected" event callback on successful write. */ + bufferevent_socket_connect(conn->bufev, NULL, 0); + } + connection_configure_bufferevent_callbacks(conn); + } else if (conn->linked && conn->linked_conn && + connection_type_uses_bufferevent(conn->linked_conn)) { + tor_assert(conn->s < 0); + if (!conn->bufev) { + struct bufferevent *pair[2] = { NULL, NULL }; + /* XXXX CHECK FOR ERROR RETURN! */ + bufferevent_pair_new(tor_libevent_get_base(), + BEV_OPT_DEFER_CALLBACKS, + pair); + tor_assert(pair[0]); + conn->bufev = pair[0]; + conn->linked_conn->bufev = pair[1]; + } /* else the other side already was added, and got a bufferevent_pair */ + connection_configure_bufferevent_callbacks(conn); + } + + if (conn->bufev && conn->inbuf) { + /* XXX Instead we should assert that there is no inbuf, once we + * have linked connections using bufferevents. */ + free_old_inbuf(conn); + } + + if (conn->linked_conn && conn->linked_conn->bufev && + conn->linked_conn->inbuf) { + /* XXX Instead we should assert that there is no inbuf, once we + * have linked connections using bufferevents. */ + free_old_inbuf(conn->linked_conn); + } + } +#else + (void) is_connecting; +#endif + + if (!HAS_BUFFEREVENT(conn) && (conn->s >= 0 || conn->linked)) { conn->read_event = tor_event_new(tor_libevent_get_base(), conn->s, EV_READ|EV_PERSIST, conn_read_callback, conn); conn->write_event = tor_event_new(tor_libevent_get_base(), conn->s, EV_WRITE|EV_PERSIST, conn_write_callback, conn); + /* XXXX CHECK FOR NULL RETURN! */ } log_debug(LD_NET,"new conn type %s, socket %d, address %s, n_conns %d.", @@ -196,6 +279,12 @@ connection_unregister_events(connection_t *conn) log_warn(LD_BUG, "Error removing write event for %d", conn->s); tor_free(conn->write_event); } +#ifdef USE_BUFFEREVENTS + if (conn->bufev) { + bufferevent_free(conn->bufev); + conn->bufev = NULL; + } +#endif if (conn->dns_server_port) { dnsserv_close_listener(conn); } @@ -310,6 +399,17 @@ get_connection_array(void) void connection_watch_events(connection_t *conn, watchable_events_t events) { + IF_HAS_BUFFEREVENT(conn, { + short ev = ((short)events) & (EV_READ|EV_WRITE); + short old_ev = bufferevent_get_enabled(conn->bufev); + if ((ev & ~old_ev) != 0) { + bufferevent_enable(conn->bufev, ev); + } + if ((old_ev & ~ev) != 0) { + bufferevent_disable(conn->bufev, old_ev & ~ev); + } + return; + }); if (events & READ_EVENT) connection_start_reading(conn); else @@ -327,6 +427,9 @@ connection_is_reading(connection_t *conn) { tor_assert(conn); + IF_HAS_BUFFEREVENT(conn, + return (bufferevent_get_enabled(conn->bufev) & EV_READ) != 0; + ); return conn->reading_from_linked_conn || (conn->read_event && event_pending(conn->read_event, EV_READ, NULL)); } @@ -336,6 +439,12 @@ void connection_stop_reading(connection_t *conn) { tor_assert(conn); + + IF_HAS_BUFFEREVENT(conn, { + bufferevent_disable(conn->bufev, EV_READ); + return; + }); + tor_assert(conn->read_event); if (conn->linked) { @@ -355,6 +464,12 @@ void connection_start_reading(connection_t *conn) { tor_assert(conn); + + IF_HAS_BUFFEREVENT(conn, { + bufferevent_enable(conn->bufev, EV_READ); + return; + }); + tor_assert(conn->read_event); if (conn->linked) { @@ -376,6 +491,10 @@ connection_is_writing(connection_t *conn) { tor_assert(conn); + IF_HAS_BUFFEREVENT(conn, + return (bufferevent_get_enabled(conn->bufev) & EV_WRITE) != 0; + ); + return conn->writing_to_linked_conn || (conn->write_event && event_pending(conn->write_event, EV_WRITE, NULL)); } @@ -385,6 +504,12 @@ void connection_stop_writing(connection_t *conn) { tor_assert(conn); + + IF_HAS_BUFFEREVENT(conn, { + bufferevent_disable(conn->bufev, EV_WRITE); + return; + }); + tor_assert(conn->write_event); if (conn->linked) { @@ -405,6 +530,12 @@ void connection_start_writing(connection_t *conn) { tor_assert(conn); + + IF_HAS_BUFFEREVENT(conn, { + bufferevent_enable(conn->bufev, EV_WRITE); + return; + }); + tor_assert(conn->write_event); if (conn->linked) { @@ -590,7 +721,20 @@ conn_close_if_marked(int i) assert_connection_ok(conn, now); /* assert_all_pending_dns_resolves_ok(); */ +#ifdef USE_BUFFEREVENTS + if (conn->bufev && conn->hold_open_until_flushed) { + if (conn->linked) { + /* We need to do this explicitly so that the linked connection + * notices that there was an EOF. */ + bufferevent_flush(conn->bufev, EV_WRITE, BEV_FINISHED); + } + if (evbuffer_get_length(bufferevent_get_output(conn->bufev))) + return 0; + } +#endif + log_debug(LD_NET,"Cleaning up connection (fd %d).",conn->s); + IF_HAS_BUFFEREVENT(conn, goto unlink); if ((conn->s >= 0 || conn->linked_conn) && connection_wants_to_flush(conn)) { /* s == -1 means it's an incomplete edge connection, or that the socket * has already been closed as unflushable. */ @@ -613,8 +757,8 @@ conn_close_if_marked(int i) } log_debug(LD_GENERAL, "Flushed last %d bytes from a linked conn; " "%d left; flushlen %d; wants-to-flush==%d", retval, - (int)buf_datalen(conn->outbuf), - (int)conn->outbuf_flushlen, + (int)connection_get_outbuf_len(conn), + (int)conn->outbuf_flushlen, connection_wants_to_flush(conn)); } else if (connection_speaks_cells(conn)) { if (conn->state == OR_CONN_STATE_OPEN) { @@ -651,13 +795,17 @@ conn_close_if_marked(int i) "something is wrong with your network connection, or " "something is wrong with theirs. " "(fd %d, type %s, state %d, marked at %s:%d).", - (int)buf_datalen(conn->outbuf), + (int)connection_get_outbuf_len(conn), escaped_safe_str_client(conn->address), conn->s, conn_type_to_string(conn->type), conn->state, conn->marked_for_close_file, conn->marked_for_close); } } + +#ifdef USE_BUFFEREVENTS + unlink: +#endif connection_unlink(conn); /* unlink, remove, free */ return 1; } @@ -703,10 +851,13 @@ directory_info_has_arrived(time_t now, int from_cache) "I learned some more directory information, but not enough to " "build a circuit: %s", get_dir_info_status_string()); update_router_descriptor_downloads(now); + update_microdesc_downloads(now); return; } else { - if (directory_fetches_from_authorities(options)) + if (directory_fetches_from_authorities(options)) { update_router_descriptor_downloads(now); + update_microdesc_downloads(now); + } /* if we have enough dir info, then update our guard status with * whatever we just learned. */ @@ -744,7 +895,8 @@ run_connection_housekeeping(int i, time_t now) int past_keepalive = now >= conn->timestamp_lastwritten + options->KeepalivePeriod; - if (conn->outbuf && !buf_datalen(conn->outbuf) && conn->type == CONN_TYPE_OR) + if (conn->outbuf && !connection_get_outbuf_len(conn) && + conn->type == CONN_TYPE_OR) TO_OR_CONN(conn)->timestamp_lastempty = now; if (conn->marked_for_close) { @@ -764,7 +916,7 @@ run_connection_housekeeping(int i, time_t now) /* This check is temporary; it's to let us know whether we should consider * parsing partial serverdesc responses. */ if (conn->purpose == DIR_PURPOSE_FETCH_SERVERDESC && - buf_datalen(conn->inbuf)>=1024) { + connection_get_inbuf_len(conn) >= 1024) { log_info(LD_DIR,"Trying to extract information from wedged server desc " "download."); connection_dir_reached_eof(TO_DIR_CONN(conn)); @@ -781,7 +933,11 @@ run_connection_housekeeping(int i, time_t now) the connection or send a keepalive, depending. */ or_conn = TO_OR_CONN(conn); +#ifdef USE_BUFFEREVENTS + tor_assert(conn->bufev); +#else tor_assert(conn->outbuf); +#endif if (or_conn->is_bad_for_new_circs && !or_conn->n_circuits) { /* It's bad for new circuits, and has no unmarked circuits on it: @@ -803,13 +959,12 @@ run_connection_housekeeping(int i, time_t now) connection_mark_for_close(conn); } } else if (we_are_hibernating() && !or_conn->n_circuits && - !buf_datalen(conn->outbuf)) { + !connection_get_outbuf_len(conn)) { /* We're hibernating, there's no circuits, and nothing to flush.*/ log_info(LD_OR,"Expiring non-used OR connection to fd %d (%s:%d) " "[Hibernating or exiting].", conn->s,conn->address, conn->port); - connection_mark_for_close(conn); - conn->hold_open_until_flushed = 1; + connection_mark_and_flush(conn); } else if (!or_conn->n_circuits && now >= or_conn->timestamp_last_added_nonpadding + IDLE_OR_CONN_TIMEOUT) { @@ -817,7 +972,6 @@ run_connection_housekeeping(int i, time_t now) "[idle %d].", conn->s,conn->address, conn->port, (int)(now - or_conn->timestamp_last_added_nonpadding)); connection_mark_for_close(conn); - conn->hold_open_until_flushed = 1; } else if ( now >= or_conn->timestamp_lastempty + options->KeepalivePeriod*10 && now >= conn->timestamp_lastwritten + options->KeepalivePeriod*10) { @@ -825,10 +979,10 @@ run_connection_housekeeping(int i, time_t now) "Expiring stuck OR connection to fd %d (%s:%d). (%d bytes to " "flush; %d seconds since last write)", conn->s, conn->address, conn->port, - (int)buf_datalen(conn->outbuf), + (int)connection_get_outbuf_len(conn), (int)(now-conn->timestamp_lastwritten)); connection_mark_for_close(conn); - } else if (past_keepalive && !buf_datalen(conn->outbuf)) { + } else if (past_keepalive && !connection_get_outbuf_len(conn)) { /* send a padding cell */ log_fn(LOG_DEBUG,LD_OR,"Sending keepalive to (%s:%d)", conn->address, conn->port); @@ -872,6 +1026,7 @@ run_scheduled_events(time_t now) static time_t time_to_check_for_expired_networkstatus = 0; static time_t time_to_write_stats_files = 0; static time_t time_to_write_bridge_stats = 0; + static time_t time_to_check_port_forwarding = 0; static int should_init_bridge_stats = 1; static time_t time_to_retry_dns_init = 0; or_options_t *options = get_options(); @@ -911,6 +1066,7 @@ run_scheduled_events(time_t now) if (time_to_try_getting_descriptors < now) { update_router_descriptor_downloads(now); update_extrainfo_downloads(now); + update_microdesc_downloads(now); if (options->UseBridges) fetch_bridge_descriptors(options, now); if (router_have_minimum_dir_info()) @@ -1230,6 +1386,17 @@ run_scheduled_events(time_t now) #define BRIDGE_STATUSFILE_INTERVAL (30*60) time_to_write_bridge_status_file = now+BRIDGE_STATUSFILE_INTERVAL; } + + if (time_to_check_port_forwarding < now && + options->PortForwarding && + server_mode(options)) { +#define PORT_FORWARDING_CHECK_INTERVAL 5 + tor_check_port_forwarding(options->PortForwardingHelper, + options->DirPort, + options->ORPort, + now); + time_to_check_port_forwarding = now+PORT_FORWARDING_CHECK_INTERVAL; + } } /** Timer: used to invoke second_elapsed_callback() once per second. */ @@ -1249,6 +1416,9 @@ second_elapsed_callback(periodic_timer_t *timer, void *arg) size_t bytes_written; size_t bytes_read; int seconds_elapsed; +#ifdef USE_BUFFEREVENTS + uint64_t cur_read,cur_written; +#endif or_options_t *options = get_options(); (void)timer; (void)arg; @@ -1260,9 +1430,15 @@ second_elapsed_callback(periodic_timer_t *timer, void *arg) update_approx_time(now); /* the second has rolled over. check more stuff. */ + seconds_elapsed = current_second ? (int)(now - current_second) : 0; +#ifdef USE_BUFFEREVENTS + connection_get_rate_limit_totals(&cur_read, &cur_written); + bytes_written = (size_t)(cur_written - stats_prev_n_written); + bytes_read = (size_t)(cur_read - stats_prev_n_read); +#else bytes_written = stats_prev_global_write_bucket - global_write_bucket; bytes_read = stats_prev_global_read_bucket - global_read_bucket; - seconds_elapsed = current_second ? (int)(now - current_second) : 0; +#endif stats_n_bytes_read += bytes_read; stats_n_bytes_written += bytes_written; if (accounting_is_enabled(options) && seconds_elapsed >= 0) @@ -1272,8 +1448,13 @@ second_elapsed_callback(periodic_timer_t *timer, void *arg) if (seconds_elapsed > 0) connection_bucket_refill(seconds_elapsed, now); +#ifdef USE_BUFFEREVENTS + stats_prev_n_written = cur_written; + stats_prev_n_read = cur_read; +#else stats_prev_global_read_bucket = global_read_bucket; stats_prev_global_write_bucket = global_write_bucket; +#endif if (server_mode(options) && !we_are_hibernating() && @@ -1474,8 +1655,10 @@ do_main_loop(void) /* Set up our buckets */ connection_bucket_init(); +#ifndef USE_BUFFEREVENTS stats_prev_global_read_bucket = global_read_bucket; stats_prev_global_write_bucket = global_write_bucket; +#endif /* initialize the bootstrap status events to know we're starting up */ control_event_bootstrap(BOOTSTRAP_STATUS_STARTING, 0); @@ -1719,13 +1902,13 @@ dumpstats(int severity) log(severity,LD_GENERAL, "Conn %d: %d bytes waiting on inbuf (len %d, last read %d secs ago)", i, - (int)buf_datalen(conn->inbuf), + (int)connection_get_inbuf_len(conn), (int)buf_allocation(conn->inbuf), (int)(now - conn->timestamp_lastread)); log(severity,LD_GENERAL, "Conn %d: %d bytes waiting on outbuf " "(len %d, last written %d secs ago)",i, - (int)buf_datalen(conn->outbuf), + (int)connection_get_outbuf_len(conn), (int)buf_allocation(conn->outbuf), (int)(now - conn->timestamp_lastwritten)); if (conn->type == CONN_TYPE_OR) { diff --git a/src/or/main.h b/src/or/main.h index ef38dc9351..10b8aad0fb 100644 --- a/src/or/main.h +++ b/src/or/main.h @@ -14,7 +14,9 @@ extern int can_complete_circuit; -int connection_add(connection_t *conn); +int connection_add_impl(connection_t *conn, int is_connecting); +#define connection_add(conn) connection_add_impl((conn), 0) +#define connection_add_connecting(conn) connection_add_impl((conn), 1) int connection_remove(connection_t *conn); void connection_unregister_events(connection_t *conn); int connection_in_array(connection_t *conn); @@ -24,6 +26,7 @@ int connection_is_on_closeable_list(connection_t *conn); smartlist_t *get_connection_array(void); typedef enum watchable_events { + /* Yes, it is intentional that these match Libevent's EV_READ and EV_WRITE */ READ_EVENT=0x02, WRITE_EVENT=0x04 } watchable_events_t; diff --git a/src/or/microdesc.c b/src/or/microdesc.c index e8f3e7c59f..2752d15c8c 100644 --- a/src/or/microdesc.c +++ b/src/or/microdesc.c @@ -3,8 +3,12 @@ #include "or.h" #include "config.h" +#include "directory.h" #include "microdesc.h" #include "routerparse.h" +#include "networkstatus.h" +#include "routerlist.h" +#include "dirserv.h" /** A data structure to hold a bunch of cached microdescriptors. There are * two active files in the cache: a "cache file" that we mmap, and a "journal @@ -23,6 +27,8 @@ struct microdesc_cache_t { tor_mmap_t *cache_content; /** Number of bytes used in the journal file. */ size_t journal_len; + /** Number of bytes in descriptors removed as too old. */ + size_t bytes_dropped; /** Total bytes of microdescriptor bodies we have added to this cache */ uint64_t total_len_seen; @@ -119,15 +125,19 @@ get_microdesc_cache(void) * ending at <b>eos</b>, and store them in <b>cache</b>. If <b>no-save</b>, * mark them as non-writable to disk. If <b>where</b> is SAVED_IN_CACHE, * leave their bodies as pointers to the mmap'd cache. If where is - * <b>SAVED_NOWHERE</b>, do not allow annotations. Return a list of the added - * microdescriptors. */ + * <b>SAVED_NOWHERE</b>, do not allow annotations. If listed_at is positive, + * set the last_listed field of every microdesc to listed_at. If + * requested_digests is non-null, then it contains a list of digests we mean + * to allow, so we should reject any non-requested microdesc with a different + * digest, and alter the list to contain only the digests of those microdescs + * we didn't find. + * Return a list of the added microdescriptors. */ smartlist_t * microdescs_add_to_cache(microdesc_cache_t *cache, const char *s, const char *eos, saved_location_t where, - int no_save) + int no_save, time_t listed_at, + smartlist_t *requested_digests256) { - /*XXXX need an argument that sets last_listed as appropriate. */ - smartlist_t *descriptors, *added; const int allow_annotations = (where != SAVED_NOWHERE); const int copy_body = (where != SAVED_IN_CACHE); @@ -135,6 +145,33 @@ microdescs_add_to_cache(microdesc_cache_t *cache, descriptors = microdescs_parse_from_string(s, eos, allow_annotations, copy_body); + if (listed_at > 0) { + SMARTLIST_FOREACH(descriptors, microdesc_t *, md, + md->last_listed = listed_at); + } + if (requested_digests256) { + digestmap_t *requested; /* XXXX actuqlly we should just use a + digest256map */ + requested = digestmap_new(); + SMARTLIST_FOREACH(requested_digests256, const char *, cp, + digestmap_set(requested, cp, (void*)1)); + SMARTLIST_FOREACH_BEGIN(descriptors, microdesc_t *, md) { + if (digestmap_get(requested, md->digest)) { + digestmap_set(requested, md->digest, (void*)2); + } else { + log_fn(LOG_PROTOCOL_WARN, LD_DIR, "Received non-requested microcdesc"); + microdesc_free(md); + SMARTLIST_DEL_CURRENT(descriptors, md); + } + } SMARTLIST_FOREACH_END(md); + SMARTLIST_FOREACH_BEGIN(requested_digests256, char *, cp) { + if (digestmap_get(requested, cp) == (void*)2) { + tor_free(cp); + SMARTLIST_DEL_CURRENT(requested_digests256, cp); + } + } SMARTLIST_FOREACH_END(cp); + digestmap_free(requested, NULL); + } added = microdescs_add_list_to_cache(cache, descriptors, where, no_save); smartlist_free(descriptors); @@ -207,10 +244,9 @@ microdescs_add_list_to_cache(microdesc_cache_t *cache, { size_t old_content_len = cache->cache_content ? cache->cache_content->size : 0; - if (cache->journal_len > 16384 + old_content_len && - cache->journal_len > old_content_len * 2) { + if ((cache->journal_len > 16384 + old_content_len && + cache->journal_len > old_content_len / 2)) microdesc_cache_rebuild(cache); - } } return added; @@ -251,7 +287,7 @@ microdesc_cache_reload(microdesc_cache_t *cache) mm = cache->cache_content = tor_mmap_file(cache->cache_fname); if (mm) { added = microdescs_add_to_cache(cache, mm->data, mm->data+mm->size, - SAVED_IN_CACHE, 0); + SAVED_IN_CACHE, 0, -1, NULL); if (added) { total += smartlist_len(added); smartlist_free(added); @@ -263,7 +299,7 @@ microdesc_cache_reload(microdesc_cache_t *cache) if (journal_content) { added = microdescs_add_to_cache(cache, journal_content, journal_content+st.st_size, - SAVED_IN_JOURNAL, 0); + SAVED_IN_JOURNAL, 0, -1, NULL); if (added) { total += smartlist_len(added); smartlist_free(added); @@ -275,6 +311,47 @@ microdesc_cache_reload(microdesc_cache_t *cache) return 0; } +/** DOCDOC */ +#define TOLERATE_MICRODESC_AGE (7*24*60*60) + +/** DOCDOC */ +void +microdesc_cache_clean(microdesc_cache_t *cache) +{ + networkstatus_t *consensus; + time_t cutoff; + microdesc_t **mdp, *victim; + int dropped=0, kept=0; + size_t bytes_dropped = 0; + time_t now = time(NULL); + + /* If we don't know a consensus, never believe last_listed values */ + consensus = networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC); + if (consensus == NULL) + return; + + cutoff = now - TOLERATE_MICRODESC_AGE; + + for (mdp = HT_START(microdesc_map, &cache->map); mdp != NULL; ) { + if ((*mdp)->last_listed < cutoff) { + ++dropped; + victim = *mdp; + mdp = HT_NEXT_RMV(microdesc_map, &cache->map, mdp); + bytes_dropped += victim->bodylen; + microdesc_free(victim); + } else { + ++kept; + mdp = HT_NEXT(microdesc_map, &cache->map, mdp); + } + } + + if (dropped) { + log_notice(LD_DIR, "Removed %d/%d microdescriptors as old.", + dropped,dropped+kept); + cache->bytes_dropped += bytes_dropped; + } +} + /** Regenerate the main cache file for <b>cache</b>, clear the journal file, * and update every microdesc_t in the cache with pointers to its new * location. */ @@ -290,6 +367,9 @@ microdesc_cache_rebuild(microdesc_cache_t *cache) int orig_size, new_size; log_info(LD_DIR, "Rebuilding the microdescriptor cache..."); + + microdesc_cache_clean(cache); + orig_size = (int)(cache->cache_content ? cache->cache_content->size : 0); orig_size += (int)cache->journal_len; @@ -344,6 +424,7 @@ microdesc_cache_rebuild(microdesc_cache_t *cache) write_str_to_file(cache->journal_fname, "", 1); cache->journal_len = 0; + cache->bytes_dropped = 0; new_size = (int)cache->cache_content->size; log_info(LD_DIR, "Done rebuilding microdesc cache. " @@ -412,3 +493,90 @@ microdesc_average_size(microdesc_cache_t *cache) return (size_t)(cache->total_len_seen / cache->n_seen); } +/** Return a smartlist of all the sha256 digest of the microdescriptors that + * are listed in <b>ns</b> but not present in <b>cache</b>. Returns pointers + * to internals of <b>ns</b>; you should not free the members of the resulting + * smartlist. Omit all microdescriptors whose digest appear in <b>skip</b>. */ +smartlist_t * +microdesc_list_missing_digest256(networkstatus_t *ns, microdesc_cache_t *cache, + int downloadable_only, digestmap_t *skip) +{ + smartlist_t *result = smartlist_create(); + time_t now = time(NULL); + tor_assert(ns->flavor == FLAV_MICRODESC); + SMARTLIST_FOREACH_BEGIN(ns->routerstatus_list, routerstatus_t *, rs) { + if (microdesc_cache_lookup_by_digest256(cache, rs->descriptor_digest)) + continue; + if (downloadable_only && + !download_status_is_ready(&rs->dl_status, now, + MAX_MICRODESC_DOWNLOAD_FAILURES)) + continue; + if (skip && digestmap_get(skip, rs->descriptor_digest)) + continue; + /* XXXX Also skip if we're a noncache and wouldn't use this router. + * XXXX NM Microdesc + */ + smartlist_add(result, rs->descriptor_digest); + } SMARTLIST_FOREACH_END(rs); + return result; +} + +/** DOCDOC */ +void +update_microdesc_downloads(time_t now) +{ + or_options_t *options = get_options(); + networkstatus_t *consensus; + smartlist_t *missing; + digestmap_t *pending; + + if (should_delay_dir_fetches(options)) + return; + if (directory_too_idle_to_fetch_descriptors(options, now)) + return; + + consensus = networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC); + if (!consensus) + return; + + if (!directory_caches_dir_info(options)) { + /* Right now, only caches fetch microdescriptors. + * XXXX NM Microdescs */ + return; + } + + pending = digestmap_new(); + list_pending_microdesc_downloads(pending); + + missing = microdesc_list_missing_digest256(consensus, + get_microdesc_cache(), + 1, + pending); + digestmap_free(pending, NULL); + + launch_descriptor_downloads(DIR_PURPOSE_FETCH_MICRODESC, + missing, NULL, now); + + smartlist_free(missing); +} + +/** DOCDOC */ +void +update_microdescs_from_networkstatus(time_t now) +{ + microdesc_cache_t *cache = get_microdesc_cache(); + microdesc_t *md; + networkstatus_t *ns = + networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC); + + if (! ns) + return; + + tor_assert(ns->flavor == FLAV_MICRODESC); + + SMARTLIST_FOREACH_BEGIN(ns->routerstatus_list, routerstatus_t *, rs) { + md = microdesc_cache_lookup_by_digest256(cache, rs->descriptor_digest); + if (md && ns->valid_after > md->last_listed) + md->last_listed = ns->valid_after; + } SMARTLIST_FOREACH_END(rs); +} diff --git a/src/or/microdesc.h b/src/or/microdesc.h index 2d1a60ad0a..1dfe3ae826 100644 --- a/src/or/microdesc.h +++ b/src/or/microdesc.h @@ -16,11 +16,13 @@ microdesc_cache_t *get_microdesc_cache(void); smartlist_t *microdescs_add_to_cache(microdesc_cache_t *cache, const char *s, const char *eos, saved_location_t where, - int no_save); + int no_save, time_t listed_at, + smartlist_t *requested_digests256); smartlist_t *microdescs_add_list_to_cache(microdesc_cache_t *cache, smartlist_t *descriptors, saved_location_t where, int no_save); +void microdesc_cache_clean(microdesc_cache_t *cache); int microdesc_cache_rebuild(microdesc_cache_t *cache); int microdesc_cache_reload(microdesc_cache_t *cache); void microdesc_cache_clear(microdesc_cache_t *cache); @@ -30,8 +32,16 @@ microdesc_t *microdesc_cache_lookup_by_digest256(microdesc_cache_t *cache, size_t microdesc_average_size(microdesc_cache_t *cache); +smartlist_t *microdesc_list_missing_digest256(networkstatus_t *ns, + microdesc_cache_t *cache, + int downloadable_only, + digestmap_t *skip); + void microdesc_free(microdesc_t *md); void microdesc_free_all(void); +void update_microdesc_downloads(time_t now); +void update_microdescs_from_networkstatus(time_t now); + #endif diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c index 1d8a20be11..cba02f3709 100644 --- a/src/or/networkstatus.c +++ b/src/or/networkstatus.c @@ -20,6 +20,7 @@ #include "dirserv.h" #include "dirvote.h" #include "main.h" +#include "microdesc.h" #include "networkstatus.h" #include "relay.h" #include "router.h" @@ -44,8 +45,19 @@ static strmap_t *named_server_map = NULL; * as unnamed for some server in the consensus. */ static strmap_t *unnamed_server_map = NULL; -/** Most recently received and validated v3 consensus network status. */ -static networkstatus_t *current_consensus = NULL; +/** Most recently received and validated v3 consensus network status, + * of whichever type we are using for our own circuits. This will be the same + * as one of current_ns_consensus or current_md_consensus. + */ +#define current_consensus current_ns_consensus + +/** Most recently received and validated v3 "ns"-flavored consensus network + * status. */ +static networkstatus_t *current_ns_consensus = NULL; + +/** Most recently received and validated v3 "microdec"-flavored consensus + * network status. */ +static networkstatus_t *current_md_consensus = NULL; /** A v3 consensus networkstatus that we've received, but which we don't * have enough certificates to be happy about. */ @@ -271,6 +283,7 @@ router_reload_consensus_networkstatus(void) update_certificate_downloads(time(NULL)); routers_update_all_from_networkstatus(time(NULL), 3); + update_microdescs_from_networkstatus(time(NULL)); return 0; } @@ -967,21 +980,25 @@ networkstatus_get_v2_list(void) } /** Return the consensus view of the status of the router whose current - * <i>descriptor</i> digest is <b>digest</b>, or NULL if no such router is - * known. */ + * <i>descriptor</i> digest in <b>consensus</b> is <b>digest</b>, or NULL if + * no such router is known. */ routerstatus_t * -router_get_consensus_status_by_descriptor_digest(const char *digest) +router_get_consensus_status_by_descriptor_digest(networkstatus_t *consensus, + const char *digest) { - if (!current_consensus) return NULL; - if (!current_consensus->desc_digest_map) { - digestmap_t * m = current_consensus->desc_digest_map = digestmap_new(); - SMARTLIST_FOREACH(current_consensus->routerstatus_list, + if (!consensus) + consensus = current_consensus; + if (!consensus) + return NULL; + if (!consensus->desc_digest_map) { + digestmap_t *m = consensus->desc_digest_map = digestmap_new(); + SMARTLIST_FOREACH(consensus->routerstatus_list, routerstatus_t *, rs, { digestmap_set(m, rs->descriptor_digest, rs); }); } - return digestmap_get(current_consensus->desc_digest_map, digest); + return digestmap_get(consensus->desc_digest_map, digest); } /** Given the digest of a router descriptor, return its current download @@ -990,7 +1007,10 @@ download_status_t * router_get_dl_status_by_descriptor_digest(const char *d) { routerstatus_t *rs; - if ((rs = router_get_consensus_status_by_descriptor_digest(d))) + if (!current_ns_consensus) + return NULL; + if ((rs = router_get_consensus_status_by_descriptor_digest( + current_ns_consensus, d))) return &rs->dl_status; if (v2_download_status_map) return digestmap_get(v2_download_status_map, d); @@ -1199,6 +1219,25 @@ update_v2_networkstatus_cache_downloads(time_t now) } } +/** DOCDOC */ +static int +we_want_to_fetch_flavor(or_options_t *options, int flavor) +{ + if (flavor < 0 || flavor > N_CONSENSUS_FLAVORS) { + /* This flavor is crazy; we don't want it */ + /*XXXX handle unrecognized flavors later */ + return 0; + } + if (authdir_mode_v3(options) || directory_caches_dir_info(options)) { + /* We want to serve all flavors to others, regardless if we would use + * it ourselves. */ + return 1; + } + /* Otherwise, we want the flavor only if we want to use it to build + * circuits. */ + return (flavor == USABLE_CONSENSUS_FLAVOR); +} + /** How many times will we try to fetch a consensus before we give up? */ #define CONSENSUS_NETWORKSTATUS_MAX_DL_TRIES 8 /** How long will we hang onto a possibly live consensus for which we're @@ -1211,48 +1250,65 @@ static void update_consensus_networkstatus_downloads(time_t now) { int i; + or_options_t *options = get_options(); + if (!networkstatus_get_live_consensus(now)) time_to_download_next_consensus = now; /* No live consensus? Get one now!*/ if (time_to_download_next_consensus > now) return; /* Wait until the current consensus is older. */ - /* XXXXNM Microdescs: may need to download more types. */ - if (!download_status_is_ready(&consensus_dl_status[FLAV_NS], now, - CONSENSUS_NETWORKSTATUS_MAX_DL_TRIES)) - return; /* We failed downloading a consensus too recently. */ - if (connection_get_by_type_purpose(CONN_TYPE_DIR, - DIR_PURPOSE_FETCH_CONSENSUS)) - return; /* There's an in-progress download.*/ for (i=0; i < N_CONSENSUS_FLAVORS; ++i) { - consensus_waiting_for_certs_t *waiting = &consensus_waiting_for_certs[i]; + /* XXXX need some way to download unknown flavors if we are caching. */ + const char *resource; + consensus_waiting_for_certs_t *waiting; + + if (! we_want_to_fetch_flavor(options, i)) + continue; + + resource = i==FLAV_NS ? NULL : networkstatus_get_flavor_name(i); + + if (!download_status_is_ready(&consensus_dl_status[i], now, + CONSENSUS_NETWORKSTATUS_MAX_DL_TRIES)) + continue; /* We failed downloading a consensus too recently. */ + if (connection_dir_get_by_purpose_and_resource( + DIR_PURPOSE_FETCH_CONSENSUS, resource)) + continue; /* There's an in-progress download.*/ + + waiting = &consensus_waiting_for_certs[i]; if (waiting->consensus) { /* XXXX make sure this doesn't delay sane downloads. */ - if (waiting->set_at + DELAY_WHILE_FETCHING_CERTS > now) - return; /* We're still getting certs for this one. */ - else { + if (waiting->set_at + DELAY_WHILE_FETCHING_CERTS > now) { + continue; /* We're still getting certs for this one. */ + } else { if (!waiting->dl_failed) { - download_status_failed(&consensus_dl_status[FLAV_NS], 0); + download_status_failed(&consensus_dl_status[i], 0); waiting->dl_failed=1; } } } - } - log_info(LD_DIR, "Launching networkstatus consensus download."); - directory_get_from_dirserver(DIR_PURPOSE_FETCH_CONSENSUS, - ROUTER_PURPOSE_GENERAL, NULL, - PDS_RETRY_IF_NO_SERVERS); + log_info(LD_DIR, "Launching %s networkstatus consensus download.", + networkstatus_get_flavor_name(i)); + + directory_get_from_dirserver(DIR_PURPOSE_FETCH_CONSENSUS, + ROUTER_PURPOSE_GENERAL, resource, + PDS_RETRY_IF_NO_SERVERS); + } } /** Called when an attempt to download a consensus fails: note that the * failure occurred, and possibly retry. */ void -networkstatus_consensus_download_failed(int status_code) +networkstatus_consensus_download_failed(int status_code, const char *flavname) { - /* XXXXNM Microdescs: may need to handle more types. */ - download_status_failed(&consensus_dl_status[FLAV_NS], status_code); - /* Retry immediately, if appropriate. */ - update_consensus_networkstatus_downloads(time(NULL)); + int flav = networkstatus_parse_flavor_name(flavname); + if (flav >= 0) { + tor_assert(flav < N_CONSENSUS_FLAVORS); + /* XXXX handle unrecognized flavors */ + download_status_failed(&consensus_dl_status[flav], status_code); + /* Retry immediately, if appropriate. */ + update_consensus_networkstatus_downloads(time(NULL)); + } } /** How long do we (as a cache) wait after a consensus becomes non-fresh @@ -1373,7 +1429,10 @@ update_certificate_downloads(time_t now) now); } - authority_certs_fetch_missing(current_consensus, now); + if (current_ns_consensus) + authority_certs_fetch_missing(current_ns_consensus, now); + if (current_ns_consensus) + authority_certs_fetch_missing(current_md_consensus, now); } /** Return 1 if we have a consensus but we don't have enough certificates @@ -1405,6 +1464,18 @@ networkstatus_get_latest_consensus(void) return current_consensus; } +/** DOCDOC */ +networkstatus_t * +networkstatus_get_latest_consensus_by_flavor(consensus_flavor_t f) +{ + if (f == FLAV_NS) + return current_ns_consensus; + else if (f == FLAV_MICRODESC) + return current_md_consensus; + else + tor_assert(0); +} + /** Return the most recent consensus that we have downloaded, or NULL if it is * no longer live. */ networkstatus_t * @@ -1424,13 +1495,15 @@ networkstatus_get_live_consensus(time_t now) /** As networkstatus_get_live_consensus(), but is way more tolerant of expired * consensuses. */ networkstatus_t * -networkstatus_get_reasonably_live_consensus(time_t now) +networkstatus_get_reasonably_live_consensus(time_t now, int flavor) { #define REASONABLY_LIVE_TIME (24*60*60) - if (current_consensus && - current_consensus->valid_after <= now && - now <= current_consensus->valid_until+REASONABLY_LIVE_TIME) - return current_consensus; + networkstatus_t *consensus = + networkstatus_get_latest_consensus_by_flavor(flavor); + if (consensus && + consensus->valid_after <= now && + now <= consensus->valid_until+REASONABLY_LIVE_TIME) + return consensus; else return NULL; } @@ -1569,6 +1642,7 @@ networkstatus_set_current_consensus(const char *consensus, const digests_t *current_digests = NULL; consensus_waiting_for_certs_t *waiting = NULL; time_t current_valid_after = 0; + int free_consensus = 1; if (flav < 0) { /* XXXX we don't handle unrecognized flavors yet. */ @@ -1614,9 +1688,16 @@ networkstatus_set_current_consensus(const char *consensus, if (!strcmp(flavor, "ns")) { consensus_fname = get_datadir_fname("cached-consensus"); unverified_fname = get_datadir_fname("unverified-consensus"); - if (current_consensus) { - current_digests = ¤t_consensus->digests; - current_valid_after = current_consensus->valid_after; + if (current_ns_consensus) { + current_digests = ¤t_ns_consensus->digests; + current_valid_after = current_ns_consensus->valid_after; + } + } else if (!strcmp(flavor, "microdesc")) { + consensus_fname = get_datadir_fname("cached-microdesc-consensus"); + unverified_fname = get_datadir_fname("unverified-microdesc-consensus"); + if (current_md_consensus) { + current_digests = ¤t_md_consensus->digests; + current_valid_after = current_md_consensus->valid_after; } } else { cached_dir_t *cur; @@ -1702,11 +1783,21 @@ networkstatus_set_current_consensus(const char *consensus, if (flav == USABLE_CONSENSUS_FLAVOR) { notify_control_networkstatus_changed(current_consensus, c); - - if (current_consensus) { - networkstatus_copy_old_consensus_info(c, current_consensus); - networkstatus_vote_free(current_consensus); + } + if (flav == FLAV_NS) { + if (current_ns_consensus) { + networkstatus_copy_old_consensus_info(c, current_ns_consensus); + networkstatus_vote_free(current_ns_consensus); + } + current_ns_consensus = c; + free_consensus = 0; /* avoid free */ + } else if (flav == FLAV_MICRODESC) { + if (current_md_consensus) { + networkstatus_copy_old_consensus_info(c, current_md_consensus); + networkstatus_vote_free(current_md_consensus); } + current_md_consensus = c; + free_consensus = 0; /* avoid free */ } waiting = &consensus_waiting_for_certs[flav]; @@ -1739,11 +1830,9 @@ networkstatus_set_current_consensus(const char *consensus, } if (flav == USABLE_CONSENSUS_FLAVOR) { - current_consensus = c; - c = NULL; /* Prevent free. */ - /* XXXXNM Microdescs: needs a non-ns variant. */ update_consensus_networkstatus_fetch_time(now); + dirvote_recalculate_timing(options, now); routerstatus_list_update_named_server_map(); cell_ewma_set_scale_factor(options, current_consensus); @@ -1758,11 +1847,11 @@ networkstatus_set_current_consensus(const char *consensus, write_str_to_file(consensus_fname, consensus, 0); } - if (ftime_definitely_before(now, current_consensus->valid_after)) { + if (ftime_definitely_before(now, c->valid_after)) { char tbuf[ISO_TIME_LEN+1]; char dbuf[64]; - long delta = now - current_consensus->valid_after; - format_iso_time(tbuf, current_consensus->valid_after); + long delta = now - c->valid_after; + format_iso_time(tbuf, c->valid_after); format_time_interval(dbuf, sizeof(dbuf), delta); log_warn(LD_GENERAL, "Our clock is %s behind the time published in the " "consensus network status document (%s GMT). Tor needs an " @@ -1776,7 +1865,8 @@ networkstatus_set_current_consensus(const char *consensus, result = 0; done: - networkstatus_vote_free(c); + if (free_consensus) + networkstatus_vote_free(c); tor_free(consensus_fname); tor_free(unverified_fname); return result; @@ -1812,7 +1902,8 @@ void routers_update_all_from_networkstatus(time_t now, int dir_version) { routerlist_t *rl = router_get_routerlist(); - networkstatus_t *consensus = networkstatus_get_live_consensus(now); + networkstatus_t *consensus = networkstatus_get_reasonably_live_consensus(now, + FLAV_NS); if (networkstatus_v2_list_has_changed) download_status_map_update_from_v2_networkstatus(); @@ -2028,7 +2119,7 @@ routers_update_status_from_consensus_networkstatus(smartlist_t *routers, void signed_descs_update_status_from_consensus_networkstatus(smartlist_t *descs) { - networkstatus_t *ns = current_consensus; + networkstatus_t *ns = current_ns_consensus; if (!ns) return; @@ -2036,7 +2127,7 @@ signed_descs_update_status_from_consensus_networkstatus(smartlist_t *descs) char dummy[DIGEST_LEN]; /* instantiates the digest map. */ memset(dummy, 0, sizeof(dummy)); - router_get_consensus_status_by_descriptor_digest(dummy); + router_get_consensus_status_by_descriptor_digest(ns, dummy); } SMARTLIST_FOREACH(descs, signed_descriptor_t *, d, { @@ -2264,8 +2355,9 @@ networkstatus_free_all(void) digestmap_free(v2_download_status_map, _tor_free); v2_download_status_map = NULL; - networkstatus_vote_free(current_consensus); - current_consensus = NULL; + networkstatus_vote_free(current_ns_consensus); + networkstatus_vote_free(current_md_consensus); + current_md_consensus = current_ns_consensus = NULL; for (i=0; i < N_CONSENSUS_FLAVORS; ++i) { consensus_waiting_for_certs_t *waiting = &consensus_waiting_for_certs[i]; diff --git a/src/or/networkstatus.h b/src/or/networkstatus.h index 32b71a9ceb..adaddc4982 100644 --- a/src/or/networkstatus.h +++ b/src/or/networkstatus.h @@ -48,12 +48,14 @@ const smartlist_t *networkstatus_get_v2_list(void); download_status_t *router_get_dl_status_by_descriptor_digest(const char *d); routerstatus_t *router_get_consensus_status_by_id(const char *digest); routerstatus_t *router_get_consensus_status_by_descriptor_digest( - const char *digest); + networkstatus_t *consensus, + const char *digest); routerstatus_t *router_get_consensus_status_by_nickname(const char *nickname, int warn_if_unnamed); const char *networkstatus_get_router_digest_by_nickname(const char *nickname); int networkstatus_nickname_is_unnamed(const char *nickname); -void networkstatus_consensus_download_failed(int status_code); +void networkstatus_consensus_download_failed(int status_code, + const char *flavname); void update_consensus_networkstatus_fetch_time(time_t now); int should_delay_dir_fetches(or_options_t *options); void update_networkstatus_downloads(time_t now); @@ -61,8 +63,11 @@ void update_certificate_downloads(time_t now); int consensus_is_waiting_for_certs(void); networkstatus_v2_t *networkstatus_v2_get_by_digest(const char *digest); networkstatus_t *networkstatus_get_latest_consensus(void); +networkstatus_t *networkstatus_get_latest_consensus_by_flavor( + consensus_flavor_t f); networkstatus_t *networkstatus_get_live_consensus(time_t now); -networkstatus_t *networkstatus_get_reasonably_live_consensus(time_t now); +networkstatus_t *networkstatus_get_reasonably_live_consensus(time_t now, + int flavor); #define NSSET_FROM_CACHE 1 #define NSSET_WAS_WAITING_FOR_CERTS 2 #define NSSET_DONT_DOWNLOAD_CERTS 4 diff --git a/src/or/or.h b/src/or/or.h index 2e65459d46..673a3920cf 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -83,6 +83,13 @@ #define snprintf _snprintf #endif +#ifdef USE_BUFFEREVENTS +#include <event2/bufferevent.h> +#include <event2/buffer.h> +#include <event2/util.h> +#endif + +#include "crypto.h" #include "tortls.h" #include "../common/torlog.h" #include "container.h" @@ -384,7 +391,9 @@ typedef enum { /** A connection to a hidden service directory server: download a v2 rendezvous * descriptor. */ #define DIR_PURPOSE_FETCH_RENDDESC_V2 18 -#define _DIR_PURPOSE_MAX 18 +/** A connection to a directory server: download a microdescriptor. */ +#define DIR_PURPOSE_FETCH_MICRODESC 19 +#define _DIR_PURPOSE_MAX 19 /** True iff <b>p</b> is a purpose corresponding to uploading data to a * directory server. */ @@ -963,6 +972,7 @@ typedef struct connection_t { /** Our socket; -1 if this connection is closed, or has no socket. */ evutil_socket_t s; int conn_array_index; /**< Index into the global connection array. */ + struct event *read_event; /**< Libevent event structure. */ struct event *write_event; /**< Libevent event structure. */ buf_t *inbuf; /**< Buffer holding data read over this connection. */ @@ -973,6 +983,11 @@ typedef struct connection_t { * read? */ time_t timestamp_lastwritten; /**< When was the last time libevent said we * could write? */ + +#ifdef USE_BUFFEREVENTS + struct bufferevent *bufev; /**< A Libevent buffered IO structure. */ +#endif + time_t timestamp_created; /**< When was this connection_t created? */ /* XXXX_IP6 make this IPv6-capable */ @@ -1072,10 +1087,16 @@ typedef struct or_connection_t { /* bandwidth* and *_bucket only used by ORs in OPEN state: */ int bandwidthrate; /**< Bytes/s added to the bucket. (OPEN ORs only.) */ int bandwidthburst; /**< Max bucket size for this conn. (OPEN ORs only.) */ +#ifndef USE_BUFFEREVENTS int read_bucket; /**< When this hits 0, stop receiving. Every second we * add 'bandwidthrate' to this, capping it at * bandwidthburst. (OPEN ORs only) */ int write_bucket; /**< When this hits 0, stop writing. Like read_bucket. */ +#else + /** DOCDOC */ + /* XXXX we could share this among all connections. */ + struct ev_token_bucket_cfg *bucket_cfg; +#endif int n_circuits; /**< How many circuits use this connection as p_conn or * n_conn ? */ @@ -1179,8 +1200,13 @@ typedef struct edge_connection_t { typedef struct dir_connection_t { connection_t _base; - char *requested_resource; /**< Which 'resource' did we ask the directory - * for? */ + /** Which 'resource' did we ask the directory for? This is typically the part + * of the URL string that defines, relative to the directory conn purpose, + * what thing we want. For example, in router descriptor downloads by + * descriptor digest, it contains "d/", then one ore more +-separated + * fingerprints. + **/ + char *requested_resource; unsigned int dirconn_direct:1; /**< Is this dirconn direct, or via Tor? */ /* Used only for server sides of some dir connections, to implement @@ -1271,6 +1297,51 @@ static INLINE control_connection_t *TO_CONTROL_CONN(connection_t *c) return DOWNCAST(control_connection_t, c); } +/* Conditional macros to help write code that works whether bufferevents are + disabled or not. + + We can't just write: + if (conn->bufev) { + do bufferevent stuff; + } else { + do other stuff; + } + because the bufferevent stuff won't even compile unless we have a fairly + new version of Libevent. Instead, we say: + IF_HAS_BUFFEREVENT(conn, { do_bufferevent_stuff } ); + or: + IF_HAS_BUFFEREVENT(conn, { + do bufferevent stuff; + }) ELSE_IF_NO_BUFFEREVENT { + do non-bufferevent stuff; + } + If we're compiling with bufferevent support, then the macros expand more or + less to: + if (conn->bufev) { + do_bufferevent_stuff; + } else { + do non-bufferevent stuff; + } + and if we aren't using bufferevents, they expand more or less to: + { do non-bufferevent stuff; } +*/ +#ifdef USE_BUFFEREVENTS +#define HAS_BUFFEREVENT(c) (((c)->bufev) != NULL) +#define IF_HAS_BUFFEREVENT(c, stmt) \ + if ((c)->bufev) do { \ + stmt ; \ + } while (0) +#define ELSE_IF_NO_BUFFEREVENT ; else +#define IF_HAS_NO_BUFFEREVENT(c) \ + if (NULL == (c)->bufev) +#else +#define HAS_BUFFEREVENT(c) (0) +#define IF_HAS_BUFFEREVENT(c, stmt) (void)0 +#define ELSE_IF_NO_BUFFEREVENT ; +#define IF_HAS_NO_BUFFEREVENT(c) \ + if (1) +#endif + /** What action type does an address policy indicate: accept or reject? */ typedef enum { ADDR_POLICY_ACCEPT=1, @@ -1434,12 +1505,10 @@ typedef struct { char *contact_info; /**< Declared contact info for this router. */ unsigned int is_hibernating:1; /**< Whether the router claims to be * hibernating */ - unsigned int has_old_dnsworkers:1; /**< Whether the router is using - * dnsworker code. */ - unsigned int caches_extra_info:1; /**< Whether the router caches and serves - * extrainfo documents. */ - unsigned int allow_single_hop_exits:1; /**< Whether the router allows - * single hop exits. */ + unsigned int caches_extra_info:1; /**< Whether the router says it caches and + * serves extrainfo documents. */ + unsigned int allow_single_hop_exits:1; /**< Whether the router says + * it allows single hop exits. */ /* local info */ unsigned int is_running:1; /**< As far as we know, is this OR currently @@ -1512,8 +1581,9 @@ typedef struct routerstatus_t { * has. */ char identity_digest[DIGEST_LEN]; /**< Digest of the router's identity * key. */ - char descriptor_digest[DIGEST_LEN]; /**< Digest of the router's most recent - * descriptor. */ + /** Digest of the router's most recent descriptor or microdescriptor. + * If it's a descriptor, we only use the first DIGEST_LEN bytes. */ + char descriptor_digest[DIGEST256_LEN]; uint32_t addr; /**< IPv4 address for this router. */ uint16_t or_port; /**< OR port for this router. */ uint16_t dir_port; /**< Directory port for this router. */ @@ -1632,6 +1702,10 @@ typedef struct microdesc_t { * up? */ #define MAX_ROUTERDESC_DOWNLOAD_FAILURES 8 +/** How many times will we try to download a microdescriptor before giving + * up? */ +#define MAX_MICRODESC_DOWNLOAD_FAILURES 8 + /** Contents of a v2 (non-consensus, non-vote) network status object. */ typedef struct networkstatus_v2_t { /** When did we receive the network-status document? */ @@ -2698,6 +2772,10 @@ typedef struct { * possible. */ int PreferTunneledDirConns; /**< If true, avoid dirservers that don't * support BEGIN_DIR, when possible. */ + int PortForwarding; /**< If true, use NAT-PMP or UPnP to automatically + * forward the DirPort and ORPort on the NAT device */ + char *PortForwardingHelper; /** < Filename or full path of the port + forwarding helper executable */ int AllowNonRFC953Hostnames; /**< If true, we allow connections to hostnames * with weird characters. */ /** If true, we try resolving hostnames with weird characters. */ @@ -2827,6 +2905,10 @@ typedef struct { */ double CircuitPriorityHalflife; + /** If true, do not enable IOCP on windows with bufferevents, even if + * we think we could. */ + int DisableIOCP; + } or_options_t; /** Persistent state for an onion router, as saved to disk. */ @@ -2935,8 +3017,6 @@ struct socks_request_t { * every connection. */ }; -/* all the function prototypes go here */ - /********************************* circuitbuild.c **********************/ /** How many hops does a general-purpose circuit have by default? */ @@ -3436,6 +3516,8 @@ typedef struct trusted_dir_server_t { * fetches to _any_ single directory server.] */ #define PDS_NO_EXISTING_SERVERDESC_FETCH (1<<3) +#define PDS_NO_EXISTING_MICRODESC_FETCH (1<<4) + #define _PDS_PREFER_TUNNELED_DIR_CONNS (1<<16) /** Possible ways to weight routers when choosing one randomly. See diff --git a/src/or/relay.c b/src/or/relay.c index b12cef4912..f9a44cf16a 100644 --- a/src/or/relay.c +++ b/src/or/relay.c @@ -1157,8 +1157,7 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, if (!conn->_base.marked_for_close) { /* only mark it if not already marked. it's possible to * get the 'end' right around when the client hangs up on us. */ - connection_mark_for_close(TO_CONN(conn)); - conn->_base.hold_open_until_flushed = 1; + connection_mark_and_flush(TO_CONN(conn)); } return 0; case RELAY_COMMAND_EXTEND: @@ -1365,7 +1364,7 @@ connection_edge_package_raw_inbuf(edge_connection_t *conn, int package_partial, return 0; } - amount_to_process = buf_datalen(conn->_base.inbuf); + amount_to_process = connection_get_inbuf_len(TO_CONN(conn)); if (!amount_to_process) return 0; @@ -1384,7 +1383,7 @@ connection_edge_package_raw_inbuf(edge_connection_t *conn, int package_partial, connection_fetch_from_buf(payload, length, TO_CONN(conn)); log_debug(domain,"(%d) Packaging %d bytes (%d waiting).", conn->_base.s, - (int)length, (int)buf_datalen(conn->_base.inbuf)); + (int)length, (int)connection_get_inbuf_len(TO_CONN(conn))); if (connection_edge_send_command(conn, RELAY_COMMAND_DATA, payload, length) < 0 ) @@ -2214,7 +2213,7 @@ set_streams_blocked_on_circ(circuit_t *circ, or_connection_t *orconn, edge->edge_blocked_on_circ = block; } - if (!conn->read_event) { + if (!conn->read_event && !HAS_BUFFEREVENT(conn)) { /* This connection is a placeholder for something; probably a DNS * request. It can't actually stop or start reading.*/ continue; @@ -2415,7 +2414,7 @@ append_cell_to_circuit_queue(circuit_t *circ, or_connection_t *orconn, make_circuit_active_on_conn(circ, orconn); } - if (! buf_datalen(orconn->_base.outbuf)) { + if (! connection_get_outbuf_len(TO_CONN(orconn))) { /* There is no data at all waiting to be sent on the outbuf. Add a * cell, so that we can notice when it gets flushed, flushed_some can * get called, and we can start putting more data onto the buffer then. diff --git a/src/or/routerlist.c b/src/or/routerlist.c index e52165ed84..43be8346cc 100644 --- a/src/or/routerlist.c +++ b/src/or/routerlist.c @@ -22,6 +22,7 @@ #include "geoip.h" #include "hibernate.h" #include "main.h" +#include "microdesc.h" #include "networkstatus.h" #include "policies.h" #include "reasons.h" @@ -44,9 +45,6 @@ static routerstatus_t *router_pick_trusteddirserver_impl( static void mark_all_trusteddirservers_up(void); static int router_nickname_matches(routerinfo_t *router, const char *nickname); static void trusted_dir_server_free(trusted_dir_server_t *ds); -static void launch_router_descriptor_downloads(smartlist_t *downloadable, - routerstatus_t *source, - time_t now); static int signed_desc_digest_is_recognized(signed_descriptor_t *desc); static void update_router_have_minimum_dir_info(void); static const char *signed_descriptor_get_body_impl(signed_descriptor_t *desc, @@ -1047,7 +1045,8 @@ router_pick_trusteddirserver(authority_type_t type, int flags) /* If the reason that we got no server is that servers are "busy", * we must be excluding good servers because we already have serverdesc * fetches with them. Do not mark down servers up because of this. */ - tor_assert((flags & PDS_NO_EXISTING_SERVERDESC_FETCH)); + tor_assert((flags & (PDS_NO_EXISTING_SERVERDESC_FETCH| + PDS_NO_EXISTING_MICRODESC_FETCH))); return NULL; } @@ -1174,6 +1173,7 @@ router_pick_trusteddirserver_impl(authority_type_t type, int flags, const int fascistfirewall = ! (flags & PDS_IGNORE_FASCISTFIREWALL); const int prefer_tunnel = (flags & _PDS_PREFER_TUNNELED_DIR_CONNS); const int no_serverdesc_fetching =(flags & PDS_NO_EXISTING_SERVERDESC_FETCH); + const int no_microdesc_fetching =(flags & PDS_NO_EXISTING_MICRODESC_FETCH); int n_busy = 0; if (!trusted_dir_servers) @@ -1212,6 +1212,13 @@ router_pick_trusteddirserver_impl(authority_type_t type, int flags, continue; } } + if (no_microdesc_fetching) { + if (connection_get_by_type_addr_port_purpose( + CONN_TYPE_DIR, &addr, d->dir_port, DIR_PURPOSE_FETCH_MICRODESC)) { + ++n_busy; + continue; + } + } if (prefer_tunnel && d->or_port && @@ -3899,6 +3906,7 @@ routerlist_retry_directory_downloads(time_t now) router_reset_descriptor_download_failures(); update_networkstatus_downloads(now); update_router_descriptor_downloads(now); + update_microdesc_downloads(now); } /** Return 1 if all running sufficiently-stable routers will reject @@ -4060,7 +4068,9 @@ any_trusted_dir_is_v1_authority(void) /** For every current directory connection whose purpose is <b>purpose</b>, * and where the resource being downloaded begins with <b>prefix</b>, split * rest of the resource into base16 fingerprints, decode them, and set the - * corresponding elements of <b>result</b> to a nonzero value. */ + * corresponding elements of <b>result</b> to a nonzero value. + * DOCDOC purpose==microdesc + */ static void list_pending_downloads(digestmap_t *result, int purpose, const char *prefix) @@ -4068,20 +4078,23 @@ list_pending_downloads(digestmap_t *result, const size_t p_len = strlen(prefix); smartlist_t *tmp = smartlist_create(); smartlist_t *conns = get_connection_array(); + int flags = DSR_HEX; + if (purpose == DIR_PURPOSE_FETCH_MICRODESC) + flags = DSR_DIGEST256|DSR_BASE64; tor_assert(result); - SMARTLIST_FOREACH(conns, connection_t *, conn, - { + SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) { if (conn->type == CONN_TYPE_DIR && conn->purpose == purpose && !conn->marked_for_close) { const char *resource = TO_DIR_CONN(conn)->requested_resource; if (!strcmpstart(resource, prefix)) dir_split_resource_into_fingerprints(resource + p_len, - tmp, NULL, DSR_HEX); + tmp, NULL, flags); } - }); + } SMARTLIST_FOREACH_END(conn); + SMARTLIST_FOREACH(tmp, char *, d, { digestmap_set(result, d, (void*)1); @@ -4101,10 +4114,18 @@ list_pending_descriptor_downloads(digestmap_t *result, int extrainfo) list_pending_downloads(result, purpose, "d/"); } -/** Launch downloads for all the descriptors whose digests are listed - * as digests[i] for lo <= i < hi. (Lo and hi may be out of range.) - * If <b>source</b> is given, download from <b>source</b>; otherwise, - * download from an appropriate random directory server. +/** DOCDOC */ +/*XXXX NM should use digest256, if one comes into being. */ +void +list_pending_microdesc_downloads(digestmap_t *result) +{ + list_pending_downloads(result, DIR_PURPOSE_FETCH_MICRODESC, "d/"); +} + +/** Launch downloads for all the descriptors whose digests or digests256 + * are listed as digests[i] for lo <= i < hi. (Lo and hi may be out of + * range.) If <b>source</b> is given, download from <b>source</b>; + * otherwise, download from an appropriate random directory server. */ static void initiate_descriptor_downloads(routerstatus_t *source, @@ -4115,6 +4136,20 @@ initiate_descriptor_downloads(routerstatus_t *source, int i, n = hi-lo; char *resource, *cp; size_t r_len; + + int digest_len = DIGEST_LEN, enc_digest_len = HEX_DIGEST_LEN; + char sep = '+'; + int b64_256 = 0; + + if (purpose == DIR_PURPOSE_FETCH_MICRODESC) { + /* Microdescriptors are downloaded by "-"-separated base64-encoded + * 256-bit digests. */ + digest_len = DIGEST256_LEN; + enc_digest_len = BASE64_DIGEST256_LEN; + sep = '-'; + b64_256 = 1; + } + if (n <= 0) return; if (lo < 0) @@ -4122,15 +4157,19 @@ initiate_descriptor_downloads(routerstatus_t *source, if (hi > smartlist_len(digests)) hi = smartlist_len(digests); - r_len = 8 + (HEX_DIGEST_LEN+1)*n; + r_len = 8 + (enc_digest_len+1)*n; cp = resource = tor_malloc(r_len); memcpy(cp, "d/", 2); cp += 2; for (i = lo; i < hi; ++i) { - base16_encode(cp, r_len-(cp-resource), - smartlist_get(digests,i), DIGEST_LEN); - cp += HEX_DIGEST_LEN; - *cp++ = '+'; + if (b64_256) { + digest256_to_base64(cp, smartlist_get(digests, i)); + } else { + base16_encode(cp, r_len-(cp-resource), + smartlist_get(digests,i), digest_len); + } + cp += enc_digest_len; + *cp++ = sep; } memcpy(cp-1, ".z", 3); @@ -4177,6 +4216,7 @@ client_would_use_router(routerstatus_t *rs, time_t now, or_options_t *options) * So use 96 because it's a nice number. */ #define MAX_DL_PER_REQUEST 96 +#define MAX_MICRODESC_DL_PER_REQUEST 92 /** Don't split our requests so finely that we are requesting fewer than * this number per server. */ #define MIN_DL_PER_REQUEST 4 @@ -4191,21 +4231,33 @@ client_would_use_router(routerstatus_t *rs, time_t now, or_options_t *options) * them until they have more, or until this amount of time has passed. */ #define MAX_CLIENT_INTERVAL_WITHOUT_REQUEST (10*60) -/** Given a list of router descriptor digests in <b>downloadable</b>, decide - * whether to delay fetching until we have more. If we don't want to delay, - * launch one or more requests to the appropriate directory authorities. */ -static void -launch_router_descriptor_downloads(smartlist_t *downloadable, - routerstatus_t *source, time_t now) +/** Given a <b>purpose</b> (FETCH_MICRODESC or FETCH_SERVERDESC) and a list of + * router descriptor digests or microdescriptor digest256s in + * <b>downloadable</b>, decide whether to delay fetching until we have more. + * If we don't want to delay, launch one or more requests to the appropriate + * directory authorities. + */ +void +launch_descriptor_downloads(int purpose, + smartlist_t *downloadable, + routerstatus_t *source, time_t now) { int should_delay = 0, n_downloadable; or_options_t *options = get_options(); + const char *descname; + + tor_assert(purpose == DIR_PURPOSE_FETCH_SERVERDESC || + purpose == DIR_PURPOSE_FETCH_MICRODESC); + + descname = (purpose == DIR_PURPOSE_FETCH_SERVERDESC) ? + "routerdesc" : "microdesc"; n_downloadable = smartlist_len(downloadable); if (!directory_fetches_dir_info_early(options)) { if (n_downloadable >= MAX_DL_TO_DELAY) { log_debug(LD_DIR, - "There are enough downloadable routerdescs to launch requests."); + "There are enough downloadable %ss to launch requests.", + descname); should_delay = 0; } else { should_delay = (last_routerdesc_download_attempted + @@ -4213,13 +4265,15 @@ launch_router_descriptor_downloads(smartlist_t *downloadable, if (!should_delay && n_downloadable) { if (last_routerdesc_download_attempted) { log_info(LD_DIR, - "There are not many downloadable routerdescs, but we've " + "There are not many downloadable %ss, but we've " "been waiting long enough (%d seconds). Downloading.", + descname, (int)(now-last_routerdesc_download_attempted)); } else { log_info(LD_DIR, - "There are not many downloadable routerdescs, but we haven't " - "tried downloading descriptors recently. Downloading."); + "There are not many downloadable %ss, but we haven't " + "tried downloading descriptors recently. Downloading.", + descname); } } } @@ -4246,12 +4300,19 @@ launch_router_descriptor_downloads(smartlist_t *downloadable, * update_router_descriptor_downloads() later on, once the connections * have succeeded or failed. */ - pds_flags |= PDS_NO_EXISTING_SERVERDESC_FETCH; + pds_flags |= (purpose == DIR_PURPOSE_FETCH_MICRODESC) ? + PDS_NO_EXISTING_MICRODESC_FETCH : + PDS_NO_EXISTING_SERVERDESC_FETCH; } n_per_request = CEIL_DIV(n_downloadable, MIN_REQUESTS); - if (n_per_request > MAX_DL_PER_REQUEST) - n_per_request = MAX_DL_PER_REQUEST; + if (purpose == DIR_PURPOSE_FETCH_MICRODESC) { + if (n_per_request > MAX_MICRODESC_DL_PER_REQUEST) + n_per_request = MAX_MICRODESC_DL_PER_REQUEST; + } else { + if (n_per_request > MAX_DL_PER_REQUEST) + n_per_request = MAX_DL_PER_REQUEST; + } if (n_per_request < MIN_DL_PER_REQUEST) n_per_request = MIN_DL_PER_REQUEST; @@ -4266,7 +4327,7 @@ launch_router_descriptor_downloads(smartlist_t *downloadable, req_plural, n_downloadable, rtr_plural, n_per_request); smartlist_sort_digests(downloadable); for (i=0; i < n_downloadable; i += n_per_request) { - initiate_descriptor_downloads(source, DIR_PURPOSE_FETCH_SERVERDESC, + initiate_descriptor_downloads(source, purpose, downloadable, i, i+n_per_request, pds_flags); } @@ -4539,7 +4600,8 @@ update_consensus_router_descriptor_downloads(time_t now, int is_vote, smartlist_len(downloadable), n_delayed, n_have, n_in_oldrouters, n_would_reject, n_wouldnt_use, n_inprogress); - launch_router_descriptor_downloads(downloadable, source, now); + launch_descriptor_downloads(DIR_PURPOSE_FETCH_SERVERDESC, + downloadable, source, now); digestmap_free(map, NULL); done: @@ -4564,10 +4626,13 @@ update_router_descriptor_downloads(time_t now) if (directory_fetches_dir_info_early(options)) { update_router_descriptor_cache_downloads_v2(now); } + update_consensus_router_descriptor_downloads(now, 0, - networkstatus_get_reasonably_live_consensus(now)); + networkstatus_get_reasonably_live_consensus(now, FLAV_NS)); /* XXXX021 we could be smarter here; see notes on bug 652. */ + /* XXXX NM Microdescs: if we're not fetching microdescriptors, we need + * to make something else invoke this. */ /* If we're a server that doesn't have a configured address, we rely on * directory fetches to learn when our address changes. So if we haven't * tried to get any routerdescs in a long time, try a dummy fetch now. */ @@ -4738,7 +4803,7 @@ count_loading_descriptors_progress(void) int num_present = 0, num_usable=0; time_t now = time(NULL); const networkstatus_t *consensus = - networkstatus_get_reasonably_live_consensus(now); + networkstatus_get_reasonably_live_consensus(now, FLAV_NS); double fraction; if (!consensus) @@ -4768,7 +4833,7 @@ update_router_have_minimum_dir_info(void) int res; or_options_t *options = get_options(); const networkstatus_t *consensus = - networkstatus_get_reasonably_live_consensus(now); + networkstatus_get_reasonably_live_consensus(now, FLAV_NS); if (!consensus) { if (!networkstatus_get_latest_consensus()) @@ -4905,7 +4970,6 @@ router_differences_are_cosmetic(routerinfo_t *r1, routerinfo_t *r2) (r1->contact_info && r2->contact_info && strcasecmp(r1->contact_info, r2->contact_info)) || r1->is_hibernating != r2->is_hibernating || - r1->has_old_dnsworkers != r2->has_old_dnsworkers || cmp_addr_policies(r1->exit_policy, r2->exit_policy)) return 0; if ((r1->declared_family == NULL) != (r2->declared_family == NULL)) diff --git a/src/or/routerlist.h b/src/or/routerlist.h index 574bce7ffc..804be2abaa 100644 --- a/src/or/routerlist.h +++ b/src/or/routerlist.h @@ -192,5 +192,11 @@ int hid_serv_get_responsible_directories(smartlist_t *responsible_dirs, int hid_serv_acting_as_directory(void); int hid_serv_responsible_for_desc_id(const char *id); +void list_pending_microdesc_downloads(digestmap_t *result); +void launch_descriptor_downloads(int purpose, + smartlist_t *downloadable, + routerstatus_t *source, + time_t now); + #endif diff --git a/src/or/routerparse.c b/src/or/routerparse.c index 96749e5a74..f7e645e8fd 100644 --- a/src/or/routerparse.c +++ b/src/or/routerparse.c @@ -69,7 +69,6 @@ typedef enum { K_V, K_W, K_M, - K_EVENTDNS, K_EXTRA_INFO, K_EXTRA_INFO_DIGEST, K_CACHES_EXTRA_INFO, @@ -286,7 +285,6 @@ static token_rule_t routerdesc_token_table[] = { T01("family", K_FAMILY, ARGS, NO_OBJ ), T01("caches-extra-info", K_CACHES_EXTRA_INFO, NO_ARGS, NO_OBJ ), - T01("eventdns", K_EVENTDNS, ARGS, NO_OBJ ), T0N("opt", K_OPT, CONCAT_ARGS, OBJ_OK ), T1( "bandwidth", K_BANDWIDTH, GE(3), NO_OBJ ), @@ -1487,13 +1485,6 @@ router_parse_entry_from_string(const char *s, const char *end, router->contact_info = tor_strdup(tok->args[0]); } - if ((tok = find_opt_by_keyword(tokens, K_EVENTDNS))) { - router->has_old_dnsworkers = tok->n_args && !strcmp(tok->args[0], "0"); - } else if (router->platform) { - if (! tor_version_as_new_as(router->platform, "0.1.2.2-alpha")) - router->has_old_dnsworkers = 1; - } - exit_policy_tokens = find_all_exitpolicy(tokens); if (!smartlist_len(exit_policy_tokens)) { log_warn(LD_DIR, "No exit policy tokens in descriptor."); @@ -1943,6 +1934,7 @@ routerstatus_parse_entry_from_string(memarea_t *area, if (!consensus_method) flav = FLAV_NS; + tor_assert(flav == FLAV_NS || flav == FLAV_MICRODESC); eos = find_start_of_next_routerstatus(*s); @@ -1955,15 +1947,16 @@ routerstatus_parse_entry_from_string(memarea_t *area, goto err; } tok = find_by_keyword(tokens, K_R); - tor_assert(tok->n_args >= 7); + tor_assert(tok->n_args >= 7); /* guaranteed by GE(7) in K_R setup */ if (flav == FLAV_NS) { if (tok->n_args < 8) { log_warn(LD_DIR, "Too few arguments to r"); goto err; } - } else { - offset = -1; + } else if (flav == FLAV_MICRODESC) { + offset = -1; /* There is no identity digest */ } + if (vote_rs) { rs = &vote_rs->status; } else { @@ -2139,6 +2132,16 @@ routerstatus_parse_entry_from_string(memarea_t *area, vote_rs->microdesc = line; } } SMARTLIST_FOREACH_END(t); + } else if (flav == FLAV_MICRODESC) { + tok = find_opt_by_keyword(tokens, K_M); + if (tok) { + tor_assert(tok->n_args); + if (digest256_from_base64(rs->descriptor_digest, tok->args[0])) { + log_warn(LD_DIR, "Error decoding microdescriptor digest %s", + escaped(tok->args[0])); + goto err; + } + } } if (!strcasecmp(rs->nickname, UNNAMED_ROUTER_NICKNAME)) diff --git a/src/test/Makefile.am b/src/test/Makefile.am index 546fa2f4b7..cfe330c744 100644 --- a/src/test/Makefile.am +++ b/src/test/Makefile.am @@ -21,10 +21,19 @@ test_SOURCES = \ test_util.c \ tinytest.c +if USE_BUFFEREVENTS +levent_openssl_lib = -levent_openssl +else +levent_openssl_lib = +endif + test_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ \ @TOR_LDFLAGS_libevent@ test_LDADD = ../or/libtor.a ../common/libor.a ../common/libor-crypto.a \ ../common/libor-event.a \ - @TOR_ZLIB_LIBS@ -lm @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ + @TOR_ZLIB_LIBS@ -lm @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ \ + @TOR_LIB_WS32@ @TOR_LIB_GDI@ $(levent_openssl_lib) noinst_HEADERS = tinytest.h tinytest_macros.h test.h + + diff --git a/src/test/test_util.c b/src/test/test_util.c index a14d548b8e..68a0ca2984 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -6,6 +6,7 @@ #include "orconfig.h" #define CONTROL_PRIVATE #define MEMPOOL_PRIVATE +#define UTIL_PRIVATE #include "or.h" #include "config.h" #include "control.h" @@ -1208,6 +1209,45 @@ test_util_load_win_lib(void *ptr) } #endif +static void +clear_hex_errno(char *hex_errno) +{ + memset(hex_errno, ' ', HEX_ERRNO_SIZE - 2); + hex_errno[HEX_ERRNO_SIZE - 1] = '\n'; + hex_errno[HEX_ERRNO_SIZE] = '\0'; +} + +static void +test_util_exit_status(void *ptr) +{ + char hex_errno[HEX_ERRNO_SIZE + 1]; + + (void)ptr; + + clear_hex_errno(hex_errno); + format_helper_exit_status(0, 0, hex_errno); + tt_str_op(hex_errno, ==, " 0/0\n"); + + clear_hex_errno(hex_errno); + format_helper_exit_status(0, 0x7FFFFFFF, hex_errno); + tt_str_op(hex_errno, ==, " 0/7FFFFFFF\n"); + + clear_hex_errno(hex_errno); + format_helper_exit_status(0xFF, -0x80000000, hex_errno); + tt_str_op(hex_errno, ==, "FF/-80000000\n"); + + clear_hex_errno(hex_errno); + format_helper_exit_status(0x7F, 0, hex_errno); + tt_str_op(hex_errno, ==, " 7F/0\n"); + + clear_hex_errno(hex_errno); + format_helper_exit_status(0x08, -0x242, hex_errno); + tt_str_op(hex_errno, ==, " 8/-242\n"); + + done: + ; +} + #define UTIL_LEGACY(name) \ { #name, legacy_test_helper, 0, &legacy_setup, test_util_ ## name } @@ -1234,6 +1274,7 @@ struct testcase_t util_tests[] = { #ifdef MS_WINDOWS UTIL_TEST(load_win_lib, 0), #endif + UTIL_TEST(exit_status, 0), END_OF_TESTCASES }; diff --git a/src/tools/Makefile.am b/src/tools/Makefile.am index 1bb5076849..de90797409 100644 --- a/src/tools/Makefile.am +++ b/src/tools/Makefile.am @@ -16,3 +16,7 @@ tor_checkkey_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ \ @TOR_LDFLAGS_libevent@ tor_checkkey_LDADD = ../common/libor.a ../common/libor-crypto.a \ -lm @TOR_ZLIB_LIBS@ @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ + +SUBDIRS = tor-fw-helper +DIST_SUBDIRS = tor-fw-helper + diff --git a/src/tools/tor-fw-helper/Makefile.am b/src/tools/tor-fw-helper/Makefile.am new file mode 100644 index 0000000000..f01f94c252 --- /dev/null +++ b/src/tools/tor-fw-helper/Makefile.am @@ -0,0 +1,33 @@ +if USE_FW_HELPER +bin_PROGRAMS = tor-fw-helper +else +bin_PROGRAMS = +endif + +tor_fw_helper_SOURCES = tor-fw-helper.c \ + tor-fw-helper-natpmp.c tor-fw-helper-upnp.c +tor_fw_helper_INCLUDES = tor-fw-helper.h tor-fw-helper-natpmp.h tor-fw-helper-upnp.h + +if NAT_PMP +nat_pmp_ldflags = @TOR_LDFLAGS_libnatpmp@ +nat_pmp_ldadd = -lnatpmp +nat_pmp_cppflags = @TOR_CPPFLAGS_libnatpmp@ +else +nat_pmp_ldflags = +nat_pmp_ldadd = +nat_pmp_cppflags = +endif + +if MINIUPNPC +miniupnpc_ldflags = @TOR_LDFLAGS_libminiupnpc@ +miniupnpc_ldadd = -lminiupnpc -lm +miniupnpc_cppflags = @TOR_CPPFLAGS_libminiupnpc@ +else +miniupnpc_ldflags = +miniupnpc_ldadd = +miniupnpc_cppflags = +endif + +tor_fw_helper_LDFLAGS = $(nat_pmp_ldflags) $(miniupnpc_ldflags) +tor_fw_helper_LDADD = $(nat_pmp_ldadd) $(miniupnpc_ldadd) ../../common/libor.a @TOR_LIB_WS32@ +tor_fw_helper_CPPFLAGS = $(nat_pmp_cppflags) $(miniupnpc_cppflags) diff --git a/src/tools/tor-fw-helper/tor-fw-helper-natpmp.c b/src/tools/tor-fw-helper/tor-fw-helper-natpmp.c new file mode 100644 index 0000000000..51b456a8c7 --- /dev/null +++ b/src/tools/tor-fw-helper/tor-fw-helper-natpmp.c @@ -0,0 +1,233 @@ +/* Copyright (c) 2010, Jacob Appelbaum, Steven J. Murdoch. + * Copyright (c) 2010, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file tor-fw-helper-natpmp.c + * \brief The implementation of our NAT-PMP firewall helper. + **/ + +#include "orconfig.h" +#ifdef NAT_PMP +#include <stdint.h> +#include <stdio.h> +#include <string.h> +#include <errno.h> +#include <arpa/inet.h> + +// debugging stuff +#include <assert.h> + +#include "tor-fw-helper.h" +#include "tor-fw-helper-natpmp.h" + +/** This hooks NAT-PMP into our multi-backend API. */ +static tor_fw_backend_t tor_natpmp_backend = { + "natpmp", + sizeof(struct natpmp_state_t), + tor_natpmp_init, + tor_natpmp_cleanup, + tor_natpmp_fetch_public_ip, + tor_natpmp_add_tcp_mapping +}; + +/** Return the backend for NAT-PMP. */ +const tor_fw_backend_t * +tor_fw_get_natpmp_backend(void) +{ + return &tor_natpmp_backend; +} + +/** Initialize the NAT-PMP backend and store the results in + * <b>backend_state</b>.*/ +int +tor_natpmp_init(tor_fw_options_t *tor_fw_options, void *backend_state) +{ + natpmp_state_t *state = (natpmp_state_t *) backend_state; + int r = 0; + + memset(&(state->natpmp), 0, sizeof(natpmp_t)); + memset(&(state->response), 0, sizeof(natpmpresp_t)); + state->init = 0; + state->protocol = NATPMP_PROTOCOL_TCP; + state->lease = NATPMP_DEFAULT_LEASE; + + if (tor_fw_options->verbose) + fprintf(stdout, "V: natpmp init...\n"); + + r = initnatpmp(&(state->natpmp)); + if (r == 0) { + state->init = 1; + fprintf(stdout, "tor-fw-helper: natpmp initialized...\n"); + return r; + } else { + fprintf(stderr, "tor-fw-helper: natpmp failed to initialize...\n"); + return r; + } +} + +/** Tear down the NAT-PMP connection stored in <b>backend_state</b>.*/ +int +tor_natpmp_cleanup(tor_fw_options_t *tor_fw_options, void *backend_state) +{ + natpmp_state_t *state = (natpmp_state_t *) backend_state; + int r = 0; + if (tor_fw_options->verbose) + fprintf(stdout, "V: natpmp cleanup...\n"); + r = closenatpmp(&(state->natpmp)); + if (tor_fw_options->verbose) + fprintf(stdout, "V: closing natpmp socket: %d\n", r); + return r; +} + +/** Use select() to wait until we can read on fd. */ +static int +wait_until_fd_readable(int fd, struct timeval *timeout) +{ + int r; + fd_set fds; + if (fd >= FD_SETSIZE) { + fprintf(stderr, "E: NAT-PMP FD_SETSIZE error %d\n", fd); + return -1; + } + FD_ZERO(&fds); + FD_SET(fd, &fds); + r = select(fd+1, &fds, NULL, NULL, timeout); + if (r == -1) { + fprintf(stdout, "V: select failed in wait_until_fd_readable: %s\n", + strerror(errno)); + return -1; + } + /* XXXX we should really check to see whether fd was readable, or we timed + out. */ + return 0; +} + +/** Add a TCP port mapping for a single port stored in <b>tor_fw_options</b> + * using the <b>natpmp_t</b> stored in <b>backend_state</b>. */ +int +tor_natpmp_add_tcp_mapping(tor_fw_options_t *tor_fw_options, + void *backend_state) +{ + natpmp_state_t *state = (natpmp_state_t *) backend_state; + int r = 0; + int x = 0; + int sav_errno; + + struct timeval timeout; + + if (tor_fw_options->verbose) + fprintf(stdout, "V: sending natpmp portmapping request...\n"); + r = sendnewportmappingrequest(&(state->natpmp), state->protocol, + tor_fw_options->internal_port, + tor_fw_options->external_port, + state->lease); + if (tor_fw_options->verbose) + fprintf(stdout, "tor-fw-helper: NAT-PMP sendnewportmappingrequest " + "returned %d (%s)\n", r, r==12?"SUCCESS":"FAILED"); + + do { + getnatpmprequesttimeout(&(state->natpmp), &timeout); + x = wait_until_fd_readable(state->natpmp.s, &timeout); + if (x == -1) + return -1; + + if (tor_fw_options->verbose) + fprintf(stdout, "V: attempting to readnatpmpreponseorretry...\n"); + r = readnatpmpresponseorretry(&(state->natpmp), &(state->response)); + sav_errno = errno; + + if (r<0 && r!=NATPMP_TRYAGAIN) { + fprintf(stderr, "E: readnatpmpresponseorretry failed %d\n", r); + fprintf(stderr, "E: errno=%d '%s'\n", sav_errno, + strerror(sav_errno)); + } + + } while (r == NATPMP_TRYAGAIN); + + if (r != 0) { + /* XXX TODO: NATPMP_* should be formatted into useful error strings */ + fprintf(stderr, "E: NAT-PMP It appears that something went wrong:" + " %d\n", r); + if (r == -51) + fprintf(stderr, "E: NAT-PMP It appears that the request was " + "unauthorized\n"); + return r; + } + + if (r == NATPMP_SUCCESS) { + fprintf(stdout, "tor-fw-helper: NAT-PMP mapped public port %hu to" + " localport %hu liftime %u\n", + (state->response).pnu.newportmapping.mappedpublicport, + (state->response).pnu.newportmapping.privateport, + (state->response).pnu.newportmapping.lifetime); + } + + tor_fw_options->nat_pmp_status = 1; + + return r; +} + +/** Fetch our likely public IP from our upstream NAT-PMP enabled NAT device. + * Use the connection context stored in <b>backend_state</b>. */ +int +tor_natpmp_fetch_public_ip(tor_fw_options_t *tor_fw_options, + void *backend_state) +{ + int r = 0; + int x = 0; + int sav_errno; + natpmp_state_t *state = (natpmp_state_t *) backend_state; + + struct timeval timeout; + + r = sendpublicaddressrequest(&(state->natpmp)); + fprintf(stdout, "tor-fw-helper: NAT-PMP sendpublicaddressrequest returned" + " %d (%s)\n", r, r==2?"SUCCESS":"FAILED"); + + do { + getnatpmprequesttimeout(&(state->natpmp), &timeout); + + x = wait_until_fd_readable(state->natpmp.s, &timeout); + if (x == -1) + return -1; + + if (tor_fw_options->verbose) + fprintf(stdout, "V: NAT-PMP attempting to read reponse...\n"); + r = readnatpmpresponseorretry(&(state->natpmp), &(state->response)); + sav_errno = errno; + + if (tor_fw_options->verbose) + fprintf(stdout, "V: NAT-PMP readnatpmpresponseorretry returned" + " %d\n", r); + + if ( r < 0 && r != NATPMP_TRYAGAIN) { + fprintf(stderr, "E: NAT-PMP readnatpmpresponseorretry failed %d\n", + r); + fprintf(stderr, "E: NAT-PMP errno=%d '%s'\n", sav_errno, + strerror(sav_errno)); + } + + } while (r == NATPMP_TRYAGAIN ); + + if (r != 0) { + fprintf(stderr, "E: NAT-PMP It appears that something went wrong:" + " %d\n", r); + return r; + } + + fprintf(stdout, "tor-fw-helper: ExternalIPAddress = %s\n", + inet_ntoa((state->response).pnu.publicaddress.addr)); + tor_fw_options->public_ip_status = 1; + + if (tor_fw_options->verbose) { + fprintf(stdout, "V: result = %u\n", r); + fprintf(stdout, "V: type = %u\n", (state->response).type); + fprintf(stdout, "V: resultcode = %u\n", (state->response).resultcode); + fprintf(stdout, "V: epoch = %u\n", (state->response).epoch); + } + + return r; +} +#endif + diff --git a/src/tools/tor-fw-helper/tor-fw-helper-natpmp.h b/src/tools/tor-fw-helper/tor-fw-helper-natpmp.h new file mode 100644 index 0000000000..5a6540b68f --- /dev/null +++ b/src/tools/tor-fw-helper/tor-fw-helper-natpmp.h @@ -0,0 +1,47 @@ +/* Copyright (c) 2010, Jacob Appelbaum, Steven J. Murdoch. + * Copyright (c) 2010, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file tor-fw-helper-natpmp.h + **/ + +#ifdef NAT_PMP +#ifndef _TOR_FW_HELPER_NATPMP_H +#define _TOR_FW_HELPER_NATPMP_H + +#include <natpmp.h> + +/** This is the default NAT-PMP lease time in seconds. */ +#define NATPMP_DEFAULT_LEASE 3600 +/** NAT-PMP has many codes for success; this is one of them. */ +#define NATPMP_SUCCESS 0 + +/** This is our NAT-PMP meta structure - it holds our request data, responses, + * various NAT-PMP parameters, and of course the status of the motion in the + * NAT-PMP ocean. */ +typedef struct natpmp_state_t { + natpmp_t natpmp; + natpmpresp_t response; + int fetch_public_ip; + int status; + int init; /**< Have we been initialized? */ + int protocol; /**< This will only be TCP. */ + int lease; +} natpmp_state_t; + +const tor_fw_backend_t *tor_fw_get_natpmp_backend(void); + +int tor_natpmp_init(tor_fw_options_t *tor_fw_options, void *backend_state); + +int tor_natpmp_cleanup(tor_fw_options_t *tor_fw_options, void *backend_state); + +int tor_natpmp_add_tcp_mapping(tor_fw_options_t *tor_fw_options, + void *backend_state); + +int tor_natpmp_fetch_public_ip(tor_fw_options_t *tor_fw_options, + void *backend_state); + +#endif +#endif + diff --git a/src/tools/tor-fw-helper/tor-fw-helper-upnp.c b/src/tools/tor-fw-helper/tor-fw-helper-upnp.c new file mode 100644 index 0000000000..b3b80d6987 --- /dev/null +++ b/src/tools/tor-fw-helper/tor-fw-helper-upnp.c @@ -0,0 +1,186 @@ +/* Copyright (c) 2010, Jacob Appelbaum, Steven J. Murdoch. + * Copyright (c) 2010, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file tor-fw-helper-upnp.c + * \brief The implementation of our UPnP firewall helper. + **/ + +#include "orconfig.h" +#ifdef MINIUPNPC +#include <stdint.h> +#include <string.h> +#include <stdio.h> + +#include <assert.h> + +#include "compat.h" +#include "tor-fw-helper.h" +#include "tor-fw-helper-upnp.h" + +/** UPnP timeout value. */ +#define UPNP_DISCOVER_TIMEOUT 2000 +/** Description of the port mapping in the UPnP table. */ +#define UPNP_DESC "Tor relay" + +/* XXX TODO: We should print these as a useful user string when we return the + * number to a user */ +/** Magic numbers as miniupnpc return codes. */ +#define UPNP_ERR_SUCCESS 0 +#define UPNP_ERR_NODEVICESFOUND 1 +#define UPNP_ERR_NOIGDFOUND 2 +#define UPNP_ERR_ADDPORTMAPPING 3 +#define UPNP_ERR_GETPORTMAPPING 4 +#define UPNP_ERR_DELPORTMAPPING 5 +#define UPNP_ERR_GETEXTERNALIP 6 +#define UPNP_ERR_INVAL 7 +#define UPNP_ERR_OTHER 8 +#define UPNP_SUCCESS 1 + +/** This hooks miniupnpc into our multi-backend API. */ +static tor_fw_backend_t tor_miniupnp_backend = { + "miniupnp", + sizeof(struct miniupnpc_state_t), + tor_upnp_init, + tor_upnp_cleanup, + tor_upnp_fetch_public_ip, + tor_upnp_add_tcp_mapping +}; + +/** Return the backend for miniupnp. */ +const tor_fw_backend_t * +tor_fw_get_miniupnp_backend(void) +{ + return &tor_miniupnp_backend; +} + +/** Initialize the UPnP backend and store the results in + * <b>backend_state</b>.*/ +int +tor_upnp_init(tor_fw_options_t *options, void *backend_state) +{ + /* + This leaks the user agent from the client to the router - perhaps we don't + want to do that? eg: + + User-Agent: Ubuntu/10.04, UPnP/1.0, MiniUPnPc/1.4 + + */ + miniupnpc_state_t *state = (miniupnpc_state_t *) backend_state; + struct UPNPDev *devlist; + int r; + + memset(&(state->urls), 0, sizeof(struct UPNPUrls)); + memset(&(state->data), 0, sizeof(struct IGDdatas)); + state->init = 0; + + devlist = upnpDiscover(UPNP_DISCOVER_TIMEOUT, NULL, NULL, 0); + if (NULL == devlist) { + fprintf(stderr, "E: upnpDiscover returned: NULL\n"); + return UPNP_ERR_NODEVICESFOUND; + } + + assert(options); + r = UPNP_GetValidIGD(devlist, &(state->urls), &(state->data), + state->lanaddr, UPNP_LANADDR_SZ); + fprintf(stdout, "tor-fw-helper: UPnP GetValidIGD returned: %d (%s)\n", r, + r==UPNP_SUCCESS?"SUCCESS":"FAILED"); + + freeUPNPDevlist(devlist); + + if (r != 1 && r != 2) + return UPNP_ERR_NOIGDFOUND; + + state->init = 1; + return UPNP_ERR_SUCCESS; +} + +/** Tear down the UPnP connection stored in <b>backend_state</b>.*/ +int +tor_upnp_cleanup(tor_fw_options_t *options, void *backend_state) +{ + + miniupnpc_state_t *state = (miniupnpc_state_t *) backend_state; + assert(options); + + if (state->init) + FreeUPNPUrls(&(state->urls)); + state->init = 0; + + return UPNP_ERR_SUCCESS; +} + +/** Fetch our likely public IP from our upstream UPnP IGD enabled NAT device. + * Use the connection context stored in <b>backend_state</b>. */ +int +tor_upnp_fetch_public_ip(tor_fw_options_t *options, void *backend_state) +{ + miniupnpc_state_t *state = (miniupnpc_state_t *) backend_state; + int r; + char externalIPAddress[16]; + + if (!state->init) { + r = tor_upnp_init(options, state); + if (r != UPNP_ERR_SUCCESS) + return r; + } + + r = UPNP_GetExternalIPAddress(state->urls.controlURL, + state->data.first.servicetype, + externalIPAddress); + + if (r != UPNPCOMMAND_SUCCESS) + goto err; + + if (externalIPAddress[0]) { + fprintf(stdout, "tor-fw-helper: ExternalIPAddress = %s\n", + externalIPAddress); tor_upnp_cleanup(options, state); + options->public_ip_status = 1; + return UPNP_ERR_SUCCESS; + } else { + goto err; + } + + err: + tor_upnp_cleanup(options, state); + return UPNP_ERR_GETEXTERNALIP; +} + +/** Add a TCP port mapping for a single port stored in <b>tor_fw_options</b> + * and store the results in <b>backend_state</b>. */ +int +tor_upnp_add_tcp_mapping(tor_fw_options_t *options, void *backend_state) +{ + miniupnpc_state_t *state = (miniupnpc_state_t *) backend_state; + int r; + char internal_port_str[6]; + char external_port_str[6]; + + if (!state->init) { + r = tor_upnp_init(options, state); + if (r != UPNP_ERR_SUCCESS) + return r; + } + + if (options->verbose) + fprintf(stdout, "V: internal port: %d, external port: %d\n", + (int)options->internal_port, (int)options->external_port); + + tor_snprintf(internal_port_str, sizeof(internal_port_str), + "%d", (int)options->internal_port); + tor_snprintf(external_port_str, sizeof(external_port_str), + "%d", (int)options->external_port); + + r = UPNP_AddPortMapping(state->urls.controlURL, + state->data.first.servicetype, + external_port_str, internal_port_str, + state->lanaddr, UPNP_DESC, "TCP", 0); + if (r != UPNPCOMMAND_SUCCESS) + return UPNP_ERR_ADDPORTMAPPING; + + options->upnp_status = 1; + return UPNP_ERR_SUCCESS; +} +#endif + diff --git a/src/tools/tor-fw-helper/tor-fw-helper-upnp.h b/src/tools/tor-fw-helper/tor-fw-helper-upnp.h new file mode 100644 index 0000000000..fdbd2dd185 --- /dev/null +++ b/src/tools/tor-fw-helper/tor-fw-helper-upnp.h @@ -0,0 +1,43 @@ +/* Copyright (c) 2010, Jacob Appelbaum, Steven J. Murdoch. + * Copyright (c) 2010, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file tor-fw-helper-upnp.h + * \brief The main header for our firewall helper. + **/ + +#ifdef MINIUPNPC +#ifndef _TOR_FW_HELPER_UPNP_H +#define _TOR_FW_HELPER_UPNP_H + +#include <miniupnpc/miniwget.h> +#include <miniupnpc/miniupnpc.h> +#include <miniupnpc/upnpcommands.h> +#include <miniupnpc/upnperrors.h> + +/** This is a magic number for miniupnpc lan address size. */ +#define UPNP_LANADDR_SZ 64 + +/** This is our miniupnpc meta structure - it holds our request data, + * responses, and various miniupnpc parameters. */ +typedef struct miniupnpc_state_t { + struct UPNPUrls urls; + struct IGDdatas data; + char lanaddr[UPNP_LANADDR_SZ]; + int init; +} miniupnpc_state_t; + +const tor_fw_backend_t *tor_fw_get_miniupnp_backend(void); + +int tor_upnp_init(tor_fw_options_t *options, void *backend_state); + +int tor_upnp_cleanup(tor_fw_options_t *options, void *backend_state); + +int tor_upnp_fetch_public_ip(tor_fw_options_t *options, void *backend_state); + +int tor_upnp_add_tcp_mapping(tor_fw_options_t *options, void *backend_state); + +#endif +#endif + diff --git a/src/tools/tor-fw-helper/tor-fw-helper.c b/src/tools/tor-fw-helper/tor-fw-helper.c new file mode 100644 index 0000000000..378672a20a --- /dev/null +++ b/src/tools/tor-fw-helper/tor-fw-helper.c @@ -0,0 +1,363 @@ +/* Copyright (c) 2010, Jacob Appelbaum, Steven J. Murdoch. + * Copyright (c) 2010, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file tor-fw-helper.c + * \brief The main wrapper around our firewall helper logic. + **/ + +/* + * tor-fw-helper is a tool for opening firewalls with NAT-PMP and UPnP; this + * tool is designed to be called by hand or by Tor by way of a exec() at a + * later date. + */ + +#include <stdio.h> +#include <stdint.h> +#include <stdlib.h> +#include <getopt.h> +#include <time.h> +#include <string.h> + +#include "orconfig.h" +#include "tor-fw-helper.h" +#ifdef NAT_PMP +#include "tor-fw-helper-natpmp.h" +#endif +#ifdef MINIUPNPC +#include "tor-fw-helper-upnp.h" +#endif + +/** This is our meta storage type - it holds information about each helper + including the total number of helper backends, function pointers, and helper + state. */ +typedef struct backends_t { + /** The total number of backends */ + int n_backends; + /** The backend functions as an array */ + tor_fw_backend_t backend_ops[MAX_BACKENDS]; + /** The internal backend state */ + void *backend_state[MAX_BACKENDS]; +} backends_t; + +/** Initalize each backend helper with the user input stored in <b>options</b> + * and put the results in the <b>backends</b> struct. */ +static int +init_backends(tor_fw_options_t *options, backends_t *backends) +{ + int n_available = 0; + int i, r, n; + tor_fw_backend_t *backend_ops_list[MAX_BACKENDS]; + void *data = NULL; + /* First, build a list of the working backends. */ + n = 0; +#ifdef MINIUPNPC + backend_ops_list[n++] = (tor_fw_backend_t *) tor_fw_get_miniupnp_backend(); +#endif +#ifdef NAT_PMP + backend_ops_list[n++] = (tor_fw_backend_t *) tor_fw_get_natpmp_backend(); +#endif + n_available = n; + + /* Now, for each backend that might work, try to initialize it. + * That's how we roll, initialized. + */ + n = 0; + for (i=0; i<n_available; ++i) { + data = calloc(1, backend_ops_list[i]->state_len); + if (!data) { + perror("calloc"); + exit(1); + } + r = backend_ops_list[i]->init(options, data); + if (r == 0) { + backends->backend_ops[n] = *backend_ops_list[i]; + backends->backend_state[n] = data; + n++; + } else { + free(data); + } + } + backends->n_backends = n; + + return n; +} + +/** Return the proper commandline switches when the user needs information. */ +static void +usage(void) +{ + fprintf(stderr, "tor-fw-helper usage:\n" + " [-h|--help]\n" + " [-T|--Test]\n" + " [-v|--verbose]\n" + " [-g|--fetch-public-ip]\n" + " -i|--internal-or-port [TCP port]\n" + " [-e|--external-or-port [TCP port]]\n" + " [-d|--internal-dir-port [TCP port]\n" + " [-p|--external-dir-port [TCP port]]]\n"); +} + +/** Log commandline options to a hardcoded file <b>tor-fw-helper.log</b> in the + * current working directory. */ +static int +log_commandline_options(int argc, char **argv) +{ + int i, retval; + FILE *logfile; + time_t now; + + /* Open the log file */ + logfile = fopen("tor-fw-helper.log", "a"); + if (NULL == logfile) + return -1; + + /* Send all commandline arguments to the file */ + now = time(NULL); + retval = fprintf(logfile, "START: %s\n", ctime(&now)); + for (i = 0; i < argc; i++) { + retval = fprintf(logfile, "ARG: %d: %s\n", i, argv[i]); + if (retval < 0) + goto error; + + retval = fprintf(stdout, "ARG: %d: %s\n", i, argv[i]); + if (retval < 0) + goto error; + } + now = time(NULL); + retval = fprintf(logfile, "END: %s\n", ctime(&now)); + + /* Close and clean up */ + retval = fclose(logfile); + return retval; + + /* If there was an error during writing */ + error: + fclose(logfile); + return -1; +} + +/** Iterate over over each of the supported <b>backends</b> and attempt to + * fetch the public ip. */ +static void +tor_fw_fetch_public_ip(tor_fw_options_t *tor_fw_options, + backends_t *backends) +{ + int i; + int r = 0; + + if (tor_fw_options->verbose) + fprintf(stdout, "V: tor_fw_fetch_public_ip\n"); + + for (i=0; i<backends->n_backends; ++i) { + if (tor_fw_options->verbose) { + fprintf(stdout, "V: running backend_state now: %i\n", i); + fprintf(stdout, "V: size of backend state: %u\n", + (int)(backends->backend_ops)[i].state_len); + fprintf(stdout, "V: backend state name: %s\n", + (char *)(backends->backend_ops)[i].name); + } + r = backends->backend_ops[i].fetch_public_ip(tor_fw_options, + backends->backend_state[i]); + fprintf(stdout, "tor-fw-helper: tor_fw_fetch_public_ip backend %s " + " returned: %i\n", (char *)(backends->backend_ops)[i].name, r); + } +} + +/** Iterate over each of the supported <b>backends</b> and attempt to add a + * port forward for the OR port stored in <b>tor_fw_options</b>. */ +static void +tor_fw_add_or_port(tor_fw_options_t *tor_fw_options, + backends_t *backends) +{ + int i; + int r = 0; + + if (tor_fw_options->verbose) + fprintf(stdout, "V: tor_fw_add_or_port\n"); + + for (i=0; i<backends->n_backends; ++i) { + if (tor_fw_options->verbose) { + fprintf(stdout, "V: running backend_state now: %i\n", i); + fprintf(stdout, "V: size of backend state: %u\n", + (int)(backends->backend_ops)[i].state_len); + fprintf(stdout, "V: backend state name: %s\n", + (const char *) backends->backend_ops[i].name); + } + r = backends->backend_ops[i].add_tcp_mapping(tor_fw_options, + backends->backend_state[i]); + fprintf(stdout, "tor-fw-helper: tor_fw_add_or_port backend %s " + "returned: %i\n", (const char *) backends->backend_ops[i].name, r); + } +} + +/** Iterate over each of the supported <b>backends</b> and attempt to add a + * port forward for the Dir port stored in <b>tor_fw_options</b>. */ +static void +tor_fw_add_dir_port(tor_fw_options_t *tor_fw_options, + backends_t *backends) +{ + int i; + int r = 0; + + if (tor_fw_options->verbose) + fprintf(stdout, "V: tor_fw_add_dir_port\n"); + + for (i=0; i<backends->n_backends; ++i) { + if (tor_fw_options->verbose) { + fprintf(stdout, "V: running backend_state now: %i\n", i); + fprintf(stdout, "V: size of backend state: %u\n", + (int)(backends->backend_ops)[i].state_len); + fprintf(stdout, "V: backend state name: %s\n", + (char *)(backends->backend_ops)[i].name); + } + r = backends->backend_ops[i].add_tcp_mapping(tor_fw_options, + backends->backend_state[i]); + fprintf(stdout, "tor-fw-helper: tor_fw_add_dir_port backend %s " + "returned: %i\n", (const char *)backends->backend_ops[i].name, r); + } +} + +int +main(int argc, char **argv) +{ + int r = 0; + int c = 0; + + tor_fw_options_t tor_fw_options; + backends_t backend_state; + + memset(&tor_fw_options, 0, sizeof(tor_fw_options)); + + while (1) { + int option_index = 0; + static struct option long_options[] = + { + {"verbose", 0, 0, 'v'}, + {"help", 0, 0, 'h'}, + {"internal-or-port", 1, 0, 'i'}, + {"external-or-port", 1, 0, 'e'}, + {"internal-dir-port", 1, 0, 'd'}, + {"external-dir-port", 1, 0, 'p'}, + {"fetch-public-ip", 0, 0, 'g'}, + {"test-commandline", 0, 0, 'T'}, + {0, 0, 0, 0} + }; + + c = getopt_long(argc, argv, "vhi:e:d:p:gT", + long_options, &option_index); + if (c == -1) + break; + + switch (c) { + case 'v': tor_fw_options.verbose = 1; break; + case 'h': tor_fw_options.help = 1; usage(); exit(1); break; + case 'i': sscanf(optarg, "%hu", &tor_fw_options.private_or_port); + break; + case 'e': sscanf(optarg, "%hu", &tor_fw_options.public_or_port); + break; + case 'd': sscanf(optarg, "%hu", &tor_fw_options.private_dir_port); + break; + case 'p': sscanf(optarg, "%hu", &tor_fw_options.public_dir_port); + break; + case 'g': tor_fw_options.fetch_public_ip = 1; break; + case 'T': tor_fw_options.test_commandline = 1; break; + case '?': break; + default : fprintf(stderr, "Unknown option!\n"); usage(); exit(1); + } + } + + if (tor_fw_options.verbose) { + fprintf(stderr, "V: tor-fw-helper version %s\n" + "V: We were called with the following arguments:\n" + "V: verbose = %d, help = %d, pub or port = %u, " + "priv or port = %u\n" + "V: pub dir port = %u, priv dir port = %u\n" + "V: fetch_public_ip = %u\n", + tor_fw_version, tor_fw_options.verbose, tor_fw_options.help, + tor_fw_options.private_or_port, tor_fw_options.public_or_port, + tor_fw_options.private_dir_port, tor_fw_options.public_dir_port, + tor_fw_options.fetch_public_ip); + } + + if (tor_fw_options.test_commandline) { + return log_commandline_options(argc, argv); + } + + /* At the very least, we require an ORPort; + Given a private ORPort, we can ask for a mapping that matches the port + externally. + */ + if (!tor_fw_options.private_or_port && !tor_fw_options.fetch_public_ip) { + fprintf(stderr, "E: We require an ORPort or fetch_public_ip" + " request!\n"); + usage(); + exit(1); + } else { + /* When we only have one ORPort, internal/external are + set to be the same.*/ + if (!tor_fw_options.public_or_port && tor_fw_options.private_or_port) { + if (tor_fw_options.verbose) + fprintf(stdout, "V: We're setting public_or_port = " + "private_or_port.\n"); + tor_fw_options.public_or_port = tor_fw_options.private_or_port; + } + } + if (!tor_fw_options.private_dir_port) { + if (tor_fw_options.verbose) + fprintf(stdout, "V: We have no DirPort; no hole punching for " + "DirPorts\n"); + + } else { + /* When we only have one DirPort, internal/external are + set to be the same.*/ + if (!tor_fw_options.public_dir_port && tor_fw_options.private_dir_port) { + if (tor_fw_options.verbose) + fprintf(stdout, "V: We're setting public_or_port = " + "private_or_port.\n"); + + tor_fw_options.public_dir_port = tor_fw_options.private_dir_port; + } + } + + if (tor_fw_options.verbose) { + fprintf(stdout, "V: pub or port = %u, priv or port = %u\n" + "V: pub dir port = %u, priv dir port = %u\n", + tor_fw_options.private_or_port, tor_fw_options.public_or_port, + tor_fw_options.private_dir_port, + tor_fw_options.public_dir_port); + } + + // Initalize the various fw-helper backend helpers + r = init_backends(&tor_fw_options, &backend_state); + if (r) + printf("tor-fw-helper: %i NAT traversal helper(s) loaded\n", r); + + if (tor_fw_options.fetch_public_ip) { + tor_fw_fetch_public_ip(&tor_fw_options, &backend_state); + } + + if (tor_fw_options.private_or_port) { + tor_fw_options.internal_port = tor_fw_options.private_or_port; + tor_fw_options.external_port = tor_fw_options.private_or_port; + tor_fw_add_or_port(&tor_fw_options, &backend_state); + } + + if (tor_fw_options.private_dir_port) { + tor_fw_options.internal_port = tor_fw_options.private_dir_port; + tor_fw_options.external_port = tor_fw_options.private_dir_port; + tor_fw_add_dir_port(&tor_fw_options, &backend_state); + } + + r = (((tor_fw_options.nat_pmp_status | tor_fw_options.upnp_status) + |tor_fw_options.public_ip_status)); + if (r > 0) { + fprintf(stdout, "tor-fw-helper: SUCCESS\n"); + } else { + fprintf(stderr, "tor-fw-helper: FAILURE\n"); + } + + exit(r); +} + diff --git a/src/tools/tor-fw-helper/tor-fw-helper.h b/src/tools/tor-fw-helper/tor-fw-helper.h new file mode 100644 index 0000000000..b83d4b7d8f --- /dev/null +++ b/src/tools/tor-fw-helper/tor-fw-helper.h @@ -0,0 +1,57 @@ +/* Copyright (c) 2010, Jacob Appelbaum, Steven J. Murdoch. + * Copyright (c) 2010, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file tor-fw-helper.h + * \brief The main header for our firewall helper. + **/ + +#ifndef _TOR_FW_HELPER_H +#define _TOR_FW_HELPER_H + +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <getopt.h> +#include <time.h> + +/** The current version of tor-fw-helper. */ +#define tor_fw_version "0.1" + +/** This is an arbitrary hard limit - We currently have two (NAT-PMP and UPnP). + We're likely going to add the Intel UPnP library but nothing else comes to + mind at the moment. */ +#define MAX_BACKENDS 23 + +/** This is where we store parsed commandline options. */ +typedef struct { + int verbose; + int help; + int test_commandline; + uint16_t private_dir_port; + uint16_t private_or_port; + uint16_t public_dir_port; + uint16_t public_or_port; + uint16_t internal_port; + uint16_t external_port; + int fetch_public_ip; + int nat_pmp_status; + int upnp_status; + int public_ip_status; +} tor_fw_options_t; + +/** This is our main structure that defines our backend helper API; each helper + * must conform to these public methods if it expects to be handled in a + * non-special way. */ +typedef struct tor_fw_backend_t { + const char *name; + size_t state_len; + int (*init)(tor_fw_options_t *options, void *backend_state); + int (*cleanup)(tor_fw_options_t *options, void *backend_state); + int (*fetch_public_ip)(tor_fw_options_t *options, void *backend_state); + int (*add_tcp_mapping)(tor_fw_options_t *options, void *backend_state); +} tor_fw_backend_t; + +#endif + diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h index ce84f14f13..7708e90b8c 100644 --- a/src/win32/orconfig.h +++ b/src/win32/orconfig.h @@ -233,5 +233,5 @@ #define USING_TWOS_COMPLEMENT /* Version number of package */ -#define VERSION "0.2.2.17-alpha-dev" +#define VERSION "0.2.3.0-alpha-dev" |