diff options
Diffstat (limited to 'src/or')
76 files changed, 18306 insertions, 12648 deletions
diff --git a/src/or/Makefile.am b/src/or/Makefile.am index b56d496431..a9ac3cdee1 100644 --- a/src/or/Makefile.am +++ b/src/or/Makefile.am @@ -1,8 +1,5 @@ -TESTS = test - -noinst_PROGRAMS = test - bin_PROGRAMS = tor +noinst_LIBRARIES = libtor.a if BUILD_NT_SERVICES tor_platform_source=ntmain.c @@ -10,18 +7,30 @@ else tor_platform_source= endif -EXTRA_DIST=ntmain.c +EXTRA_DIST=ntmain.c or_sha1.i + +if USE_EXTERNAL_EVDNS +evdns_source= +else +evdns_source=eventdns.c +endif -tor_SOURCES = buffers.c circuitbuild.c circuitlist.c \ +libtor_a_SOURCES = buffers.c circuitbuild.c circuitlist.c \ circuituse.c command.c config.c \ connection.c connection_edge.c connection_or.c control.c \ cpuworker.c directory.c dirserv.c dirvote.c \ dns.c dnsserv.c geoip.c hibernate.c main.c $(tor_platform_source) \ + microdesc.c \ networkstatus.c onion.c policies.c \ reasons.c relay.c rendcommon.c rendclient.c rendmid.c \ rendservice.c rephist.c router.c routerlist.c routerparse.c \ - eventdns.c \ - tor_main.c + $(evdns_source) config_codedigest.c + +#libtor_a_LIBADD = ../common/libor.a ../common/libor-crypto.a \ +# ../common/libor-event.a + + +tor_SOURCES = tor_main.c AM_CPPFLAGS = -DSHARE_DATADIR="\"$(datadir)\"" \ -DLOCALSTATEDIR="\"$(localstatedir)\"" \ @@ -32,42 +41,48 @@ AM_CPPFLAGS = -DSHARE_DATADIR="\"$(datadir)\"" \ # matters a lot there, and is quite hard to debug if you forget to do it. tor_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ @TOR_LDFLAGS_libevent@ -tor_LDADD = ../common/libor.a ../common/libor-crypto.a \ - -lz @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ -test_SOURCES = buffers.c circuitbuild.c circuitlist.c \ - circuituse.c command.c config.c \ - connection.c connection_edge.c connection_or.c control.c \ - cpuworker.c directory.c dirserv.c dirvote.c \ - dns.c dnsserv.c geoip.c hibernate.c main.c $(tor_platform_source) \ - networkstatus.c onion.c policies.c \ - reasons.c relay.c rendcommon.c rendclient.c rendmid.c \ - rendservice.c rephist.c router.c routerlist.c routerparse.c \ - eventdns.c \ - test_data.c test.c +tor_LDADD = ./libtor.a ../common/libor.a ../common/libor-crypto.a \ + ../common/libor-event.a \ + @TOR_ZLIB_LIBS@ -lm @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ -test_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ \ - @TOR_LDFLAGS_libevent@ -test_LDADD = ../common/libor.a ../common/libor-crypto.a \ - -lz @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ +noinst_HEADERS = buffers.h circuitbuild.h circuitlist.h circuituse.h \ + command.h config.h connection_edge.h connection.h connection_or.h \ + control.h cpuworker.h directory.h dirserv.h dirvote.h dns.h \ + dnsserv.h geoip.h hibernate.h main.h microdesc.h networkstatus.h \ + ntmain.h onion.h policies.h reasons.h relay.h rendclient.h \ + rendcommon.h rendmid.h rendservice.h rephist.h router.h routerlist.h \ + routerparse.h or.h eventdns.h eventdns_tor.h micro-revision.i -noinst_HEADERS = or.h eventdns.h eventdns_tor.h micro-revision.i +config_codedigest.o: or_sha1.i tor_main.o: micro-revision.i micro-revision.i: FORCE @rm -f micro-revision.tmp; \ - if test -d ../../.git && test -x "`which git 2>&1;true`"; then \ - HASH="`git rev-parse --short=16 HEAD`"; \ - echo \"$$HASH\" > micro-revision.tmp; \ - fi; \ + if test -d ../../.git && test -x "`which git 2>&1;true`"; then \ + HASH="`git rev-parse --short=16 HEAD`"; \ + echo \"$$HASH\" > micro-revision.tmp; \ + fi; \ if test ! -f micro-revision.tmp ; then \ if test ! -f micro-revision.i ; then \ echo '""' > micro-revision.i; \ fi; \ elif test ! -f micro-revision.i || \ - test x"`cat micro-revision.tmp`" != x"`cat micro-revision.i`"; then \ + test x"`cat micro-revision.tmp`" != x"`cat micro-revision.i`"; then \ mv micro-revision.tmp micro-revision.i; \ fi; true +or_sha1.i: $(tor_SOURCES) + if test "@SHA1SUM@" != none; then \ + @SHA1SUM@ $(tor_SOURCES) | @SED@ -n 's/^\(.*\)$$/"\1\\n"/p' > or_sha1.i; \ + elif test "@OPENSSL@" != none; then \ + @OPENSSL@ sha1 $(tor_SOURCES) | @SED@ -n 's/SHA1(\(.*\))= \(.*\)/"\2 \1\\n"/p' > or_sha1.i; \ + else \ + rm or_sha1.i; \ + touch or_sha1.i; \ + fi + +CLEANFILES = micro-revision.i + #Dummy target to ensure that micro-revision.i _always_ gets built. FORCE: diff --git a/src/or/buffers.c b/src/or/buffers.c index 710374638b..db926955b4 100644 --- a/src/or/buffers.c +++ b/src/or/buffers.c @@ -12,6 +12,14 @@ **/ #define BUFFERS_PRIVATE #include "or.h" +#include "buffers.h" +#include "config.h" +#include "connection_edge.h" +#include "connection_or.h" +#include "control.h" +#include "reasons.h" +#include "../common/util.h" +#include "../common/torlog.h" #ifdef HAVE_UNISTD_H #include <unistd.h> #endif @@ -145,10 +153,13 @@ get_freelist(size_t alloc) /** Deallocate a chunk or put it on a freelist */ static void -chunk_free(chunk_t *chunk) +chunk_free_unchecked(chunk_t *chunk) { - size_t alloc = CHUNK_ALLOC_SIZE(chunk->memlen); - chunk_freelist_t *freelist = get_freelist(alloc); + size_t alloc; + chunk_freelist_t *freelist; + + alloc = CHUNK_ALLOC_SIZE(chunk->memlen); + freelist = get_freelist(alloc); if (freelist && freelist->cur_length < freelist->max_length) { chunk->next = freelist->head; freelist->head = chunk; @@ -193,7 +204,7 @@ chunk_new_with_alloc_size(size_t alloc) } #else static void -chunk_free(chunk_t *chunk) +chunk_free_unchecked(chunk_t *chunk) { tor_free(chunk); } @@ -259,13 +270,22 @@ buf_shrink_freelists(int free_all) int n_to_free = free_all ? freelists[i].cur_length : (freelists[i].lowest_length - slack); int n_to_skip = freelists[i].cur_length - n_to_free; + int orig_length = freelists[i].cur_length; int orig_n_to_free = n_to_free, n_freed=0; int orig_n_to_skip = n_to_skip; int new_length = n_to_skip; chunk_t **chp = &freelists[i].head; chunk_t *chunk; while (n_to_skip) { - tor_assert((*chp)->next); + if (! (*chp)->next) { + log_warn(LD_BUG, "I wanted to skip %d chunks in the freelist for " + "%d-byte chunks, but only found %d. (Length %d)", + orig_n_to_skip, (int)freelists[i].alloc_size, + orig_n_to_skip-n_to_skip, freelists[i].cur_length); + assert_freelist_ok(&freelists[i]); + goto done; + } + // tor_assert((*chp)->next); chp = &(*chp)->next; --n_to_skip; } @@ -290,13 +310,15 @@ buf_shrink_freelists(int free_all) } // tor_assert(!n_to_free); freelists[i].cur_length = new_length; - log_info(LD_MM, "Cleaned freelist for %d-byte chunks: kept %d, " - "dropped %d.", - (int)freelists[i].alloc_size, orig_n_to_skip, orig_n_to_free); + log_info(LD_MM, "Cleaned freelist for %d-byte chunks: original " + "length %d, kept %d, dropped %d.", + (int)freelists[i].alloc_size, orig_length, + orig_n_to_skip, orig_n_to_free); } freelists[i].lowest_length = freelists[i].cur_length; assert_freelist_ok(&freelists[i]); } + done: enable_control_logging(); #else (void) free_all; @@ -404,7 +426,7 @@ buf_pullup(buf_t *buf, size_t bytes, int nulterminate) dest->next = src->next; if (buf->tail == src) buf->tail = dest; - chunk_free(src); + chunk_free_unchecked(src); } else { memcpy(CHUNK_WRITE_PTR(dest), src->data, n); dest->datalen += n; @@ -450,7 +472,7 @@ buf_remove_from_front(buf_t *buf, size_t n) buf->head = victim->next; if (buf->tail == victim) buf->tail = NULL; - chunk_free(victim); + chunk_free_unchecked(victim); } } check(); @@ -484,7 +506,7 @@ buf_clear(buf_t *buf) buf->datalen = 0; for (chunk = buf->head; chunk; chunk = next) { next = chunk->next; - chunk_free(chunk); + chunk_free_unchecked(chunk); } buf->head = buf->tail = NULL; } @@ -523,6 +545,8 @@ buf_slack(const buf_t *buf) void buf_free(buf_t *buf) { + if (!buf) + return; buf_clear(buf); buf->magic = 0xdeadbeef; tor_free(buf); @@ -642,12 +666,12 @@ read_to_chunk_tls(buf_t *buf, chunk_t *chunk, tor_tls_t *tls, * (because of EOF), set *<b>reached_eof</b> to 1 and return 0. Return -1 on * error; else return the number of bytes read. */ -/* XXXX021 indicate "read blocked" somehow? */ +/* XXXX023 indicate "read blocked" somehow? */ int read_to_buf(int s, size_t at_most, buf_t *buf, int *reached_eof, int *socket_error) { - /* XXXX021 It's stupid to overload the return values for these functions: + /* XXXX023 It's stupid to overload the return values for these functions: * "error status" and "number of bytes read" are not mutually exclusive. */ int r = 0; @@ -832,7 +856,7 @@ flush_chunk_tls(tor_tls_t *tls, buf_t *buf, chunk_t *chunk, int flush_buf(int s, buf_t *buf, size_t sz, size_t *buf_flushlen) { - /* XXXX021 It's stupid to overload the return values for these functions: + /* XXXX023 It's stupid to overload the return values for these functions: * "error status" and "number of bytes flushed" are not mutually exclusive. */ int r; @@ -1275,6 +1299,47 @@ fetch_from_buf_http(buf_t *buf, return 1; } +/** + * Wait this many seconds before warning the user about using SOCKS unsafely + * again (requires that WarnUnsafeSocks is turned on). */ +#define SOCKS_WARN_INTERVAL 5 + +/** Warn that the user application has made an unsafe socks request using + * protocol <b>socks_protocol</b> on port <b>port</b>. Don't warn more than + * once per SOCKS_WARN_INTERVAL, unless <b>safe_socks</b> is set. */ +static void +log_unsafe_socks_warning(int socks_protocol, const char *address, + uint16_t port, int safe_socks) +{ + static ratelim_t socks_ratelim = RATELIM_INIT(SOCKS_WARN_INTERVAL); + + or_options_t *options = get_options(); + char *m = NULL; + if (! options->WarnUnsafeSocks) + return; + if (safe_socks || (m = rate_limit_log(&socks_ratelim, approx_time()))) { + log_warn(LD_APP, + "Your application (using socks%d to port %d) is giving " + "Tor only an IP address. Applications that do DNS resolves " + "themselves may leak information. Consider using Socks4A " + "(e.g. via privoxy or socat) instead. For more information, " + "please see https://wiki.torproject.org/TheOnionRouter/" + "TorFAQ#SOCKSAndDNS.%s%s", + socks_protocol, + (int)port, + safe_socks ? " Rejecting." : "", + m ? m : ""); + tor_free(m); + } + control_event_client_status(LOG_WARN, + "DANGEROUS_SOCKS PROTOCOL=SOCKS%d ADDRESS=%s:%d", + socks_protocol, address, (int)port); +} + +/** Do not attempt to parse socks messages longer than this. This value is + * actually significantly higher than the longest possible socks message. */ +#define MAX_SOCKS_MESSAGE_LEN 512 + /** There is a (possibly incomplete) socks handshake on <b>buf</b>, of one * of the forms * - socks4: "socksheader username\\0" @@ -1313,14 +1378,10 @@ fetch_from_buf_socks(buf_t *buf, socks_request_t *req, char *next, *startaddr; struct in_addr in; - /* If the user connects with socks4 or the wrong variant of socks5, - * then log a warning to let him know that it might be unwise. */ - static int have_warned_about_unsafe_socks = 0; - if (buf->datalen < 2) /* version and another byte */ return 0; - buf_pullup(buf, 128, 0); + buf_pullup(buf, MAX_SOCKS_MESSAGE_LEN, 0); tor_assert(buf->head && buf->head->datalen >= 2); socksver = *buf->head->data; @@ -1396,21 +1457,8 @@ fetch_from_buf_socks(buf_t *buf, socks_request_t *req, req->port = ntohs(get_uint16(buf->head->data+4+addrlen)); buf_remove_from_front(buf, 6+addrlen); if (req->command != SOCKS_COMMAND_RESOLVE_PTR && - !addressmap_have_mapping(req->address,0) && - !have_warned_about_unsafe_socks) { - log_warn(LD_APP, - "Your application (using socks5 to port %d) is giving " - "Tor only an IP address. Applications that do DNS resolves " - "themselves may leak information. Consider using Socks4A " - "(e.g. via privoxy or socat) instead. For more information, " - "please see http://wiki.noreply.org/noreply/TheOnionRouter/" - "TorFAQ#SOCKSAndDNS.%s", req->port, - safe_socks ? " Rejecting." : ""); - /*have_warned_about_unsafe_socks = 1;*/ - /*(for now, warn every time)*/ - control_event_client_status(LOG_WARN, - "DANGEROUS_SOCKS PROTOCOL=SOCKS5 ADDRESS=%s:%d", - req->address, req->port); + !addressmap_have_mapping(req->address,0)) { + log_unsafe_socks_warning(5, req->address, req->port, safe_socks); if (safe_socks) return -1; } @@ -1475,8 +1523,8 @@ fetch_from_buf_socks(buf_t *buf, socks_request_t *req, return -1; } - req->port = ntohs(*(uint16_t*)(buf->head->data+2)); - destip = ntohl(*(uint32_t*)(buf->head->data+4)); + req->port = ntohs(get_uint16(buf->head->data+2)); + destip = ntohl(get_uint32(buf->head->data+4)); if ((!req->port && req->command!=SOCKS_COMMAND_RESOLVE) || !destip) { log_warn(LD_APP,"socks4: Port or DestIP is zero. Rejecting."); return -1; @@ -1491,7 +1539,8 @@ fetch_from_buf_socks(buf_t *buf, socks_request_t *req, return -1; } log_debug(LD_APP, - "socks4: successfully read destip (%s)", safe_str(tmpbuf)); + "socks4: successfully read destip (%s)", + safe_str_client(tmpbuf)); socks4_prot = socks4; } @@ -1509,20 +1558,9 @@ fetch_from_buf_socks(buf_t *buf, socks_request_t *req, startaddr = NULL; if (socks4_prot != socks4a && - !addressmap_have_mapping(tmpbuf,0) && - !have_warned_about_unsafe_socks) { - log_warn(LD_APP, - "Your application (using socks4 to port %d) is giving Tor " - "only an IP address. Applications that do DNS resolves " - "themselves may leak information. Consider using Socks4A " - "(e.g. via privoxy or socat) instead. For more information, " - "please see http://wiki.noreply.org/noreply/TheOnionRouter/" - "TorFAQ#SOCKSAndDNS.%s", req->port, - safe_socks ? " Rejecting." : ""); - /*have_warned_about_unsafe_socks = 1;*/ /*(for now, warn every time)*/ - control_event_client_status(LOG_WARN, - "DANGEROUS_SOCKS PROTOCOL=SOCKS4 ADDRESS=%s:%d", - tmpbuf, req->port); + !addressmap_have_mapping(tmpbuf,0)) { + log_unsafe_socks_warning(4, tmpbuf, req->port, safe_socks); + if (safe_socks) return -1; } @@ -1614,6 +1652,132 @@ fetch_from_buf_socks(buf_t *buf, socks_request_t *req, } } +/** Inspect a reply from SOCKS server stored in <b>buf</b> according + * to <b>state</b>, removing the protocol data upon success. Return 0 on + * incomplete response, 1 on success and -1 on error, in which case + * <b>reason</b> is set to a descriptive message (free() when finished + * with it). + * + * As a special case, 2 is returned when user/pass is required + * during SOCKS5 handshake and user/pass is configured. + */ +int +fetch_from_buf_socks_client(buf_t *buf, int state, char **reason) +{ + unsigned char *data; + size_t addrlen; + + if (buf->datalen < 2) + return 0; + + buf_pullup(buf, MAX_SOCKS_MESSAGE_LEN, 0); + tor_assert(buf->head && buf->head->datalen >= 2); + + data = (unsigned char *) buf->head->data; + + switch (state) { + case PROXY_SOCKS4_WANT_CONNECT_OK: + /* Wait for the complete response */ + if (buf->head->datalen < 8) + return 0; + + if (data[1] != 0x5a) { + *reason = tor_strdup(socks4_response_code_to_string(data[1])); + return -1; + } + + /* Success */ + buf_remove_from_front(buf, 8); + return 1; + + case PROXY_SOCKS5_WANT_AUTH_METHOD_NONE: + /* we don't have any credentials */ + if (data[1] != 0x00) { + *reason = tor_strdup("server doesn't support any of our " + "available authentication methods"); + return -1; + } + + log_info(LD_NET, "SOCKS 5 client: continuing without authentication"); + buf_clear(buf); + return 1; + + case PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929: + /* we have a username and password. return 1 if we can proceed without + * providing authentication, or 2 otherwise. */ + switch (data[1]) { + case 0x00: + log_info(LD_NET, "SOCKS 5 client: we have auth details but server " + "doesn't require authentication."); + buf_clear(buf); + return 1; + case 0x02: + log_info(LD_NET, "SOCKS 5 client: need authentication."); + buf_clear(buf); + return 2; + /* fall through */ + } + + *reason = tor_strdup("server doesn't support any of our available " + "authentication methods"); + return -1; + + case PROXY_SOCKS5_WANT_AUTH_RFC1929_OK: + /* handle server reply to rfc1929 authentication */ + if (data[1] != 0x00) { + *reason = tor_strdup("authentication failed"); + return -1; + } + + log_info(LD_NET, "SOCKS 5 client: authentication successful."); + buf_clear(buf); + return 1; + + case PROXY_SOCKS5_WANT_CONNECT_OK: + /* response is variable length. BND.ADDR, etc, isn't needed + * (don't bother with buf_pullup()), but make sure to eat all + * the data used */ + + /* wait for address type field to arrive */ + if (buf->datalen < 4) + return 0; + + switch (data[3]) { + case 0x01: /* ip4 */ + addrlen = 4; + break; + case 0x04: /* ip6 */ + addrlen = 16; + break; + case 0x03: /* fqdn (can this happen here?) */ + if (buf->datalen < 5) + return 0; + addrlen = 1 + data[4]; + break; + default: + *reason = tor_strdup("invalid response to connect request"); + return -1; + } + + /* wait for address and port */ + if (buf->datalen < 6 + addrlen) + return 0; + + if (data[1] != 0x00) { + *reason = tor_strdup(socks5_response_code_to_string(data[1])); + return -1; + } + + buf_remove_from_front(buf, 6 + addrlen); + return 1; + } + + /* shouldn't get here... */ + tor_assert(0); + + return -1; +} + /** Return 1 iff buf looks more like it has an (obsolete) v0 controller * command on it than any valid v1 controller command. */ int diff --git a/src/or/buffers.h b/src/or/buffers.h new file mode 100644 index 0000000000..e50b9ff6fb --- /dev/null +++ b/src/or/buffers.h @@ -0,0 +1,58 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file buffers.h + * \brief Header file for buffers.c. + **/ + +#ifndef _TOR_BUFFERS_H +#define _TOR_BUFFERS_H + +buf_t *buf_new(void); +buf_t *buf_new_with_capacity(size_t size); +void buf_free(buf_t *buf); +void buf_clear(buf_t *buf); +void buf_shrink(buf_t *buf); +void buf_shrink_freelists(int free_all); +void buf_dump_freelist_sizes(int severity); + +size_t buf_datalen(const buf_t *buf); +size_t buf_allocation(const buf_t *buf); +size_t buf_slack(const buf_t *buf); + +int read_to_buf(int s, size_t at_most, buf_t *buf, int *reached_eof, + int *socket_error); +int read_to_buf_tls(tor_tls_t *tls, size_t at_most, buf_t *buf); + +int flush_buf(int s, buf_t *buf, size_t sz, size_t *buf_flushlen); +int flush_buf_tls(tor_tls_t *tls, buf_t *buf, size_t sz, size_t *buf_flushlen); + +int write_to_buf(const char *string, size_t string_len, buf_t *buf); +int write_to_buf_zlib(buf_t *buf, tor_zlib_state_t *state, + const char *data, size_t data_len, int done); +int move_buf_to_buf(buf_t *buf_out, buf_t *buf_in, size_t *buf_flushlen); +int fetch_from_buf(char *string, size_t string_len, buf_t *buf); +int fetch_var_cell_from_buf(buf_t *buf, var_cell_t **out, int linkproto); +int fetch_from_buf_http(buf_t *buf, + char **headers_out, size_t max_headerlen, + char **body_out, size_t *body_used, size_t max_bodylen, + int force_complete); +int fetch_from_buf_socks(buf_t *buf, socks_request_t *req, + int log_sockstype, int safe_socks); +int fetch_from_buf_socks_client(buf_t *buf, int state, char **reason); +int fetch_from_buf_line(buf_t *buf, char *data_out, size_t *data_len); + +int peek_buf_has_control0_command(buf_t *buf); + +void assert_buf_ok(buf_t *buf); + +#ifdef BUFFERS_PRIVATE +int buf_find_string_offset(const buf_t *buf, const char *s, size_t n); +#endif + +#endif + diff --git a/src/or/circuitbuild.c b/src/or/circuitbuild.c index 208a9cb11b..108007e384 100644 --- a/src/or/circuitbuild.c +++ b/src/or/circuitbuild.c @@ -9,9 +9,46 @@ * \brief The actual details of building circuits. **/ +#define CIRCUIT_PRIVATE + #include "or.h" +#include "circuitbuild.h" +#include "circuitlist.h" +#include "circuituse.h" +#include "config.h" +#include "connection.h" +#include "connection_edge.h" +#include "connection_or.h" +#include "control.h" +#include "directory.h" +#include "main.h" +#include "networkstatus.h" +#include "onion.h" +#include "policies.h" +#include "relay.h" +#include "rephist.h" +#include "router.h" +#include "routerlist.h" +#include "routerparse.h" +#include "crypto.h" +#undef log +#include <math.h> + +#ifndef MIN +#define MIN(a,b) ((a)<(b)?(a):(b)) +#endif + +#define CBT_BIN_TO_MS(bin) ((bin)*CBT_BIN_WIDTH + (CBT_BIN_WIDTH/2)) /********* START VARIABLES **********/ +/** Global list of circuit build times */ +// XXXX023: Add this as a member for entry_guard_t instead of global? +// Then we could do per-guard statistics, as guards are likely to +// vary in their own latency. The downside of this is that guards +// can change frequently, so we'd be building a lot more circuits +// most likely. +/* XXXX023 Make this static; add accessor functions. */ +circuit_build_times_t circ_times; /** A global list of all circuits at this hop. */ extern circuit_t *global_circuitlist; @@ -47,6 +84,10 @@ static smartlist_t *entry_guards = NULL; * and those changes need to be flushed to disk. */ static int entry_guards_dirty = 0; +/** If set, we're running the unit tests: we should avoid clobbering + * our state file or accessing get_options() or get_or_state() */ +static int unit_tests = 0; + /********* END VARIABLES ************/ static int circuit_deliver_create_cell(circuit_t *circ, @@ -59,6 +100,1358 @@ static int onion_append_hop(crypt_path_t **head_ptr, extend_info_t *choice); static void entry_guards_changed(void); +/** + * This function decides if CBT learning should be disabled. It returns + * true if one or more of the following four conditions are met: + * + * 1. If the cbtdisabled consensus parameter is set. + * 2. If the torrc option LearnCircuitBuildTimeout is false. + * 3. If we are a directory authority + * 4. If we fail to write circuit build time history to our state file. + */ +static int +circuit_build_times_disabled(void) +{ + if (unit_tests) { + return 0; + } else { + int consensus_disabled = networkstatus_get_param(NULL, "cbtdisabled", + 0, 0, 1); + int config_disabled = !get_options()->LearnCircuitBuildTimeout; + int dirauth_disabled = get_options()->AuthoritativeDir; + int state_disabled = did_last_state_file_write_fail() ? 1 : 0; + + if (consensus_disabled || config_disabled || dirauth_disabled || + state_disabled) { + log_info(LD_CIRC, + "CircuitBuildTime learning is disabled. " + "Consensus=%d, Config=%d, AuthDir=%d, StateFile=%d", + consensus_disabled, config_disabled, dirauth_disabled, + state_disabled); + return 1; + } else { + return 0; + } + } +} + +/** + * Retrieve and bounds-check the cbtmaxtimeouts consensus paramter. + * + * Effect: When this many timeouts happen in the last 'cbtrecentcount' + * circuit attempts, the client should discard all of its history and + * begin learning a fresh timeout value. + */ +static int32_t +circuit_build_times_max_timeouts(void) +{ + return networkstatus_get_param(NULL, "cbtmaxtimeouts", + CBT_DEFAULT_MAX_RECENT_TIMEOUT_COUNT, + CBT_MIN_MAX_RECENT_TIMEOUT_COUNT, + CBT_MAX_MAX_RECENT_TIMEOUT_COUNT); +} + +/** + * Retrieve and bounds-check the cbtnummodes consensus paramter. + * + * Effect: This value governs how many modes to use in the weighted + * average calculation of Pareto parameter Xm. A value of 3 introduces + * some bias (2-5% of CDF) under ideal conditions, but allows for better + * performance in the event that a client chooses guard nodes of radically + * different performance characteristics. + */ +static int32_t +circuit_build_times_default_num_xm_modes(void) +{ + int32_t num = networkstatus_get_param(NULL, "cbtnummodes", + CBT_DEFAULT_NUM_XM_MODES, + CBT_MIN_NUM_XM_MODES, + CBT_MAX_NUM_XM_MODES); + return num; +} + +/** + * Retrieve and bounds-check the cbtmincircs consensus paramter. + * + * Effect: This is the minimum number of circuits to build before + * computing a timeout. + */ +static int32_t +circuit_build_times_min_circs_to_observe(void) +{ + int32_t num = networkstatus_get_param(NULL, "cbtmincircs", + CBT_DEFAULT_MIN_CIRCUITS_TO_OBSERVE, + CBT_MIN_MIN_CIRCUITS_TO_OBSERVE, + CBT_MAX_MIN_CIRCUITS_TO_OBSERVE); + return num; +} + +/** Return true iff <b>cbt</b> has recorded enough build times that we + * want to start acting on the timeout it implies. */ +int +circuit_build_times_enough_to_compute(circuit_build_times_t *cbt) +{ + return cbt->total_build_times >= circuit_build_times_min_circs_to_observe(); +} + +/** + * Retrieve and bounds-check the cbtquantile consensus paramter. + * + * Effect: This is the position on the quantile curve to use to set the + * timeout value. It is a percent (10-99). + */ +double +circuit_build_times_quantile_cutoff(void) +{ + int32_t num = networkstatus_get_param(NULL, "cbtquantile", + CBT_DEFAULT_QUANTILE_CUTOFF, + CBT_MIN_QUANTILE_CUTOFF, + CBT_MAX_QUANTILE_CUTOFF); + return num/100.0; +} + +int +circuit_build_times_get_bw_scale(networkstatus_t *ns) +{ + return networkstatus_get_param(ns, "bwweightscale", + BW_WEIGHT_SCALE, + BW_MIN_WEIGHT_SCALE, + BW_MAX_WEIGHT_SCALE); +} + +/** + * Retrieve and bounds-check the cbtclosequantile consensus paramter. + * + * Effect: This is the position on the quantile curve to use to set the + * timeout value to use to actually close circuits. It is a percent + * (0-99). + */ +static double +circuit_build_times_close_quantile(void) +{ + int32_t param; + /* Cast is safe - circuit_build_times_quantile_cutoff() is capped */ + int32_t min = (int)tor_lround(100*circuit_build_times_quantile_cutoff()); + param = networkstatus_get_param(NULL, "cbtclosequantile", + CBT_DEFAULT_CLOSE_QUANTILE, + CBT_MIN_CLOSE_QUANTILE, + CBT_MAX_CLOSE_QUANTILE); + if (param < min) { + log_warn(LD_DIR, "Consensus parameter cbtclosequantile is " + "too small, raising to %d", min); + param = min; + } + return param / 100.0; +} + +/** + * Retrieve and bounds-check the cbttestfreq consensus paramter. + * + * Effect: Describes how often in seconds to build a test circuit to + * gather timeout values. Only applies if less than 'cbtmincircs' + * have been recorded. + */ +static int32_t +circuit_build_times_test_frequency(void) +{ + int32_t num = networkstatus_get_param(NULL, "cbttestfreq", + CBT_DEFAULT_TEST_FREQUENCY, + CBT_MIN_TEST_FREQUENCY, + CBT_MAX_TEST_FREQUENCY); + return num; +} + +/** + * Retrieve and bounds-check the cbtmintimeout consensus paramter. + * + * Effect: This is the minimum allowed timeout value in milliseconds. + * The minimum is to prevent rounding to 0 (we only check once + * per second). + */ +static int32_t +circuit_build_times_min_timeout(void) +{ + int32_t num = networkstatus_get_param(NULL, "cbtmintimeout", + CBT_DEFAULT_TIMEOUT_MIN_VALUE, + CBT_MIN_TIMEOUT_MIN_VALUE, + CBT_MAX_TIMEOUT_MIN_VALUE); + return num; +} + +/** + * Retrieve and bounds-check the cbtinitialtimeout consensus paramter. + * + * Effect: This is the timeout value to use before computing a timeout, + * in milliseconds. + */ +int32_t +circuit_build_times_initial_timeout(void) +{ + int32_t min = circuit_build_times_min_timeout(); + int32_t param = networkstatus_get_param(NULL, "cbtinitialtimeout", + CBT_DEFAULT_TIMEOUT_INITIAL_VALUE, + CBT_MIN_TIMEOUT_INITIAL_VALUE, + CBT_MAX_TIMEOUT_INITIAL_VALUE); + if (param < min) { + log_warn(LD_DIR, "Consensus parameter cbtinitialtimeout is too small, " + "raising to %d", min); + param = min; + } + return param; +} + +/** + * Retrieve and bounds-check the cbtrecentcount consensus paramter. + * + * Effect: This is the number of circuit build times to keep track of + * for deciding if we hit cbtmaxtimeouts and need to reset our state + * and learn a new timeout. + */ +static int32_t +circuit_build_times_recent_circuit_count(networkstatus_t *ns) +{ + return networkstatus_get_param(ns, "cbtrecentcount", + CBT_DEFAULT_RECENT_CIRCUITS, + CBT_MIN_RECENT_CIRCUITS, + CBT_MAX_RECENT_CIRCUITS); +} + +/** + * This function is called when we get a consensus update. + * + * It checks to see if we have changed any consensus parameters + * that require reallocation or discard of previous stats. + */ +void +circuit_build_times_new_consensus_params(circuit_build_times_t *cbt, + networkstatus_t *ns) +{ + int32_t num = circuit_build_times_recent_circuit_count(ns); + + if (num > 0 && num != cbt->liveness.num_recent_circs) { + int8_t *recent_circs; + log_notice(LD_CIRC, "The Tor Directory Consensus has changed how many " + "circuits we must track to detect network failures from %d " + "to %d.", cbt->liveness.num_recent_circs, num); + + tor_assert(cbt->liveness.timeouts_after_firsthop); + + /* + * Technically this is a circular array that we are reallocating + * and memcopying. However, since it only consists of either 1s + * or 0s, and is only used in a statistical test to determine when + * we should discard our history after a sufficient number of 1's + * have been reached, it is fine if order is not preserved or + * elements are lost. + * + * cbtrecentcount should only be changing in cases of severe network + * distress anyway, so memory correctness here is paramount over + * doing acrobatics to preserve the array. + */ + recent_circs = tor_malloc_zero(sizeof(int8_t)*num); + memcpy(recent_circs, cbt->liveness.timeouts_after_firsthop, + sizeof(int8_t)*MIN(num, cbt->liveness.num_recent_circs)); + + // Adjust the index if it needs it. + if (num < cbt->liveness.num_recent_circs) { + cbt->liveness.after_firsthop_idx = MIN(num-1, + cbt->liveness.after_firsthop_idx); + } + + tor_free(cbt->liveness.timeouts_after_firsthop); + cbt->liveness.timeouts_after_firsthop = recent_circs; + cbt->liveness.num_recent_circs = num; + } +} + +/** Make a note that we're running unit tests (rather than running Tor + * itself), so we avoid clobbering our state file. */ +void +circuitbuild_running_unit_tests(void) +{ + unit_tests = 1; +} + +/** + * Return the initial default or configured timeout in milliseconds + */ +static double +circuit_build_times_get_initial_timeout(void) +{ + double timeout; + if (!unit_tests && get_options()->CircuitBuildTimeout) { + timeout = get_options()->CircuitBuildTimeout*1000; + if (timeout < circuit_build_times_min_timeout()) { + log_warn(LD_CIRC, "Config CircuitBuildTimeout too low. Setting to %ds", + circuit_build_times_min_timeout()/1000); + timeout = circuit_build_times_min_timeout(); + } + } else { + timeout = circuit_build_times_initial_timeout(); + } + return timeout; +} + +/** + * Reset the build time state. + * + * Leave estimated parameters, timeout and network liveness intact + * for future use. + */ +void +circuit_build_times_reset(circuit_build_times_t *cbt) +{ + memset(cbt->circuit_build_times, 0, sizeof(cbt->circuit_build_times)); + cbt->total_build_times = 0; + cbt->build_times_idx = 0; + cbt->have_computed_timeout = 0; +} + +/** + * Initialize the buildtimes structure for first use. + * + * Sets the initial timeout values based on either the config setting, + * the consensus param, or the default (CBT_DEFAULT_TIMEOUT_INITIAL_VALUE). + */ +void +circuit_build_times_init(circuit_build_times_t *cbt) +{ + memset(cbt, 0, sizeof(*cbt)); + cbt->liveness.num_recent_circs = + circuit_build_times_recent_circuit_count(NULL); + cbt->liveness.timeouts_after_firsthop = tor_malloc_zero(sizeof(int8_t)* + cbt->liveness.num_recent_circs); + cbt->close_ms = cbt->timeout_ms = circuit_build_times_get_initial_timeout(); + control_event_buildtimeout_set(cbt, BUILDTIMEOUT_SET_EVENT_RESET); +} + +#if 0 +/** + * Rewind our build time history by n positions. + */ +static void +circuit_build_times_rewind_history(circuit_build_times_t *cbt, int n) +{ + int i = 0; + + cbt->build_times_idx -= n; + cbt->build_times_idx %= CBT_NCIRCUITS_TO_OBSERVE; + + for (i = 0; i < n; i++) { + cbt->circuit_build_times[(i+cbt->build_times_idx) + %CBT_NCIRCUITS_TO_OBSERVE]=0; + } + + if (cbt->total_build_times > n) { + cbt->total_build_times -= n; + } else { + cbt->total_build_times = 0; + } + + log_info(LD_CIRC, + "Rewound history by %d places. Current index: %d. " + "Total: %d", n, cbt->build_times_idx, cbt->total_build_times); +} +#endif + +/** + * Add a new build time value <b>time</b> to the set of build times. Time + * units are milliseconds. + * + * circuit_build_times <b>cbt</b> is a circular array, so loop around when + * array is full. + */ +int +circuit_build_times_add_time(circuit_build_times_t *cbt, build_time_t time) +{ + if (time <= 0 || time > CBT_BUILD_TIME_MAX) { + log_warn(LD_BUG, "Circuit build time is too large (%u)." + "This is probably a bug.", time); + tor_fragile_assert(); + return -1; + } + + log_debug(LD_CIRC, "Adding circuit build time %u", time); + + cbt->circuit_build_times[cbt->build_times_idx] = time; + cbt->build_times_idx = (cbt->build_times_idx + 1) % CBT_NCIRCUITS_TO_OBSERVE; + if (cbt->total_build_times < CBT_NCIRCUITS_TO_OBSERVE) + cbt->total_build_times++; + + if ((cbt->total_build_times % CBT_SAVE_STATE_EVERY) == 0) { + /* Save state every n circuit builds */ + if (!unit_tests && !get_options()->AvoidDiskWrites) + or_state_mark_dirty(get_or_state(), 0); + } + + return 0; +} + +/** + * Return maximum circuit build time + */ +static build_time_t +circuit_build_times_max(circuit_build_times_t *cbt) +{ + int i = 0; + build_time_t max_build_time = 0; + for (i = 0; i < CBT_NCIRCUITS_TO_OBSERVE; i++) { + if (cbt->circuit_build_times[i] > max_build_time + && cbt->circuit_build_times[i] != CBT_BUILD_ABANDONED) + max_build_time = cbt->circuit_build_times[i]; + } + return max_build_time; +} + +#if 0 +/** Return minimum circuit build time */ +build_time_t +circuit_build_times_min(circuit_build_times_t *cbt) +{ + int i = 0; + build_time_t min_build_time = CBT_BUILD_TIME_MAX; + for (i = 0; i < CBT_NCIRCUITS_TO_OBSERVE; i++) { + if (cbt->circuit_build_times[i] && /* 0 <-> uninitialized */ + cbt->circuit_build_times[i] < min_build_time) + min_build_time = cbt->circuit_build_times[i]; + } + if (min_build_time == CBT_BUILD_TIME_MAX) { + log_warn(LD_CIRC, "No build times less than CBT_BUILD_TIME_MAX!"); + } + return min_build_time; +} +#endif + +/** + * Calculate and return a histogram for the set of build times. + * + * Returns an allocated array of histrogram bins representing + * the frequency of index*CBT_BIN_WIDTH millisecond + * build times. Also outputs the number of bins in nbins. + * + * The return value must be freed by the caller. + */ +static uint32_t * +circuit_build_times_create_histogram(circuit_build_times_t *cbt, + build_time_t *nbins) +{ + uint32_t *histogram; + build_time_t max_build_time = circuit_build_times_max(cbt); + int i, c; + + *nbins = 1 + (max_build_time / CBT_BIN_WIDTH); + histogram = tor_malloc_zero(*nbins * sizeof(build_time_t)); + + // calculate histogram + for (i = 0; i < CBT_NCIRCUITS_TO_OBSERVE; i++) { + if (cbt->circuit_build_times[i] == 0 + || cbt->circuit_build_times[i] == CBT_BUILD_ABANDONED) + continue; /* 0 <-> uninitialized */ + + c = (cbt->circuit_build_times[i] / CBT_BIN_WIDTH); + histogram[c]++; + } + + return histogram; +} + +/** + * Return the Pareto start-of-curve parameter Xm. + * + * Because we are not a true Pareto curve, we compute this as the + * weighted average of the N most frequent build time bins. N is either + * 1 if we don't have enough circuit build time data collected, or + * determined by the consensus parameter cbtnummodes (default 3). + */ +static build_time_t +circuit_build_times_get_xm(circuit_build_times_t *cbt) +{ + build_time_t i, nbins; + build_time_t *nth_max_bin; + int32_t bin_counts=0; + build_time_t ret = 0; + uint32_t *histogram = circuit_build_times_create_histogram(cbt, &nbins); + int n=0; + int num_modes = circuit_build_times_default_num_xm_modes(); + + tor_assert(nbins > 0); + tor_assert(num_modes > 0); + + // Only use one mode if < 1000 buildtimes. Not enough data + // for multiple. + if (cbt->total_build_times < CBT_NCIRCUITS_TO_OBSERVE) + num_modes = 1; + + nth_max_bin = (build_time_t*)tor_malloc_zero(num_modes*sizeof(build_time_t)); + + /* Determine the N most common build times */ + for (i = 0; i < nbins; i++) { + if (histogram[i] >= histogram[nth_max_bin[0]]) { + nth_max_bin[0] = i; + } + + for (n = 1; n < num_modes; n++) { + if (histogram[i] >= histogram[nth_max_bin[n]] && + (!histogram[nth_max_bin[n-1]] + || histogram[i] < histogram[nth_max_bin[n-1]])) { + nth_max_bin[n] = i; + } + } + } + + for (n = 0; n < num_modes; n++) { + bin_counts += histogram[nth_max_bin[n]]; + ret += CBT_BIN_TO_MS(nth_max_bin[n])*histogram[nth_max_bin[n]]; + log_info(LD_CIRC, "Xm mode #%d: %u %u", n, CBT_BIN_TO_MS(nth_max_bin[n]), + histogram[nth_max_bin[n]]); + } + + /* The following assert is safe, because we don't get called when we + * haven't observed at least CBT_MIN_MIN_CIRCUITS_TO_OBSERVE circuits. */ + tor_assert(bin_counts > 0); + + ret /= bin_counts; + tor_free(histogram); + tor_free(nth_max_bin); + + return ret; +} + +/** + * Output a histogram of current circuit build times to + * the or_state_t state structure. + */ +void +circuit_build_times_update_state(circuit_build_times_t *cbt, + or_state_t *state) +{ + uint32_t *histogram; + build_time_t i = 0; + build_time_t nbins = 0; + config_line_t **next, *line; + + histogram = circuit_build_times_create_histogram(cbt, &nbins); + // write to state + config_free_lines(state->BuildtimeHistogram); + next = &state->BuildtimeHistogram; + *next = NULL; + + state->TotalBuildTimes = cbt->total_build_times; + state->CircuitBuildAbandonedCount = 0; + + for (i = 0; i < CBT_NCIRCUITS_TO_OBSERVE; i++) { + if (cbt->circuit_build_times[i] == CBT_BUILD_ABANDONED) + state->CircuitBuildAbandonedCount++; + } + + for (i = 0; i < nbins; i++) { + // compress the histogram by skipping the blanks + if (histogram[i] == 0) continue; + *next = line = tor_malloc_zero(sizeof(config_line_t)); + line->key = tor_strdup("CircuitBuildTimeBin"); + line->value = tor_malloc(25); + tor_snprintf(line->value, 25, "%d %d", + CBT_BIN_TO_MS(i), histogram[i]); + next = &(line->next); + } + + if (!unit_tests) { + if (!get_options()->AvoidDiskWrites) + or_state_mark_dirty(get_or_state(), 0); + } + + tor_free(histogram); +} + +/** + * Shuffle the build times array. + * + * Adapted from http://en.wikipedia.org/wiki/Fisher-Yates_shuffle + */ +static void +circuit_build_times_shuffle_and_store_array(circuit_build_times_t *cbt, + build_time_t *raw_times, + uint32_t num_times) +{ + uint32_t n = num_times; + if (num_times > CBT_NCIRCUITS_TO_OBSERVE) { + log_notice(LD_CIRC, "The number of circuit times that this Tor version " + "uses to calculate build times is less than the number stored " + "in your state file. Decreasing the circuit time history from " + "%d to %d.", num_times, CBT_NCIRCUITS_TO_OBSERVE); + } + + /* This code can only be run on a compact array */ + while (n-- > 1) { + int k = crypto_rand_int(n + 1); /* 0 <= k <= n. */ + build_time_t tmp = raw_times[k]; + raw_times[k] = raw_times[n]; + raw_times[n] = tmp; + } + + /* Since the times are now shuffled, take a random CBT_NCIRCUITS_TO_OBSERVE + * subset (ie the first CBT_NCIRCUITS_TO_OBSERVE values) */ + for (n = 0; n < MIN(num_times, CBT_NCIRCUITS_TO_OBSERVE); n++) { + circuit_build_times_add_time(cbt, raw_times[n]); + } +} + +/** + * Filter old synthetic timeouts that were created before the + * new right-censored Pareto calculation was deployed. + * + * Once all clients before 0.2.1.13-alpha are gone, this code + * will be unused. + */ +static int +circuit_build_times_filter_timeouts(circuit_build_times_t *cbt) +{ + int num_filtered=0, i=0; + double timeout_rate = 0; + build_time_t max_timeout = 0; + + timeout_rate = circuit_build_times_timeout_rate(cbt); + max_timeout = (build_time_t)cbt->close_ms; + + for (i = 0; i < CBT_NCIRCUITS_TO_OBSERVE; i++) { + if (cbt->circuit_build_times[i] > max_timeout) { + build_time_t replaced = cbt->circuit_build_times[i]; + num_filtered++; + cbt->circuit_build_times[i] = CBT_BUILD_ABANDONED; + + log_debug(LD_CIRC, "Replaced timeout %d with %d", replaced, + cbt->circuit_build_times[i]); + } + } + + log_info(LD_CIRC, + "We had %d timeouts out of %d build times, " + "and filtered %d above the max of %u", + (int)(cbt->total_build_times*timeout_rate), + cbt->total_build_times, num_filtered, max_timeout); + + return num_filtered; +} + +/** + * Load histogram from <b>state</b>, shuffling the resulting array + * after we do so. Use this result to estimate parameters and + * calculate the timeout. + * + * Return -1 on error. + */ +int +circuit_build_times_parse_state(circuit_build_times_t *cbt, + or_state_t *state) +{ + int tot_values = 0; + uint32_t loaded_cnt = 0, N = 0; + config_line_t *line; + unsigned int i; + build_time_t *loaded_times; + int err = 0; + circuit_build_times_init(cbt); + + if (circuit_build_times_disabled()) { + return 0; + } + + /* build_time_t 0 means uninitialized */ + loaded_times = tor_malloc_zero(sizeof(build_time_t)*state->TotalBuildTimes); + + for (line = state->BuildtimeHistogram; line; line = line->next) { + smartlist_t *args = smartlist_create(); + smartlist_split_string(args, line->value, " ", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + if (smartlist_len(args) < 2) { + log_warn(LD_GENERAL, "Unable to parse circuit build times: " + "Too few arguments to CircuitBuildTime"); + err = 1; + SMARTLIST_FOREACH(args, char*, cp, tor_free(cp)); + smartlist_free(args); + break; + } else { + const char *ms_str = smartlist_get(args,0); + const char *count_str = smartlist_get(args,1); + uint32_t count, k; + build_time_t ms; + int ok; + ms = (build_time_t)tor_parse_ulong(ms_str, 0, 0, + CBT_BUILD_TIME_MAX, &ok, NULL); + if (!ok) { + log_warn(LD_GENERAL, "Unable to parse circuit build times: " + "Unparsable bin number"); + err = 1; + SMARTLIST_FOREACH(args, char*, cp, tor_free(cp)); + smartlist_free(args); + break; + } + count = (uint32_t)tor_parse_ulong(count_str, 0, 0, + UINT32_MAX, &ok, NULL); + if (!ok) { + log_warn(LD_GENERAL, "Unable to parse circuit build times: " + "Unparsable bin count"); + err = 1; + SMARTLIST_FOREACH(args, char*, cp, tor_free(cp)); + smartlist_free(args); + break; + } + + if (loaded_cnt+count+state->CircuitBuildAbandonedCount + > state->TotalBuildTimes) { + log_warn(LD_CIRC, + "Too many build times in state file. " + "Stopping short before %d", + loaded_cnt+count); + SMARTLIST_FOREACH(args, char*, cp, tor_free(cp)); + smartlist_free(args); + break; + } + + for (k = 0; k < count; k++) { + loaded_times[loaded_cnt++] = ms; + } + N++; + SMARTLIST_FOREACH(args, char*, cp, tor_free(cp)); + smartlist_free(args); + } + } + + log_info(LD_CIRC, + "Adding %d timeouts.", state->CircuitBuildAbandonedCount); + for (i=0; i < state->CircuitBuildAbandonedCount; i++) { + loaded_times[loaded_cnt++] = CBT_BUILD_ABANDONED; + } + + if (loaded_cnt != state->TotalBuildTimes) { + log_warn(LD_CIRC, + "Corrupt state file? Build times count mismatch. " + "Read %d times, but file says %d", loaded_cnt, + state->TotalBuildTimes); + err = 1; + circuit_build_times_reset(cbt); + goto done; + } + + circuit_build_times_shuffle_and_store_array(cbt, loaded_times, loaded_cnt); + + /* Verify that we didn't overwrite any indexes */ + for (i=0; i < CBT_NCIRCUITS_TO_OBSERVE; i++) { + if (!cbt->circuit_build_times[i]) + break; + tot_values++; + } + log_info(LD_CIRC, + "Loaded %d/%d values from %d lines in circuit time histogram", + tot_values, cbt->total_build_times, N); + + if (cbt->total_build_times != tot_values + || cbt->total_build_times > CBT_NCIRCUITS_TO_OBSERVE) { + log_warn(LD_CIRC, + "Corrupt state file? Shuffled build times mismatch. " + "Read %d times, but file says %d", tot_values, + state->TotalBuildTimes); + err = 1; + circuit_build_times_reset(cbt); + goto done; + } + + circuit_build_times_set_timeout(cbt); + + if (!state->CircuitBuildAbandonedCount && cbt->total_build_times) { + circuit_build_times_filter_timeouts(cbt); + } + + done: + tor_free(loaded_times); + return err ? -1 : 0; +} + +/** + * Estimates the Xm and Alpha parameters using + * http://en.wikipedia.org/wiki/Pareto_distribution#Parameter_estimation + * + * The notable difference is that we use mode instead of min to estimate Xm. + * This is because our distribution is frechet-like. We claim this is + * an acceptable approximation because we are only concerned with the + * accuracy of the CDF of the tail. + */ +int +circuit_build_times_update_alpha(circuit_build_times_t *cbt) +{ + build_time_t *x=cbt->circuit_build_times; + double a = 0; + int n=0,i=0,abandoned_count=0; + build_time_t max_time=0; + + /* http://en.wikipedia.org/wiki/Pareto_distribution#Parameter_estimation */ + /* We sort of cheat here and make our samples slightly more pareto-like + * and less frechet-like. */ + cbt->Xm = circuit_build_times_get_xm(cbt); + + tor_assert(cbt->Xm > 0); + + for (i=0; i< CBT_NCIRCUITS_TO_OBSERVE; i++) { + if (!x[i]) { + continue; + } + + if (x[i] < cbt->Xm) { + a += tor_mathlog(cbt->Xm); + } else if (x[i] == CBT_BUILD_ABANDONED) { + abandoned_count++; + } else { + a += tor_mathlog(x[i]); + if (x[i] > max_time) + max_time = x[i]; + } + n++; + } + + /* + * We are erring and asserting here because this can only happen + * in codepaths other than startup. The startup state parsing code + * performs this same check, and resets state if it hits it. If we + * hit it at runtime, something serious has gone wrong. + */ + if (n!=cbt->total_build_times) { + log_err(LD_CIRC, "Discrepancy in build times count: %d vs %d", n, + cbt->total_build_times); + } + tor_assert(n==cbt->total_build_times); + + if (max_time <= 0) { + /* This can happen if Xm is actually the *maximum* value in the set. + * It can also happen if we've abandoned every single circuit somehow. + * In either case, tell the caller not to compute a new build timeout. */ + log_warn(LD_BUG, + "Could not determine largest build time (%d). " + "Xm is %dms and we've abandoned %d out of %d circuits.", max_time, + cbt->Xm, abandoned_count, n); + return 0; + } + + a += abandoned_count*tor_mathlog(max_time); + + a -= n*tor_mathlog(cbt->Xm); + // Estimator comes from Eq #4 in: + // "Bayesian estimation based on trimmed samples from Pareto populations" + // by Arturo J. Fernández. We are right-censored only. + a = (n-abandoned_count)/a; + + cbt->alpha = a; + + return 1; +} + +/** + * This is the Pareto Quantile Function. It calculates the point x + * in the distribution such that F(x) = quantile (ie quantile*100% + * of the mass of the density function is below x on the curve). + * + * We use it to calculate the timeout and also to generate synthetic + * values of time for circuits that timeout before completion. + * + * See http://en.wikipedia.org/wiki/Quantile_function, + * http://en.wikipedia.org/wiki/Inverse_transform_sampling and + * http://en.wikipedia.org/wiki/Pareto_distribution#Generating_a_ + * random_sample_from_Pareto_distribution + * That's right. I'll cite wikipedia all day long. + * + * Return value is in milliseconds. + */ +double +circuit_build_times_calculate_timeout(circuit_build_times_t *cbt, + double quantile) +{ + double ret; + tor_assert(quantile >= 0); + tor_assert(1.0-quantile > 0); + tor_assert(cbt->Xm > 0); + + ret = cbt->Xm/pow(1.0-quantile,1.0/cbt->alpha); + if (ret > INT32_MAX) { + ret = INT32_MAX; + } + tor_assert(ret > 0); + return ret; +} + +/** Pareto CDF */ +double +circuit_build_times_cdf(circuit_build_times_t *cbt, double x) +{ + double ret; + tor_assert(cbt->Xm > 0); + ret = 1.0-pow(cbt->Xm/x,cbt->alpha); + tor_assert(0 <= ret && ret <= 1.0); + return ret; +} + +/** + * Generate a synthetic time using our distribution parameters. + * + * The return value will be within the [q_lo, q_hi) quantile points + * on the CDF. + */ +build_time_t +circuit_build_times_generate_sample(circuit_build_times_t *cbt, + double q_lo, double q_hi) +{ + double randval = crypto_rand_double(); + build_time_t ret; + double u; + + /* Generate between [q_lo, q_hi) */ + /*XXXX This is what nextafter is supposed to be for; we should use it on the + * platforms that support it. */ + q_hi -= 1.0/(INT32_MAX); + + tor_assert(q_lo >= 0); + tor_assert(q_hi < 1); + tor_assert(q_lo < q_hi); + + u = q_lo + (q_hi-q_lo)*randval; + + tor_assert(0 <= u && u < 1.0); + /* circuit_build_times_calculate_timeout returns <= INT32_MAX */ + ret = (build_time_t) + tor_lround(circuit_build_times_calculate_timeout(cbt, u)); + tor_assert(ret > 0); + return ret; +} + +/** + * Estimate an initial alpha parameter by solving the quantile + * function with a quantile point and a specific timeout value. + */ +void +circuit_build_times_initial_alpha(circuit_build_times_t *cbt, + double quantile, double timeout_ms) +{ + // Q(u) = Xm/((1-u)^(1/a)) + // Q(0.8) = Xm/((1-0.8))^(1/a)) = CircBuildTimeout + // CircBuildTimeout = Xm/((1-0.8))^(1/a)) + // CircBuildTimeout = Xm*((1-0.8))^(-1/a)) + // ln(CircBuildTimeout) = ln(Xm)+ln(((1-0.8)))*(-1/a) + // -ln(1-0.8)/(ln(CircBuildTimeout)-ln(Xm))=a + tor_assert(quantile >= 0); + tor_assert(cbt->Xm > 0); + cbt->alpha = tor_mathlog(1.0-quantile)/ + (tor_mathlog(cbt->Xm)-tor_mathlog(timeout_ms)); + tor_assert(cbt->alpha > 0); +} + +/** + * Returns true if we need circuits to be built + */ +int +circuit_build_times_needs_circuits(circuit_build_times_t *cbt) +{ + /* Return true if < MIN_CIRCUITS_TO_OBSERVE */ + return !circuit_build_times_enough_to_compute(cbt); +} + +/** + * Returns true if we should build a timeout test circuit + * right now. + */ +int +circuit_build_times_needs_circuits_now(circuit_build_times_t *cbt) +{ + return circuit_build_times_needs_circuits(cbt) && + approx_time()-cbt->last_circ_at > circuit_build_times_test_frequency(); +} + +/** + * Called to indicate that the network showed some signs of liveness, + * i.e. we received a cell. + * + * This is used by circuit_build_times_network_check_live() to decide + * if we should record the circuit build timeout or not. + * + * This function is called every time we receive a cell. Avoid + * syscalls, events, and other high-intensity work. + */ +void +circuit_build_times_network_is_live(circuit_build_times_t *cbt) +{ + time_t now = approx_time(); + if (cbt->liveness.nonlive_timeouts > 0) { + log_notice(LD_CIRC, + "Tor now sees network activity. Restoring circuit build " + "timeout recording. Network was down for %d seconds " + "during %d circuit attempts.", + (int)(now - cbt->liveness.network_last_live), + cbt->liveness.nonlive_timeouts); + } + cbt->liveness.network_last_live = now; + cbt->liveness.nonlive_timeouts = 0; +} + +/** + * Called to indicate that we completed a circuit. Because this circuit + * succeeded, it doesn't count as a timeout-after-the-first-hop. + * + * This is used by circuit_build_times_network_check_changed() to determine + * if we had too many recent timeouts and need to reset our learned timeout + * to something higher. + */ +void +circuit_build_times_network_circ_success(circuit_build_times_t *cbt) +{ + cbt->liveness.timeouts_after_firsthop[cbt->liveness.after_firsthop_idx] = 0; + cbt->liveness.after_firsthop_idx++; + cbt->liveness.after_firsthop_idx %= cbt->liveness.num_recent_circs; +} + +/** + * A circuit just timed out. If it failed after the first hop, record it + * in our history for later deciding if the network speed has changed. + * + * This is used by circuit_build_times_network_check_changed() to determine + * if we had too many recent timeouts and need to reset our learned timeout + * to something higher. + */ +static void +circuit_build_times_network_timeout(circuit_build_times_t *cbt, + int did_onehop) +{ + if (did_onehop) { + cbt->liveness.timeouts_after_firsthop[cbt->liveness.after_firsthop_idx]=1; + cbt->liveness.after_firsthop_idx++; + cbt->liveness.after_firsthop_idx %= cbt->liveness.num_recent_circs; + } +} + +/** + * A circuit was just forcibly closed. If there has been no recent network + * activity at all, but this circuit was launched back when we thought the + * network was live, increment the number of "nonlive" circuit timeouts. + * + * This is used by circuit_build_times_network_check_live() to decide + * if we should record the circuit build timeout or not. + */ +static void +circuit_build_times_network_close(circuit_build_times_t *cbt, + int did_onehop, time_t start_time) +{ + time_t now = time(NULL); + /* + * Check if this is a timeout that was for a circuit that spent its + * entire existence during a time where we have had no network activity. + */ + if (cbt->liveness.network_last_live < start_time) { + if (did_onehop) { + char last_live_buf[ISO_TIME_LEN+1]; + char start_time_buf[ISO_TIME_LEN+1]; + char now_buf[ISO_TIME_LEN+1]; + format_local_iso_time(last_live_buf, cbt->liveness.network_last_live); + format_local_iso_time(start_time_buf, start_time); + format_local_iso_time(now_buf, now); + log_warn(LD_BUG, + "Circuit somehow completed a hop while the network was " + "not live. Network was last live at %s, but circuit launched " + "at %s. It's now %s.", last_live_buf, start_time_buf, + now_buf); + } + cbt->liveness.nonlive_timeouts++; + if (cbt->liveness.nonlive_timeouts == 1) { + log_notice(LD_CIRC, + "Tor has not observed any network activity for the past %d " + "seconds. Disabling circuit build timeout recording.", + (int)(now - cbt->liveness.network_last_live)); + } else { + log_info(LD_CIRC, + "Got non-live timeout. Current count is: %d", + cbt->liveness.nonlive_timeouts); + } + } +} + +/** + * When the network is not live, we do not record circuit build times. + * + * The network is considered not live if there has been at least one + * circuit build that began and ended (had its close_ms measurement + * period expire) since we last received a cell. + * + * Also has the side effect of rewinding the circuit time history + * in the case of recent liveness changes. + */ +int +circuit_build_times_network_check_live(circuit_build_times_t *cbt) +{ + if (cbt->liveness.nonlive_timeouts > 0) { + return 0; + } + + return 1; +} + +/** + * Returns true if we have seen more than MAX_RECENT_TIMEOUT_COUNT of + * the past RECENT_CIRCUITS time out after the first hop. Used to detect + * if the network connection has changed significantly, and if so, + * resets our circuit build timeout to the default. + * + * Also resets the entire timeout history in this case and causes us + * to restart the process of building test circuits and estimating a + * new timeout. + */ +int +circuit_build_times_network_check_changed(circuit_build_times_t *cbt) +{ + int total_build_times = cbt->total_build_times; + int timeout_count=0; + int i; + + /* how many of our recent circuits made it to the first hop but then + * timed out? */ + for (i = 0; i < cbt->liveness.num_recent_circs; i++) { + timeout_count += cbt->liveness.timeouts_after_firsthop[i]; + } + + /* If 80% of our recent circuits are timing out after the first hop, + * we need to re-estimate a new initial alpha and timeout. */ + if (timeout_count < circuit_build_times_max_timeouts()) { + return 0; + } + + circuit_build_times_reset(cbt); + memset(cbt->liveness.timeouts_after_firsthop, 0, + sizeof(*cbt->liveness.timeouts_after_firsthop)* + cbt->liveness.num_recent_circs); + cbt->liveness.after_firsthop_idx = 0; + + /* Check to see if this has happened before. If so, double the timeout + * to give people on abysmally bad network connections a shot at access */ + if (cbt->timeout_ms >= circuit_build_times_get_initial_timeout()) { + if (cbt->timeout_ms > INT32_MAX/2 || cbt->close_ms > INT32_MAX/2) { + log_warn(LD_CIRC, "Insanely large circuit build timeout value. " + "(timeout = %lfmsec, close = %lfmsec)", + cbt->timeout_ms, cbt->close_ms); + } else { + cbt->timeout_ms *= 2; + cbt->close_ms *= 2; + } + } else { + cbt->close_ms = cbt->timeout_ms + = circuit_build_times_get_initial_timeout(); + } + + control_event_buildtimeout_set(cbt, BUILDTIMEOUT_SET_EVENT_RESET); + + log_notice(LD_CIRC, + "Your network connection speed appears to have changed. Resetting " + "timeout to %lds after %d timeouts and %d buildtimes.", + tor_lround(cbt->timeout_ms/1000), timeout_count, + total_build_times); + + return 1; +} + +/** + * Count the number of timeouts in a set of cbt data. + */ +double +circuit_build_times_timeout_rate(const circuit_build_times_t *cbt) +{ + int i=0,timeouts=0; + for (i = 0; i < CBT_NCIRCUITS_TO_OBSERVE; i++) { + if (cbt->circuit_build_times[i] >= cbt->timeout_ms) { + timeouts++; + } + } + + if (!cbt->total_build_times) + return 0; + + return ((double)timeouts)/cbt->total_build_times; +} + +/** + * Count the number of closed circuits in a set of cbt data. + */ +double +circuit_build_times_close_rate(const circuit_build_times_t *cbt) +{ + int i=0,closed=0; + for (i = 0; i < CBT_NCIRCUITS_TO_OBSERVE; i++) { + if (cbt->circuit_build_times[i] == CBT_BUILD_ABANDONED) { + closed++; + } + } + + if (!cbt->total_build_times) + return 0; + + return ((double)closed)/cbt->total_build_times; +} + +/** + * Store a timeout as a synthetic value. + * + * Returns true if the store was successful and we should possibly + * update our timeout estimate. + */ +int +circuit_build_times_count_close(circuit_build_times_t *cbt, + int did_onehop, + time_t start_time) +{ + if (circuit_build_times_disabled()) { + cbt->close_ms = cbt->timeout_ms + = circuit_build_times_get_initial_timeout(); + return 0; + } + + /* Record this force-close to help determine if the network is dead */ + circuit_build_times_network_close(cbt, did_onehop, start_time); + + /* Only count timeouts if network is live.. */ + if (!circuit_build_times_network_check_live(cbt)) { + return 0; + } + + circuit_build_times_add_time(cbt, CBT_BUILD_ABANDONED); + return 1; +} + +/** + * Update timeout counts to determine if we need to expire + * our build time history due to excessive timeouts. + * + * We do not record any actual time values at this stage; + * we are only interested in recording the fact that a timeout + * happened. We record the time values via + * circuit_build_times_count_close() and circuit_build_times_add_time(). + */ +void +circuit_build_times_count_timeout(circuit_build_times_t *cbt, + int did_onehop) +{ + if (circuit_build_times_disabled()) { + cbt->close_ms = cbt->timeout_ms + = circuit_build_times_get_initial_timeout(); + return; + } + + /* Register the fact that a timeout just occurred. */ + circuit_build_times_network_timeout(cbt, did_onehop); + + /* If there are a ton of timeouts, we should reset + * the circuit build timeout. */ + circuit_build_times_network_check_changed(cbt); +} + +/** + * Estimate a new timeout based on history and set our timeout + * variable accordingly. + */ +static int +circuit_build_times_set_timeout_worker(circuit_build_times_t *cbt) +{ + build_time_t max_time; + if (!circuit_build_times_enough_to_compute(cbt)) + return 0; + + if (!circuit_build_times_update_alpha(cbt)) + return 0; + + cbt->timeout_ms = circuit_build_times_calculate_timeout(cbt, + circuit_build_times_quantile_cutoff()); + + cbt->close_ms = circuit_build_times_calculate_timeout(cbt, + circuit_build_times_close_quantile()); + + max_time = circuit_build_times_max(cbt); + + /* Sometimes really fast guard nodes give us such a steep curve + * that this ends up being not that much greater than timeout_ms. + * Make it be at least 1 min to handle this case. */ + cbt->close_ms = MAX(cbt->close_ms, circuit_build_times_initial_timeout()); + + if (cbt->timeout_ms > max_time) { + log_notice(LD_CIRC, + "Circuit build timeout of %dms is beyond the maximum build " + "time we have ever observed. Capping it to %dms.", + (int)cbt->timeout_ms, max_time); + cbt->timeout_ms = max_time; + } + + if (max_time < INT32_MAX/2 && cbt->close_ms > 2*max_time) { + log_info(LD_CIRC, + "Circuit build measurement period of %dms is more than twice " + "the maximum build time we have ever observed. Capping it to " + "%dms.", (int)cbt->close_ms, 2*max_time); + cbt->close_ms = 2*max_time; + } + + cbt->have_computed_timeout = 1; + return 1; +} + +/** + * Exposed function to compute a new timeout. Dispatches events and + * also filters out extremely high timeout values. + */ +void +circuit_build_times_set_timeout(circuit_build_times_t *cbt) +{ + long prev_timeout = tor_lround(cbt->timeout_ms/1000); + double timeout_rate; + + if (!circuit_build_times_set_timeout_worker(cbt)) + return; + + if (cbt->timeout_ms < circuit_build_times_min_timeout()) { + log_warn(LD_CIRC, "Set buildtimeout to low value %lfms. Setting to %dms", + cbt->timeout_ms, circuit_build_times_min_timeout()); + cbt->timeout_ms = circuit_build_times_min_timeout(); + if (cbt->close_ms < cbt->timeout_ms) { + /* This shouldn't happen because of MAX() in timeout_worker above, + * but doing it just in case */ + cbt->close_ms = circuit_build_times_initial_timeout(); + } + } + + control_event_buildtimeout_set(cbt, BUILDTIMEOUT_SET_EVENT_COMPUTED); + + timeout_rate = circuit_build_times_timeout_rate(cbt); + + if (prev_timeout > tor_lround(cbt->timeout_ms/1000)) { + log_notice(LD_CIRC, + "Based on %d circuit times, it looks like we don't need to " + "wait so long for circuits to finish. We will now assume a " + "circuit is too slow to use after waiting %ld seconds.", + cbt->total_build_times, + tor_lround(cbt->timeout_ms/1000)); + log_info(LD_CIRC, + "Circuit timeout data: %lfms, %lfms, Xm: %d, a: %lf, r: %lf", + cbt->timeout_ms, cbt->close_ms, cbt->Xm, cbt->alpha, + timeout_rate); + } else if (prev_timeout < tor_lround(cbt->timeout_ms/1000)) { + log_notice(LD_CIRC, + "Based on %d circuit times, it looks like we need to wait " + "longer for circuits to finish. We will now assume a " + "circuit is too slow to use after waiting %ld seconds.", + cbt->total_build_times, + tor_lround(cbt->timeout_ms/1000)); + log_info(LD_CIRC, + "Circuit timeout data: %lfms, %lfms, Xm: %d, a: %lf, r: %lf", + cbt->timeout_ms, cbt->close_ms, cbt->Xm, cbt->alpha, + timeout_rate); + } else { + log_info(LD_CIRC, + "Set circuit build timeout to %lds (%lfms, %lfms, Xm: %d, a: %lf," + " r: %lf) based on %d circuit times", + tor_lround(cbt->timeout_ms/1000), + cbt->timeout_ms, cbt->close_ms, cbt->Xm, cbt->alpha, timeout_rate, + cbt->total_build_times); + } +} + /** Iterate over values of circ_id, starting from conn-\>next_circ_id, * and with the high bit specified by conn-\>circ_id_type, until we get * a circ_id that is not in use by any other circuit on that conn. @@ -112,21 +1505,21 @@ circuit_list_path_impl(origin_circuit_t *circ, int verbose, int verbose_names) crypt_path_t *hop; smartlist_t *elements; const char *states[] = {"closed", "waiting for keys", "open"}; - char buf[128]; char *s; elements = smartlist_create(); if (verbose) { const char *nickname = build_state_get_exit_nickname(circ->build_state); - tor_snprintf(buf, sizeof(buf), "%s%s circ (length %d%s%s):", + char *cp; + tor_asprintf(&cp, "%s%s circ (length %d%s%s):", circ->build_state->is_internal ? "internal" : "exit", circ->build_state->need_uptime ? " (high-uptime)" : "", circ->build_state->desired_path_len, - circ->_base.state == CIRCUIT_STATE_OPEN ? "" : ", exit ", + circ->_base.state == CIRCUIT_STATE_OPEN ? "" : ", last hop ", circ->_base.state == CIRCUIT_STATE_OPEN ? "" : (nickname?nickname:"*unnamed*")); - smartlist_add(elements, tor_strdup(buf)); + smartlist_add(elements, cp); } hop = circ->cpath; @@ -148,8 +1541,7 @@ circuit_list_path_impl(origin_circuit_t *circ, int verbose, int verbose_names) router_get_verbose_nickname(elt, ri); } else if ((rs = router_get_consensus_status_by_id(id))) { routerstatus_get_verbose_nickname(elt, rs); - } else if (hop->extend_info->nickname && - is_legal_nickname(hop->extend_info->nickname)) { + } else if (is_legal_nickname(hop->extend_info->nickname)) { elt[0] = '$'; base16_encode(elt+1, HEX_DIGEST_LEN+1, id, DIGEST_LEN); elt[HEX_DIGEST_LEN+1]= '~'; @@ -217,7 +1609,7 @@ void circuit_log_path(int severity, unsigned int domain, origin_circuit_t *circ) { char *s = circuit_list_path(circ,1); - log(severity,domain,"%s",s); + tor_log(severity,domain,"%s",s); tor_free(s); } @@ -269,7 +1661,7 @@ static int onion_populate_cpath(origin_circuit_t *circ) { int r; -again: + again: r = onion_extend_cpath(circ); if (r < 0) { log_info(LD_CIRC,"Generating cpath hop failed."); @@ -361,9 +1753,10 @@ circuit_handle_first_hop(origin_circuit_t *circ) if (!n_conn) { /* not currently connected in a useful way. */ - const char *name = firsthop->extend_info->nickname ? + const char *name = strlen(firsthop->extend_info->nickname) ? firsthop->extend_info->nickname : fmt_addr(&firsthop->extend_info->addr); - log_info(LD_CIRC, "Next router is %s: %s ", safe_str(name), msg?msg:"???"); + log_info(LD_CIRC, "Next router is %s: %s", + safe_str_client(name), msg?msg:"???"); circ->_base.n_hop = extend_info_dup(firsthop->extend_info); if (should_launch) { @@ -506,7 +1899,8 @@ circuit_deliver_create_cell(circuit_t *circ, uint8_t cell_type, cell.circ_id = circ->n_circ_id; memcpy(cell.payload, payload, ONIONSKIN_CHALLENGE_LEN); - append_cell_to_circuit_queue(circ, circ->n_conn, &cell, CELL_DIRECTION_OUT); + append_cell_to_circuit_queue(circ, circ->n_conn, &cell, + CELL_DIRECTION_OUT, 0); if (CIRCUIT_IS_ORIGIN(circ)) { /* mark it so it gets better rate limiting treatment. */ @@ -536,7 +1930,7 @@ inform_testing_reachability(void) "CHECKING_REACHABILITY DIRADDRESS=%s:%d", me->address, me->dir_port); } - log(LOG_NOTICE, LD_OR, "Now checking whether ORPort %s:%d%s %s reachable... " + log_notice(LD_OR, "Now checking whether ORPort %s:%d%s %s reachable... " "(this may take up to %d minutes -- look for log " "messages indicating success)", me->address, me->or_port, @@ -569,6 +1963,18 @@ should_use_create_fast_for_circuit(origin_circuit_t *circ) return 1; } +/** Return true if <b>circ</b> is the type of circuit we want to count + * timeouts from. In particular, we want it to have not completed yet + * (already completing indicates we cannibalized it), and we want it to + * have exactly three hops. + */ +int +circuit_timeout_want_to_count_circ(origin_circuit_t *circ) +{ + return !circ->has_opened + && circ->build_state->desired_path_len == DEFAULT_ROUTE_LEN; +} + /** This is the backbone function for building circuits. * * If circ's first hop is closed, then we need to build a create @@ -642,15 +2048,43 @@ circuit_send_next_onion_skin(origin_circuit_t *circ) if (!hop) { /* done building the circuit. whew. */ circuit_set_state(TO_CIRCUIT(circ), CIRCUIT_STATE_OPEN); + if (circuit_timeout_want_to_count_circ(circ)) { + struct timeval end; + long timediff; + tor_gettimeofday(&end); + timediff = tv_mdiff(&circ->_base.timestamp_created, &end); + + /* + * If the circuit build time is much greater than we would have cut + * it off at, we probably had a suspend event along this codepath, + * and we should discard the value. + */ + if (timediff < 0 || timediff > 2*circ_times.close_ms+1000) { + log_notice(LD_CIRC, "Strange value for circuit build time: %ldmsec. " + "Assuming clock jump. Purpose %d (%s)", timediff, + circ->_base.purpose, + circuit_purpose_to_string(circ->_base.purpose)); + } else if (!circuit_build_times_disabled()) { + /* Only count circuit times if the network is live */ + if (circuit_build_times_network_check_live(&circ_times)) { + circuit_build_times_add_time(&circ_times, (build_time_t)timediff); + circuit_build_times_set_timeout(&circ_times); + } + + if (circ->_base.purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) { + circuit_build_times_network_circ_success(&circ_times); + } + } + } log_info(LD_CIRC,"circuit built!"); circuit_reset_failure_count(0); if (circ->build_state->onehop_tunnel) control_event_bootstrap(BOOTSTRAP_STATUS_REQUESTING_STATUS, 0); - if (!has_completed_circuit && !circ->build_state->onehop_tunnel) { + if (!can_complete_circuit && !circ->build_state->onehop_tunnel) { or_options_t *options = get_options(); - has_completed_circuit=1; + can_complete_circuit=1; /* FFFF Log a count of known routers here */ - log(LOG_NOTICE, LD_GENERAL, + log_notice(LD_GENERAL, "Tor has successfully opened a circuit. " "Looks like client functionality is working."); control_event_bootstrap(BOOTSTRAP_STATUS_DONE, 0); @@ -662,6 +2096,10 @@ circuit_send_next_onion_skin(origin_circuit_t *circ) } circuit_rep_hist_note_result(circ); circuit_has_opened(circ); /* do other actions as necessary */ + + /* We're done with measurement circuits here. Just close them */ + if (circ->_base.purpose == CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) + circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_FINISHED); return 0; } @@ -705,13 +2143,13 @@ void circuit_note_clock_jumped(int seconds_elapsed) { int severity = server_mode(get_options()) ? LOG_WARN : LOG_NOTICE; - log(severity, LD_GENERAL, "Your system clock just jumped %d seconds %s; " + tor_log(severity, LD_GENERAL, "Your system clock just jumped %d seconds %s; " "assuming established circuits no longer work.", seconds_elapsed >=0 ? seconds_elapsed : -seconds_elapsed, seconds_elapsed >=0 ? "forward" : "backward"); control_event_general_status(LOG_WARN, "CLOCK_JUMPED TIME=%d", seconds_elapsed); - has_completed_circuit=0; /* so it'll log when it works again */ + can_complete_circuit=0; /* so it'll log when it works again */ control_event_client_status(severity, "CIRCUIT_NOT_ESTABLISHED REASON=%s", "CLOCK_JUMPED"); circuit_mark_all_unused_circs(); @@ -944,10 +2382,9 @@ circuit_finish_handshake(origin_circuit_t *circ, uint8_t reply_type, return -END_CIRC_REASON_TORPROTOCOL; } - if (hop->dh_handshake_state) { - crypto_dh_free(hop->dh_handshake_state); /* don't need it anymore */ - hop->dh_handshake_state = NULL; - } + crypto_dh_free(hop->dh_handshake_state); /* don't need it anymore */ + hop->dh_handshake_state = NULL; + memset(hop->fast_handshake_state, 0, sizeof(hop->fast_handshake_state)); if (circuit_init_cpath_crypto(hop, keys, 0)<0) { @@ -1035,8 +2472,8 @@ onionskin_answer(or_circuit_t *circ, uint8_t cell_type, const char *payload, cell_type == CELL_CREATED ? ONIONSKIN_REPLY_LEN : DIGEST_LEN*2); log_debug(LD_CIRC,"init digest forward 0x%.8x, backward 0x%.8x.", - (unsigned int)*(uint32_t*)(keys), - (unsigned int)*(uint32_t*)(keys+20)); + (unsigned int)get_uint32(keys), + (unsigned int)get_uint32(keys+20)); if (circuit_init_cpath_crypto(tmp_cpath, keys, 0)<0) { log_warn(LD_BUG,"Circuit initialization failed"); tor_free(tmp_cpath); @@ -1057,7 +2494,7 @@ onionskin_answer(or_circuit_t *circ, uint8_t cell_type, const char *payload, circ->is_first_hop = (cell_type == CELL_CREATED_FAST); append_cell_to_circuit_queue(TO_CIRCUIT(circ), - circ->p_conn, &cell, CELL_DIRECTION_IN); + circ->p_conn, &cell, CELL_DIRECTION_IN, 0); log_debug(LD_CIRC,"Finished sending 'created' cell."); if (!is_local_addr(&circ->p_conn->_base.addr) && @@ -1086,7 +2523,7 @@ new_route_len(uint8_t purpose, extend_info_t *exit, tor_assert(routers); - routelen = 3; + routelen = DEFAULT_ROUTE_LEN; if (exit && purpose != CIRCUIT_PURPOSE_TESTING && purpose != CIRCUIT_PURPOSE_S_ESTABLISH_INTRO) @@ -1151,6 +2588,8 @@ circuit_all_predicted_ports_handled(time_t now, int *need_uptime, smartlist_t *LongLivedServices = get_options()->LongLivedPorts; tor_assert(need_uptime); tor_assert(need_capacity); + // Always predict need_capacity + *need_capacity = 1; enough = (smartlist_len(sl) == 0); for (i = 0; i < smartlist_len(sl); ++i) { port = smartlist_get(sl, i); @@ -1173,6 +2612,8 @@ router_handles_some_port(routerinfo_t *router, smartlist_t *needed_ports) for (i = 0; i < smartlist_len(needed_ports); ++i) { addr_policy_result_t r; + /* alignment issues aren't a worry for this dereference, since + needed_ports is explicitly a smartlist of uint16_t's */ port = *(uint16_t *)smartlist_get(needed_ports, i); tor_assert(port); r = compare_addr_to_addr_policy(0, port, router->exit_policy); @@ -1255,9 +2696,24 @@ choose_good_exit_server_general(routerlist_t *dir, int need_uptime, n_supported[i] = -1; continue; /* skip routers that are known to be down or bad exits */ } + + if (options->_ExcludeExitNodesUnion && + routerset_contains_router(options->_ExcludeExitNodesUnion, router)) { + n_supported[i] = -1; + continue; /* user asked us not to use it, no matter what */ + } + if (options->ExitNodes && + !routerset_contains_router(options->ExitNodes, router)) { + n_supported[i] = -1; + continue; /* not one of our chosen exit nodes */ + } + if (router_is_unreliable(router, need_uptime, need_capacity, 0)) { n_supported[i] = -1; - continue; /* skip routers that are not suitable */ + continue; /* skip routers that are not suitable. Don't worry if + * this makes us reject all the possible routers: if so, + * we'll retry later in this function with need_update and + * need_capacity set to 0. */ } if (!(router->is_valid || options->_AllowInvalid & ALLOW_INVALID_EXIT)) { /* if it's invalid and we don't want it */ @@ -1316,28 +2772,21 @@ choose_good_exit_server_general(routerlist_t *dir, int need_uptime, /* If any routers definitely support any pending connections, choose one * at random. */ if (best_support > 0) { - smartlist_t *supporting = smartlist_create(), *use = smartlist_create(); + smartlist_t *supporting = smartlist_create(); for (i = 0; i < smartlist_len(dir->routers); i++) if (n_supported[i] == best_support) smartlist_add(supporting, smartlist_get(dir->routers, i)); - routersets_get_disjunction(use, supporting, options->ExitNodes, - options->_ExcludeExitNodesUnion, 1); - if (smartlist_len(use) == 0 && !options->StrictExitNodes) { - routersets_get_disjunction(use, supporting, NULL, - options->_ExcludeExitNodesUnion, 1); - } - router = routerlist_sl_choose_by_bandwidth(use, WEIGHT_FOR_EXIT); - smartlist_free(use); + router = routerlist_sl_choose_by_bandwidth(supporting, WEIGHT_FOR_EXIT); smartlist_free(supporting); } else { /* Either there are no pending connections, or no routers even seem to * possibly support any of them. Choose a router at random that satisfies * at least one predicted exit port. */ - int try; - smartlist_t *needed_ports, *supporting, *use; + int attempt; + smartlist_t *needed_ports, *supporting; if (best_support == -1) { if (need_uptime || need_capacity) { @@ -1349,42 +2798,32 @@ choose_good_exit_server_general(routerlist_t *dir, int need_uptime, tor_free(n_supported); return choose_good_exit_server_general(dir, 0, 0); } - log_notice(LD_CIRC, "All routers are down or won't exit -- choosing a " - "doomed exit at random."); + log_notice(LD_CIRC, "All routers are down or won't exit%s -- " + "choosing a doomed exit at random.", + options->_ExcludeExitNodesUnion ? " or are Excluded" : ""); } supporting = smartlist_create(); - use = smartlist_create(); needed_ports = circuit_get_unhandled_ports(time(NULL)); - for (try = 0; try < 2; try++) { + for (attempt = 0; attempt < 2; attempt++) { /* try once to pick only from routers that satisfy a needed port, * then if there are none, pick from any that support exiting. */ for (i = 0; i < smartlist_len(dir->routers); i++) { router = smartlist_get(dir->routers, i); if (n_supported[i] != -1 && - (try || router_handles_some_port(router, needed_ports))) { + (attempt || router_handles_some_port(router, needed_ports))) { // log_fn(LOG_DEBUG,"Try %d: '%s' is a possibility.", // try, router->nickname); smartlist_add(supporting, router); } } - routersets_get_disjunction(use, supporting, options->ExitNodes, - options->_ExcludeExitNodesUnion, 1); - if (smartlist_len(use) == 0 && !options->StrictExitNodes) { - routersets_get_disjunction(use, supporting, NULL, - options->_ExcludeExitNodesUnion, 1); - } - /* XXX sometimes the above results in null, when the requested - * exit node is down. we should pick it anyway. */ - router = routerlist_sl_choose_by_bandwidth(use, WEIGHT_FOR_EXIT); + router = routerlist_sl_choose_by_bandwidth(supporting, WEIGHT_FOR_EXIT); if (router) break; smartlist_clear(supporting); - smartlist_clear(use); } SMARTLIST_FOREACH(needed_ports, uint16_t *, cp, tor_free(cp)); smartlist_free(needed_ports); - smartlist_free(use); smartlist_free(supporting); } @@ -1393,10 +2832,11 @@ choose_good_exit_server_general(routerlist_t *dir, int need_uptime, log_info(LD_CIRC, "Chose exit server '%s'", router->nickname); return router; } - if (options->StrictExitNodes) { + if (options->ExitNodes) { log_warn(LD_CIRC, - "No specified exit routers seem to be running, and " - "StrictExitNodes is set: can't choose an exit."); + "No specified %sexit routers seem to be running: " + "can't choose an exit.", + options->_ExcludeExitNodesUnion ? "non-excluded " : ""); } return NULL; } @@ -1427,15 +2867,13 @@ choose_good_exit_server(uint8_t purpose, routerlist_t *dir, if (options->_AllowInvalid & ALLOW_INVALID_MIDDLE) flags |= CRN_ALLOW_INVALID; if (is_internal) /* pick it like a middle hop */ - return router_choose_random_node(NULL, NULL, - options->ExcludeNodes, flags); + return router_choose_random_node(NULL, options->ExcludeNodes, flags); else return choose_good_exit_server_general(dir,need_uptime,need_capacity); case CIRCUIT_PURPOSE_C_ESTABLISH_REND: if (options->_AllowInvalid & ALLOW_INVALID_RENDEZVOUS) flags |= CRN_ALLOW_INVALID; - return router_choose_random_node(NULL, NULL, - options->ExcludeNodes, flags); + return router_choose_random_node(NULL, options->ExcludeNodes, flags); } log_warn(LD_BUG,"Unhandled purpose %d", purpose); tor_fragile_assert(); @@ -1450,7 +2888,6 @@ warn_if_last_router_excluded(origin_circuit_t *circ, const extend_info_t *exit) or_options_t *options = get_options(); routerset_t *rs = options->ExcludeNodes; const char *description; - int domain = LD_CIRC; uint8_t purpose = circ->_base.purpose; if (circ->build_state->onehop_tunnel) @@ -1463,13 +2900,14 @@ warn_if_last_router_excluded(origin_circuit_t *circ, const extend_info_t *exit) case CIRCUIT_PURPOSE_INTRO_POINT: case CIRCUIT_PURPOSE_REND_POINT_WAITING: case CIRCUIT_PURPOSE_REND_ESTABLISHED: - log_warn(LD_BUG, "Called on non-origin circuit (purpose %d)", - (int)purpose); + log_warn(LD_BUG, "Called on non-origin circuit (purpose %d, %s)", + (int)purpose, + circuit_purpose_to_string(purpose)); return; case CIRCUIT_PURPOSE_C_GENERAL: if (circ->build_state->is_internal) return; - description = "Requested exit node"; + description = "requested exit node"; rs = options->_ExcludeExitNodesUnion; break; case CIRCUIT_PURPOSE_C_INTRODUCING: @@ -1484,22 +2922,34 @@ warn_if_last_router_excluded(origin_circuit_t *circ, const extend_info_t *exit) case CIRCUIT_PURPOSE_C_REND_READY: case CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED: case CIRCUIT_PURPOSE_C_REND_JOINED: - description = "Chosen rendezvous point"; - domain = LD_BUG; + description = "chosen rendezvous point"; break; case CIRCUIT_PURPOSE_CONTROLLER: rs = options->_ExcludeExitNodesUnion; - description = "Controller-selected circuit target"; + description = "controller-selected circuit target"; break; } if (routerset_contains_extendinfo(rs, exit)) { - log_fn(LOG_WARN, domain, "%s '%s' is in ExcludeNodes%s. Using anyway " - "(circuit purpose %d).", - description,exit->nickname, - rs==options->ExcludeNodes?"":" or ExcludeExitNodes", - (int)purpose); - circuit_log_path(LOG_WARN, domain, circ); + /* We should never get here if StrictNodes is set to 1. */ + if (options->StrictNodes) { + log_warn(LD_BUG, "Using %s '%s' which is listed in ExcludeNodes%s, " + "even though StrictNodes is set. Please report. " + "(Circuit purpose: %s)", + description, exit->nickname, + rs==options->ExcludeNodes?"":" or ExcludeExitNodes", + circuit_purpose_to_string(purpose)); + } else { + log_warn(LD_CIRC, "Using %s '%s' which is listed in " + "ExcludeNodes%s, because no better options were available. To " + "prevent this (and possibly break your Tor functionality), " + "set the StrictNodes configuration option. " + "(Circuit purpose: %s)", + description, exit->nickname, + rs==options->ExcludeNodes?"":" or ExcludeExitNodes", + circuit_purpose_to_string(purpose)); + } + circuit_log_path(LOG_WARN, LD_CIRC, circ); } return; @@ -1555,8 +3005,7 @@ circuit_append_new_exit(origin_circuit_t *circ, extend_info_t *exit) state = circ->build_state; tor_assert(state); - if (state->chosen_exit) - extend_info_free(state->chosen_exit); + extend_info_free(state->chosen_exit); state->chosen_exit = extend_info_dup(exit); ++circ->build_state->desired_path_len; @@ -1678,8 +3127,7 @@ choose_good_middle_server(uint8_t purpose, flags |= CRN_NEED_CAPACITY; if (options->_AllowInvalid & ALLOW_INVALID_MIDDLE) flags |= CRN_ALLOW_INVALID; - choice = router_choose_random_node(NULL, - excluded, options->ExcludeNodes, flags); + choice = router_choose_random_node(excluded, options->ExcludeNodes, flags); smartlist_free(excluded); return choice; } @@ -1743,11 +3191,7 @@ choose_good_entry_server(uint8_t purpose, cpath_build_state_t *state) if (options->_AllowInvalid & ALLOW_INVALID_ENTRY) flags |= CRN_ALLOW_INVALID; - choice = router_choose_random_node( - NULL, - excluded, - options->ExcludeNodes, - flags); + choice = router_choose_random_node(excluded, options->ExcludeNodes, flags); smartlist_free(excluded); return choice; } @@ -1868,9 +3312,9 @@ extend_info_from_router(routerinfo_t *r) void extend_info_free(extend_info_t *info) { - tor_assert(info); - if (info->onion_key) - crypto_free_pk_env(info->onion_key); + if (!info) + return; + crypto_free_pk_env(info->onion_key); tor_free(info); } @@ -1929,8 +3373,6 @@ entry_guard_set_status(entry_guard_t *e, routerinfo_t *ri, char buf[HEX_DIGEST_LEN+1]; int changed = 0; - tor_assert(options); - *reason = NULL; /* Do we want to mark this guard as bad? */ @@ -1993,35 +3435,58 @@ entry_is_time_to_retry(entry_guard_t *e, time_t now) * - Listed as either up or never yet contacted; * - Present in the routerlist; * - Listed as 'stable' or 'fast' by the current dirserver consensus, - * if demanded by <b>need_uptime</b> or <b>need_capacity</b>; - * (This check is currently redundant with the Guard flag, but in - * the future that might change. Best to leave it in for now.) + * if demanded by <b>need_uptime</b> or <b>need_capacity</b> + * (unless it's a configured EntryNode); * - Allowed by our current ReachableORAddresses config option; and - * - Currently thought to be reachable by us (unless assume_reachable + * - Currently thought to be reachable by us (unless <b>assume_reachable</b> * is true). + * + * If the answer is no, set *<b>msg</b> to an explanation of why. */ static INLINE routerinfo_t * entry_is_live(entry_guard_t *e, int need_uptime, int need_capacity, - int assume_reachable) + int assume_reachable, const char **msg) { routerinfo_t *r; - if (e->bad_since) + or_options_t *options = get_options(); + tor_assert(msg); + + if (e->bad_since) { + *msg = "bad"; return NULL; + } /* no good if it's unreachable, unless assume_unreachable or can_retry. */ if (!assume_reachable && !e->can_retry && - e->unreachable_since && !entry_is_time_to_retry(e, time(NULL))) + e->unreachable_since && !entry_is_time_to_retry(e, time(NULL))) { + *msg = "unreachable"; return NULL; + } r = router_get_by_digest(e->identity); - if (!r) + if (!r) { + *msg = "no descriptor"; return NULL; - if (get_options()->UseBridges && r->purpose != ROUTER_PURPOSE_BRIDGE) + } + if (get_options()->UseBridges && r->purpose != ROUTER_PURPOSE_BRIDGE) { + *msg = "not a bridge"; return NULL; - if (!get_options()->UseBridges && r->purpose != ROUTER_PURPOSE_GENERAL) + } + if (!get_options()->UseBridges && r->purpose != ROUTER_PURPOSE_GENERAL) { + *msg = "not general-purpose"; return NULL; - if (router_is_unreliable(r, need_uptime, need_capacity, 0)) + } + if (options->EntryNodes && + routerset_contains_router(options->EntryNodes, r)) { + /* they asked for it, they get it */ + need_uptime = need_capacity = 0; + } + if (router_is_unreliable(r, need_uptime, need_capacity, 0)) { + *msg = "not fast/stable"; return NULL; - if (!fascist_firewall_allows_or(r)) + } + if (!fascist_firewall_allows_or(r)) { + *msg = "unreachable by config"; return NULL; + } return r; } @@ -2030,11 +3495,12 @@ static int num_live_entry_guards(void) { int n = 0; + const char *msg; if (! entry_guards) return 0; SMARTLIST_FOREACH(entry_guards, entry_guard_t *, entry, { - if (entry_is_live(entry, 0, 1, 0)) + if (entry_is_live(entry, 0, 1, 0, &msg)) ++n; }); return n; @@ -2058,16 +3524,21 @@ static void log_entry_guards(int severity) { smartlist_t *elements = smartlist_create(); - char buf[1024]; char *s; SMARTLIST_FOREACH(entry_guards, entry_guard_t *, e, { - tor_snprintf(buf, sizeof(buf), "%s (%s%s)", - e->nickname, - entry_is_live(e, 0, 1, 0) ? "up " : "down ", - e->made_contact ? "made-contact" : "never-contacted"); - smartlist_add(elements, tor_strdup(buf)); + const char *msg = NULL; + char *cp; + if (entry_is_live(e, 0, 1, 0, &msg)) + tor_asprintf(&cp, "%s (up %s)", + e->nickname, + e->made_contact ? "made-contact" : "never-contacted"); + else + tor_asprintf(&cp, "%s (%s, %s)", + e->nickname, msg, + e->made_contact ? "made-contact" : "never-contacted"); + smartlist_add(elements, cp); }); s = smartlist_join_strings(elements, ",", 0, NULL); @@ -2091,12 +3562,13 @@ control_event_guard_deferred(void) **/ #if 0 int n = 0; + const char *msg; or_options_t *options = get_options(); if (!entry_guards) return; SMARTLIST_FOREACH(entry_guards, entry_guard_t *, entry, { - if (entry_is_live(entry, 0, 1, 0)) { + if (entry_is_live(entry, 0, 1, 0, &msg)) { if (n++ == options->NumEntryGuards) { control_event_guard(entry->nickname, entry->identity, "DEFERRED"); return; @@ -2158,9 +3630,8 @@ add_an_entry_guard(routerinfo_t *chosen, int reset_status) /** If the use of entry guards is configured, choose more entry guards * until we have enough in the list. */ static void -pick_entry_guards(void) +pick_entry_guards(or_options_t *options) { - or_options_t *options = get_options(); int changed = 0; tor_assert(entry_guards); @@ -2182,7 +3653,8 @@ pick_entry_guards(void) static void entry_guard_free(entry_guard_t *e) { - tor_assert(e); + if (!e) + return; tor_free(e->chosen_by_version); tor_free(e); } @@ -2191,10 +3663,9 @@ entry_guard_free(entry_guard_t *e) * or which was selected by a version of Tor that's known to select * entry guards badly. */ static int -remove_obsolete_entry_guards(void) +remove_obsolete_entry_guards(time_t now) { int changed = 0, i; - time_t now = time(NULL); for (i = 0; i < smartlist_len(entry_guards); ++i) { entry_guard_t *entry = smartlist_get(entry_guards, i); @@ -2254,11 +3725,10 @@ remove_obsolete_entry_guards(void) * long that we don't think they'll come up again. Return 1 if we * removed any, or 0 if we did nothing. */ static int -remove_dead_entry_guards(void) +remove_dead_entry_guards(time_t now) { char dbuf[HEX_DIGEST_LEN+1]; char tbuf[ISO_TIME_LEN+1]; - time_t now = time(NULL); int i; int changed = 0; @@ -2293,19 +3763,16 @@ remove_dead_entry_guards(void) * think that things are unlisted. */ void -entry_guards_compute_status(void) +entry_guards_compute_status(or_options_t *options, time_t now) { - time_t now; int changed = 0; - int severity = LOG_DEBUG; - or_options_t *options; digestmap_t *reasons; + if (! entry_guards) return; - options = get_options(); - - now = time(NULL); + if (options->EntryNodes) /* reshuffle the entry guard list if needed */ + entry_nodes_should_be_added(); reasons = digestmap_new(); SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, entry) @@ -2322,21 +3789,22 @@ entry_guards_compute_status(void) } SMARTLIST_FOREACH_END(entry); - if (remove_dead_entry_guards()) + if (remove_dead_entry_guards(now)) changed = 1; - severity = changed ? LOG_DEBUG : LOG_INFO; - if (changed) { SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, entry) { const char *reason = digestmap_get(reasons, entry->identity); - log_info(LD_CIRC, "Summary: Entry '%s' is %s, %s%s%s, and %s.", + const char *live_msg = ""; + routerinfo_t *r = entry_is_live(entry, 0, 1, 0, &live_msg); + log_info(LD_CIRC, "Summary: Entry '%s' is %s, %s%s%s, and %s%s.", entry->nickname, entry->unreachable_since ? "unreachable" : "reachable", entry->bad_since ? "unusable" : "usable", reason ? ", ": "", reason ? reason : "", - entry_is_live(entry, 0, 1, 0) ? "live" : "not live"); + r ? "live" : "not live / ", + r ? "" : live_msg); } SMARTLIST_FOREACH_END(entry); log_info(LD_CIRC, " (%d/%d entry guards are usable/new)", num_live_entry_guards(), smartlist_len(entry_guards)); @@ -2355,7 +3823,7 @@ entry_guards_compute_status(void) * If <b>mark_relay_status</b>, also call router_set_status() on this * relay. * - * XXX022 change succeeded and mark_relay_status into 'int flags'. + * XXX023 change succeeded and mark_relay_status into 'int flags'. */ int entry_guard_register_connect_status(const char *digest, int succeeded, @@ -2407,6 +3875,7 @@ entry_guard_register_connect_status(const char *digest, int succeeded, "Removing from the list. %d/%d entry guards usable/new.", entry->nickname, buf, num_live_entry_guards()-1, smartlist_len(entry_guards)-1); + control_event_guard(entry->nickname, entry->identity, "DROPPED"); entry_guard_free(entry); smartlist_del_keeporder(entry_guards, idx); log_entry_guards(LOG_INFO); @@ -2443,7 +3912,8 @@ entry_guard_register_connect_status(const char *digest, int succeeded, if (e == entry) break; if (e->made_contact) { - routerinfo_t *r = entry_is_live(e, 0, 1, 1); + const char *msg; + routerinfo_t *r = entry_is_live(e, 0, 1, 1, &msg); if (r && e->unreachable_since) { refuse_conn = 1; e->can_retry = 1; @@ -2474,16 +3944,16 @@ static int should_add_entry_nodes = 0; void entry_nodes_should_be_added(void) { - log_info(LD_CIRC, "New EntryNodes config option detected. Will use."); + log_info(LD_CIRC, "EntryNodes config option set. Putting configured " + "relays at the front of the entry guard list."); should_add_entry_nodes = 1; } /** Add all nodes in EntryNodes that aren't currently guard nodes to the list * of guard nodes, at the front. */ static void -entry_guards_prepend_from_config(void) +entry_guards_prepend_from_config(or_options_t *options) { - or_options_t *options = get_options(); smartlist_t *entry_routers, *entry_fps; smartlist_t *old_entry_guards_on_list, *old_entry_guards_not_on_list; tor_assert(entry_guards); @@ -2498,7 +3968,7 @@ entry_guards_prepend_from_config(void) return; } - if (options->EntryNodes) { + { char *string = routerset_to_string(options->EntryNodes); log_info(LD_CIRC,"Adding configured EntryNodes '%s'.", string); tor_free(string); @@ -2511,13 +3981,14 @@ entry_guards_prepend_from_config(void) /* Split entry guards into those on the list and those not. */ - /* XXXX022 Now that we allow countries and IP ranges in EntryNodes, this is + /* XXXX023 Now that we allow countries and IP ranges in EntryNodes, this is * potentially an enormous list. For now, we disable such values for * EntryNodes in options_validate(); really, this wants a better solution. * Perhaps we should do this calculation once whenever the list of routers * changes or the entrynodes setting changes. */ - routerset_get_all_routers(entry_routers, options->EntryNodes, 0); + routerset_get_all_routers(entry_routers, options->EntryNodes, + options->ExcludeNodes, 0); SMARTLIST_FOREACH(entry_routers, routerinfo_t *, ri, smartlist_add(entry_fps,ri->cache_info.identity_digest)); SMARTLIST_FOREACH(entry_guards, entry_guard_t *, e, { @@ -2542,13 +4013,10 @@ entry_guards_prepend_from_config(void) SMARTLIST_FOREACH(entry_routers, routerinfo_t *, ri, { add_an_entry_guard(ri, 0); }); - /* Finally, the remaining EntryNodes, unless we're strict */ - if (options->StrictEntryNodes) { - SMARTLIST_FOREACH(old_entry_guards_not_on_list, entry_guard_t *, e, - entry_guard_free(e)); - } else { - smartlist_add_all(entry_guards, old_entry_guards_not_on_list); - } + /* Finally, free the remaining previously configured guards that are not in + * EntryNodes. */ + SMARTLIST_FOREACH(old_entry_guards_not_on_list, entry_guard_t *, e, + entry_guard_free(e)); smartlist_free(entry_routers); smartlist_free(entry_fps); @@ -2557,16 +4025,18 @@ entry_guards_prepend_from_config(void) entry_guards_changed(); } -/** Return 1 if we're fine adding arbitrary routers out of the - * directory to our entry guard list. Else return 0. */ +/** Return 0 if we're fine adding arbitrary routers out of the + * directory to our entry guard list, or return 1 if we have a + * list already and we must stick to it. + */ int -entry_list_can_grow(or_options_t *options) +entry_list_is_constrained(or_options_t *options) { - if (options->StrictEntryNodes) - return 0; + if (options->EntryNodes) + return 1; if (options->UseBridges) - return 0; - return 1; + return 1; + return 0; } /** Pick a live (up and listed) entry guard from entry_guards. If @@ -2584,10 +4054,9 @@ choose_random_entry(cpath_build_state_t *state) routerinfo_t *r = NULL; int need_uptime = state ? state->need_uptime : 0; int need_capacity = state ? state->need_capacity : 0; - int consider_exit_family = 0; + int preferred_min, consider_exit_family = 0; if (chosen_exit) { - smartlist_add(exit_family, chosen_exit); routerlist_add_family(exit_family, chosen_exit); consider_exit_family = 1; } @@ -2596,38 +4065,66 @@ choose_random_entry(cpath_build_state_t *state) entry_guards = smartlist_create(); if (should_add_entry_nodes) - entry_guards_prepend_from_config(); + entry_guards_prepend_from_config(options); - if (entry_list_can_grow(options) && - (! entry_guards || - smartlist_len(entry_guards) < options->NumEntryGuards)) - pick_entry_guards(); + if (!entry_list_is_constrained(options) && + smartlist_len(entry_guards) < options->NumEntryGuards) + pick_entry_guards(options); retry: smartlist_clear(live_entry_guards); SMARTLIST_FOREACH(entry_guards, entry_guard_t *, entry, { - r = entry_is_live(entry, need_uptime, need_capacity, 0); - if (r && (!consider_exit_family || !smartlist_isin(exit_family, r))) { - smartlist_add(live_entry_guards, r); - if (!entry->made_contact) { - /* Always start with the first not-yet-contacted entry - * guard. Otherwise we might add several new ones, pick - * the second new one, and now we've expanded our entry - * guard list without needing to. */ - goto choose_and_finish; + const char *msg; + r = entry_is_live(entry, need_uptime, need_capacity, 0, &msg); + if (!r) + continue; /* down, no point */ + if (r == chosen_exit) + continue; /* don't pick the same node for entry and exit */ + if (consider_exit_family && smartlist_isin(exit_family, r)) + continue; /* avoid relays that are family members of our exit */ +#if 0 /* since EntryNodes is always strict now, this clause is moot */ + if (options->EntryNodes && + !routerset_contains_router(options->EntryNodes, r)) { + /* We've come to the end of our preferred entry nodes. */ + if (smartlist_len(live_entry_guards)) + goto choose_and_finish; /* only choose from the ones we like */ + if (options->StrictNodes) { + /* in theory this case should never happen, since + * entry_guards_prepend_from_config() drops unwanted relays */ + tor_fragile_assert(); + } else { + log_info(LD_CIRC, + "No relays from EntryNodes available. Using others."); } - if (smartlist_len(live_entry_guards) >= options->NumEntryGuards) - break; /* we have enough */ } +#endif + smartlist_add(live_entry_guards, r); + if (!entry->made_contact) { + /* Always start with the first not-yet-contacted entry + * guard. Otherwise we might add several new ones, pick + * the second new one, and now we've expanded our entry + * guard list without needing to. */ + goto choose_and_finish; + } + if (smartlist_len(live_entry_guards) >= options->NumEntryGuards) + break; /* we have enough */ }); - /* Try to have at least 2 choices available. This way we don't - * get stuck with a single live-but-crummy entry and just keep - * using him. - * (We might get 2 live-but-crummy entry guards, but so be it.) */ - if (smartlist_len(live_entry_guards) < 2) { - if (entry_list_can_grow(options)) { + if (entry_list_is_constrained(options)) { + /* If we prefer the entry nodes we've got, and we have at least + * one choice, that's great. Use it. */ + preferred_min = 1; + } else { + /* Try to have at least 2 choices available. This way we don't + * get stuck with a single live-but-crummy entry and just keep + * using him. + * (We might get 2 live-but-crummy entry guards, but so be it.) */ + preferred_min = 2; + } + + if (smartlist_len(live_entry_guards) < preferred_min) { + if (!entry_list_is_constrained(options)) { /* still no? try adding a new entry then */ /* XXX if guard doesn't imply fast and stable, then we need * to tell add_an_entry_guard below what we want, or it might @@ -2652,26 +4149,31 @@ choose_random_entry(cpath_build_state_t *state) need_capacity = 0; goto retry; } - if (!r && !entry_list_can_grow(options) && consider_exit_family) { - /* still no? if we're using bridges or have strictentrynodes - * set, and our chosen exit is in the same family as all our - * bridges/entry guards, then be flexible about families. */ +#if 0 + /* Removing this retry logic: if we only allow one exit, and it is in the + same family as all our entries, then we are just plain not going to win + here. */ + if (!r && entry_list_is_constrained(options) && consider_exit_family) { + /* still no? if we're using bridges, + * and our chosen exit is in the same family as all our + * bridges, then be flexible about families. */ consider_exit_family = 0; goto retry; } +#endif /* live_entry_guards may be empty below. Oh well, we tried. */ } choose_and_finish: - if (entry_list_can_grow(options)) { + if (entry_list_is_constrained(options)) { + /* We need to weight by bandwidth, because our bridges or entryguards + * were not already selected proportional to their bandwidth. */ + r = routerlist_sl_choose_by_bandwidth(live_entry_guards, WEIGHT_FOR_GUARD); + } else { /* We choose uniformly at random here, because choose_good_entry_server() * already weights its choices by bandwidth, so we don't want to * *double*-weight our guard selection. */ r = smartlist_choose(live_entry_guards); - } else { - /* We need to weight by bandwidth, because our bridges or entryguards - * were not already selected proportional to their bandwidth. */ - r = routerlist_sl_choose_by_bandwidth(live_entry_guards, WEIGHT_FOR_GUARD); } smartlist_free(live_entry_guards); smartlist_free(exit_family); @@ -2803,9 +4305,9 @@ entry_guards_parse_state(or_state_t *state, int set, char **msg) } entry_guards = new_entry_guards; entry_guards_dirty = 0; - /* XXX022 hand new_entry_guards to this func, and move it up a + /* XXX023 hand new_entry_guards to this func, and move it up a * few lines, so we don't have to re-dirty it */ - if (remove_obsolete_entry_guards()) + if (remove_obsolete_entry_guards(now)) entry_guards_dirty = 1; } digestmap_free(added_by, _tor_free); @@ -2904,9 +4406,11 @@ entry_guards_update_state(or_state_t *state) * */ int getinfo_helper_entry_guards(control_connection_t *conn, - const char *question, char **answer) + const char *question, char **answer, + const char **errmsg) { - int use_long_names = conn->use_long_names; + (void) conn; + (void) errmsg; if (!strcmp(question,"entry-guards") || !strcmp(question,"helper-nodes")) { @@ -2915,12 +4419,13 @@ getinfo_helper_entry_guards(control_connection_t *conn, char nbuf[MAX_VERBOSE_NICKNAME_LEN+1]; if (!entry_guards) entry_guards = smartlist_create(); - SMARTLIST_FOREACH(entry_guards, entry_guard_t *, e, - { + SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, e) { size_t len = MAX_VERBOSE_NICKNAME_LEN+ISO_TIME_LEN+32; char *c = tor_malloc(len); const char *status = NULL; time_t when = 0; + routerinfo_t *ri; + if (!e->made_contact) { status = "never-connected"; } else if (e->bad_since) { @@ -2929,19 +4434,17 @@ getinfo_helper_entry_guards(control_connection_t *conn, } else { status = "up"; } - if (use_long_names) { - routerinfo_t *ri = router_get_by_digest(e->identity); - if (ri) { - router_get_verbose_nickname(nbuf, ri); - } else { - nbuf[0] = '$'; - base16_encode(nbuf+1, sizeof(nbuf)-1, e->identity, DIGEST_LEN); - /* e->nickname field is not very reliable if we don't know about - * this router any longer; don't include it. */ - } + + ri = router_get_by_digest(e->identity); + if (ri) { + router_get_verbose_nickname(nbuf, ri); } else { - base16_encode(nbuf, sizeof(nbuf), e->identity, DIGEST_LEN); + nbuf[0] = '$'; + base16_encode(nbuf+1, sizeof(nbuf)-1, e->identity, DIGEST_LEN); + /* e->nickname field is not very reliable if we don't know about + * this router any longer; don't include it. */ } + if (when) { format_iso_time(tbuf, when); tor_snprintf(c, len, "%s %s %s\n", nbuf, status, tbuf); @@ -2949,7 +4452,7 @@ getinfo_helper_entry_guards(control_connection_t *conn, tor_snprintf(c, len, "%s %s\n", nbuf, status); } smartlist_add(sl, c); - }); + } SMARTLIST_FOREACH_END(e); *answer = smartlist_join_strings(sl, "", 0, NULL); SMARTLIST_FOREACH(sl, char *, c, tor_free(c)); smartlist_free(sl); @@ -2965,6 +4468,9 @@ typedef struct { tor_addr_t addr; /** TLS port for the bridge. */ uint16_t port; + /** Boolean: We are re-parsing our bridge list, and we are going to remove + * this one if we don't find it in the list of configured bridges. */ + unsigned marked_for_removal : 1; /** Expected identity digest, or all zero bytes if we don't know what the * digest should be. */ char identity[DIGEST_LEN]; @@ -2973,11 +4479,39 @@ typedef struct { } bridge_info_t; /** A list of configured bridges. Whenever we actually get a descriptor - * for one, we add it as an entry guard. */ + * for one, we add it as an entry guard. Note that the order of bridges + * in this list does not necessarily correspond to the order of bridges + * in the torrc. */ static smartlist_t *bridge_list = NULL; -/** Initialize the bridge list to empty, creating it if needed. */ +/** Mark every entry of the bridge list to be removed on our next call to + * sweep_bridge_list unless it has first been un-marked. */ void +mark_bridge_list(void) +{ + if (!bridge_list) + bridge_list = smartlist_create(); + SMARTLIST_FOREACH(bridge_list, bridge_info_t *, b, + b->marked_for_removal = 1); +} + +/** Remove every entry of the bridge list that was marked with + * mark_bridge_list if it has not subsequently been un-marked. */ +void +sweep_bridge_list(void) +{ + if (!bridge_list) + bridge_list = smartlist_create(); + SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, b) { + if (b->marked_for_removal) { + SMARTLIST_DEL_CURRENT(bridge_list, b); + tor_free(b); + } + } SMARTLIST_FOREACH_END(b); +} + +/** Initialize the bridge list to empty, creating it if needed. */ +static void clear_bridge_list(void) { if (!bridge_list) @@ -2990,37 +4524,75 @@ clear_bridge_list(void) * (either by comparing keys if possible, else by comparing addr/port). * Else return NULL. */ static bridge_info_t * -routerinfo_get_configured_bridge(routerinfo_t *ri) +get_configured_bridge_by_addr_port_digest(const tor_addr_t *addr, + uint16_t port, + const char *digest) { if (!bridge_list) return NULL; SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, bridge) { if (tor_digest_is_zero(bridge->identity) && - tor_addr_eq_ipv4h(&bridge->addr, ri->addr) && - bridge->port == ri->or_port) + !tor_addr_compare(&bridge->addr, addr, CMP_EXACT) && + bridge->port == port) return bridge; - if (tor_memeq(bridge->identity, ri->cache_info.identity_digest, - DIGEST_LEN)) + if (digest && tor_memeq(bridge->identity, digest, DIGEST_LEN)) return bridge; } SMARTLIST_FOREACH_END(bridge); return NULL; } +/** Wrapper around get_configured_bridge_by_addr_port_digest() to look + * it up via router descriptor <b>ri</b>. */ +static bridge_info_t * +get_configured_bridge_by_routerinfo(routerinfo_t *ri) +{ + tor_addr_t addr; + tor_addr_from_ipv4h(&addr, ri->addr); + return get_configured_bridge_by_addr_port_digest(&addr, + ri->or_port, ri->cache_info.identity_digest); +} + /** Return 1 if <b>ri</b> is one of our known bridges, else 0. */ int routerinfo_is_a_configured_bridge(routerinfo_t *ri) { - return routerinfo_get_configured_bridge(ri) ? 1 : 0; + return get_configured_bridge_by_routerinfo(ri) ? 1 : 0; +} + +/** We made a connection to a router at <b>addr</b>:<b>port</b> + * without knowing its digest. Its digest turned out to be <b>digest</b>. + * If it was a bridge, and we still don't know its digest, record it. + */ +void +learned_router_identity(const tor_addr_t *addr, uint16_t port, + const char *digest) +{ + bridge_info_t *bridge = + get_configured_bridge_by_addr_port_digest(addr, port, digest); + if (bridge && tor_digest_is_zero(bridge->identity)) { + memcpy(bridge->identity, digest, DIGEST_LEN); + log_notice(LD_DIR, "Learned fingerprint %s for bridge %s:%d", + hex_str(digest, DIGEST_LEN), fmt_addr(addr), port); + } } /** Remember a new bridge at <b>addr</b>:<b>port</b>. If <b>digest</b> - * is set, it tells us the identity key too. */ + * is set, it tells us the identity key too. If we already had the + * bridge in our list, unmark it, and don't actually add anything new. */ void -bridge_add_from_config(const tor_addr_t *addr, uint16_t port, char *digest) +bridge_add_from_config(const tor_addr_t *addr, uint16_t port, + const char *digest) { - bridge_info_t *b = tor_malloc_zero(sizeof(bridge_info_t)); + bridge_info_t *b; + + if ((b = get_configured_bridge_by_addr_port_digest(addr, port, digest))) { + b->marked_for_removal = 0; + return; + } + + b = tor_malloc_zero(sizeof(bridge_info_t)); tor_addr_copy(&b->addr, addr); b->port = port; if (digest) @@ -3028,9 +4600,28 @@ bridge_add_from_config(const tor_addr_t *addr, uint16_t port, char *digest) b->fetch_status.schedule = DL_SCHED_BRIDGE; if (!bridge_list) bridge_list = smartlist_create(); + smartlist_add(bridge_list, b); } +/** Return true iff <b>routerset</b> contains the bridge <b>bridge</b>. */ +static int +routerset_contains_bridge(const routerset_t *routerset, + const bridge_info_t *bridge) +{ + int result; + extend_info_t *extinfo; + tor_assert(bridge); + if (!routerset) + return 0; + + extinfo = extend_info_alloc( + NULL, bridge->identity, NULL, &bridge->addr, bridge->port); + result = routerset_contains_extendinfo(routerset, extinfo); + extend_info_free(extinfo); + return result; +} + /** If <b>digest</b> is one of our known bridges, return it. */ static bridge_info_t * find_bridge_by_digest(const char *digest) @@ -3043,12 +4634,12 @@ find_bridge_by_digest(const char *digest) return NULL; } -/** We need to ask <b>bridge</b> for its server descriptor. <b>address</b> - * is a helpful string describing this bridge. */ +/** We need to ask <b>bridge</b> for its server descriptor. */ static void launch_direct_bridge_descriptor_fetch(bridge_info_t *bridge) { char *address; + or_options_t *options = get_options(); if (connection_get_by_type_addr_port_purpose( CONN_TYPE_DIR, &bridge->addr, bridge->port, @@ -3056,6 +4647,13 @@ launch_direct_bridge_descriptor_fetch(bridge_info_t *bridge) return; /* it's already on the way */ address = tor_dup_addr(&bridge->addr); + if (routerset_contains_bridge(options->ExcludeNodes, bridge)) { + download_status_mark_impossible(&bridge->fetch_status); + log_warn(LD_APP, "Not using bridge at %s: it is in ExcludeNodes.", + safe_str_client(fmt_addr(&bridge->addr))); + return; + } + directory_initiate_command(address, &bridge->addr, bridge->port, 0, 0, /* does not matter */ @@ -3082,9 +4680,8 @@ retry_bridge_descriptor_fetch_directly(const char *digest) * descriptor, fetch a new copy of its descriptor -- either directly * from the bridge or via a bridge authority. */ void -fetch_bridge_descriptors(time_t now) +fetch_bridge_descriptors(or_options_t *options, time_t now) { - or_options_t *options = get_options(); int num_bridge_auths = get_n_authorities(BRIDGE_AUTHORITY); int ask_bridge_directly; int can_use_bridge_authority; @@ -3097,6 +4694,12 @@ fetch_bridge_descriptors(time_t now) if (!download_status_is_ready(&bridge->fetch_status, now, IMPOSSIBLE_TO_DOWNLOAD)) continue; /* don't bother, no need to retry yet */ + if (routerset_contains_bridge(options->ExcludeNodes, bridge)) { + download_status_mark_impossible(&bridge->fetch_status); + log_warn(LD_APP, "Not using bridge at %s: it is in ExcludeNodes.", + safe_str_client(fmt_addr(&bridge->addr))); + continue; + } /* schedule another fetch as if this one will fail, in case it does */ download_status_failed(&bridge->fetch_status, 0); @@ -3143,6 +4746,29 @@ fetch_bridge_descriptors(time_t now) SMARTLIST_FOREACH_END(bridge); } +/** If our <b>bridge</b> is configured to be a different address than + * the bridge gives in its routerinfo <b>ri</b>, rewrite the routerinfo + * we received to use the address we meant to use. Now we handle + * multihomed bridges better. + */ +static void +rewrite_routerinfo_address_for_bridge(bridge_info_t *bridge, routerinfo_t *ri) +{ + tor_addr_t addr; + tor_addr_from_ipv4h(&addr, ri->addr); + + if (!tor_addr_compare(&bridge->addr, &addr, CMP_EXACT) && + bridge->port == ri->or_port) + return; /* they match, so no need to do anything */ + + ri->addr = tor_addr_to_ipv4h(&bridge->addr); + tor_free(ri->address); + ri->address = tor_dup_ip(ri->addr); + ri->or_port = bridge->port; + log_info(LD_DIR, "Adjusted bridge '%s' to match configured address %s:%d.", + ri->nickname, ri->address, ri->or_port); +} + /** We just learned a descriptor for a bridge. See if that * digest is in our entry guard list, and add it if not. */ void @@ -3152,7 +4778,7 @@ learned_bridge_descriptor(routerinfo_t *ri, int from_cache) tor_assert(ri->purpose == ROUTER_PURPOSE_BRIDGE); if (get_options()->UseBridges) { int first = !any_bridge_descriptors_known(); - bridge_info_t *bridge = routerinfo_get_configured_bridge(ri); + bridge_info_t *bridge = get_configured_bridge_by_routerinfo(ri); time_t now = time(NULL); ri->is_running = 1; @@ -3161,6 +4787,8 @@ learned_bridge_descriptor(routerinfo_t *ri, int from_cache) if (!from_cache) download_status_reset(&bridge->fetch_status); + rewrite_routerinfo_address_for_bridge(bridge, ri); + add_an_entry_guard(ri, 1); log_notice(LD_DIR, "new bridge descriptor '%s' (%s)", ri->nickname, from_cache ? "cached" : "fresh"); @@ -3209,26 +4837,38 @@ any_pending_bridge_descriptor_fetches(void) return 0; } -/** Return 1 if we have at least one descriptor for a bridge and - * all descriptors we know are down. Else return 0. If <b>act</b> is - * 1, then mark the down bridges up; else just observe and report. */ +/** Return 1 if we have at least one descriptor for an entry guard + * (bridge or member of EntryNodes) and all descriptors we know are + * down. Else return 0. If <b>act</b> is 1, then mark the down guards + * up; else just observe and report. */ static int -bridges_retry_helper(int act) +entries_retry_helper(or_options_t *options, int act) { routerinfo_t *ri; int any_known = 0; int any_running = 0; + int purpose = options->UseBridges ? + ROUTER_PURPOSE_BRIDGE : ROUTER_PURPOSE_GENERAL; if (!entry_guards) entry_guards = smartlist_create(); SMARTLIST_FOREACH(entry_guards, entry_guard_t *, e, { ri = router_get_by_digest(e->identity); - if (ri && ri->purpose == ROUTER_PURPOSE_BRIDGE) { + if (ri && ri->purpose == purpose) { any_known = 1; if (ri->is_running) - any_running = 1; /* some bridge is both known and running */ - else if (act) { /* mark it for retry */ - ri->is_running = 1; + any_running = 1; /* some entry is both known and running */ + else if (act) { + /* Mark all current connections to this OR as unhealthy, since + * otherwise there could be one that started 30 seconds + * ago, and in 30 seconds it will time out, causing us to mark + * the node down and undermine the retry attempt. We mark even + * the established conns, since if the network just came back + * we'll want to attach circuits to fresh conns. */ + connection_or_set_bad_connections(ri->cache_info.identity_digest, 1); + + /* mark this entry node for retry */ + router_set_status(ri->cache_info.identity_digest, 1); e->can_retry = 1; e->bad_since = 0; } @@ -3239,19 +4879,21 @@ bridges_retry_helper(int act) return any_known && !any_running; } -/** Do we know any descriptors for our bridges, and are they all - * down? */ +/** Do we know any descriptors for our bridges / entrynodes, and are + * all the ones we have descriptors for down? */ int -bridges_known_but_down(void) +entries_known_but_down(or_options_t *options) { - return bridges_retry_helper(0); + tor_assert(entry_list_is_constrained(options)); + return entries_retry_helper(options, 0); } -/** Mark all down known bridges up. */ +/** Mark all down known bridges / entrynodes up. */ void -bridges_retry_all(void) +entries_retry_all(or_options_t *options) { - bridges_retry_helper(1); + tor_assert(entry_list_is_constrained(options)); + entries_retry_helper(options, 1); } /** Release all storage held by the list of entry guards and related diff --git a/src/or/circuitbuild.h b/src/or/circuitbuild.h new file mode 100644 index 0000000000..0e673e1c05 --- /dev/null +++ b/src/or/circuitbuild.h @@ -0,0 +1,128 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file circuitbuild.h + * \brief Header file for circuitbuild.c. + **/ + +#ifndef _TOR_CIRCUITBUILD_H +#define _TOR_CIRCUITBUILD_H + +char *circuit_list_path(origin_circuit_t *circ, int verbose); +char *circuit_list_path_for_controller(origin_circuit_t *circ); +void circuit_log_path(int severity, unsigned int domain, + origin_circuit_t *circ); +void circuit_rep_hist_note_result(origin_circuit_t *circ); +origin_circuit_t *origin_circuit_init(uint8_t purpose, int flags); +origin_circuit_t *circuit_establish_circuit(uint8_t purpose, + extend_info_t *exit, + int flags); +int circuit_handle_first_hop(origin_circuit_t *circ); +void circuit_n_conn_done(or_connection_t *or_conn, int status); +int inform_testing_reachability(void); +int circuit_timeout_want_to_count_circ(origin_circuit_t *circ); +int circuit_send_next_onion_skin(origin_circuit_t *circ); +void circuit_note_clock_jumped(int seconds_elapsed); +int circuit_extend(cell_t *cell, circuit_t *circ); +int circuit_init_cpath_crypto(crypt_path_t *cpath, const char *key_data, + int reverse); +int circuit_finish_handshake(origin_circuit_t *circ, uint8_t cell_type, + const uint8_t *reply); +int circuit_truncated(origin_circuit_t *circ, crypt_path_t *layer); +int onionskin_answer(or_circuit_t *circ, uint8_t cell_type, + const char *payload, const char *keys); +int circuit_all_predicted_ports_handled(time_t now, int *need_uptime, + int *need_capacity); + +int circuit_append_new_exit(origin_circuit_t *circ, extend_info_t *info); +int circuit_extend_to_new_exit(origin_circuit_t *circ, extend_info_t *info); +void onion_append_to_cpath(crypt_path_t **head_ptr, crypt_path_t *new_hop); +extend_info_t *extend_info_alloc(const char *nickname, const char *digest, + crypto_pk_env_t *onion_key, + const tor_addr_t *addr, uint16_t port); +extend_info_t *extend_info_from_router(routerinfo_t *r); +extend_info_t *extend_info_dup(extend_info_t *info); +void extend_info_free(extend_info_t *info); +routerinfo_t *build_state_get_exit_router(cpath_build_state_t *state); +const char *build_state_get_exit_nickname(cpath_build_state_t *state); + +void entry_guards_compute_status(or_options_t *options, time_t now); +int entry_guard_register_connect_status(const char *digest, int succeeded, + int mark_relay_status, time_t now); +void entry_nodes_should_be_added(void); +int entry_list_is_constrained(or_options_t *options); +routerinfo_t *choose_random_entry(cpath_build_state_t *state); +int entry_guards_parse_state(or_state_t *state, int set, char **msg); +void entry_guards_update_state(or_state_t *state); +int getinfo_helper_entry_guards(control_connection_t *conn, + const char *question, char **answer, + const char **errmsg); + +void mark_bridge_list(void); +void sweep_bridge_list(void); +int routerinfo_is_a_configured_bridge(routerinfo_t *ri); +void learned_router_identity(const tor_addr_t *addr, uint16_t port, + const char *digest); +void bridge_add_from_config(const tor_addr_t *addr, uint16_t port, + const char *digest); +void retry_bridge_descriptor_fetch_directly(const char *digest); +void fetch_bridge_descriptors(or_options_t *options, time_t now); +void learned_bridge_descriptor(routerinfo_t *ri, int from_cache); +int any_bridge_descriptors_known(void); +int any_pending_bridge_descriptor_fetches(void); +int entries_known_but_down(or_options_t *options); +void entries_retry_all(or_options_t *options); + +void entry_guards_free_all(void); + +extern circuit_build_times_t circ_times; +int circuit_build_times_enough_to_compute(circuit_build_times_t *cbt); +void circuit_build_times_update_state(circuit_build_times_t *cbt, + or_state_t *state); +int circuit_build_times_parse_state(circuit_build_times_t *cbt, + or_state_t *state); +void circuit_build_times_count_timeout(circuit_build_times_t *cbt, + int did_onehop); +int circuit_build_times_count_close(circuit_build_times_t *cbt, + int did_onehop, time_t start_time); +void circuit_build_times_set_timeout(circuit_build_times_t *cbt); +int circuit_build_times_add_time(circuit_build_times_t *cbt, + build_time_t time); +int circuit_build_times_needs_circuits(circuit_build_times_t *cbt); + +int circuit_build_times_needs_circuits_now(circuit_build_times_t *cbt); +void circuit_build_times_init(circuit_build_times_t *cbt); +void circuit_build_times_new_consensus_params(circuit_build_times_t *cbt, + networkstatus_t *ns); +double circuit_build_times_timeout_rate(const circuit_build_times_t *cbt); +double circuit_build_times_close_rate(const circuit_build_times_t *cbt); + +#ifdef CIRCUIT_PRIVATE +double circuit_build_times_calculate_timeout(circuit_build_times_t *cbt, + double quantile); +build_time_t circuit_build_times_generate_sample(circuit_build_times_t *cbt, + double q_lo, double q_hi); +void circuit_build_times_initial_alpha(circuit_build_times_t *cbt, + double quantile, double time_ms); +int circuit_build_times_update_alpha(circuit_build_times_t *cbt); +double circuit_build_times_cdf(circuit_build_times_t *cbt, double x); +void circuitbuild_running_unit_tests(void); +void circuit_build_times_reset(circuit_build_times_t *cbt); + +/* Network liveness functions */ +int circuit_build_times_network_check_changed(circuit_build_times_t *cbt); +#endif + +/* Network liveness functions */ +void circuit_build_times_network_is_live(circuit_build_times_t *cbt); +int circuit_build_times_network_check_live(circuit_build_times_t *cbt); +void circuit_build_times_network_circ_success(circuit_build_times_t *cbt); + +int circuit_build_times_get_bw_scale(networkstatus_t *ns); + +#endif + diff --git a/src/or/circuitlist.c b/src/or/circuitlist.c index 9cf331ecf2..8ec46186d9 100644 --- a/src/or/circuitlist.c +++ b/src/or/circuitlist.c @@ -10,6 +10,21 @@ **/ #include "or.h" +#include "circuitbuild.h" +#include "circuitlist.h" +#include "circuituse.h" +#include "connection.h" +#include "config.h" +#include "connection_edge.h" +#include "connection_or.h" +#include "control.h" +#include "networkstatus.h" +#include "onion.h" +#include "relay.h" +#include "rendclient.h" +#include "rendcommon.h" +#include "rephist.h" +#include "routerlist.h" #include "ht.h" /********* START VARIABLES **********/ @@ -352,6 +367,8 @@ circuit_purpose_to_controller_string(uint8_t purpose) case CIRCUIT_PURPOSE_TESTING: return "TESTING"; + case CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT: + return "MEASURE_TIMEOUT"; case CIRCUIT_PURPOSE_CONTROLLER: return "CONTROLLER"; @@ -361,13 +378,71 @@ circuit_purpose_to_controller_string(uint8_t purpose) } } +/** Return a human-readable string for the circuit purpose <b>purpose</b>. */ +const char * +circuit_purpose_to_string(uint8_t purpose) +{ + static char buf[32]; + + switch (purpose) + { + case CIRCUIT_PURPOSE_OR: + return "Circuit at relay"; + case CIRCUIT_PURPOSE_INTRO_POINT: + return "Acting as intro point"; + case CIRCUIT_PURPOSE_REND_POINT_WAITING: + return "Acting as rendevous (pending)"; + case CIRCUIT_PURPOSE_REND_ESTABLISHED: + return "Acting as rendevous (established)"; + case CIRCUIT_PURPOSE_C_GENERAL: + return "General-purpose client"; + case CIRCUIT_PURPOSE_C_INTRODUCING: + return "Hidden service client: Connecting to intro point"; + case CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT: + return "Hidden service client: Waiting for ack from intro point"; + case CIRCUIT_PURPOSE_C_INTRODUCE_ACKED: + return "Hidden service client: Received ack from intro point"; + case CIRCUIT_PURPOSE_C_ESTABLISH_REND: + return "Hidden service client: Establishing rendezvous point"; + case CIRCUIT_PURPOSE_C_REND_READY: + return "Hidden service client: Pending rendezvous point"; + case CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED: + return "Hidden service client: Pending rendezvous point (ack received)"; + case CIRCUIT_PURPOSE_C_REND_JOINED: + return "Hidden service client: Active rendezvous point"; + case CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT: + return "Measuring circuit timeout"; + + case CIRCUIT_PURPOSE_S_ESTABLISH_INTRO: + return "Hidden service: Establishing introduction point"; + case CIRCUIT_PURPOSE_S_INTRO: + return "Hidden service: Introduction point"; + case CIRCUIT_PURPOSE_S_CONNECT_REND: + return "Hidden service: Connecting to rendezvous point"; + case CIRCUIT_PURPOSE_S_REND_JOINED: + return "Hidden service: Active rendezvous point"; + + case CIRCUIT_PURPOSE_TESTING: + return "Testing circuit"; + + case CIRCUIT_PURPOSE_CONTROLLER: + return "Circuit made by controller"; + + default: + tor_snprintf(buf, sizeof(buf), "UNKNOWN_%d", (int)purpose); + return buf; + } +} + /** Pick a reasonable package_window to start out for our circuits. * Originally this was hard-coded at 1000, but now the consensus votes * on the answer. See proposal 168. */ int32_t circuit_initial_package_window(void) { - int32_t num = networkstatus_get_param(NULL, "circwindow", CIRCWINDOW_START); + int32_t num = networkstatus_get_param(NULL, "circwindow", CIRCWINDOW_START, + CIRCWINDOW_START_MIN, + CIRCWINDOW_START_MAX); /* If the consensus tells us a negative number, we'd assert. */ if (num < 0) num = CIRCWINDOW_START; @@ -379,11 +454,17 @@ circuit_initial_package_window(void) static void init_circuit_base(circuit_t *circ) { - circ->timestamp_created = time(NULL); + tor_gettimeofday(&circ->timestamp_created); circ->package_window = circuit_initial_package_window(); circ->deliver_window = CIRCWINDOW_START; + /* Initialize the cell_ewma_t structure */ + circ->n_cell_ewma.last_adjusted_tick = cell_ewma_get_tick(); + circ->n_cell_ewma.cell_count = 0.0; + circ->n_cell_ewma.heap_index = -1; + circ->n_cell_ewma.is_for_p_conn = 0; + circuit_add(circ); } @@ -408,6 +489,8 @@ origin_circuit_new(void) init_circuit_base(TO_CIRCUIT(circ)); + circ_times.last_circ_at = approx_time(); + return circ; } @@ -429,6 +512,16 @@ or_circuit_new(circid_t p_circ_id, or_connection_t *p_conn) init_circuit_base(TO_CIRCUIT(circ)); + /* Initialize the cell_ewma_t structure */ + + /* Initialize the cell counts to 0 */ + circ->p_cell_ewma.cell_count = 0.0; + circ->p_cell_ewma.last_adjusted_tick = cell_ewma_get_tick(); + circ->p_cell_ewma.is_for_p_conn = 1; + + /* It's not in any heap yet. */ + circ->p_cell_ewma.heap_index = -1; + return circ; } @@ -439,39 +532,37 @@ circuit_free(circuit_t *circ) { void *mem; size_t memlen; - tor_assert(circ); + if (!circ) + return; + if (CIRCUIT_IS_ORIGIN(circ)) { origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); mem = ocirc; memlen = sizeof(origin_circuit_t); tor_assert(circ->magic == ORIGIN_CIRCUIT_MAGIC); if (ocirc->build_state) { - if (ocirc->build_state->chosen_exit) extend_info_free(ocirc->build_state->chosen_exit); - if (ocirc->build_state->pending_final_cpath) circuit_free_cpath_node(ocirc->build_state->pending_final_cpath); } tor_free(ocirc->build_state); circuit_free_cpath(ocirc->cpath); - if (ocirc->intro_key) - crypto_free_pk_env(ocirc->intro_key); - if (ocirc->rend_data) - rend_data_free(ocirc->rend_data); + + crypto_free_pk_env(ocirc->intro_key); + rend_data_free(ocirc->rend_data); } else { or_circuit_t *ocirc = TO_OR_CIRCUIT(circ); + /* Remember cell statistics for this circuit before deallocating. */ + if (get_options()->CellStatistics) + rep_hist_buffer_stats_add_circ(circ, time(NULL)); mem = ocirc; memlen = sizeof(or_circuit_t); tor_assert(circ->magic == OR_CIRCUIT_MAGIC); - if (ocirc->p_crypto) - crypto_free_cipher_env(ocirc->p_crypto); - if (ocirc->p_digest) - crypto_free_digest_env(ocirc->p_digest); - if (ocirc->n_crypto) - crypto_free_cipher_env(ocirc->n_crypto); - if (ocirc->n_digest) - crypto_free_digest_env(ocirc->n_digest); + crypto_free_cipher_env(ocirc->p_crypto); + crypto_free_digest_env(ocirc->p_digest); + crypto_free_cipher_env(ocirc->n_crypto); + crypto_free_digest_env(ocirc->n_digest); if (ocirc->rend_splice) { or_circuit_t *other = ocirc->rend_splice; @@ -487,8 +578,7 @@ circuit_free(circuit_t *circ) cell_queue_clear(ô->p_conn_cells); } - if (circ->n_hop) - extend_info_free(circ->n_hop); + extend_info_free(circ->n_hop); tor_free(circ->n_conn_onionskin); /* Remove from map. */ @@ -498,7 +588,7 @@ circuit_free(circuit_t *circ) * "active" checks will be violated. */ cell_queue_clear(&circ->n_conn_cells); - memset(circ, 0xAA, memlen); /* poison memory */ + memset(mem, 0xAA, memlen); /* poison memory */ tor_free(mem); } @@ -541,10 +631,10 @@ circuit_free_all(void) circuit_free(global_circuitlist); global_circuitlist = next; } - if (circuits_pending_or_conns) { - smartlist_free(circuits_pending_or_conns); - circuits_pending_or_conns = NULL; - } + + smartlist_free(circuits_pending_or_conns); + circuits_pending_or_conns = NULL; + HT_CLEAR(orconn_circid_map, &orconn_circid_circuit_map); } @@ -552,18 +642,15 @@ circuit_free_all(void) static void circuit_free_cpath_node(crypt_path_t *victim) { - if (victim->f_crypto) - crypto_free_cipher_env(victim->f_crypto); - if (victim->b_crypto) - crypto_free_cipher_env(victim->b_crypto); - if (victim->f_digest) - crypto_free_digest_env(victim->f_digest); - if (victim->b_digest) - crypto_free_digest_env(victim->b_digest); - if (victim->dh_handshake_state) - crypto_dh_free(victim->dh_handshake_state); - if (victim->extend_info) - extend_info_free(victim->extend_info); + if (!victim) + return; + + crypto_free_cipher_env(victim->f_crypto); + crypto_free_cipher_env(victim->b_crypto); + crypto_free_digest_env(victim->f_digest); + crypto_free_digest_env(victim->b_digest); + crypto_dh_free(victim->dh_handshake_state); + extend_info_free(victim->extend_info); memset(victim, 0xBB, sizeof(crypt_path_t)); /* poison memory */ tor_free(victim); @@ -577,9 +664,10 @@ circuit_dump_details(int severity, circuit_t *circ, int conn_array_index, const char *type, int this_circid, int other_circid) { log(severity, LD_CIRC, "Conn %d has %s circuit: circID %d (other side %d), " - "state %d (%s), born %d:", + "state %d (%s), born %ld:", conn_array_index, type, this_circid, other_circid, circ->state, - circuit_state_to_string(circ->state), (int)circ->timestamp_created); + circuit_state_to_string(circ->state), + (long)circ->timestamp_created.tv_sec); if (CIRCUIT_IS_ORIGIN(circ)) { /* circ starts at this node */ circuit_log_path(severity, LD_CIRC, TO_ORIGIN_CIRCUIT(circ)); } @@ -891,6 +979,11 @@ circuit_find_to_cannibalize(uint8_t purpose, extend_info_t *info, int need_uptime = (flags & CIRCLAUNCH_NEED_UPTIME) != 0; int need_capacity = (flags & CIRCLAUNCH_NEED_CAPACITY) != 0; int internal = (flags & CIRCLAUNCH_IS_INTERNAL) != 0; + or_options_t *options = get_options(); + + /* Make sure we're not trying to create a onehop circ by + * cannibalization. */ + tor_assert(!(flags & CIRCLAUNCH_ONEHOP_TUNNEL)); log_debug(LD_CIRC, "Hunting for a circ to cannibalize: purpose %d, uptime %d, " @@ -907,7 +1000,8 @@ circuit_find_to_cannibalize(uint8_t purpose, extend_info_t *info, if ((!need_uptime || circ->build_state->need_uptime) && (!need_capacity || circ->build_state->need_capacity) && (internal == circ->build_state->is_internal) && - circ->remaining_relay_early_cells) { + circ->remaining_relay_early_cells && + !circ->build_state->onehop_tunnel) { if (info) { /* need to make sure we don't duplicate hops */ crypt_path_t *hop = circ->cpath; @@ -924,6 +1018,19 @@ circuit_find_to_cannibalize(uint8_t purpose, extend_info_t *info, hop=hop->next; } while (hop!=circ->cpath); } + if (options->ExcludeNodes) { + /* Make sure no existing nodes in the circuit are excluded for + * general use. (This may be possible if StrictNodes is 0, and we + * thought we needed to use an otherwise excluded node for, say, a + * directory operation.) */ + crypt_path_t *hop = circ->cpath; + do { + if (routerset_contains_extendinfo(options->ExcludeNodes, + hop->extend_info)) + goto next; + hop = hop->next; + } while (hop != circ->cpath); + } if (!best || (best->build_state->need_uptime && !need_uptime)) best = circ; next: ; @@ -986,6 +1093,7 @@ circuit_mark_all_unused_circs(void) * This is useful for letting the user change pseudonyms, so new * streams will not be linkable to old streams. */ +/* XXX023 this is a bad name for what this function does */ void circuit_expire_all_dirty_circs(void) { @@ -996,6 +1104,8 @@ circuit_expire_all_dirty_circs(void) if (CIRCUIT_IS_ORIGIN(circ) && !circ->marked_for_close && circ->timestamp_dirty) + /* XXXX023 This is a screwed-up way to say "This is too dirty + * for new circuits. */ circ->timestamp_dirty -= options->MaxCircuitDirtiness; } } @@ -1083,14 +1193,16 @@ _circuit_mark_for_close(circuit_t *circ, int reason, int line, tor_assert(ocirc->rend_data); /* treat this like getting a nack from it */ log_info(LD_REND, "Failed intro circ %s to %s (awaiting ack). " - "Removing from descriptor.", - safe_str(ocirc->rend_data->onion_address), - safe_str(build_state_get_exit_nickname(ocirc->build_state))); + "Removing from descriptor.", + safe_str_client(ocirc->rend_data->onion_address), + safe_str_client(build_state_get_exit_nickname(ocirc->build_state))); rend_client_remove_intro_point(ocirc->build_state->chosen_exit, ocirc->rend_data); } - if (circ->n_conn) + if (circ->n_conn) { + circuit_clear_cell_queue(circ, circ->n_conn); connection_or_send_destroy(circ->n_circ_id, circ->n_conn, reason); + } if (! CIRCUIT_IS_ORIGIN(circ)) { or_circuit_t *or_circ = TO_OR_CIRCUIT(circ); @@ -1114,8 +1226,10 @@ _circuit_mark_for_close(circuit_t *circ, int reason, int line, conn->on_circuit = NULL; } - if (or_circ->p_conn) + if (or_circ->p_conn) { + circuit_clear_cell_queue(circ, or_circ->p_conn); connection_or_send_destroy(or_circ->p_circ_id, or_circ->p_conn, reason); + } } else { origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); edge_connection_t *conn; @@ -1236,11 +1350,6 @@ assert_circuit_ok(const circuit_t *c) tor_assert(c == c2); } } -#if 0 /* false now that rendezvous exits are attached to p_streams */ - if (origin_circ) - for (conn = origin_circ->p_streams; conn; conn = conn->next_stream) - tor_assert(conn->_base.type == CONN_TYPE_AP); -#endif if (or_circ) for (conn = or_circ->n_streams; conn; conn = conn->next_stream) tor_assert(conn->_base.type == CONN_TYPE_EXIT); diff --git a/src/or/circuitlist.h b/src/or/circuitlist.h new file mode 100644 index 0000000000..7b01ca3ae2 --- /dev/null +++ b/src/or/circuitlist.h @@ -0,0 +1,61 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file circuitlist.h + * \brief Header file for circuitlist.c. + **/ + +#ifndef _TOR_CIRCUITLIST_H +#define _TOR_CIRCUITLIST_H + +circuit_t * _circuit_get_global_list(void); +const char *circuit_state_to_string(int state); +const char *circuit_purpose_to_controller_string(uint8_t purpose); +const char *circuit_purpose_to_string(uint8_t purpose); +void circuit_dump_by_conn(connection_t *conn, int severity); +void circuit_set_p_circid_orconn(or_circuit_t *circ, circid_t id, + or_connection_t *conn); +void circuit_set_n_circid_orconn(circuit_t *circ, circid_t id, + or_connection_t *conn); +void circuit_set_state(circuit_t *circ, uint8_t state); +void circuit_close_all_marked(void); +int32_t circuit_initial_package_window(void); +origin_circuit_t *origin_circuit_new(void); +or_circuit_t *or_circuit_new(circid_t p_circ_id, or_connection_t *p_conn); +circuit_t *circuit_get_by_circid_orconn(circid_t circ_id, + or_connection_t *conn); +int circuit_id_in_use_on_orconn(circid_t circ_id, or_connection_t *conn); +circuit_t *circuit_get_by_edge_conn(edge_connection_t *conn); +void circuit_unlink_all_from_or_conn(or_connection_t *conn, int reason); +origin_circuit_t *circuit_get_by_global_id(uint32_t id); +origin_circuit_t *circuit_get_by_rend_query_and_purpose(const char *rend_query, + uint8_t purpose); +origin_circuit_t *circuit_get_next_by_pk_and_purpose(origin_circuit_t *start, + const char *digest, uint8_t purpose); +or_circuit_t *circuit_get_rendezvous(const char *cookie); +or_circuit_t *circuit_get_intro_point(const char *digest); +origin_circuit_t *circuit_find_to_cannibalize(uint8_t purpose, + extend_info_t *info, int flags); +void circuit_mark_all_unused_circs(void); +void circuit_expire_all_dirty_circs(void); +void _circuit_mark_for_close(circuit_t *circ, int reason, + int line, const char *file); +int circuit_get_cpath_len(origin_circuit_t *circ); +crypt_path_t *circuit_get_cpath_hop(origin_circuit_t *circ, int hopnum); +void circuit_get_all_pending_on_or_conn(smartlist_t *out, + or_connection_t *or_conn); +int circuit_count_pending_on_or_conn(or_connection_t *or_conn); + +#define circuit_mark_for_close(c, reason) \ + _circuit_mark_for_close((c), (reason), __LINE__, _SHORT_FILE_) + +void assert_cpath_layer_ok(const crypt_path_t *cp); +void assert_circuit_ok(const circuit_t *c); +void circuit_free_all(void); + +#endif + diff --git a/src/or/circuituse.c b/src/or/circuituse.c index 41c1899c3a..7289aa5c11 100644 --- a/src/or/circuituse.c +++ b/src/or/circuituse.c @@ -10,6 +10,20 @@ **/ #include "or.h" +#include "circuitbuild.h" +#include "circuitlist.h" +#include "circuituse.h" +#include "config.h" +#include "connection.h" +#include "connection_edge.h" +#include "control.h" +#include "policies.h" +#include "rendclient.h" +#include "rendcommon.h" +#include "rendservice.h" +#include "rephist.h" +#include "router.h" +#include "routerlist.h" /********* START VARIABLES **********/ @@ -17,7 +31,7 @@ extern circuit_t *global_circuitlist; /* from circuitlist.c */ /********* END VARIABLES ************/ -static void circuit_expire_old_circuits_clientside(time_t now); +static void circuit_expire_old_circuits_clientside(void); static void circuit_increment_failure_count(void); /** Return 1 if <b>circ</b> could be returned by circuit_get_best(). @@ -148,10 +162,14 @@ circuit_is_better(circuit_t *a, circuit_t *b, uint8_t purpose) return 1; } else { if (a->timestamp_dirty || - a->timestamp_created > b->timestamp_created) + timercmp(&a->timestamp_created, &b->timestamp_created, >)) return 1; if (CIRCUIT_IS_ORIGIN(b) && TO_ORIGIN_CIRCUIT(b)->build_state->is_internal) + /* XXX023 what the heck is this internal thing doing here. I + * think we can get rid of it. circuit_is_acceptable() already + * makes sure that is_internal is exactly what we need it to + * be. -RD */ return 1; } break; @@ -190,7 +208,7 @@ circuit_get_best(edge_connection_t *conn, int must_be_open, uint8_t purpose, int need_uptime, int need_internal) { circuit_t *circ, *best=NULL; - time_t now = time(NULL); + struct timeval now; int intro_going_on_but_too_old = 0; tor_assert(conn); @@ -199,17 +217,16 @@ circuit_get_best(edge_connection_t *conn, int must_be_open, uint8_t purpose, purpose == CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT || purpose == CIRCUIT_PURPOSE_C_REND_JOINED); + tor_gettimeofday(&now); + for (circ=global_circuitlist;circ;circ = circ->next) { if (!circuit_is_acceptable(circ,conn,must_be_open,purpose, - need_uptime,need_internal,now)) + need_uptime,need_internal,now.tv_sec)) continue; -/* XXX022 make this 15 be a function of circuit finishing times we've - * seen lately, a la Fallon Chen's GSoC work -RD */ -#define REND_PARALLEL_INTRO_DELAY 15 if (purpose == CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT && - !must_be_open && circ->state != CIRCUIT_STATE_OPEN && - circ->timestamp_created + REND_PARALLEL_INTRO_DELAY < now) { + !must_be_open && circ->state != CIRCUIT_STATE_OPEN && + tv_mdiff(&now, &circ->timestamp_created) > circ_times.timeout_ms) { intro_going_on_but_too_old = 1; continue; } @@ -229,50 +246,73 @@ circuit_get_best(edge_connection_t *conn, int must_be_open, uint8_t purpose, return best ? TO_ORIGIN_CIRCUIT(best) : NULL; } +#if 0 /** Check whether, according to the policies in <b>options</b>, the * circuit <b>circ</b> makes sense. */ -/* XXXX currently only checks Exclude{Exit}Nodes. It should check more. */ +/* XXXX currently only checks Exclude{Exit}Nodes; it should check more. + * Also, it doesn't have the right definition of an exit circuit. Also, + * it's never called. */ int circuit_conforms_to_options(const origin_circuit_t *circ, const or_options_t *options) { const crypt_path_t *cpath, *cpath_next = NULL; - for (cpath = circ->cpath; cpath && cpath_next != circ->cpath; - cpath = cpath_next) { + /* first check if it includes any excluded nodes */ + for (cpath = circ->cpath; cpath_next != circ->cpath; cpath = cpath_next) { cpath_next = cpath->next; - if (routerset_contains_extendinfo(options->ExcludeNodes, cpath->extend_info)) return 0; + } - if (cpath->next == circ->cpath) { - /* This is apparently the exit node. */ + /* then consider the final hop */ + if (routerset_contains_extendinfo(options->ExcludeExitNodes, + circ->cpath->prev->extend_info)) + return 0; - if (routerset_contains_extendinfo(options->ExcludeExitNodes, - cpath->extend_info)) - return 0; - } - } return 1; } +#endif /** Close all circuits that start at us, aren't open, and were born * at least CircuitBuildTimeout seconds ago. */ void -circuit_expire_building(time_t now) +circuit_expire_building(void) { - circuit_t *victim, *circ = global_circuitlist; - time_t general_cutoff = now - get_options()->CircuitBuildTimeout; - time_t begindir_cutoff = now - get_options()->CircuitBuildTimeout/2; - time_t introcirc_cutoff = begindir_cutoff; + circuit_t *victim, *next_circ = global_circuitlist; + /* circ_times.timeout_ms and circ_times.close_ms are from + * circuit_build_times_get_initial_timeout() if we haven't computed + * custom timeouts yet */ + struct timeval general_cutoff, begindir_cutoff, fourhop_cutoff, + cannibalize_cutoff, close_cutoff, extremely_old_cutoff; + struct timeval now; + struct timeval introcirc_cutoff; cpath_build_state_t *build_state; - while (circ) { - time_t cutoff; - victim = circ; - circ = circ->next; + tor_gettimeofday(&now); +#define SET_CUTOFF(target, msec) do { \ + long ms = tor_lround(msec); \ + struct timeval diff; \ + diff.tv_sec = ms / 1000; \ + diff.tv_usec = (int)((ms % 1000) * 1000); \ + timersub(&now, &diff, &target); \ + } while (0) + + SET_CUTOFF(general_cutoff, circ_times.timeout_ms); + SET_CUTOFF(begindir_cutoff, circ_times.timeout_ms / 2.0); + SET_CUTOFF(fourhop_cutoff, circ_times.timeout_ms * (4/3.0)); + SET_CUTOFF(cannibalize_cutoff, circ_times.timeout_ms / 2.0); + SET_CUTOFF(close_cutoff, circ_times.close_ms); + SET_CUTOFF(extremely_old_cutoff, circ_times.close_ms*2 + 1000); + + introcirc_cutoff = begindir_cutoff; + + while (next_circ) { + struct timeval cutoff; + victim = next_circ; + next_circ = next_circ->next; if (!CIRCUIT_IS_ORIGIN(victim) || /* didn't originate here */ victim->marked_for_close) /* don't mess with marked circs */ continue; @@ -280,11 +320,19 @@ circuit_expire_building(time_t now) build_state = TO_ORIGIN_CIRCUIT(victim)->build_state; if (build_state && build_state->onehop_tunnel) cutoff = begindir_cutoff; + else if (build_state && build_state->desired_path_len == 4 + && !TO_ORIGIN_CIRCUIT(victim)->has_opened) + cutoff = fourhop_cutoff; + else if (TO_ORIGIN_CIRCUIT(victim)->has_opened) + cutoff = cannibalize_cutoff; else if (victim->purpose == CIRCUIT_PURPOSE_C_INTRODUCING) cutoff = introcirc_cutoff; + else if (victim->purpose == CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) + cutoff = close_cutoff; else cutoff = general_cutoff; - if (victim->timestamp_created > cutoff) + + if (timercmp(&victim->timestamp_created, &cutoff, >)) continue; /* it's still young, leave it alone */ #if 0 @@ -330,7 +378,7 @@ circuit_expire_building(time_t now) * because that's set when they switch purposes */ if (TO_ORIGIN_CIRCUIT(victim)->rend_data || - victim->timestamp_dirty > cutoff) + victim->timestamp_dirty > cutoff.tv_sec) continue; break; case CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED: @@ -339,10 +387,60 @@ circuit_expire_building(time_t now) * make an introduction attempt. so timestamp_dirty * will reflect the time since the last attempt. */ - if (victim->timestamp_dirty > cutoff) + if (victim->timestamp_dirty > cutoff.tv_sec) continue; break; } + } else { /* circuit not open, consider recording failure as timeout */ + int first_hop_succeeded = TO_ORIGIN_CIRCUIT(victim)->cpath && + TO_ORIGIN_CIRCUIT(victim)->cpath->state == CPATH_STATE_OPEN; + + if (TO_ORIGIN_CIRCUIT(victim)->p_streams != NULL) { + log_warn(LD_BUG, "Circuit %d (purpose %d, %s) has timed out, " + "yet has attached streams!", + TO_ORIGIN_CIRCUIT(victim)->global_identifier, + victim->purpose, + circuit_purpose_to_string(victim->purpose)); + tor_fragile_assert(); + continue; + } + + if (circuit_timeout_want_to_count_circ(TO_ORIGIN_CIRCUIT(victim)) && + circuit_build_times_enough_to_compute(&circ_times)) { + /* Circuits are allowed to last longer for measurement. + * Switch their purpose and wait. */ + if (victim->purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) { + control_event_circuit_status(TO_ORIGIN_CIRCUIT(victim), + CIRC_EVENT_FAILED, + END_CIRC_REASON_TIMEOUT); + victim->purpose = CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT; + /* Record this failure to check for too many timeouts + * in a row. This function does not record a time value yet + * (we do that later); it only counts the fact that we did + * have a timeout. */ + circuit_build_times_count_timeout(&circ_times, + first_hop_succeeded); + continue; + } + + /* + * If the circuit build time is much greater than we would have cut + * it off at, we probably had a suspend event along this codepath, + * and we should discard the value. + */ + if (timercmp(&victim->timestamp_created, &extremely_old_cutoff, <)) { + log_notice(LD_CIRC, + "Extremely large value for circuit build timeout: %lds. " + "Assuming clock jump. Purpose %d (%s)", + (long)(now.tv_sec - victim->timestamp_created.tv_sec), + victim->purpose, + circuit_purpose_to_string(victim->purpose)); + } else if (circuit_build_times_count_close(&circ_times, + first_hop_succeeded, + victim->timestamp_created.tv_sec)) { + circuit_build_times_set_timeout(&circ_times); + } + } } if (victim->n_conn) @@ -357,7 +455,10 @@ circuit_expire_building(time_t now) circuit_state_to_string(victim->state), victim->purpose); circuit_log_path(LOG_INFO,LD_CIRC,TO_ORIGIN_CIRCUIT(victim)); - circuit_mark_for_close(victim, END_CIRC_REASON_TIMEOUT); + if (victim->purpose == CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) + circuit_mark_for_close(victim, END_CIRC_REASON_MEASUREMENT_EXPIRED); + else + circuit_mark_for_close(victim, END_CIRC_REASON_TIMEOUT); } } @@ -431,11 +532,11 @@ circuit_stream_is_being_handled(edge_connection_t *conn, } /** Don't keep more than this many unused open circuits around. */ -#define MAX_UNUSED_OPEN_CIRCUITS 12 +#define MAX_UNUSED_OPEN_CIRCUITS 14 /** Figure out how many circuits we have open that are clean. Make * sure it's enough for all the upcoming behaviors we predict we'll have. - * But if we have too many, close the not-so-useful ones. + * But put an upper bound on the total number of circuits. */ static void circuit_predict_and_launch_new(void) @@ -517,6 +618,19 @@ circuit_predict_and_launch_new(void) circuit_launch_by_router(CIRCUIT_PURPOSE_C_GENERAL, NULL, flags); return; } + + /* Finally, check to see if we still need more circuits to learn + * a good build timeout. But if we're close to our max number we + * want, don't do another -- we want to leave a few slots open so + * we can still build circuits preemptively as needed. */ + if (num < MAX_UNUSED_OPEN_CIRCUITS-2 && + circuit_build_times_needs_circuits_now(&circ_times)) { + flags = CIRCLAUNCH_NEED_CAPACITY; + log_info(LD_CIRC, + "Have %d clean circs need another buildtime test circ.", num); + circuit_launch_by_router(CIRCUIT_PURPOSE_C_GENERAL, NULL, flags); + return; + } } /** Build a new test circuit every 5 minutes */ @@ -544,7 +658,7 @@ circuit_build_needed_circs(time_t now) time_to_new_circuit = now + options->NewCircuitPeriod; if (proxy_mode(get_options())) addressmap_clean(now); - circuit_expire_old_circuits_clientside(now); + circuit_expire_old_circuits_clientside(); #if 0 /* disable for now, until predict-and-launch-new can cull leftovers */ circ = circuit_get_youngest_clean_open(CIRCUIT_PURPOSE_C_GENERAL); @@ -624,37 +738,77 @@ circuit_detach_stream(circuit_t *circ, edge_connection_t *conn) tor_fragile_assert(); } +/** If we haven't yet decided on a good timeout value for circuit + * building, we close idles circuits aggressively so we can get more + * data points. */ +#define IDLE_TIMEOUT_WHILE_LEARNING (10*60) + /** Find each circuit that has been unused for too long, or dirty * for too long and has no streams on it: mark it for close. */ static void -circuit_expire_old_circuits_clientside(time_t now) +circuit_expire_old_circuits_clientside(void) { circuit_t *circ; - time_t cutoff = now - get_options()->CircuitIdleTimeout; + struct timeval cutoff, now; + + tor_gettimeofday(&now); + cutoff = now; + + if (circuit_build_times_needs_circuits(&circ_times)) { + /* Circuits should be shorter lived if we need more of them + * for learning a good build timeout */ + cutoff.tv_sec -= IDLE_TIMEOUT_WHILE_LEARNING; + } else { + cutoff.tv_sec -= get_options()->CircuitIdleTimeout; + } for (circ = global_circuitlist; circ; circ = circ->next) { - if (circ->marked_for_close || ! CIRCUIT_IS_ORIGIN(circ)) + if (circ->marked_for_close || !CIRCUIT_IS_ORIGIN(circ)) continue; /* If the circuit has been dirty for too long, and there are no streams * on it, mark it for close. */ if (circ->timestamp_dirty && - circ->timestamp_dirty + get_options()->MaxCircuitDirtiness < now && + circ->timestamp_dirty + get_options()->MaxCircuitDirtiness < + now.tv_sec && !TO_ORIGIN_CIRCUIT(circ)->p_streams /* nothing attached */ ) { - log_debug(LD_CIRC, "Closing n_circ_id %d (dirty %d secs ago, " + log_debug(LD_CIRC, "Closing n_circ_id %d (dirty %ld sec ago, " "purpose %d)", - circ->n_circ_id, (int)(now - circ->timestamp_dirty), + circ->n_circ_id, (long)(now.tv_sec - circ->timestamp_dirty), circ->purpose); circuit_mark_for_close(circ, END_CIRC_REASON_FINISHED); - } else if (!circ->timestamp_dirty && - circ->state == CIRCUIT_STATE_OPEN && - circ->purpose == CIRCUIT_PURPOSE_C_GENERAL) { - if (circ->timestamp_created < cutoff) { - log_debug(LD_CIRC, - "Closing circuit that has been unused for %d seconds.", - (int)(now - circ->timestamp_created)); - circuit_mark_for_close(circ, END_CIRC_REASON_FINISHED); + } else if (!circ->timestamp_dirty && circ->state == CIRCUIT_STATE_OPEN) { + if (timercmp(&circ->timestamp_created, &cutoff, <)) { + if (circ->purpose == CIRCUIT_PURPOSE_C_GENERAL || + circ->purpose == CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT || + circ->purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO || + circ->purpose == CIRCUIT_PURPOSE_TESTING || + (circ->purpose >= CIRCUIT_PURPOSE_C_INTRODUCING && + circ->purpose <= CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED) || + circ->purpose == CIRCUIT_PURPOSE_S_CONNECT_REND) { + log_debug(LD_CIRC, + "Closing circuit that has been unused for %ld msec.", + tv_mdiff(&circ->timestamp_created, &now)); + circuit_mark_for_close(circ, END_CIRC_REASON_FINISHED); + } else if (!TO_ORIGIN_CIRCUIT(circ)->is_ancient) { + /* Server-side rend joined circuits can end up really old, because + * they are reused by clients for longer than normal. The client + * controls their lifespan. (They never become dirty, because + * connection_exit_begin_conn() never marks anything as dirty.) + * Similarly, server-side intro circuits last a long time. */ + if (circ->purpose != CIRCUIT_PURPOSE_S_REND_JOINED && + circ->purpose != CIRCUIT_PURPOSE_S_INTRO) { + log_notice(LD_CIRC, + "Ancient non-dirty circuit %d is still around after " + "%ld milliseconds. Purpose: %d (%s)", + TO_ORIGIN_CIRCUIT(circ)->global_identifier, + tv_mdiff(&circ->timestamp_created, &now), + circ->purpose, + circuit_purpose_to_string(circ->purpose)); + TO_ORIGIN_CIRCUIT(circ)->is_ancient = 1; + } + } } } } @@ -794,6 +948,11 @@ circuit_has_opened(origin_circuit_t *circ) { control_event_circuit_status(circ, CIRC_EVENT_BUILT, 0); + /* Remember that this circuit has finished building. Now if we start + * it building again later (e.g. by extending it), we will know not + * to consider its build time. */ + circ->has_opened = 1; + switch (TO_CIRCUIT(circ)->purpose) { case CIRCUIT_PURPOSE_C_ESTABLISH_REND: rend_client_rendcirc_has_opened(circ); @@ -846,15 +1005,29 @@ circuit_build_failed(origin_circuit_t *circ) * to blame, blame it. Also, avoid this relay for a while, and * fail any one-hop directory fetches destined for it. */ const char *n_conn_id = circ->cpath->extend_info->identity_digest; + int already_marked = 0; if (circ->_base.n_conn) { or_connection_t *n_conn = circ->_base.n_conn; + if (n_conn->is_bad_for_new_circs) { + /* We only want to blame this router when a fresh healthy + * connection fails. So don't mark this router as newly failed, + * since maybe this was just an old circuit attempt that's + * finally timing out now. Also, there's no need to blow away + * circuits/streams/etc, since the failure of an unhealthy conn + * doesn't tell us much about whether a healthy conn would + * succeed. */ + already_marked = 1; + } log_info(LD_OR, "Our circuit failed to get a response from the first hop " "(%s:%d). I'm going to try to rotate to a better connection.", n_conn->_base.address, n_conn->_base.port); n_conn->is_bad_for_new_circs = 1; + } else { + log_info(LD_OR, + "Our circuit died before the first hop with no connection"); } - if (n_conn_id) { + if (n_conn_id && !already_marked) { entry_guard_register_connect_status(n_conn_id, 0, 1, time(NULL)); /* if there are any one-hop streams waiting on this circuit, fail * them now so they can retry elsewhere. */ @@ -936,8 +1109,8 @@ circuit_launch_by_router(uint8_t purpose, if (exit) info = extend_info_from_router(exit); circ = circuit_launch_by_extend_info(purpose, info, flags); - if (info) - extend_info_free(info); + + extend_info_free(info); return circ; } @@ -969,13 +1142,14 @@ circuit_launch_by_extend_info(uint8_t purpose, * internal circs rather than exit circs? -RD */ circ = circuit_find_to_cannibalize(purpose, extend_info, flags); if (circ) { - log_info(LD_CIRC,"Cannibalizing circ '%s' for purpose %d", - build_state_get_exit_nickname(circ->build_state), purpose); + log_info(LD_CIRC,"Cannibalizing circ '%s' for purpose %d (%s)", + build_state_get_exit_nickname(circ->build_state), purpose, + circuit_purpose_to_string(purpose)); circ->_base.purpose = purpose; /* reset the birth date of this circ, else expire_building * will see it and think it's been trying to build since it * began. */ - circ->_base.timestamp_created = time(NULL); + tor_gettimeofday(&circ->_base.timestamp_created); switch (purpose) { case CIRCUIT_PURPOSE_C_ESTABLISH_REND: case CIRCUIT_PURPOSE_S_ESTABLISH_INTRO: @@ -1080,15 +1254,17 @@ circuit_get_open_circ_or_launch(edge_connection_t *conn, int severity = LOG_NOTICE; /* FFFF if this is a tunneled directory fetch, don't yell * as loudly. the user doesn't even know it's happening. */ - if (options->UseBridges && bridges_known_but_down()) { + if (entry_list_is_constrained(options) && + entries_known_but_down(options)) { log_fn(severity, LD_APP|LD_DIR, - "Application request when we're believed to be " - "offline. Optimistically trying known bridges again."); - bridges_retry_all(); + "Application request when we haven't used client functionality " + "lately. Optimistically trying known %s again.", + options->UseBridges ? "bridges" : "entrynodes"); + entries_retry_all(options); } else if (!options->UseBridges || any_bridge_descriptors_known()) { log_fn(severity, LD_APP|LD_DIR, - "Application request when we're believed to be " - "offline. Optimistically trying directory fetches again."); + "Application request when we haven't used client functionality " + "lately. Optimistically trying directory fetches again."); routerlist_retry_directory_downloads(time(NULL)); } } @@ -1111,17 +1287,19 @@ circuit_get_open_circ_or_launch(edge_connection_t *conn, need_uptime)) { log_notice(LD_APP, "No Tor server allows exit to %s:%d. Rejecting.", - safe_str(conn->socks_request->address), + safe_str_client(conn->socks_request->address), conn->socks_request->port); return -1; } } else { - /* XXXX022 Duplicates checks in connection_ap_handshake_attach_circuit */ + /* XXXX023 Duplicates checks in connection_ap_handshake_attach_circuit: + * refactor into a single function? */ routerinfo_t *router = router_get_by_nickname(conn->chosen_exit_name, 1); int opt = conn->chosen_exit_optional; if (router && !connection_ap_can_use_exit(conn, router)) { log_fn(opt ? LOG_INFO : LOG_WARN, LD_APP, - "Requested exit point '%s' would refuse request. %s.", + "Requested exit point '%s' is excluded or " + "would refuse request. %s.", conn->chosen_exit_name, opt ? "Trying others" : "Closing"); if (opt) { conn->chosen_exit_optional = 0; @@ -1152,19 +1330,14 @@ circuit_get_open_circ_or_launch(edge_connection_t *conn, if (!extend_info) { log_info(LD_REND, "No intro points for '%s': re-fetching service descriptor.", - safe_str(conn->rend_data->onion_address)); - /* Fetch both, v0 and v2 rend descriptors in parallel. Use whichever - * arrives first. Exception: When using client authorization, only - * fetch v2 descriptors.*/ + safe_str_client(conn->rend_data->onion_address)); rend_client_refetch_v2_renddesc(conn->rend_data); - if (conn->rend_data->auth_type == REND_NO_AUTH) - rend_client_refetch_renddesc(conn->rend_data->onion_address); conn->_base.state = AP_CONN_STATE_RENDDESC_WAIT; return 0; } log_info(LD_REND,"Chose '%s' as intro point for '%s'.", extend_info->nickname, - safe_str(conn->rend_data->onion_address)); + safe_str_client(conn->rend_data->onion_address)); } /* If we have specified a particular exit node for our @@ -1193,7 +1366,7 @@ circuit_get_open_circ_or_launch(edge_connection_t *conn, } if (tor_addr_from_str(&addr, conn->socks_request->address) < 0) { log_info(LD_DIR, "Broken address %s on tunnel conn. Closing.", - escaped_safe_str(conn->socks_request->address)); + escaped_safe_str_client(conn->socks_request->address)); return -1; } extend_info = extend_info_alloc(conn->chosen_exit_name+1, @@ -1235,10 +1408,20 @@ circuit_get_open_circ_or_launch(edge_connection_t *conn, flags); } - if (extend_info) - extend_info_free(extend_info); + extend_info_free(extend_info); - if (desired_circuit_purpose != CIRCUIT_PURPOSE_C_GENERAL) { + if (desired_circuit_purpose == CIRCUIT_PURPOSE_C_GENERAL) { + /* We just caused a circuit to get built because of this stream. + * If this stream has caused a _lot_ of circuits to be built, that's + * a bad sign: we should tell the user. */ + if (conn->num_circuits_launched < NUM_CIRCUITS_LAUNCHED_THRESHOLD && + ++conn->num_circuits_launched == NUM_CIRCUITS_LAUNCHED_THRESHOLD) + log_info(LD_CIRC, "The application request to %s:%d has launched " + "%d circuits without finding one it likes.", + escaped_safe_str_client(conn->socks_request->address), + conn->socks_request->port, + conn->num_circuits_launched); + } else { /* help predict this next time */ rep_hist_note_used_internal(time(NULL), need_uptime, 1); if (circ) { @@ -1302,15 +1485,35 @@ link_apconn_to_circ(edge_connection_t *apconn, origin_circuit_t *circ, } } -/** If an exit wasn't specifically chosen, save the history for future - * use. */ +/** Return true iff <b>address</b> is matched by one of the entries in + * TrackHostExits. */ +int +hostname_in_track_host_exits(or_options_t *options, const char *address) +{ + if (!options->TrackHostExits) + return 0; + SMARTLIST_FOREACH_BEGIN(options->TrackHostExits, const char *, cp) { + if (cp[0] == '.') { /* match end */ + if (cp[1] == '\0' || + !strcasecmpend(address, cp) || + !strcasecmp(address, &cp[1])) + return 1; + } else if (strcasecmp(cp, address) == 0) { + return 1; + } + } SMARTLIST_FOREACH_END(cp); + return 0; +} + +/** If an exit wasn't explicitly specified for <b>conn</b>, consider saving + * the exit that we *did* choose for use by future connections to + * <b>conn</b>'s destination. + */ static void consider_recording_trackhost(edge_connection_t *conn, origin_circuit_t *circ) { - int found_needle = 0; or_options_t *options = get_options(); - size_t len; - char *new_address; + char *new_address = NULL; char fp[HEX_DIGEST_LEN+1]; /* Search the addressmap for this conn's destination. */ @@ -1320,18 +1523,8 @@ consider_recording_trackhost(edge_connection_t *conn, origin_circuit_t *circ) options->TrackHostExitsExpire)) return; /* nothing to track, or already mapped */ - SMARTLIST_FOREACH(options->TrackHostExits, const char *, cp, { - if (cp[0] == '.') { /* match end */ - if (cp[1] == '\0' || - !strcasecmpend(conn->socks_request->address, cp) || - !strcasecmp(conn->socks_request->address, &cp[1])) - found_needle = 1; - } else if (strcasecmp(cp, conn->socks_request->address) == 0) { - found_needle = 1; - } - }); - - if (!found_needle || !circ->build_state->chosen_exit) + if (!hostname_in_track_host_exits(options, conn->socks_request->address) || + !circ->build_state->chosen_exit) return; /* write down the fingerprint of the chosen exit, not the nickname, @@ -1340,12 +1533,7 @@ consider_recording_trackhost(edge_connection_t *conn, origin_circuit_t *circ) circ->build_state->chosen_exit->identity_digest, DIGEST_LEN); /* Add this exit/hostname pair to the addressmap. */ - len = strlen(conn->socks_request->address) + 1 /* '.' */ + - strlen(fp) + 1 /* '.' */ + - strlen("exit") + 1 /* '\0' */; - new_address = tor_malloc(len); - - tor_snprintf(new_address, len, "%s.%s.exit", + tor_asprintf(&new_address, "%s.%s.exit", conn->socks_request->address, fp); addressmap_register(conn->socks_request->address, new_address, @@ -1418,7 +1606,7 @@ connection_ap_handshake_attach_circuit(edge_connection_t *conn) LOG_INFO : LOG_NOTICE; log_fn(severity, LD_APP, "Tried for %d seconds to get a connection to %s:%d. Giving up.", - conn_age, safe_str(conn->socks_request->address), + conn_age, safe_str_client(conn->socks_request->address), conn->socks_request->port); return -1; } @@ -1447,7 +1635,8 @@ connection_ap_handshake_attach_circuit(edge_connection_t *conn) } if (router && !connection_ap_can_use_exit(conn, router)) { log_fn(opt ? LOG_INFO : LOG_WARN, LD_APP, - "Requested exit point '%s' would refuse request. %s.", + "Requested exit point '%s' is excluded or " + "would refuse request. %s.", conn->chosen_exit_name, opt ? "Trying others" : "Closing"); if (opt) { conn->chosen_exit_optional = 0; @@ -1461,7 +1650,7 @@ connection_ap_handshake_attach_circuit(edge_connection_t *conn) /* find the circuit that we should use, if there is one. */ retval = circuit_get_open_circ_or_launch( conn, CIRCUIT_PURPOSE_C_GENERAL, &circ); - if (retval < 1) // XXX021 if we totally fail, this still returns 0 -RD + if (retval < 1) // XXX022 if we totally fail, this still returns 0 -RD return retval; log_debug(LD_APP|LD_CIRC, diff --git a/src/or/circuituse.h b/src/or/circuituse.h new file mode 100644 index 0000000000..bfeaea20dc --- /dev/null +++ b/src/or/circuituse.h @@ -0,0 +1,57 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file circuituse.h + * \brief Header file for circuituse.c. + **/ + +#ifndef _TOR_CIRCUITUSE_H +#define _TOR_CIRCUITUSE_H + +void circuit_expire_building(void); +void circuit_remove_handled_ports(smartlist_t *needed_ports); +int circuit_stream_is_being_handled(edge_connection_t *conn, uint16_t port, + int min); +#if 0 +int circuit_conforms_to_options(const origin_circuit_t *circ, + const or_options_t *options); +#endif +void circuit_build_needed_circs(time_t now); +void circuit_detach_stream(circuit_t *circ, edge_connection_t *conn); + +void circuit_expire_old_circuits_serverside(time_t now); + +void reset_bandwidth_test(void); +int circuit_enough_testing_circs(void); + +void circuit_has_opened(origin_circuit_t *circ); +void circuit_build_failed(origin_circuit_t *circ); + +/** Flag to set when a circuit should have only a single hop. */ +#define CIRCLAUNCH_ONEHOP_TUNNEL (1<<0) +/** Flag to set when a circuit needs to be built of high-uptime nodes */ +#define CIRCLAUNCH_NEED_UPTIME (1<<1) +/** Flag to set when a circuit needs to be built of high-capacity nodes */ +#define CIRCLAUNCH_NEED_CAPACITY (1<<2) +/** Flag to set when the last hop of a circuit doesn't need to be an + * exit node. */ +#define CIRCLAUNCH_IS_INTERNAL (1<<3) +origin_circuit_t *circuit_launch_by_extend_info(uint8_t purpose, + extend_info_t *info, + int flags); +origin_circuit_t *circuit_launch_by_router(uint8_t purpose, + routerinfo_t *exit, int flags); +void circuit_reset_failure_count(int timeout); +int connection_ap_handshake_attach_chosen_circuit(edge_connection_t *conn, + origin_circuit_t *circ, + crypt_path_t *cpath); +int connection_ap_handshake_attach_circuit(edge_connection_t *conn); + +int hostname_in_track_host_exits(or_options_t *options, const char *address); + +#endif + diff --git a/src/or/command.c b/src/or/command.c index 9269456805..e377f4fb67 100644 --- a/src/or/command.c +++ b/src/or/command.c @@ -16,6 +16,19 @@ */ #include "or.h" +#include "circuitbuild.h" +#include "circuitlist.h" +#include "command.h" +#include "connection.h" +#include "connection_or.h" +#include "config.h" +#include "control.h" +#include "cpuworker.h" +#include "hibernate.h" +#include "onion.h" +#include "relay.h" +#include "router.h" +#include "routerlist.h" /** How many CELL_PADDING cells have we received, ever? */ uint64_t stats_n_padding_cells_processed = 0; @@ -275,7 +288,14 @@ command_process_create_cell(cell_t *cell, or_connection_t *conn) /* hand it off to the cpuworkers, and then return. */ if (assign_onionskin_to_cpuworker(NULL, circ, onionskin) < 0) { - log_warn(LD_GENERAL,"Failed to hand off onionskin. Closing."); +#define WARN_HANDOFF_FAILURE_INTERVAL (6*60*60) + static ratelim_t handoff_warning = + RATELIM_INIT(WARN_HANDOFF_FAILURE_INTERVAL); + char *m; + if ((m = rate_limit_log(&handoff_warning, approx_time()))) { + log_warn(LD_GENERAL,"Failed to hand off onionskin. Closing.%s",m); + tor_free(m); + } circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_INTERNAL); return; } @@ -396,15 +416,18 @@ command_process_relay_cell(cell_t *cell, or_connection_t *conn) * gotten no more than MAX_RELAY_EARLY_CELLS_PER_CIRCUIT of them. */ if (cell->command == CELL_RELAY_EARLY) { if (direction == CELL_DIRECTION_IN) { - /* XXX Allow an unlimited number of inbound relay_early cells for - * now, for hidden service compatibility. See bug 1038. -RD */ + /* Allow an unlimited number of inbound relay_early cells, + * for hidden service compatibility. There isn't any way to make + * a long circuit through inbound relay_early cells anyway. See + * bug 1038. -RD */ } else { or_circuit_t *or_circ = TO_OR_CIRCUIT(circ); if (or_circ->remaining_relay_early_cells == 0) { log_fn(LOG_PROTOCOL_WARN, LD_OR, "Received too many RELAY_EARLY cells on circ %d from %s:%d." " Closing circuit.", - cell->circ_id, safe_str(conn->_base.address), conn->_base.port); + cell->circ_id, safe_str(conn->_base.address), + conn->_base.port); circuit_mark_for_close(circ, END_CIRC_REASON_TORPROTOCOL); return; } @@ -512,7 +535,8 @@ command_process_versions_cell(var_cell_t *cell, or_connection_t *conn) conn->handshake_state->received_versions = 1; log_info(LD_OR, "Negotiated version %d with %s:%d; sending NETINFO.", - highest_supported_version, safe_str(conn->_base.address), + highest_supported_version, + safe_str_client(conn->_base.address), conn->_base.port); tor_assert(conn->link_proto >= 2); @@ -621,14 +645,15 @@ command_process_netinfo_cell(cell_t *cell, or_connection_t *conn) /* XXX maybe act on my_apparent_addr, if the source is sufficiently * trustworthy. */ + (void)my_apparent_addr; if (connection_or_set_state_open(conn)<0) connection_mark_for_close(TO_CONN(conn)); else log_info(LD_OR, "Got good NETINFO cell from %s:%d; OR connection is now " "open, using protocol version %d", - safe_str(conn->_base.address), conn->_base.port, - (int)conn->link_proto); + safe_str_client(conn->_base.address), + conn->_base.port, (int)conn->link_proto); assert_connection_ok(TO_CONN(conn),time(NULL)); } diff --git a/src/or/command.h b/src/or/command.h new file mode 100644 index 0000000000..95b0f3a931 --- /dev/null +++ b/src/or/command.h @@ -0,0 +1,25 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file command.h + * \brief Header file for command.c. + **/ + +#ifndef _TOR_COMMAND_H +#define _TOR_COMMAND_H + +void command_process_cell(cell_t *cell, or_connection_t *conn); +void command_process_var_cell(var_cell_t *cell, or_connection_t *conn); + +extern uint64_t stats_n_padding_cells_processed; +extern uint64_t stats_n_create_cells_processed; +extern uint64_t stats_n_created_cells_processed; +extern uint64_t stats_n_relay_cells_processed; +extern uint64_t stats_n_destroy_cells_processed; + +#endif + diff --git a/src/or/config.c b/src/or/config.c index fbfa771ed3..46b0cef413 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -12,15 +12,41 @@ #define CONFIG_PRIVATE #include "or.h" +#include "circuitbuild.h" +#include "circuitlist.h" +#include "config.h" +#include "connection.h" +#include "connection_edge.h" +#include "connection_or.h" +#include "control.h" +#include "cpuworker.h" +#include "dirserv.h" +#include "dirvote.h" +#include "dns.h" +#include "geoip.h" +#include "hibernate.h" +#include "main.h" +#include "networkstatus.h" +#include "policies.h" +#include "relay.h" +#include "rendclient.h" +#include "rendservice.h" +#include "rephist.h" +#include "router.h" +#include "routerlist.h" #ifdef MS_WINDOWS #include <shlobj.h> #endif +#include "procmon.h" + /** Enumeration of types which option values can take */ typedef enum config_type_t { CONFIG_TYPE_STRING = 0, /**< An arbitrary string. */ CONFIG_TYPE_FILENAME, /**< A filename: some prefixes get expanded. */ CONFIG_TYPE_UINT, /**< A non-negative integer less than MAX_INT */ + CONFIG_TYPE_PORT, /**< A port from 1...65535, 0 for "not set", or + * "auto". */ CONFIG_TYPE_INTERVAL, /**< A number of seconds, with optional units*/ CONFIG_TYPE_MEMUNIT, /**< A number of bytes, with optional units*/ CONFIG_TYPE_DOUBLE, /**< A floating-point value */ @@ -61,11 +87,12 @@ static config_abbrev_t _option_abbrevs[] = { PLURAL(LongLivedPort), PLURAL(HiddenServiceNode), PLURAL(HiddenServiceExcludeNode), - PLURAL(NumCpu), + PLURAL(NumCPU), PLURAL(RendNode), PLURAL(RendExcludeNode), PLURAL(StrictEntryNode), PLURAL(StrictExitNode), + PLURAL(StrictNode), { "l", "Log", 1, 0}, { "AllowUnverifiedNodes", "AllowInvalidNodes", 0, 0}, { "AutomapHostSuffixes", "AutomapHostsSuffixes", 0, 0}, @@ -83,10 +110,12 @@ static config_abbrev_t _option_abbrevs[] = { { "NumEntryNodes", "NumEntryGuards", 0, 0}, { "ResolvConf", "ServerDNSResolvConfFile", 0, 1}, { "SearchDomains", "ServerDNSSearchDomains", 0, 1}, - { "ServerDNSAllowBrokenResolvConf", "ServerDNSAllowBrokenConfig", 0, 0 }, + { "ServerDNSAllowBrokenResolvConf", "ServerDNSAllowBrokenConfig", 0, 0}, { "PreferTunnelledDirConns", "PreferTunneledDirConns", 0, 0}, { "BridgeAuthoritativeDirectory", "BridgeAuthoritativeDir", 0, 0}, { "HashedControlPassword", "__HashedControlSessionPassword", 1, 0}, + { "StrictEntryNodes", "StrictNodes", 0, 1}, + { "StrictExitNodes", "StrictNodes", 0, 1}, { NULL, NULL, 0, 0}, }; @@ -134,6 +163,7 @@ static config_var_t _option_vars[] = { V(AccountingMax, MEMUNIT, "0 bytes"), V(AccountingStart, STRING, NULL), V(Address, STRING, NULL), + V(AllowDotExit, BOOL, "0"), V(AllowInvalidNodes, CSV, "middle,rendezvous"), V(AllowNonRFC953Hostnames, BOOL, "0"), V(AllowSingleHopCircuits, BOOL, "0"), @@ -162,17 +192,26 @@ static config_var_t _option_vars[] = { V(BridgePassword, STRING, NULL), V(BridgeRecordUsageByCountry, BOOL, "1"), V(BridgeRelay, BOOL, "0"), - V(CircuitBuildTimeout, INTERVAL, "1 minute"), + V(CellStatistics, BOOL, "0"), + V(LearnCircuitBuildTimeout, BOOL, "1"), + V(CircuitBuildTimeout, INTERVAL, "0"), V(CircuitIdleTimeout, INTERVAL, "1 hour"), + V(CircuitStreamTimeout, INTERVAL, "0"), + V(CircuitPriorityHalflife, DOUBLE, "-100.0"), /*negative:'Use default'*/ V(ClientDNSRejectInternalAddresses, BOOL,"1"), + V(ClientRejectInternalAddresses, BOOL, "1"), V(ClientOnly, BOOL, "0"), + V(ConsensusParams, STRING, NULL), V(ConnLimit, UINT, "1000"), V(ConstrainedSockets, BOOL, "0"), V(ConstrainedSockSize, MEMUNIT, "8192"), V(ContactInfo, STRING, NULL), V(ControlListenAddress, LINELIST, NULL), - V(ControlPort, UINT, "0"), + V(ControlPort, PORT, "0"), + V(ControlPortFileGroupReadable,BOOL, "0"), + V(ControlPortWriteToFile, FILENAME, NULL), V(ControlSocket, LINELIST, NULL), + V(ControlSocketsGroupWritable, BOOL, "0"), V(CookieAuthentication, BOOL, "0"), V(CookieAuthFileGroupReadable, BOOL, "0"), V(CookieAuthFile, STRING, NULL), @@ -183,21 +222,22 @@ static config_var_t _option_vars[] = { V(DirListenAddress, LINELIST, NULL), OBSOLETE("DirFetchPeriod"), V(DirPolicy, LINELIST, NULL), - V(DirPort, UINT, "0"), + V(DirPort, PORT, "0"), V(DirPortFrontPage, FILENAME, NULL), OBSOLETE("DirPostPeriod"), -#ifdef ENABLE_GEOIP_STATS - V(DirRecordUsageByCountry, BOOL, "0"), - V(DirRecordUsageGranularity, UINT, "4"), - V(DirRecordUsageRetainIPs, INTERVAL, "14 days"), - V(DirRecordUsageSaveInterval, INTERVAL, "6 hours"), -#endif + OBSOLETE("DirRecordUsageByCountry"), + OBSOLETE("DirRecordUsageGranularity"), + OBSOLETE("DirRecordUsageRetainIPs"), + OBSOLETE("DirRecordUsageSaveInterval"), + V(DirReqStatistics, BOOL, "0"), VAR("DirServer", LINELIST, DirServers, NULL), - V(DNSPort, UINT, "0"), + V(DisableAllSwap, BOOL, "0"), + V(DNSPort, PORT, "0"), V(DNSListenAddress, LINELIST, NULL), V(DownloadExtraInfo, BOOL, "0"), V(EnforceDistinctSubnets, BOOL, "1"), V(EntryNodes, ROUTERSET, NULL), + V(EntryStatistics, BOOL, "0"), V(TestingEstimatedDescriptorPropagationTime, INTERVAL, "10 minutes"), V(ExcludeNodes, ROUTERSET, NULL), V(ExcludeExitNodes, ROUTERSET, NULL), @@ -205,15 +245,24 @@ static config_var_t _option_vars[] = { V(ExitNodes, ROUTERSET, NULL), V(ExitPolicy, LINELIST, NULL), V(ExitPolicyRejectPrivate, BOOL, "1"), + V(ExitPortStatistics, BOOL, "0"), + V(ExtraInfoStatistics, BOOL, "0"), + +#if defined (WINCE) + V(FallbackNetworkstatusFile, FILENAME, "fallback-consensus"), +#else V(FallbackNetworkstatusFile, FILENAME, SHARE_DATADIR PATH_SEPARATOR "tor" PATH_SEPARATOR "fallback-consensus"), +#endif V(FascistFirewall, BOOL, "0"), V(FirewallPorts, CSV, ""), V(FastFirstHopPK, BOOL, "1"), V(FetchDirInfoEarly, BOOL, "0"), + V(FetchDirInfoExtraEarly, BOOL, "0"), V(FetchServerDescriptors, BOOL, "1"), V(FetchHidServDescriptors, BOOL, "1"), V(FetchUselessDescriptors, BOOL, "0"), + V(FetchV2Networkstatus, BOOL, "0"), #ifdef WIN32 V(GeoIPFile, FILENAME, "<default>"), #else @@ -222,6 +271,8 @@ static config_var_t _option_vars[] = { #endif OBSOLETE("Group"), V(HardwareAccel, BOOL, "0"), + V(AccelName, STRING, NULL), + V(AccelDir, FILENAME, NULL), V(HashedControlPassword, LINELIST, NULL), V(HidServDirectoryV2, BOOL, "1"), VAR("HiddenServiceDir", LINELIST_S, RendConfigLines, NULL), @@ -233,14 +284,19 @@ static config_var_t _option_vars[] = { VAR("HiddenServiceAuthorizeClient",LINELIST_S,RendConfigLines, NULL), V(HidServAuth, LINELIST, NULL), V(HSAuthoritativeDir, BOOL, "0"), - V(HSAuthorityRecordStats, BOOL, "0"), - V(HttpProxy, STRING, NULL), - V(HttpProxyAuthenticator, STRING, NULL), - V(HttpsProxy, STRING, NULL), - V(HttpsProxyAuthenticator, STRING, NULL), + OBSOLETE("HSAuthorityRecordStats"), + V(HTTPProxy, STRING, NULL), + V(HTTPProxyAuthenticator, STRING, NULL), + V(HTTPSProxy, STRING, NULL), + V(HTTPSProxyAuthenticator, STRING, NULL), + V(Socks4Proxy, STRING, NULL), + V(Socks5Proxy, STRING, NULL), + V(Socks5ProxyUsername, STRING, NULL), + V(Socks5ProxyPassword, STRING, NULL), OBSOLETE("IgnoreVersion"), V(KeepalivePeriod, INTERVAL, "5 minutes"), VAR("Log", LINELIST, Logs, NULL), + V(LogMessageDomains, BOOL, "0"), OBSOLETE("LinkPadding"), OBSOLETE("LogLevel"), OBSOLETE("LogFile"), @@ -254,17 +310,20 @@ static config_var_t _option_vars[] = { V(MyFamily, STRING, NULL), V(NewCircuitPeriod, INTERVAL, "30 seconds"), VAR("NamingAuthoritativeDirectory",BOOL, NamingAuthoritativeDir, "0"), - V(NatdListenAddress, LINELIST, NULL), - V(NatdPort, UINT, "0"), + V(NATDListenAddress, LINELIST, NULL), + V(NATDPort, PORT, "0"), V(Nickname, STRING, NULL), - V(NoPublish, BOOL, "0"), + V(WarnUnsafeSocks, BOOL, "1"), + OBSOLETE("NoPublish"), VAR("NodeFamily", LINELIST, NodeFamilies, NULL), - V(NumCpus, UINT, "1"), + V(NumCPUs, UINT, "1"), V(NumEntryGuards, UINT, "3"), V(ORListenAddress, LINELIST, NULL), - V(ORPort, UINT, "0"), + V(ORPort, PORT, "0"), V(OutboundBindAddress, STRING, NULL), OBSOLETE("PathlenCoinWeight"), + V(PerConnBWBurst, MEMUNIT, "0"), + V(PerConnBWRate, MEMUNIT, "0"), V(PidFile, STRING, NULL), V(TestingTorNetwork, BOOL, "0"), V(PreferTunneledDirConns, BOOL, "1"), @@ -278,6 +337,7 @@ static config_var_t _option_vars[] = { V(RecommendedClientVersions, LINELIST, NULL), V(RecommendedServerVersions, LINELIST, NULL), OBSOLETE("RedirectExit"), + V(RefuseUnknownExits, STRING, "auto"), V(RejectPlaintextPorts, CSV, ""), V(RelayBandwidthBurst, MEMUNIT, "0"), V(RelayBandwidthRate, MEMUNIT, "0"), @@ -287,8 +347,9 @@ static config_var_t _option_vars[] = { V(RephistTrackTime, INTERVAL, "24 hours"), OBSOLETE("RouterFile"), V(RunAsDaemon, BOOL, "0"), - V(RunTesting, BOOL, "0"), - V(SafeLogging, BOOL, "1"), +// V(RunTesting, BOOL, "0"), + OBSOLETE("RunTesting"), // currently unused + V(SafeLogging, STRING, "1"), V(SafeSocks, BOOL, "0"), V(ServerDNSAllowBrokenConfig, BOOL, "1"), V(ServerDNSAllowNonRFC953Hostnames, BOOL,"0"), @@ -301,11 +362,10 @@ static config_var_t _option_vars[] = { V(ShutdownWaitLength, INTERVAL, "30 seconds"), V(SocksListenAddress, LINELIST, NULL), V(SocksPolicy, LINELIST, NULL), - V(SocksPort, UINT, "9050"), + V(SocksPort, PORT, "9050"), V(SocksTimeout, INTERVAL, "2 minutes"), OBSOLETE("StatusFetchPeriod"), - V(StrictEntryNodes, BOOL, "0"), - V(StrictExitNodes, BOOL, "0"), + V(StrictNodes, BOOL, "0"), OBSOLETE("SysLog"), V(TestSocks, BOOL, "0"), OBSOLETE("TestVia"), @@ -313,7 +373,7 @@ static config_var_t _option_vars[] = { V(TrackHostExitsExpire, INTERVAL, "30 minutes"), OBSOLETE("TrafficShaping"), V(TransListenAddress, LINELIST, NULL), - V(TransPort, UINT, "0"), + V(TransPort, PORT, "0"), V(TunnelDirConns, BOOL, "1"), V(UpdateBridgesFromAuthority, BOOL, "0"), V(UseBridges, BOOL, "0"), @@ -330,6 +390,7 @@ static config_var_t _option_vars[] = { V(V3AuthDistDelay, INTERVAL, "5 minutes"), V(V3AuthNIntervalsValid, UINT, "3"), V(V3AuthUseLegacyKey, BOOL, "0"), + V(V3BandwidthsFile, FILENAME, NULL), VAR("VersioningAuthoritativeDirectory",BOOL,VersioningAuthoritativeDir, "0"), V(VirtualAddrNetwork, STRING, "127.192.0.0/10"), V(WarnPlaintextPorts, CSV, "23,109,110,143"), @@ -339,7 +400,10 @@ static config_var_t _option_vars[] = { VAR("__LeaveStreamsUnattached",BOOL, LeaveStreamsUnattached, "0"), VAR("__HashedControlSessionPassword", LINELIST, HashedControlSessionPassword, NULL), + VAR("__OwningControllerProcess",STRING,OwningControllerProcess, NULL), V(MinUptimeHidServDirectoryV2, INTERVAL, "24 hours"), + V(_UsingTestNetworkDefaults, BOOL, "0"), + { NULL, CONFIG_TYPE_OBSOLETE, 0, NULL } }; @@ -353,6 +417,7 @@ static config_var_t testing_tor_network_defaults[] = { V(AuthDirMaxServersPerAddr, UINT, "0"), V(AuthDirMaxServersPerAuthAddr,UINT, "0"), V(ClientDNSRejectInternalAddresses, BOOL,"0"), + V(ClientRejectInternalAddresses, BOOL, "0"), V(ExitPolicyRejectPrivate, BOOL, "0"), V(V3AuthVotingInterval, INTERVAL, "5 minutes"), V(V3AuthVoteDelay, INTERVAL, "20 seconds"), @@ -362,6 +427,8 @@ static config_var_t testing_tor_network_defaults[] = { V(TestingV3AuthInitialDistDelay, INTERVAL, "20 seconds"), V(TestingAuthDirTimeToLearnReachability, INTERVAL, "0 minutes"), V(TestingEstimatedDescriptorPropagationTime, INTERVAL, "0 minutes"), + V(MinUptimeHidServDirectoryV2, INTERVAL, "0 minutes"), + V(_UsingTestNetworkDefaults, BOOL, "1"), { NULL, CONFIG_TYPE_OBSOLETE, 0, NULL } }; #undef VAR @@ -377,6 +444,9 @@ static config_var_t _state_vars[] = { V(AccountingExpectedUsage, MEMUNIT, NULL), V(AccountingIntervalStart, ISOTIME, NULL), V(AccountingSecondsActive, INTERVAL, NULL), + V(AccountingSecondsToReachSoftLimit,INTERVAL, NULL), + V(AccountingSoftLimitHitAt, ISOTIME, NULL), + V(AccountingBytesAtSoftLimit, MEMUNIT, NULL), VAR("EntryGuard", LINELIST_S, EntryGuards, NULL), VAR("EntryGuardDownSince", LINELIST_S, EntryGuards, NULL), @@ -387,15 +457,30 @@ static config_var_t _state_vars[] = { V(BWHistoryReadEnds, ISOTIME, NULL), V(BWHistoryReadInterval, UINT, "900"), V(BWHistoryReadValues, CSV, ""), + V(BWHistoryReadMaxima, CSV, ""), V(BWHistoryWriteEnds, ISOTIME, NULL), V(BWHistoryWriteInterval, UINT, "900"), V(BWHistoryWriteValues, CSV, ""), + V(BWHistoryWriteMaxima, CSV, ""), + V(BWHistoryDirReadEnds, ISOTIME, NULL), + V(BWHistoryDirReadInterval, UINT, "900"), + V(BWHistoryDirReadValues, CSV, ""), + V(BWHistoryDirReadMaxima, CSV, ""), + V(BWHistoryDirWriteEnds, ISOTIME, NULL), + V(BWHistoryDirWriteInterval, UINT, "900"), + V(BWHistoryDirWriteValues, CSV, ""), + V(BWHistoryDirWriteMaxima, CSV, ""), V(TorVersion, STRING, NULL), V(LastRotatedOnionKey, ISOTIME, NULL), V(LastWritten, ISOTIME, NULL), + V(TotalBuildTimes, UINT, NULL), + V(CircuitBuildAbandonedCount, UINT, "0"), + VAR("CircuitBuildTimeBin", LINELIST_S, BuildtimeHistogram, NULL), + VAR("BuildtimeHistogram", LINELIST_V, BuildtimeHistogram, NULL), + { NULL, CONFIG_TYPE_OBSOLETE, 0, NULL } }; @@ -410,213 +495,6 @@ typedef struct config_var_description_t { const char *description; } config_var_description_t; -/** Descriptions of the configuration options, to be displayed by online - * option browsers */ -/* XXXX022 did anybody want this? at all? If not, kill it.*/ -static config_var_description_t options_description[] = { - /* ==== general options */ - { "AvoidDiskWrites", "If non-zero, try to write to disk less frequently than" - " we would otherwise." }, - { "BandwidthRate", "A token bucket limits the average incoming bandwidth on " - "this node to the specified number of bytes per second." }, - { "BandwidthBurst", "Limit the maximum token buffer size (also known as " - "burst) to the given number of bytes." }, - { "ConnLimit", "Minimum number of simultaneous sockets we must have." }, - { "ConstrainedSockets", "Shrink tx and rx buffers for sockets to avoid " - "system limits on vservers and related environments. See man page for " - "more information regarding this option." }, - { "ConstrainedSockSize", "Limit socket buffers to this size when " - "ConstrainedSockets is enabled." }, - /* ControlListenAddress */ - { "ControlPort", "If set, Tor will accept connections from the same machine " - "(localhost only) on this port, and allow those connections to control " - "the Tor process using the Tor Control Protocol (described in " - "control-spec.txt).", }, - { "CookieAuthentication", "If this option is set to 1, don't allow any " - "connections to the control port except when the connecting process " - "can read a file that Tor creates in its data directory." }, - { "DataDirectory", "Store working data, state, keys, and caches here." }, - { "DirServer", "Tor only trusts directories signed with one of these " - "servers' keys. Used to override the standard list of directory " - "authorities." }, - /* { "FastFirstHopPK", "" }, */ - /* FetchServerDescriptors, FetchHidServDescriptors, - * FetchUselessDescriptors */ - { "HardwareAccel", "If set, Tor tries to use hardware crypto accelerators " - "when it can." }, - /* HashedControlPassword */ - { "HTTPProxy", "Force Tor to make all HTTP directory requests through this " - "host:port (or host:80 if port is not set)." }, - { "HTTPProxyAuthenticator", "A username:password pair to be used with " - "HTTPProxy." }, - { "HTTPSProxy", "Force Tor to make all TLS (SSL) connections through this " - "host:port (or host:80 if port is not set)." }, - { "HTTPSProxyAuthenticator", "A username:password pair to be used with " - "HTTPSProxy." }, - { "KeepalivePeriod", "Send a padding cell every N seconds to keep firewalls " - "from closing our connections while Tor is not in use." }, - { "Log", "Where to send logging messages. Format is " - "minSeverity[-maxSeverity] (stderr|stdout|syslog|file FILENAME)." }, - { "OutboundBindAddress", "Make all outbound connections originate from the " - "provided IP address (only useful for multiple network interfaces)." }, - { "PIDFile", "On startup, write our PID to this file. On clean shutdown, " - "remove the file." }, - { "PreferTunneledDirConns", "If non-zero, avoid directory servers that " - "don't support tunneled connections." }, - /* PreferTunneledDirConns */ - /* ProtocolWarnings */ - /* RephistTrackTime */ - { "RunAsDaemon", "If set, Tor forks and daemonizes to the background when " - "started. Unix only." }, - { "SafeLogging", "If set to 0, Tor logs potentially sensitive strings " - "rather than replacing them with the string [scrubbed]." }, - { "TunnelDirConns", "If non-zero, when a directory server we contact " - "supports it, we will build a one-hop circuit and make an encrypted " - "connection via its ORPort." }, - { "User", "On startup, setuid to this user." }, - - /* ==== client options */ - { "AllowInvalidNodes", "Where on our circuits should Tor allow servers " - "that the directory authorities haven't called \"valid\"?" }, - { "AllowNonRFC953Hostnames", "If set to 1, we don't automatically reject " - "hostnames for having invalid characters." }, - /* CircuitBuildTimeout, CircuitIdleTimeout */ - { "ClientOnly", "If set to 1, Tor will under no circumstances run as a " - "server, even if ORPort is enabled." }, - { "EntryNodes", "A list of preferred entry nodes to use for the first hop " - "in circuits, when possible." }, - /* { "EnforceDistinctSubnets" , "" }, */ - { "ExitNodes", "A list of preferred nodes to use for the last hop in " - "circuits, when possible." }, - { "ExcludeNodes", "A list of nodes never to use when building a circuit." }, - { "FascistFirewall", "If set, Tor will only create outgoing connections to " - "servers running on the ports listed in FirewallPorts." }, - { "FirewallPorts", "A list of ports that we can connect to. Only used " - "when FascistFirewall is set." }, - { "LongLivedPorts", "A list of ports for services that tend to require " - "high-uptime connections." }, - { "MapAddress", "Force Tor to treat all requests for one address as if " - "they were for another." }, - { "NewCircuitPeriod", "Force Tor to consider whether to build a new circuit " - "every NUM seconds." }, - { "MaxCircuitDirtiness", "Do not attach new streams to a circuit that has " - "been used more than this many seconds ago." }, - /* NatdPort, NatdListenAddress */ - { "NodeFamily", "A list of servers that constitute a 'family' and should " - "never be used in the same circuit." }, - { "NumEntryGuards", "How many entry guards should we keep at a time?" }, - /* PathlenCoinWeight */ - { "ReachableAddresses", "Addresses we can connect to, as IP/bits:port-port. " - "By default, we assume all addresses are reachable." }, - /* reachablediraddresses, reachableoraddresses. */ - /* SafeSOCKS */ - { "SOCKSPort", "The port where we listen for SOCKS connections from " - "applications." }, - { "SOCKSListenAddress", "Bind to this address to listen to connections from " - "SOCKS-speaking applications." }, - { "SOCKSPolicy", "Set an entry policy to limit which addresses can connect " - "to the SOCKSPort." }, - /* SocksTimeout */ - { "StrictExitNodes", "If set, Tor will fail to operate when none of the " - "configured ExitNodes can be used." }, - { "StrictEntryNodes", "If set, Tor will fail to operate when none of the " - "configured EntryNodes can be used." }, - /* TestSocks */ - { "TrackHostsExit", "Hosts and domains which should, if possible, be " - "accessed from the same exit node each time we connect to them." }, - { "TrackHostsExitExpire", "Time after which we forget which exit we were " - "using to connect to hosts in TrackHostsExit." }, - /* "TransPort", "TransListenAddress */ - { "UseEntryGuards", "Set to 0 if we want to pick from the whole set of " - "servers for the first position in each circuit, rather than picking a " - "set of 'Guards' to prevent profiling attacks." }, - - /* === server options */ - { "Address", "The advertised (external) address we should use." }, - /* Accounting* options. */ - /* AssumeReachable */ - { "ContactInfo", "Administrative contact information to advertise for this " - "server." }, - { "ExitPolicy", "Address/port ranges for which to accept or reject outgoing " - "connections on behalf of Tor users." }, - /* { "ExitPolicyRejectPrivate, "" }, */ - { "MaxAdvertisedBandwidth", "If set, we will not advertise more than this " - "amount of bandwidth for our bandwidth rate, regardless of how much " - "bandwidth we actually detect." }, - { "MaxOnionsPending", "Reject new attempts to extend circuits when we " - "already have this many pending." }, - { "MyFamily", "Declare a list of other servers as belonging to the same " - "family as this one, so that clients will not use two from the same " - "family in the same circuit." }, - { "Nickname", "Set the server nickname." }, - { "NoPublish", "{DEPRECATED}" }, - { "NumCPUs", "How many processes to use at once for public-key crypto." }, - { "ORPort", "Advertise this port to listen for connections from Tor clients " - "and servers." }, - { "ORListenAddress", "Bind to this address to listen for connections from " - "clients and servers, instead of the default 0.0.0.0:ORPort." }, - { "PublishServerDescriptor", "Set to 0 to keep the server from " - "uploading info to the directory authorities." }, - /* ServerDNS: DetectHijacking, ResolvConfFile, SearchDomains */ - { "ShutdownWaitLength", "Wait this long for clients to finish when " - "shutting down because of a SIGINT." }, - - /* === directory cache options */ - { "DirPort", "Serve directory information from this port, and act as a " - "directory cache." }, - { "DirPortFrontPage", "Serve a static html disclaimer on DirPort." }, - { "DirListenAddress", "Bind to this address to listen for connections from " - "clients and servers, instead of the default 0.0.0.0:DirPort." }, - { "DirPolicy", "Set a policy to limit who can connect to the directory " - "port." }, - - /* Authority options: AuthDirBadExit, AuthDirInvalid, AuthDirReject, - * AuthDirRejectUnlisted, AuthDirListBadExits, AuthoritativeDirectory, - * DirAllowPrivateAddresses, HSAuthoritativeDir, - * NamingAuthoritativeDirectory, RecommendedVersions, - * RecommendedClientVersions, RecommendedServerVersions, RendPostPeriod, - * RunTesting, V1AuthoritativeDirectory, VersioningAuthoritativeDirectory, */ - - /* Hidden service options: HiddenService: dir,excludenodes, nodes, - * options, port. PublishHidServDescriptor */ - - /* Nonpersistent options: __LeaveStreamsUnattached, __AllDirActionsPrivate */ - { NULL, NULL }, -}; - -/** Online description of state variables. */ -static config_var_description_t state_description[] = { - { "AccountingBytesReadInInterval", - "How many bytes have we read in this accounting period?" }, - { "AccountingBytesWrittenInInterval", - "How many bytes have we written in this accounting period?" }, - { "AccountingExpectedUsage", - "How many bytes did we expect to use per minute? (0 for no estimate.)" }, - { "AccountingIntervalStart", "When did this accounting period begin?" }, - { "AccountingSecondsActive", "How long have we been awake in this period?" }, - - { "BWHistoryReadEnds", "When does the last-recorded read-interval end?" }, - { "BWHistoryReadInterval", "How long is each read-interval (in seconds)?" }, - { "BWHistoryReadValues", "Number of bytes read in each interval." }, - { "BWHistoryWriteEnds", "When does the last-recorded write-interval end?" }, - { "BWHistoryWriteInterval", "How long is each write-interval (in seconds)?"}, - { "BWHistoryWriteValues", "Number of bytes written in each interval." }, - - { "EntryGuard", "One of the nodes we have chosen as a fixed entry" }, - { "EntryGuardDownSince", - "The last entry guard has been unreachable since this time." }, - { "EntryGuardUnlistedSince", - "The last entry guard has been unusable since this time." }, - - { "LastRotatedOnionKey", - "The last time at which we changed the medium-term private key used for " - "building circuits." }, - { "LastWritten", "When was this state file last regenerated?" }, - - { "TorVersion", "Which version of Tor generated this state file?" }, - { NULL, NULL }, -}; - /** Type of a callback to validate whether a given configuration is * well-formed and consistent. See options_trial_assign() for documentation * of arguments. */ @@ -635,8 +513,6 @@ typedef struct { config_var_t *vars; /**< List of variables we recognize, their default * values, and where we stick them in the structure. */ validate_fn_t validate_fn; /**< Function to validate config. */ - /** Documentation for configuration variables. */ - config_var_description_t *descriptions; /** If present, extra is a LINELIST variable for unrecognized * lines. Otherwise, unrecognized lines are an error. */ config_var_t *extra; @@ -693,27 +569,13 @@ static int or_state_validate(or_state_t *old_options, or_state_t *options, static int or_state_load(void); static int options_init_logs(or_options_t *options, int validate_only); -static int is_listening_on_low_port(uint16_t port_option, +static int is_listening_on_low_port(int port_option, const config_line_t *listen_options); static uint64_t config_parse_memunit(const char *s, int *ok); static int config_parse_interval(const char *s, int *ok); static void init_libevent(void); static int opt_streq(const char *s1, const char *s2); -/** Versions of libevent. */ -typedef enum { - /* Note: we compare these, so it's important that "old" precede everything, - * and that "other" come last. */ - LE_OLD=0, LE_10C, LE_10D, LE_10E, LE_11, LE_11A, LE_11B, LE_12, LE_12A, - LE_13, LE_13A, LE_13B, LE_13C, LE_13D, LE_13E, - LE_140, LE_141, LE_142, LE_143, LE_144, LE_145, LE_146, LE_147, LE_148, - LE_1499, - LE_OTHER -} le_version_t; -static le_version_t decode_libevent_version(const char *v, int *bincompat_out); -#if defined(HAVE_EVENT_GET_VERSION) && defined(HAVE_EVENT_GET_METHOD) -static void check_libevent_version(const char *m, int server); -#endif /** Magic value for or_options_t. */ #define OR_OPTIONS_MAGIC 9090909 @@ -726,7 +588,6 @@ static config_format_t options_format = { _option_abbrevs, _option_vars, (validate_fn_t)options_validate, - options_description, NULL }; @@ -747,7 +608,6 @@ static config_format_t state_format = { _state_abbrevs, _state_vars, (validate_fn_t)or_state_validate, - state_description, &state_extra_var, }; @@ -812,13 +672,13 @@ set_options(or_options_t *new_val, char **msg) "Acting on config options left us in a broken state. Dying."); exit(1); } - if (old_options) - config_free(&options_format, old_options); + + config_free(&options_format, old_options); return 0; } -extern const char tor_svn_revision[]; /* from tor_main.c */ +extern const char tor_git_revision[]; /* from tor_main.c */ /** The version of this Tor process, as parsed. */ static char *_version = NULL; @@ -828,10 +688,10 @@ const char * get_version(void) { if (_version == NULL) { - if (strlen(tor_svn_revision)) { - size_t len = strlen(VERSION)+strlen(tor_svn_revision)+16; + if (strlen(tor_git_revision)) { + size_t len = strlen(VERSION)+strlen(tor_git_revision)+16; _version = tor_malloc(len); - tor_snprintf(_version, len, "%s (git-%s)", VERSION, tor_svn_revision); + tor_snprintf(_version, len, "%s (git-%s)", VERSION, tor_git_revision); } else { _version = tor_strdup(VERSION); } @@ -844,8 +704,10 @@ get_version(void) static void or_options_free(or_options_t *options) { - if (options->_ExcludeExitNodesUnion) - routerset_free(options->_ExcludeExitNodesUnion); + if (!options) + return; + + routerset_free(options->_ExcludeExitNodesUnion); config_free(&options_format, options); } @@ -854,43 +716,72 @@ or_options_free(or_options_t *options) void config_free_all(void) { - if (global_options) { - or_options_free(global_options); - global_options = NULL; - } - if (global_state) { - config_free(&state_format, global_state); - global_state = NULL; - } - if (global_cmdline_options) { - config_free_lines(global_cmdline_options); - global_cmdline_options = NULL; - } + or_options_free(global_options); + global_options = NULL; + + config_free(&state_format, global_state); + global_state = NULL; + + config_free_lines(global_cmdline_options); + global_cmdline_options = NULL; + tor_free(torrc_fname); tor_free(_version); tor_free(global_dirfrontpagecontents); } -/** If options->SafeLogging is on, return a not very useful string, - * else return address. +/** Make <b>address</b> -- a piece of information related to our operation as + * a client -- safe to log according to the settings in options->SafeLogging, + * and return it. + * + * (We return "[scrubbed]" if SafeLogging is "1", and address otherwise.) + */ +const char * +safe_str_client(const char *address) +{ + tor_assert(address); + if (get_options()->_SafeLogging == SAFELOG_SCRUB_ALL) + return "[scrubbed]"; + else + return address; +} + +/** Make <b>address</b> -- a piece of information of unspecified sensitivity + * -- safe to log according to the settings in options->SafeLogging, and + * return it. + * + * (We return "[scrubbed]" if SafeLogging is anything besides "0", and address + * otherwise.) */ const char * safe_str(const char *address) { tor_assert(address); - if (get_options()->SafeLogging) + if (get_options()->_SafeLogging != SAFELOG_SCRUB_NONE) return "[scrubbed]"; else return address; } +/** Equivalent to escaped(safe_str_client(address)). See reentrancy note on + * escaped(): don't use this outside the main thread, or twice in the same + * log statement. */ +const char * +escaped_safe_str_client(const char *address) +{ + if (get_options()->_SafeLogging == SAFELOG_SCRUB_ALL) + return "[scrubbed]"; + else + return escaped(address); +} + /** Equivalent to escaped(safe_str(address)). See reentrancy note on * escaped(): don't use this outside the main thread, or twice in the same * log statement. */ const char * escaped_safe_str(const char *address) { - if (get_options()->SafeLogging) + if (get_options()->_SafeLogging != SAFELOG_SCRUB_NONE) return "[scrubbed]"; else return escaped(address); @@ -1065,9 +956,15 @@ options_act_reversible(or_options_t *old_options, char **msg) } #ifndef HAVE_SYS_UN_H - if (options->ControlSocket) { - *msg = tor_strdup("Unix domain sockets (ControlSocket) not supported" - " on this OS/with this build."); + if (options->ControlSocket || options->ControlSocketsGroupWritable) { + *msg = tor_strdup("Unix domain sockets (ControlSocket) not supported " + "on this OS/with this build."); + goto rollback; + } +#else + if (options->ControlSocketsGroupWritable && !options->ControlSocket) { + *msg = tor_strdup("Setting ControlSocketGroupWritable without setting" + "a ControlSocket makes no sense."); goto rollback; } #endif @@ -1089,10 +986,12 @@ options_act_reversible(or_options_t *old_options, char **msg) } /* Launch the listeners. (We do this before we setuid, so we can bind to - * ports under 1024.) */ - if (retry_all_listeners(replaced_listeners, new_listeners) < 0) { - *msg = tor_strdup("Failed to bind one of the listener ports."); - goto rollback; + * ports under 1024.) We don't want to rebind if we're hibernating. */ + if (!we_are_hibernating()) { + if (retry_all_listeners(replaced_listeners, new_listeners) < 0) { + *msg = tor_strdup("Failed to bind one of the listener ports."); + goto rollback; + } } } @@ -1106,6 +1005,15 @@ options_act_reversible(or_options_t *old_options, char **msg) } #endif + /* Attempt to lock all current and future memory with mlockall() only once */ + if (options->DisableAllSwap) { + if (tor_mlockall() == -1) { + *msg = tor_strdup("DisableAllSwap failure. Do you have proper " + "permissions?"); + goto done; + } + } + /* Setuid/setgid as appropriate */ if (options->User) { if (switch_id(options->User) != 0) { @@ -1118,11 +1026,9 @@ options_act_reversible(or_options_t *old_options, char **msg) /* Ensure data directory is private; create if possible. */ if (check_private_dir(options->DataDirectory, running_tor ? CPD_CREATE : CPD_CHECK)<0) { - char buf[1024]; - int tmp = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "Couldn't access/create private data directory \"%s\"", options->DataDirectory); - *msg = tor_strdup(tmp >= 0 ? buf : "internal error"); goto done; /* No need to roll back, since you can't change the value. */ } @@ -1133,10 +1039,8 @@ options_act_reversible(or_options_t *old_options, char **msg) tor_snprintf(fn, len, "%s"PATH_SEPARATOR"cached-status", options->DataDirectory); if (check_private_dir(fn, running_tor ? CPD_CREATE : CPD_CHECK) < 0) { - char buf[1024]; - int tmp = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "Couldn't access/create private data directory \"%s\"", fn); - *msg = tor_strdup(tmp >= 0 ? buf : "internal error"); tor_free(fn); goto done; } @@ -1236,7 +1140,6 @@ get_effective_bwrate(or_options_t *options) bw = options->MaxAdvertisedBandwidth; if (options->RelayBandwidthRate > 0 && bw > options->RelayBandwidthRate) bw = options->RelayBandwidthRate; - /* ensure_bandwidth_cap() makes sure that this cast can't overflow. */ return (uint32_t)bw; } @@ -1279,7 +1182,7 @@ options_act(or_options_t *old_options) return -1; if (options->Bridges) { - clear_bridge_list(); + mark_bridge_list(); for (cl = options->Bridges; cl; cl = cl->next) { if (parse_bridge_line(cl->value, 0)<0) { log_warn(LD_BUG, @@ -1287,6 +1190,7 @@ options_act(or_options_t *old_options) return -1; } } + sweep_bridge_list(); } if (running_tor && rend_config_services(options, 0)<0) { @@ -1314,14 +1218,14 @@ options_act(or_options_t *old_options) return 0; /* Finish backgrounding the process */ - if (running_tor && options->RunAsDaemon) { + if (options->RunAsDaemon) { /* We may be calling this for the n'th time (on SIGHUP), but it's safe. */ finish_daemon(options->DataDirectory); } /* Write our PID to the PID file. If we do not have write permissions we * will log a warning */ - if (running_tor && options->PidFile) + if (options->PidFile) write_pidfile(options->PidFile); /* Register addressmap directives */ @@ -1340,6 +1244,8 @@ options_act(or_options_t *old_options) return -1; } + monitor_owning_controller_process(options->OwningControllerProcess); + /* reload keys as needed for rendezvous services. */ if (rend_service_load_keys()<0) { log_warn(LD_GENERAL,"Error loading rendezvous service keys"); @@ -1354,30 +1260,103 @@ options_act(or_options_t *old_options) if (accounting_is_enabled(options)) configure_accounting(time(NULL)); + /* parse RefuseUnknownExits tristate */ + if (!strcmp(options->RefuseUnknownExits, "0")) + options->RefuseUnknownExits_ = 0; + else if (!strcmp(options->RefuseUnknownExits, "1")) + options->RefuseUnknownExits_ = 1; + else if (!strcmp(options->RefuseUnknownExits, "auto")) + options->RefuseUnknownExits_ = -1; + else { + /* Should have caught this in options_validate */ + return -1; + } + + /* Change the cell EWMA settings */ + cell_ewma_set_scale_factor(options, networkstatus_get_latest_consensus()); + /* Check for transitions that need action. */ if (old_options) { - if (options->UseEntryGuards && !old_options->UseEntryGuards) { + int revise_trackexithosts = 0; + int revise_automap_entries = 0; + if ((options->UseEntryGuards && !old_options->UseEntryGuards) || + options->UseBridges != old_options->UseBridges || + (options->UseBridges && + !config_lines_eq(options->Bridges, old_options->Bridges)) || + !routerset_equal(old_options->ExcludeNodes,options->ExcludeNodes) || + !routerset_equal(old_options->ExcludeExitNodes, + options->ExcludeExitNodes) || + !routerset_equal(old_options->EntryNodes, options->EntryNodes) || + !routerset_equal(old_options->ExitNodes, options->ExitNodes) || + options->StrictNodes != old_options->StrictNodes) { log_info(LD_CIRC, - "Switching to entry guards; abandoning previous circuits"); + "Changed to using entry guards or bridges, or changed " + "preferred or excluded node lists. " + "Abandoning previous circuits."); circuit_mark_all_unused_circs(); circuit_expire_all_dirty_circs(); + revise_trackexithosts = 1; + } + + if (!smartlist_strings_eq(old_options->TrackHostExits, + options->TrackHostExits)) + revise_trackexithosts = 1; + + if (revise_trackexithosts) + addressmap_clear_excluded_trackexithosts(options); + + if (!options->AutomapHostsOnResolve) { + if (old_options->AutomapHostsOnResolve) + revise_automap_entries = 1; + } else { + if (!smartlist_strings_eq(old_options->AutomapHostsSuffixes, + options->AutomapHostsSuffixes)) + revise_automap_entries = 1; + else if (!opt_streq(old_options->VirtualAddrNetwork, + options->VirtualAddrNetwork)) + revise_automap_entries = 1; } + if (revise_automap_entries) + addressmap_clear_invalid_automaps(options); + +/* How long should we delay counting bridge stats after becoming a bridge? + * We use this so we don't count people who used our bridge thinking it is + * a relay. If you change this, don't forget to change the log message + * below. It's 4 hours (the time it takes to stop being used by clients) + * plus some extra time for clock skew. */ +#define RELAY_BRIDGE_STATS_DELAY (6 * 60 * 60) + if (! bool_eq(options->BridgeRelay, old_options->BridgeRelay)) { - log_info(LD_GENERAL, "Bridge status changed. Forgetting GeoIP stats."); - geoip_remove_old_clients(time(NULL)+(2*60*60)); + int was_relay = 0; + if (options->BridgeRelay) { + time_t int_start = time(NULL); + if (old_options->ORPort == options->ORPort) { + int_start += RELAY_BRIDGE_STATS_DELAY; + was_relay = 1; + } + geoip_bridge_stats_init(int_start); + log_info(LD_CONFIG, "We are acting as a bridge now. Starting new " + "GeoIP stats interval%s.", was_relay ? " in 6 " + "hours from now" : ""); + } else { + geoip_bridge_stats_term(); + log_info(LD_GENERAL, "We are no longer acting as a bridge. " + "Forgetting GeoIP stats."); + } } if (options_transition_affects_workers(old_options, options)) { log_info(LD_GENERAL, "Worker-related options changed. Rotating workers."); + + if (init_keys() < 0) { + log_warn(LD_BUG,"Error initializing keys; exiting"); + return -1; + } if (server_mode(options) && !server_mode(old_options)) { - if (init_keys() < 0) { - log_warn(LD_BUG,"Error initializing keys; exiting"); - return -1; - } ip_address_changed(0); - if (has_completed_circuit || !any_predicted_circuits(time(NULL))) + if (can_complete_circuit || !any_predicted_circuits(time(NULL))) inform_testing_reachability(); } cpuworkers_rotate(); @@ -1390,6 +1369,10 @@ options_act(or_options_t *old_options) if (options->V3AuthoritativeDir && !old_options->V3AuthoritativeDir) init_keys(); + + if (options->PerConnBWRate != old_options->PerConnBWRate || + options->PerConnBWBurst != old_options->PerConnBWBurst) + connection_or_update_token_buckets(get_connection_array(), options); } /* Maybe load geoip file */ @@ -1398,7 +1381,7 @@ options_act(or_options_t *old_options) || !geoip_is_loaded())) { /* XXXX Don't use this "<default>" junk; make our filename options * understand prefixes somehow. -NM */ - /* XXXX021 Reload GeoIPFile on SIGHUP. -NM */ + /* XXXX023 Reload GeoIPFile on SIGHUP. -NM */ char *actual_fname = tor_strdup(options->GeoIPFile); #ifdef WIN32 if (!strcmp(actual_fname, "<default>")) { @@ -1412,17 +1395,68 @@ options_act(or_options_t *old_options) geoip_load_file(actual_fname, options); tor_free(actual_fname); } -#ifdef ENABLE_GEOIP_STATS - log_warn(LD_CONFIG, "We are configured to measure GeoIP statistics, but " - "the way these statistics are measured has changed " - "significantly in later versions of Tor. The results may not be " - "as expected if you are used to later versions. Be sure you " - "know what you are doing."); -#endif + + if (options->DirReqStatistics && !geoip_is_loaded()) { + /* Check if GeoIP database could be loaded. */ + log_warn(LD_CONFIG, "Configured to measure directory request " + "statistics, but no GeoIP database found!"); + return -1; + } + + if (options->EntryStatistics) { + if (should_record_bridge_info(options)) { + /* Don't allow measuring statistics on entry guards when configured + * as bridge. */ + log_warn(LD_CONFIG, "Bridges cannot be configured to measure " + "additional GeoIP statistics as entry guards."); + return -1; + } else if (!geoip_is_loaded()) { + /* Check if GeoIP database could be loaded. */ + log_warn(LD_CONFIG, "Configured to measure entry node statistics, " + "but no GeoIP database found!"); + return -1; + } + } + + if (options->CellStatistics || options->DirReqStatistics || + options->EntryStatistics || options->ExitPortStatistics) { + time_t now = time(NULL); + if ((!old_options || !old_options->CellStatistics) && + options->CellStatistics) + rep_hist_buffer_stats_init(now); + if ((!old_options || !old_options->DirReqStatistics) && + options->DirReqStatistics) + geoip_dirreq_stats_init(now); + if ((!old_options || !old_options->EntryStatistics) && + options->EntryStatistics) + geoip_entry_stats_init(now); + if ((!old_options || !old_options->ExitPortStatistics) && + options->ExitPortStatistics) + rep_hist_exit_stats_init(now); + if (!old_options) + log_notice(LD_CONFIG, "Configured to measure statistics. Look for " + "the *-stats files that will first be written to the " + "data directory in 24 hours from now."); + } + + if (old_options && old_options->CellStatistics && + !options->CellStatistics) + rep_hist_buffer_stats_term(); + if (old_options && old_options->DirReqStatistics && + !options->DirReqStatistics) + geoip_dirreq_stats_term(); + if (old_options && old_options->EntryStatistics && + !options->EntryStatistics) + geoip_entry_stats_term(); + if (old_options && old_options->ExitPortStatistics && + !options->ExitPortStatistics) + rep_hist_exit_stats_term(); + /* Check if we need to parse and add the EntryNodes config option. */ if (options->EntryNodes && (!old_options || - (!routerset_equal(old_options->EntryNodes,options->EntryNodes)))) + !routerset_equal(old_options->EntryNodes,options->EntryNodes) || + !routerset_equal(old_options->ExcludeNodes,options->ExcludeNodes))) entry_nodes_should_be_added(); /* Since our options changed, we might need to regenerate and upload our @@ -1430,7 +1464,7 @@ options_act(or_options_t *old_options) */ if (!old_options || options_transition_affects_descriptor(old_options, options)) - mark_my_descriptor_dirty(); + mark_my_descriptor_dirty("config change"); /* We may need to reschedule some directory stuff if our status changed. */ if (old_options) { @@ -1491,7 +1525,10 @@ expand_abbrev(config_format_t *fmt, const char *option, int command_line, fmt->abbrevs[i].abbreviated, fmt->abbrevs[i].full); } - return fmt->abbrevs[i].full; + /* Keep going through the list in case we want to rewrite it more. + * (We could imagine recursing here, but I don't want to get the + * user into an infinite loop if we craft our list wrong.) */ + option = fmt->abbrevs[i].full; } } return option; @@ -1536,7 +1573,10 @@ config_get_commandlines(int argc, char **argv, config_line_t **result) *new = tor_malloc_zero(sizeof(config_line_t)); s = argv[i]; - while (*s == '-') + /* Each keyword may be prefixed with one or two dashes. */ + if (*s == '-') + s++; + if (*s == '-') s++; (*new)->key = tor_strdup(expand_abbrev(&options_format, s, 1, 1)); @@ -1628,19 +1668,6 @@ config_free_lines(config_line_t *front) } } -/** Return the description for a given configuration variable, or NULL if no - * description exists. */ -static const char * -config_find_description(config_format_t *fmt, const char *name) -{ - int i; - for (i=0; fmt->descriptions[i].name; ++i) { - if (!strcasecmp(name, fmt->descriptions[i].name)) - return fmt->descriptions[i].description; - } - return NULL; -} - /** If <b>key</b> is a configuration option, return the corresponding * config_var_t. Otherwise, if <b>key</b> is a non-standard abbreviation, * warn, and return the corresponding config_var_t. Otherwise return NULL. @@ -1671,6 +1698,16 @@ config_find_option(config_format_t *fmt, const char *key) return NULL; } +/** Return the number of option entries in <b>fmt</b>. */ +static int +config_count_options(config_format_t *fmt) +{ + int i; + for (i=0; fmt->vars[i].name; ++i) + ; + return i; +} + /* * Functions to assign config options. */ @@ -1684,8 +1721,7 @@ static int config_assign_value(config_format_t *fmt, or_options_t *options, config_line_t *c, char **msg) { - int i, r, ok; - char buf[1024]; + int i, ok; config_var_t *var; void *lvalue; @@ -1698,13 +1734,20 @@ config_assign_value(config_format_t *fmt, or_options_t *options, switch (var->type) { + case CONFIG_TYPE_PORT: + if (!strcasecmp(c->value, "auto")) { + *(int *)lvalue = CFG_AUTO_PORT; + break; + } + /* fall through */ case CONFIG_TYPE_UINT: - i = (int)tor_parse_long(c->value, 10, 0, INT_MAX, &ok, NULL); + i = (int)tor_parse_long(c->value, 10, 0, + var->type==CONFIG_TYPE_PORT ? 65535 : INT_MAX, + &ok, NULL); if (!ok) { - r = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "Int keyword '%s %s' is malformed or out of bounds.", c->key, c->value); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); return -1; } *(int *)lvalue = i; @@ -1713,10 +1756,9 @@ config_assign_value(config_format_t *fmt, or_options_t *options, case CONFIG_TYPE_INTERVAL: { i = config_parse_interval(c->value, &ok); if (!ok) { - r = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "Interval '%s %s' is malformed or out of bounds.", c->key, c->value); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); return -1; } *(int *)lvalue = i; @@ -1726,10 +1768,9 @@ config_assign_value(config_format_t *fmt, or_options_t *options, case CONFIG_TYPE_MEMUNIT: { uint64_t u64 = config_parse_memunit(c->value, &ok); if (!ok) { - r = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "Value '%s %s' is malformed or out of bounds.", c->key, c->value); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); return -1; } *(uint64_t *)lvalue = u64; @@ -1739,10 +1780,9 @@ config_assign_value(config_format_t *fmt, or_options_t *options, case CONFIG_TYPE_BOOL: i = (int)tor_parse_long(c->value, 10, 0, 1, &ok, NULL); if (!ok) { - r = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "Boolean '%s %s' expects 0 or 1.", c->key, c->value); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); return -1; } *(int *)lvalue = i; @@ -1760,9 +1800,8 @@ config_assign_value(config_format_t *fmt, or_options_t *options, case CONFIG_TYPE_ISOTIME: if (parse_iso_time(c->value, (time_t *)lvalue)) { - r = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "Invalid time '%s' for keyword '%s'", c->value, c->key); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); return -1; } break; @@ -1773,9 +1812,8 @@ config_assign_value(config_format_t *fmt, or_options_t *options, } *(routerset_t**)lvalue = routerset_new(); if (routerset_parse(*(routerset_t**)lvalue, c->value, c->key)<0) { - tor_snprintf(buf, sizeof(buf), "Invalid exit list '%s' for option '%s'", + tor_asprintf(msg, "Invalid exit list '%s' for option '%s'", c->value, c->key); - *msg = tor_strdup(buf); return -1; } break; @@ -1800,9 +1838,8 @@ config_assign_value(config_format_t *fmt, or_options_t *options, log_warn(LD_CONFIG, "Skipping obsolete configuration option '%s'", c->key); break; case CONFIG_TYPE_LINELIST_V: - r = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "You may not provide a value for virtual option '%s'", c->key); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); return -1; default: tor_assert(0); @@ -1823,7 +1860,7 @@ config_assign_value(config_format_t *fmt, or_options_t *options, static int config_assign_line(config_format_t *fmt, or_options_t *options, config_line_t *c, int use_defaults, - int clear_first, char **msg) + int clear_first, bitarray_t *options_seen, char **msg) { config_var_t *var; @@ -1838,13 +1875,12 @@ config_assign_line(config_format_t *fmt, or_options_t *options, config_line_append((config_line_t**)lvalue, c->key, c->value); return 0; } else { - char buf[1024]; - int tmp = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "Unknown option '%s'. Failing.", c->key); - *msg = tor_strdup(tmp >= 0 ? buf : "internal error"); return -1; } } + /* Put keyword into canonical case. */ if (strcmp(var->name, c->key)) { tor_free(c->key); @@ -1867,6 +1903,18 @@ config_assign_line(config_format_t *fmt, or_options_t *options, return 0; } + if (options_seen && (var->type != CONFIG_TYPE_LINELIST && + var->type != CONFIG_TYPE_LINELIST_S)) { + /* We're tracking which options we've seen, and this option is not + * supposed to occur more than once. */ + int var_index = (int)(var - fmt->vars); + if (bitarray_is_set(options_seen, var_index)) { + log_warn(LD_CONFIG, "Option '%s' used more than once; all but the last " + "value will be ignored.", var->name); + } + bitarray_set(options_seen, var_index); + } + if (config_assign_value(fmt, options, c, msg) < 0) return -2; return 0; @@ -1967,7 +2015,6 @@ get_assigned_option(config_format_t *fmt, void *options, { config_var_t *var; const void *value; - char buf[32]; config_line_t *result; tor_assert(options && key); @@ -2004,23 +2051,26 @@ get_assigned_option(config_format_t *fmt, void *options, } escape_val = 0; /* Can't need escape. */ break; + case CONFIG_TYPE_PORT: + if (*(int*)value == CFG_AUTO_PORT) { + result->value = tor_strdup("auto"); + escape_val = 0; + break; + } case CONFIG_TYPE_INTERVAL: case CONFIG_TYPE_UINT: /* This means every or_options_t uint or bool element * needs to be an int. Not, say, a uint16_t or char. */ - tor_snprintf(buf, sizeof(buf), "%d", *(int*)value); - result->value = tor_strdup(buf); + tor_asprintf(&result->value, "%d", *(int*)value); escape_val = 0; /* Can't need escape. */ break; case CONFIG_TYPE_MEMUNIT: - tor_snprintf(buf, sizeof(buf), U64_FORMAT, + tor_asprintf(&result->value, U64_FORMAT, U64_PRINTF_ARG(*(uint64_t*)value)); - result->value = tor_strdup(buf); escape_val = 0; /* Can't need escape. */ break; case CONFIG_TYPE_DOUBLE: - tor_snprintf(buf, sizeof(buf), "%f", *(double*)value); - result->value = tor_strdup(buf); + tor_asprintf(&result->value, "%f", *(double*)value); escape_val = 0; /* Can't need escape. */ break; case CONFIG_TYPE_BOOL: @@ -2139,6 +2189,8 @@ config_assign(config_format_t *fmt, void *options, config_line_t *list, int use_defaults, int clear_first, char **msg) { config_line_t *p; + bitarray_t *options_seen; + const int n_options = config_count_options(fmt); CHECK(fmt, options); @@ -2158,14 +2210,18 @@ config_assign(config_format_t *fmt, void *options, config_line_t *list, config_reset_line(fmt, options, p->key, use_defaults); } + options_seen = bitarray_init_zero(n_options); /* pass 3: assign. */ while (list) { int r; if ((r=config_assign_line(fmt, options, list, use_defaults, - clear_first, msg))) + clear_first, options_seen, msg))) { + bitarray_free(options_seen); return r; + } list = list->next; } + bitarray_free(options_seen); return 0; } @@ -2230,6 +2286,7 @@ option_clear(config_format_t *fmt, or_options_t *options, config_var_t *var) break; case CONFIG_TYPE_INTERVAL: case CONFIG_TYPE_UINT: + case CONFIG_TYPE_PORT: case CONFIG_TYPE_BOOL: *(int*)lvalue = 0; break; @@ -2308,20 +2365,10 @@ list_torrc_options(void) smartlist_t *lines = smartlist_create(); for (i = 0; _option_vars[i].name; ++i) { config_var_t *var = &_option_vars[i]; - const char *desc; if (var->type == CONFIG_TYPE_OBSOLETE || var->type == CONFIG_TYPE_LINELIST_V) continue; - desc = config_find_description(&options_format, var->name); printf("%s\n", var->name); - if (desc) { - wrap_string(lines, desc, 76, " ", " "); - SMARTLIST_FOREACH(lines, char *, cp, { - printf("%s", cp); - tor_free(cp); - }); - smartlist_clear(lines); - } } smartlist_free(lines); } @@ -2340,7 +2387,7 @@ resolve_my_address(int warn_severity, or_options_t *options, uint32_t *addr_out, char **hostname_out) { struct in_addr in; - uint32_t addr; + uint32_t addr; /* host order */ char hostname[256]; int explicit_ip=1; int explicit_hostname=1; @@ -2370,8 +2417,8 @@ resolve_my_address(int warn_severity, or_options_t *options, if (tor_inet_aton(hostname, &in) == 0) { /* then we have to resolve it */ explicit_ip = 0; - if (tor_lookup_hostname(hostname, &addr)) { - uint32_t interface_ip; + if (tor_lookup_hostname(hostname, &addr)) { /* failed to resolve */ + uint32_t interface_ip; /* host order */ if (explicit_hostname) { log_fn(warn_severity, LD_CONFIG, @@ -2392,7 +2439,7 @@ resolve_my_address(int warn_severity, or_options_t *options, log_fn(notice_severity, LD_CONFIG, "Learned IP address '%s' for " "local interface. Using that.", tmpbuf); strlcpy(hostname, "<guessed from interfaces>", sizeof(hostname)); - } else { + } else { /* resolved hostname into addr */ in.s_addr = htonl(addr); if (!explicit_hostname && @@ -2495,7 +2542,7 @@ is_local_addr(const tor_addr_t *addr) if (get_options()->EnforceDistinctSubnets == 0) return 0; if (tor_addr_family(addr) == AF_INET) { - /*XXXX022 IP6 what corresponds to an /24? */ + /*XXXX023 IP6 what corresponds to an /24? */ uint32_t ip = tor_addr_to_ipv4h(addr); /* It's possible that this next check will hit before the first time @@ -2512,61 +2559,16 @@ is_local_addr(const tor_addr_t *addr) return 0; } -/** Called when we don't have a nickname set. Try to guess a good nickname - * based on the hostname, and return it in a newly allocated string. If we - * can't, return NULL and let the caller warn if it wants to. */ -static char * -get_default_nickname(void) -{ - static const char * const bad_default_nicknames[] = { - "localhost", - NULL, - }; - char localhostname[256]; - char *cp, *out, *outp; - int i; - - if (gethostname(localhostname, sizeof(localhostname)) < 0) - return NULL; - - /* Put it in lowercase; stop at the first dot. */ - if ((cp = strchr(localhostname, '.'))) - *cp = '\0'; - tor_strlower(localhostname); - - /* Strip invalid characters. */ - cp = localhostname; - out = outp = tor_malloc(strlen(localhostname) + 1); - while (*cp) { - if (strchr(LEGAL_NICKNAME_CHARACTERS, *cp)) - *outp++ = *cp++; - else - cp++; - } - *outp = '\0'; - - /* Enforce length. */ - if (strlen(out) > MAX_NICKNAME_LEN) - out[MAX_NICKNAME_LEN]='\0'; - - /* Check for dumb names. */ - for (i = 0; bad_default_nicknames[i]; ++i) { - if (!strcmp(out, bad_default_nicknames[i])) { - tor_free(out); - return NULL; - } - } - - return out; -} - /** Release storage held by <b>options</b>. */ static void config_free(config_format_t *fmt, void *options) { int i; - tor_assert(options); + if (!options) + return; + + tor_assert(fmt); for (i=0; fmt->vars[i].name; ++i) option_clear(fmt, options, &(fmt->vars[i])); @@ -2664,10 +2666,12 @@ options_init(or_options_t *options) * it is, or 0 if it isn't or the concept of a low port isn't applicable for * the platform we're on. */ static int -is_listening_on_low_port(uint16_t port_option, +is_listening_on_low_port(int port_option, const config_line_t *listen_options) { #ifdef MS_WINDOWS + (void) port_option; + (void) listen_options; return 0; /* No port is too low for windows. */ #else const config_line_t *l; @@ -2717,7 +2721,6 @@ config_dump(config_format_t *fmt, void *options, int minimal, config_line_t *line, *assigned; char *result; int i; - const char *desc; char *msg = NULL; defaults = config_alloc(fmt); @@ -2745,24 +2748,13 @@ config_dump(config_format_t *fmt, void *options, int minimal, option_is_same(fmt, options, defaults, fmt->vars[i].name)) comment_option = 1; - desc = config_find_description(fmt, fmt->vars[i].name); line = assigned = get_assigned_option(fmt, options, fmt->vars[i].name, 1); - if (line && desc) { - /* Only dump the description if there's something to describe. */ - wrap_string(elements, desc, 78, "# ", "# "); - } - for (; line; line = line->next) { - size_t len = strlen(line->key) + strlen(line->value) + 5; char *tmp; - tmp = tor_malloc(len); - if (tor_snprintf(tmp, len, "%s%s %s\n", - comment_option ? "# " : "", - line->key, line->value)<0) { - log_err(LD_BUG,"Internal error writing option value"); - tor_assert(0); - } + tor_asprintf(&tmp, "%s%s %s\n", + comment_option ? "# " : "", + line->key, line->value); smartlist_add(elements, tmp); } config_free_lines(assigned); @@ -2771,13 +2763,8 @@ config_dump(config_format_t *fmt, void *options, int minimal, if (fmt->extra) { line = *(config_line_t**)STRUCT_VAR_P(options, fmt->extra->var_offset); for (; line; line = line->next) { - size_t len = strlen(line->key) + strlen(line->value) + 3; char *tmp; - tmp = tor_malloc(len); - if (tor_snprintf(tmp, len, "%s %s\n", line->key, line->value)<0) { - log_err(LD_BUG,"Internal error writing option value"); - tor_assert(0); - } + tor_asprintf(&tmp, "%s %s\n", line->key, line->value); smartlist_add(elements, tmp); } } @@ -2793,7 +2780,7 @@ config_dump(config_format_t *fmt, void *options, int minimal, * the configuration in <b>options</b>. If <b>minimal</b> is true, do not * include options that are the same as Tor's defaults. */ -static char * +char * options_dump(or_options_t *options, int minimal) { return config_dump(&options_format, options, minimal, 0); @@ -2806,7 +2793,6 @@ static int validate_ports_csv(smartlist_t *sl, const char *name, char **msg) { int i; - char buf[1024]; tor_assert(name); if (!sl) @@ -2816,9 +2802,7 @@ validate_ports_csv(smartlist_t *sl, const char *name, char **msg) { i = atoi(cp); if (i < 1 || i > 65535) { - int r = tor_snprintf(buf, sizeof(buf), - "Port '%s' out of range in %s", cp, name); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); + tor_asprintf(msg, "Port '%s' out of range in %s", cp, name); return -1; } }); @@ -2832,18 +2816,15 @@ validate_ports_csv(smartlist_t *sl, const char *name, char **msg) static int ensure_bandwidth_cap(uint64_t *value, const char *desc, char **msg) { - int r; - char buf[1024]; if (*value > ROUTER_MAX_DECLARED_BANDWIDTH) { /* This handles an understandable special case where somebody says "2gb" * whereas our actual maximum is 2gb-1 (INT_MAX) */ --*value; } if (*value > ROUTER_MAX_DECLARED_BANDWIDTH) { - r = tor_snprintf(buf, sizeof(buf), "%s ("U64_FORMAT") must be at most %d", - desc, U64_PRINTF_ARG(*value), - ROUTER_MAX_DECLARED_BANDWIDTH); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); + tor_asprintf(msg, "%s ("U64_FORMAT") must be at most %d", + desc, U64_PRINTF_ARG(*value), + ROUTER_MAX_DECLARED_BANDWIDTH); return -1; } return 0; @@ -2896,15 +2877,14 @@ compute_publishserverdescriptor(or_options_t *options) /** Highest allowable value for RendPostPeriod. */ #define MAX_DIR_PERIOD (MIN_ONION_KEY_LIFETIME/2) -/** Lowest allowable value for CircuitBuildTimeout; values too low will - * increase network load because of failing connections being retried, and - * might prevent users from connecting to the network at all. */ -#define MIN_CIRCUIT_BUILD_TIMEOUT 30 - /** Lowest allowable value for MaxCircuitDirtiness; if this is too low, Tor * will generate too many circuits and potentially overload the network. */ #define MIN_MAX_CIRCUIT_DIRTINESS 10 +/** Lowest allowable value for CircuitStreamTimeout; if this is too low, Tor + * will generate too many circuits and potentially overload the network. */ +#define MIN_CIRCUIT_STREAM_TIMEOUT 10 + /** Return 0 if every setting in <b>options</b> is reasonable, and a * permissible transition from <b>old_options</b>. Else return -1. * Should have no side effects, except for normalizing the contents of @@ -2921,10 +2901,9 @@ static int options_validate(or_options_t *old_options, or_options_t *options, int from_setconf, char **msg) { - int i, r; + int i; config_line_t *cl; const char *uname = get_uname(); - char buf[1024]; #define REJECT(arg) \ STMT_BEGIN *msg = tor_strdup(arg); return -1; STMT_END #define COMPLAIN(arg) STMT_BEGIN log(LOG_WARN, LD_CONFIG, arg); STMT_END @@ -2932,16 +2911,13 @@ options_validate(or_options_t *old_options, or_options_t *options, tor_assert(msg); *msg = NULL; - if (options->ORPort < 0 || options->ORPort > 65535) - REJECT("ORPort option out of bounds."); - if (server_mode(options) && (!strcmpstart(uname, "Windows 95") || !strcmpstart(uname, "Windows 98") || !strcmpstart(uname, "Windows Me"))) { log(LOG_WARN, LD_CONFIG, "Tor is running as a server, but you are " "running %s; this probably won't work. See " - "http://wiki.noreply.org/noreply/TheOnionRouter/TorFAQ#ServerOS " + "https://wiki.torproject.org/TheOnionRouter/TorFAQ#ServerOS " "for details.", uname); } @@ -2960,8 +2936,8 @@ options_validate(or_options_t *old_options, or_options_t *options, if (options->TransPort == 0 && options->TransListenAddress != NULL) REJECT("TransPort must be defined if TransListenAddress is defined."); - if (options->NatdPort == 0 && options->NatdListenAddress != NULL) - REJECT("NatdPort must be defined if NatdListenAddress is defined."); + if (options->NATDPort == 0 && options->NATDListenAddress != NULL) + REJECT("NATDPort must be defined if NATDListenAddress is defined."); /* Don't gripe about SocksPort 0 with SocksListenAddress set; a standard * configuration does this. */ @@ -2980,8 +2956,8 @@ options_validate(or_options_t *old_options, or_options_t *options, old = old_options ? old_options->TransListenAddress : NULL; tp = "transparent proxy"; } else { - opt = options->NatdListenAddress; - old = old_options ? old_options->NatdListenAddress : NULL; + opt = options->NATDListenAddress; + old = old_options ? old_options->NATDListenAddress : NULL; tp = "natd proxy"; } @@ -3008,21 +2984,13 @@ options_validate(or_options_t *old_options, or_options_t *options, if (options->Nickname == NULL) { if (server_mode(options)) { - if (!(options->Nickname = get_default_nickname())) { - log_notice(LD_CONFIG, "Couldn't pick a nickname based on " - "our hostname; using %s instead.", UNNAMED_ROUTER_NICKNAME); options->Nickname = tor_strdup(UNNAMED_ROUTER_NICKNAME); - } else { - log_notice(LD_CONFIG, "Choosing default nickname '%s'", - options->Nickname); - } } } else { if (!is_legal_nickname(options->Nickname)) { - r = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "Nickname '%s' is wrong length or contains illegal characters.", options->Nickname); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); return -1; } } @@ -3039,14 +3007,6 @@ options_validate(or_options_t *old_options, or_options_t *options, if (options_init_logs(options, 1)<0) /* Validate the log(s) */ REJECT("Failed to validate Log options. See logs for details."); - if (options->NoPublish) { - log(LOG_WARN, LD_CONFIG, - "NoPublish is obsolete. Use PublishServerDescriptor instead."); - SMARTLIST_FOREACH(options->PublishServerDescriptor, char *, s, - tor_free(s)); - smartlist_clear(options->PublishServerDescriptor); - } - if (authdir_mode(options)) { /* confirm that our address isn't broken, so we can complain now */ uint32_t tmp; @@ -3054,37 +3014,25 @@ options_validate(or_options_t *old_options, or_options_t *options, REJECT("Failed to resolve/guess local address. See logs for details."); } + if (strcmp(options->RefuseUnknownExits, "0") && + strcmp(options->RefuseUnknownExits, "1") && + strcmp(options->RefuseUnknownExits, "auto")) { + REJECT("RefuseUnknownExits must be 0, 1, or auto"); + } + #ifndef MS_WINDOWS if (options->RunAsDaemon && torrc_fname && path_is_relative(torrc_fname)) REJECT("Can't use a relative path to torrc when RunAsDaemon is set."); #endif - if (options->SocksPort < 0 || options->SocksPort > 65535) - REJECT("SocksPort option out of bounds."); - - if (options->DNSPort < 0 || options->DNSPort > 65535) - REJECT("DNSPort option out of bounds."); - - if (options->TransPort < 0 || options->TransPort > 65535) - REJECT("TransPort option out of bounds."); - - if (options->NatdPort < 0 || options->NatdPort > 65535) - REJECT("NatdPort option out of bounds."); - if (options->SocksPort == 0 && options->TransPort == 0 && - options->NatdPort == 0 && options->ORPort == 0 && + options->NATDPort == 0 && options->ORPort == 0 && options->DNSPort == 0 && !options->RendConfigLines) log(LOG_WARN, LD_CONFIG, - "SocksPort, TransPort, NatdPort, DNSPort, and ORPort are all " + "SocksPort, TransPort, NATDPort, DNSPort, and ORPort are all " "undefined, and there aren't any hidden services configured. " "Tor will still run, but probably won't do anything."); - if (options->ControlPort < 0 || options->ControlPort > 65535) - REJECT("ControlPort option out of bounds."); - - if (options->DirPort < 0 || options->DirPort > 65535) - REJECT("DirPort option out of bounds."); - #ifndef USE_TRANSPARENT if (options->TransPort || options->TransListenAddress) REJECT("TransPort and TransListenAddress are disabled in this build."); @@ -3109,19 +3057,11 @@ options_validate(or_options_t *old_options, or_options_t *options, routerset_union(options->_ExcludeExitNodesUnion,options->ExcludeNodes); } - if (options->StrictExitNodes && - (!options->ExitNodes) && - (!old_options || - (old_options->StrictExitNodes != options->StrictExitNodes) || - (!routerset_equal(old_options->ExitNodes,options->ExitNodes)))) - COMPLAIN("StrictExitNodes set, but no ExitNodes listed."); - - if (options->StrictEntryNodes && - (!options->EntryNodes) && - (!old_options || - (old_options->StrictEntryNodes != options->StrictEntryNodes) || - (!routerset_equal(old_options->EntryNodes,options->EntryNodes)))) - COMPLAIN("StrictEntryNodes set, but no EntryNodes listed."); + if (options->ExcludeNodes && options->StrictNodes) { + COMPLAIN("You have asked to exclude certain relays from all positions " + "in your circuits. Expect hidden services and other Tor " + "features to be broken in unpredictable ways."); + } if (options->EntryNodes && !routerset_is_list(options->EntryNodes)) { /* XXXX fix this; see entry_guards_prepend_from_config(). */ @@ -3159,6 +3099,10 @@ options_validate(or_options_t *old_options, or_options_t *options, options->V3AuthoritativeDir)) REJECT("AuthoritativeDir is set, but none of " "(Bridge/HS/V1/V2/V3)AuthoritativeDir is set."); + /* If we have a v3bandwidthsfile and it's broken, complain on startup */ + if (options->V3BandwidthsFile && !old_options) { + dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL); + } } if (options->AuthoritativeDir && !options->DirPort) @@ -3170,15 +3114,18 @@ options_validate(or_options_t *old_options, or_options_t *options, if (options->AuthoritativeDir && options->ClientOnly) REJECT("Running as authoritative directory, but ClientOnly also set."); - if (options->HSAuthorityRecordStats && !options->HSAuthoritativeDir) - REJECT("HSAuthorityRecordStats is set but we're not running as " - "a hidden service authority."); + if (options->FetchDirInfoExtraEarly && !options->FetchDirInfoEarly) + REJECT("FetchDirInfoExtraEarly requires that you also set " + "FetchDirInfoEarly"); + + if (options->HSAuthoritativeDir && proxy_mode(options)) + REJECT("Running as authoritative v0 HS directory, but also configured " + "as a client."); if (options->ConnLimit <= 0) { - r = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "ConnLimit must be greater than 0, but was set to %d", options->ConnLimit); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); return -1; } @@ -3283,6 +3230,12 @@ options_validate(or_options_t *old_options, or_options_t *options, REJECT("Servers must be able to freely connect to the rest " "of the Internet, so they must not set UseBridges."); + /* If both of these are set, we'll end up with funny behavior where we + * demand enough entrynodes be up and running else we won't build + * circuits, yet we never actually use them. */ + if (options->UseBridges && options->EntryNodes) + REJECT("You cannot set both UseBridges and EntryNodes."); + options->_AllowInvalid = 0; if (options->AllowInvalidNodes) { SMARTLIST_FOREACH(options->AllowInvalidNodes, const char *, cp, { @@ -3297,18 +3250,29 @@ options_validate(or_options_t *old_options, or_options_t *options, else if (!strcasecmp(cp, "rendezvous")) options->_AllowInvalid |= ALLOW_INVALID_RENDEZVOUS; else { - r = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "Unrecognized value '%s' in AllowInvalidNodes", cp); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); return -1; } }); } + if (!options->SafeLogging || + !strcasecmp(options->SafeLogging, "0")) { + options->_SafeLogging = SAFELOG_SCRUB_NONE; + } else if (!strcasecmp(options->SafeLogging, "relay")) { + options->_SafeLogging = SAFELOG_SCRUB_RELAY; + } else if (!strcasecmp(options->SafeLogging, "1")) { + options->_SafeLogging = SAFELOG_SCRUB_ALL; + } else { + tor_asprintf(msg, + "Unrecognized value '%s' in SafeLogging", + escaped(options->SafeLogging)); + return -1; + } + if (compute_publishserverdescriptor(options) < 0) { - r = tor_snprintf(buf, sizeof(buf), - "Unrecognized value in PublishServerDescriptor"); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); + tor_asprintf(msg, "Unrecognized value in PublishServerDescriptor"); return -1; } @@ -3321,6 +3285,12 @@ options_validate(or_options_t *old_options, or_options_t *options, "PublishServerDescriptor line."); } + if (options->BridgeRelay && options->DirPort) { + log_warn(LD_CONFIG, "Can't set a DirPort on a bridge relay; disabling " + "DirPort"); + options->DirPort = 0; + } + if (options->MinUptimeHidServDirectoryV2 < 0) { log_warn(LD_CONFIG, "MinUptimeHidServDirectoryV2 option must be at " "least 0 seconds. Changing to 0."); @@ -3328,29 +3298,30 @@ options_validate(or_options_t *old_options, or_options_t *options, } if (options->RendPostPeriod < MIN_REND_POST_PERIOD) { - log(LOG_WARN,LD_CONFIG,"RendPostPeriod option is too short; " - "raising to %d seconds.", MIN_REND_POST_PERIOD); + log_warn(LD_CONFIG, "RendPostPeriod option is too short; " + "raising to %d seconds.", MIN_REND_POST_PERIOD); options->RendPostPeriod = MIN_REND_POST_PERIOD; } if (options->RendPostPeriod > MAX_DIR_PERIOD) { - log(LOG_WARN, LD_CONFIG, "RendPostPeriod is too large; clipping to %ds.", - MAX_DIR_PERIOD); + log_warn(LD_CONFIG, "RendPostPeriod is too large; clipping to %ds.", + MAX_DIR_PERIOD); options->RendPostPeriod = MAX_DIR_PERIOD; } - if (options->CircuitBuildTimeout < MIN_CIRCUIT_BUILD_TIMEOUT) { - log(LOG_WARN, LD_CONFIG, "CircuitBuildTimeout option is too short; " - "raising to %d seconds.", MIN_CIRCUIT_BUILD_TIMEOUT); - options->CircuitBuildTimeout = MIN_CIRCUIT_BUILD_TIMEOUT; - } - if (options->MaxCircuitDirtiness < MIN_MAX_CIRCUIT_DIRTINESS) { - log(LOG_WARN, LD_CONFIG, "MaxCircuitDirtiness option is too short; " - "raising to %d seconds.", MIN_MAX_CIRCUIT_DIRTINESS); + log_warn(LD_CONFIG, "MaxCircuitDirtiness option is too short; " + "raising to %d seconds.", MIN_MAX_CIRCUIT_DIRTINESS); options->MaxCircuitDirtiness = MIN_MAX_CIRCUIT_DIRTINESS; } + if (options->CircuitStreamTimeout && + options->CircuitStreamTimeout < MIN_CIRCUIT_STREAM_TIMEOUT) { + log_warn(LD_CONFIG, "CircuitStreamTimeout option is too short; " + "raising to %d seconds.", MIN_CIRCUIT_STREAM_TIMEOUT); + options->CircuitStreamTimeout = MIN_CIRCUIT_STREAM_TIMEOUT; + } + if (options->KeepalivePeriod < 1) REJECT("KeepalivePeriod option must be positive."); @@ -3369,6 +3340,12 @@ options_validate(or_options_t *old_options, or_options_t *options, if (ensure_bandwidth_cap(&options->RelayBandwidthBurst, "RelayBandwidthBurst", msg) < 0) return -1; + if (ensure_bandwidth_cap(&options->PerConnBWRate, + "PerConnBWRate", msg) < 0) + return -1; + if (ensure_bandwidth_cap(&options->PerConnBWBurst, + "PerConnBWBurst", msg) < 0) + return -1; if (options->RelayBandwidthRate && !options->RelayBandwidthBurst) options->RelayBandwidthBurst = options->RelayBandwidthRate; @@ -3377,31 +3354,28 @@ options_validate(or_options_t *old_options, or_options_t *options, if (server_mode(options)) { if (options->BandwidthRate < ROUTER_REQUIRED_MIN_BANDWIDTH) { - r = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "BandwidthRate is set to %d bytes/second. " "For servers, it must be at least %d.", (int)options->BandwidthRate, ROUTER_REQUIRED_MIN_BANDWIDTH); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); return -1; } else if (options->MaxAdvertisedBandwidth < ROUTER_REQUIRED_MIN_BANDWIDTH/2) { - r = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "MaxAdvertisedBandwidth is set to %d bytes/second. " "For servers, it must be at least %d.", (int)options->MaxAdvertisedBandwidth, ROUTER_REQUIRED_MIN_BANDWIDTH/2); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); return -1; } if (options->RelayBandwidthRate && options->RelayBandwidthRate < ROUTER_REQUIRED_MIN_BANDWIDTH) { - r = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "RelayBandwidthRate is set to %d bytes/second. " "For servers, it must be at least %d.", (int)options->RelayBandwidthRate, ROUTER_REQUIRED_MIN_BANDWIDTH); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); return -1; } } @@ -3423,34 +3397,73 @@ options_validate(or_options_t *old_options, or_options_t *options, if (accounting_parse_options(options, 1)<0) REJECT("Failed to parse accounting options. See logs for details."); - if (options->HttpProxy) { /* parse it now */ - if (parse_addr_port(LOG_WARN, options->HttpProxy, NULL, - &options->HttpProxyAddr, &options->HttpProxyPort) < 0) - REJECT("HttpProxy failed to parse or resolve. Please fix."); - if (options->HttpProxyPort == 0) { /* give it a default */ - options->HttpProxyPort = 80; + if (options->HTTPProxy) { /* parse it now */ + if (tor_addr_port_parse(options->HTTPProxy, + &options->HTTPProxyAddr, &options->HTTPProxyPort) < 0) + REJECT("HTTPProxy failed to parse or resolve. Please fix."); + if (options->HTTPProxyPort == 0) { /* give it a default */ + options->HTTPProxyPort = 80; } } - if (options->HttpProxyAuthenticator) { - if (strlen(options->HttpProxyAuthenticator) >= 48) - REJECT("HttpProxyAuthenticator is too long (>= 48 chars)."); + if (options->HTTPProxyAuthenticator) { + if (strlen(options->HTTPProxyAuthenticator) >= 512) + REJECT("HTTPProxyAuthenticator is too long (>= 512 chars)."); } - if (options->HttpsProxy) { /* parse it now */ - if (parse_addr_port(LOG_WARN, options->HttpsProxy, NULL, - &options->HttpsProxyAddr, &options->HttpsProxyPort) <0) - REJECT("HttpsProxy failed to parse or resolve. Please fix."); - if (options->HttpsProxyPort == 0) { /* give it a default */ - options->HttpsProxyPort = 443; + if (options->HTTPSProxy) { /* parse it now */ + if (tor_addr_port_parse(options->HTTPSProxy, + &options->HTTPSProxyAddr, &options->HTTPSProxyPort) <0) + REJECT("HTTPSProxy failed to parse or resolve. Please fix."); + if (options->HTTPSProxyPort == 0) { /* give it a default */ + options->HTTPSProxyPort = 443; } } - if (options->HttpsProxyAuthenticator) { - if (strlen(options->HttpsProxyAuthenticator) >= 48) - REJECT("HttpsProxyAuthenticator is too long (>= 48 chars)."); + if (options->HTTPSProxyAuthenticator) { + if (strlen(options->HTTPSProxyAuthenticator) >= 512) + REJECT("HTTPSProxyAuthenticator is too long (>= 512 chars)."); } + if (options->Socks4Proxy) { /* parse it now */ + if (tor_addr_port_parse(options->Socks4Proxy, + &options->Socks4ProxyAddr, + &options->Socks4ProxyPort) <0) + REJECT("Socks4Proxy failed to parse or resolve. Please fix."); + if (options->Socks4ProxyPort == 0) { /* give it a default */ + options->Socks4ProxyPort = 1080; + } + } + + if (options->Socks5Proxy) { /* parse it now */ + if (tor_addr_port_parse(options->Socks5Proxy, + &options->Socks5ProxyAddr, + &options->Socks5ProxyPort) <0) + REJECT("Socks5Proxy failed to parse or resolve. Please fix."); + if (options->Socks5ProxyPort == 0) { /* give it a default */ + options->Socks5ProxyPort = 1080; + } + } + + if (options->Socks4Proxy && options->Socks5Proxy) + REJECT("You cannot specify both Socks4Proxy and SOCKS5Proxy"); + + if (options->Socks5ProxyUsername) { + size_t len; + + len = strlen(options->Socks5ProxyUsername); + if (len < 1 || len > 255) + REJECT("Socks5ProxyUsername must be between 1 and 255 characters."); + + if (!options->Socks5ProxyPassword) + REJECT("Socks5ProxyPassword must be included with Socks5ProxyUsername."); + + len = strlen(options->Socks5ProxyPassword); + if (len < 1 || len > 255) + REJECT("Socks5ProxyPassword must be between 1 and 255 characters."); + } else if (options->Socks5ProxyPassword) + REJECT("Socks5ProxyPassword must be included with Socks5ProxyUsername."); + if (options->HashedControlPassword) { smartlist_t *sl = decode_hashed_passwords(options->HashedControlPassword); if (!sl) { @@ -3472,6 +3485,16 @@ options_validate(or_options_t *old_options, or_options_t *options, } } + if (options->OwningControllerProcess) { + const char *validate_pspec_msg = NULL; + if (tor_validate_process_specifier(options->OwningControllerProcess, + &validate_pspec_msg)) { + tor_asprintf(msg, "Bad OwningControllerProcess: %s", + validate_pspec_msg); + return -1; + } + } + if (options->ControlListenAddress) { int all_are_local = 1; config_line_t *ln; @@ -3514,6 +3537,12 @@ options_validate(or_options_t *old_options, or_options_t *options, "upgrade your Tor controller as soon as possible."); } + if (options->CookieAuthFileGroupReadable && !options->CookieAuthFile) { + log_warn(LD_CONFIG, "CookieAuthFileGroupReadable is set, but will have " + "no effect: you must specify an explicit CookieAuthFile to " + "have it group-readable."); + } + if (options->UseEntryGuards && ! options->NumEntryGuards) REJECT("Cannot enable UseEntryGuards with NumEntryGuards set to 0"); @@ -3547,11 +3576,10 @@ options_validate(or_options_t *old_options, or_options_t *options, if (options->ConstrainedSockSize < MIN_CONSTRAINED_TCP_BUFFER || options->ConstrainedSockSize > MAX_CONSTRAINED_TCP_BUFFER || options->ConstrainedSockSize % 1024) { - r = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "ConstrainedSockSize is invalid. Must be a value between %d and %d " "in 1024 byte increments.", MIN_CONSTRAINED_TCP_BUFFER, MAX_CONSTRAINED_TCP_BUFFER); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); return -1; } if (options->DirPort) { @@ -3599,6 +3627,12 @@ options_validate(or_options_t *old_options, or_options_t *options, if (options->PreferTunneledDirConns && !options->TunnelDirConns) REJECT("Must set TunnelDirConns if PreferTunneledDirConns is set."); + if ((options->Socks4Proxy || options->Socks5Proxy) && + !options->HTTPProxy && !options->PreferTunneledDirConns) + REJECT("When Socks4Proxy or Socks5Proxy is configured, " + "PreferTunneledDirConns and TunnelDirConns must both be " + "set to 1, or HTTPProxy must be configured."); + if (options->AutomapHostsSuffixes) { SMARTLIST_FOREACH(options->AutomapHostsSuffixes, char *, suf, { @@ -3613,12 +3647,19 @@ options_validate(or_options_t *old_options, or_options_t *options, "a non-default set of DirServers."); } - /*XXXX022 checking for defaults manually like this is a bit fragile.*/ + if (options->AllowSingleHopExits && !options->DirServers) { + COMPLAIN("You have set AllowSingleHopExits; now your relay will allow " + "others to make one-hop exits. However, since by default most " + "clients avoid relays that set this option, most clients will " + "ignore you."); + } + + /*XXXX023 checking for defaults manually like this is a bit fragile.*/ /* Keep changes to hard-coded values synchronous to man page and default * values table. */ if (options->TestingV3AuthInitialVotingInterval != 30*60 && - !options->TestingTorNetwork) { + !options->TestingTorNetwork && !options->_UsingTestNetworkDefaults) { REJECT("TestingV3AuthInitialVotingInterval may only be changed in testing " "Tor networks!"); } else if (options->TestingV3AuthInitialVotingInterval < MIN_VOTE_INTERVAL) { @@ -3629,7 +3670,8 @@ options_validate(or_options_t *old_options, or_options_t *options, } if (options->TestingV3AuthInitialVoteDelay != 5*60 && - !options->TestingTorNetwork) { + !options->TestingTorNetwork && !options->_UsingTestNetworkDefaults) { + REJECT("TestingV3AuthInitialVoteDelay may only be changed in testing " "Tor networks!"); } else if (options->TestingV3AuthInitialVoteDelay < MIN_VOTE_SECONDS) { @@ -3637,7 +3679,7 @@ options_validate(or_options_t *old_options, or_options_t *options, } if (options->TestingV3AuthInitialDistDelay != 5*60 && - !options->TestingTorNetwork) { + !options->TestingTorNetwork && !options->_UsingTestNetworkDefaults) { REJECT("TestingV3AuthInitialDistDelay may only be changed in testing " "Tor networks!"); } else if (options->TestingV3AuthInitialDistDelay < MIN_DIST_SECONDS) { @@ -3652,7 +3694,7 @@ options_validate(or_options_t *old_options, or_options_t *options, } if (options->TestingAuthDirTimeToLearnReachability != 30*60 && - !options->TestingTorNetwork) { + !options->TestingTorNetwork && !options->_UsingTestNetworkDefaults) { REJECT("TestingAuthDirTimeToLearnReachability may only be changed in " "testing Tor networks!"); } else if (options->TestingAuthDirTimeToLearnReachability < 0) { @@ -3662,7 +3704,7 @@ options_validate(or_options_t *old_options, or_options_t *options, } if (options->TestingEstimatedDescriptorPropagationTime != 10*60 && - !options->TestingTorNetwork) { + !options->TestingTorNetwork && !options->_UsingTestNetworkDefaults) { REJECT("TestingEstimatedDescriptorPropagationTime may only be changed in " "testing Tor networks!"); } else if (options->TestingEstimatedDescriptorPropagationTime < 0) { @@ -3678,6 +3720,26 @@ options_validate(or_options_t *old_options, or_options_t *options, "testing Tor network!"); } + if (options->AccelName && !options->HardwareAccel) + options->HardwareAccel = 1; + if (options->AccelDir && !options->AccelName) + REJECT("Can't use hardware crypto accelerator dir without engine name."); + + if (options->PublishServerDescriptor) + SMARTLIST_FOREACH(options->PublishServerDescriptor, const char *, pubdes, { + if (!strcmp(pubdes, "1") || !strcmp(pubdes, "0")) + if (smartlist_len(options->PublishServerDescriptor) > 1) { + COMPLAIN("You have passed a list of multiple arguments to the " + "PublishServerDescriptor option that includes 0 or 1. " + "0 or 1 should only be used as the sole argument. " + "This configuration will be rejected in a future release."); + break; + } + }); + + if (options->BridgeRelay == 1 && options->ORPort == 0) + REJECT("BridgeRelay is 1, ORPort is 0. This is an invalid combination."); + return 0; #undef REJECT #undef COMPLAIN @@ -3716,12 +3778,10 @@ options_transition_allowed(or_options_t *old, or_options_t *new_val, } if (strcmp(old->DataDirectory,new_val->DataDirectory)!=0) { - char buf[1024]; - int r = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "While Tor is running, changing DataDirectory " "(\"%s\"->\"%s\") is not allowed.", old->DataDirectory, new_val->DataDirectory); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); return -1; } @@ -3730,19 +3790,22 @@ options_transition_allowed(or_options_t *old, or_options_t *new_val, return -1; } - if (!opt_streq(old->Group, new_val->Group)) { - *msg = tor_strdup("While Tor is running, changing Group is not allowed."); + if ((old->HardwareAccel != new_val->HardwareAccel) + || !opt_streq(old->AccelName, new_val->AccelName) + || !opt_streq(old->AccelDir, new_val->AccelDir)) { + *msg = tor_strdup("While Tor is running, changing OpenSSL hardware " + "acceleration engine is not allowed."); return -1; } - if (old->HardwareAccel != new_val->HardwareAccel) { - *msg = tor_strdup("While Tor is running, changing HardwareAccel is " - "not allowed."); + if (old->TestingTorNetwork != new_val->TestingTorNetwork) { + *msg = tor_strdup("While Tor is running, changing TestingTorNetwork " + "is not allowed."); return -1; } - if (old->TestingTorNetwork != new_val->TestingTorNetwork) { - *msg = tor_strdup("While Tor is running, changing TestingTorNetwork " + if (old->DisableAllSwap != new_val->DisableAllSwap) { + *msg = tor_strdup("While Tor is running, changing DisableAllSwap " "is not allowed."); return -1; } @@ -3757,13 +3820,15 @@ options_transition_affects_workers(or_options_t *old_options, or_options_t *new_options) { if (!opt_streq(old_options->DataDirectory, new_options->DataDirectory) || - old_options->NumCpus != new_options->NumCpus || + old_options->NumCPUs != new_options->NumCPUs || old_options->ORPort != new_options->ORPort || old_options->ServerDNSSearchDomains != new_options->ServerDNSSearchDomains || - old_options->SafeLogging != new_options->SafeLogging || + old_options->_SafeLogging != new_options->_SafeLogging || old_options->ClientOnly != new_options->ClientOnly || - !config_lines_eq(old_options->Logs, new_options->Logs)) + public_server_mode(old_options) != public_server_mode(new_options) || + !config_lines_eq(old_options->Logs, new_options->Logs) || + old_options->LogMessageDomains != new_options->LogMessageDomains) return 1; /* Check whether log options match. */ @@ -3789,7 +3854,6 @@ options_transition_affects_descriptor(or_options_t *old_options, old_options->ORPort != new_options->ORPort || old_options->DirPort != new_options->DirPort || old_options->ClientOnly != new_options->ClientOnly || - old_options->NoPublish != new_options->NoPublish || old_options->_PublishServerDescriptor != new_options->_PublishServerDescriptor || get_effective_bwrate(old_options) != get_effective_bwrate(new_options) || @@ -3812,6 +3876,7 @@ get_windows_conf_root(void) { static int is_set = 0; static char path[MAX_PATH+1]; + TCHAR tpath[MAX_PATH] = {0}; LPITEMIDLIST idl; IMalloc *m; @@ -3829,7 +3894,7 @@ get_windows_conf_root(void) #define APPDATA_PATH CSIDL_APPDATA #endif if (!SUCCEEDED(SHGetSpecialFolderLocation(NULL, APPDATA_PATH, &idl))) { - GetCurrentDirectory(MAX_PATH, path); + getcwd(path,MAX_PATH); is_set = 1; log_warn(LD_CONFIG, "I couldn't find your application data folder: are you " @@ -3838,8 +3903,15 @@ get_windows_conf_root(void) return path; } /* Convert the path from an "ID List" (whatever that is!) to a path. */ - result = SHGetPathFromIDList(idl, path); - /* Now we need to free the */ + result = SHGetPathFromIDList(idl, tpath); +#ifdef UNICODE + wcstombs(path,tpath,MAX_PATH); +#else + strlcpy(path,tpath,sizeof(path)); +#endif + + /* Now we need to free the memory that the path-idl was stored in. In + * typical Windows fashion, we can't just call 'free()' on it. */ SHGetMalloc(&m); if (m) { m->lpVtbl->Free(m, idl); @@ -3887,10 +3959,7 @@ check_nickname_list(const char *lst, const char *name, char **msg) SMARTLIST_FOREACH(sl, const char *, s, { if (!is_legal_nickname_or_hexdigest(s)) { - char buf[1024]; - int tmp = tor_snprintf(buf, sizeof(buf), - "Invalid nickname '%s' in %s line", s, name); - *msg = tor_strdup(tmp >= 0 ? buf : "internal error"); + tor_asprintf(msg, "Invalid nickname '%s' in %s line", s, name); r = -1; break; } @@ -3914,13 +3983,7 @@ find_torrc_filename(int argc, char **argv, log(LOG_WARN, LD_CONFIG, "Duplicate -f options on command line."); tor_free(fname); } -#ifdef MS_WINDOWS - /* XXX one day we might want to extend expand_filename to work - * under Windows as well. */ - fname = tor_strdup(argv[i+1]); -#else fname = expand_filename(argv[i+1]); -#endif *using_default_torrc = 0; ++i; } else if (!strcmp(argv[i],"--ignore-missing-torrc")) { @@ -4026,6 +4089,12 @@ options_init_from_torrc(int argc, char **argv) printf("Tor version %s.\n",get_version()); exit(0); } + if (argc > 1 && (!strcmp(argv[1],"--digests"))) { + printf("Tor version %s.\n",get_version()); + printf("%s", libor_get_digests()); + printf("%s", tor_get_digests()); + exit(0); + } /* Go through command-line variables */ if (!global_cmdline_options) { @@ -4193,12 +4262,9 @@ options_init_from_string(const char *cf, err: config_free(&options_format, newoptions); if (*msg) { - int len = (int)strlen(*msg)+256; - char *newmsg = tor_malloc(len); - - tor_snprintf(newmsg, len, "Failed to parse/validate config: %s", *msg); - tor_free(*msg); - *msg = newmsg; + char *old_msg = *msg; + tor_asprintf(msg, "Failed to parse/validate config: %s", old_msg); + tor_free(old_msg); } return err; } @@ -4318,11 +4384,13 @@ options_init_logs(or_options_t *options, int validate_only) if (smartlist_len(elts) == 2 && !strcasecmp(smartlist_get(elts,0), "file")) { if (!validate_only) { - if (add_file_log(severity, smartlist_get(elts, 1)) < 0) { + char *fname = expand_filename(smartlist_get(elts, 1)); + if (add_file_log(severity, fname) < 0) { log_warn(LD_CONFIG, "Couldn't open file for 'Log %s': %s", opt->value, strerror(errno)); ok = 0; } + tor_free(fname); } goto cleanup; } @@ -4338,6 +4406,9 @@ options_init_logs(or_options_t *options, int validate_only) } smartlist_free(elts); + if (ok && !validate_only) + logs_set_domain_logging(options->LogMessageDomains); + return ok?0:-1; } @@ -4586,7 +4657,7 @@ normalize_data_directory(or_options_t *options) } /** Check and normalize the value of options->DataDirectory; return 0 if it - * sane, -1 otherwise. */ + * is sane, -1 otherwise. */ static int validate_data_directory(or_options_t *options) { @@ -4618,7 +4689,6 @@ write_configuration_file(const char *fname, or_options_t *options) { char *old_val=NULL, *new_val=NULL, *new_conf=NULL; int rename_old = 0, r; - size_t len; tor_assert(fname); @@ -4645,9 +4715,7 @@ write_configuration_file(const char *fname, or_options_t *options) goto err; } - len = strlen(new_conf)+256; - new_val = tor_malloc(len); - tor_snprintf(new_val, len, "%s\n%s\n\n%s", + tor_asprintf(&new_val, "%s\n%s\n\n%s", GENERATED_FILE_PREFIX, GENERATED_FILE_COMMENT, new_conf); if (rename_old) { @@ -4697,22 +4765,19 @@ write_configuration_file(const char *fname, or_options_t *options) int options_save_current(void) { - if (torrc_fname) { - /* This fails if we can't write to our configuration file. - * - * If we try falling back to datadirectory or something, we have a better - * chance of saving the configuration, but a better chance of doing - * something the user never expected. Let's just warn instead. */ - return write_configuration_file(torrc_fname, get_options()); - } - return write_configuration_file(get_default_conf_file(), get_options()); + /* This fails if we can't write to our configuration file. + * + * If we try falling back to datadirectory or something, we have a better + * chance of saving the configuration, but a better chance of doing + * something the user never expected. */ + return write_configuration_file(get_torrc_fname(), get_options()); } /** Mapping from a unit name to a multiplier for converting that unit into a - * base unit. */ + * base unit. Used by config_parse_unit. */ struct unit_table_t { - const char *unit; - uint64_t multiplier; + const char *unit; /**< The name of the unit */ + uint64_t multiplier; /**< How many of the base unit appear in this unit */ }; /** Table to map the names of memory units to the number of bytes they @@ -4770,30 +4835,47 @@ static struct unit_table_t time_units[] = { static uint64_t config_parse_units(const char *val, struct unit_table_t *u, int *ok) { - uint64_t v; + uint64_t v = 0; + double d = 0; + int use_float = 0; char *cp; tor_assert(ok); v = tor_parse_uint64(val, 10, 0, UINT64_MAX, ok, &cp); - if (!*ok) - return 0; + if (!*ok || (cp && *cp == '.')) { + d = tor_parse_double(val, 0, UINT64_MAX, ok, &cp); + if (!*ok) + goto done; + use_float = 1; + } + if (!cp) { *ok = 1; - return v; + v = use_float ? DBL_TO_U64(d) : v; + goto done; } - while (TOR_ISSPACE(*cp)) - ++cp; + + cp = (char*) eat_whitespace(cp); + for ( ;u->unit;++u) { if (!strcasecmp(u->unit, cp)) { - v *= u->multiplier; + if (use_float) + v = u->multiplier * d; + else + v *= u->multiplier; *ok = 1; - return v; + goto done; } } log_warn(LD_CONFIG, "Unknown unit '%s'.", cp); *ok = 0; - return 0; + done: + + if (*ok) + return v; + else + return 0; } /** Parse a string in the format "number unit", where unit is a unit of @@ -4803,7 +4885,8 @@ config_parse_units(const char *val, struct unit_table_t *u, int *ok) static uint64_t config_parse_memunit(const char *s, int *ok) { - return config_parse_units(s, memory_units, ok); + uint64_t u = config_parse_units(s, memory_units, ok); + return u; } /** Parse a string in the format "number unit", where unit is a unit of time. @@ -4825,256 +4908,37 @@ config_parse_interval(const char *s, int *ok) return (int)r; } -/* This is what passes for version detection on OSX. We set - * MACOSX_KQUEUE_IS_BROKEN to true iff we're on a version of OSX before - * 10.4.0 (aka 1040). */ -#ifdef __APPLE__ -#ifdef __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ -#define MACOSX_KQUEUE_IS_BROKEN \ - (__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 1040) -#else -#define MACOSX_KQUEUE_IS_BROKEN 0 -#endif -#endif - /** * Initialize the libevent library. */ static void init_libevent(void) { + const char *badness=NULL; + configure_libevent_logging(); /* If the kernel complains that some method (say, epoll) doesn't * exist, we don't care about it, since libevent will cope. */ suppress_libevent_log_msg("Function not implemented"); -#ifdef __APPLE__ - if (MACOSX_KQUEUE_IS_BROKEN || - decode_libevent_version(event_get_version(), NULL) < LE_11B) { - setenv("EVENT_NOKQUEUE","1",1); - } -#endif - /* In libevent versions before 2.0, it's hard to keep binary compatibility - * between upgrades, and unpleasant to detect when the version we compiled - * against is unlike the version we have linked against. Here's how. */ -#if defined(_EVENT_VERSION) && defined(HAVE_EVENT_GET_VERSION) - /* We have a header-file version and a function-call version. Easy. */ - if (strcmp(_EVENT_VERSION, event_get_version())) { - int compat1 = -1, compat2 = -1; - int verybad, prettybad ; - decode_libevent_version(_EVENT_VERSION, &compat1); - decode_libevent_version(event_get_version(), &compat2); - verybad = compat1 != compat2; - prettybad = (compat1 == -1 || compat2 == -1) && compat1 != compat2; - - log(verybad ? LOG_WARN : (prettybad ? LOG_NOTICE : LOG_INFO), - LD_GENERAL, "We were compiled with headers from version %s " - "of Libevent, but we're using a Libevent library that says it's " - "version %s.", _EVENT_VERSION, event_get_version()); - if (verybad) - log_warn(LD_GENERAL, "This will almost certainly make Tor crash."); - else if (prettybad) - log_notice(LD_GENERAL, "If Tor crashes, this might be why."); - else - log_info(LD_GENERAL, "I think these versions are binary-compatible."); - } -#elif defined(HAVE_EVENT_GET_VERSION) - /* event_get_version but no _EVENT_VERSION. We might be in 1.4.0-beta or - earlier, where that's normal. To see whether we were compiled with an - earlier version, let's see whether the struct event defines MIN_HEAP_IDX. - */ -#ifdef HAVE_STRUCT_EVENT_MIN_HEAP_IDX - /* The header files are 1.4.0-beta or later. If the version is not - * 1.4.0-beta, we are incompatible. */ - { - if (strcmp(event_get_version(), "1.4.0-beta")) { - log_warn(LD_GENERAL, "It's a little hard to tell, but you seem to have " - "Libevent 1.4.0-beta header files, whereas you have linked " - "against Libevent %s. This will probably make Tor crash.", - event_get_version()); - } - } -#else - /* Our headers are 1.3e or earlier. If the library version is not 1.4.x or - later, we're probably fine. */ - { - const char *v = event_get_version(); - if ((v[0] == '1' && v[2] == '.' && v[3] > '3') || v[0] > '1') { - log_warn(LD_GENERAL, "It's a little hard to tell, but you seem to have " - "Libevent header file from 1.3e or earlier, whereas you have " - "linked against Libevent %s. This will probably make Tor " - "crash.", event_get_version()); - } - } -#endif + tor_check_libevent_header_compatibility(); -#elif defined(_EVENT_VERSION) -#warn "_EVENT_VERSION is defined but not get_event_version(): Libevent is odd." -#else - /* Your libevent is ancient. */ -#endif + tor_libevent_initialize(); - event_init(); suppress_libevent_log_msg(NULL); -#if defined(HAVE_EVENT_GET_VERSION) && defined(HAVE_EVENT_GET_METHOD) - /* Making this a NOTICE for now so we can link bugs to a libevent versions - * or methods better. */ - log(LOG_NOTICE, LD_GENERAL, - "Initialized libevent version %s using method %s. Good.", - event_get_version(), event_get_method()); - check_libevent_version(event_get_method(), get_options()->ORPort != 0); -#else - log(LOG_NOTICE, LD_GENERAL, - "Initialized old libevent (version 1.0b or earlier)."); - log(LOG_WARN, LD_GENERAL, - "You have a *VERY* old version of libevent. It is likely to be buggy; " - "please build Tor with a more recent version."); -#endif -} - -/** Table mapping return value of event_get_version() to le_version_t. */ -static const struct { - const char *name; le_version_t version; int bincompat; -} le_version_table[] = { - /* earlier versions don't have get_version. */ - { "1.0c", LE_10C, 1}, - { "1.0d", LE_10D, 1}, - { "1.0e", LE_10E, 1}, - { "1.1", LE_11, 1 }, - { "1.1a", LE_11A, 1 }, - { "1.1b", LE_11B, 1 }, - { "1.2", LE_12, 1 }, - { "1.2a", LE_12A, 1 }, - { "1.3", LE_13, 1 }, - { "1.3a", LE_13A, 1 }, - { "1.3b", LE_13B, 1 }, - { "1.3c", LE_13C, 1 }, - { "1.3d", LE_13D, 1 }, - { "1.3e", LE_13E, 1 }, - { "1.4.0-beta", LE_140, 2 }, - { "1.4.1-beta", LE_141, 2 }, - { "1.4.2-rc", LE_142, 2 }, - { "1.4.3-stable", LE_143, 2 }, - { "1.4.4-stable", LE_144, 2 }, - { "1.4.5-stable", LE_145, 2 }, - { "1.4.6-stable", LE_146, 2 }, - { "1.4.7-stable", LE_147, 2 }, - { "1.4.8-stable", LE_148, 2 }, - { "1.4.99-trunk", LE_1499, 3 }, - { NULL, LE_OTHER, 0 } -}; - -/** Return the le_version_t for the current version of libevent. If the - * version is very new, return LE_OTHER. If the version is so old that it - * doesn't support event_get_version(), return LE_OLD. */ -static le_version_t -decode_libevent_version(const char *v, int *bincompat_out) -{ - int i; - for (i=0; le_version_table[i].name; ++i) { - if (!strcmp(le_version_table[i].name, v)) { - if (bincompat_out) - *bincompat_out = le_version_table[i].bincompat; - return le_version_table[i].version; - } - } - if (v[0] != '1' && bincompat_out) - *bincompat_out = 100; - else if (!strcmpstart(v, "1.4") && bincompat_out) - *bincompat_out = 2; - return LE_OTHER; -} -#if defined(HAVE_EVENT_GET_VERSION) && defined(HAVE_EVENT_GET_METHOD) -/** - * Compare the given libevent method and version to a list of versions - * which are known not to work. Warn the user as appropriate. - */ -static void -check_libevent_version(const char *m, int server) -{ - int buggy = 0, iffy = 0, slow = 0, thread_unsafe = 0; - le_version_t version; - const char *v = event_get_version(); - const char *badness = NULL; - const char *sad_os = ""; - - version = decode_libevent_version(v, NULL); - - /* XXX Would it be worthwhile disabling the methods that we know - * are buggy, rather than just warning about them and then proceeding - * to use them? If so, we should probably not wrap this whole thing - * in HAVE_EVENT_GET_VERSION and HAVE_EVENT_GET_METHOD. -RD */ - /* XXXX The problem is that it's not trivial to get libevent to change it's - * method once it's initialized, and it's not trivial to tell what method it - * will use without initializing it. I guess we could preemptively disable - * buggy libevent modes based on the version _before_ initializing it, - * though, but then there's no good way (afaict) to warn "I would have used - * kqueue, but instead I'm using select." -NM */ - if (!strcmp(m, "kqueue")) { - if (version < LE_11B) - buggy = 1; - } else if (!strcmp(m, "epoll")) { - if (version < LE_11) - iffy = 1; - } else if (!strcmp(m, "poll")) { - if (version < LE_10E) - buggy = 1; - else if (version < LE_11) - slow = 1; - } else if (!strcmp(m, "select")) { - if (version < LE_11) - slow = 1; - } else if (!strcmp(m, "win32")) { - if (version < LE_11B) - buggy = 1; - } - - /* Libevent versions before 1.3b do very badly on operating systems with - * user-space threading implementations. */ -#if defined(__OpenBSD__) || defined(__FreeBSD__) || defined(__NetBSD__) - if (server && version < LE_13B) { - thread_unsafe = 1; - sad_os = "BSD variants"; - } -#elif defined(__APPLE__) || defined(__darwin__) - if (server && version < LE_13B) { - thread_unsafe = 1; - sad_os = "Mac OS X"; - } -#endif - - if (thread_unsafe) { - log(LOG_WARN, LD_GENERAL, - "Libevent version %s often crashes when running a Tor server with %s. " - "Please use the latest version of libevent (1.3b or later)",v,sad_os); - badness = "BROKEN"; - } else if (buggy) { - log(LOG_WARN, LD_GENERAL, - "There are serious bugs in using %s with libevent %s. " - "Please use the latest version of libevent.", m, v); - badness = "BROKEN"; - } else if (iffy) { - log(LOG_WARN, LD_GENERAL, - "There are minor bugs in using %s with libevent %s. " - "You may want to use the latest version of libevent.", m, v); - badness = "BUGGY"; - } else if (slow && server) { - log(LOG_WARN, LD_GENERAL, - "libevent %s can be very slow with %s. " - "When running a server, please use the latest version of libevent.", - v,m); - badness = "SLOW"; - } + tor_check_libevent_version(tor_libevent_get_method(), + get_options()->ORPort != 0, + &badness); if (badness) { + const char *v = tor_libevent_get_version_str(); + const char *m = tor_libevent_get_method(); control_event_general_status(LOG_WARN, "BAD_LIBEVENT VERSION=%s METHOD=%s BADNESS=%s RECOVERED=NO", v, m, badness); } - } -#endif /** Return the persistent state struct for this Tor. */ or_state_t * @@ -5156,22 +5020,61 @@ or_state_validate(or_state_t *old_state, or_state_t *state, } /** Replace the current persistent state with <b>new_state</b> */ -static void +static int or_state_set(or_state_t *new_state) { char *err = NULL; + int ret = 0; tor_assert(new_state); - if (global_state) - config_free(&state_format, global_state); + config_free(&state_format, global_state); global_state = new_state; if (entry_guards_parse_state(global_state, 1, &err)<0) { log_warn(LD_GENERAL,"%s",err); tor_free(err); + ret = -1; } if (rep_hist_load_state(global_state, &err)<0) { log_warn(LD_GENERAL,"Unparseable bandwidth history state: %s",err); tor_free(err); + ret = -1; } + if (circuit_build_times_parse_state(&circ_times, global_state) < 0) { + ret = -1; + } + return ret; +} + +/** + * Save a broken state file to a backup location. + */ +static void +or_state_save_broken(char *fname) +{ + int i; + file_status_t status; + size_t len = strlen(fname)+16; + char *fname2 = tor_malloc(len); + for (i = 0; i < 100; ++i) { + tor_snprintf(fname2, len, "%s.%d", fname, i); + status = file_status(fname2); + if (status == FN_NOENT) + break; + } + if (i == 100) { + log_warn(LD_BUG, "Unable to parse state in \"%s\"; too many saved bad " + "state files to move aside. Discarding the old state file.", + fname); + unlink(fname); + } else { + log_warn(LD_BUG, "Unable to parse state in \"%s\". Moving it aside " + "to \"%s\". This could be a bug in Tor; please tell " + "the developers.", fname, fname2); + if (rename(fname, fname2) < 0) { + log_warn(LD_BUG, "Weirdly, I couldn't even move the state aside. The " + "OS gave an error of %s", strerror(errno)); + } + } + tor_free(fname2); } /** Reload the persistent state from disk, generating a new state as needed. @@ -5233,31 +5136,8 @@ or_state_load(void) " This is a bug in Tor."); goto done; } else if (badstate && contents) { - int i; - file_status_t status; - size_t len = strlen(fname)+16; - char *fname2 = tor_malloc(len); - for (i = 0; i < 100; ++i) { - tor_snprintf(fname2, len, "%s.%d", fname, i); - status = file_status(fname2); - if (status == FN_NOENT) - break; - } - if (i == 100) { - log_warn(LD_BUG, "Unable to parse state in \"%s\"; too many saved bad " - "state files to move aside. Discarding the old state file.", - fname); - unlink(fname); - } else { - log_warn(LD_BUG, "Unable to parse state in \"%s\". Moving it aside " - "to \"%s\". This could be a bug in Tor; please tell " - "the developers.", fname, fname2); - if (rename(fname, fname2) < 0) { - log_warn(LD_BUG, "Weirdly, I couldn't even move the state aside. The " - "OS gave an error of %s", strerror(errno)); - } - } - tor_free(fname2); + or_state_save_broken(fname); + tor_free(contents); config_free(&state_format, new_state); @@ -5269,7 +5149,9 @@ or_state_load(void) } else { log_info(LD_GENERAL, "Initialized state"); } - or_state_set(new_state); + if (or_state_set(new_state) == -1) { + or_state_save_broken(fname); + } new_state = NULL; if (!contents) { global_state->next_write = 0; @@ -5286,13 +5168,32 @@ or_state_load(void) return r; } +/** Did the last time we tried to write the state file fail? If so, we + * should consider disabling such features as preemptive circuit generation + * to compute circuit-build-time. */ +static int last_state_file_write_failed = 0; + +/** Return whether the state file failed to write last time we tried. */ +int +did_last_state_file_write_fail(void) +{ + return last_state_file_write_failed; +} + +/** If writing the state to disk fails, try again after this many seconds. */ +#define STATE_WRITE_RETRY_INTERVAL 3600 + +/** If we're a relay, how often should we checkpoint our state file even + * if nothing else dirties it? This will checkpoint ongoing stats like + * bandwidth used, per-country user stats, etc. */ +#define STATE_RELAY_CHECKPOINT_INTERVAL (12*60*60) + /** Write the persistent state to disk. Return 0 for success, <0 on failure. */ int or_state_save(time_t now) { char *state, *contents; char tbuf[ISO_TIME_LEN+1]; - size_t len; char *fname; tor_assert(global_state); @@ -5304,20 +5205,18 @@ or_state_save(time_t now) * to avoid redundant writes. */ entry_guards_update_state(global_state); rep_hist_update_state(global_state); + circuit_build_times_update_state(&circ_times, global_state); if (accounting_is_enabled(get_options())) accounting_run_housekeeping(now); - global_state->LastWritten = time(NULL); + global_state->LastWritten = now; + tor_free(global_state->TorVersion); - len = strlen(get_version())+8; - global_state->TorVersion = tor_malloc(len); - tor_snprintf(global_state->TorVersion, len, "Tor %s", get_version()); + tor_asprintf(&global_state->TorVersion, "Tor %s", get_version()); state = config_dump(&state_format, global_state, 1, 0); - len = strlen(state)+256; - contents = tor_malloc(len); - format_local_iso_time(tbuf, time(NULL)); - tor_snprintf(contents, len, + format_local_iso_time(tbuf, now); + tor_asprintf(&contents, "# Tor state file last generated on %s local time\n" "# Other times below are in GMT\n" "# You *do not* need to edit this file.\n\n%s", @@ -5327,15 +5226,25 @@ or_state_save(time_t now) if (write_str_to_file(fname, contents, 0)<0) { log_warn(LD_FS, "Unable to write state to file \"%s\"; " "will try again later", fname); + last_state_file_write_failed = 1; tor_free(fname); tor_free(contents); + /* Try again after STATE_WRITE_RETRY_INTERVAL (or sooner, if the state + * changes sooner). */ + global_state->next_write = now + STATE_WRITE_RETRY_INTERVAL; return -1; } + + last_state_file_write_failed = 0; log_info(LD_GENERAL, "Saved state to \"%s\"", fname); tor_free(fname); tor_free(contents); - global_state->next_write = TIME_MAX; + if (server_mode(get_options())) + global_state->next_write = now + STATE_RELAY_CHECKPOINT_INTERVAL; + else + global_state->next_write = TIME_MAX; + return 0; } @@ -5362,22 +5271,23 @@ remove_file_if_very_old(const char *fname, time_t now) * types. */ int getinfo_helper_config(control_connection_t *conn, - const char *question, char **answer) + const char *question, char **answer, + const char **errmsg) { (void) conn; + (void) errmsg; if (!strcmp(question, "config/names")) { smartlist_t *sl = smartlist_create(); int i; for (i = 0; _option_vars[i].name; ++i) { config_var_t *var = &_option_vars[i]; - const char *type, *desc; + const char *type; char *line; - size_t len; - desc = config_find_description(&options_format, var->name); switch (var->type) { case CONFIG_TYPE_STRING: type = "String"; break; case CONFIG_TYPE_FILENAME: type = "Filename"; break; case CONFIG_TYPE_UINT: type = "Integer"; break; + case CONFIG_TYPE_PORT: type = "Port"; break; case CONFIG_TYPE_INTERVAL: type = "TimeInterval"; break; case CONFIG_TYPE_MEMUNIT: type = "DataSize"; break; case CONFIG_TYPE_DOUBLE: type = "Float"; break; @@ -5394,14 +5304,7 @@ getinfo_helper_config(control_connection_t *conn, } if (!type) continue; - len = strlen(var->name)+strlen(type)+16; - if (desc) - len += strlen(desc); - line = tor_malloc(len); - if (desc) - tor_snprintf(line, len, "%s %s %s\n",var->name,type,desc); - else - tor_snprintf(line, len, "%s %s\n",var->name,type); + tor_asprintf(&line, "%s %s\n",var->name,type); smartlist_add(sl, line); } *answer = smartlist_join_strings(sl, "", 0, NULL); diff --git a/src/or/config.h b/src/or/config.h new file mode 100644 index 0000000000..78a67dddf5 --- /dev/null +++ b/src/or/config.h @@ -0,0 +1,79 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file config.h + * \brief Header file for config.c. + **/ + +#ifndef _TOR_CONFIG_H +#define _TOR_CONFIG_H + +const char *get_dirportfrontpage(void); +or_options_t *get_options(void); +int set_options(or_options_t *new_val, char **msg); +void config_free_all(void); +const char *safe_str_client(const char *address); +const char *safe_str(const char *address); +const char *escaped_safe_str_client(const char *address); +const char *escaped_safe_str(const char *address); +const char *get_version(void); + +int config_get_lines(const char *string, config_line_t **result); +void config_free_lines(config_line_t *front); +setopt_err_t options_trial_assign(config_line_t *list, int use_defaults, + int clear_first, char **msg); +int resolve_my_address(int warn_severity, or_options_t *options, + uint32_t *addr, char **hostname_out); +int is_local_addr(const tor_addr_t *addr) ATTR_PURE; +void options_init(or_options_t *options); +char *options_dump(or_options_t *options, int minimal); +int options_init_from_torrc(int argc, char **argv); +setopt_err_t options_init_from_string(const char *cf, + int command, const char *command_arg, char **msg); +int option_is_recognized(const char *key); +const char *option_get_canonical_name(const char *key); +config_line_t *option_get_assignment(or_options_t *options, + const char *key); +int options_save_current(void); +const char *get_torrc_fname(void); +char *options_get_datadir_fname2_suffix(or_options_t *options, + const char *sub1, const char *sub2, + const char *suffix); +#define get_datadir_fname2_suffix(sub1, sub2, suffix) \ + options_get_datadir_fname2_suffix(get_options(), (sub1), (sub2), (suffix)) +/** Return a newly allocated string containing datadir/sub1. See + * get_datadir_fname2_suffix. */ +#define get_datadir_fname(sub1) get_datadir_fname2_suffix((sub1), NULL, NULL) +/** Return a newly allocated string containing datadir/sub1/sub2. See + * get_datadir_fname2_suffix. */ +#define get_datadir_fname2(sub1,sub2) \ + get_datadir_fname2_suffix((sub1), (sub2), NULL) +/** Return a newly allocated string containing datadir/sub1suffix. See + * get_datadir_fname2_suffix. */ +#define get_datadir_fname_suffix(sub1, suffix) \ + get_datadir_fname2_suffix((sub1), NULL, (suffix)) + +or_state_t *get_or_state(void); +int did_last_state_file_write_fail(void); +int or_state_save(time_t now); + +int options_need_geoip_info(or_options_t *options, const char **reason_out); +int getinfo_helper_config(control_connection_t *conn, + const char *question, char **answer, + const char **errmsg); + +const char *tor_get_digests(void); +uint32_t get_effective_bwrate(or_options_t *options); +uint32_t get_effective_bwburst(or_options_t *options); + +#ifdef CONFIG_PRIVATE +/* Used only by config.c and test.c */ +or_options_t *options_new(void); +#endif + +#endif + diff --git a/src/or/config_codedigest.c b/src/or/config_codedigest.c new file mode 100644 index 0000000000..be9eaa331d --- /dev/null +++ b/src/or/config_codedigest.c @@ -0,0 +1,11 @@ + +const char *tor_get_digests(void); + +const char * +tor_get_digests(void) +{ + return "" +#include "or_sha1.i" + ; +} + diff --git a/src/or/connection.c b/src/or/connection.c index 4869a2439a..4d3e897185 100644 --- a/src/or/connection.c +++ b/src/or/connection.c @@ -11,9 +11,33 @@ **/ #include "or.h" +#include "buffers.h" +#include "circuitbuild.h" +#include "circuitlist.h" +#include "circuituse.h" +#include "config.h" +#include "connection.h" +#include "connection_edge.h" +#include "connection_or.h" +#include "control.h" +#include "cpuworker.h" +#include "directory.h" +#include "dirserv.h" +#include "dns.h" +#include "dnsserv.h" +#include "geoip.h" +#include "main.h" +#include "policies.h" +#include "reasons.h" +#include "relay.h" +#include "rendclient.h" +#include "rendcommon.h" +#include "rephist.h" +#include "router.h" +#include "routerparse.h" static connection_t *connection_create_listener( - struct sockaddr *listensockaddr, + const struct sockaddr *listensockaddr, socklen_t listensocklen, int type, char* address); static void connection_init(time_t now, connection_t *conn, int type, @@ -21,17 +45,22 @@ static void connection_init(time_t now, connection_t *conn, int type, static int connection_init_accepted_conn(connection_t *conn, uint8_t listener_type); static int connection_handle_listener_read(connection_t *conn, int new_type); -static int connection_read_bucket_should_increase(or_connection_t *conn); +static int connection_bucket_should_increase(int bucket, + or_connection_t *conn); static int connection_finished_flushing(connection_t *conn); static int connection_flushed_some(connection_t *conn); static int connection_finished_connecting(connection_t *conn); static int connection_reached_eof(connection_t *conn); -static int connection_read_to_buf(connection_t *conn, int *max_to_read, +static int connection_read_to_buf(connection_t *conn, ssize_t *max_to_read, int *socket_error); static int connection_process_inbuf(connection_t *conn, int package_partial); static void client_check_address_changed(int sock); static void set_constrained_socket_buffers(int sock, int size); +static const char *connection_proxy_state_to_string(int state); +static int connection_read_https_proxy_response(connection_t *conn); +static void connection_send_socks5_connect(connection_t *conn); + /** The last IPv4 address that our network interface seemed to have been * binding to, in host order. We use this to detect when our IP changes. */ static uint32_t last_interface_ip = 0; @@ -92,8 +121,7 @@ conn_state_to_string(int type, int state) case CONN_TYPE_OR: switch (state) { case OR_CONN_STATE_CONNECTING: return "connect()ing"; - case OR_CONN_STATE_PROXY_FLUSHING: return "proxy flushing"; - case OR_CONN_STATE_PROXY_READING: return "proxy reading"; + case OR_CONN_STATE_PROXY_HANDSHAKING: return "handshaking (proxy)"; case OR_CONN_STATE_TLS_HANDSHAKING: return "handshaking (TLS)"; case OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING: return "renegotiating (TLS)"; @@ -177,6 +205,9 @@ or_connection_new(int socket_family) or_conn->timestamp_last_added_nonpadding = time(NULL); or_conn->next_circ_id = crypto_rand_int(1<<15); + or_conn->active_circuit_pqueue = smartlist_create(); + or_conn->active_circuit_pqueue_last_recalibrated = cell_ewma_get_tick(); + return or_conn; } @@ -202,6 +233,7 @@ control_connection_new(int socket_family) tor_malloc_zero(sizeof(control_connection_t)); connection_init(time(NULL), TO_CONN(control_conn), CONN_TYPE_CONTROL, socket_family); + log_notice(LD_CONTROL, "New control connection opened."); return control_conn; } @@ -299,25 +331,6 @@ connection_link_connections(connection_t *conn_a, connection_t *conn_b) conn_b->linked_conn = conn_a; } -/** Tell libevent that we don't care about <b>conn</b> any more. */ -void -connection_unregister_events(connection_t *conn) -{ - if (conn->read_event) { - if (event_del(conn->read_event)) - log_warn(LD_BUG, "Error removing read event for %d", conn->s); - tor_free(conn->read_event); - } - if (conn->write_event) { - if (event_del(conn->write_event)) - log_warn(LD_BUG, "Error removing write event for %d", conn->s); - tor_free(conn->write_event); - } - if (conn->dns_server_port) { - dnsserv_close_listener(conn); - } -} - /** Deallocate memory used by <b>conn</b>. Deallocate its buffers if * necessary, close its socket if necessary, and mark the directory as dirty * if <b>conn</b> is an OR or OP connection. @@ -327,6 +340,9 @@ _connection_free(connection_t *conn) { void *mem; size_t memlen; + if (!conn) + return; + switch (conn->type) { case CONN_TYPE_OR: tor_assert(conn->magic == OR_CONNECTION_MAGIC); @@ -384,14 +400,11 @@ _connection_free(connection_t *conn) if (connection_speaks_cells(conn)) { or_connection_t *or_conn = TO_OR_CONN(conn); - if (or_conn->tls) { - tor_tls_free(or_conn->tls); - or_conn->tls = NULL; - } - if (or_conn->handshake_state) { - or_handshake_state_free(or_conn->handshake_state); - or_conn->handshake_state = NULL; - } + tor_tls_free(or_conn->tls); + or_conn->tls = NULL; + or_handshake_state_free(or_conn->handshake_state); + or_conn->handshake_state = NULL; + smartlist_free(or_conn->active_circuit_pqueue); tor_free(or_conn->nickname); } if (CONN_IS_EDGE(conn)) { @@ -401,8 +414,8 @@ _connection_free(connection_t *conn) memset(edge_conn->socks_request, 0xcc, sizeof(socks_request_t)); tor_free(edge_conn->socks_request); } - if (edge_conn->rend_data) - rend_data_free(edge_conn->rend_data); + + rend_data_free(edge_conn->rend_data); } if (conn->type == CONN_TYPE_CONTROL) { control_connection_t *control_conn = TO_CONTROL_CONN(conn); @@ -415,16 +428,15 @@ _connection_free(connection_t *conn) if (conn->type == CONN_TYPE_DIR) { dir_connection_t *dir_conn = TO_DIR_CONN(conn); tor_free(dir_conn->requested_resource); - if (dir_conn->zlib_state) - tor_zlib_free(dir_conn->zlib_state); + + tor_zlib_free(dir_conn->zlib_state); if (dir_conn->fingerprint_stack) { SMARTLIST_FOREACH(dir_conn->fingerprint_stack, char *, cp, tor_free(cp)); smartlist_free(dir_conn->fingerprint_stack); } - if (dir_conn->cached_dir) - cached_dir_decref(dir_conn->cached_dir); - if (dir_conn->rend_data) - rend_data_free(dir_conn->rend_data); + + cached_dir_decref(dir_conn->cached_dir); + rend_data_free(dir_conn->rend_data); } if (conn->s >= 0) { @@ -439,7 +451,7 @@ _connection_free(connection_t *conn) connection_or_remove_from_identity_map(TO_OR_CONN(conn)); } - memset(conn, 0xAA, memlen); /* poison memory */ + memset(mem, 0xCC, memlen); /* poison memory */ tor_free(mem); } @@ -448,7 +460,8 @@ _connection_free(connection_t *conn) void connection_free(connection_t *conn) { - tor_assert(conn); + if (!conn) + return; tor_assert(!connection_is_on_closeable_list(conn)); tor_assert(!connection_in_array(conn)); if (conn->linked_conn) { @@ -466,8 +479,7 @@ connection_free(connection_t *conn) } } if (conn->type == CONN_TYPE_CONTROL) { - TO_CONTROL_CONN(conn)->event_mask = 0; - control_update_global_event_mask(); + connection_control_closed(TO_CONTROL_CONN(conn)); } connection_unregister_events(conn); _connection_free(conn); @@ -544,13 +556,6 @@ connection_about_to_close_connection(connection_t *conn) * failed: forget about this router, and maybe try again. */ connection_dir_request_failed(dir_conn); } - if (conn->purpose == DIR_PURPOSE_FETCH_RENDDESC && dir_conn->rend_data) { - /* Give it a try. However, there is no re-fetching for v0 rend - * descriptors; if the response is empty or the descriptor is - * unusable, close pending connections (unless a v2 request is - * still in progress). */ - rend_client_desc_trynow(dir_conn->rend_data->onion_address, 0); - } /* If we were trying to fetch a v2 rend desc and did not succeed, * retry as needed. (If a fetch is successful, the connection state * is changed to DIR_PURPOSE_HAS_FETCHED_RENDDESC to mark that @@ -573,7 +578,7 @@ connection_about_to_close_connection(connection_t *conn) or_options_t *options = get_options(); rep_hist_note_connect_failed(or_conn->identity_digest, now); entry_guard_register_connect_status(or_conn->identity_digest,0, - !options->HttpsProxy, now); + !options->HTTPSProxy, now); if (conn->state >= OR_CONN_STATE_TLS_HANDSHAKING) { int reason = tls_error_to_orconn_end_reason(or_conn->tls_error); control_event_or_conn_status(or_conn, OR_CONN_EVENT_FAILED, @@ -589,7 +594,7 @@ connection_about_to_close_connection(connection_t *conn) rep_hist_note_disconnect(or_conn->identity_digest, now); control_event_or_conn_status(or_conn, OR_CONN_EVENT_CLOSED, tls_error_to_orconn_end_reason(or_conn->tls_error)); - } else if (or_conn->identity_digest) { + } else if (!tor_digest_is_zero(or_conn->identity_digest)) { rep_hist_note_connection_died(or_conn->identity_digest, now); control_event_or_conn_status(or_conn, OR_CONN_EVENT_CLOSED, tls_error_to_orconn_end_reason(or_conn->tls_error)); @@ -753,7 +758,7 @@ connection_expire_held_open(void) * The listenaddr struct has to be freed by the caller. */ static struct sockaddr_in * -create_inet_sockaddr(const char *listenaddress, uint16_t listenport, +create_inet_sockaddr(const char *listenaddress, int listenport, char **readable_address, socklen_t *socklen_out) { struct sockaddr_in *listenaddr = NULL; uint32_t addr; @@ -765,8 +770,10 @@ create_inet_sockaddr(const char *listenaddress, uint16_t listenport, "Error parsing/resolving ListenAddress %s", listenaddress); goto err; } - if (usePort==0) - usePort = listenport; + if (usePort==0) { + if (listenport != CFG_AUTO_PORT) + usePort = listenport; + } listenaddr = tor_malloc_zero(sizeof(struct sockaddr_in)); listenaddr->sin_addr.s_addr = htonl(addr); @@ -833,16 +840,72 @@ static void warn_too_many_conns(void) { #define WARN_TOO_MANY_CONNS_INTERVAL (6*60*60) - static time_t last_warned = 0; - time_t now = time(NULL); - int n_conns = get_n_open_sockets(); - if (last_warned + WARN_TOO_MANY_CONNS_INTERVAL < now) { + static ratelim_t last_warned = RATELIM_INIT(WARN_TOO_MANY_CONNS_INTERVAL); + char *m; + if ((m = rate_limit_log(&last_warned, approx_time()))) { + int n_conns = get_n_open_sockets(); log_warn(LD_NET,"Failing because we have %d connections already. Please " - "raise your ulimit -n.", n_conns); - last_warned = now; + "raise your ulimit -n.%s", n_conns, m); + tor_free(m); + control_event_general_status(LOG_WARN, "TOO_MANY_CONNECTIONS CURRENT=%d", + n_conns); } - control_event_general_status(LOG_WARN, "TOO_MANY_CONNECTIONS CURRENT=%d", - n_conns); +} + +#ifdef HAVE_SYS_UN_H +/** Check whether we should be willing to open an AF_UNIX socket in + * <b>path</b>. Return 0 if we should go ahead and -1 if we shouldn't. */ +static int +check_location_for_unix_socket(or_options_t *options, const char *path) +{ + int r = -1; + char *p = tor_strdup(path); + cpd_check_t flags = CPD_CHECK_MODE_ONLY; + if (get_parent_directory(p)<0) + goto done; + + if (options->ControlSocketsGroupWritable) + flags |= CPD_GROUP_OK; + + if (check_private_dir(p, flags) < 0) { + char *escpath, *escdir; + escpath = esc_for_log(path); + escdir = esc_for_log(p); + log_warn(LD_GENERAL, "Before Tor can create a control socket in %s, the " + "directory %s needs to exist, and to be accessible only by the " + "user%s account that is running Tor. (On some Unix systems, " + "anybody who can list a socket can conect to it, so Tor is " + "being careful.)", escpath, escdir, + options->ControlSocketsGroupWritable ? " and group" : ""); + tor_free(escpath); + tor_free(escdir); + goto done; + } + + r = 0; + done: + tor_free(p); + return r; +} +#endif + +/** Tell the TCP stack that it shouldn't wait for a long time after + * <b>sock</b> has closed before reusing its port. */ +static void +make_socket_reuseable(int sock) +{ +#ifdef MS_WINDOWS + (void) sock; +#else + int one=1; + + /* REUSEADDR on normal places means you can rebind to the port + * right after somebody else has let it go. But REUSEADDR on win32 + * means you can bind to the port _even when somebody else + * already has it bound_. So, don't do that on Win32. */ + setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (void*) &one, + (socklen_t)sizeof(one)); +#endif } /** Bind a new non-blocking socket listening to the socket described @@ -852,12 +915,13 @@ warn_too_many_conns(void) * to the conn. */ static connection_t * -connection_create_listener(struct sockaddr *listensockaddr, socklen_t socklen, +connection_create_listener(const struct sockaddr *listensockaddr, + socklen_t socklen, int type, char* address) { connection_t *conn; int s; /* the socket we're going to make */ - uint16_t usePort = 0; + uint16_t usePort = 0, gotPort = 0; int start_reading = 0; if (get_n_open_sockets() >= get_options()->_ConnLimit-1) { @@ -866,18 +930,15 @@ connection_create_listener(struct sockaddr *listensockaddr, socklen_t socklen, } if (listensockaddr->sa_family == AF_INET) { + tor_addr_t addr; int is_tcp = (type != CONN_TYPE_AP_DNS_LISTENER); -#ifndef MS_WINDOWS - int one=1; -#endif if (is_tcp) start_reading = 1; - usePort = ntohs( (uint16_t) - ((struct sockaddr_in *)listensockaddr)->sin_port); + tor_addr_from_sockaddr(&addr, listensockaddr, &usePort); log_notice(LD_NET, "Opening %s on %s:%d", - conn_type_to_string(type), address, usePort); + conn_type_to_string(type), fmt_addr(&addr), usePort); s = tor_open_socket(PF_INET, is_tcp ? SOCK_STREAM : SOCK_DGRAM, @@ -887,14 +948,7 @@ connection_create_listener(struct sockaddr *listensockaddr, socklen_t socklen, goto err; } -#ifndef MS_WINDOWS - /* REUSEADDR on normal places means you can rebind to the port - * right after somebody else has let it go. But REUSEADDR on win32 - * means you can bind to the port _even when somebody else - * already has it bound_. So, don't do that on Win32. */ - setsockopt(s, SOL_SOCKET, SO_REUSEADDR, (void*) &one, - (socklen_t)sizeof(one)); -#endif + make_socket_reuseable(s); if (bind(s,listensockaddr,socklen) < 0) { const char *helpfulhint = ""; @@ -915,6 +969,21 @@ connection_create_listener(struct sockaddr *listensockaddr, socklen_t socklen, goto err; } } + + if (usePort != 0) { + gotPort = usePort; + } else { + tor_addr_t addr2; + struct sockaddr_storage ss; + socklen_t ss_len=sizeof(ss); + if (getsockname(s, (struct sockaddr*)&ss, &ss_len)<0) { + log_warn(LD_NET, "getsockname() couldn't learn address for %s: %s", + conn_type_to_string(type), + tor_socket_strerror(tor_socket_errno(s))); + gotPort = 0; + } + tor_addr_from_sockaddr(&addr2, (struct sockaddr*)&ss, &gotPort); + } #ifdef HAVE_SYS_UN_H } else if (listensockaddr->sa_family == AF_UNIX) { start_reading = 1; @@ -923,6 +992,9 @@ connection_create_listener(struct sockaddr *listensockaddr, socklen_t socklen, * and listeners at the same time */ tor_assert(type == CONN_TYPE_CONTROL_LISTENER); + if (check_location_for_unix_socket(get_options(), address) < 0) + goto err; + log_notice(LD_NET, "Opening %s on %s", conn_type_to_string(type), address); @@ -942,6 +1014,15 @@ connection_create_listener(struct sockaddr *listensockaddr, socklen_t socklen, tor_socket_strerror(tor_socket_errno(s))); goto err; } + if (get_options()->ControlSocketsGroupWritable) { + /* We need to use chmod; fchmod doesn't work on sockets on all + * platforms. */ + if (chmod(address, 0660) < 0) { + log_warn(LD_FS,"Unable to make %s group-writable.", address); + tor_close_socket(s); + goto err; + } + } if (listen(s,SOMAXCONN) < 0) { log_warn(LD_NET, "Could not listen on %s: %s", address, @@ -962,7 +1043,7 @@ connection_create_listener(struct sockaddr *listensockaddr, socklen_t socklen, conn->socket_family = listensockaddr->sa_family; conn->s = s; conn->address = tor_strdup(address); - conn->port = usePort; + conn->port = gotPort; if (connection_add(conn) < 0) { /* no space, forget it */ log_warn(LD_NET,"connection_add for listener failed. Giving up."); @@ -970,8 +1051,12 @@ connection_create_listener(struct sockaddr *listensockaddr, socklen_t socklen, goto err; } - log_debug(LD_NET,"%s listening on port %u.", - conn_type_to_string(type), usePort); + log_fn(usePort==gotPort ? LOG_DEBUG : LOG_NOTICE, LD_NET, + "%s listening on port %u.", + conn_type_to_string(type), gotPort); + + if (type == CONN_TYPE_CONTROL_LISTENER) + control_ports_write_to_file(); conn->state = LISTENER_STATE_READY; if (start_reading) { @@ -1082,6 +1167,7 @@ connection_handle_listener_read(connection_t *conn, int new_type) "Connection accepted on socket %d (child of fd %d).", news,conn->s); + make_socket_reuseable(news); set_socket_nonblocking(news); if (options->ConstrainedSockets) @@ -1172,7 +1258,8 @@ connection_handle_listener_read(connection_t *conn, int new_type) } if (connection_init_accepted_conn(newconn, conn->type) < 0) { - connection_mark_for_close(newconn); + if (! newconn->marked_for_close) + connection_mark_for_close(newconn); return 0; } return 0; @@ -1198,9 +1285,11 @@ connection_init_accepted_conn(connection_t *conn, uint8_t listener_type) conn->state = AP_CONN_STATE_SOCKS_WAIT; break; case CONN_TYPE_AP_TRANS_LISTENER: + TO_EDGE_CONN(conn)->is_transparent_ap = 1; conn->state = AP_CONN_STATE_CIRCUIT_WAIT; return connection_ap_process_transparent(TO_EDGE_CONN(conn)); case CONN_TYPE_AP_NATD_LISTENER: + TO_EDGE_CONN(conn)->is_transparent_ap = 1; conn->state = AP_CONN_STATE_NATD_WAIT; break; } @@ -1231,7 +1320,7 @@ connection_connect(connection_t *conn, const char *address, { int s, inprogress = 0; char addrbuf[256]; - struct sockaddr *dest_addr = (struct sockaddr*) addrbuf; + struct sockaddr *dest_addr; socklen_t dest_addr_len; or_options_t *options = get_options(); int protocol_family; @@ -1254,7 +1343,7 @@ connection_connect(connection_t *conn, const char *address, return -1; } - if (options->OutboundBindAddress) { + if (options->OutboundBindAddress && !tor_addr_is_loopback(addr)) { struct sockaddr_in ext_addr; memset(&ext_addr, 0, sizeof(ext_addr)); @@ -1285,7 +1374,10 @@ connection_connect(connection_t *conn, const char *address, dest_addr_len = tor_addr_to_sockaddr(addr, port, dest_addr, sizeof(addrbuf)); tor_assert(dest_addr_len > 0); - log_debug(LD_NET,"Connecting to %s:%u.",escaped_safe_str(address),port); + log_debug(LD_NET, "Connecting to %s:%u.", + escaped_safe_str_client(address), port); + + make_socket_reuseable(s); if (connect(s, dest_addr, dest_addr_len) < 0) { int e = tor_socket_errno(s); @@ -1293,7 +1385,8 @@ connection_connect(connection_t *conn, const char *address, /* yuck. kill it. */ *socket_error = e; log_info(LD_NET, - "connect() to %s:%u failed: %s",escaped_safe_str(address), + "connect() to %s:%u failed: %s", + escaped_safe_str_client(address), port, tor_socket_strerror(e)); tor_close_socket(s); return -1; @@ -1307,7 +1400,8 @@ connection_connect(connection_t *conn, const char *address, /* it succeeded. we're connected. */ log_fn(inprogress?LOG_DEBUG:LOG_INFO, LD_NET, - "Connection to %s:%u %s (sock %d).",escaped_safe_str(address), + "Connection to %s:%u %s (sock %d).", + escaped_safe_str_client(address), port, inprogress?"in progress":"established", s); conn->s = s; if (connection_add(conn) < 0) /* no space, forget it */ @@ -1315,6 +1409,363 @@ connection_connect(connection_t *conn, const char *address, return inprogress ? 0 : 1; } +/** Convert state number to string representation for logging purposes. + */ +static const char * +connection_proxy_state_to_string(int state) +{ + static const char *unknown = "???"; + static const char *states[] = { + "PROXY_NONE", + "PROXY_HTTPS_WANT_CONNECT_OK", + "PROXY_SOCKS4_WANT_CONNECT_OK", + "PROXY_SOCKS5_WANT_AUTH_METHOD_NONE", + "PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929", + "PROXY_SOCKS5_WANT_AUTH_RFC1929_OK", + "PROXY_SOCKS5_WANT_CONNECT_OK", + "PROXY_CONNECTED", + }; + + if (state < PROXY_NONE || state > PROXY_CONNECTED) + return unknown; + + return states[state]; +} + +/** Write a proxy request of <b>type</b> (socks4, socks5, https) to conn + * for conn->addr:conn->port, authenticating with the auth details given + * in the configuration (if available). SOCKS 5 and HTTP CONNECT proxies + * support authentication. + * + * Returns -1 if conn->addr is incompatible with the proxy protocol, and + * 0 otherwise. + * + * Use connection_read_proxy_handshake() to complete the handshake. + */ +int +connection_proxy_connect(connection_t *conn, int type) +{ + or_options_t *options; + + tor_assert(conn); + + options = get_options(); + + switch (type) { + case PROXY_CONNECT: { + char buf[1024]; + char *base64_authenticator=NULL; + const char *authenticator = options->HTTPSProxyAuthenticator; + + /* Send HTTP CONNECT and authentication (if available) in + * one request */ + + if (authenticator) { + base64_authenticator = alloc_http_authenticator(authenticator); + if (!base64_authenticator) + log_warn(LD_OR, "Encoding https authenticator failed"); + } + + if (base64_authenticator) { + tor_snprintf(buf, sizeof(buf), "CONNECT %s:%d HTTP/1.1\r\n" + "Proxy-Authorization: Basic %s\r\n\r\n", + fmt_addr(&conn->addr), + conn->port, base64_authenticator); + tor_free(base64_authenticator); + } else { + tor_snprintf(buf, sizeof(buf), "CONNECT %s:%d HTTP/1.0\r\n\r\n", + fmt_addr(&conn->addr), conn->port); + } + + connection_write_to_buf(buf, strlen(buf), conn); + conn->proxy_state = PROXY_HTTPS_WANT_CONNECT_OK; + break; + } + + case PROXY_SOCKS4: { + unsigned char buf[9]; + uint16_t portn; + uint32_t ip4addr; + + /* Send a SOCKS4 connect request with empty user id */ + + if (tor_addr_family(&conn->addr) != AF_INET) { + log_warn(LD_NET, "SOCKS4 client is incompatible with IPv6"); + return -1; + } + + ip4addr = tor_addr_to_ipv4n(&conn->addr); + portn = htons(conn->port); + + buf[0] = 4; /* version */ + buf[1] = SOCKS_COMMAND_CONNECT; /* command */ + memcpy(buf + 2, &portn, 2); /* port */ + memcpy(buf + 4, &ip4addr, 4); /* addr */ + buf[8] = 0; /* userid (empty) */ + + connection_write_to_buf((char *)buf, sizeof(buf), conn); + conn->proxy_state = PROXY_SOCKS4_WANT_CONNECT_OK; + break; + } + + case PROXY_SOCKS5: { + unsigned char buf[4]; /* fields: vers, num methods, method list */ + + /* Send a SOCKS5 greeting (connect request must wait) */ + + buf[0] = 5; /* version */ + + /* number of auth methods */ + if (options->Socks5ProxyUsername) { + buf[1] = 2; + buf[2] = 0x00; /* no authentication */ + buf[3] = 0x02; /* rfc1929 Username/Passwd auth */ + conn->proxy_state = PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929; + } else { + buf[1] = 1; + buf[2] = 0x00; /* no authentication */ + conn->proxy_state = PROXY_SOCKS5_WANT_AUTH_METHOD_NONE; + } + + connection_write_to_buf((char *)buf, 2 + buf[1], conn); + break; + } + + default: + log_err(LD_BUG, "Invalid proxy protocol, %d", type); + tor_fragile_assert(); + return -1; + } + + log_debug(LD_NET, "set state %s", + connection_proxy_state_to_string(conn->proxy_state)); + + return 0; +} + +/** Read conn's inbuf. If the http response from the proxy is all + * here, make sure it's good news, then return 1. If it's bad news, + * return -1. Else return 0 and hope for better luck next time. + */ +static int +connection_read_https_proxy_response(connection_t *conn) +{ + char *headers; + char *reason=NULL; + int status_code; + time_t date_header; + + switch (fetch_from_buf_http(conn->inbuf, + &headers, MAX_HEADERS_SIZE, + NULL, NULL, 10000, 0)) { + case -1: /* overflow */ + log_warn(LD_PROTOCOL, + "Your https proxy sent back an oversized response. Closing."); + return -1; + case 0: + log_info(LD_NET,"https proxy response not all here yet. Waiting."); + return 0; + /* case 1, fall through */ + } + + if (parse_http_response(headers, &status_code, &date_header, + NULL, &reason) < 0) { + log_warn(LD_NET, + "Unparseable headers from proxy (connecting to '%s'). Closing.", + conn->address); + tor_free(headers); + return -1; + } + if (!reason) reason = tor_strdup("[no reason given]"); + + if (status_code == 200) { + log_info(LD_NET, + "HTTPS connect to '%s' successful! (200 %s) Starting TLS.", + conn->address, escaped(reason)); + tor_free(reason); + return 1; + } + /* else, bad news on the status code */ + switch (status_code) { + case 403: + log_warn(LD_NET, + "The https proxy refused to allow connection to %s " + "(status code %d, %s). Closing.", + conn->address, status_code, escaped(reason)); + break; + default: + log_warn(LD_NET, + "The https proxy sent back an unexpected status code %d (%s). " + "Closing.", + status_code, escaped(reason)); + break; + } + tor_free(reason); + return -1; +} + +/** Send SOCKS5 CONNECT command to <b>conn</b>, copying <b>conn->addr</b> + * and <b>conn->port</b> into the request. + */ +static void +connection_send_socks5_connect(connection_t *conn) +{ + unsigned char buf[1024]; + size_t reqsize = 6; + uint16_t port = htons(conn->port); + + buf[0] = 5; /* version */ + buf[1] = SOCKS_COMMAND_CONNECT; /* command */ + buf[2] = 0; /* reserved */ + + if (tor_addr_family(&conn->addr) == AF_INET) { + uint32_t addr = tor_addr_to_ipv4n(&conn->addr); + + buf[3] = 1; + reqsize += 4; + memcpy(buf + 4, &addr, 4); + memcpy(buf + 8, &port, 2); + } else { /* AF_INET6 */ + buf[3] = 4; + reqsize += 16; + memcpy(buf + 4, tor_addr_to_in6(&conn->addr), 16); + memcpy(buf + 20, &port, 2); + } + + connection_write_to_buf((char *)buf, reqsize, conn); + + conn->proxy_state = PROXY_SOCKS5_WANT_CONNECT_OK; +} + +/** Call this from connection_*_process_inbuf() to advance the proxy + * handshake. + * + * No matter what proxy protocol is used, if this function returns 1, the + * handshake is complete, and the data remaining on inbuf may contain the + * start of the communication with the requested server. + * + * Returns 0 if the current buffer contains an incomplete response, and -1 + * on error. + */ +int +connection_read_proxy_handshake(connection_t *conn) +{ + int ret = 0; + char *reason = NULL; + + log_debug(LD_NET, "enter state %s", + connection_proxy_state_to_string(conn->proxy_state)); + + switch (conn->proxy_state) { + case PROXY_HTTPS_WANT_CONNECT_OK: + ret = connection_read_https_proxy_response(conn); + if (ret == 1) + conn->proxy_state = PROXY_CONNECTED; + break; + + case PROXY_SOCKS4_WANT_CONNECT_OK: + ret = fetch_from_buf_socks_client(conn->inbuf, + conn->proxy_state, + &reason); + if (ret == 1) + conn->proxy_state = PROXY_CONNECTED; + break; + + case PROXY_SOCKS5_WANT_AUTH_METHOD_NONE: + ret = fetch_from_buf_socks_client(conn->inbuf, + conn->proxy_state, + &reason); + /* no auth needed, do connect */ + if (ret == 1) { + connection_send_socks5_connect(conn); + ret = 0; + } + break; + + case PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929: + ret = fetch_from_buf_socks_client(conn->inbuf, + conn->proxy_state, + &reason); + + /* send auth if needed, otherwise do connect */ + if (ret == 1) { + connection_send_socks5_connect(conn); + ret = 0; + } else if (ret == 2) { + unsigned char buf[1024]; + size_t reqsize, usize, psize; + const char *user, *pass; + + user = get_options()->Socks5ProxyUsername; + pass = get_options()->Socks5ProxyPassword; + tor_assert(user && pass); + + /* XXX len of user and pass must be <= 255 !!! */ + usize = strlen(user); + psize = strlen(pass); + tor_assert(usize <= 255 && psize <= 255); + reqsize = 3 + usize + psize; + + buf[0] = 1; /* negotiation version */ + buf[1] = usize; + memcpy(buf + 2, user, usize); + buf[2 + usize] = psize; + memcpy(buf + 3 + usize, pass, psize); + + connection_write_to_buf((char *)buf, reqsize, conn); + + conn->proxy_state = PROXY_SOCKS5_WANT_AUTH_RFC1929_OK; + ret = 0; + } + break; + + case PROXY_SOCKS5_WANT_AUTH_RFC1929_OK: + ret = fetch_from_buf_socks_client(conn->inbuf, + conn->proxy_state, + &reason); + /* send the connect request */ + if (ret == 1) { + connection_send_socks5_connect(conn); + ret = 0; + } + break; + + case PROXY_SOCKS5_WANT_CONNECT_OK: + ret = fetch_from_buf_socks_client(conn->inbuf, + conn->proxy_state, + &reason); + if (ret == 1) + conn->proxy_state = PROXY_CONNECTED; + break; + + default: + log_err(LD_BUG, "Invalid proxy_state for reading, %d", + conn->proxy_state); + tor_fragile_assert(); + ret = -1; + break; + } + + log_debug(LD_NET, "leaving state %s", + connection_proxy_state_to_string(conn->proxy_state)); + + if (ret < 0) { + if (reason) { + log_warn(LD_NET, "Proxy Client: unable to connect to %s:%d (%s)", + conn->address, conn->port, escaped(reason)); + tor_free(reason); + } else { + log_warn(LD_NET, "Proxy Client: unable to connect to %s:%d", + conn->address, conn->port); + } + } else if (ret == 1) { + log_info(LD_NET, "Proxy Client: connection to %s:%d successful", + conn->address, conn->port); + } + + return ret; +} + /** * Launch any configured listener connections of type <b>type</b>. (A * listener is configured if <b>port_option</b> is non-zero. If any @@ -1384,10 +1835,23 @@ retry_listeners(int type, config_line_t *cfg, if (!parse_addr_port(LOG_WARN, wanted->value, &address, NULL, &port)) { int addr_matches = !strcasecmp(address, conn->address); + int port_matches; tor_free(address); - if (! port) - port = port_option; - if (port == conn->port && addr_matches) { + if (port) { + /* The Listener line has a port */ + port_matches = (port == conn->port); + } else if (port_option == CFG_AUTO_PORT) { + /* The Listener line has no port, and the Port line is "auto". + * "auto" matches anything; transitions from any port to + * "auto" succeed. */ + port_matches = 1; + } else { + /* The Listener line has no port, and the Port line is "auto". + * "auto" matches anything; transitions from any port to + * "auto" succeed. */ + port_matches = (port_option == conn->port); + } + if (port_matches && addr_matches) { line = wanted; break; } @@ -1435,7 +1899,7 @@ retry_listeners(int type, config_line_t *cfg, case AF_INET: listensockaddr = (struct sockaddr *) create_inet_sockaddr(cfg_line->value, - (uint16_t) port_option, + port_option, &address, &listensocklen); break; case AF_UNIX: @@ -1506,8 +1970,8 @@ retry_all_listeners(smartlist_t *replaced_conns, replaced_conns, new_conns, 0, AF_INET)<0) return -1; - if (retry_listeners(CONN_TYPE_AP_NATD_LISTENER, options->NatdListenAddress, - options->NatdPort, "127.0.0.1", + if (retry_listeners(CONN_TYPE_AP_NATD_LISTENER, options->NATDListenAddress, + options->NATDPort, "127.0.0.1", replaced_conns, new_conns, 0, AF_INET)<0) return -1; @@ -1643,6 +2107,7 @@ connection_bucket_write_limit(connection_t *conn, time_t now) int base = connection_speaks_cells(conn) ? CELL_NETWORK_SIZE : RELAY_PAYLOAD_SIZE; int priority = conn->type != CONN_TYPE_DIR; + int conn_bucket = (int)conn->outbuf_flushlen; int global_bucket = global_write_bucket; if (!connection_is_rate_limited(conn)) { @@ -1650,12 +2115,22 @@ connection_bucket_write_limit(connection_t *conn, time_t now) return conn->outbuf_flushlen; } + if (connection_speaks_cells(conn)) { + /* use the per-conn write limit if it's lower, but if it's less + * than zero just use zero */ + or_connection_t *or_conn = TO_OR_CONN(conn); + if (conn->state == OR_CONN_STATE_OPEN) + if (or_conn->write_bucket < conn_bucket) + conn_bucket = or_conn->write_bucket >= 0 ? + or_conn->write_bucket : 0; + } + if (connection_counts_as_relayed_traffic(conn, now) && global_relayed_write_bucket <= global_write_bucket) global_bucket = global_relayed_write_bucket; - return connection_bucket_round_robin(base, priority, global_bucket, - conn->outbuf_flushlen); + return connection_bucket_round_robin(base, priority, + global_bucket, conn_bucket); } /** Return 1 if the global write buckets are low enough that we @@ -1709,14 +2184,12 @@ global_write_bucket_low(connection_t *conn, size_t attempt, int priority) return 0; } -/** We just read num_read and wrote num_written onto conn. - * Decrement buckets appropriately. */ +/** We just read <b>num_read</b> and wrote <b>num_written</b> bytes + * onto <b>conn</b>. Decrement buckets appropriately. */ static void connection_buckets_decrement(connection_t *conn, time_t now, size_t num_read, size_t num_written) { - if (!connection_is_rate_limited(conn)) - return; /* local IPs are free */ if (num_written >= INT_MAX || num_read >= INT_MAX) { log_err(LD_BUG, "Value out of range. num_read=%lu, num_written=%lu, " "connection type=%s, state=%s", @@ -1728,10 +2201,24 @@ connection_buckets_decrement(connection_t *conn, time_t now, tor_fragile_assert(); } - if (num_read > 0) + /* Count bytes of answering direct and tunneled directory requests */ + if (conn->type == CONN_TYPE_DIR && conn->purpose == DIR_PURPOSE_SERVER) { + if (num_read > 0) + rep_hist_note_dir_bytes_read(num_read, now); + if (num_written > 0) + rep_hist_note_dir_bytes_written(num_written, now); + } + + if (!connection_is_rate_limited(conn)) + return; /* local IPs are free */ + if (num_read > 0) { rep_hist_note_bytes_read(num_read, now); - if (num_written > 0) + } + if (num_written > 0) { rep_hist_note_bytes_written(num_written, now); + } + if (conn->type == CONN_TYPE_EXIT) + rep_hist_note_exit_bytes(conn->port, num_written, num_read); if (connection_counts_as_relayed_traffic(conn, now)) { global_relayed_read_bucket -= (int)num_read; @@ -1739,8 +2226,10 @@ connection_buckets_decrement(connection_t *conn, time_t now, } global_read_bucket -= (int)num_read; global_write_bucket -= (int)num_written; - if (connection_speaks_cells(conn) && conn->state == OR_CONN_STATE_OPEN) + if (connection_speaks_cells(conn) && conn->state == OR_CONN_STATE_OPEN) { TO_OR_CONN(conn)->read_bucket -= (int)num_read; + TO_OR_CONN(conn)->write_bucket -= (int)num_written; + } } /** If we have exhausted our global buckets, or the buckets for conn, @@ -1779,12 +2268,10 @@ connection_consider_empty_write_buckets(connection_t *conn) } else if (connection_counts_as_relayed_traffic(conn, approx_time()) && global_relayed_write_bucket <= 0) { reason = "global relayed write bucket exhausted. Pausing."; -#if 0 } else if (connection_speaks_cells(conn) && conn->state == OR_CONN_STATE_OPEN && TO_OR_CONN(conn)->write_bucket <= 0) { reason = "connection write bucket exhausted. Pausing."; -#endif } else return; /* all good, no need to stop it */ @@ -1880,14 +2367,19 @@ connection_bucket_refill(int seconds_elapsed, time_t now) { if (connection_speaks_cells(conn)) { or_connection_t *or_conn = TO_OR_CONN(conn); - if (connection_read_bucket_should_increase(or_conn)) { + if (connection_bucket_should_increase(or_conn->read_bucket, or_conn)) { connection_bucket_refill_helper(&or_conn->read_bucket, or_conn->bandwidthrate, or_conn->bandwidthburst, seconds_elapsed, "or_conn->read_bucket"); - //log_fn(LOG_DEBUG,"Receiver bucket %d now %d.", i, - // conn->read_bucket); + } + if (connection_bucket_should_increase(or_conn->write_bucket, or_conn)) { + connection_bucket_refill_helper(&or_conn->write_bucket, + or_conn->bandwidthrate, + or_conn->bandwidthburst, + seconds_elapsed, + "or_conn->write_bucket"); } } @@ -1908,8 +2400,10 @@ connection_bucket_refill(int seconds_elapsed, time_t now) if (conn->write_blocked_on_bw == 1 && global_write_bucket > 0 /* and we're allowed to write */ && (!connection_counts_as_relayed_traffic(conn, now) || - global_relayed_write_bucket > 0)) { - /* even if we're relayed traffic */ + global_relayed_write_bucket > 0) /* even if it's relayed traffic */ + && (!connection_speaks_cells(conn) || + conn->state != OR_CONN_STATE_OPEN || + TO_OR_CONN(conn)->write_bucket > 0)) { LOG_FN_CONN(conn, (LOG_DEBUG,LD_NET, "waking up conn (fd %d) for write", conn->s)); conn->write_blocked_on_bw = 0; @@ -1918,17 +2412,17 @@ connection_bucket_refill(int seconds_elapsed, time_t now) }); } -/** Is the receiver bucket for connection <b>conn</b> low enough that we +/** Is the <b>bucket</b> for connection <b>conn</b> low enough that we * should add another pile of tokens to it? */ static int -connection_read_bucket_should_increase(or_connection_t *conn) +connection_bucket_should_increase(int bucket, or_connection_t *conn) { tor_assert(conn); if (conn->_base.state != OR_CONN_STATE_OPEN) return 0; /* only open connections play the rate limiting game */ - if (conn->read_bucket >= conn->bandwidthburst) + if (bucket >= conn->bandwidthburst) return 0; return 1; @@ -1946,10 +2440,10 @@ connection_read_bucket_should_increase(or_connection_t *conn) * Mark the connection and return -1 if you want to close it, else * return 0. */ -int -connection_handle_read(connection_t *conn) +static int +connection_handle_read_impl(connection_t *conn) { - int max_to_read=-1, try_to_read; + ssize_t max_to_read=-1, try_to_read; size_t before, n_read = 0; int socket_error = 0; @@ -1975,7 +2469,7 @@ connection_handle_read(connection_t *conn) return 0; } -loop_again: + loop_again: try_to_read = max_to_read; tor_assert(!conn->marked_for_close); @@ -2016,12 +2510,16 @@ loop_again: return -1; } if (conn->linked_conn) { - /* The other side's handle_write will never actually get called, so + /* The other side's handle_write() will never actually get called, so * we need to invoke the appropriate callbacks ourself. */ connection_t *linked = conn->linked_conn; if (n_read) { - /* Probably a no-op, but hey. */ + /* Probably a no-op, since linked conns typically don't count for + * bandwidth rate limiting. But do it anyway so we can keep stats + * accurately. Note that since we read the bytes from conn, and + * we're writing the bytes onto the linked connection, we count + * these as <i>written</i> bytes. */ connection_buckets_decrement(linked, approx_time(), 0, n_read); if (connection_flushed_some(linked) < 0) @@ -2033,7 +2531,7 @@ loop_again: if (!buf_datalen(linked->outbuf) && conn->active_on_link) connection_stop_reading_from_linked_conn(conn); } - /* If we hit the EOF, call connection_reached_eof. */ + /* If we hit the EOF, call connection_reached_eof(). */ if (!conn->marked_for_close && conn->inbuf_reached_eof && connection_reached_eof(conn) < 0) { @@ -2042,6 +2540,16 @@ loop_again: return 0; } +int +connection_handle_read(connection_t *conn) +{ + int res; + + tor_gettimeofday_cache_clear(); + res = connection_handle_read_impl(conn); + return res; +} + /** Pull in new bytes from conn-\>s or conn-\>linked_conn onto conn-\>inbuf, * either directly or via TLS. Reduce the token buckets by the number of bytes * read. @@ -2053,7 +2561,8 @@ loop_again: * Return -1 if we want to break conn, else return 0. */ static int -connection_read_to_buf(connection_t *conn, int *max_to_read, int *socket_error) +connection_read_to_buf(connection_t *conn, ssize_t *max_to_read, + int *socket_error) { int result; ssize_t at_most = *max_to_read; @@ -2075,7 +2584,7 @@ connection_read_to_buf(connection_t *conn, int *max_to_read, int *socket_error) } if (connection_speaks_cells(conn) && - conn->state > OR_CONN_STATE_PROXY_READING) { + conn->state > OR_CONN_STATE_PROXY_HANDSHAKING) { int pending; or_connection_t *or_conn = TO_OR_CONN(conn); size_t initial_size; @@ -2171,15 +2680,19 @@ connection_read_to_buf(connection_t *conn, int *max_to_read, int *socket_error) n_read = (size_t) result; } - if (n_read > 0) { /* change *max_to_read */ - /*XXXX021 check for overflow*/ - *max_to_read = (int)(at_most - n_read); - } + if (n_read > 0) { + /* change *max_to_read */ + *max_to_read = at_most - n_read; - if (conn->type == CONN_TYPE_AP) { - edge_connection_t *edge_conn = TO_EDGE_CONN(conn); - /*XXXX021 check for overflow*/ - edge_conn->n_read += (int)n_read; + /* Update edge_conn->n_read */ + if (conn->type == CONN_TYPE_AP) { + edge_connection_t *edge_conn = TO_EDGE_CONN(conn); + /* Check for overflow: */ + if (PREDICT_LIKELY(UINT32_MAX - edge_conn->n_read > n_read)) + edge_conn->n_read += (int)n_read; + else + edge_conn->n_read = UINT32_MAX; + } } connection_buckets_decrement(conn, approx_time(), n_read, n_written); @@ -2243,8 +2756,8 @@ connection_outbuf_too_full(connection_t *conn) * Mark the connection and return -1 if you want to close it, else * return 0. */ -int -connection_handle_write(connection_t *conn, int force) +static int +connection_handle_write_impl(connection_t *conn, int force) { int e; socklen_t len=(socklen_t)sizeof(e); @@ -2259,7 +2772,7 @@ connection_handle_write(connection_t *conn, int force) return 0; /* do nothing */ if (conn->in_flushed_some) { - log_warn(LD_BUG, "called recursively from inside conn->in_flushed_some()"); + log_warn(LD_BUG, "called recursively from inside conn->in_flushed_some"); return 0; } @@ -2303,7 +2816,7 @@ connection_handle_write(connection_t *conn, int force) : connection_bucket_write_limit(conn, now); if (connection_speaks_cells(conn) && - conn->state > OR_CONN_STATE_PROXY_READING) { + conn->state > OR_CONN_STATE_PROXY_HANDSHAKING) { or_connection_t *or_conn = TO_OR_CONN(conn); if (conn->state == OR_CONN_STATE_TLS_HANDSHAKING || conn->state == OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING) { @@ -2322,6 +2835,13 @@ connection_handle_write(connection_t *conn, int force) /* else open, or closing */ result = flush_buf_tls(or_conn->tls, conn->outbuf, max_to_write, &conn->outbuf_flushlen); + + /* If we just flushed the last bytes, check if this tunneled dir + * request is done. */ + if (buf_datalen(conn->outbuf) == 0 && conn->dirreq_id) + geoip_change_dirreq_state(conn->dirreq_id, DIRREQ_TUNNELED, + DIRREQ_OR_CONN_BUFFER_FLUSHED); + switch (result) { CASE_TOR_TLS_ERROR_ANY: case TOR_TLS_CLOSE: @@ -2341,8 +2861,8 @@ connection_handle_write(connection_t *conn, int force) if (!connection_is_reading(conn)) { connection_stop_writing(conn); conn->write_blocked_on_bw = 1; - /* we'll start reading again when the next second arrives, - * and then also start writing again. + /* we'll start reading again when we get more tokens in our + * read bucket; then we'll start writing again too. */ } /* else no problem, we're already reading */ @@ -2371,10 +2891,13 @@ connection_handle_write(connection_t *conn, int force) n_written = (size_t) result; } - if (conn->type == CONN_TYPE_AP) { + if (n_written && conn->type == CONN_TYPE_AP) { edge_connection_t *edge_conn = TO_EDGE_CONN(conn); - /*XXXX021 check for overflow.*/ - edge_conn->n_written += (int)n_written; + /* Check for overflow: */ + if (PREDICT_LIKELY(UINT32_MAX - edge_conn->n_written > n_written)) + edge_conn->n_written += (int)n_written; + else + edge_conn->n_written = UINT32_MAX; } connection_buckets_decrement(conn, approx_time(), n_read, n_written); @@ -2404,6 +2927,15 @@ connection_handle_write(connection_t *conn, int force) return 0; } +int +connection_handle_write(connection_t *conn, int force) +{ + int res; + tor_gettimeofday_cache_clear(); + res = connection_handle_write_impl(conn, force); + return res; +} + /** OpenSSL TLS record size is 16383; this is close. The goal here is to * push data out as soon as we know there's enough for a TLS record, so * during periods of high load we won't read entire megabytes from @@ -2577,13 +3109,11 @@ connection_get_by_type_state(int type, int state) /** Return a connection of type <b>type</b> that has rendquery equal * to <b>rendquery</b>, and that is not marked for close. If state - * is non-zero, conn must be of that state too. If rendversion is - * nonnegative, conn must be fetching that rendversion, too. + * is non-zero, conn must be of that state too. */ connection_t * connection_get_by_type_state_rendquery(int type, int state, - const char *rendquery, - int rendversion) + const char *rendquery) { smartlist_t *conns = get_connection_array(); @@ -2598,8 +3128,6 @@ connection_get_by_type_state_rendquery(int type, int state, (!state || state == conn->state)) { if (type == CONN_TYPE_DIR && TO_DIR_CONN(conn)->rend_data && - (rendversion < 0 || - rendversion == TO_DIR_CONN(conn)->rend_data->rend_desc_version) && !rend_cmp_service_ids(rendquery, TO_DIR_CONN(conn)->rend_data->onion_address)) return conn; @@ -2703,8 +3231,17 @@ alloc_http_authenticator(const char *authenticator) authenticator, authenticator_length) < 0) { tor_free(base64_authenticator); /* free and set to null */ } else { - /* remove extra \n at end of encoding */ - base64_authenticator[strlen(base64_authenticator) - 1] = 0; + int i = 0, j = 0; + ssize_t len = strlen(base64_authenticator); + + /* remove all newline occurrences within the string */ + for (i=0; i < len; ++i) { + if ('\n' != base64_authenticator[i]) { + base64_authenticator[j] = base64_authenticator[i]; + ++j; + } + } + base64_authenticator[j]='\0'; } return base64_authenticator; } @@ -2717,10 +3254,10 @@ alloc_http_authenticator(const char *authenticator) static void client_check_address_changed(int sock) { - uint32_t iface_ip, ip_out; + uint32_t iface_ip, ip_out; /* host order */ struct sockaddr_in out_addr; socklen_t out_addr_len = (socklen_t) sizeof(out_addr); - uint32_t *ip; + uint32_t *ip; /* host order */ if (!last_interface_ip) get_interface_address(LOG_INFO, &last_interface_ip); @@ -2734,7 +3271,7 @@ client_check_address_changed(int sock) return; } - /* Okay. If we've used this address previously, we're okay. */ + /* If we've used this address previously, we're okay. */ ip_out = ntohl(out_addr.sin_addr.s_addr); SMARTLIST_FOREACH(outgoing_addrs, uint32_t*, ip_ptr, if (*ip_ptr == ip_out) return; @@ -2830,6 +3367,8 @@ connection_flushed_some(connection_t *conn) r = connection_dirserv_flushed_some(TO_DIR_CONN(conn)); } else if (conn->type == CONN_TYPE_OR) { r = connection_or_flushed_some(TO_OR_CONN(conn)); + } else if (CONN_IS_EDGE(conn)) { + r = connection_edge_flushed_some(TO_EDGE_CONN(conn)); } conn->in_flushed_some = 0; return r; @@ -3033,7 +3572,7 @@ assert_connection_ok(connection_t *conn, time_t now) } // tor_assert(conn->addr && conn->port); tor_assert(conn->address); - if (conn->state > OR_CONN_STATE_PROXY_READING) + if (conn->state > OR_CONN_STATE_PROXY_HANDSHAKING) tor_assert(or_conn->tls); } diff --git a/src/or/connection.h b/src/or/connection.h new file mode 100644 index 0000000000..576d3a63e1 --- /dev/null +++ b/src/or/connection.h @@ -0,0 +1,100 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file connection.h + * \brief Header file for connection.c. + **/ + +#ifndef _TOR_CONNECTION_H +#define _TOR_CONNECTION_H + +const char *conn_type_to_string(int type); +const char *conn_state_to_string(int type, int state); + +dir_connection_t *dir_connection_new(int socket_family); +or_connection_t *or_connection_new(int socket_family); +edge_connection_t *edge_connection_new(int type, int socket_family); +control_connection_t *control_connection_new(int socket_family); +connection_t *connection_new(int type, int socket_family); + +void connection_link_connections(connection_t *conn_a, connection_t *conn_b); +void connection_free(connection_t *conn); +void connection_free_all(void); +void connection_about_to_close_connection(connection_t *conn); +void connection_close_immediate(connection_t *conn); +void _connection_mark_for_close(connection_t *conn,int line, const char *file); + +#define connection_mark_for_close(c) \ + _connection_mark_for_close((c), __LINE__, _SHORT_FILE_) + +void connection_expire_held_open(void); + +int connection_connect(connection_t *conn, const char *address, + const tor_addr_t *addr, + uint16_t port, int *socket_error); + +int connection_proxy_connect(connection_t *conn, int type); +int connection_read_proxy_handshake(connection_t *conn); + +int retry_all_listeners(smartlist_t *replaced_conns, + smartlist_t *new_conns); + +ssize_t connection_bucket_write_limit(connection_t *conn, time_t now); +int global_write_bucket_low(connection_t *conn, size_t attempt, int priority); +void connection_bucket_init(void); +void connection_bucket_refill(int seconds_elapsed, time_t now); + +int connection_handle_read(connection_t *conn); + +int connection_fetch_from_buf(char *string, size_t len, connection_t *conn); + +int connection_wants_to_flush(connection_t *conn); +int connection_outbuf_too_full(connection_t *conn); +int connection_handle_write(connection_t *conn, int force); +void _connection_write_to_buf_impl(const char *string, size_t len, + connection_t *conn, int zlib); +static void connection_write_to_buf(const char *string, size_t len, + connection_t *conn); +static void connection_write_to_buf_zlib(const char *string, size_t len, + dir_connection_t *conn, int done); +static INLINE void +connection_write_to_buf(const char *string, size_t len, connection_t *conn) +{ + _connection_write_to_buf_impl(string, len, conn, 0); +} +static INLINE void +connection_write_to_buf_zlib(const char *string, size_t len, + dir_connection_t *conn, int done) +{ + _connection_write_to_buf_impl(string, len, TO_CONN(conn), done ? -1 : 1); +} + +connection_t *connection_get_by_global_id(uint64_t id); + +connection_t *connection_get_by_type(int type); +connection_t *connection_get_by_type_purpose(int type, int purpose); +connection_t *connection_get_by_type_addr_port_purpose(int type, + const tor_addr_t *addr, + uint16_t port, int purpose); +connection_t *connection_get_by_type_state(int type, int state); +connection_t *connection_get_by_type_state_rendquery(int type, int state, + const char *rendquery); + +#define connection_speaks_cells(conn) ((conn)->type == CONN_TYPE_OR) +int connection_is_listener(connection_t *conn); +int connection_state_is_open(connection_t *conn); +int connection_state_is_connecting(connection_t *conn); + +char *alloc_http_authenticator(const char *authenticator); + +void assert_connection_ok(connection_t *conn, time_t now); +int connection_or_nonopen_was_started_here(or_connection_t *conn); +void connection_dump_buffer_mem_stats(int severity); +void remove_file_if_very_old(const char *fname, time_t now); + +#endif + diff --git a/src/or/connection_edge.c b/src/or/connection_edge.c index 0ec2002afe..7828f16386 100644 --- a/src/or/connection_edge.c +++ b/src/or/connection_edge.c @@ -10,6 +10,28 @@ **/ #include "or.h" +#include "buffers.h" +#include "circuitlist.h" +#include "circuituse.h" +#include "config.h" +#include "connection.h" +#include "connection_edge.h" +#include "connection_or.h" +#include "control.h" +#include "dns.h" +#include "dnsserv.h" +#include "dirserv.h" +#include "hibernate.h" +#include "main.h" +#include "policies.h" +#include "reasons.h" +#include "relay.h" +#include "rendclient.h" +#include "rendcommon.h" +#include "rendservice.h" +#include "rephist.h" +#include "router.h" +#include "routerlist.h" #ifdef HAVE_LINUX_TYPES_H #include <linux/types.h> @@ -125,7 +147,7 @@ connection_edge_process_inbuf(edge_connection_t *conn, int package_partial) return 0; case AP_CONN_STATE_OPEN: case EXIT_CONN_STATE_OPEN: - if (connection_edge_package_raw_inbuf(conn, package_partial) < 0) { + if (connection_edge_package_raw_inbuf(conn, package_partial, NULL) < 0) { /* (We already sent an end cell if possible) */ connection_mark_for_close(TO_CONN(conn)); return -1; @@ -279,6 +301,23 @@ connection_edge_end_errno(edge_connection_t *conn) return connection_edge_end(conn, reason); } +/** We just wrote some data to <b>conn</b>; act appropriately. + * + * (That is, if it's open, consider sending a stream-level sendme cell if we + * have just flushed enough.) + */ +int +connection_edge_flushed_some(edge_connection_t *conn) +{ + switch (conn->_base.state) { + case AP_CONN_STATE_OPEN: + case EXIT_CONN_STATE_OPEN: + connection_edge_consider_sending_sendme(conn); + break; + } + return 0; +} + /** Connection <b>conn</b> has finished writing and has no bytes left on * its outbuf. * @@ -330,11 +369,13 @@ connection_edge_finished_connecting(edge_connection_t *edge_conn) tor_assert(conn->state == EXIT_CONN_STATE_CONNECTING); log_info(LD_EXIT,"Exit connection to %s:%u (%s) established.", - escaped_safe_str(conn->address),conn->port, + escaped_safe_str(conn->address), conn->port, safe_str(fmt_addr(&conn->addr))); + rep_hist_note_exit_stream_opened(conn->port); + conn->state = EXIT_CONN_STATE_OPEN; - connection_watch_events(conn, EV_READ); /* stop writing, continue reading */ + connection_watch_events(conn, READ_EVENT); /* stop writing, keep reading */ if (connection_wants_to_flush(conn)) /* in case there are any queued relay * cells */ connection_start_writing(conn); @@ -375,13 +416,16 @@ connection_edge_finished_connecting(edge_connection_t *edge_conn) static int compute_retry_timeout(edge_connection_t *conn) { + int timeout = get_options()->CircuitStreamTimeout; + if (timeout) /* if our config options override the default, use them */ + return timeout; if (conn->num_socks_retries < 2) /* try 0 and try 1 */ return 10; return 15; } /** Find all general-purpose AP streams waiting for a response that sent their - * begin/resolve cell >=15 seconds ago. Detach from their current circuit, and + * begin/resolve cell too long ago. Detach from their current circuit, and * mark their current circuit as unsuitable for new streams. Then call * connection_ap_handshake_attach_circuit() to attach to a new circuit (if * available) or launch a new one. @@ -423,7 +467,8 @@ connection_ap_expire_beginning(void) log_fn(severity, LD_APP, "Tried for %d seconds to get a connection to %s:%d. " "Giving up. (%s)", - seconds_since_born, safe_str(conn->socks_request->address), + seconds_since_born, + safe_str_client(conn->socks_request->address), conn->socks_request->port, conn_state_to_string(CONN_TYPE_AP, conn->_base.state)); connection_mark_unattached_ap(conn, END_STREAM_REASON_TIMEOUT); @@ -440,7 +485,7 @@ connection_ap_expire_beginning(void) circ = circuit_get_by_edge_conn(conn); if (!circ) { /* it's vanished? */ log_info(LD_APP,"Conn is waiting (address %s), but lost its circ.", - safe_str(conn->socks_request->address)); + safe_str_client(conn->socks_request->address)); connection_mark_unattached_ap(conn, END_STREAM_REASON_TIMEOUT); continue; } @@ -450,7 +495,7 @@ connection_ap_expire_beginning(void) "Rend stream is %d seconds late. Giving up on address" " '%s.onion'.", seconds_idle, - safe_str(conn->socks_request->address)); + safe_str_client(conn->socks_request->address)); connection_edge_end(conn, END_STREAM_REASON_TIMEOUT); connection_mark_unattached_ap(conn, END_STREAM_REASON_TIMEOUT); } @@ -460,7 +505,8 @@ connection_ap_expire_beginning(void) log_fn(cutoff < 15 ? LOG_INFO : severity, LD_APP, "We tried for %d seconds to connect to '%s' using exit '%s'." " Retrying on a new circuit.", - seconds_idle, safe_str(conn->socks_request->address), + seconds_idle, + safe_str_client(conn->socks_request->address), conn->cpath_layer ? conn->cpath_layer->extend_info->nickname : "*unnamed*"); /* send an end down the circuit */ @@ -471,6 +517,7 @@ connection_ap_expire_beginning(void) /* kludge to make us not try this circuit again, yet to allow * current streams on it to survive if they can: make it * unattractive to use for new streams */ + /* XXXX023 this is a kludgy way to do this. */ tor_assert(circ->timestamp_dirty); circ->timestamp_dirty -= options->MaxCircuitDirtiness; /* give our stream another 'cutoff' seconds to try */ @@ -511,7 +558,7 @@ connection_ap_attach_pending(void) /** Tell any AP streams that are waiting for a one-hop tunnel to * <b>failed_digest</b> that they are going to fail. */ -/* XXX022 We should get rid of this function, and instead attach +/* XXX023 We should get rid of this function, and instead attach * one-hop streams to circ->p_streams so they get marked in * circuit_mark_for_close like normal p_streams. */ void @@ -577,8 +624,8 @@ circuit_discard_optional_exit_enclaves(extend_info_t *info) tor_assert(edge_conn->socks_request); if (edge_conn->chosen_exit_optional) { log_info(LD_APP, "Giving up on enclave exit '%s' for destination %s.", - safe_str(edge_conn->chosen_exit_name), - escaped_safe_str(edge_conn->socks_request->address)); + safe_str_client(edge_conn->chosen_exit_name), + escaped_safe_str_client(edge_conn->socks_request->address)); edge_conn->chosen_exit_optional = 0; tor_free(edge_conn->chosen_exit_name); /* clears it */ /* if this port is dangerous, warn or reject it now that we don't @@ -683,7 +730,11 @@ addressmap_init(void) static void addressmap_ent_free(void *_ent) { - addressmap_entry_t *ent = _ent; + addressmap_entry_t *ent; + if (!_ent) + return; + + ent = _ent; tor_free(ent->new_address); tor_free(ent); } @@ -692,7 +743,11 @@ addressmap_ent_free(void *_ent) static void addressmap_virtaddress_ent_free(void *_ent) { - virtaddress_entry_t *ent = _ent; + virtaddress_entry_t *ent; + if (!_ent) + return; + + ent = _ent; tor_free(ent->ipv4_address); tor_free(ent->hostname_address); tor_free(ent); @@ -744,7 +799,8 @@ clear_trackexithost_mappings(const char *exitname) tor_strlower(suffix); STRMAP_FOREACH_MODIFY(addressmap, address, addressmap_entry_t *, ent) { - if (ent->source == ADDRMAPSRC_TRACKEXIT && !strcmpend(address, suffix)) { + if (ent->source == ADDRMAPSRC_TRACKEXIT && + !strcmpend(ent->new_address, suffix)) { addressmap_ent_remove(address, ent); MAP_DEL_CURRENT(address); } @@ -753,6 +809,101 @@ clear_trackexithost_mappings(const char *exitname) tor_free(suffix); } +/** Remove all TRACKEXIT mappings from the addressmap for which the target + * host is unknown or no longer allowed, or for which the source address + * is no longer in trackexithosts. */ +void +addressmap_clear_excluded_trackexithosts(or_options_t *options) +{ + const routerset_t *allow_nodes = options->ExitNodes; + const routerset_t *exclude_nodes = options->_ExcludeExitNodesUnion; + + if (!addressmap) + return; + if (routerset_is_empty(allow_nodes)) + allow_nodes = NULL; + if (allow_nodes == NULL && routerset_is_empty(exclude_nodes)) + return; + + STRMAP_FOREACH_MODIFY(addressmap, address, addressmap_entry_t *, ent) { + size_t len; + const char *target = ent->new_address, *dot; + char *nodename; + routerinfo_t *ri; /* XXX023 Use node_t. */ + + if (strcmpend(target, ".exit")) { + /* Not a .exit mapping */ + continue; + } else if (ent->source != ADDRMAPSRC_TRACKEXIT) { + /* Not a trackexit mapping. */ + continue; + } + len = strlen(target); + if (len < 6) + continue; /* malformed. */ + dot = target + len - 6; /* dot now points to just before .exit */ + dot = strrchr(dot, '.'); /* dot now points to the . before .exit or NULL */ + if (!dot) { + nodename = tor_strndup(target, len-5); + } else { + nodename = tor_strndup(dot+1, strlen(dot+1)-5); + } + ri = router_get_by_nickname(nodename, 0); + tor_free(nodename); + if (!ri || + (allow_nodes && !routerset_contains_router(allow_nodes, ri)) || + routerset_contains_router(exclude_nodes, ri) || + !hostname_in_track_host_exits(options, address)) { + /* We don't know this one, or we want to be rid of it. */ + addressmap_ent_remove(address, ent); + MAP_DEL_CURRENT(address); + } + } STRMAP_FOREACH_END; +} + +/** Remove all AUTOMAP mappings from the addressmap for which the + * source address no longer matches AutomapHostsSuffixes, which is + * no longer allowed by AutomapHostsOnResolve, or for which the + * target address is no longer in the virtual network. */ +void +addressmap_clear_invalid_automaps(or_options_t *options) +{ + int clear_all = !options->AutomapHostsOnResolve; + const smartlist_t *suffixes = options->AutomapHostsSuffixes; + + if (!addressmap) + return; + + if (!suffixes) + clear_all = 1; /* This should be impossible, but let's be sure. */ + + STRMAP_FOREACH_MODIFY(addressmap, src_address, addressmap_entry_t *, ent) { + int remove = clear_all; + if (ent->source != ADDRMAPSRC_AUTOMAP) + continue; /* not an automap mapping. */ + + if (!remove) { + int suffix_found = 0; + SMARTLIST_FOREACH(suffixes, const char *, suffix, { + if (!strcasecmpend(src_address, suffix)) { + suffix_found = 1; + break; + } + }); + if (!suffix_found) + remove = 1; + } + + if (!remove && ! address_is_in_virtual_range(ent->new_address)) + remove = 1; + + if (remove) { + addressmap_ent_remove(src_address, ent); + MAP_DEL_CURRENT(src_address); + } + } STRMAP_FOREACH_END; +} + /** Remove all entries from the addressmap that were set via the * configuration file or the command line. */ void @@ -782,14 +933,11 @@ addressmap_clean(time_t now) void addressmap_free_all(void) { - if (addressmap) { - strmap_free(addressmap, addressmap_ent_free); - addressmap = NULL; - } - if (virtaddress_reversemap) { - strmap_free(virtaddress_reversemap, addressmap_virtaddress_ent_free); - virtaddress_reversemap = NULL; - } + strmap_free(addressmap, addressmap_ent_free); + addressmap = NULL; + + strmap_free(virtaddress_reversemap, addressmap_virtaddress_ent_free); + virtaddress_reversemap = NULL; } /** Look at address, and rewrite it until it doesn't want any @@ -816,9 +964,9 @@ addressmap_rewrite(char *address, size_t maxlen, time_t *expires_out) return (rewrites > 0); /* done, no rewrite needed */ } - cp = tor_strdup(escaped_safe_str(ent->new_address)); + cp = tor_strdup(escaped_safe_str_client(ent->new_address)); log_info(LD_APP, "Addressmap: rewriting %s to %s", - escaped_safe_str(address), cp); + escaped_safe_str_client(address), cp); if (ent->expires > 1 && ent->expires < expires) expires = ent->expires; tor_free(cp); @@ -826,7 +974,7 @@ addressmap_rewrite(char *address, size_t maxlen, time_t *expires_out) } log_warn(LD_CONFIG, "Loop detected: we've rewritten %s 16 times! Using it as-is.", - escaped_safe_str(address)); + escaped_safe_str_client(address)); /* it's fine to rewrite a rewrite, but don't loop forever */ if (expires_out) *expires_out = TIME_MAX; @@ -848,9 +996,9 @@ addressmap_rewrite_reverse(char *address, size_t maxlen, time_t *expires_out) tor_snprintf(s, len, "REVERSE[%s]", address); ent = strmap_get(addressmap, s); if (ent) { - cp = tor_strdup(escaped_safe_str(ent->new_address)); + cp = tor_strdup(escaped_safe_str_client(ent->new_address)); log_info(LD_APP, "Rewrote reverse lookup %s -> %s", - escaped_safe_str(s), cp); + escaped_safe_str_client(s), cp); tor_free(cp); strlcpy(address, ent->new_address, maxlen); r = 1; @@ -912,7 +1060,9 @@ addressmap_register(const char *address, char *new_address, time_t expires, if (expires > 1) { log_info(LD_APP,"Temporary addressmap ('%s' to '%s') not performed, " "since it's already mapped to '%s'", - safe_str(address), safe_str(new_address), safe_str(ent->new_address)); + safe_str_client(address), + safe_str_client(new_address), + safe_str_client(ent->new_address)); tor_free(new_address); return; } @@ -931,7 +1081,8 @@ addressmap_register(const char *address, char *new_address, time_t expires, ent->source = source; log_info(LD_CONFIG, "Addressmap: (re)mapped '%s' to '%s'", - safe_str(address), safe_str(ent->new_address)); + safe_str_client(address), + safe_str_client(ent->new_address)); control_event_address_mapped(address, ent->new_address, expires, NULL); } @@ -951,7 +1102,8 @@ client_dns_incr_failures(const char *address) if (ent->num_resolve_failures < SHORT_MAX) ++ent->num_resolve_failures; /* don't overflow */ log_info(LD_APP, "Address %s now has %d resolve failures.", - safe_str(address), ent->num_resolve_failures); + safe_str_client(address), + ent->num_resolve_failures); return ent->num_resolve_failures; } @@ -1247,8 +1399,10 @@ addressmap_register_virtual_address(int type, char *new_address) log_warn(LD_BUG, "Internal confusion: I thought that '%s' was mapped to by " "'%s', but '%s' really maps to '%s'. This is a harmless bug.", - safe_str(new_address), safe_str(*addrp), safe_str(*addrp), - ent?safe_str(ent->new_address):"(nothing)"); + safe_str_client(new_address), + safe_str_client(*addrp), + safe_str_client(*addrp), + ent?safe_str_client(ent->new_address):"(nothing)"); } tor_free(*addrp); @@ -1261,7 +1415,7 @@ addressmap_register_virtual_address(int type, char *new_address) log_info(LD_APP, "Registering map from %s to %s", *addrp, new_address); if (vent_needs_to_be_added) strmap_set(virtaddress_reversemap, new_address, vent); - addressmap_register(*addrp, new_address, 2, ADDRMAPSRC_CONTROLLER); + addressmap_register(*addrp, new_address, 2, ADDRMAPSRC_AUTOMAP); #if 0 { @@ -1276,7 +1430,8 @@ addressmap_register_virtual_address(int type, char *new_address) (type == RESOLVED_TYPE_IPV4) ? vent->ipv4_address : vent->hostname_address)); log_info(LD_APP, "Map from %s to %s okay.", - safe_str(*addrp),safe_str(new_address)); + safe_str_client(*addrp), + safe_str_client(new_address)); } #endif @@ -1391,6 +1546,26 @@ consider_plaintext_ports(edge_connection_t *conn, uint16_t port) * different one? */ #define TRACKHOSTEXITS_RETRIES 5 +/** Call connection_ap_handshake_rewrite_and_attach() unless a controller + * asked us to leave streams unattached. Return 0 in that case. + * + * See connection_ap_handshake_rewrite_and_attach()'s + * documentation for arguments and return value. + */ +int +connection_ap_rewrite_and_attach_if_allowed(edge_connection_t *conn, + origin_circuit_t *circ, + crypt_path_t *cpath) +{ + or_options_t *options = get_options(); + + if (options->LeaveStreamsUnattached) { + conn->_base.state = AP_CONN_STATE_CONTROLLER_WAIT; + return 0; + } + return connection_ap_handshake_rewrite_and_attach(conn, circ, cpath); +} + /** Connection <b>conn</b> just finished its socks handshake, or the * controller asked us to take care of it. If <b>circ</b> is defined, * then that's where we'll want to attach it. Otherwise we have to @@ -1415,16 +1590,20 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, hostname_type_t addresstype; or_options_t *options = get_options(); struct in_addr addr_tmp; + /* We set this to true if this is an address we should automatically + * remap to a local address in VirtualAddrNetwork */ int automap = 0; char orig_address[MAX_SOCKS_ADDR_LEN]; time_t map_expires = TIME_MAX; + /* This will be set to true iff the address starts out as a non-.exit + address, and we remap it to one because of an entry in the addressmap. */ int remapped_to_exit = 0; time_t now = time(NULL); tor_strlower(socks->address); /* normalize it */ strlcpy(orig_address, socks->address, sizeof(orig_address)); log_debug(LD_APP,"Client asked for %s:%d", - safe_str(socks->address), + safe_str_client(socks->address), socks->port); if (socks->command == SOCKS_COMMAND_RESOLVE && @@ -1446,7 +1625,8 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, return -1; } log_info(LD_APP, "Automapping %s to %s", - escaped_safe_str(socks->address), safe_str(new_addr)); + escaped_safe_str_client(socks->address), + safe_str_client(new_addr)); strlcpy(socks->address, new_addr, sizeof(socks->address)); } } @@ -1503,7 +1683,7 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, * information. */ log_warn(LD_APP,"Missing mapping for virtual address '%s'. Refusing.", - socks->address); /* don't safe_str() this yet. */ + safe_str_client(socks->address)); connection_mark_unattached_ap(conn, END_STREAM_REASON_INTERNAL); return -1; } @@ -1511,11 +1691,12 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, /* Parse the address provided by SOCKS. Modify it in-place if it * specifies a hidden-service (.onion) or particular exit node (.exit). */ - addresstype = parse_extended_hostname(socks->address); + addresstype = parse_extended_hostname(socks->address, + remapped_to_exit || options->AllowDotExit); if (addresstype == BAD_HOSTNAME) { log_warn(LD_APP, "Invalid onion hostname %s; rejecting", - safe_str(socks->address)); + safe_str_client(socks->address)); control_event_client_status(LOG_WARN, "SOCKS_BAD_HOSTNAME HOSTNAME=%s", escaped(socks->address)); connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL); @@ -1524,38 +1705,61 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, if (addresstype == EXIT_HOSTNAME) { /* foo.exit -- modify conn->chosen_exit_node to specify the exit - * node, and conn->address to hold only the address portion.*/ + * node, and conn->address to hold only the address portion. */ char *s = strrchr(socks->address,'.'); + + /* If StrictNodes is not set, then .exit overrides ExcludeNodes. */ + routerset_t *excludeset = options->StrictNodes ? + options->_ExcludeExitNodesUnion : options->ExcludeExitNodes; + /*XXX023 make this a node_t. */ + routerinfo_t *router; + tor_assert(!automap); if (s) { + /* The address was of the form "(stuff).(name).exit */ if (s[1] != '\0') { conn->chosen_exit_name = tor_strdup(s+1); + router = router_get_by_nickname(conn->chosen_exit_name, 1); if (remapped_to_exit) /* 5 tries before it expires the addressmap */ conn->chosen_exit_retries = TRACKHOSTEXITS_RETRIES; *s = 0; } else { + /* Oops, the address was (stuff)..exit. That's not okay. */ log_warn(LD_APP,"Malformed exit address '%s.exit'. Refusing.", - safe_str(socks->address)); + safe_str_client(socks->address)); control_event_client_status(LOG_WARN, "SOCKS_BAD_HOSTNAME HOSTNAME=%s", escaped(socks->address)); connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL); return -1; } } else { - routerinfo_t *r; + /* It looks like they just asked for "foo.exit". */ conn->chosen_exit_name = tor_strdup(socks->address); - r = router_get_by_nickname(conn->chosen_exit_name, 1); - *socks->address = 0; - if (r) { - strlcpy(socks->address, r->address, sizeof(socks->address)); - } else { - log_warn(LD_APP, - "Unrecognized server in exit address '%s.exit'. Refusing.", - safe_str(socks->address)); - connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL); - return -1; + router = router_get_by_nickname(conn->chosen_exit_name, 1); + if (router) { + *socks->address = 0; + strlcpy(socks->address, router->address, sizeof(socks->address)); } } + /* Now make sure that the chosen exit exists... */ + if (!router) { + log_warn(LD_APP, + "Unrecognized relay in exit address '%s.exit'. Refusing.", + safe_str_client(socks->address)); + connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL); + return -1; + } + /* ...and make sure that it isn't excluded. */ + if (routerset_contains_router(excludeset, router)) { + log_warn(LD_APP, + "Excluded relay in exit address '%s.exit'. Refusing.", + safe_str_client(socks->address)); + connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL); + return -1; + } + /* XXXX022-1090 Should we also allow foo.bar.exit if ExitNodes is set and + Bar is not listed in it? I say yes, but our revised manpage branch + implies no. */ } if (addresstype != ONION_HOSTNAME) { @@ -1565,7 +1769,7 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, escaped(socks->address)); log_warn(LD_APP, "Destination '%s' seems to be an invalid hostname. Failing.", - safe_str(socks->address)); + safe_str_client(socks->address)); connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL); return -1; } @@ -1574,18 +1778,6 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, uint32_t answer; struct in_addr in; /* Reply to resolves immediately if we can. */ - if (strlen(socks->address) > RELAY_PAYLOAD_SIZE) { - log_warn(LD_APP,"Address to be resolved is too large. Failing."); - control_event_client_status(LOG_WARN, "SOCKS_BAD_HOSTNAME HOSTNAME=%s", - escaped(socks->address)); - connection_ap_handshake_socks_resolved(conn, - RESOLVED_TYPE_ERROR_TRANSIENT, - 0,NULL,-1,TIME_MAX); - connection_mark_unattached_ap(conn, - END_STREAM_REASON_SOCKSPROTOCOL | - END_STREAM_REASON_FLAG_ALREADY_SOCKS_REPLIED); - return -1; - } if (tor_inet_aton(socks->address, &in)) { /* see if it's an IP already */ /* leave it in network order */ answer = in.s_addr; @@ -1608,6 +1800,28 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL); return -1; } + if (options->ClientRejectInternalAddresses && + !conn->use_begindir && !conn->chosen_exit_name && !circ) { + tor_addr_t addr; + if (tor_addr_from_str(&addr, socks->address) >= 0 && + tor_addr_is_internal(&addr, 0)) { + /* If this is an explicit private address with no chosen exit node, + * then we really don't want to try to connect to it. That's + * probably an error. */ + if (conn->is_transparent_ap) { + log_warn(LD_NET, + "Rejecting request for anonymous connection to private " + "address %s on a TransPort or NATDPort. Possible loop " + "in your NAT rules?", safe_str_client(socks->address)); + } else { + log_warn(LD_NET, + "Rejecting SOCKS request for anonymous connection to " + "private address %s", safe_str_client(socks->address)); + } + connection_mark_unattached_ap(conn, END_STREAM_REASON_PRIVATE_ADDR); + return -1; + } + } if (!conn->use_begindir && !conn->chosen_exit_name && !circ) { /* see if we can find a suitable enclave exit */ @@ -1616,7 +1830,7 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, if (r) { log_info(LD_APP, "Redirecting address %s to exit at enclave router %s", - safe_str(socks->address), r->nickname); + safe_str_client(socks->address), r->nickname); /* use the hex digest, not nickname, in case there are two routers with this nickname */ conn->chosen_exit_name = @@ -1680,12 +1894,12 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, strlcpy(conn->rend_data->onion_address, socks->address, sizeof(conn->rend_data->onion_address)); log_info(LD_REND,"Got a hidden service request for ID '%s'", - safe_str(conn->rend_data->onion_address)); + safe_str_client(conn->rend_data->onion_address)); /* see if we already have it cached */ r = rend_cache_lookup_entry(conn->rend_data->onion_address, -1, &entry); if (r<0) { log_warn(LD_BUG,"Invalid service name '%s'", - safe_str(conn->rend_data->onion_address)); + safe_str_client(conn->rend_data->onion_address)); connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL); return -1; } @@ -1707,32 +1921,15 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, if (r==0) { conn->_base.state = AP_CONN_STATE_RENDDESC_WAIT; log_info(LD_REND, "Unknown descriptor %s. Fetching.", - safe_str(conn->rend_data->onion_address)); - /* Fetch both, v0 and v2 rend descriptors in parallel. Use whichever - * arrives first. Exception: When using client authorization, only - * fetch v2 descriptors.*/ + safe_str_client(conn->rend_data->onion_address)); rend_client_refetch_v2_renddesc(conn->rend_data); - if (conn->rend_data->auth_type == REND_NO_AUTH) - rend_client_refetch_renddesc(conn->rend_data->onion_address); } else { /* r > 0 */ - if (now - entry->received < NUM_SECONDS_BEFORE_HS_REFETCH) { - conn->_base.state = AP_CONN_STATE_CIRCUIT_WAIT; - log_info(LD_REND, "Descriptor is here and fresh enough. Great."); - if (connection_ap_handshake_attach_circuit(conn) < 0) { - if (!conn->_base.marked_for_close) - connection_mark_unattached_ap(conn, END_STREAM_REASON_CANT_ATTACH); - return -1; - } - } else { - conn->_base.state = AP_CONN_STATE_RENDDESC_WAIT; - log_info(LD_REND, "Stale descriptor %s. Re-fetching.", - safe_str(conn->rend_data->onion_address)); - /* Fetch both, v0 and v2 rend descriptors in parallel. Use whichever - * arrives first. Exception: When using client authorization, only - * fetch v2 descriptors.*/ - rend_client_refetch_v2_renddesc(conn->rend_data); - if (conn->rend_data->auth_type == REND_NO_AUTH) - rend_client_refetch_renddesc(conn->rend_data->onion_address); + conn->_base.state = AP_CONN_STATE_CIRCUIT_WAIT; + log_info(LD_REND, "Descriptor is here. Great."); + if (connection_ap_handshake_attach_circuit(conn) < 0) { + if (!conn->_base.marked_for_close) + connection_mark_unattached_ap(conn, END_STREAM_REASON_CANT_ATTACH); + return -1; } } return 0; @@ -1920,24 +2117,12 @@ connection_ap_handshake_process_socks(edge_connection_t *conn) return -1; } /* else socks handshake is done, continue processing */ - if (hostname_is_noconnect_address(socks->address)) - { - control_event_stream_status(conn, STREAM_EVENT_NEW, 0); - control_event_stream_status(conn, STREAM_EVENT_CLOSED, 0); - connection_mark_unattached_ap(conn, END_STREAM_REASON_DONE); - return -1; - } - if (SOCKS_COMMAND_IS_CONNECT(socks->command)) control_event_stream_status(conn, STREAM_EVENT_NEW, 0); else control_event_stream_status(conn, STREAM_EVENT_NEW_RESOLVE, 0); - if (options->LeaveStreamsUnattached) { - conn->_base.state = AP_CONN_STATE_CONTROLLER_WAIT; - return 0; - } - return connection_ap_handshake_rewrite_and_attach(conn, NULL, NULL); + return connection_ap_rewrite_and_attach_if_allowed(conn, NULL, NULL); } /** connection_init_accepted_conn() found a new trans AP conn. @@ -1951,7 +2136,6 @@ int connection_ap_process_transparent(edge_connection_t *conn) { socks_request_t *socks; - or_options_t *options = get_options(); tor_assert(conn); tor_assert(conn->_base.type == CONN_TYPE_AP); @@ -1975,11 +2159,7 @@ connection_ap_process_transparent(edge_connection_t *conn) control_event_stream_status(conn, STREAM_EVENT_NEW, 0); - if (options->LeaveStreamsUnattached) { - conn->_base.state = AP_CONN_STATE_CONTROLLER_WAIT; - return 0; - } - return connection_ap_handshake_rewrite_and_attach(conn, NULL, NULL); + return connection_ap_rewrite_and_attach_if_allowed(conn, NULL, NULL); } /** connection_edge_process_inbuf() found a conn in state natd_wait. See if @@ -2000,7 +2180,6 @@ connection_ap_process_natd(edge_connection_t *conn) size_t tlen = 30; int err, port_ok; socks_request_t *socks; - or_options_t *options = get_options(); tor_assert(conn); tor_assert(conn->_base.type == CONN_TYPE_AP); @@ -2016,13 +2195,13 @@ connection_ap_process_natd(edge_connection_t *conn) if (err == 0) return 0; if (err < 0) { - log_warn(LD_APP,"Natd handshake failed (DEST too long). Closing"); + log_warn(LD_APP,"NATD handshake failed (DEST too long). Closing"); connection_mark_unattached_ap(conn, END_STREAM_REASON_INVALID_NATD_DEST); return -1; } if (strcmpstart(tmp_buf, "[DEST ")) { - log_warn(LD_APP,"Natd handshake was ill-formed; closing. The client " + log_warn(LD_APP,"NATD handshake was ill-formed; closing. The client " "said: %s", escaped(tmp_buf)); connection_mark_unattached_ap(conn, END_STREAM_REASON_INVALID_NATD_DEST); @@ -2031,7 +2210,7 @@ connection_ap_process_natd(edge_connection_t *conn) daddr = tbuf = &tmp_buf[0] + 6; /* after end of "[DEST " */ if (!(tbuf = strchr(tbuf, ' '))) { - log_warn(LD_APP,"Natd handshake was ill-formed; closing. The client " + log_warn(LD_APP,"NATD handshake was ill-formed; closing. The client " "said: %s", escaped(tmp_buf)); connection_mark_unattached_ap(conn, END_STREAM_REASON_INVALID_NATD_DEST); @@ -2045,7 +2224,7 @@ connection_ap_process_natd(edge_connection_t *conn) socks->port = (uint16_t) tor_parse_long(tbuf, 10, 1, 65535, &port_ok, &daddr); if (!port_ok) { - log_warn(LD_APP,"Natd handshake failed; port %s is ill-formed or out " + log_warn(LD_APP,"NATD handshake failed; port %s is ill-formed or out " "of range.", escaped(tbuf)); connection_mark_unattached_ap(conn, END_STREAM_REASON_INVALID_NATD_DEST); return -1; @@ -2056,13 +2235,9 @@ connection_ap_process_natd(edge_connection_t *conn) control_event_stream_status(conn, STREAM_EVENT_NEW, 0); - if (options->LeaveStreamsUnattached) { - conn->_base.state = AP_CONN_STATE_CONTROLLER_WAIT; - return 0; - } conn->_base.state = AP_CONN_STATE_CIRCUIT_WAIT; - return connection_ap_handshake_rewrite_and_attach(conn, NULL, NULL); + return connection_ap_rewrite_and_attach_if_allowed(conn, NULL, NULL); } /** Iterate over the two bytes of stream_id until we get one that is not @@ -2075,7 +2250,7 @@ get_unique_stream_id_by_circ(origin_circuit_t *circ) streamid_t test_stream_id; uint32_t attempts=0; -again: + again: test_stream_id = circ->next_stream_id++; if (++attempts > 1<<16) { /* Make sure we don't loop forever if all stream_id's are used. */ @@ -2112,8 +2287,14 @@ connection_ap_handshake_send_begin(edge_connection_t *ap_conn) ap_conn->stream_id = get_unique_stream_id_by_circ(circ); if (ap_conn->stream_id==0) { + /* XXXX023 Instead of closing this stream, we should make it get + * retried on another circuit. */ connection_mark_unattached_ap(ap_conn, END_STREAM_REASON_INTERNAL); - circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_RESOURCELIMIT); + + /* Mark this circuit "unusable for new streams". */ + /* XXXX023 this is a kludgy way to do this. */ + tor_assert(circ->_base.timestamp_dirty); + circ->_base.timestamp_dirty -= get_options()->MaxCircuitDirtiness; return -1; } @@ -2171,9 +2352,14 @@ connection_ap_handshake_send_resolve(edge_connection_t *ap_conn) ap_conn->stream_id = get_unique_stream_id_by_circ(circ); if (ap_conn->stream_id==0) { + /* XXXX023 Instead of closing this stream, we should make it get + * retried on another circuit. */ connection_mark_unattached_ap(ap_conn, END_STREAM_REASON_INTERNAL); - /*XXXX022 _close_ the circuit because it's full? That sounds dumb. */ - circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_RESOURCELIMIT); + + /* Mark this circuit "unusable for new streams". */ + /* XXXX023 this is a kludgy way to do this. */ + tor_assert(circ->_base.timestamp_dirty); + circ->_base.timestamp_dirty -= get_options()->MaxCircuitDirtiness; return -1; } @@ -2191,7 +2377,7 @@ connection_ap_handshake_send_resolve(edge_connection_t *ap_conn) r = tor_addr_parse_reverse_lookup_name(&addr, a, AF_INET, 1); if (r <= 0) { log_warn(LD_APP, "Rejecting ill-formed reverse lookup of %s", - safe_str(a)); + safe_str_client(a)); connection_mark_unattached_ap(ap_conn, END_STREAM_REASON_INTERNAL); return -1; } @@ -2199,7 +2385,7 @@ connection_ap_handshake_send_resolve(edge_connection_t *ap_conn) r = tor_addr_to_reverse_lookup_name(inaddr_buf, sizeof(inaddr_buf), &addr); if (r < 0) { log_warn(LD_BUG, "Couldn't generate reverse lookup hostname of %s", - safe_str(a)); + safe_str_client(a)); connection_mark_unattached_ap(ap_conn, END_STREAM_REASON_INTERNAL); return -1; } @@ -2209,12 +2395,6 @@ connection_ap_handshake_send_resolve(edge_connection_t *ap_conn) tor_assert(payload_len <= (int)sizeof(inaddr_buf)); } - if (payload_len > RELAY_PAYLOAD_SIZE) { - /* This should be impossible: we don't accept addresses this big. */ - connection_mark_unattached_ap(ap_conn, END_STREAM_REASON_INTERNAL); - return -1; - } - log_debug(LD_APP, "Sending relay cell to begin stream %d.", ap_conn->stream_id); @@ -2246,7 +2426,8 @@ connection_ap_make_link(char *address, uint16_t port, edge_connection_t *conn; log_info(LD_APP,"Making internal %s tunnel to %s:%d ...", - want_onehop ? "direct" : "anonymized" , safe_str(address),port); + want_onehop ? "direct" : "anonymized", + safe_str_client(address), port); conn = edge_connection_new(CONN_TYPE_AP, AF_INET); conn->_base.linked = 1; /* so that we can add it safely below. */ @@ -2336,7 +2517,7 @@ tell_controller_about_resolved_result(edge_connection_t *conn, * certain errors or for values that didn't come via DNS. <b>expires</b> is * a time when the answer expires, or -1 or TIME_MAX if there's a good TTL. **/ -/* XXXX022 the use of the ttl and expires fields is nutty. Let's make this +/* XXXX023 the use of the ttl and expires fields is nutty. Let's make this * interface and those that use it less ugly. */ void connection_ap_handshake_socks_resolved(edge_connection_t *conn, @@ -2512,6 +2693,7 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ) char *address=NULL; uint16_t port; or_circuit_t *or_circ = NULL; + or_options_t *options = get_options(); assert_circuit_ok(circ); if (!CIRCUIT_IS_ORIGIN(circ)) @@ -2526,7 +2708,7 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ) * that we have a stream connected to a circuit, and we don't connect to a * circuit until we have a pending/successful resolve. */ - if (!server_mode(get_options()) && + if (!server_mode(options) && circ->purpose != CIRCUIT_PURPOSE_S_REND_JOINED) { log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "Relay begin cell at non-server. Closing."); @@ -2560,21 +2742,30 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ) tor_free(address); return 0; } - if (or_circ && or_circ->is_first_hop && - !get_options()->AllowSingleHopExits) { + if (or_circ && or_circ->p_conn && !options->AllowSingleHopExits && + (or_circ->is_first_hop || + (!connection_or_digest_is_known_relay( + or_circ->p_conn->identity_digest) && + should_refuse_unknown_exits(options)))) { /* Don't let clients use us as a single-hop proxy, unless the user - * has explicitly allowed that in the config. It attracts attackers + * has explicitly allowed that in the config. It attracts attackers * and users who'd be better off with, well, single-hop proxies. */ log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, - "Attempt to open a stream on first hop of circuit. Closing."); + "Attempt by %s to open a stream %s. Closing.", + safe_str(or_circ->p_conn->_base.address), + or_circ->is_first_hop ? "on first hop of circuit" : + "from unknown relay"); relay_send_end_cell_from_edge(rh.stream_id, circ, - END_STREAM_REASON_TORPROTOCOL, NULL); + or_circ->is_first_hop ? + END_STREAM_REASON_TORPROTOCOL : + END_STREAM_REASON_MISC, + NULL); tor_free(address); return 0; } } else if (rh.command == RELAY_COMMAND_BEGIN_DIR) { - if (!directory_permits_begindir_requests(get_options()) || + if (!directory_permits_begindir_requests(options) || circ->purpose != CIRCUIT_PURPOSE_OR) { relay_send_end_cell_from_edge(rh.stream_id, circ, END_STREAM_REASON_NOTDIRECTORY, NULL); @@ -2600,6 +2791,11 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ) log_debug(LD_EXIT,"Creating new exit connection."); n_stream = edge_connection_new(CONN_TYPE_EXIT, AF_INET); + + /* Remember the tunneled request ID in the new edge connection, so that + * we can measure download times. */ + TO_CONN(n_stream)->dirreq_id = circ->dirreq_id; + n_stream->_base.purpose = EXIT_PURPOSE_CONNECT; n_stream->stream_id = rh.stream_id; @@ -2630,7 +2826,7 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ) log_debug(LD_REND,"Finished assigning addr/port"); n_stream->cpath_layer = origin_circ->cpath->prev; /* link it */ - /* add it into the linked list of n_streams on this circuit */ + /* add it into the linked list of p_streams on this circuit */ n_stream->next_stream = origin_circ->p_streams; n_stream->on_circuit = circ; origin_circ->p_streams = n_stream; @@ -2657,7 +2853,7 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ) if (rh.command == RELAY_COMMAND_BEGIN_DIR) { tor_assert(or_circ); if (or_circ->p_conn && !tor_addr_is_null(&or_circ->p_conn->real_addr)) - tor_addr_assign(&n_stream->_base.addr, &or_circ->p_conn->real_addr); + tor_addr_copy(&n_stream->_base.addr, &or_circ->p_conn->real_addr); return connection_exit_connect_dir(n_stream); } @@ -2750,7 +2946,7 @@ connection_exit_connect(edge_connection_t *edge_conn) if (!connection_edge_is_rendezvous_stream(edge_conn) && router_compare_to_my_exit_policy(edge_conn)) { log_info(LD_EXIT,"%s:%d failed exit policy. Closing.", - escaped_safe_str(conn->address), conn->port); + escaped_safe_str_client(conn->address), conn->port); connection_edge_end(edge_conn, END_STREAM_REASON_EXITPOLICY); circuit_detach_stream(circuit_get_by_edge_conn(edge_conn), edge_conn); connection_free(conn); @@ -2762,17 +2958,17 @@ connection_exit_connect(edge_connection_t *edge_conn) log_debug(LD_EXIT,"about to try connecting"); switch (connection_connect(conn, conn->address, addr, port, &socket_error)) { - case -1: - /* XXX021 use socket_error below rather than trying to piece things - * together from the current errno, which may have been clobbered. */ - connection_edge_end_errno(edge_conn); + case -1: { + int reason = errno_to_stream_end_reason(socket_error); + connection_edge_end(edge_conn, reason); circuit_detach_stream(circuit_get_by_edge_conn(edge_conn), edge_conn); connection_free(conn); return; + } case 0: conn->state = EXIT_CONN_STATE_CONNECTING; - connection_watch_events(conn, EV_WRITE | EV_READ); + connection_watch_events(conn, READ_EVENT | WRITE_EVENT); /* writable indicates finish; * readable/error indicates broken link in windows-land. */ return; @@ -2785,7 +2981,7 @@ connection_exit_connect(edge_connection_t *edge_conn) log_warn(LD_BUG,"newly connected conn had data waiting!"); // connection_start_writing(conn); } - connection_watch_events(conn, EV_READ); + connection_watch_events(conn, READ_EVENT); /* also, deliver a 'connected' cell back through the circuit. */ if (connection_edge_is_rendezvous_stream(edge_conn)) { @@ -2832,13 +3028,17 @@ connection_exit_connect_dir(edge_connection_t *exitconn) dirconn = dir_connection_new(AF_INET); - tor_addr_assign(&dirconn->_base.addr, &exitconn->_base.addr); + tor_addr_copy(&dirconn->_base.addr, &exitconn->_base.addr); dirconn->_base.port = 0; dirconn->_base.address = tor_strdup(exitconn->_base.address); dirconn->_base.type = CONN_TYPE_DIR; dirconn->_base.purpose = DIR_PURPOSE_SERVER; dirconn->_base.state = DIR_CONN_STATE_SERVER_COMMAND_WAIT; + /* Note that the new dir conn belongs to the same tunneled request as + * the edge conn, so that we can measure download times. */ + TO_CONN(dirconn)->dirreq_id = TO_CONN(exitconn)->dirreq_id; + connection_link_connections(TO_CONN(dirconn), TO_CONN(exitconn)); if (connection_add(TO_CONN(exitconn))<0) { @@ -2893,6 +3093,8 @@ connection_edge_is_rendezvous_stream(edge_connection_t *conn) int connection_ap_can_use_exit(edge_connection_t *conn, routerinfo_t *exit) { + or_options_t *options = get_options(); + tor_assert(conn); tor_assert(conn->_base.type == CONN_TYPE_AP); tor_assert(conn->socks_request); @@ -2938,20 +3140,26 @@ connection_ap_can_use_exit(edge_connection_t *conn, routerinfo_t *exit) if (!conn->chosen_exit_name && policy_is_reject_star(exit->exit_policy)) return 0; } + if (options->_ExcludeExitNodesUnion && + routerset_contains_router(options->_ExcludeExitNodesUnion, exit)) { + /* Not a suitable exit. Refuse it. */ + return 0; + } + return 1; } /** If address is of the form "y.onion" with a well-formed handle y: * Put a NUL after y, lower-case it, and return ONION_HOSTNAME. * - * If address is of the form "y.exit": + * If address is of the form "y.exit" and <b>allowdotexit</b> is true: * Put a NUL after y and return EXIT_HOSTNAME. * * Otherwise: * Return NORMAL_HOSTNAME and change nothing. */ hostname_type_t -parse_extended_hostname(char *address) +parse_extended_hostname(char *address, int allowdotexit) { char *s; char query[REND_SERVICE_ID_LEN_BASE32+1]; @@ -2960,8 +3168,16 @@ parse_extended_hostname(char *address) if (!s) return NORMAL_HOSTNAME; /* no dot, thus normal */ if (!strcmp(s+1,"exit")) { - *s = 0; /* NUL-terminate it */ - return EXIT_HOSTNAME; /* .exit */ + if (allowdotexit) { + *s = 0; /* NUL-terminate it */ + return EXIT_HOSTNAME; /* .exit */ + } else { + log_warn(LD_APP, "The \".exit\" notation is disabled in Tor due to " + "security risks. Set AllowDotExit in your torrc to enable " + "it."); + /* FFFF send a controller event too to notify Vidalia users */ + return BAD_HOSTNAME; + } } if (strcmp(s+1,"onion")) return NORMAL_HOSTNAME; /* neither .exit nor .onion, thus normal */ @@ -2974,17 +3190,9 @@ parse_extended_hostname(char *address) if (rend_valid_service_id(query)) { return ONION_HOSTNAME; /* success */ } -failed: + failed: /* otherwise, return to previous state and return 0 */ *s = '.'; return BAD_HOSTNAME; } -/** Check if the address is of the form "y.noconnect" - */ -int -hostname_is_noconnect_address(const char *address) -{ - return ! strcasecmpend(address, ".noconnect"); -} - diff --git a/src/or/connection_edge.h b/src/or/connection_edge.h new file mode 100644 index 0000000000..8ba2fafd08 --- /dev/null +++ b/src/or/connection_edge.h @@ -0,0 +1,102 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file connection_edge.h + * \brief Header file for connection_edge.c. + **/ + +#ifndef _TOR_CONNECTION_EDGE_H +#define _TOR_CONNECTION_EDGE_H + +#define connection_mark_unattached_ap(conn, endreason) \ + _connection_mark_unattached_ap((conn), (endreason), __LINE__, _SHORT_FILE_) + +void _connection_mark_unattached_ap(edge_connection_t *conn, int endreason, + int line, const char *file); +int connection_edge_reached_eof(edge_connection_t *conn); +int connection_edge_process_inbuf(edge_connection_t *conn, + int package_partial); +int connection_edge_destroy(circid_t circ_id, edge_connection_t *conn); +int connection_edge_end(edge_connection_t *conn, uint8_t reason); +int connection_edge_end_errno(edge_connection_t *conn); +int connection_edge_flushed_some(edge_connection_t *conn); +int connection_edge_finished_flushing(edge_connection_t *conn); +int connection_edge_finished_connecting(edge_connection_t *conn); + +int connection_ap_handshake_send_begin(edge_connection_t *ap_conn); +int connection_ap_handshake_send_resolve(edge_connection_t *ap_conn); + +edge_connection_t *connection_ap_make_link(char *address, uint16_t port, + const char *digest, + int use_begindir, int want_onehop); +void connection_ap_handshake_socks_reply(edge_connection_t *conn, char *reply, + size_t replylen, + int endreason); +void connection_ap_handshake_socks_resolved(edge_connection_t *conn, + int answer_type, + size_t answer_len, + const uint8_t *answer, + int ttl, + time_t expires); + +int connection_exit_begin_conn(cell_t *cell, circuit_t *circ); +int connection_exit_begin_resolve(cell_t *cell, or_circuit_t *circ); +void connection_exit_connect(edge_connection_t *conn); +int connection_edge_is_rendezvous_stream(edge_connection_t *conn); +int connection_ap_can_use_exit(edge_connection_t *conn, routerinfo_t *exit); +void connection_ap_expire_beginning(void); +void connection_ap_attach_pending(void); +void connection_ap_fail_onehop(const char *failed_digest, + cpath_build_state_t *build_state); +void circuit_discard_optional_exit_enclaves(extend_info_t *info); +int connection_ap_detach_retriable(edge_connection_t *conn, + origin_circuit_t *circ, + int reason); +int connection_ap_process_transparent(edge_connection_t *conn); + +int address_is_invalid_destination(const char *address, int client); + +void addressmap_init(void); +void addressmap_clear_excluded_trackexithosts(or_options_t *options); +void addressmap_clear_invalid_automaps(or_options_t *options); +void addressmap_clean(time_t now); +void addressmap_clear_configured(void); +void addressmap_clear_transient(void); +void addressmap_free_all(void); +int addressmap_rewrite(char *address, size_t maxlen, time_t *expires_out); +int addressmap_have_mapping(const char *address, int update_timeout); + +void addressmap_register(const char *address, char *new_address, + time_t expires, addressmap_entry_source_t source); +int parse_virtual_addr_network(const char *val, int validate_only, + char **msg); +int client_dns_incr_failures(const char *address); +void client_dns_clear_failures(const char *address); +void client_dns_set_addressmap(const char *address, uint32_t val, + const char *exitname, int ttl); +const char *addressmap_register_virtual_address(int type, char *new_address); +void addressmap_get_mappings(smartlist_t *sl, time_t min_expires, + time_t max_expires, int want_expiry); +int connection_ap_rewrite_and_attach_if_allowed(edge_connection_t *conn, + origin_circuit_t *circ, + crypt_path_t *cpath); +int connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, + origin_circuit_t *circ, + crypt_path_t *cpath); + +/** Possible return values for parse_extended_hostname. */ +typedef enum hostname_type_t { + NORMAL_HOSTNAME, ONION_HOSTNAME, EXIT_HOSTNAME, BAD_HOSTNAME +} hostname_type_t; +hostname_type_t parse_extended_hostname(char *address, int allowdotexit); + +#if defined(HAVE_NET_IF_H) && defined(HAVE_NET_PFVAR_H) +int get_pf_socket(void); +#endif + +#endif + diff --git a/src/or/connection_or.c b/src/or/connection_or.c index d402563fb5..ed174c922e 100644 --- a/src/or/connection_or.c +++ b/src/or/connection_or.c @@ -11,6 +11,22 @@ **/ #include "or.h" +#include "buffers.h" +#include "circuitbuild.h" +#include "command.h" +#include "config.h" +#include "connection.h" +#include "connection_or.h" +#include "control.h" +#include "dirserv.h" +#include "geoip.h" +#include "main.h" +#include "networkstatus.h" +#include "reasons.h" +#include "relay.h" +#include "rephist.h" +#include "router.h" +#include "routerlist.h" static int connection_tls_finish_handshake(or_connection_t *conn); static int connection_or_process_cells_from_inbuf(or_connection_t *conn); @@ -80,10 +96,8 @@ connection_or_clear_identity_map(void) } }); - if (orconn_identity_map) { - digestmap_free(orconn_identity_map, NULL); - orconn_identity_map = NULL; - } + digestmap_free(orconn_identity_map, NULL); + orconn_identity_map = NULL; } /** Change conn->identity_digest to digest, and add conn into @@ -133,7 +147,7 @@ void cell_pack(packed_cell_t *dst, const cell_t *src) { char *dest = dst->body; - *(uint16_t*)dest = htons(src->circ_id); + set_uint16(dest, htons(src->circ_id)); *(uint8_t*)(dest+2) = src->command; memcpy(dest+3, src->payload, CELL_PAYLOAD_SIZE); } @@ -144,7 +158,7 @@ cell_pack(packed_cell_t *dst, const cell_t *src) static void cell_unpack(cell_t *dest, const char *src) { - dest->circ_id = ntohs(*(uint16_t*)(src)); + dest->circ_id = ntohs(get_uint16(src)); dest->command = *(uint8_t*)(src+2); memcpy(dest->payload, src+3, CELL_PAYLOAD_SIZE); } @@ -187,66 +201,6 @@ connection_or_reached_eof(or_connection_t *conn) return 0; } -/** Read conn's inbuf. If the http response from the proxy is all - * here, make sure it's good news, and begin the tls handshake. If - * it's bad news, close the connection and return -1. Else return 0 - * and hope for better luck next time. - */ -static int -connection_or_read_proxy_response(or_connection_t *or_conn) -{ - char *headers; - char *reason=NULL; - int status_code; - time_t date_header; - connection_t *conn = TO_CONN(or_conn); - - switch (fetch_from_buf_http(conn->inbuf, - &headers, MAX_HEADERS_SIZE, - NULL, NULL, 10000, 0)) { - case -1: /* overflow */ - log_warn(LD_PROTOCOL, - "Your https proxy sent back an oversized response. Closing."); - return -1; - case 0: - log_info(LD_OR,"https proxy response not all here yet. Waiting."); - return 0; - /* case 1, fall through */ - } - - if (parse_http_response(headers, &status_code, &date_header, - NULL, &reason) < 0) { - log_warn(LD_OR, - "Unparseable headers from proxy (connecting to '%s'). Closing.", - conn->address); - tor_free(headers); - return -1; - } - if (!reason) reason = tor_strdup("[no reason given]"); - - if (status_code == 200) { - log_info(LD_OR, - "HTTPS connect to '%s' successful! (200 %s) Starting TLS.", - conn->address, escaped(reason)); - tor_free(reason); - if (connection_tls_start_handshake(or_conn, 0) < 0) { - /* TLS handshaking error of some kind. */ - connection_mark_for_close(conn); - - return -1; - } - return 0; - } - /* else, bad news on the status code */ - log_warn(LD_OR, - "The https proxy sent back an unexpected status code %d (%s). " - "Closing.", - status_code, escaped(reason)); - tor_free(reason); - connection_mark_for_close(conn); - return -1; -} - /** Handle any new bytes that have come in on connection <b>conn</b>. * If conn is in 'open' state, hand it to * connection_or_process_cells_from_inbuf() @@ -255,11 +209,24 @@ connection_or_read_proxy_response(or_connection_t *or_conn) int connection_or_process_inbuf(or_connection_t *conn) { + int ret; tor_assert(conn); switch (conn->_base.state) { - case OR_CONN_STATE_PROXY_READING: - return connection_or_read_proxy_response(conn); + case OR_CONN_STATE_PROXY_HANDSHAKING: + ret = connection_read_proxy_handshake(TO_CONN(conn)); + + /* start TLS after handshake completion, or deal with error */ + if (ret == 1) { + tor_assert(TO_CONN(conn)->proxy_state == PROXY_CONNECTED); + if (connection_tls_start_handshake(conn, 0) < 0) + ret = -1; + } + if (ret < 0) { + connection_mark_for_close(TO_CONN(conn)); + } + + return ret; case OR_CONN_STATE_OPEN: case OR_CONN_STATE_OR_HANDSHAKING: return connection_or_process_cells_from_inbuf(conn); @@ -285,8 +252,7 @@ connection_or_flushed_some(or_connection_t *conn) /* If we're under the low water mark, add cells until we're just over the * high water mark. */ if (datalen < OR_CONN_LOWWATER) { - ssize_t n = (OR_CONN_HIGHWATER - datalen + CELL_NETWORK_SIZE-1) - / CELL_NETWORK_SIZE; + ssize_t n = CEIL_DIV(OR_CONN_HIGHWATER - datalen, CELL_NETWORK_SIZE); time_t now = approx_time(); while (conn->active_circuits && n > 0) { int flushed; @@ -312,11 +278,7 @@ connection_or_finished_flushing(or_connection_t *conn) assert_connection_ok(TO_CONN(conn),0); switch (conn->_base.state) { - case OR_CONN_STATE_PROXY_FLUSHING: - log_debug(LD_OR,"finished sending CONNECT to proxy."); - conn->_base.state = OR_CONN_STATE_PROXY_READING; - connection_stop_writing(TO_CONN(conn)); - break; + case OR_CONN_STATE_PROXY_HANDSHAKING: case OR_CONN_STATE_OPEN: case OR_CONN_STATE_OR_HANDSHAKING: connection_stop_writing(TO_CONN(conn)); @@ -334,37 +296,34 @@ connection_or_finished_flushing(or_connection_t *conn) int connection_or_finished_connecting(or_connection_t *or_conn) { + int proxy_type; connection_t *conn; tor_assert(or_conn); conn = TO_CONN(or_conn); tor_assert(conn->state == OR_CONN_STATE_CONNECTING); - log_debug(LD_OR,"OR connect() to router at %s:%u finished.", + log_debug(LD_HANDSHAKE,"OR connect() to router at %s:%u finished.", conn->address,conn->port); control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE, 0); - if (get_options()->HttpsProxy) { - char buf[1024]; - char *base64_authenticator=NULL; - const char *authenticator = get_options()->HttpsProxyAuthenticator; + proxy_type = PROXY_NONE; - if (authenticator) { - base64_authenticator = alloc_http_authenticator(authenticator); - if (!base64_authenticator) - log_warn(LD_OR, "Encoding https authenticator failed"); - } - if (base64_authenticator) { - tor_snprintf(buf, sizeof(buf), "CONNECT %s:%d HTTP/1.1\r\n" - "Proxy-Authorization: Basic %s\r\n\r\n", - fmt_addr(&conn->addr), - conn->port, base64_authenticator); - tor_free(base64_authenticator); - } else { - tor_snprintf(buf, sizeof(buf), "CONNECT %s:%d HTTP/1.0\r\n\r\n", - fmt_addr(&conn->addr), conn->port); + if (get_options()->HTTPSProxy) + proxy_type = PROXY_CONNECT; + else if (get_options()->Socks4Proxy) + proxy_type = PROXY_SOCKS4; + else if (get_options()->Socks5Proxy) + proxy_type = PROXY_SOCKS5; + + if (proxy_type != PROXY_NONE) { + /* start proxy handshake */ + if (connection_proxy_connect(conn, proxy_type) < 0) { + connection_mark_for_close(conn); + return -1; } - connection_write_to_buf(buf, strlen(buf), conn); - conn->state = OR_CONN_STATE_PROXY_FLUSHING; + + connection_start_reading(conn); + conn->state = OR_CONN_STATE_PROXY_HANDSHAKING; return 0; } @@ -376,6 +335,78 @@ connection_or_finished_connecting(or_connection_t *or_conn) return 0; } +/** Return 1 if identity digest <b>id_digest</b> is known to be a + * currently or recently running relay. Otherwise return 0. */ +int +connection_or_digest_is_known_relay(const char *id_digest) +{ + if (router_get_consensus_status_by_id(id_digest)) + return 1; /* It's in the consensus: "yes" */ + if (router_get_by_digest(id_digest)) + return 1; /* Not in the consensus, but we have a descriptor for + * it. Probably it was in a recent consensus. "Yes". */ + return 0; +} + +/** Set the per-conn read and write limits for <b>conn</b>. If it's a known + * relay, we will rely on the global read and write buckets, so give it + * per-conn limits that are big enough they'll never matter. But if it's + * not a known relay, first check if we set PerConnBwRate/Burst, then + * check if the consensus sets them, else default to 'big enough'. + * + * If <b>reset</b> is true, set the bucket to be full. Otherwise, just + * clip the bucket if it happens to be <em>too</em> full. + */ +static void +connection_or_update_token_buckets_helper(or_connection_t *conn, int reset, + or_options_t *options) +{ + int rate, burst; /* per-connection rate limiting params */ + if (connection_or_digest_is_known_relay(conn->identity_digest)) { + /* It's in the consensus, or we have a descriptor for it meaning it + * was probably in a recent consensus. It's a recognized relay: + * give it full bandwidth. */ + rate = (int)options->BandwidthRate; + burst = (int)options->BandwidthBurst; + } else { + /* Not a recognized relay. Squeeze it down based on the suggested + * bandwidth parameters in the consensus, but allow local config + * options to override. */ + rate = options->PerConnBWRate ? (int)options->PerConnBWRate : + networkstatus_get_param(NULL, "perconnbwrate", + (int)options->BandwidthRate, 1, INT32_MAX); + burst = options->PerConnBWBurst ? (int)options->PerConnBWBurst : + networkstatus_get_param(NULL, "perconnbwburst", + (int)options->BandwidthBurst, 1, INT32_MAX); + } + + conn->bandwidthrate = rate; + conn->bandwidthburst = burst; + if (reset) { /* set up the token buckets to be full */ + conn->read_bucket = conn->write_bucket = burst; + return; + } + /* If the new token bucket is smaller, take out the extra tokens. + * (If it's larger, don't -- the buckets can grow to reach the cap.) */ + if (conn->read_bucket > burst) + conn->read_bucket = burst; + if (conn->write_bucket > burst) + conn->write_bucket = burst; +} + +/** Either our set of relays or our per-conn rate limits have changed. + * Go through all the OR connections and update their token buckets to make + * sure they don't exceed their maximum values. */ +void +connection_or_update_token_buckets(smartlist_t *conns, or_options_t *options) +{ + SMARTLIST_FOREACH(conns, connection_t *, conn, + { + if (connection_speaks_cells(conn)) + connection_or_update_token_buckets_helper(TO_OR_CONN(conn), 0, options); + }); +} + /** If we don't necessarily know the router we're connecting to, but we * have an addr/port/id_digest, then fill in as much as we can. Start * by checking to see if this describes a router we know. */ @@ -385,11 +416,9 @@ connection_or_init_conn_from_address(or_connection_t *conn, const char *id_digest, int started_here) { - or_options_t *options = get_options(); routerinfo_t *r = router_get_by_digest(id_digest); - conn->bandwidthrate = (int)options->BandwidthRate; - conn->read_bucket = conn->bandwidthburst = (int)options->BandwidthBurst; connection_or_set_identity_digest(conn, id_digest); + connection_or_update_token_buckets_helper(conn, 1, get_options()); conn->_base.port = port; tor_addr_copy(&conn->_base.addr, addr); @@ -442,7 +471,7 @@ connection_or_init_conn_from_address(or_connection_t *conn, * Requires that both input connections are open; not is_bad_for_new_circs, * and not impossibly non-canonical. * - * If </b>forgive_new_connections</b> is true, then we do not call + * If <b>forgive_new_connections</b> is true, then we do not call * <b>a</b>better than <b>b</b> simply because b has no circuits, * unless b is also relatively old. */ @@ -581,11 +610,24 @@ connection_or_get_for_extend(const char *digest, #define TIME_BEFORE_OR_CONN_IS_TOO_OLD (60*60*24*7) /** Given the head of the linked list for all the or_connections with a given - * identity, set elements of that list as is_bad_for_new_circs() as - * appropriate. Helper for connection_or_set_bad_connections(). + * identity, set elements of that list as is_bad_for_new_circs as + * appropriate. Helper for connection_or_set_bad_connections(). + * + * Specifically, we set the is_bad_for_new_circs flag on: + * - all connections if <b>force</b> is true. + * - all connections that are too old. + * - all open non-canonical connections for which a canonical connection + * exists to the same router. + * - all open canonical connections for which a 'better' canonical + * connection exists to the same router. + * - all open non-canonical connections for which a 'better' non-canonical + * connection exists to the same router at the same address. + * + * See connection_or_is_better() for our idea of what makes one OR connection + * better than another. */ static void -connection_or_group_set_badness(or_connection_t *head) +connection_or_group_set_badness(or_connection_t *head, int force) { or_connection_t *or_conn = NULL, *best = NULL; int n_old = 0, n_inprogress = 0, n_canonical = 0, n_other = 0; @@ -597,8 +639,9 @@ connection_or_group_set_badness(or_connection_t *head) if (or_conn->_base.marked_for_close || or_conn->is_bad_for_new_circs) continue; - if (or_conn->_base.timestamp_created + TIME_BEFORE_OR_CONN_IS_TOO_OLD - < now) { + if (force || + or_conn->_base.timestamp_created + TIME_BEFORE_OR_CONN_IS_TOO_OLD + < now) { log_info(LD_OR, "Marking OR conn to %s:%d as too old for new circuits " "(fd %d, %d secs old).", @@ -631,7 +674,7 @@ connection_or_group_set_badness(or_connection_t *head) /* We have at least one open canonical connection to this router, * and this one is open but not canonical. Mark it bad. */ log_info(LD_OR, - "Marking OR conn to %s:%d as too old for new circuits: " + "Marking OR conn to %s:%d as unsuitable for new circuits: " "(fd %d, %d secs old). It is not canonical, and we have " "another connection to that OR that is.", or_conn->_base.address, or_conn->_base.port, or_conn->_base.s, @@ -671,7 +714,7 @@ connection_or_group_set_badness(or_connection_t *head) even when we're being forgiving. */ if (best->is_canonical) { log_info(LD_OR, - "Marking OR conn to %s:%d as too old for new circuits: " + "Marking OR conn to %s:%d as unsuitable for new circuits: " "(fd %d, %d secs old). We have a better canonical one " "(fd %d; %d secs old).", or_conn->_base.address, or_conn->_base.port, or_conn->_base.s, @@ -681,9 +724,9 @@ connection_or_group_set_badness(or_connection_t *head) } else if (!tor_addr_compare(&or_conn->real_addr, &best->real_addr, CMP_EXACT)) { log_info(LD_OR, - "Marking OR conn to %s:%d as too old for new circuits: " - "(fd %d, %d secs old). We have a better one " - "(fd %d; %d secs old).", + "Marking OR conn to %s:%d as unsuitable for new circuits: " + "(fd %d, %d secs old). We have a better one with the " + "same address (fd %d; %d secs old).", or_conn->_base.address, or_conn->_base.port, or_conn->_base.s, (int)(now - or_conn->_base.timestamp_created), best->_base.s, (int)(now - best->_base.timestamp_created)); @@ -693,27 +736,20 @@ connection_or_group_set_badness(or_connection_t *head) } } -/** Go through all the OR connections, and set the is_bad_for_new_circs - * flag on: - * - all connections that are too old. - * - all open non-canonical connections for which a canonical connection - * exists to the same router. - * - all open canonical connections for which a 'better' canonical - * connection exists to the same router. - * - all open non-canonical connections for which a 'better' non-canonical - * connection exists to the same router at the same address. - * - * See connection_or_is_better() for our idea of what makes one OR connection - * better than another. +/** Go through all the OR connections (or if <b>digest</b> is non-NULL, just + * the OR connections with that digest), and set the is_bad_for_new_circs + * flag based on the rules in connection_or_group_set_badness() (or just + * always set it if <b>force</b> is true). */ void -connection_or_set_bad_connections(void) +connection_or_set_bad_connections(const char *digest, int force) { if (!orconn_identity_map) return; DIGESTMAP_FOREACH(orconn_identity_map, identity, or_connection_t *, conn) { - connection_or_group_set_badness(conn); + if (!digest || tor_memeq(digest, conn->identity_digest, DIGEST_LEN)) + connection_or_group_set_badness(conn, force); } DIGESTMAP_FOREACH_END; } @@ -753,6 +789,7 @@ connection_or_connect(const tor_addr_t *_addr, uint16_t port, or_connection_t *conn; or_options_t *options = get_options(); int socket_error = 0; + int using_proxy = 0; tor_addr_t addr; tor_assert(_addr); @@ -771,19 +808,27 @@ connection_or_connect(const tor_addr_t *_addr, uint16_t port, conn->_base.state = OR_CONN_STATE_CONNECTING; control_event_or_conn_status(conn, OR_CONN_EVENT_LAUNCHED, 0); - if (options->HttpsProxy) { - /* we shouldn't connect directly. use the https proxy instead. */ - tor_addr_from_ipv4h(&addr, options->HttpsProxyAddr); - port = options->HttpsProxyPort; + /* use a proxy server if available */ + if (options->HTTPSProxy) { + using_proxy = 1; + tor_addr_copy(&addr, &options->HTTPSProxyAddr); + port = options->HTTPSProxyPort; + } else if (options->Socks4Proxy) { + using_proxy = 1; + tor_addr_copy(&addr, &options->Socks4ProxyAddr); + port = options->Socks4ProxyPort; + } else if (options->Socks5Proxy) { + using_proxy = 1; + tor_addr_copy(&addr, &options->Socks5ProxyAddr); + port = options->Socks5ProxyPort; } switch (connection_connect(TO_CONN(conn), conn->_base.address, &addr, port, &socket_error)) { case -1: /* If the connection failed immediately, and we're using - * an https proxy, our https proxy is down. Don't blame the - * Tor server. */ - if (!options->HttpsProxy) + * a proxy, our proxy is down. Don't blame the Tor server. */ + if (!using_proxy) entry_guard_register_connect_status(conn->identity_digest, 0, 1, time(NULL)); connection_or_connect_failed(conn, @@ -792,7 +837,7 @@ connection_or_connect(const tor_addr_t *_addr, uint16_t port, connection_free(TO_CONN(conn)); return NULL; case 0: - connection_watch_events(TO_CONN(conn), EV_READ | EV_WRITE); + connection_watch_events(TO_CONN(conn), READ_EVENT | WRITE_EVENT); /* writable indicates finish, readable indicates broken link, error indicates broken link on windows */ return conn; @@ -819,13 +864,14 @@ connection_tls_start_handshake(or_connection_t *conn, int receiving) { conn->_base.state = OR_CONN_STATE_TLS_HANDSHAKING; conn->tls = tor_tls_new(conn->_base.s, receiving); - tor_tls_set_logged_address(conn->tls, escaped_safe_str(conn->_base.address)); + tor_tls_set_logged_address(conn->tls, // XXX client and relay? + escaped_safe_str(conn->_base.address)); if (!conn->tls) { log_warn(LD_BUG,"tor_tls_new failed. Closing."); return -1; } connection_start_reading(TO_CONN(conn)); - log_debug(LD_OR,"starting TLS handshake on fd %d", conn->_base.s); + log_debug(LD_HANDSHAKE,"starting TLS handshake on fd %d", conn->_base.s); note_crypto_pk_op(receiving ? TLS_HANDSHAKE_S : TLS_HANDSHAKE_C); if (connection_tls_continue_handshake(conn) < 0) { @@ -932,23 +978,26 @@ connection_or_nonopen_was_started_here(or_connection_t *conn) * return -1 if he is lying, broken, or otherwise something is wrong. * * If we initiated this connection (<b>started_here</b> is true), make sure - * the other side sent sent a correctly formed certificate. If I initiated the + * the other side sent a correctly formed certificate. If I initiated the * connection, make sure it's the right guy. * * Otherwise (if we _didn't_ initiate this connection), it's okay for * the certificate to be weird or absent. * * If we return 0, and the certificate is as expected, write a hash of the - * identity key into digest_rcvd, which must have DIGEST_LEN space in it. (If - * we return -1 this buffer is undefined.) If the certificate is invalid - * or missing on an incoming connection, we return 0 and set digest_rcvd to - * DIGEST_LEN 0 bytes. + * identity key into <b>digest_rcvd_out</b>, which must have DIGEST_LEN + * space in it. + * If the certificate is invalid or missing on an incoming connection, + * we return 0 and set <b>digest_rcvd_out</b> to DIGEST_LEN NUL bytes. + * (If we return -1, the contents of this buffer are undefined.) * * As side effects, * 1) Set conn->circ_id_type according to tor-spec.txt. * 2) If we're an authdirserver and we initiated the connection: drop all * descriptors that claim to be on that IP/port but that aren't * this guy; and note that this guy is reachable. + * 3) If this is a bridge and we didn't configure its identity + * fingerprint, remember the keyid we just learned. */ static int connection_or_check_valid_tls_handshake(or_connection_t *conn, @@ -959,19 +1008,23 @@ connection_or_check_valid_tls_handshake(or_connection_t *conn, or_options_t *options = get_options(); int severity = server_mode(options) ? LOG_PROTOCOL_WARN : LOG_WARN; const char *safe_address = - started_here ? conn->_base.address : safe_str(conn->_base.address); + started_here ? conn->_base.address : + safe_str_client(conn->_base.address); const char *conn_type = started_here ? "outgoing" : "incoming"; + crypto_pk_env_t *our_identity = + started_here ? get_tlsclient_identity_key() : + get_server_identity_key(); int has_cert = 0, has_identity=0; check_no_tls_errors(); has_cert = tor_tls_peer_has_cert(conn->tls); if (started_here && !has_cert) { - log_info(LD_PROTOCOL,"Tried connecting to router at %s:%d, but it didn't " + log_info(LD_HANDSHAKE,"Tried connecting to router at %s:%d, but it didn't " "send a cert! Closing.", safe_address, conn->_base.port); return -1; } else if (!has_cert) { - log_debug(LD_PROTOCOL,"Got incoming connection with no certificate. " + log_debug(LD_HANDSHAKE,"Got incoming connection with no certificate. " "That's ok."); } check_no_tls_errors(); @@ -980,15 +1033,16 @@ connection_or_check_valid_tls_handshake(or_connection_t *conn, int v = tor_tls_verify(started_here?severity:LOG_INFO, conn->tls, &identity_rcvd); if (started_here && v<0) { - log_fn(severity,LD_OR,"Tried connecting to router at %s:%d: It" + log_fn(severity,LD_HANDSHAKE,"Tried connecting to router at %s:%d: It" " has a cert but it's invalid. Closing.", safe_address, conn->_base.port); return -1; } else if (v<0) { - log_info(LD_PROTOCOL,"Incoming connection gave us an invalid cert " + log_info(LD_HANDSHAKE,"Incoming connection gave us an invalid cert " "chain; ignoring."); } else { - log_debug(LD_OR,"The certificate seems to be valid on %s connection " + log_debug(LD_HANDSHAKE, + "The certificate seems to be valid on %s connection " "with %s:%d", conn_type, safe_address, conn->_base.port); } check_no_tls_errors(); @@ -997,7 +1051,7 @@ connection_or_check_valid_tls_handshake(or_connection_t *conn, if (identity_rcvd) { has_identity = 1; crypto_pk_get_digest(identity_rcvd, digest_rcvd_out); - if (crypto_pk_cmp_keys(get_identity_key(), identity_rcvd)<0) { + if (crypto_pk_cmp_keys(our_identity, identity_rcvd)<0) { conn->circ_id_type = CIRC_ID_TYPE_LOWER; } else { conn->circ_id_type = CIRC_ID_TYPE_HIGHER; @@ -1015,9 +1069,13 @@ connection_or_check_valid_tls_handshake(or_connection_t *conn, conn->nickname[0] = '$'; base16_encode(conn->nickname+1, HEX_DIGEST_LEN+1, conn->identity_digest, DIGEST_LEN); - log_info(LD_OR, "Connected to router %s at %s:%d without knowing " + log_info(LD_HANDSHAKE, "Connected to router %s at %s:%d without knowing " "its key. Hoping for the best.", conn->nickname, conn->_base.address, conn->_base.port); + /* if it's a bridge and we didn't know its identity fingerprint, now + * we do -- remember it for future attempts. */ + learned_router_identity(&conn->_base.addr, conn->_base.port, + digest_rcvd_out); } if (started_here) { @@ -1031,7 +1089,7 @@ connection_or_check_valid_tls_handshake(or_connection_t *conn, base16_encode(seen, sizeof(seen), digest_rcvd_out, DIGEST_LEN); base16_encode(expected, sizeof(expected), conn->identity_digest, DIGEST_LEN); - log_fn(severity, LD_OR, + log_fn(severity, LD_HANDSHAKE, "Tried connecting to router at %s:%d, but identity key was not " "as expected: wanted %s but got %s.", conn->_base.address, conn->_base.port, expected, seen); @@ -1044,9 +1102,6 @@ connection_or_check_valid_tls_handshake(or_connection_t *conn, as_advertised = 0; } if (authdir_mode_tests_reachability(options)) { - /* We initiated this connection to address:port. Drop all routers - * with the same address:port and a different key. - */ dirserv_orconn_tls_done(conn->_base.address, conn->_base.port, digest_rcvd_out, as_advertised); } @@ -1073,8 +1128,8 @@ connection_tls_finish_handshake(or_connection_t *conn) char digest_rcvd[DIGEST_LEN]; int started_here = connection_or_nonopen_was_started_here(conn); - log_debug(LD_OR,"tls handshake with %s done. verifying.", - safe_str(conn->_base.address)); + log_debug(LD_HANDSHAKE,"tls handshake with %s done. verifying.", + safe_str_client(conn->_base.address)); directory_set_dirty(); @@ -1082,6 +1137,8 @@ connection_tls_finish_handshake(or_connection_t *conn) digest_rcvd) < 0) return -1; + circuit_build_times_network_is_live(&circ_times); + if (tor_tls_used_v1_handshake(conn->tls)) { conn->link_proto = 1; if (!started_here) { @@ -1117,7 +1174,8 @@ connection_init_or_handshake_state(or_connection_t *conn, int started_here) void or_handshake_state_free(or_handshake_state_t *state) { - tor_assert(state); + if (!state) + return; memset(state, 0xBE, sizeof(or_handshake_state_t)); tor_free(state); } @@ -1134,6 +1192,7 @@ connection_or_set_state_open(or_connection_t *conn) control_event_or_conn_status(conn, OR_CONN_EVENT_CONNECTED, 0); if (started_here) { + circuit_build_times_network_is_live(&circ_times); rep_hist_note_connect_succeeded(conn->identity_digest, now); if (entry_guard_register_connect_status(conn->identity_digest, 1, 0, now) < 0) { @@ -1158,10 +1217,10 @@ connection_or_set_state_open(or_connection_t *conn) } } } - if (conn->handshake_state) { - or_handshake_state_free(conn->handshake_state); - conn->handshake_state = NULL; - } + + or_handshake_state_free(conn->handshake_state); + conn->handshake_state = NULL; + connection_start_reading(TO_CONN(conn)); circuit_n_conn_done(conn, 1); /* send the pending creates, if any. */ @@ -1235,6 +1294,7 @@ connection_or_process_cells_from_inbuf(or_connection_t *conn) if (connection_fetch_var_cell_from_buf(conn, &var_cell)) { if (!var_cell) return 0; /* not yet. */ + circuit_build_times_network_is_live(&circ_times); command_process_var_cell(var_cell, conn); var_cell_free(var_cell); } else { @@ -1244,6 +1304,7 @@ connection_or_process_cells_from_inbuf(or_connection_t *conn) available? */ return 0; /* not yet */ + circuit_build_times_network_is_live(&circ_times); connection_fetch_from_buf(buf, CELL_NETWORK_SIZE, TO_CONN(conn)); /* retrieve cell info from buf (create the host-order struct from the @@ -1274,10 +1335,6 @@ connection_or_send_destroy(circid_t circ_id, or_connection_t *conn, int reason) cell.payload[0] = (uint8_t) reason; log_debug(LD_OR,"Sending destroy (circID %d).", circ_id); - /* XXXX It's possible that under some circumstances, we want the destroy - * to take precedence over other data waiting on the circuit's cell queue. - */ - connection_or_write_cell_to_buf(&cell, conn); return 0; } @@ -1357,9 +1414,8 @@ connection_or_send_netinfo(or_connection_t *conn) len = append_address_to_payload(out, &my_addr); if (len < 0) return -1; - out += len; } else { - *out++ = 0; + *out = 0; } connection_or_write_cell_to_buf(&cell, conn); diff --git a/src/or/connection_or.h b/src/or/connection_or.h new file mode 100644 index 0000000000..70ef96a335 --- /dev/null +++ b/src/or/connection_or.h @@ -0,0 +1,57 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file connection_or.h + * \brief Header file for connection_or.c. + **/ + +#ifndef _TOR_CONNECTION_OR_H +#define _TOR_CONNECTION_OR_H + +void connection_or_remove_from_identity_map(or_connection_t *conn); +void connection_or_clear_identity_map(void); +or_connection_t *connection_or_get_for_extend(const char *digest, + const tor_addr_t *target_addr, + const char **msg_out, + int *launch_out); +void connection_or_set_bad_connections(const char *digest, int force); + +int connection_or_reached_eof(or_connection_t *conn); +int connection_or_process_inbuf(or_connection_t *conn); +int connection_or_flushed_some(or_connection_t *conn); +int connection_or_finished_flushing(or_connection_t *conn); +int connection_or_finished_connecting(or_connection_t *conn); +int connection_or_digest_is_known_relay(const char *id_digest); +void connection_or_update_token_buckets(smartlist_t *conns, + or_options_t *options); + +void connection_or_connect_failed(or_connection_t *conn, + int reason, const char *msg); +or_connection_t *connection_or_connect(const tor_addr_t *addr, uint16_t port, + const char *id_digest); + +int connection_tls_start_handshake(or_connection_t *conn, int receiving); +int connection_tls_continue_handshake(or_connection_t *conn); + +void or_handshake_state_free(or_handshake_state_t *state); +int connection_or_set_state_open(or_connection_t *conn); +void connection_or_write_cell_to_buf(const cell_t *cell, + or_connection_t *conn); +void connection_or_write_var_cell_to_buf(const var_cell_t *cell, + or_connection_t *conn); +int connection_or_send_destroy(circid_t circ_id, or_connection_t *conn, + int reason); +int connection_or_send_netinfo(or_connection_t *conn); +int is_or_protocol_version_known(uint16_t version); + +void cell_pack(packed_cell_t *dest, const cell_t *src); +void var_cell_pack_header(const var_cell_t *cell, char *hdr_out); +var_cell_t *var_cell_new(uint16_t payload_len); +void var_cell_free(var_cell_t *cell); + +#endif + diff --git a/src/or/control.c b/src/or/control.c index f91afafeee..f75ac67758 100644 --- a/src/or/control.c +++ b/src/or/control.c @@ -11,6 +11,28 @@ #define CONTROL_PRIVATE #include "or.h" +#include "buffers.h" +#include "circuitbuild.h" +#include "circuitlist.h" +#include "circuituse.h" +#include "config.h" +#include "connection.h" +#include "connection_edge.h" +#include "control.h" +#include "directory.h" +#include "dirserv.h" +#include "dnsserv.h" +#include "geoip.h" +#include "hibernate.h" +#include "main.h" +#include "networkstatus.h" +#include "policies.h" +#include "reasons.h" +#include "router.h" +#include "routerlist.h" +#include "routerparse.h" + +#include "procmon.h" /** Yield true iff <b>s</b> is the state of a control_connection_t that has * finished authentication and is accepting commands. */ @@ -43,7 +65,8 @@ #define EVENT_STREAM_BANDWIDTH_USED 0x0014 #define EVENT_CLIENTS_SEEN 0x0015 #define EVENT_NEWCONSENSUS 0x0016 -#define _EVENT_MAX 0x0016 +#define EVENT_BUILDTIMEOUT_SET 0x0017 +#define _EVENT_MAX 0x0017 /* If _EVENT_MAX ever hits 0x0020, we need to make the mask wider. */ /** Bitfield: The bit 1<<e is set if <b>any</b> open control @@ -54,13 +77,9 @@ **/ typedef uint32_t event_mask_t; -/** An event mask of all the events that controller with the LONG_NAMES option - * set is interested in receiving. */ -static event_mask_t global_event_mask1long = 0; - -/** An event mask of all the events that controller with the SHORT_NAMES option - * set is interested in receiving. */ -static event_mask_t global_event_mask1short = 0; +/** An event mask of all the events that any controller is interested in + * receiving. */ +static event_mask_t global_event_mask = 0; /** True iff we have disabled log messages from being sent to the controller */ static int disable_log_messages = 0; @@ -68,13 +87,7 @@ static int disable_log_messages = 0; /** Macro: true if any control connection is interested in events of type * <b>e</b>. */ #define EVENT_IS_INTERESTING(e) \ - ((global_event_mask1long|global_event_mask1short) & (1<<(e))) -/** Macro: true if any control connection with the LONG_NAMES option is - * interested in events of type <b>e</b>. */ -#define EVENT_IS_INTERESTING1L(e) (global_event_mask1long & (1<<(e))) -/** Macro: true if any control connection with the SHORT_NAMES option is - * interested in events of type <b>e</b>. */ -#define EVENT_IS_INTERESTING1S(e) (global_event_mask1short & (1<<(e))) + (global_event_mask & (1<<(e))) /** If we're using cookie-type authentication, how long should our cookies be? */ @@ -95,25 +108,13 @@ static char authentication_cookie[AUTHENTICATION_COOKIE_LEN]; * of this so we can respond to getinfo status/bootstrap-phase queries. */ static char last_sent_bootstrap_message[BOOTSTRAP_MSG_LEN]; -/** Flag for event_format_t. Indicates that we should use the old - * name format of nickname|hexdigest - */ -#define SHORT_NAMES 1 -/** Flag for event_format_t. Indicates that we should use the new - * name format of $hexdigest[=~]nickname +/** Flag for event_format_t. Indicates that we should use the one standard + format. */ -#define LONG_NAMES 2 -#define ALL_NAMES (SHORT_NAMES|LONG_NAMES) -/** Flag for event_format_t. Indicates that we should use the new event - * format where extra event fields are allowed using a NAME=VAL format. */ -#define EXTENDED_FORMAT 4 -/** Flag for event_format_t. Indicates that we are using the old event format - * where extra fields aren't allowed. */ -#define NONEXTENDED_FORMAT 8 -#define ALL_FORMATS (EXTENDED_FORMAT|NONEXTENDED_FORMAT) +#define ALL_FORMATS 1 /** Bit field of flags to select how to format a controller event. Recognized - * flags are SHORT_NAMES, LONG_NAMES, EXTENDED_FORMAT, NONEXTENDED_FORMAT. */ + * flag is ALL_FORMATS. */ typedef int event_format_t; static void connection_printf_to_buf(control_connection_t *conn, @@ -123,9 +124,6 @@ static void send_control_done(control_connection_t *conn); static void send_control_event(uint16_t event, event_format_t which, const char *format, ...) CHECK_PRINTF(3,4); -static void send_control_event_extended(uint16_t event, event_format_t which, - const char *format, ...) - CHECK_PRINTF(3,4); static int handle_control_setconf(control_connection_t *conn, uint32_t len, char *body); static int handle_control_resetconf(control_connection_t *conn, uint32_t len, @@ -139,8 +137,6 @@ static int handle_control_setevents(control_connection_t *conn, uint32_t len, static int handle_control_authenticate(control_connection_t *conn, uint32_t len, const char *body); -static int handle_control_saveconf(control_connection_t *conn, uint32_t len, - const char *body); static int handle_control_signal(control_connection_t *conn, uint32_t len, const char *body); static int handle_control_mapaddress(control_connection_t *conn, uint32_t len, @@ -174,7 +170,7 @@ static int handle_control_usefeature(control_connection_t *conn, const char *body); static int write_stream_target_to_buf(edge_connection_t *conn, char *buf, size_t len); -static void orconn_target_get_name(int long_names, char *buf, size_t len, +static void orconn_target_get_name(char *buf, size_t len, or_connection_t *conn); static char *get_cookie_file(void); @@ -214,25 +210,19 @@ control_update_global_event_mask(void) { smartlist_t *conns = get_connection_array(); event_mask_t old_mask, new_mask; - old_mask = global_event_mask1short; - old_mask |= global_event_mask1long; + old_mask = global_event_mask; - global_event_mask1short = 0; - global_event_mask1long = 0; + global_event_mask = 0; SMARTLIST_FOREACH(conns, connection_t *, _conn, { if (_conn->type == CONN_TYPE_CONTROL && STATE_IS_OPEN(_conn->state)) { control_connection_t *conn = TO_CONTROL_CONN(_conn); - if (conn->use_long_names) - global_event_mask1long |= conn->event_mask; - else - global_event_mask1short |= conn->event_mask; + global_event_mask |= conn->event_mask; } }); - new_mask = global_event_mask1short; - new_mask |= global_event_mask1long; + new_mask = global_event_mask; /* Handle the aftermath. Set up the log callback to tell us only what * we want to hear...*/ @@ -312,7 +302,7 @@ connection_write_str_to_buf(const char *s, control_connection_t *conn) /** Given a <b>len</b>-character string in <b>data</b>, made of lines * terminated by CRLF, allocate a new string in *<b>out</b>, and copy the * contents of <b>data</b> into *<b>out</b>, adding a period before any period - * that that appears at the start of a line, and adding a period-CRLF line at + * that appears at the start of a line, and adding a period-CRLF line at * the end. Replace all LF characters sequences with CRLF. Return the number * of bytes in *<b>out</b>. */ @@ -520,6 +510,53 @@ connection_printf_to_buf(control_connection_t *conn, const char *format, ...) connection_write_to_buf(buf, len, TO_CONN(conn)); } +/** Write all of the open control ports to ControlPortWriteToFile */ +void +control_ports_write_to_file(void) +{ + smartlist_t *lines; + char *joined = NULL; + or_options_t *options = get_options(); + + if (!options->ControlPortWriteToFile) + return; + + lines = smartlist_create(); + + SMARTLIST_FOREACH_BEGIN(get_connection_array(), const connection_t *, conn) { + char *port_str = NULL; + if (conn->type != CONN_TYPE_CONTROL_LISTENER || conn->marked_for_close) + continue; +#ifdef AF_UNIX + if (conn->socket_family == AF_UNIX) { + tor_asprintf(&port_str, "UNIX_PORT=%s\n", conn->address); + smartlist_add(lines, port_str); + continue; + } +#endif + tor_asprintf(&port_str, "PORT=%s:%d\n", conn->address, conn->port); + smartlist_add(lines, port_str); + } SMARTLIST_FOREACH_END(conn); + + joined = smartlist_join_strings(lines, "", 0, NULL); + + if (write_str_to_file(options->ControlPortWriteToFile, joined, 0) < 0) { + log_warn(LD_CONTROL, "Writing %s failed: %s", + options->ControlPortWriteToFile, strerror(errno)); + } +#ifndef MS_WINDOWS + if (options->ControlPortFileGroupReadable) { + if (chmod(options->ControlPortWriteToFile, 0640)) { + log_warn(LD_FS,"Unable to make %s group-readable.", + options->ControlPortWriteToFile); + } + } +#endif + tor_free(joined); + SMARTLIST_FOREACH(lines, char *, cp, tor_free(cp)); + smartlist_free(lines); +} + /** Send a "DONE" message down the control connection <b>conn</b>. */ static void send_control_done(control_connection_t *conn) @@ -542,28 +579,15 @@ send_control_event_string(uint16_t event, event_format_t which, const char *msg) { smartlist_t *conns = get_connection_array(); + (void)which; tor_assert(event >= _EVENT_MIN && event <= _EVENT_MAX); - SMARTLIST_FOREACH(conns, connection_t *, conn, - { + SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) { if (conn->type == CONN_TYPE_CONTROL && !conn->marked_for_close && conn->state == CONTROL_CONN_STATE_OPEN) { control_connection_t *control_conn = TO_CONTROL_CONN(conn); - if (control_conn->use_long_names) { - if (!(which & LONG_NAMES)) - continue; - } else { - if (!(which & SHORT_NAMES)) - continue; - } - if (control_conn->use_extended_events) { - if (!(which & EXTENDED_FORMAT)) - continue; - } else { - if (!(which & NONEXTENDED_FORMAT)) - continue; - } + if (control_conn->event_mask & (1<<event)) { int is_err = 0; connection_write_to_buf(msg, strlen(msg), TO_CONN(control_conn)); @@ -579,7 +603,7 @@ send_control_event_string(uint16_t event, event_format_t which, connection_handle_write(TO_CONN(control_conn), 1); } } - }); + } SMARTLIST_FOREACH_END(conn); } /** Helper for send_control1_event and send_control1_event_extended: @@ -587,22 +611,17 @@ send_control_event_string(uint16_t event, event_format_t which, * <b>event</b>. The event's body is created by the printf-style format in * <b>format</b>, and other arguments as provided. * - * If <b>extended</b> is true, and the format contains a single '@' character, - * it will be replaced with a space and all text after that character will be - * sent only to controllers that have enabled extended events. - * * Currently the length of the message is limited to 1024 (including the * ending \\r\\n\\0). */ static void -send_control_event_impl(uint16_t event, event_format_t which, int extended, - const char *format, va_list ap) +send_control_event_impl(uint16_t event, event_format_t which, + const char *format, va_list ap) { /* This is just a little longer than the longest allowed log message */ #define SEND_CONTROL1_EVENT_BUFFERSIZE 10064 int r; char buf[SEND_CONTROL1_EVENT_BUFFERSIZE]; size_t len; - char *cp; r = tor_vsnprintf(buf, sizeof(buf), format, ap); if (r<0) { @@ -618,15 +637,7 @@ send_control_event_impl(uint16_t event, event_format_t which, int extended, buf[SEND_CONTROL1_EVENT_BUFFERSIZE-3] = '\r'; } - if (extended && (cp = strchr(buf, '@'))) { - which &= ~ALL_FORMATS; - *cp = ' '; - send_control_event_string(event, which|EXTENDED_FORMAT, buf); - memcpy(cp, "\r\n\0", 3); - send_control_event_string(event, which|NONEXTENDED_FORMAT, buf); - } else { - send_control_event_string(event, which|ALL_FORMATS, buf); - } + send_control_event_string(event, which|ALL_FORMATS, buf); } /** Send an event to all v1 controllers that are listening for code @@ -641,27 +652,7 @@ send_control_event(uint16_t event, event_format_t which, { va_list ap; va_start(ap, format); - send_control_event_impl(event, which, 0, format, ap); - va_end(ap); -} - -/** Send an event to all v1 controllers that are listening for code - * <b>event</b>. The event's body is created by the printf-style format in - * <b>format</b>, and other arguments as provided. - * - * If the format contains a single '@' character, it will be replaced with a - * space and all text after that character will be sent only to controllers - * that have enabled extended events. - * - * Currently the length of the message is limited to 1024 (including the - * ending \\n\\r\\0. */ -static void -send_control_event_extended(uint16_t event, event_format_t which, - const char *format, ...) -{ - va_list ap; - va_start(ap, format); - send_control_event_impl(event, which, 1, format, ap); + send_control_event_impl(event, which, format, ap); va_end(ap); } @@ -907,36 +898,37 @@ handle_control_loadconf(control_connection_t *conn, uint32_t len, retval = options_init_from_string(body, CMD_RUN_TOR, NULL, &errstring); - if (retval != SETOPT_OK) { + if (retval != SETOPT_OK) log_warn(LD_CONTROL, "Controller gave us config file that didn't validate: %s", errstring); - switch (retval) { - case SETOPT_ERR_PARSE: - msg = "552 Invalid config file"; - break; - case SETOPT_ERR_TRANSITION: - msg = "553 Transition not allowed"; - break; - case SETOPT_ERR_SETTING: - msg = "553 Unable to set option"; - break; - case SETOPT_ERR_MISC: - default: - msg = "550 Unable to load config"; - break; - case SETOPT_OK: - tor_fragile_assert(); - break; - } + + switch (retval) { + case SETOPT_ERR_PARSE: + msg = "552 Invalid config file"; + break; + case SETOPT_ERR_TRANSITION: + msg = "553 Transition not allowed"; + break; + case SETOPT_ERR_SETTING: + msg = "553 Unable to set option"; + break; + case SETOPT_ERR_MISC: + default: + msg = "550 Unable to load config"; + break; + case SETOPT_OK: + break; + } + if (msg) { if (errstring) connection_printf_to_buf(conn, "%s: %s\r\n", msg, errstring); else connection_printf_to_buf(conn, "%s\r\n", msg); - tor_free(errstring); - return 0; + } else { + send_control_done(conn); } - send_control_done(conn); + tor_free(errstring); return 0; } @@ -948,7 +940,6 @@ handle_control_setevents(control_connection_t *conn, uint32_t len, { uint16_t event_code; uint32_t event_mask = 0; - unsigned int extended = 0; smartlist_t *events = smartlist_create(); (void) len; @@ -958,7 +949,6 @@ handle_control_setevents(control_connection_t *conn, uint32_t len, SMARTLIST_FOREACH_BEGIN(events, const char *, ev) { if (!strcasecmp(ev, "EXTENDED")) { - extended = 1; continue; } else if (!strcasecmp(ev, "CIRC")) event_code = EVENT_CIRCUIT_STATUS; @@ -1002,6 +992,8 @@ handle_control_setevents(control_connection_t *conn, uint32_t len, event_code = EVENT_CLIENTS_SEEN; else if (!strcasecmp(ev, "NEWCONSENSUS")) event_code = EVENT_NEWCONSENSUS; + else if (!strcasecmp(ev, "BUILDTIMEOUT_SET")) + event_code = EVENT_BUILDTIMEOUT_SET; else { connection_printf_to_buf(conn, "552 Unrecognized event \"%s\"\r\n", ev); @@ -1016,8 +1008,6 @@ handle_control_setevents(control_connection_t *conn, uint32_t len, smartlist_free(events); conn->event_mask = event_mask; - if (extended) - conn->use_extended_events = 1; control_update_global_event_mask(); send_control_done(conn); @@ -1281,7 +1271,29 @@ handle_control_signal(control_connection_t *conn, uint32_t len, /* Flush the "done" first if the signal might make us shut down. */ if (sig == SIGTERM || sig == SIGINT) connection_handle_write(TO_CONN(conn), 1); - control_signal_act(sig); + + process_signal(sig); + + return 0; +} + +/** Called when we get a TAKEOWNERSHIP command. Mark this connection + * as an owning connection, so that we will exit if the connection + * closes. */ +static int +handle_control_takeownership(control_connection_t *conn, uint32_t len, + const char *body) +{ + (void)len; + (void)body; + + conn->is_owning_control_connection = 1; + + log_info(LD_CONTROL, "Control connection %d has taken ownership of this " + "Tor instance.", + (int)(conn->_base.s)); + + send_control_done(conn); return 0; } @@ -1328,7 +1340,7 @@ handle_control_mapaddress(control_connection_t *conn, uint32_t len, smartlist_add(reply, ans); log_warn(LD_CONTROL, "Unable to allocate address for '%s' in MapAddress msg", - safe_str(line)); + safe_str_client(line)); } else { tor_snprintf(ans, anslen, "250-%s=%s", address, to); smartlist_add(reply, ans); @@ -1345,7 +1357,8 @@ handle_control_mapaddress(control_connection_t *conn, uint32_t len, "not of expected form 'foo=bar'.", line); smartlist_add(reply, ans); log_info(LD_CONTROL, "Skipping MapAddress '%s': wrong " - "number of items.", safe_str(line)); + "number of items.", + safe_str_client(line)); } SMARTLIST_FOREACH(elts, char *, cp, tor_free(cp)); smartlist_clear(elts); @@ -1374,13 +1387,15 @@ handle_control_mapaddress(control_connection_t *conn, uint32_t len, * trivial-to-implement questions. */ static int getinfo_helper_misc(control_connection_t *conn, const char *question, - char **answer) + char **answer, const char **errmsg) { (void) conn; if (!strcmp(question, "version")) { *answer = tor_strdup(get_version()); } else if (!strcmp(question, "config-file")) { *answer = tor_strdup(get_torrc_fname()); + } else if (!strcmp(question, "config-text")) { + *answer = options_dump(get_options(), 1); } else if (!strcmp(question, "info/names")) { *answer = list_getinfo_options(); } else if (!strcmp(question, "events/names")) { @@ -1392,15 +1407,19 @@ getinfo_helper_misc(control_connection_t *conn, const char *question, *answer = tor_strdup("VERBOSE_NAMES EXTENDED_EVENTS"); } else if (!strcmp(question, "address")) { uint32_t addr; - if (router_pick_published_address(get_options(), &addr) < 0) + if (router_pick_published_address(get_options(), &addr) < 0) { + *errmsg = "Address unknown"; return -1; + } *answer = tor_dup_ip(addr); } else if (!strcmp(question, "dir-usage")) { *answer = directory_dump_request_log(); } else if (!strcmp(question, "fingerprint")) { routerinfo_t *me = router_get_my_routerinfo(); - if (!me) + if (!me) { + *errmsg = "No routerdesc known; am I really a server?"; return -1; + } *answer = tor_malloc(HEX_DIGEST_LEN+1); base16_encode(*answer, HEX_DIGEST_LEN+1, me->cache_info.identity_digest, DIGEST_LEN); @@ -1457,12 +1476,71 @@ munge_extrainfo_into_routerinfo(const char *ri_body, signed_descriptor_t *ri, return tor_strndup(ri_body, ri->signed_descriptor_len); } +/** Implementation helper for GETINFO: answers requests for information about + * which ports are bound. */ +static int +getinfo_helper_listeners(control_connection_t *control_conn, + const char *question, + char **answer, const char **errmsg) +{ + int type; + smartlist_t *res; + + (void)control_conn; + (void)errmsg; + + if (!strcmp(question, "net/listeners/or")) + type = CONN_TYPE_OR_LISTENER; + else if (!strcmp(question, "net/listeners/dir")) + type = CONN_TYPE_DIR_LISTENER; + else if (!strcmp(question, "net/listeners/socks")) + type = CONN_TYPE_AP_LISTENER; + else if (!strcmp(question, "net/listeners/trans")) + type = CONN_TYPE_AP_TRANS_LISTENER; + else if (!strcmp(question, "net/listeners/natd")) + type = CONN_TYPE_AP_NATD_LISTENER; + else if (!strcmp(question, "net/listeners/dns")) + type = CONN_TYPE_AP_DNS_LISTENER; + else if (!strcmp(question, "net/listeners/control")) + type = CONN_TYPE_CONTROL_LISTENER; + else + return 0; /* unknown key */ + + res = smartlist_create(); + SMARTLIST_FOREACH_BEGIN(get_connection_array(), connection_t *, conn) { + char *addr; + struct sockaddr_storage ss; + socklen_t ss_len = sizeof(ss); + + if (conn->type != type || conn->marked_for_close || conn->s < 0) + continue; + + if (getsockname(conn->s, (struct sockaddr *)&ss, &ss_len) < 0) { + tor_asprintf(&addr, "%s:%d", conn->address, (int)conn->port); + } else { + char *tmp = tor_sockaddr_to_str((struct sockaddr *)&ss); + addr = esc_for_log(tmp); + tor_free(tmp); + } + if (addr) + smartlist_add(res, addr); + } SMARTLIST_FOREACH_END(conn); + + *answer = smartlist_join_strings(res, " ", 0, NULL); + + SMARTLIST_FOREACH(res, char *, cp, tor_free(cp)); + smartlist_free(res); + return 0; +} + /** Implementation helper for GETINFO: knows the answers for questions about * directory information. */ static int getinfo_helper_dir(control_connection_t *control_conn, - const char *question, char **answer) + const char *question, char **answer, + const char **errmsg) { + (void) control_conn; if (!strcmpstart(question, "desc/id/")) { routerinfo_t *ri = router_get_by_hexdigest(question+strlen("desc/id/")); if (ri) { @@ -1537,6 +1615,7 @@ getinfo_helper_dir(control_connection_t *control_conn, log_warn(LD_CONTROL, "getinfo '%s': %s", question, msg); smartlist_free(descs); tor_free(url); + *errmsg = msg; return -1; } SMARTLIST_FOREACH(descs, signed_descriptor_t *, sd, @@ -1587,7 +1666,7 @@ getinfo_helper_dir(control_connection_t *control_conn, } } else if (!strcmp(question, "dir/status-vote/current/consensus")) { /* v3 */ if (directory_caches_dir_info(get_options())) { - const cached_dir_t *consensus = dirserv_get_consensus(); + const cached_dir_t *consensus = dirserv_get_consensus("ns"); if (consensus) *answer = tor_strdup(consensus->dir); } @@ -1598,10 +1677,8 @@ getinfo_helper_dir(control_connection_t *control_conn, } } else if (!strcmp(question, "network-status")) { /* v1 */ routerlist_t *routerlist = router_get_routerlist(); - int verbose = control_conn->use_long_names; if (!routerlist || !routerlist->routers || - list_server_status_v1(routerlist->routers, answer, - verbose ? 2 : 1) < 0) { + list_server_status_v1(routerlist->routers, answer, 1) < 0) { return -1; } } else if (!strcmpstart(question, "extra-info/digest/")) { @@ -1635,8 +1712,10 @@ getinfo_helper_dir(control_connection_t *control_conn, * current states of things we send events about. */ static int getinfo_helper_events(control_connection_t *control_conn, - const char *question, char **answer) + const char *question, char **answer, + const char **errmsg) { + (void) control_conn; if (!strcmp(question, "circuit-status")) { circuit_t *circ; smartlist_t *status = smartlist_create(); @@ -1647,10 +1726,9 @@ getinfo_helper_events(control_connection_t *control_conn, const char *purpose; if (! CIRCUIT_IS_ORIGIN(circ) || circ->marked_for_close) continue; - if (control_conn->use_long_names) - path = circuit_list_path_for_controller(TO_ORIGIN_CIRCUIT(circ)); - else - path = circuit_list_path(TO_ORIGIN_CIRCUIT(circ),0); + + path = circuit_list_path_for_controller(TO_ORIGIN_CIRCUIT(circ)); + if (circ->state == CIRCUIT_STATE_OPEN) state = "BUILT"; else if (strlen(path)) @@ -1728,8 +1806,7 @@ getinfo_helper_events(control_connection_t *control_conn, } else if (!strcmp(question, "orconn-status")) { smartlist_t *conns = get_connection_array(); smartlist_t *status = smartlist_create(); - SMARTLIST_FOREACH(conns, connection_t *, base_conn, - { + SMARTLIST_FOREACH_BEGIN(conns, connection_t *, base_conn) { const char *state; char *s; char name[128]; @@ -1744,29 +1821,19 @@ getinfo_helper_events(control_connection_t *control_conn, state = "LAUNCHED"; else state = "NEW"; - orconn_target_get_name(control_conn->use_long_names, name, sizeof(name), - conn); + orconn_target_get_name(name, sizeof(name), conn); slen = strlen(name)+strlen(state)+2; s = tor_malloc(slen+1); tor_snprintf(s, slen, "%s %s", name, state); smartlist_add(status, s); - }); + } SMARTLIST_FOREACH_END(base_conn); *answer = smartlist_join_strings(status, "\r\n", 0, NULL); SMARTLIST_FOREACH(status, char *, cp, tor_free(cp)); smartlist_free(status); - } else if (!strcmpstart(question, "addr-mappings/") || - !strcmpstart(question, "address-mappings/")) { + } else if (!strcmpstart(question, "address-mappings/")) { time_t min_e, max_e; smartlist_t *mappings; - int want_expiry = !strcmpstart(question, "address-mappings/"); - if (!strcmpstart(question, "addr-mappings/")) { - /* XXXX022 This has been deprecated since 0.2.0.3-alpha, and has - generated a warning since 0.2.1.10-alpha; remove late in 0.2.2.x. */ - log_warn(LD_CONTROL, "Controller used obsolete addr-mappings/ GETINFO " - "key; use address-mappings/ instead."); - } - question += strlen(want_expiry ? "address-mappings/" - : "addr-mappings/"); + question += strlen("address-mappings/"); if (!strcmp(question, "all")) { min_e = 0; max_e = TIME_MAX; } else if (!strcmp(question, "cache")) { @@ -1779,7 +1846,7 @@ getinfo_helper_events(control_connection_t *control_conn, return 0; } mappings = smartlist_create(); - addressmap_get_mappings(mappings, min_e, max_e, want_expiry); + addressmap_get_mappings(mappings, min_e, max_e, 1); *answer = smartlist_join_strings(mappings, "\r\n", 0, NULL); SMARTLIST_FOREACH(mappings, char *, cp, tor_free(cp)); smartlist_free(mappings); @@ -1787,7 +1854,7 @@ getinfo_helper_events(control_connection_t *control_conn, /* Note that status/ is not a catch-all for events; there's only supposed * to be a status GETINFO if there's a corresponding STATUS event. */ if (!strcmp(question, "status/circuit-established")) { - *answer = tor_strdup(has_completed_circuit ? "1" : "0"); + *answer = tor_strdup(can_complete_circuit ? "1" : "0"); } else if (!strcmp(question, "status/enough-dir-info")) { *answer = tor_strdup(router_have_minimum_dir_info() ? "1" : "0"); } else if (!strcmp(question, "status/good-server-descriptor") || @@ -1846,21 +1913,12 @@ getinfo_helper_events(control_connection_t *control_conn, "information", question); } } else if (!strcmp(question, "status/clients-seen")) { - char geoip_start[ISO_TIME_LEN+1]; - size_t answer_len; - char *geoip_summary = extrainfo_get_client_geoip_summary(time(NULL)); - - if (!geoip_summary) + char *bridge_stats = geoip_get_bridge_stats_controller(time(NULL)); + if (!bridge_stats) { + *errmsg = "No bridge-client stats available"; return -1; - - answer_len = strlen("TimeStarted=\"\" CountrySummary=") + - ISO_TIME_LEN + strlen(geoip_summary) + 1; - *answer = tor_malloc(answer_len); - format_iso_time(geoip_start, geoip_get_history_start()); - tor_snprintf(*answer, answer_len, - "TimeStarted=\"%s\" CountrySummary=%s", - geoip_start, geoip_summary); - tor_free(geoip_summary); + } + *answer = bridge_stats; } else { return 0; } @@ -1870,11 +1928,14 @@ getinfo_helper_events(control_connection_t *control_conn, /** Callback function for GETINFO: on a given control connection, try to * answer the question <b>q</b> and store the newly-allocated answer in - * *<b>a</b>. If there's no answer, or an error occurs, just don't set - * <b>a</b>. Return 0. + * *<b>a</b>. If an internal error occurs, return -1 and optionally set + * *<b>error_out</b> to point to an error message to be delivered to the + * controller. On success, _or if the key is not recognized_, return 0. Do not + * set <b>a</b> if the key is not recognized. */ typedef int (*getinfo_helper_t)(control_connection_t *, - const char *q, char **a); + const char *q, char **a, + const char **error_out); /** A single item for the GETINFO question-to-answer-function table. */ typedef struct getinfo_item_t { @@ -1894,6 +1955,8 @@ typedef struct getinfo_item_t { static const getinfo_item_t getinfo_items[] = { ITEM("version", misc, "The current version of Tor."), ITEM("config-file", misc, "Current location of the \"torrc\" file."), + ITEM("config-text", misc, + "Return the string that would be written by a saveconf command."), ITEM("accounting/bytes", accounting, "Number of bytes read/written so far in the accounting interval."), ITEM("accounting/bytes-left", accounting, @@ -1924,6 +1987,7 @@ static const getinfo_item_t getinfo_items[] = { "All non-expired, non-superseded router descriptors."), ITEM("desc/all-recent-extrainfo-hack", dir, NULL), /* Hack. */ PREFIX("extra-info/digest/", dir, "Extra-info documents by digest."), + PREFIX("net/listeners/", listeners, "Bound addresses by type"), ITEM("ns/all", networkstatus, "Brief summary of router status (v2 directory format)"), PREFIX("ns/id/", networkstatus, @@ -1933,7 +1997,6 @@ static const getinfo_item_t getinfo_items[] = { PREFIX("ns/purpose/", networkstatus, "Brief summary of router status by purpose (v2 directory format)."), - PREFIX("unregistered-servers-", dirserv_unregistered, NULL), ITEM("network-status", dir, "Brief summary of router status (v1 directory format)"), ITEM("circuit-status", events, "List of current circuits originating here."), @@ -1945,14 +2008,6 @@ static const getinfo_item_t getinfo_items[] = { DOC("address-mappings/config", "Current address mappings from configuration."), DOC("address-mappings/control", "Current address mappings from controller."), - PREFIX("addr-mappings/", events, NULL), - DOC("addr-mappings/all", "Current address mappings without expiry times."), - DOC("addr-mappings/cache", - "Current cached DNS replies without expiry times."), - DOC("addr-mappings/config", - "Current address mappings from configuration without expiry times."), - DOC("addr-mappings/control", - "Current address mappings from controller without expiry times."), PREFIX("status/", events, NULL), DOC("status/circuit-established", "Whether we think client functionality is working."), @@ -1977,8 +2032,8 @@ static const getinfo_item_t getinfo_items[] = { "v2 networkstatus docs as retrieved from a DirPort."), ITEM("dir/status-vote/current/consensus", dir, "v3 Networkstatus consensus as retrieved from a DirPort."), - PREFIX("exit-policy/default", policies, - "The default value appended to the configured exit policy."), + ITEM("exit-policy/default", policies, + "The default value appended to the configured exit policy."), PREFIX("ip-to-country/", geoip, "Perform a GEOIP lookup"), { NULL, NULL, NULL, 0 } }; @@ -1988,18 +2043,18 @@ static char * list_getinfo_options(void) { int i; - char buf[300]; + char *buf=NULL; smartlist_t *lines = smartlist_create(); char *ans; for (i = 0; getinfo_items[i].varname; ++i) { if (!getinfo_items[i].desc) continue; - tor_snprintf(buf, sizeof(buf), "%s%s -- %s\n", + tor_asprintf(&buf, "%s%s -- %s\n", getinfo_items[i].varname, getinfo_items[i].is_prefix ? "*" : "", getinfo_items[i].desc); - smartlist_add(lines, tor_strdup(buf)); + smartlist_add(lines, buf); } smartlist_sort_strings(lines); @@ -2016,7 +2071,8 @@ list_getinfo_options(void) * internal error. */ static int handle_getinfo_helper(control_connection_t *control_conn, - const char *question, char **answer) + const char *question, char **answer, + const char **err_out) { int i; *answer = NULL; /* unrecognized key by default */ @@ -2029,7 +2085,7 @@ handle_getinfo_helper(control_connection_t *control_conn, match = !strcmp(question, getinfo_items[i].varname); if (match) { tor_assert(getinfo_items[i].fn); - return getinfo_items[i].fn(control_conn, question, answer); + return getinfo_items[i].fn(control_conn, question, answer, err_out); } } @@ -2051,10 +2107,12 @@ handle_control_getinfo(control_connection_t *conn, uint32_t len, smartlist_split_string(questions, body, " ", SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - SMARTLIST_FOREACH(questions, const char *, q, - { - if (handle_getinfo_helper(conn, q, &ans) < 0) { - connection_write_str_to_buf("551 Internal error\r\n", conn); + SMARTLIST_FOREACH_BEGIN(questions, const char *, q) { + const char *errmsg = NULL; + if (handle_getinfo_helper(conn, q, &ans, &errmsg) < 0) { + if (!errmsg) + errmsg = "Internal error"; + connection_printf_to_buf(conn, "551 %s\r\n", errmsg); goto done; } if (!ans) { @@ -2063,7 +2121,7 @@ handle_control_getinfo(control_connection_t *conn, uint32_t len, smartlist_add(answers, tor_strdup(q)); smartlist_add(answers, ans); } - }); + } SMARTLIST_FOREACH_END(q); if (smartlist_len(unrecognized)) { for (i=0; i < smartlist_len(unrecognized)-1; ++i) connection_printf_to_buf(conn, @@ -2108,12 +2166,12 @@ handle_control_getinfo(control_connection_t *conn, uint32_t len, static uint8_t circuit_purpose_from_string(const char *string) { - if (!strcmpstart(string, "purpose=")) + if (!strcasecmpstart(string, "purpose=")) string += strlen("purpose="); - if (!strcmp(string, "general")) + if (!strcasecmp(string, "general")) return CIRCUIT_PURPOSE_C_GENERAL; - else if (!strcmp(string, "controller")) + else if (!strcasecmp(string, "controller")) return CIRCUIT_PURPOSE_CONTROLLER; else return CIRCUIT_PURPOSE_UNKNOWN; @@ -2145,6 +2203,31 @@ getargs_helper(const char *command, control_connection_t *conn, return NULL; } +/** Helper. Return the first element of <b>sl</b> at index <b>start_at</b> or + * higher that starts with <b>prefix</b>, case-insensitive. Return NULL if no + * such element exists. */ +static const char * +find_element_starting_with(smartlist_t *sl, int start_at, const char *prefix) +{ + int i; + for (i = start_at; i < smartlist_len(sl); ++i) { + const char *elt = smartlist_get(sl, i); + if (!strcasecmpstart(elt, prefix)) + return elt; + } + return NULL; +} + +/** Helper. Return true iff s is an argument that we should treat as a + * key-value pair. */ +static int +is_keyval_pair(const char *s) +{ + /* An argument is a key-value pair if it has an =, and it isn't of the form + * $fingeprint=name */ + return strchr(s, '=') && s[0] != '$'; +} + /** Called when we get an EXTENDCIRCUIT message. Try to extend the listed * circuit, and report success or failure. */ static int @@ -2160,33 +2243,57 @@ handle_control_extendcircuit(control_connection_t *conn, uint32_t len, router_nicknames = smartlist_create(); - args = getargs_helper("EXTENDCIRCUIT", conn, body, 2, -1); + args = getargs_helper("EXTENDCIRCUIT", conn, body, 1, -1); if (!args) goto done; zero_circ = !strcmp("0", (char*)smartlist_get(args,0)); - if (!zero_circ && !(circ = get_circ(smartlist_get(args,0)))) { - connection_printf_to_buf(conn, "552 Unknown circuit \"%s\"\r\n", - (char*)smartlist_get(args, 0)); - } - smartlist_split_string(router_nicknames, smartlist_get(args,1), ",", 0, 0); - if (zero_circ && smartlist_len(args)>2) { - char *purp = smartlist_get(args,2); - intended_purpose = circuit_purpose_from_string(purp); - if (intended_purpose == CIRCUIT_PURPOSE_UNKNOWN) { - connection_printf_to_buf(conn, "552 Unknown purpose \"%s\"\r\n", purp); + if (zero_circ) { + const char *purp = find_element_starting_with(args, 1, "PURPOSE="); + + if (purp) { + intended_purpose = circuit_purpose_from_string(purp); + if (intended_purpose == CIRCUIT_PURPOSE_UNKNOWN) { + connection_printf_to_buf(conn, "552 Unknown purpose \"%s\"\r\n", purp); + SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); + smartlist_free(args); + goto done; + } + } + + if ((smartlist_len(args) == 1) || + (smartlist_len(args) >= 2 && is_keyval_pair(smartlist_get(args, 1)))) { + // "EXTENDCIRCUIT 0" || EXTENDCIRCUIT 0 foo=bar" + circ = circuit_launch_by_router(intended_purpose, NULL, + CIRCLAUNCH_NEED_CAPACITY); + if (!circ) { + connection_write_str_to_buf("551 Couldn't start circuit\r\n", conn); + } else { + connection_printf_to_buf(conn, "250 EXTENDED %lu\r\n", + (unsigned long)circ->global_identifier); + } SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); smartlist_free(args); goto done; } + // "EXTENDCIRCUIT 0 router1,router2" || + // "EXTENDCIRCUIT 0 router1,router2 PURPOSE=foo" } - SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); - smartlist_free(args); - if (!zero_circ && !circ) { + + if (!zero_circ && !(circ = get_circ(smartlist_get(args,0)))) { + connection_printf_to_buf(conn, "552 Unknown circuit \"%s\"\r\n", + (char*)smartlist_get(args, 0)); + SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); + smartlist_free(args); goto done; } + smartlist_split_string(router_nicknames, smartlist_get(args,1), ",", 0, 0); + + SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); + smartlist_free(args); + routers = smartlist_create(); SMARTLIST_FOREACH(router_nicknames, const char *, n, { @@ -2244,8 +2351,7 @@ handle_control_extendcircuit(control_connection_t *conn, uint32_t len, done: SMARTLIST_FOREACH(router_nicknames, char *, n, tor_free(n)); smartlist_free(router_nicknames); - if (routers) - smartlist_free(routers); + smartlist_free(routers); return 0; } @@ -2271,7 +2377,7 @@ handle_control_setcircuitpurpose(control_connection_t *conn, } { - char *purp = smartlist_get(args,1); + const char *purp = find_element_starting_with(args,1,"PURPOSE="); new_purpose = circuit_purpose_from_string(purp); if (new_purpose == CIRCUIT_PURPOSE_UNKNOWN) { connection_printf_to_buf(conn, "552 Unknown purpose \"%s\"\r\n", purp); @@ -2282,7 +2388,7 @@ handle_control_setcircuitpurpose(control_connection_t *conn, circ->_base.purpose = new_purpose; connection_write_str_to_buf("250 OK\r\n", conn); -done: + done: if (args) { SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); smartlist_free(args); @@ -2316,9 +2422,9 @@ handle_control_attachstream(control_connection_t *conn, uint32_t len, } else if (!zero_circ && !(circ = get_circ(smartlist_get(args, 1)))) { connection_printf_to_buf(conn, "552 Unknown circuit \"%s\"\r\n", (char*)smartlist_get(args, 1)); - } else if (circ && smartlist_len(args) > 2) { - char *hopstring = smartlist_get(args, 2); - if (!strcasecmpstart(hopstring, "HOP=")) { + } else if (circ) { + const char *hopstring = find_element_starting_with(args,2,"HOP="); + if (hopstring) { hopstring += strlen("HOP="); hop = (int) tor_parse_ulong(hopstring, 10, 0, INT_MAX, &hop_line_ok, NULL); @@ -2365,7 +2471,7 @@ handle_control_attachstream(control_connection_t *conn, uint32_t len, char* exit_digest; if (circ->build_state && circ->build_state->chosen_exit && - circ->build_state->chosen_exit->identity_digest) { + !tor_digest_is_zero(circ->build_state->chosen_exit->identity_digest)) { exit_digest = circ->build_state->chosen_exit->identity_digest; r = router_get_by_digest(exit_digest); } @@ -2426,9 +2532,9 @@ handle_control_postdescriptor(control_connection_t *conn, uint32_t len, } } else if (!strcasecmpstart(option, "cache=")) { option += strlen("cache="); - if (!strcmp(option, "no")) + if (!strcasecmp(option, "no")) cache = 0; - else if (!strcmp(option, "yes")) + else if (!strcasecmp(option, "yes")) cache = 1; else { connection_printf_to_buf(conn, "552 Unknown cache request \"%s\"\r\n", @@ -2610,17 +2716,17 @@ handle_control_resolve(control_connection_t *conn, uint32_t len, args = smartlist_create(); smartlist_split_string(args, body, " ", SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - if (smartlist_len(args) && - !strcasecmp(smartlist_get(args, 0), "mode=reverse")) { - char *cp = smartlist_get(args, 0); - smartlist_del_keeporder(args, 0); - tor_free(cp); - is_reverse = 1; + { + const char *modearg = find_element_starting_with(args, 0, "mode="); + if (modearg && !strcasecmp(modearg, "mode=reverse")) + is_reverse = 1; } failed = smartlist_create(); SMARTLIST_FOREACH(args, const char *, arg, { - if (dnsserv_launch_request(arg, is_reverse)<0) - smartlist_add(failed, (char*)arg); + if (!is_keyval_pair(arg)) { + if (dnsserv_launch_request(arg, is_reverse)<0) + smartlist_add(failed, (char*)arg); + } }); send_control_done(conn); @@ -2710,7 +2816,6 @@ handle_control_usefeature(control_connection_t *conn, const char *body) { smartlist_t *args; - int verbose_names = 0, extended_events = 0; int bad = 0; (void) len; /* body is nul-terminated; it's safe to ignore the length */ args = smartlist_create(); @@ -2718,9 +2823,9 @@ handle_control_usefeature(control_connection_t *conn, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); SMARTLIST_FOREACH(args, const char *, arg, { if (!strcasecmp(arg, "VERBOSE_NAMES")) - verbose_names = 1; + ; else if (!strcasecmp(arg, "EXTENDED_EVENTS")) - extended_events = 1; + ; else { connection_printf_to_buf(conn, "552 Unrecognized feature \"%s\"\r\n", arg); @@ -2730,12 +2835,6 @@ handle_control_usefeature(control_connection_t *conn, }); if (!bad) { - if (verbose_names) { - conn->use_long_names = 1; - control_update_global_event_mask(); - } - if (extended_events) - conn->use_extended_events = 1; send_control_done(conn); } @@ -2765,6 +2864,43 @@ connection_control_reached_eof(control_connection_t *conn) return 0; } +/** Shut down this Tor instance in the same way that SIGINT would, but + * with a log message appropriate for the loss of an owning controller. */ +static void +lost_owning_controller(const char *owner_type, const char *loss_manner) +{ + int shutdown_slowly = server_mode(get_options()); + + log_notice(LD_CONTROL, "Owning controller %s has %s -- %s.", + owner_type, loss_manner, + shutdown_slowly ? "shutting down" : "exiting now"); + + /* XXXX Perhaps this chunk of code should be a separate function, + * called here and by process_signal(SIGINT). */ + + if (!shutdown_slowly) { + tor_cleanup(); + exit(0); + } + /* XXXX This will close all listening sockets except control-port + * listeners. Perhaps we should close those too. */ + hibernate_begin_shutdown(); +} + +/** Called when <b>conn</b> is being freed. */ +void +connection_control_closed(control_connection_t *conn) +{ + tor_assert(conn); + + conn->event_mask = 0; + control_update_global_event_mask(); + + if (conn->is_owning_control_connection) { + lost_owning_controller("connection", "closed"); + } +} + /** Return true iff <b>cmd</b> is allowable (or at least forgivable) at this * stage of the protocol. */ static int @@ -2885,9 +3021,10 @@ connection_control_process_inbuf(control_connection_t *conn) && !TOR_ISSPACE(conn->incoming_cmd[cmd_len])) ++cmd_len; - data_len -= cmd_len; conn->incoming_cmd[cmd_len]='\0'; args = conn->incoming_cmd+cmd_len+1; + tor_assert(data_len>(size_t)cmd_len); + data_len -= (cmd_len+1); /* skip the command and NUL we added after it */ while (*args == ' ' || *args == '\t') { ++args; --data_len; @@ -2919,6 +3056,9 @@ connection_control_process_inbuf(control_connection_t *conn) return 0; } + /* XXXX Why is this not implemented as a table like the GETINFO + * items are? Even handling the plus signs at the beginnings of + * commands wouldn't be very hard with proper macros. */ cmd_data_len = (uint32_t)data_len; if (!strcasecmp(conn->incoming_cmd, "SETCONF")) { if (handle_control_setconf(conn, cmd_data_len, args)) @@ -2944,6 +3084,9 @@ connection_control_process_inbuf(control_connection_t *conn) } else if (!strcasecmp(conn->incoming_cmd, "SIGNAL")) { if (handle_control_signal(conn, cmd_data_len, args)) return -1; + } else if (!strcasecmp(conn->incoming_cmd, "TAKEOWNERSHIP")) { + if (handle_control_takeownership(conn, cmd_data_len, args)) + return -1; } else if (!strcasecmp(conn->incoming_cmd, "MAPADDRESS")) { if (handle_control_mapaddress(conn, cmd_data_len, args)) return -1; @@ -2999,7 +3142,6 @@ control_event_circuit_status(origin_circuit_t *circ, circuit_status_event_t tp, { const char *status; char extended_buf[96]; - int providing_reason=0; if (!EVENT_IS_INTERESTING(EVENT_CIRCUIT_STATUS)) return 0; tor_assert(circ); @@ -3023,7 +3165,6 @@ control_event_circuit_status(origin_circuit_t *circ, circuit_status_event_t tp, const char *reason_str = circuit_end_reason_to_control_string(reason_code); char *reason = NULL; size_t n=strlen(extended_buf); - providing_reason=1; if (!reason_str) { reason = tor_malloc(16); tor_snprintf(reason, 16, "UNKNOWN_%d", reason_code); @@ -3039,20 +3180,11 @@ control_event_circuit_status(origin_circuit_t *circ, circuit_status_event_t tp, tor_free(reason); } - if (EVENT_IS_INTERESTING1S(EVENT_CIRCUIT_STATUS)) { - char *path = circuit_list_path(circ,0); - const char *sp = strlen(path) ? " " : ""; - send_control_event_extended(EVENT_CIRCUIT_STATUS, SHORT_NAMES, - "650 CIRC %lu %s%s%s@%s\r\n", - (unsigned long)circ->global_identifier, - status, sp, path, extended_buf); - tor_free(path); - } - if (EVENT_IS_INTERESTING1L(EVENT_CIRCUIT_STATUS)) { + { char *vpath = circuit_list_path_for_controller(circ); const char *sp = strlen(vpath) ? " " : ""; - send_control_event_extended(EVENT_CIRCUIT_STATUS, LONG_NAMES, - "650 CIRC %lu %s%s%s@%s\r\n", + send_control_event(EVENT_CIRCUIT_STATUS, ALL_FORMATS, + "650 CIRC %lu %s%s%s %s\r\n", (unsigned long)circ->global_identifier, status, sp, vpath, extended_buf); tor_free(vpath); @@ -3131,26 +3263,26 @@ control_event_stream_status(edge_connection_t *conn, stream_status_event_t tp, char *r = NULL; if (!reason_str) { r = tor_malloc(16); - tor_snprintf(r, 16, "UNKNOWN_%d", reason_code); + tor_snprintf(r, 16, " UNKNOWN_%d", reason_code); reason_str = r; } if (reason_code & END_STREAM_REASON_FLAG_REMOTE) tor_snprintf(reason_buf, sizeof(reason_buf), - "REASON=END REMOTE_REASON=%s", reason_str); + " REASON=END REMOTE_REASON=%s", reason_str); else tor_snprintf(reason_buf, sizeof(reason_buf), - "REASON=%s", reason_str); + " REASON=%s", reason_str); tor_free(r); } else if (reason_code && tp == STREAM_EVENT_REMAP) { switch (reason_code) { case REMAP_STREAM_SOURCE_CACHE: - strlcpy(reason_buf, "SOURCE=CACHE", sizeof(reason_buf)); + strlcpy(reason_buf, " SOURCE=CACHE", sizeof(reason_buf)); break; case REMAP_STREAM_SOURCE_EXIT: - strlcpy(reason_buf, "SOURCE=EXIT", sizeof(reason_buf)); + strlcpy(reason_buf, " SOURCE=EXIT", sizeof(reason_buf)); break; default: - tor_snprintf(reason_buf, sizeof(reason_buf), "REASON=UNKNOWN_%d", + tor_snprintf(reason_buf, sizeof(reason_buf), " REASON=UNKNOWN_%d", reason_code); /* XXX do we want SOURCE=UNKNOWN_%d above instead? -RD */ break; @@ -3158,8 +3290,7 @@ control_event_stream_status(edge_connection_t *conn, stream_status_event_t tp, } if (tp == STREAM_EVENT_NEW) { - tor_snprintf(addrport_buf,sizeof(addrport_buf), "%sSOURCE_ADDR=%s:%d", - strlen(reason_buf) ? " " : "", + tor_snprintf(addrport_buf,sizeof(addrport_buf), " SOURCE_ADDR=%s:%d", TO_CONN(conn)->address, TO_CONN(conn)->port ); } else { addrport_buf[0] = '\0'; @@ -3188,8 +3319,8 @@ control_event_stream_status(edge_connection_t *conn, stream_status_event_t tp, circ = circuit_get_by_edge_conn(conn); if (circ && CIRCUIT_IS_ORIGIN(circ)) origin_circ = TO_ORIGIN_CIRCUIT(circ); - send_control_event_extended(EVENT_STREAM_STATUS, ALL_NAMES, - "650 STREAM "U64_FORMAT" %s %lu %s@%s%s%s\r\n", + send_control_event(EVENT_STREAM_STATUS, ALL_FORMATS, + "650 STREAM "U64_FORMAT" %s %lu %s%s%s%s\r\n", U64_PRINTF_ARG(conn->_base.global_identifier), status, origin_circ? (unsigned long)origin_circ->global_identifier : 0ul, @@ -3202,30 +3333,21 @@ control_event_stream_status(edge_connection_t *conn, stream_status_event_t tp, /** Figure out the best name for the target router of an OR connection * <b>conn</b>, and write it into the <b>len</b>-character buffer - * <b>name</b>. Use verbose names if <b>long_names</b> is set. */ + * <b>name</b>. */ static void -orconn_target_get_name(int long_names, - char *name, size_t len, or_connection_t *conn) +orconn_target_get_name(char *name, size_t len, or_connection_t *conn) { - if (! long_names) { - if (conn->nickname) - strlcpy(name, conn->nickname, len); - else - tor_snprintf(name, len, "%s:%d", - conn->_base.address, conn->_base.port); + routerinfo_t *ri = router_get_by_digest(conn->identity_digest); + if (ri) { + tor_assert(len > MAX_VERBOSE_NICKNAME_LEN); + router_get_verbose_nickname(name, ri); + } else if (! tor_digest_is_zero(conn->identity_digest)) { + name[0] = '$'; + base16_encode(name+1, len-1, conn->identity_digest, + DIGEST_LEN); } else { - routerinfo_t *ri = router_get_by_digest(conn->identity_digest); - if (ri) { - tor_assert(len > MAX_VERBOSE_NICKNAME_LEN); - router_get_verbose_nickname(name, ri); - } else if (! tor_digest_is_zero(conn->identity_digest)) { - name[0] = '$'; - base16_encode(name+1, len-1, conn->identity_digest, - DIGEST_LEN); - } else { - tor_snprintf(name, len, "%s:%d", - conn->_base.address, conn->_base.port); - } + tor_snprintf(name, len, "%s:%d", + conn->_base.address, conn->_base.port); } } @@ -3264,24 +3386,13 @@ control_event_or_conn_status(or_connection_t *conn, or_conn_status_event_t tp, reason ? " " : "", ncircs); } - if (EVENT_IS_INTERESTING1S(EVENT_OR_CONN_STATUS)) { - orconn_target_get_name(0, name, sizeof(name), conn); - send_control_event_extended(EVENT_OR_CONN_STATUS, SHORT_NAMES, - "650 ORCONN %s %s@%s%s%s\r\n", - name, status, - reason ? "REASON=" : "", - orconn_end_reason_to_control_string(reason), - ncircs_buf); - } - if (EVENT_IS_INTERESTING1L(EVENT_OR_CONN_STATUS)) { - orconn_target_get_name(1, name, sizeof(name), conn); - send_control_event_extended(EVENT_OR_CONN_STATUS, LONG_NAMES, - "650 ORCONN %s %s@%s%s%s\r\n", - name, status, - reason ? "REASON=" : "", - orconn_end_reason_to_control_string(reason), - ncircs_buf); - } + orconn_target_get_name(name, sizeof(name), conn); + send_control_event(EVENT_OR_CONN_STATUS, ALL_FORMATS, + "650 ORCONN %s %s %s%s%s\r\n", + name, status, + reason ? "REASON=" : "", + orconn_end_reason_to_control_string(reason), + ncircs_buf); return 0; } @@ -3296,7 +3407,7 @@ control_event_stream_bandwidth(edge_connection_t *edge_conn) if (!edge_conn->n_read && !edge_conn->n_written) return 0; - send_control_event(EVENT_STREAM_BANDWIDTH_USED, ALL_NAMES, + send_control_event(EVENT_STREAM_BANDWIDTH_USED, ALL_FORMATS, "650 STREAM_BW "U64_FORMAT" %lu %lu\r\n", U64_PRINTF_ARG(edge_conn->_base.global_identifier), (unsigned long)edge_conn->n_read, @@ -3325,7 +3436,7 @@ control_event_stream_bandwidth_used(void) if (!edge_conn->n_read && !edge_conn->n_written) continue; - send_control_event(EVENT_STREAM_BANDWIDTH_USED, ALL_NAMES, + send_control_event(EVENT_STREAM_BANDWIDTH_USED, ALL_FORMATS, "650 STREAM_BW "U64_FORMAT" %lu %lu\r\n", U64_PRINTF_ARG(edge_conn->_base.global_identifier), (unsigned long)edge_conn->n_read, @@ -3345,7 +3456,7 @@ int control_event_bandwidth_used(uint32_t n_read, uint32_t n_written) { if (EVENT_IS_INTERESTING(EVENT_BANDWIDTH_USED)) { - send_control_event(EVENT_BANDWIDTH_USED, ALL_NAMES, + send_control_event(EVENT_BANDWIDTH_USED, ALL_FORMATS, "650 BW %lu %lu\r\n", (unsigned long)n_read, (unsigned long)n_written); @@ -3415,7 +3526,7 @@ control_event_logmsg(int severity, uint32_t domain, const char *msg) default: s = "UnknownLogSeverity"; break; } ++disable_log_messages; - send_control_event(event, ALL_NAMES, "650 %s %s\r\n", s, b?b:msg); + send_control_event(event, ALL_FORMATS, "650 %s %s\r\n", s, b?b:msg); --disable_log_messages; tor_free(b); } @@ -3428,31 +3539,12 @@ control_event_logmsg(int severity, uint32_t domain, const char *msg) int control_event_descriptors_changed(smartlist_t *routers) { - size_t len; char *msg; - smartlist_t *identities = NULL; - char buf[HEX_DIGEST_LEN+1]; if (!EVENT_IS_INTERESTING(EVENT_NEW_DESC)) return 0; - if (EVENT_IS_INTERESTING1S(EVENT_NEW_DESC)) { - identities = smartlist_create(); - SMARTLIST_FOREACH(routers, routerinfo_t *, r, - { - base16_encode(buf,sizeof(buf),r->cache_info.identity_digest,DIGEST_LEN); - smartlist_add(identities, tor_strdup(buf)); - }); - } - if (EVENT_IS_INTERESTING1S(EVENT_NEW_DESC)) { - char *ids = smartlist_join_strings(identities, " ", 0, &len); - size_t ids_len = strlen(ids)+32; - msg = tor_malloc(ids_len); - tor_snprintf(msg, ids_len, "650 NEWDESC %s\r\n", ids); - send_control_event_string(EVENT_NEW_DESC, SHORT_NAMES|ALL_FORMATS, msg); - tor_free(ids); - tor_free(msg); - } - if (EVENT_IS_INTERESTING1L(EVENT_NEW_DESC)) { + + { smartlist_t *names = smartlist_create(); char *ids; size_t names_len; @@ -3465,16 +3557,12 @@ control_event_descriptors_changed(smartlist_t *routers) names_len = strlen(ids)+32; msg = tor_malloc(names_len); tor_snprintf(msg, names_len, "650 NEWDESC %s\r\n", ids); - send_control_event_string(EVENT_NEW_DESC, LONG_NAMES|ALL_FORMATS, msg); + send_control_event_string(EVENT_NEW_DESC, ALL_FORMATS, msg); tor_free(ids); tor_free(msg); SMARTLIST_FOREACH(names, char *, cp, tor_free(cp)); smartlist_free(names); } - if (identities) { - SMARTLIST_FOREACH(identities, char *, cp, tor_free(cp)); - smartlist_free(identities); - } return 0; } @@ -3491,17 +3579,17 @@ control_event_address_mapped(const char *from, const char *to, time_t expires, return 0; if (expires < 3 || expires == TIME_MAX) - send_control_event_extended(EVENT_ADDRMAP, ALL_NAMES, - "650 ADDRMAP %s %s NEVER@%s\r\n", from, to, + send_control_event(EVENT_ADDRMAP, ALL_FORMATS, + "650 ADDRMAP %s %s NEVER %s\r\n", from, to, error?error:""); else { char buf[ISO_TIME_LEN+1]; char buf2[ISO_TIME_LEN+1]; format_local_iso_time(buf,expires); format_iso_time(buf2,expires); - send_control_event_extended(EVENT_ADDRMAP, ALL_NAMES, + send_control_event(EVENT_ADDRMAP, ALL_FORMATS, "650 ADDRMAP %s %s \"%s\"" - "@%s%sEXPIRES=\"%s\"\r\n", + " %s%sEXPIRES=\"%s\"\r\n", from, to, buf, error?error:"", error?" ":"", buf2); @@ -3541,9 +3629,9 @@ control_event_or_authdir_new_descriptor(const char *action, buf = tor_malloc(totallen); strlcpy(buf, firstline, totallen); strlcpy(buf+strlen(firstline), esc, totallen); - send_control_event_string(EVENT_AUTHDIR_NEWDESCS, ALL_NAMES|ALL_FORMATS, + send_control_event_string(EVENT_AUTHDIR_NEWDESCS, ALL_FORMATS, buf); - send_control_event_string(EVENT_AUTHDIR_NEWDESCS, ALL_NAMES|ALL_FORMATS, + send_control_event_string(EVENT_AUTHDIR_NEWDESCS, ALL_FORMATS, "650 OK\r\n"); tor_free(esc); tor_free(buf); @@ -3581,8 +3669,8 @@ control_event_networkstatus_changed_helper(smartlist_t *statuses, SMARTLIST_FOREACH(strs, char *, cp, tor_free(cp)); smartlist_free(strs); tor_free(s); - send_control_event_string(event, ALL_NAMES|ALL_FORMATS, esc); - send_control_event_string(event, ALL_NAMES|ALL_FORMATS, + send_control_event_string(event, ALL_FORMATS, esc); + send_control_event_string(event, ALL_FORMATS, "650 OK\r\n"); tor_free(esc); @@ -3608,6 +3696,55 @@ control_event_newconsensus(const networkstatus_t *consensus) consensus->routerstatus_list, EVENT_NEWCONSENSUS, "NEWCONSENSUS"); } +/** Called when we compute a new circuitbuildtimeout */ +int +control_event_buildtimeout_set(const circuit_build_times_t *cbt, + buildtimeout_set_event_t type) +{ + const char *type_string = NULL; + double qnt = circuit_build_times_quantile_cutoff(); + + if (!control_event_is_interesting(EVENT_BUILDTIMEOUT_SET)) + return 0; + + switch (type) { + case BUILDTIMEOUT_SET_EVENT_COMPUTED: + type_string = "COMPUTED"; + break; + case BUILDTIMEOUT_SET_EVENT_RESET: + type_string = "RESET"; + qnt = 1.0; + break; + case BUILDTIMEOUT_SET_EVENT_SUSPENDED: + type_string = "SUSPENDED"; + qnt = 1.0; + break; + case BUILDTIMEOUT_SET_EVENT_DISCARD: + type_string = "DISCARD"; + qnt = 1.0; + break; + case BUILDTIMEOUT_SET_EVENT_RESUME: + type_string = "RESUME"; + break; + default: + type_string = "UNKNOWN"; + break; + } + + send_control_event(EVENT_BUILDTIMEOUT_SET, ALL_FORMATS, + "650 BUILDTIMEOUT_SET %s TOTAL_TIMES=%lu " + "TIMEOUT_MS=%lu XM=%lu ALPHA=%lf CUTOFF_QUANTILE=%lf " + "TIMEOUT_RATE=%lf CLOSE_MS=%lu CLOSE_RATE=%lf\r\n", + type_string, (unsigned long)cbt->total_build_times, + (unsigned long)cbt->timeout_ms, + (unsigned long)cbt->Xm, cbt->alpha, qnt, + circuit_build_times_timeout_rate(cbt), + (unsigned long)cbt->close_ms, + circuit_build_times_close_rate(cbt)); + + return 0; +} + /** Called when a single local_routerstatus_t has changed: Sends an NS event * to any controller that cares. */ int @@ -3631,7 +3768,7 @@ control_event_networkstatus_changed_single(routerstatus_t *rs) int control_event_my_descriptor_changed(void) { - send_control_event(EVENT_DESCCHANGED, ALL_NAMES, "650 DESCCHANGED\r\n"); + send_control_event(EVENT_DESCCHANGED, ALL_FORMATS, "650 DESCCHANGED\r\n"); return 0; } @@ -3679,7 +3816,7 @@ control_event_status(int type, int severity, const char *format, va_list args) return -1; } - send_control_event_impl(type, ALL_NAMES|ALL_FORMATS, 0, format_buf, args); + send_control_event_impl(type, ALL_FORMATS, format_buf, args); return 0; } @@ -3743,7 +3880,7 @@ control_event_guard(const char *nickname, const char *digest, if (!EVENT_IS_INTERESTING(EVENT_GUARD)) return 0; - if (EVENT_IS_INTERESTING1L(EVENT_GUARD)) { + { char buf[MAX_VERBOSE_NICKNAME_LEN+1]; routerinfo_t *ri = router_get_by_digest(digest); if (ri) { @@ -3751,13 +3888,9 @@ control_event_guard(const char *nickname, const char *digest, } else { tor_snprintf(buf, sizeof(buf), "$%s~%s", hbuf, nickname); } - send_control_event(EVENT_GUARD, LONG_NAMES, + send_control_event(EVENT_GUARD, ALL_FORMATS, "650 GUARD ENTRY %s %s\r\n", buf, status); } - if (EVENT_IS_INTERESTING1S(EVENT_GUARD)) { - send_control_event(EVENT_GUARD, SHORT_NAMES, - "650 GUARD ENTRY $%s %s\r\n", hbuf, status); - } return 0; } @@ -3814,6 +3947,75 @@ init_cookie_authentication(int enabled) return 0; } +/** A copy of the process specifier of Tor's owning controller, or + * NULL if this Tor instance is not currently owned by a process. */ +static char *owning_controller_process_spec = NULL; + +/** A process-termination monitor for Tor's owning controller, or NULL + * if this Tor instance is not currently owned by a process. */ +static tor_process_monitor_t *owning_controller_process_monitor = NULL; + +/** Process-termination monitor callback for Tor's owning controller + * process. */ +static void +owning_controller_procmon_cb(void *unused) +{ + (void)unused; + + lost_owning_controller("process", "vanished"); +} + +/** Set <b>process_spec</b> as Tor's owning controller process. + * Exit on failure. */ +void +monitor_owning_controller_process(const char *process_spec) +{ + const char *msg; + + tor_assert((owning_controller_process_spec == NULL) == + (owning_controller_process_monitor == NULL)); + + if (owning_controller_process_spec != NULL) { + if ((process_spec != NULL) && !strcmp(process_spec, + owning_controller_process_spec)) { + /* Same process -- return now, instead of disposing of and + * recreating the process-termination monitor. */ + return; + } + + /* We are currently owned by a process, and we should no longer be + * owned by it. Free the process-termination monitor. */ + tor_process_monitor_free(owning_controller_process_monitor); + owning_controller_process_monitor = NULL; + + tor_free(owning_controller_process_spec); + owning_controller_process_spec = NULL; + } + + tor_assert((owning_controller_process_spec == NULL) && + (owning_controller_process_monitor == NULL)); + + if (process_spec == NULL) + return; + + owning_controller_process_spec = tor_strdup(process_spec); + owning_controller_process_monitor = + tor_process_monitor_new(tor_libevent_get_base(), + owning_controller_process_spec, + LD_CONTROL, + owning_controller_procmon_cb, NULL, + &msg); + + if (owning_controller_process_monitor == NULL) { + log_err(LD_BUG, "Couldn't create process-termination monitor for " + "owning controller: %s. Exiting.", + msg); + owning_controller_process_spec = NULL; + tor_cleanup(); + exit(0); + } +} + /** Convert the name of a bootstrapping phase <b>s</b> into strings * <b>tag</b> and <b>summary</b> suitable for display by the controller. */ static int @@ -3910,7 +4112,7 @@ static int bootstrap_problems = 0; * information and initial circuits. * * <b>status</b> is the new status, that is, what task we will be doing - * next. <b>percent</b> is zero if we just started this task, else it + * next. <b>progress</b> is zero if we just started this task, else it * represents progress on the task. */ void control_event_bootstrap(bootstrap_status_t status, int progress) @@ -3966,6 +4168,9 @@ control_event_bootstrap_problem(const char *warn, int reason) char buf[BOOTSTRAP_MSG_LEN]; const char *recommendation = "ignore"; + /* bootstrap_percent must not be in "undefined" state here. */ + tor_assert(status >= 0); + if (bootstrap_percent == 100) return; /* already bootstrapped; nothing to be done here. */ @@ -4008,10 +4213,9 @@ control_event_bootstrap_problem(const char *warn, int reason) * from recently. Send a copy to the controller in case it wants to * display it for the user. */ void -control_event_clients_seen(const char *timestarted, const char *countries) +control_event_clients_seen(const char *controller_str) { send_control_event(EVENT_CLIENTS_SEEN, 0, - "650 CLIENTS_SEEN TimeStarted=\"%s\" CountrySummary=%s\r\n", - timestarted, countries); + "650 CLIENTS_SEEN %s\r\n", controller_str); } diff --git a/src/or/control.h b/src/or/control.h new file mode 100644 index 0000000000..ddea4cd548 --- /dev/null +++ b/src/or/control.h @@ -0,0 +1,91 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file control.h + * \brief Header file for control.c. + **/ + +#ifndef _TOR_CONTROL_H +#define _TOR_CONTROL_H + +void control_update_global_event_mask(void); +void control_adjust_event_log_severity(void); + +void control_ports_write_to_file(void); + +/** Log information about the connection <b>conn</b>, protecting it as with + * CONN_LOG_PROTECT. Example: + * + * LOG_FN_CONN(conn, (LOG_DEBUG, "Socket %d wants to write", conn->s)); + **/ +#define LOG_FN_CONN(conn, args) \ + CONN_LOG_PROTECT(conn, log_fn args) + +int connection_control_finished_flushing(control_connection_t *conn); +int connection_control_reached_eof(control_connection_t *conn); +void connection_control_closed(control_connection_t *conn); + +int connection_control_process_inbuf(control_connection_t *conn); + +#define EVENT_AUTHDIR_NEWDESCS 0x000D +#define EVENT_NS 0x000F +int control_event_is_interesting(int event); + +int control_event_circuit_status(origin_circuit_t *circ, + circuit_status_event_t e, int reason); +int control_event_stream_status(edge_connection_t *conn, + stream_status_event_t e, + int reason); +int control_event_or_conn_status(or_connection_t *conn, + or_conn_status_event_t e, int reason); +int control_event_bandwidth_used(uint32_t n_read, uint32_t n_written); +int control_event_stream_bandwidth(edge_connection_t *edge_conn); +int control_event_stream_bandwidth_used(void); +void control_event_logmsg(int severity, unsigned int domain, const char *msg); +int control_event_descriptors_changed(smartlist_t *routers); +int control_event_address_mapped(const char *from, const char *to, + time_t expires, const char *error); +int control_event_or_authdir_new_descriptor(const char *action, + const char *desc, + size_t desclen, + const char *msg); +int control_event_my_descriptor_changed(void); +int control_event_networkstatus_changed(smartlist_t *statuses); + +int control_event_newconsensus(const networkstatus_t *consensus); +int control_event_networkstatus_changed_single(routerstatus_t *rs); +int control_event_general_status(int severity, const char *format, ...) + CHECK_PRINTF(2,3); +int control_event_client_status(int severity, const char *format, ...) + CHECK_PRINTF(2,3); +int control_event_server_status(int severity, const char *format, ...) + CHECK_PRINTF(2,3); +int control_event_guard(const char *nickname, const char *digest, + const char *status); +int control_event_buildtimeout_set(const circuit_build_times_t *cbt, + buildtimeout_set_event_t type); + +int init_cookie_authentication(int enabled); +smartlist_t *decode_hashed_passwords(config_line_t *passwords); +void disable_control_logging(void); +void enable_control_logging(void); + +void monitor_owning_controller_process(const char *process_spec); + +void control_event_bootstrap(bootstrap_status_t status, int progress); +void control_event_bootstrap_problem(const char *warn, int reason); + +void control_event_clients_seen(const char *controller_str); + +#ifdef CONTROL_PRIVATE +/* Used only by control.c and test.c */ +size_t write_escaped_data(const char *data, size_t len, char **out); +size_t read_escaped_data(const char *data, size_t len, char **out); +#endif + +#endif + diff --git a/src/or/cpuworker.c b/src/or/cpuworker.c index 91af5f1d61..7cbc191333 100644 --- a/src/or/cpuworker.c +++ b/src/or/cpuworker.c @@ -13,6 +13,15 @@ **/ #include "or.h" +#include "buffers.h" +#include "circuitbuild.h" +#include "circuitlist.h" +#include "config.h" +#include "connection.h" +#include "cpuworker.h" +#include "main.h" +#include "onion.h" +#include "router.h" /** The maximum number of cpuworker processes we will keep around. */ #define MAX_CPUWORKERS 16 @@ -183,7 +192,7 @@ connection_cpu_process_inbuf(connection_t *conn) tor_assert(0); /* don't ask me to do handshakes yet */ } -done_processing: + done_processing: conn->state = CPUWORKER_STATE_IDLE; num_cpuworkers_busy--; if (conn->timestamp_created < last_rotation_time) { @@ -241,7 +250,7 @@ cpuworker_main(void *data) for (;;) { ssize_t r; - if ((r = recv(fd, &question_type, 1, 0)) != 1) { + if ((r = recv(fd, (void *)&question_type, 1, 0)) != 1) { // log_fn(LOG_ERR,"read type failed. Exiting."); if (r == 0) { log_info(LD_OR, @@ -358,7 +367,7 @@ spawn_cpuworker(void) static void spawn_enough_cpuworkers(void) { - int num_cpuworkers_needed = get_options()->NumCpus; + int num_cpuworkers_needed = get_options()->NumCPUs; if (num_cpuworkers_needed < MIN_CPUWORKERS) num_cpuworkers_needed = MIN_CPUWORKERS; diff --git a/src/or/cpuworker.h b/src/or/cpuworker.h new file mode 100644 index 0000000000..04e37ee459 --- /dev/null +++ b/src/or/cpuworker.h @@ -0,0 +1,25 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file cpuworker.h + * \brief Header file for cpuworker.c. + **/ + +#ifndef _TOR_CPUWORKER_H +#define _TOR_CPUWORKER_H + +void cpu_init(void); +void cpuworkers_rotate(void); +int connection_cpu_finished_flushing(connection_t *conn); +int connection_cpu_reached_eof(connection_t *conn); +int connection_cpu_process_inbuf(connection_t *conn); +int assign_onionskin_to_cpuworker(connection_t *cpuworker, + or_circuit_t *circ, + char *onionskin); + +#endif + diff --git a/src/or/directory.c b/src/or/directory.c index 9f9b2c1577..ff0a5a427b 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -4,6 +4,26 @@ /* See LICENSE for licensing information */ #include "or.h" +#include "buffers.h" +#include "circuitbuild.h" +#include "config.h" +#include "connection.h" +#include "connection_edge.h" +#include "control.h" +#include "directory.h" +#include "dirserv.h" +#include "dirvote.h" +#include "geoip.h" +#include "main.h" +#include "networkstatus.h" +#include "policies.h" +#include "rendclient.h" +#include "rendcommon.h" +#include "rephist.h" +#include "router.h" +#include "routerlist.h" +#include "routerparse.h" + #if defined(EXPORTMALLINFO) && defined(HAVE_MALLOC_H) && defined(HAVE_MALLINFO) #ifndef OPENBSD #include <malloc.h> @@ -44,11 +64,13 @@ static int purpose_needs_anonymity(uint8_t dir_purpose, uint8_t router_purpose); static char *http_get_header(const char *headers, const char *which); static void http_set_address_origin(const char *headers, connection_t *conn); -static void connection_dir_download_networkstatus_failed( +static void connection_dir_download_v2_networkstatus_failed( dir_connection_t *conn, int status_code); static void connection_dir_download_routerdesc_failed(dir_connection_t *conn); +static void connection_dir_bridge_routerdesc_failed(dir_connection_t *conn); static void connection_dir_download_cert_failed( dir_connection_t *conn, int status_code); +static void connection_dir_retry_bridges(smartlist_t *descs); static void dir_networkstatus_download_failed(smartlist_t *failed, int status_code); static void dir_routerdesc_download_failed(smartlist_t *failed, @@ -92,17 +114,19 @@ static void directory_initiate_command_rend(const char *address, #define ROUTERDESC_CACHE_LIFETIME (30*60) #define ROUTERDESC_BY_DIGEST_CACHE_LIFETIME (48*60*60) #define ROBOTS_CACHE_LIFETIME (24*60*60) +#define MICRODESC_CACHE_LIFETIME (48*60*60) /********* END VARIABLES ************/ -/** Return true iff the directory purpose 'purpose' must use an - * anonymous connection to a directory. */ +/** Return true iff the directory purpose <b>dir_purpose</b> (and if it's + * fetching descriptors, it's fetching them for <b>router_purpose</b>) + * must use an anonymous connection to a directory. */ static int purpose_needs_anonymity(uint8_t dir_purpose, uint8_t router_purpose) { if (get_options()->AllDirActionsPrivate) return 1; - if (router_purpose == ROUTER_PURPOSE_BRIDGE && has_completed_circuit) + if (router_purpose == ROUTER_PURPOSE_BRIDGE && can_complete_circuit) return 1; /* if no circuits yet, we may need this info to bootstrap. */ if (dir_purpose == DIR_PURPOSE_UPLOAD_DIR || dir_purpose == DIR_PURPOSE_UPLOAD_VOTE || @@ -229,10 +253,13 @@ directories_have_accepted_server_descriptor(void) } /** Start a connection to every suitable directory authority, using - * connection purpose 'purpose' and uploading the payload 'payload' - * (length 'payload_len'). The purpose should be one of + * connection purpose <b>dir_purpose</b> and uploading <b>payload</b> + * (of length <b>payload_len</b>). The dir_purpose should be one of * 'DIR_PURPOSE_UPLOAD_DIR' or 'DIR_PURPOSE_UPLOAD_RENDDESC'. * + * <b>router_purpose</b> describes the type of descriptor we're + * publishing, if we're publishing a descriptor -- e.g. general or bridge. + * * <b>type</b> specifies what sort of dir authorities (V1, V2, * HIDSERV, BRIDGE) we should upload to. * @@ -248,9 +275,12 @@ directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose, const char *payload, size_t payload_len, size_t extrainfo_len) { + or_options_t *options = get_options(); int post_via_tor; smartlist_t *dirservers = router_get_trusted_dir_servers(); int found = 0; + const int exclude_self = (dir_purpose == DIR_PURPOSE_UPLOAD_VOTE || + dir_purpose == DIR_PURPOSE_UPLOAD_SIGNATURES); tor_assert(dirservers); /* This tries dirservers which we believe to be down, but ultimately, that's * harmless, and we may as well err on the side of getting things uploaded. @@ -263,6 +293,19 @@ directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose, if ((type & ds->type) == 0) continue; + if (exclude_self && router_digest_is_me(ds->digest)) + continue; + + if (options->ExcludeNodes && options->StrictNodes && + routerset_contains_routerstatus(options->ExcludeNodes, rs)) { + log_warn(LD_DIR, "Wanted to contact authority '%s' for %s, but " + "it's in our ExcludedNodes list and StrictNodes is set. " + "Skipping.", + ds->nickname, + dir_conn_purpose_to_string(dir_purpose)); + continue; + } + found = 1; /* at least one authority of this type was listed */ if (dir_purpose == DIR_PURPOSE_UPLOAD_DIR) ds->has_accepted_serverdesc = 0; @@ -315,6 +358,7 @@ directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose, break; case DIR_PURPOSE_FETCH_V2_NETWORKSTATUS: type = V2_AUTHORITY; + prefer_authority = 1; /* Only v2 authorities have these anyway. */ break; case DIR_PURPOSE_FETCH_SERVERDESC: type = (router_purpose == ROUTER_PURPOSE_BRIDGE ? BRIDGE_AUTHORITY : @@ -348,7 +392,7 @@ directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose, if (!get_via_tor) { if (options->UseBridges && type != BRIDGE_AUTHORITY) { /* want to ask a running bridge for which we have a descriptor. */ - /* XXX022 we assume that all of our bridges can answer any + /* XXX023 we assume that all of our bridges can answer any * possible directory question. This won't be true forever. -RD */ /* It certainly is not true with conditional consensus downloading, * so, for now, never assume the server supports that. */ @@ -472,12 +516,14 @@ directory_initiate_command_routerstatus_rend(routerstatus_t *status, time_t if_modified_since, const rend_data_t *rend_query) { + or_options_t *options = get_options(); routerinfo_t *router; char address_buf[INET_NTOA_BUF_LEN+1]; struct in_addr in; const char *address; tor_addr_t addr; router = router_get_by_digest(status->identity_digest); + if (!router && anonymized_connection) { log_info(LD_DIR, "Not sending anonymized request to directory '%s'; we " "don't have its router descriptor.", status->nickname); @@ -490,6 +536,17 @@ directory_initiate_command_routerstatus_rend(routerstatus_t *status, address = address_buf; } tor_addr_from_ipv4h(&addr, status->addr); + + if (options->ExcludeNodes && options->StrictNodes && + routerset_contains_routerstatus(options->ExcludeNodes, status)) { + log_warn(LD_DIR, "Wanted to contact directory mirror '%s' for %s, but " + "it's in our ExcludedNodes list and StrictNodes is set. " + "Skipping. This choice might make your Tor not work.", + status->nickname, + dir_conn_purpose_to_string(dir_purpose)); + return; + } + directory_initiate_command_rend(address, &addr, status->or_port, status->dir_port, status->version_supports_conditional_consensus, @@ -560,16 +617,18 @@ connection_dir_request_failed(dir_connection_t *conn) if (directory_conn_is_self_reachability_test(conn)) { return; /* this was a test fetch. don't retry. */ } - if (entry_list_can_grow(get_options())) + if (!entry_list_is_constrained(get_options())) router_set_status(conn->identity_digest, 0); /* don't try him again */ if (conn->_base.purpose == DIR_PURPOSE_FETCH_V2_NETWORKSTATUS) { log_info(LD_DIR, "Giving up on directory server at '%s'; retrying", conn->_base.address); - connection_dir_download_networkstatus_failed(conn, -1); + connection_dir_download_v2_networkstatus_failed(conn, -1); } else if (conn->_base.purpose == DIR_PURPOSE_FETCH_SERVERDESC || conn->_base.purpose == DIR_PURPOSE_FETCH_EXTRAINFO) { log_info(LD_DIR, "Giving up on directory server at '%s'; retrying", conn->_base.address); + if (conn->router_purpose == ROUTER_PURPOSE_BRIDGE) + connection_dir_bridge_routerdesc_failed(conn); connection_dir_download_routerdesc_failed(conn); } else if (conn->_base.purpose == DIR_PURPOSE_FETCH_CONSENSUS) { networkstatus_consensus_download_failed(0); @@ -591,7 +650,7 @@ connection_dir_request_failed(dir_connection_t *conn) * retry the fetch now, later, or never. */ static void -connection_dir_download_networkstatus_failed(dir_connection_t *conn, +connection_dir_download_v2_networkstatus_failed(dir_connection_t *conn, int status_code) { if (!conn->requested_resource) { @@ -614,7 +673,7 @@ connection_dir_download_networkstatus_failed(dir_connection_t *conn, * failed, and possibly retry them later.*/ smartlist_t *failed = smartlist_create(); dir_split_resource_into_fingerprints(conn->requested_resource+3, - failed, NULL, 0, 0); + failed, NULL, 0); if (smartlist_len(failed)) { dir_networkstatus_download_failed(failed, status_code); SMARTLIST_FOREACH(failed, char *, cp, tor_free(cp)); @@ -623,6 +682,24 @@ connection_dir_download_networkstatus_failed(dir_connection_t *conn, } } +/** Helper: Attempt to fetch directly the descriptors of each bridge + * listed in <b>failed</b>. + */ +static void +connection_dir_retry_bridges(smartlist_t *descs) +{ + char digest[DIGEST_LEN]; + SMARTLIST_FOREACH(descs, const char *, cp, + { + if (base16_decode(digest, DIGEST_LEN, cp, strlen(cp))<0) { + log_warn(LD_BUG, "Malformed fingerprint in list: %s", + escaped(cp)); + continue; + } + retry_bridge_descriptor_fetch_directly(digest); + }); +} + /** Called when an attempt to download one or more router descriptors * or extra-info documents on connection <b>conn</b> failed. */ @@ -640,6 +717,33 @@ connection_dir_download_routerdesc_failed(dir_connection_t *conn) (void) conn; } +/** Called when an attempt to download a bridge's routerdesc from + * one of the authorities failed due to a network error. If + * possible attempt to download descriptors from the bridge directly. + */ +static void +connection_dir_bridge_routerdesc_failed(dir_connection_t *conn) +{ + smartlist_t *which = NULL; + + /* Requests for bridge descriptors are in the form 'fp/', so ignore + anything else. */ + if (!conn->requested_resource || strcmpstart(conn->requested_resource,"fp/")) + return; + + which = smartlist_create(); + dir_split_resource_into_fingerprints(conn->requested_resource + + strlen("fp/"), + which, NULL, 0); + + tor_assert(conn->_base.purpose != DIR_PURPOSE_FETCH_EXTRAINFO); + if (smartlist_len(which)) { + connection_dir_retry_bridges(which); + SMARTLIST_FOREACH(which, char *, cp, tor_free(cp)); + } + smartlist_free(which); +} + /** Called when an attempt to fetch a certificate fails. */ static void connection_dir_download_cert_failed(dir_connection_t *conn, int status) @@ -651,7 +755,7 @@ connection_dir_download_cert_failed(dir_connection_t *conn, int status) return; failed = smartlist_create(); dir_split_resource_into_fingerprints(conn->requested_resource+3, - failed, NULL, 1, 0); + failed, NULL, DSR_HEX); SMARTLIST_FOREACH(failed, char *, cp, { authority_cert_dl_failed(cp, status); @@ -667,7 +771,7 @@ connection_dir_download_cert_failed(dir_connection_t *conn, int status) * 1) If or_port is 0, or it's a direct conn and or_port is firewalled * or we're a dir mirror, no. * 2) If we prefer to avoid begindir conns, and we're not fetching or - * publishing a bridge relay descriptor, no. + * publishing a bridge relay descriptor, no. * 3) Else yes. */ static int @@ -746,6 +850,15 @@ directory_initiate_command_rend(const char *address, const tor_addr_t *_addr, log_debug(LD_DIR, "Initiating %s", dir_conn_purpose_to_string(dir_purpose)); + /* ensure that we don't make direct connections when a SOCKS server is + * configured. */ + if (!anonymized_connection && !use_begindir && !options->HTTPProxy && + (options->Socks4Proxy || options->Socks5Proxy)) { + log_warn(LD_DIR, "Cannot connect to a directory server through a " + "SOCKS proxy!"); + return; + } + conn = dir_connection_new(AF_INET); /* set up conn so it's got all the data we need to remember */ @@ -770,9 +883,9 @@ directory_initiate_command_rend(const char *address, const tor_addr_t *_addr, if (!anonymized_connection && !use_begindir) { /* then we want to connect to dirport directly */ - if (options->HttpProxy) { - tor_addr_from_ipv4h(&addr, options->HttpProxyAddr); - dir_port = options->HttpProxyPort; + if (options->HTTPProxy) { + tor_addr_copy(&addr, &options->HTTPProxyAddr); + dir_port = options->HTTPProxyPort; } switch (connection_connect(TO_CONN(conn), conn->_base.address, &addr, @@ -793,7 +906,7 @@ directory_initiate_command_rend(const char *address, const tor_addr_t *_addr, payload, payload_len, supports_conditional_consensus, if_modified_since); - connection_watch_events(TO_CONN(conn), EV_READ | EV_WRITE); + connection_watch_events(TO_CONN(conn), READ_EVENT | WRITE_EVENT); /* writable indicates finish, readable indicates broken link, error indicates broken link in windowsland. */ } @@ -832,7 +945,7 @@ directory_initiate_command_rend(const char *address, const tor_addr_t *_addr, payload, payload_len, supports_conditional_consensus, if_modified_since); - connection_watch_events(TO_CONN(conn), EV_READ | EV_WRITE); + connection_watch_events(TO_CONN(conn), READ_EVENT|WRITE_EVENT); connection_start_reading(TO_CONN(linked_conn)); } } @@ -880,7 +993,7 @@ directory_get_consensus_url(int supports_conditional_consensus) if (supports_conditional_consensus) { char *authority_id_list; - smartlist_t *authority_digets = smartlist_create(); + smartlist_t *authority_digests = smartlist_create(); SMARTLIST_FOREACH(router_get_trusted_dir_servers(), trusted_dir_server_t *, ds, @@ -892,10 +1005,10 @@ directory_get_consensus_url(int supports_conditional_consensus) hex = tor_malloc(2*CONDITIONAL_CONSENSUS_FPR_LEN+1); base16_encode(hex, 2*CONDITIONAL_CONSENSUS_FPR_LEN+1, ds->v3_identity_digest, CONDITIONAL_CONSENSUS_FPR_LEN); - smartlist_add(authority_digets, hex); + smartlist_add(authority_digests, hex); }); - smartlist_sort(authority_digets, _compare_strs); - authority_id_list = smartlist_join_strings(authority_digets, + smartlist_sort(authority_digests, _compare_strs); + authority_id_list = smartlist_join_strings(authority_digests, "+", 0, NULL); len = strlen(authority_id_list)+64; @@ -903,8 +1016,8 @@ directory_get_consensus_url(int supports_conditional_consensus) tor_snprintf(url, len, "/tor/status-vote/current/consensus/%s.z", authority_id_list); - SMARTLIST_FOREACH(authority_digets, char *, cp, tor_free(cp)); - smartlist_free(authority_digets); + SMARTLIST_FOREACH(authority_digests, char *, cp, tor_free(cp)); + smartlist_free(authority_digests); tor_free(authority_id_list); } else { url = tor_strdup("/tor/status-vote/current/consensus.z"); @@ -913,7 +1026,7 @@ directory_get_consensus_url(int supports_conditional_consensus) } /** Queue an appropriate HTTP command on conn-\>outbuf. The other args - * are as in directory_initiate_command. + * are as in directory_initiate_command(). */ static void directory_send_command(dir_connection_t *conn, @@ -956,9 +1069,9 @@ directory_send_command(dir_connection_t *conn, } /* come up with some proxy lines, if we're using one. */ - if (direct && get_options()->HttpProxy) { + if (direct && get_options()->HTTPProxy) { char *base64_authenticator=NULL; - const char *authenticator = get_options()->HttpProxyAuthenticator; + const char *authenticator = get_options()->HTTPProxyAuthenticator; tor_snprintf(proxystring, sizeof(proxystring),"http://%s", hoststring); if (authenticator) { @@ -1049,31 +1162,10 @@ directory_send_command(dir_connection_t *conn, httpcommand = "POST"; url = tor_strdup("/tor/post/consensus-signature"); break; - case DIR_PURPOSE_FETCH_RENDDESC: - tor_assert(resource); - tor_assert(!payload); - - /* this must be true or we wouldn't be doing the lookup */ - tor_assert(strlen(resource) <= REND_SERVICE_ID_LEN_BASE32); - /* This breaks the function abstraction. */ - conn->rend_data = tor_malloc_zero(sizeof(rend_data_t)); - strlcpy(conn->rend_data->onion_address, resource, - sizeof(conn->rend_data->onion_address)); - conn->rend_data->rend_desc_version = 0; - - httpcommand = "GET"; - /* Request the most recent versioned descriptor. */ - // (XXXX We were going to switch this to fetch rendezvous1 descriptors, - // but that never got testing, and it wasn't a good design.) - len = strlen(resource)+32; - url = tor_malloc(len); - tor_snprintf(url, len, "/tor/rendezvous/%s", resource); - break; case DIR_PURPOSE_FETCH_RENDDESC_V2: tor_assert(resource); tor_assert(strlen(resource) <= REND_DESC_ID_V2_LEN_BASE32); tor_assert(!payload); - conn->rend_data->rend_desc_version = 2; httpcommand = "GET"; len = strlen(resource) + 32; url = tor_malloc(len); @@ -1162,7 +1254,7 @@ parse_http_url(const char *headers, char **url) if (s-tmp >= 3 && !strcmpstart(tmp,"://")) { tmp = strchr(tmp+3, '/'); if (tmp && tmp < s) { - log_debug(LD_DIR,"Skipping over 'http[s]://hostname' string"); + log_debug(LD_DIR,"Skipping over 'http[s]://hostname/' string"); start = tmp; } } @@ -1479,7 +1571,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) } (void) skewed; /* skewed isn't used yet. */ - if (status_code == 503 && body_len < 16) { + if (status_code == 503) { routerstatus_t *rs; trusted_dir_server_t *ds; log_info(LD_DIR,"Received http status code %d (%s) from server " @@ -1493,12 +1585,6 @@ connection_dir_client_reached_eof(dir_connection_t *conn) tor_free(body); tor_free(headers); tor_free(reason); return -1; - } else if (status_code == 503) { - /* XXXX022 Remove this once every server with bug 539 is obsolete. */ - log_info(LD_DIR, "Server at '%s:%d' sent us a 503 response, but included " - "a body anyway. We'll pretend it gave us a 200.", - conn->_base.address, conn->_base.port); - status_code = 200; } plausible = body_is_plausible(body, body_len, conn->_base.purpose); @@ -1565,15 +1651,21 @@ connection_dir_client_reached_eof(dir_connection_t *conn) v2_networkstatus_source_t source; char *cp; log_info(LD_DIR,"Received networkstatus objects (size %d) from server " - "'%s:%d'",(int) body_len, conn->_base.address, conn->_base.port); + "'%s:%d'", (int)body_len, conn->_base.address, conn->_base.port); if (status_code != 200) { - log_warn(LD_DIR, - "Received http status code %d (%s) from server " - "'%s:%d' while fetching \"/tor/status/%s\". I'll try again soon.", - status_code, escaped(reason), conn->_base.address, - conn->_base.port, conn->requested_resource); + static ratelim_t warning_limit = RATELIM_INIT(3600); + char *m; + if ((m = rate_limit_log(&warning_limit, now))) { + log_warn(LD_DIR, + "Received http status code %d (%s) from server " + "'%s:%d' while fetching \"/tor/status/%s\". " + "I'll try again soon.%s", + status_code, escaped(reason), conn->_base.address, + conn->_base.port, conn->requested_resource, m); + tor_free(m); + } tor_free(body); tor_free(headers); tor_free(reason); - connection_dir_download_networkstatus_failed(conn, status_code); + connection_dir_download_v2_networkstatus_failed(conn, status_code); return -1; } if (conn->requested_resource && @@ -1581,7 +1673,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) source = NS_FROM_DIR_BY_FP; which = smartlist_create(); dir_split_resource_into_fingerprints(conn->requested_resource+3, - which, NULL, 0, 0); + which, NULL, 0); } else if (conn->requested_resource && !strcmpstart(conn->requested_resource, "all")) { source = NS_FROM_DIR_ALL; @@ -1639,8 +1731,8 @@ connection_dir_client_reached_eof(dir_connection_t *conn) return -1; } log_info(LD_DIR,"Received consensus directory (size %d) from server " - "'%s:%d'",(int) body_len, conn->_base.address, conn->_base.port); - if ((r=networkstatus_set_current_consensus(body, 0))<0) { + "'%s:%d'", (int)body_len, conn->_base.address, conn->_base.port); + if ((r=networkstatus_set_current_consensus(body, "ns", 0))<0) { log_fn(r<-1?LOG_WARN:LOG_INFO, LD_DIR, "Unable to load consensus directory downloaded from " "server '%s:%d'. I'll try again soon.", @@ -1667,9 +1759,11 @@ connection_dir_client_reached_eof(dir_connection_t *conn) return -1; } log_info(LD_DIR,"Received authority certificates (size %d) from server " - "'%s:%d'",(int) body_len, conn->_base.address, conn->_base.port); + "'%s:%d'", (int)body_len, conn->_base.address, conn->_base.port); if (trusted_dirs_load_certs_from_string(body, 0, 1)<0) { log_warn(LD_DIR, "Unable to parse fetched certificates"); + /* if we fetched more than one and only some failed, the successful + * ones got flushed to disk so it's safe to call this on them */ connection_dir_download_cert_failed(conn, status_code); } else { directory_info_has_arrived(now, 0); @@ -1680,7 +1774,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) const char *msg; int st; log_info(LD_DIR,"Got votes (size %d) from server %s:%d", - (int) body_len, conn->_base.address, conn->_base.port); + (int)body_len, conn->_base.address, conn->_base.port); if (status_code != 200) { log_warn(LD_DIR, "Received http status code %d (%s) from server " @@ -1700,11 +1794,11 @@ connection_dir_client_reached_eof(dir_connection_t *conn) if (conn->_base.purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES) { const char *msg = NULL; log_info(LD_DIR,"Got detached signatures (size %d) from server %s:%d", - (int) body_len, conn->_base.address, conn->_base.port); + (int)body_len, conn->_base.address, conn->_base.port); if (status_code != 200) { log_warn(LD_DIR, - "Received http status code %d (%s) from server " - "'%s:%d' while fetching \"/tor/status-vote/consensus-signatures.z\".", + "Received http status code %d (%s) from server '%s:%d' while fetching " + "\"/tor/status-vote/next/consensus-signatures.z\".", status_code, escaped(reason), conn->_base.address, conn->_base.port); tor_free(body); tor_free(headers); tor_free(reason); @@ -1732,7 +1826,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) which = smartlist_create(); dir_split_resource_into_fingerprints(conn->requested_resource + (descriptor_digests ? 2 : 3), - which, NULL, 0, 0); + which, NULL, 0); n_asked_for = smartlist_len(which); } if (status_code != 200) { @@ -1806,7 +1900,6 @@ connection_dir_client_reached_eof(dir_connection_t *conn) router_get_trusteddirserver_by_digest(conn->identity_digest); char *rejected_hdr = http_get_header(headers, "X-Descriptor-Not-New: "); - int rejected = 0; if (rejected_hdr) { if (!strcmp(rejected_hdr, "Yes")) { log_info(LD_GENERAL, @@ -1814,12 +1907,11 @@ connection_dir_client_reached_eof(dir_connection_t *conn) ds->nickname); /* XXXX use this information; be sure to upload next one * sooner. -NM */ - /* XXXX021 On further thought, the task above implies that we're + /* XXXX023 On further thought, the task above implies that we're * basing our regenerate-descriptor time on when we uploaded the * last descriptor, not on the published time of the last * descriptor. If those are different, that's a bad thing to * do. -NM */ - rejected = 1; } tor_free(rejected_hdr); } @@ -1921,7 +2013,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) /* Success, or at least there's a v2 descriptor already * present. Notify pending connections about this. */ conn->_base.purpose = DIR_PURPOSE_HAS_FETCHED_RENDDESC; - rend_client_desc_trynow(conn->rend_data->onion_address, -1); + rend_client_desc_trynow(conn->rend_data->onion_address); } break; case 404: @@ -1968,7 +2060,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) log_info(LD_REND, "Successfully fetched v2 rendezvous " "descriptor."); conn->_base.purpose = DIR_PURPOSE_HAS_FETCHED_RENDDESC; - rend_client_desc_trynow(conn->rend_data->onion_address, -1); + rend_client_desc_trynow(conn->rend_data->onion_address); break; } break; @@ -2011,12 +2103,6 @@ connection_dir_client_reached_eof(dir_connection_t *conn) "'%s:%d'. Malformed rendezvous descriptor?", escaped(reason), conn->_base.address, conn->_base.port); break; - case 503: - log_info(LD_REND,"http status 503 (%s) response from dirserver " - "'%s:%d'. Node is (currently) not acting as v2 hidden " - "service directory.", - escaped(reason), conn->_base.address, conn->_base.port); - break; default: log_warn(LD_REND,"http status %d (%s) response unexpected (server " "'%s:%d').", @@ -2323,7 +2409,7 @@ directory_dump_request_log(void) } #endif -/** Decide whether a client would accept the consensus we have +/** Decide whether a client would accept the consensus we have. * * Clients can say they only want a consensus if it's signed by more * than half the authorities in a list. They pass this list in @@ -2344,31 +2430,32 @@ client_likes_consensus(networkstatus_t *v, const char *want_url) int need_at_least; int have = 0; - dir_split_resource_into_fingerprints(want_url, want_authorities, NULL, 0, 0); + dir_split_resource_into_fingerprints(want_url, want_authorities, NULL, 0); need_at_least = smartlist_len(want_authorities)/2+1; - SMARTLIST_FOREACH(want_authorities, const char *, d, { + SMARTLIST_FOREACH_BEGIN(want_authorities, const char *, d) { char want_digest[DIGEST_LEN]; size_t want_len = strlen(d)/2; if (want_len > DIGEST_LEN) want_len = DIGEST_LEN; if (base16_decode(want_digest, DIGEST_LEN, d, want_len*2) < 0) { - log_warn(LD_DIR,"Failed to decode requested authority digest %s.", d); + log_fn(LOG_PROTOCOL_WARN, LD_DIR, + "Failed to decode requested authority digest %s.", d); continue; }; - SMARTLIST_FOREACH(v->voters, networkstatus_voter_info_t *, vi, { - if (vi->signature && - fast_memeq(vi->identity_digest, want_digest, want_len)) { + SMARTLIST_FOREACH_BEGIN(v->voters, networkstatus_voter_info_t *, vi) { + if (smartlist_len(vi->sigs) && + tor_memeq(vi->identity_digest, want_digest, want_len)) { have++; break; }; - }); + } SMARTLIST_FOREACH_END(vi); /* early exit, if we already have enough */ if (have >= need_at_least) break; - }); + } SMARTLIST_FOREACH_END(d); SMARTLIST_FOREACH(want_authorities, char *, d, tor_free(d)); smartlist_free(want_authorities); @@ -2515,9 +2602,12 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, /* v2 or v3 network status fetch. */ smartlist_t *dir_fps = smartlist_create(); int is_v3 = !strcmpstart(url, "/tor/status-vote"); + geoip_client_action_t act = + is_v3 ? GEOIP_CLIENT_NETWORKSTATUS : GEOIP_CLIENT_NETWORKSTATUS_V2; const char *request_type = NULL; const char *key = url + strlen("/tor/status/"); long lifetime = NETWORKSTATUS_CACHE_LIFETIME; + if (!is_v3) { dirserv_get_networkstatus_v2_fingerprints(dir_fps, key); if (!strcmpstart(key, "fp/")) @@ -2532,18 +2622,44 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, } else { networkstatus_t *v = networkstatus_get_latest_consensus(); time_t now = time(NULL); + const char *want_fps = NULL; + char *flavor = NULL; #define CONSENSUS_URL_PREFIX "/tor/status-vote/current/consensus/" - if (v && - !strcmpstart(url, CONSENSUS_URL_PREFIX) && - !client_likes_consensus(v, url + strlen(CONSENSUS_URL_PREFIX))) { + #define CONSENSUS_FLAVORED_PREFIX "/tor/status-vote/current/consensus-" + /* figure out the flavor if any, and who we wanted to sign the thing */ + if (!strcmpstart(url, CONSENSUS_FLAVORED_PREFIX)) { + const char *f, *cp; + f = url + strlen(CONSENSUS_FLAVORED_PREFIX); + cp = strchr(f, '/'); + if (cp) { + want_fps = cp+1; + flavor = tor_strndup(f, cp-f); + } else { + flavor = tor_strdup(f); + } + } else { + if (!strcmpstart(url, CONSENSUS_URL_PREFIX)) + want_fps = url+strlen(CONSENSUS_URL_PREFIX); + } + + /* XXXX MICRODESC NM NM should check document of correct flavor */ + if (v && want_fps && + !client_likes_consensus(v, want_fps)) { write_http_status_line(conn, 404, "Consensus not signed by sufficient " "number of requested authorities"); smartlist_free(dir_fps); + geoip_note_ns_response(act, GEOIP_REJECT_NOT_ENOUGH_SIGS); + tor_free(flavor); goto done; } - smartlist_add(dir_fps, tor_memdup("\0\0\0\0\0\0\0\0\0\0" - "\0\0\0\0\0\0\0\0\0\0", 20)); + { + char *fp = tor_malloc_zero(DIGEST_LEN); + if (flavor) + strlcpy(fp, flavor, DIGEST_LEN); + tor_free(flavor); + smartlist_add(dir_fps, fp); + } request_type = compressed?"v3.z":"v3"; lifetime = (v && v->fresh_until > now) ? v->fresh_until - now : 0; } @@ -2551,6 +2667,7 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, if (!smartlist_len(dir_fps)) { /* we failed to create/cache cp */ write_http_status_line(conn, 503, "Network status object unavailable"); smartlist_free(dir_fps); + geoip_note_ns_response(act, GEOIP_REJECT_UNAVAILABLE); goto done; } @@ -2558,11 +2675,13 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, write_http_status_line(conn, 404, "Not found"); SMARTLIST_FOREACH(dir_fps, char *, cp, tor_free(cp)); smartlist_free(dir_fps); + geoip_note_ns_response(act, GEOIP_REJECT_NOT_FOUND); goto done; } else if (!smartlist_len(dir_fps)) { write_http_status_line(conn, 304, "Not modified"); SMARTLIST_FOREACH(dir_fps, char *, cp, tor_free(cp)); smartlist_free(dir_fps); + geoip_note_ns_response(act, GEOIP_REJECT_NOT_MODIFIED); goto done; } @@ -2574,18 +2693,25 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, write_http_status_line(conn, 503, "Directory busy, try again later"); SMARTLIST_FOREACH(dir_fps, char *, fp, tor_free(fp)); smartlist_free(dir_fps); + geoip_note_ns_response(act, GEOIP_REJECT_BUSY); goto done; } -#ifdef ENABLE_GEOIP_STATS { - geoip_client_action_t act = - is_v3 ? GEOIP_CLIENT_NETWORKSTATUS : GEOIP_CLIENT_NETWORKSTATUS_V2; struct in_addr in; - if (tor_inet_aton((TO_CONN(conn))->address, &in)) + if (tor_inet_aton((TO_CONN(conn))->address, &in)) { geoip_note_client_seen(act, ntohl(in.s_addr), time(NULL)); + geoip_note_ns_response(act, GEOIP_SUCCESS); + /* Note that a request for a network status has started, so that we + * can measure the download time later on. */ + if (TO_CONN(conn)->dirreq_id) + geoip_start_dirreq(TO_CONN(conn)->dirreq_id, dlen, act, + DIRREQ_TUNNELED); + else + geoip_start_dirreq(TO_CONN(conn)->global_identifier, dlen, act, + DIRREQ_DIRECT); + } } -#endif // note_request(request_type,dlen); (void) request_type; @@ -2611,7 +2737,7 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, ssize_t estimated_len = 0; smartlist_t *items = smartlist_create(); smartlist_t *dir_items = smartlist_create(); - int lifetime = 60; /* XXXX022 should actually use vote intervals. */ + int lifetime = 60; /* XXXX023 should actually use vote intervals. */ url += strlen("/tor/status-vote/"); current = !strcmpstart(url, "current/"); url = strchr(url, '/'); @@ -2621,7 +2747,7 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, const char *item; tor_assert(!current); /* we handle current consensus specially above, * since it wants to be spooled. */ - if ((item = dirvote_get_pending_consensus())) + if ((item = dirvote_get_pending_consensus(FLAV_NS))) smartlist_add(items, (char*)item); } else if (!current && !strcmp(url, "consensus-signatures")) { /* XXXX the spec says that we should implement @@ -2647,7 +2773,8 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, flags = DGV_BY_ID | (current ? DGV_INCLUDE_PREVIOUS : DGV_INCLUDE_PENDING); } - dir_split_resource_into_fingerprints(url, fps, NULL, 1, 1); + dir_split_resource_into_fingerprints(url, fps, NULL, + DSR_HEX|DSR_SORT_UNIQ); SMARTLIST_FOREACH(fps, char *, fp, { if ((d = dirvote_get_vote(fp, flags))) smartlist_add(dir_items, (cached_dir_t*)d); @@ -2700,6 +2827,41 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, goto done; } + if (!strcmpstart(url, "/tor/micro/d/")) { + smartlist_t *fps = smartlist_create(); + + dir_split_resource_into_fingerprints(url+strlen("/tor/micro/d/"), + fps, NULL, + DSR_DIGEST256|DSR_BASE64|DSR_SORT_UNIQ); + + if (!dirserv_have_any_microdesc(fps)) { + write_http_status_line(conn, 404, "Not found"); + SMARTLIST_FOREACH(fps, char *, fp, tor_free(fp)); + smartlist_free(fps); + goto done; + } + dlen = dirserv_estimate_microdesc_size(fps, compressed); + if (global_write_bucket_low(TO_CONN(conn), dlen, 2)) { + log_info(LD_DIRSERV, + "Client asked for server descriptors, but we've been " + "writing too many bytes lately. Sending 503 Dir busy."); + write_http_status_line(conn, 503, "Directory busy, try again later"); + SMARTLIST_FOREACH(fps, char *, fp, tor_free(fp)); + smartlist_free(fps); + goto done; + } + + write_http_response_header(conn, -1, compressed, MICRODESC_CACHE_LIFETIME); + conn->dir_spool_src = DIR_SPOOL_MICRODESC; + conn->fingerprint_stack = fps; + + if (compressed) + conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD); + + connection_dirserv_flushed_some(conn); + goto done; + } + if (!strcmpstart(url,"/tor/server/") || (!options->BridgeAuthoritativeDir && !options->BridgeRelay && !strcmpstart(url,"/tor/extra/"))) { @@ -2781,7 +2943,8 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, } else if (!strcmpstart(url, "/tor/keys/fp/")) { smartlist_t *fps = smartlist_create(); dir_split_resource_into_fingerprints(url+strlen("/tor/keys/fp/"), - fps, NULL, 1, 1); + fps, NULL, + DSR_HEX|DSR_SORT_UNIQ); SMARTLIST_FOREACH(fps, char *, d, { authority_cert_t *c = authority_cert_get_newest_by_id(d); if (c) smartlist_add(certs, c); @@ -2791,7 +2954,8 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, } else if (!strcmpstart(url, "/tor/keys/sk/")) { smartlist_t *fps = smartlist_create(); dir_split_resource_into_fingerprints(url+strlen("/tor/keys/sk/"), - fps, NULL, 1, 1); + fps, NULL, + DSR_HEX|DSR_SORT_UNIQ); SMARTLIST_FOREACH(fps, char *, d, { authority_cert_t *c = authority_cert_get_by_sk_digest(d); if (c) smartlist_add(certs, c); @@ -2893,18 +3057,9 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, note_request("/tor/rendezvous?/", desc_len); /* need to send descp separately, because it may include NULs */ connection_write_to_buf(descp, desc_len, TO_CONN(conn)); - /* report successful fetch to statistic */ - if (options->HSAuthorityRecordStats) { - hs_usage_note_fetch_total(query, time(NULL)); - hs_usage_note_fetch_successful(query, time(NULL)); - } break; case 0: /* well-formed but not present */ write_http_status_line(conn, 404, "Not found"); - /* report (unsuccessful) fetch to statistic */ - if (options->HSAuthorityRecordStats) { - hs_usage_note_fetch_total(query, time(NULL)); - } break; case -1: /* not well-formed */ write_http_status_line(conn, 400, "Bad request"); @@ -3135,6 +3290,8 @@ directory_handle_command_post(dir_connection_t *conn, const char *headers, write_http_status_line(conn, status, "Vote stored"); } else { tor_assert(msg); + log_warn(LD_DIRSERV, "Rejected vote from %s (\"%s\").", + conn->_base.address, msg); write_http_status_line(conn, status, msg); } goto done; @@ -3181,8 +3338,8 @@ directory_handle_command(dir_connection_t *conn) &body, &body_len, MAX_DIR_UL_SIZE, 0)) { case -1: /* overflow */ log_warn(LD_DIRSERV, - "Invalid input from address '%s'. Closing.", - conn->_base.address); + "Request too large from address '%s' to DirPort. Closing.", + safe_str(conn->_base.address)); return -1; case 0: log_debug(LD_DIRSERV,"command not all here yet."); @@ -3218,6 +3375,16 @@ connection_dir_finished_flushing(dir_connection_t *conn) tor_assert(conn); tor_assert(conn->_base.type == CONN_TYPE_DIR); + /* Note that we have finished writing the directory response. For direct + * connections this means we're done, for tunneled connections its only + * an intermediate step. */ + if (TO_CONN(conn)->dirreq_id) + geoip_change_dirreq_state(TO_CONN(conn)->dirreq_id, DIRREQ_TUNNELED, + DIRREQ_FLUSHING_DIR_CONN_FINISHED); + else + geoip_change_dirreq_state(TO_CONN(conn)->global_identifier, + DIRREQ_DIRECT, + DIRREQ_FLUSHING_DIR_CONN_FINISHED); switch (conn->_base.state) { case DIR_CONN_STATE_CLIENT_SENDING: log_debug(LD_DIR,"client finished sending command."); @@ -3400,6 +3567,14 @@ download_status_reset(download_status_t *dls) dls->next_attempt_at = time(NULL) + schedule[0]; } +/** Return the number of failures on <b>dls</b> since the last success (if + * any). */ +int +download_status_get_n_failures(const download_status_t *dls) +{ + return dls->n_download_failures; +} + /** Called when one or more routerdesc (or extrainfo, if <b>was_extrainfo</b>) * fetches have failed (with uppercase fingerprints listed in <b>failed</b>, * either as descriptor digests or as identity digests based on @@ -3415,16 +3590,8 @@ dir_routerdesc_download_failed(smartlist_t *failed, int status_code, int server = directory_fetches_from_authorities(get_options()); if (!was_descriptor_digests) { if (router_purpose == ROUTER_PURPOSE_BRIDGE) { - tor_assert(!was_extrainfo); /* not supported yet */ - SMARTLIST_FOREACH(failed, const char *, cp, - { - if (base16_decode(digest, DIGEST_LEN, cp, strlen(cp))<0) { - log_warn(LD_BUG, "Malformed fingerprint in list: %s", - escaped(cp)); - continue; - } - retry_bridge_descriptor_fetch_directly(digest); - }); + tor_assert(!was_extrainfo); + connection_dir_retry_bridges(failed); } return; /* FFFF should implement for other-than-router-purpose someday */ } @@ -3517,19 +3684,37 @@ dir_split_resource_into_fingerprint_pairs(const char *res, /** Given a directory <b>resource</b> request, containing zero * or more strings separated by plus signs, followed optionally by ".z", store * the strings, in order, into <b>fp_out</b>. If <b>compressed_out</b> is - * non-NULL, set it to 1 if the resource ends in ".z", else set it to 0. If - * decode_hex is true, then delete all elements that aren't hex digests, and - * decode the rest. If sort_uniq is true, then sort the list and remove - * all duplicates. + * non-NULL, set it to 1 if the resource ends in ".z", else set it to 0. + * + * If (flags & DSR_HEX), then delete all elements that aren't hex digests, and + * decode the rest. If (flags & DSR_BASE64), then use "-" rather than "+" as + * a separator, delete all the elements that aren't base64-encoded digests, + * and decode the rest. If (flags & DSR_DIGEST256), these digests should be + * 256 bits long; else they should be 160. + * + * If (flags & DSR_SORT_UNIQ), then sort the list and remove all duplicates. */ int dir_split_resource_into_fingerprints(const char *resource, smartlist_t *fp_out, int *compressed_out, - int decode_hex, int sort_uniq) + int flags) { + const int decode_hex = flags & DSR_HEX; + const int decode_base64 = flags & DSR_BASE64; + const int digests_are_256 = flags & DSR_DIGEST256; + const int sort_uniq = flags & DSR_SORT_UNIQ; + + const int digest_len = digests_are_256 ? DIGEST256_LEN : DIGEST_LEN; + const int hex_digest_len = digests_are_256 ? + HEX_DIGEST256_LEN : HEX_DIGEST_LEN; + const int base64_digest_len = digests_are_256 ? + BASE64_DIGEST256_LEN : BASE64_DIGEST_LEN; smartlist_t *fp_tmp = smartlist_create(); + + tor_assert(!(decode_hex && decode_base64)); tor_assert(fp_out); - smartlist_split_string(fp_tmp, resource, "+", 0, 0); + + smartlist_split_string(fp_tmp, resource, decode_base64?"-":"+", 0, 0); if (compressed_out) *compressed_out = 0; if (smartlist_len(fp_tmp)) { @@ -3541,22 +3726,25 @@ dir_split_resource_into_fingerprints(const char *resource, *compressed_out = 1; } } - if (decode_hex) { + if (decode_hex || decode_base64) { + const size_t encoded_len = decode_hex ? hex_digest_len : base64_digest_len; int i; char *cp, *d = NULL; for (i = 0; i < smartlist_len(fp_tmp); ++i) { cp = smartlist_get(fp_tmp, i); - if (strlen(cp) != HEX_DIGEST_LEN) { + if (strlen(cp) != encoded_len) { log_info(LD_DIR, "Skipping digest %s with non-standard length.", escaped(cp)); smartlist_del_keeporder(fp_tmp, i--); goto again; } - d = tor_malloc_zero(DIGEST_LEN); - if (base16_decode(d, DIGEST_LEN, cp, HEX_DIGEST_LEN)<0) { - log_info(LD_DIR, "Skipping non-decodable digest %s", escaped(cp)); - smartlist_del_keeporder(fp_tmp, i--); - goto again; + d = tor_malloc_zero(digest_len); + if (decode_hex ? + (base16_decode(d, digest_len, cp, hex_digest_len)<0) : + (base64_decode(d, digest_len, cp, base64_digest_len)<0)) { + log_info(LD_DIR, "Skipping non-decodable digest %s", escaped(cp)); + smartlist_del_keeporder(fp_tmp, i--); + goto again; } smartlist_set(fp_tmp, i, d); d = NULL; @@ -3566,26 +3754,18 @@ dir_split_resource_into_fingerprints(const char *resource, } } if (sort_uniq) { - smartlist_t *fp_tmp2 = smartlist_create(); - int i; - if (decode_hex) - smartlist_sort_digests(fp_tmp); - else + if (decode_hex || decode_base64) { + if (digests_are_256) { + smartlist_sort_digests256(fp_tmp); + smartlist_uniq_digests256(fp_tmp); + } else { + smartlist_sort_digests(fp_tmp); + smartlist_uniq_digests(fp_tmp); + } + } else { smartlist_sort_strings(fp_tmp); - if (smartlist_len(fp_tmp)) - smartlist_add(fp_tmp2, smartlist_get(fp_tmp, 0)); - for (i = 1; i < smartlist_len(fp_tmp); ++i) { - char *cp = smartlist_get(fp_tmp, i); - char *last = smartlist_get(fp_tmp2, smartlist_len(fp_tmp2)-1); - - if ((decode_hex && fast_memcmp(cp, last, DIGEST_LEN)) - || (!decode_hex && strcasecmp(cp, last))) - smartlist_add(fp_tmp2, cp); - else - tor_free(cp); + smartlist_uniq_strings(fp_tmp); } - smartlist_free(fp_tmp); - fp_tmp = fp_tmp2; } smartlist_add_all(fp_out, fp_tmp); smartlist_free(fp_tmp); diff --git a/src/or/directory.h b/src/or/directory.h new file mode 100644 index 0000000000..94dfb17644 --- /dev/null +++ b/src/or/directory.h @@ -0,0 +1,110 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file directory.h + * \brief Header file for directory.c. + **/ + +#ifndef _TOR_DIRECTORY_H +#define _TOR_DIRECTORY_H + +int directories_have_accepted_server_descriptor(void); +char *authority_type_to_string(authority_type_t auth); +void directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose, + authority_type_t type, const char *payload, + size_t payload_len, size_t extrainfo_len); +void directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose, + const char *resource, + int pds_flags); +void directory_get_from_all_authorities(uint8_t dir_purpose, + uint8_t router_purpose, + const char *resource); +void directory_initiate_command_routerstatus(routerstatus_t *status, + uint8_t dir_purpose, + uint8_t router_purpose, + int anonymized_connection, + const char *resource, + const char *payload, + size_t payload_len, + time_t if_modified_since); +void directory_initiate_command_routerstatus_rend(routerstatus_t *status, + uint8_t dir_purpose, + uint8_t router_purpose, + int anonymized_connection, + const char *resource, + const char *payload, + size_t payload_len, + time_t if_modified_since, + const rend_data_t *rend_query); + +int parse_http_response(const char *headers, int *code, time_t *date, + compress_method_t *compression, char **response); + +int connection_dir_is_encrypted(dir_connection_t *conn); +int connection_dir_reached_eof(dir_connection_t *conn); +int connection_dir_process_inbuf(dir_connection_t *conn); +int connection_dir_finished_flushing(dir_connection_t *conn); +int connection_dir_finished_connecting(dir_connection_t *conn); +void connection_dir_request_failed(dir_connection_t *conn); +void directory_initiate_command(const char *address, const tor_addr_t *addr, + uint16_t or_port, uint16_t dir_port, + int supports_conditional_consensus, + int supports_begindir, const char *digest, + uint8_t dir_purpose, uint8_t router_purpose, + int anonymized_connection, + const char *resource, + const char *payload, size_t payload_len, + time_t if_modified_since); + +#define DSR_HEX (1<<0) +#define DSR_BASE64 (1<<1) +#define DSR_DIGEST256 (1<<2) +#define DSR_SORT_UNIQ (1<<3) +int dir_split_resource_into_fingerprints(const char *resource, + smartlist_t *fp_out, int *compressed_out, + int flags); + +int dir_split_resource_into_fingerprint_pairs(const char *res, + smartlist_t *pairs_out); +char *directory_dump_request_log(void); +void note_request(const char *key, size_t bytes); +int router_supports_extrainfo(const char *identity_digest, int is_authority); + +time_t download_status_increment_failure(download_status_t *dls, + int status_code, const char *item, + int server, time_t now); +/** Increment the failure count of the download_status_t <b>dls</b>, with + * the optional status code <b>sc</b>. */ +#define download_status_failed(dls, sc) \ + download_status_increment_failure((dls), (sc), NULL, \ + get_options()->DirPort, time(NULL)) + +void download_status_reset(download_status_t *dls); +static int download_status_is_ready(download_status_t *dls, time_t now, + int max_failures); +/** Return true iff, as of <b>now</b>, the resource tracked by <b>dls</b> is + * ready to get its download reattempted. */ +static INLINE int +download_status_is_ready(download_status_t *dls, time_t now, + int max_failures) +{ + return (dls->n_download_failures <= max_failures + && dls->next_attempt_at <= now); +} + +static void download_status_mark_impossible(download_status_t *dl); +/** Mark <b>dl</b> as never downloadable. */ +static INLINE void +download_status_mark_impossible(download_status_t *dl) +{ + dl->n_download_failures = IMPOSSIBLE_TO_DOWNLOAD; +} + +int download_status_get_n_failures(const download_status_t *dls); + +#endif + diff --git a/src/or/dirserv.c b/src/or/dirserv.c index e367cb1c3a..79b68cdac8 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -5,6 +5,22 @@ #define DIRSERV_PRIVATE #include "or.h" +#include "buffers.h" +#include "config.h" +#include "connection.h" +#include "connection_or.h" +#include "control.h" +#include "directory.h" +#include "dirserv.h" +#include "dirvote.h" +#include "hibernate.h" +#include "microdesc.h" +#include "networkstatus.h" +#include "policies.h" +#include "rephist.h" +#include "router.h" +#include "routerlist.h" +#include "routerparse.h" /** * \file dirserv.c @@ -27,6 +43,8 @@ extern time_t time_of_process_start; /* from main.c */ +extern long stats_n_seconds_working; /* from main.c */ + /** Do we need to regenerate the v1 directory when someone asks for it? */ static time_t the_directory_is_dirty = 1; /** Do we need to regenerate the v1 runningrouters document when somebody @@ -41,7 +59,7 @@ static time_t the_v2_networkstatus_is_dirty = 1; static cached_dir_t *the_directory = NULL; /** For authoritative directories: the current (v1) network status. */ -static cached_dir_t the_runningrouters = { NULL, NULL, 0, 0, 0, -1 }; +static cached_dir_t the_runningrouters; static void directory_remove_invalid(void); static cached_dir_t *dirserv_regenerate_directory(void); @@ -63,13 +81,16 @@ static signed_descriptor_t *get_signed_descriptor_by_fp(const char *fp, time_t publish_cutoff); static int dirserv_add_extrainfo(extrainfo_t *ei, const char **msg); +/************** Measured Bandwidth parsing code ******/ +#define MAX_MEASUREMENT_AGE (3*24*60*60) /* 3 days */ + /************** Fingerprint handling code ************/ #define FP_NAMED 1 /**< Listed in fingerprint file. */ #define FP_INVALID 2 /**< Believed invalid. */ #define FP_REJECT 4 /**< We will not publish this router. */ #define FP_BADDIR 8 /**< We'll tell clients to avoid using this as a dir. */ -#define FP_BADEXIT 16 /**< We'll tell clients not to use this as an exit. */ +#define FP_BADEXIT 16 /**< We'll tell clients not to use this as an exit. */ #define FP_UNNAMED 32 /**< Another router has this name in fingerprint file. */ /** Encapsulate a nickname and an FP_* status; target of status_by_digest @@ -99,7 +120,7 @@ authdir_config_new(void) return list; } -/** Add the fingerprint <b>fp</b> for the nickname <b>nickname</b> to +/** Add the fingerprint <b>fp</b> for <b>nickname</b> to * the smartlist of fingerprint_entry_t's <b>list</b>. Return 0 if it's * new, or 1 if we replaced the old value. */ @@ -181,8 +202,7 @@ dirserv_add_own_fingerprint(const char *nickname, crypto_pk_env_t *pk) * file. The file format is line-based, with each non-blank holding one * nickname, some space, and a fingerprint for that nickname. On success, * replace the current fingerprint list with the new list and return 0. On - * failure, leave the current fingerprint list untouched, and - * return -1. */ + * failure, leave the current fingerprint list untouched, and return -1. */ int dirserv_load_fingerprint_file(void) { @@ -368,13 +388,19 @@ dirserv_get_status_impl(const char *id_digest, const char *nickname, strmap_size(fingerprint_list->fp_by_name), digestmap_size(fingerprint_list->status_by_digest)); - /* 0.1.1.17-rc was the first version that claimed to be stable, doesn't - * crash and drop circuits all the time, and is even vaguely compatible with - * the current network */ - if (platform && !tor_version_as_new_as(platform,"0.1.1.17-rc")) { + /* Tor 0.2.0.26-rc is the oldest version that currently caches the right + * directory information. Once more of them die off, we should raise this + * minimum. */ + if (platform && !tor_version_as_new_as(platform,"0.2.0.26-rc")) { if (msg) *msg = "Tor version is far too old to work."; return FP_REJECT; + } else if (platform && tor_version_as_new_as(platform,"0.2.1.3-alpha") + && !tor_version_as_new_as(platform, "0.2.1.19")) { + /* These versions mishandled RELAY_EARLY cells on rend circuits. */ + if (msg) + *msg = "Tor version is too buggy to work."; + return FP_REJECT; } result = dirserv_get_name_status(id_digest, nickname); @@ -520,7 +546,7 @@ authdir_wants_to_reject_router(routerinfo_t *ri, const char **msg, /* Okay. Now check whether the fingerprint is recognized. */ uint32_t status = dirserv_router_get_status(ri, msg); time_t now; - int severity = complain ? LOG_NOTICE : LOG_INFO; + int severity = (complain && ri->contact_info) ? LOG_NOTICE : LOG_INFO; tor_assert(msg); if (status & FP_REJECT) return -1; /* msg is already set. */ @@ -712,6 +738,10 @@ dirserv_add_descriptor(routerinfo_t *ri, const char **msg, const char *source) desc = tor_strndup(ri->cache_info.signed_descriptor_body, desclen); nickname = tor_strdup(ri->nickname); + /* Tell if we're about to need to launch a test if we add this. */ + ri->needs_retest_if_added = + dirserv_should_launch_reachability_test(ri, ri_old); + r = router_add_to_routerlist(ri, msg, 0, 0); if (!WRA_WAS_ADDED(r)) { /* unless the routerinfo was fine, just out-of-date */ @@ -726,7 +756,7 @@ dirserv_add_descriptor(routerinfo_t *ri, const char **msg, const char *source) changed = smartlist_create(); smartlist_add(changed, ri); - control_event_descriptors_changed(changed); + routerlist_descriptors_added(changed, 0); smartlist_free(changed); if (!*msg) { *msg = ri->is_valid ? "Descriptor for valid server accepted" : @@ -835,46 +865,6 @@ directory_remove_invalid(void) routerlist_assert_ok(rl); } -/** Write a list of unregistered descriptors into a newly allocated - * string and return it. Used by dirserv operators to keep track of - * fast nodes that haven't registered. - */ -int -getinfo_helper_dirserv_unregistered(control_connection_t *control_conn, - const char *question, char **answer_out) -{ - smartlist_t *answerlist; - char buf[1024]; - char *answer; - int min_bw = atoi(question); - routerlist_t *rl = router_get_routerlist(); - - (void) control_conn; - - if (strcmpstart(question, "unregistered-servers-")) - return 0; - question += strlen("unregistered-servers-"); - - answerlist = smartlist_create(); - SMARTLIST_FOREACH(rl->routers, routerinfo_t *, ent, { - uint32_t r = dirserv_router_get_status(ent, NULL); - if (router_get_advertised_bandwidth(ent) >= (size_t)min_bw && - !(r & FP_NAMED)) { - /* then log this one */ - tor_snprintf(buf, sizeof(buf), - "%s: BW %d on '%s'.", - ent->nickname, router_get_advertised_bandwidth(ent), - ent->platform ? ent->platform : ""); - smartlist_add(answerlist, tor_strdup(buf)); - } - }); - answer = smartlist_join_strings(answerlist, "\r\n", 0, NULL); - SMARTLIST_FOREACH(answerlist, char *, cp, tor_free(cp)); - smartlist_free(answerlist); - *answer_out = answer; - return 0; -} - /** Mark the directory as <b>dirty</b> -- when we're next asked for a * directory, we will rebuild it instead of reusing the most recently * generated one. @@ -933,28 +923,66 @@ list_single_server_status(routerinfo_t *desc, int is_live) return tor_strdup(buf); } +static INLINE int +running_long_enough_to_decide_unreachable(void) +{ + return time_of_process_start + + get_options()->TestingAuthDirTimeToLearnReachability < approx_time(); +} + /** Each server needs to have passed a reachability test no more * than this number of seconds ago, or he is listed as down in * the directory. */ #define REACHABLE_TIMEOUT (45*60) +/** If we tested a router and found it reachable _at least this long_ after it + * declared itself hibernating, it is probably done hibernating and we just + * missed a descriptor from it. */ +#define HIBERNATION_PUBLICATION_SKEW (60*60) + /** Treat a router as alive if * - It's me, and I'm not hibernating. * or - We've found it reachable recently. */ void dirserv_set_router_is_running(routerinfo_t *router, time_t now) { + /*XXXX023 This function is a mess. Separate out the part that calculates + whether it's reachable and the part that tells rephist that the router was + unreachable. + */ int answer; - if (router_is_me(router) && !we_are_hibernating()) + if (router_is_me(router)) { + /* We always know if we are down ourselves. */ + answer = ! we_are_hibernating(); + } else if (router->is_hibernating && + (router->cache_info.published_on + + HIBERNATION_PUBLICATION_SKEW) > router->last_reachable) { + /* A hibernating router is down unless we (somehow) had contact with it + * since it declared itself to be hibernating. */ + answer = 0; + } else if (get_options()->AssumeReachable) { + /* If AssumeReachable, everybody is up unless they say they are down! */ answer = 1; - else - answer = get_options()->AssumeReachable || - now < router->last_reachable + REACHABLE_TIMEOUT; + } else { + /* Otherwise, a router counts as up if we found it reachable in the last + REACHABLE_TIMEOUT seconds. */ + answer = (now < router->last_reachable + REACHABLE_TIMEOUT); + } + + if (!answer && running_long_enough_to_decide_unreachable()) { + /* Not considered reachable. tell rephist about that. - if (!answer) { - /* not considered reachable. tell rephist. */ - rep_hist_note_router_unreachable(router->cache_info.identity_digest, now); + Because we launch a reachability test for each router every + REACHABILITY_TEST_CYCLE_PERIOD seconds, then the router has probably + been down since at least that time after we last successfully reached + it. + */ + time_t when = now; + if (router->last_reachable && + router->last_reachable + REACHABILITY_TEST_CYCLE_PERIOD < now) + when = router->last_reachable + REACHABILITY_TEST_CYCLE_PERIOD; + rep_hist_note_router_unreachable(router->cache_info.identity_digest, when); } router->is_running = answer; @@ -965,7 +993,6 @@ dirserv_set_router_is_running(routerinfo_t *router, time_t now) * *<b>router_status_out</b>. Return 0 on success, -1 on failure. * * If for_controller is true, include the routers with very old descriptors. - * If for_controller is >1, use the verbose nickname format. */ int list_server_status_v1(smartlist_t *routers, char **router_status_out, @@ -985,23 +1012,22 @@ list_server_status_v1(smartlist_t *routers, char **router_status_out, rs_entries = smartlist_create(); - SMARTLIST_FOREACH(routers, routerinfo_t *, ri, - { + SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) { if (authdir) { /* Update router status in routerinfo_t. */ dirserv_set_router_is_running(ri, now); } - if (for_controller == 1 || ri->cache_info.published_on >= cutoff) - smartlist_add(rs_entries, list_single_server_status(ri, ri->is_running)); - else if (for_controller > 2) { + if (for_controller) { char name_buf[MAX_VERBOSE_NICKNAME_LEN+2]; char *cp = name_buf; if (!ri->is_running) *cp++ = '!'; router_get_verbose_nickname(cp, ri); smartlist_add(rs_entries, tor_strdup(name_buf)); + } else if (ri->cache_info.published_on >= cutoff) { + smartlist_add(rs_entries, list_single_server_status(ri, ri->is_running)); } - }); + } SMARTLIST_FOREACH_END(ri); *router_status_out = smartlist_join_strings(rs_entries, " ", 0, NULL); @@ -1119,7 +1145,8 @@ dirserv_dump_directory_to_string(char **dir_out, return -1; } note_crypto_pk_op(SIGN_DIR); - if (router_append_dirobj_signature(buf,buf_len,digest,private_key)<0) { + if (router_append_dirobj_signature(buf,buf_len,digest,DIGEST_LEN, + private_key)<0) { tor_free(buf); return -1; } @@ -1144,18 +1171,21 @@ directory_fetches_from_authorities(or_options_t *options) { routerinfo_t *me; uint32_t addr; + int refuseunknown; if (options->FetchDirInfoEarly) return 1; if (options->BridgeRelay == 1) return 0; if (server_mode(options) && router_pick_published_address(options, &addr)<0) return 1; /* we don't know our IP address; ask an authority. */ - if (options->DirPort == 0) + refuseunknown = ! router_my_exit_policy_is_reject_star() && + should_refuse_unknown_exits(options); + if (options->DirPort == 0 && !refuseunknown) return 0; if (!server_mode(options) || !advertised_server_mode()) return 0; me = router_get_my_routerinfo(); - if (!me || !me->dir_port) + if (!me || (!me->dir_port && !refuseunknown)) return 0; /* if dirport not advertised, return 0 too */ return 1; } @@ -1195,7 +1225,14 @@ directory_caches_v2_dir_info(or_options_t *options) int directory_caches_dir_info(or_options_t *options) { - return options->BridgeRelay != 0 || options->DirPort != 0; + if (options->BridgeRelay || options->DirPort) + return 1; + if (!server_mode(options) || !advertised_server_mode()) + return 0; + /* We need an up-to-date view of network info if we're going to try to + * block exit attempts from unknown relays. */ + return ! router_my_exit_policy_is_reject_star() && + should_refuse_unknown_exits(options); } /** Return 1 if we want to allow remote people to ask us directory @@ -1238,14 +1275,14 @@ directory_too_idle_to_fetch_descriptors(or_options_t *options, time_t now) static cached_dir_t *cached_directory = NULL; /** The v1 runningrouters document we'll serve (as a cache or as an authority) * if requested. */ -static cached_dir_t cached_runningrouters = { NULL, NULL, 0, 0, 0, -1 }; +static cached_dir_t cached_runningrouters; /** Used for other dirservers' v2 network statuses. Map from hexdigest to * cached_dir_t. */ static digestmap_t *cached_v2_networkstatus = NULL; -/** The v3 consensus network status that we're currently serving. */ -static cached_dir_t *cached_v3_networkstatus = NULL; +/** Map from flavor name to the v3 consensuses that we're currently serving. */ +static strmap_t *cached_consensuses = NULL; /** Possibly replace the contents of <b>d</b> with the value of * <b>directory</b> published on <b>when</b>, unless <b>when</b> is older than @@ -1319,7 +1356,11 @@ clear_cached_dir(cached_dir_t *d) static void _free_cached_dir(void *_d) { - cached_dir_t *d = (cached_dir_t *)_d; + cached_dir_t *d; + if (!_d) + return; + + d = (cached_dir_t *)_d; cached_dir_decref(d); } @@ -1413,17 +1454,26 @@ dirserv_set_cached_networkstatus_v2(const char *networkstatus, } } -/** Replace the v3 consensus networkstatus that we're serving with - * <b>networkstatus</b>, published at <b>published</b>. No validation is - * performed. */ +/** Replace the v3 consensus networkstatus of type <b>flavor_name</b> that + * we're serving with <b>networkstatus</b>, published at <b>published</b>. No + * validation is performed. */ void -dirserv_set_cached_networkstatus_v3(const char *networkstatus, - time_t published) +dirserv_set_cached_consensus_networkstatus(const char *networkstatus, + const char *flavor_name, + const digests_t *digests, + time_t published) { - if (cached_v3_networkstatus) - cached_dir_decref(cached_v3_networkstatus); - cached_v3_networkstatus = new_cached_dir( - tor_strdup(networkstatus), published); + cached_dir_t *new_networkstatus; + cached_dir_t *old_networkstatus; + if (!cached_consensuses) + cached_consensuses = strmap_new(); + + new_networkstatus = new_cached_dir(tor_strdup(networkstatus), published); + memcpy(&new_networkstatus->digests, digests, sizeof(digests_t)); + old_networkstatus = strmap_set(cached_consensuses, flavor_name, + new_networkstatus); + if (old_networkstatus) + cached_dir_decref(old_networkstatus); } /** Remove any v2 networkstatus from the directory cache that was published @@ -1520,7 +1570,8 @@ dirserv_regenerate_directory(void) { char *new_directory=NULL; - if (dirserv_dump_directory_to_string(&new_directory, get_identity_key())) { + if (dirserv_dump_directory_to_string(&new_directory, + get_server_identity_key())) { log_warn(LD_BUG, "Error creating directory."); tor_free(new_directory); return NULL; @@ -1550,7 +1601,7 @@ generate_runningrouters(void) char digest[DIGEST_LEN]; char published[ISO_TIME_LEN+1]; size_t len; - crypto_pk_env_t *private_key = get_identity_key(); + crypto_pk_env_t *private_key = get_server_identity_key(); char *identity_pkey; /* Identity key, DER64-encoded. */ size_t identity_pkey_len; @@ -1577,7 +1628,8 @@ generate_runningrouters(void) goto err; } note_crypto_pk_op(SIGN_DIR); - if (router_append_dirobj_signature(s, len, digest, private_key)<0) + if (router_append_dirobj_signature(s, len, digest, DIGEST_LEN, + private_key)<0) goto err; set_cached_dir(&the_runningrouters, s, time(NULL)); @@ -1605,9 +1657,11 @@ dirserv_get_runningrouters(void) /** Return the latest downloaded consensus networkstatus in encoded, signed, * optionally compressed format, suitable for sending to clients. */ cached_dir_t * -dirserv_get_consensus(void) +dirserv_get_consensus(const char *flavor_name) { - return cached_v3_networkstatus; + if (!cached_consensuses) + return NULL; + return strmap_get(cached_consensuses, flavor_name); } /** For authoritative directories: the current (v2) network status. */ @@ -1645,7 +1699,7 @@ should_generate_v2_networkstatus(void) #define TIME_KNOWN_TO_GUARANTEE_FAMILIAR (8*24*60*60) /** Similarly, every node with sufficient WFU is around enough to be a guard. */ -#define WFU_TO_GUARANTEE_GUARD (0.995) +#define WFU_TO_GUARANTEE_GUARD (0.98) /* Thresholds for server performance: set by * dirserv_compute_performance_thresholds, and used by @@ -1700,9 +1754,12 @@ dirserv_thinks_router_is_unreliable(time_t now, { if (need_uptime) { if (!enough_mtbf_info) { - /* XXX022 Once most authorities are on v3, we should change the rule from + /* XXX023 Once most authorities are on v3, we should change the rule from * "use uptime if we don't have mtbf data" to "don't advertise Stable on - * v3 if we don't have enough mtbf data." */ + * v3 if we don't have enough mtbf data." Or maybe not, since if we ever + * hit a point where we need to reset a lot of authorities at once, + * none of them would be in a position to declare Stable. + */ long uptime = real_uptime(router, now); if ((unsigned)uptime < stable_uptime && (unsigned)uptime < UPTIME_TO_GUARANTEE_STABLE) @@ -1725,7 +1782,7 @@ dirserv_thinks_router_is_unreliable(time_t now, /** Return true iff <b>router</b> should be assigned the "HSDir" flag. * Right now this means it advertises support for it, it has a high - * uptime, and it's currently considered Running. + * uptime, it has a DirPort open, and it's currently considered Running. * * This function needs to be called after router-\>is_running has * been set. @@ -1733,9 +1790,31 @@ dirserv_thinks_router_is_unreliable(time_t now, static int dirserv_thinks_router_is_hs_dir(routerinfo_t *router, time_t now) { - long uptime = real_uptime(router, now); - return (router->wants_to_be_hs_dir && + long uptime; + + /* If we haven't been running for at least + * get_options()->MinUptimeHidServDirectoryV2 seconds, we can't + * have accurate data telling us a relay has been up for at least + * that long. We also want to allow a bit of slack: Reachability + * tests aren't instant. If we haven't been running long enough, + * trust the relay. */ + + if (stats_n_seconds_working > + get_options()->MinUptimeHidServDirectoryV2 * 1.1) + uptime = MIN(rep_hist_get_uptime(router->cache_info.identity_digest, now), + real_uptime(router, now)); + else + uptime = real_uptime(router, now); + + /* XXX We shouldn't need to check dir_port, but we do because of + * bug 1693. In the future, once relays set wants_to_be_hs_dir + * correctly, we can revert to only checking dir_port if router's + * version is too old. */ + /* XXX Unfortunately, we need to keep checking dir_port until all + * *clients* suffering from bug 2722 are obsolete. The first version + * to fix the bug was 0.2.2.25-alpha. */ + return (router->wants_to_be_hs_dir && router->dir_port && uptime > get_options()->MinUptimeHidServDirectoryV2 && router->is_running); } @@ -1790,7 +1869,8 @@ dirserv_compute_performance_thresholds(routerlist_t *rl) if (router_is_active(ri, now)) { const char *id = ri->cache_info.identity_digest; uint32_t bw; - ri->is_exit = exit_policy_is_general_exit(ri->exit_policy); + ri->is_exit = (!router_exit_policy_rejects_all(ri) && + exit_policy_is_general_exit(ri->exit_policy)); uptimes[n_active] = (uint32_t)real_uptime(ri, now); mtbfs[n_active] = rep_hist_get_stability(id, now); tks [n_active] = rep_hist_get_weighted_time_known(id, now); @@ -1897,16 +1977,20 @@ version_from_platform(const char *platform) * which has at least <b>buf_len</b> free characters. Do NUL-termination. * Use the same format as in network-status documents. If <b>version</b> is * non-NULL, add a "v" line for the platform. Return 0 on success, -1 on - * failure. If <b>first_line_only</b> is true, don't include any flags - * or version line. + * failure. + * + * The format argument has three possible values: + * NS_V2 - Output an entry suitable for a V2 NS opinion document + * NS_V3_CONSENSUS - Output the first portion of a V3 NS consensus entry + * NS_V3_CONSENSUS_MICRODESC - Output the first portion of a V3 microdesc + * consensus entry. + * NS_V3_VOTE - Output a complete V3 NS vote + * NS_CONTROL_PORT - Output a NS document for the control port */ int routerstatus_format_entry(char *buf, size_t buf_len, routerstatus_t *rs, const char *version, - int first_line_only, int v2_format) -/* XXX: first_line_only and v2_format should probably be be both - * replaced by a single purpose parameter. - */ + routerstatus_format_type_t format) { int r; struct in_addr in; @@ -1925,10 +2009,11 @@ routerstatus_format_entry(char *buf, size_t buf_len, tor_inet_ntoa(&in, ipaddr, sizeof(ipaddr)); r = tor_snprintf(buf, buf_len, - "r %s %s %s %s %s %d %d\n", + "r %s %s %s%s%s %s %d %d\n", rs->nickname, identity64, - digest64, + (format==NS_V3_CONSENSUS_MICRODESC)?"":digest64, + (format==NS_V3_CONSENSUS_MICRODESC)?"":" ", published, ipaddr, (int)rs->or_port, @@ -1937,7 +2022,12 @@ routerstatus_format_entry(char *buf, size_t buf_len, log_warn(LD_BUG, "Not enough space in buffer."); return -1; } - if (first_line_only) + + /* TODO: Maybe we want to pass in what we need to build the rest of + * this here, instead of in the caller. Then we could use the + * networkstatus_type_t values, with an additional control port value + * added -MP */ + if (format == NS_V3_CONSENSUS || format == NS_V3_CONSENSUS_MICRODESC) return 0; cp = buf + strlen(buf); @@ -1974,62 +2064,87 @@ routerstatus_format_entry(char *buf, size_t buf_len, cp += strlen(cp); } - if (!v2_format) { + if (format != NS_V2) { routerinfo_t* desc = router_get_by_digest(rs->identity_digest); + uint32_t bw; + + if (format != NS_CONTROL_PORT) { + /* Blow up more or less nicely if we didn't get anything or not the + * thing we expected. + */ + if (!desc) { + char id[HEX_DIGEST_LEN+1]; + char dd[HEX_DIGEST_LEN+1]; + + base16_encode(id, sizeof(id), rs->identity_digest, DIGEST_LEN); + base16_encode(dd, sizeof(dd), rs->descriptor_digest, DIGEST_LEN); + log_warn(LD_BUG, "Cannot get any descriptor for %s " + "(wanted descriptor %s).", + id, dd); + return -1; + }; + + /* This assert can fire for the control port, because + * it can request NS documents before all descriptors + * have been fetched. */ + if (tor_memneq(desc->cache_info.signed_descriptor_digest, + rs->descriptor_digest, + DIGEST_LEN)) { + char rl_d[HEX_DIGEST_LEN+1]; + char rs_d[HEX_DIGEST_LEN+1]; + char id[HEX_DIGEST_LEN+1]; + + base16_encode(rl_d, sizeof(rl_d), + desc->cache_info.signed_descriptor_digest, DIGEST_LEN); + base16_encode(rs_d, sizeof(rs_d), rs->descriptor_digest, DIGEST_LEN); + base16_encode(id, sizeof(id), rs->identity_digest, DIGEST_LEN); + log_err(LD_BUG, "descriptor digest in routerlist does not match " + "the one in routerstatus: %s vs %s " + "(router %s)\n", + rl_d, rs_d, id); + + tor_assert(tor_memeq(desc->cache_info.signed_descriptor_digest, + rs->descriptor_digest, + DIGEST_LEN)); + }; + } - /* Blow up more or less nicely if we didn't get anything or not the - * thing we expected. - */ - if (!desc) { - char id[HEX_DIGEST_LEN+1]; - char dd[HEX_DIGEST_LEN+1]; - - base16_encode(id, sizeof(id), rs->identity_digest, DIGEST_LEN); - base16_encode(dd, sizeof(dd), rs->descriptor_digest, DIGEST_LEN); - log_warn(LD_BUG, "Cannot get any descriptor for %s " - "(wanted descriptor %s).", - id, dd); - return -1; - }; - if (fast_memcmp(desc->cache_info.signed_descriptor_digest, - rs->descriptor_digest, - DIGEST_LEN)) { - char rl_d[HEX_DIGEST_LEN+1]; - char rs_d[HEX_DIGEST_LEN+1]; - char id[HEX_DIGEST_LEN+1]; - - base16_encode(rl_d, sizeof(rl_d), - desc->cache_info.signed_descriptor_digest, DIGEST_LEN); - base16_encode(rs_d, sizeof(rs_d), rs->descriptor_digest, DIGEST_LEN); - base16_encode(id, sizeof(id), rs->identity_digest, DIGEST_LEN); - log_err(LD_BUG, "descriptor digest in routerlist does not match " - "the one in routerstatus: %s vs %s " - "(router %s)\n", - rl_d, rs_d, id); - - tor_assert(fast_memeq(desc->cache_info.signed_descriptor_digest, - rs->descriptor_digest, - DIGEST_LEN)); - }; - + if (format == NS_CONTROL_PORT && rs->has_bandwidth) { + bw = rs->bandwidth; + } else { + tor_assert(desc); + bw = router_get_advertised_bandwidth_capped(desc) / 1000; + } r = tor_snprintf(cp, buf_len - (cp-buf), - "w Bandwidth=%d\n", - router_get_advertised_bandwidth_capped(desc) / 1024); + "w Bandwidth=%d\n", bw); + if (r<0) { log_warn(LD_BUG, "Not enough space in buffer."); return -1; } cp += strlen(cp); + if (format == NS_V3_VOTE && rs->has_measured_bw) { + *--cp = '\0'; /* Kill "\n" */ + r = tor_snprintf(cp, buf_len - (cp-buf), + " Measured=%d\n", rs->measured_bw); + if (r<0) { + log_warn(LD_BUG, "Not enough space in buffer for weight line."); + return -1; + } + cp += strlen(cp); + } - summary = policy_summarize(desc->exit_policy); - r = tor_snprintf(cp, buf_len - (cp-buf), "p %s\n", summary); - if (r<0) { - log_warn(LD_BUG, "Not enough space in buffer."); + if (desc) { + summary = policy_summarize(desc->exit_policy); + r = tor_snprintf(cp, buf_len - (cp-buf), "p %s\n", summary); + if (r<0) { + log_warn(LD_BUG, "Not enough space in buffer."); + tor_free(summary); + return -1; + } + cp += strlen(cp); tor_free(summary); - return -1; } - cp += strlen(cp); - tor_free(summary); } return 0; @@ -2132,9 +2247,7 @@ get_possible_sybil_list(const smartlist_t *routers) /** Extract status information from <b>ri</b> and from other authority * functions and store it in <b>rs</b>>. If <b>naming</b>, consider setting - * the named flag in <b>rs</b>. If not <b>exits_can_be_guards</b>, never mark - * an exit as a guard. If <b>listbadexits</b>, consider setting the badexit - * flag. + * the named flag in <b>rs</b>. * * We assume that ri-\>is_running has already been set, e.g. by * dirserv_set_router_is_running(ri, now); @@ -2142,8 +2255,8 @@ get_possible_sybil_list(const smartlist_t *routers) void set_routerstatus_from_routerinfo(routerstatus_t *rs, routerinfo_t *ri, time_t now, - int naming, int exits_can_be_guards, - int listbadexits, int listbaddirs) + int naming, int listbadexits, + int listbaddirs) { int unstable_version = !tor_version_as_new_as(ri->platform,"0.1.1.16-rc-cvs"); @@ -2172,11 +2285,10 @@ set_routerstatus_from_routerinfo(routerstatus_t *rs, rs->is_valid = ri->is_valid; if (rs->is_fast && - (!rs->is_exit || exits_can_be_guards) && (router_get_advertised_bandwidth(ri) >= BANDWIDTH_TO_GUARANTEE_GUARD || router_get_advertised_bandwidth(ri) >= - (exits_can_be_guards ? guard_bandwidth_including_exits : - guard_bandwidth_excluding_exits))) { + MIN(guard_bandwidth_including_exits, + guard_bandwidth_excluding_exits))) { long tk = rep_hist_get_weighted_time_known( ri->cache_info.identity_digest, now); double wfu = rep_hist_get_weighted_fractional_uptime( @@ -2232,6 +2344,177 @@ router_clear_status_flags(routerinfo_t *router) router->is_bad_exit = router->is_bad_directory = 0; } +/** + * Helper function to parse out a line in the measured bandwidth file + * into a measured_bw_line_t output structure. Returns -1 on failure + * or 0 on success. + */ +int +measured_bw_line_parse(measured_bw_line_t *out, const char *orig_line) +{ + char *line = tor_strdup(orig_line); + char *cp = line; + int got_bw = 0; + int got_node_id = 0; + char *strtok_state; /* lame sauce d'jour */ + cp = tor_strtok_r(cp, " \t", &strtok_state); + + if (!cp) { + log_warn(LD_DIRSERV, "Invalid line in bandwidth file: %s", + escaped(orig_line)); + tor_free(line); + return -1; + } + + if (orig_line[strlen(orig_line)-1] != '\n') { + log_warn(LD_DIRSERV, "Incomplete line in bandwidth file: %s", + escaped(orig_line)); + tor_free(line); + return -1; + } + + do { + if (strcmpstart(cp, "bw=") == 0) { + int parse_ok = 0; + char *endptr; + if (got_bw) { + log_warn(LD_DIRSERV, "Double bw= in bandwidth file line: %s", + escaped(orig_line)); + tor_free(line); + return -1; + } + cp+=strlen("bw="); + + out->bw = tor_parse_long(cp, 0, 0, LONG_MAX, &parse_ok, &endptr); + if (!parse_ok || (*endptr && !TOR_ISSPACE(*endptr))) { + log_warn(LD_DIRSERV, "Invalid bandwidth in bandwidth file line: %s", + escaped(orig_line)); + tor_free(line); + return -1; + } + got_bw=1; + } else if (strcmpstart(cp, "node_id=$") == 0) { + if (got_node_id) { + log_warn(LD_DIRSERV, "Double node_id= in bandwidth file line: %s", + escaped(orig_line)); + tor_free(line); + return -1; + } + cp+=strlen("node_id=$"); + + if (strlen(cp) != HEX_DIGEST_LEN || + base16_decode(out->node_id, DIGEST_LEN, cp, HEX_DIGEST_LEN)) { + log_warn(LD_DIRSERV, "Invalid node_id in bandwidth file line: %s", + escaped(orig_line)); + tor_free(line); + return -1; + } + strncpy(out->node_hex, cp, sizeof(out->node_hex)); + got_node_id=1; + } + } while ((cp = tor_strtok_r(NULL, " \t", &strtok_state))); + + if (got_bw && got_node_id) { + tor_free(line); + return 0; + } else { + log_warn(LD_DIRSERV, "Incomplete line in bandwidth file: %s", + escaped(orig_line)); + tor_free(line); + return -1; + } +} + +/** + * Helper function to apply a parsed measurement line to a list + * of bandwidth statuses. Returns true if a line is found, + * false otherwise. + */ +int +measured_bw_line_apply(measured_bw_line_t *parsed_line, + smartlist_t *routerstatuses) +{ + routerstatus_t *rs = NULL; + if (!routerstatuses) + return 0; + + rs = smartlist_bsearch(routerstatuses, parsed_line->node_id, + compare_digest_to_routerstatus_entry); + + if (rs) { + rs->has_measured_bw = 1; + rs->measured_bw = (uint32_t)parsed_line->bw; + } else { + log_info(LD_DIRSERV, "Node ID %s not found in routerstatus list", + parsed_line->node_hex); + } + + return rs != NULL; +} + +/** + * Read the measured bandwidth file and apply it to the list of + * routerstatuses. Returns -1 on error, 0 otherwise. + */ +int +dirserv_read_measured_bandwidths(const char *from_file, + smartlist_t *routerstatuses) +{ + char line[256]; + FILE *fp = fopen(from_file, "r"); + int applied_lines = 0; + time_t file_time; + int ok; + if (fp == NULL) { + log_warn(LD_CONFIG, "Can't open bandwidth file at configured location: %s", + from_file); + return -1; + } + + if (!fgets(line, sizeof(line), fp) + || !strlen(line) || line[strlen(line)-1] != '\n') { + log_warn(LD_DIRSERV, "Long or truncated time in bandwidth file: %s", + escaped(line)); + fclose(fp); + return -1; + } + + line[strlen(line)-1] = '\0'; + file_time = tor_parse_ulong(line, 10, 0, ULONG_MAX, &ok, NULL); + if (!ok) { + log_warn(LD_DIRSERV, "Non-integer time in bandwidth file: %s", + escaped(line)); + fclose(fp); + return -1; + } + + if ((time(NULL) - file_time) > MAX_MEASUREMENT_AGE) { + log_warn(LD_DIRSERV, "Bandwidth measurement file stale. Age: %u", + (unsigned)(time(NULL) - file_time)); + fclose(fp); + return -1; + } + + if (routerstatuses) + smartlist_sort(routerstatuses, compare_routerstatus_entries); + + while (!feof(fp)) { + measured_bw_line_t parsed_line; + if (fgets(line, sizeof(line), fp) && strlen(line)) { + if (measured_bw_line_parse(&parsed_line, line) != -1) { + if (measured_bw_line_apply(&parsed_line, routerstatuses) > 0) + applied_lines++; + } + } + } + + fclose(fp); + log_info(LD_DIRSERV, + "Bandwidth measurement file successfully read. " + "Applied %d measurements.", applied_lines); + return 0; +} + /** Return a new networkstatus_t* containing our current opinion. (For v3 * authorities) */ networkstatus_t * @@ -2249,22 +2532,18 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key, int naming = options->NamingAuthoritativeDir; int listbadexits = options->AuthDirListBadExits; int listbaddirs = options->AuthDirListBadDirs; - int exits_can_be_guards; routerlist_t *rl = router_get_routerlist(); time_t now = time(NULL); time_t cutoff = now - ROUTER_MAX_AGE_TO_PUBLISH; networkstatus_voter_info_t *voter = NULL; vote_timing_t timing; digestmap_t *omit_as_sybil = NULL; - int vote_on_reachability = 1; + const int vote_on_reachability = running_long_enough_to_decide_unreachable(); + smartlist_t *microdescriptors = NULL; tor_assert(private_key); tor_assert(cert); - if (now - time_of_process_start < - options->TestingAuthDirTimeToLearnReachability) - vote_on_reachability = 0; - if (resolve_my_address(LOG_WARN, options, &addr, &hostname)<0) { log_warn(LD_NET, "Couldn't resolve my hostname"); return NULL; @@ -2299,27 +2578,24 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key, dirserv_compute_performance_thresholds(rl); - /* XXXX We should take steps to keep this from oscillating if - * total_exit_bandwidth is close to total_bandwidth/3. */ - exits_can_be_guards = total_exit_bandwidth >= (total_bandwidth / 3); - routers = smartlist_create(); smartlist_add_all(routers, rl->routers); routers_sort_by_identity(routers); omit_as_sybil = get_possible_sybil_list(routers); routerstatuses = smartlist_create(); + microdescriptors = smartlist_create(); - SMARTLIST_FOREACH(routers, routerinfo_t *, ri, { + SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) { if (ri->cache_info.published_on >= cutoff) { routerstatus_t *rs; vote_routerstatus_t *vrs; + microdesc_t *md; vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); rs = &vrs->status; set_routerstatus_from_routerinfo(rs, ri, now, - naming, exits_can_be_guards, - listbadexits, listbaddirs); + naming, listbadexits, listbaddirs); if (digestmap_get(omit_as_sybil, ri->cache_info.identity_digest)) clear_status_flags_on_sybil(rs); @@ -2328,12 +2604,39 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key, rs->is_running = 0; vrs->version = version_from_platform(ri->platform); + md = dirvote_create_microdescriptor(ri); + if (md) { + char buf[128]; + vote_microdesc_hash_t *h; + dirvote_format_microdesc_vote_line(buf, sizeof(buf), md); + h = tor_malloc(sizeof(vote_microdesc_hash_t)); + h->microdesc_hash_line = tor_strdup(buf); + h->next = NULL; + vrs->microdesc = h; + md->last_listed = now; + smartlist_add(microdescriptors, md); + } + smartlist_add(routerstatuses, vrs); } - }); + } SMARTLIST_FOREACH_END(ri); + + { + smartlist_t *added = + microdescs_add_list_to_cache(get_microdesc_cache(), + microdescriptors, SAVED_NOWHERE, 0); + smartlist_free(added); + smartlist_free(microdescriptors); + } + smartlist_free(routers); digestmap_free(omit_as_sybil, NULL); + if (options->V3BandwidthsFile) { + dirserv_read_measured_bandwidths(options->V3BandwidthsFile, + routerstatuses); + } + v3_out = tor_malloc_zero(sizeof(networkstatus_t)); v3_out->type = NS_TYPE_VOTE; @@ -2383,19 +2686,29 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key, } smartlist_sort_strings(v3_out->known_flags); + if (options->ConsensusParams) { + v3_out->net_params = smartlist_create(); + smartlist_split_string(v3_out->net_params, + options->ConsensusParams, NULL, 0, 0); + smartlist_sort_strings(v3_out->net_params); + } + voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t)); voter->nickname = tor_strdup(options->Nickname); memcpy(voter->identity_digest, identity_digest, DIGEST_LEN); + voter->sigs = smartlist_create(); voter->address = hostname; voter->addr = addr; - voter->dir_port = options->DirPort; - voter->or_port = options->ORPort; + voter->dir_port = router_get_advertised_dir_port(options); + voter->or_port = router_get_advertised_or_port(options); voter->contact = tor_strdup(contact); - memcpy(voter->signing_key_digest, signing_key_digest, DIGEST_LEN); if (options->V3AuthUseLegacyKey) { authority_cert_t *c = get_my_v3_legacy_cert(); if (c) { - crypto_pk_get_digest(c->identity_key, voter->legacy_id_digest); + if (crypto_pk_get_digest(c->identity_key, voter->legacy_id_digest)) { + log_warn(LD_BUG, "Unable to compute digest of legacy v3 identity key"); + memset(voter->legacy_id_digest, 0, DIGEST_LEN); + } } } @@ -2435,13 +2748,12 @@ generate_v2_networkstatus_opinion(void) int versioning = options->VersioningAuthoritativeDir; int listbaddirs = options->AuthDirListBadDirs; int listbadexits = options->AuthDirListBadExits; - int exits_can_be_guards; const char *contact; char *version_lines = NULL; smartlist_t *routers = NULL; digestmap_t *omit_as_sybil = NULL; - private_key = get_identity_key(); + private_key = get_server_identity_key(); if (resolve_my_address(LOG_WARN, options, &addr, &hostname)<0) { log_warn(LD_NET, "Couldn't resolve my hostname"); @@ -2494,7 +2806,7 @@ generate_v2_networkstatus_opinion(void) "dir-options%s%s%s%s\n" "%s" /* client version line, server version line. */ "dir-signing-key\n%s", - hostname, ipaddr, (int)options->DirPort, + hostname, ipaddr, (int)router_get_advertised_dir_port(options), fingerprint, contact, published, @@ -2515,10 +2827,6 @@ generate_v2_networkstatus_opinion(void) dirserv_compute_performance_thresholds(rl); - /* XXXX We should take steps to keep this from oscillating if - * total_exit_bandwidth is close to total_bandwidth/3. */ - exits_can_be_guards = total_exit_bandwidth >= (total_bandwidth / 3); - routers = smartlist_create(); smartlist_add_all(routers, rl->routers); routers_sort_by_identity(routers); @@ -2531,13 +2839,12 @@ generate_v2_networkstatus_opinion(void) char *version = version_from_platform(ri->platform); set_routerstatus_from_routerinfo(&rs, ri, now, - naming, exits_can_be_guards, - listbadexits, listbaddirs); + naming, listbadexits, listbaddirs); if (digestmap_get(omit_as_sybil, ri->cache_info.identity_digest)) clear_status_flags_on_sybil(&rs); - if (routerstatus_format_entry(outp, endp-outp, &rs, version, 0, 1)) { + if (routerstatus_format_entry(outp, endp-outp, &rs, version, NS_V2)) { log_warn(LD_BUG, "Unable to print router status."); tor_free(version); goto done; @@ -2559,7 +2866,8 @@ generate_v2_networkstatus_opinion(void) outp += strlen(outp); note_crypto_pk_op(SIGN_DIR); - if (router_append_dirobj_signature(outp,endp-outp,digest,private_key)<0) { + if (router_append_dirobj_signature(outp,endp-outp,digest,DIGEST_LEN, + private_key)<0) { log_warn(LD_BUG, "Unable to sign router status."); goto done; } @@ -2591,10 +2899,8 @@ generate_v2_networkstatus_opinion(void) tor_free(status); tor_free(hostname); tor_free(identity_pkey); - if (routers) - smartlist_free(routers); - if (omit_as_sybil) - digestmap_free(omit_as_sybil, NULL); + smartlist_free(routers); + digestmap_free(omit_as_sybil, NULL); return r; } @@ -2642,7 +2948,8 @@ dirserv_get_networkstatus_v2_fingerprints(smartlist_t *result, log_info(LD_DIRSERV, "Client requested 'all' network status objects; we have none."); } else if (!strcmpstart(key, "fp/")) { - dir_split_resource_into_fingerprints(key+3, result, NULL, 1, 1); + dir_split_resource_into_fingerprints(key+3, result, NULL, + DSR_HEX|DSR_SORT_UNIQ); } } @@ -2709,10 +3016,12 @@ dirserv_get_routerdesc_fingerprints(smartlist_t *fps_out, const char *key, } else if (!strcmpstart(key, "d/")) { by_id = 0; key += strlen("d/"); - dir_split_resource_into_fingerprints(key, fps_out, NULL, 1, 1); + dir_split_resource_into_fingerprints(key, fps_out, NULL, + DSR_HEX|DSR_SORT_UNIQ); } else if (!strcmpstart(key, "fp/")) { key += strlen("fp/"); - dir_split_resource_into_fingerprints(key, fps_out, NULL, 1, 1); + dir_split_resource_into_fingerprints(key, fps_out, NULL, + DSR_HEX|DSR_SORT_UNIQ); } else { *msg = "Key not recognized"; return -1; @@ -2777,7 +3086,8 @@ dirserv_get_routerdescs(smartlist_t *descs_out, const char *key, } else if (!strcmpstart(key, "/tor/server/d/")) { smartlist_t *digests = smartlist_create(); key += strlen("/tor/server/d/"); - dir_split_resource_into_fingerprints(key, digests, NULL, 1, 1); + dir_split_resource_into_fingerprints(key, digests, NULL, + DSR_HEX|DSR_SORT_UNIQ); SMARTLIST_FOREACH(digests, const char *, d, { signed_descriptor_t *sd = router_get_by_descriptor_digest(d); @@ -2790,7 +3100,8 @@ dirserv_get_routerdescs(smartlist_t *descs_out, const char *key, smartlist_t *digests = smartlist_create(); time_t cutoff = time(NULL) - ROUTER_MAX_AGE_TO_PUBLISH; key += strlen("/tor/server/fp/"); - dir_split_resource_into_fingerprints(key, digests, NULL, 1, 1); + dir_split_resource_into_fingerprints(key, digests, NULL, + DSR_HEX|DSR_SORT_UNIQ); SMARTLIST_FOREACH(digests, const char *, d, { if (router_digest_is_me(d)) { @@ -2841,34 +3152,82 @@ dirserv_orconn_tls_done(const char *address, tor_assert(address); tor_assert(digest_rcvd); - SMARTLIST_FOREACH(rl->routers, routerinfo_t *, ri, { + /* XXX023 Doing a loop like this is stupid. We should just look up the + * router by digest_rcvd, and see if address, orport, and as_advertised + * match up. -NM */ + SMARTLIST_FOREACH_BEGIN(rl->routers, routerinfo_t *, ri) { if (!strcasecmp(address, ri->address) && or_port == ri->or_port && as_advertised && fast_memeq(ri->cache_info.identity_digest, digest_rcvd, DIGEST_LEN)) { /* correct digest. mark this router reachable! */ if (!bridge_auth || ri->purpose == ROUTER_PURPOSE_BRIDGE) { - log_info(LD_DIRSERV, "Found router %s to be reachable. Yay.", - ri->nickname); - rep_hist_note_router_reachable(digest_rcvd, now); + tor_addr_t addr, *addrp=NULL; + log_info(LD_DIRSERV, "Found router %s to be reachable at %s:%d. Yay.", + ri->nickname, address, ri->or_port ); + if (tor_addr_from_str(&addr, ri->address) != -1) + addrp = &addr; + else + log_warn(LD_BUG, "Couldn't parse IP address \"%s\"", ri->address); + rep_hist_note_router_reachable(digest_rcvd, addrp, or_port, now); ri->last_reachable = now; } } - }); + } SMARTLIST_FOREACH_END(ri); /* FFFF Maybe we should reinstate the code that dumps routers with the same * addr/port but with nonmatching keys, but instead of dumping, we should * skip testing. */ } -/** Auth dir server only: if <b>try_all</b> is 1, launch connections to - * all known routers; else we want to load balance such that we only +/** Called when we, as an authority, receive a new router descriptor either as + * an upload or a download. Used to decide whether to relaunch reachability + * testing for the server. */ +int +dirserv_should_launch_reachability_test(routerinfo_t *ri, routerinfo_t *ri_old) +{ + if (!authdir_mode_handles_descs(get_options(), ri->purpose)) + return 0; + if (!ri_old) { + /* New router: Launch an immediate reachability test, so we will have an + * opinion soon in case we're generating a consensus soon */ + return 1; + } + if (ri_old->is_hibernating && !ri->is_hibernating) { + /* It just came out of hibernation; launch a reachability test */ + return 1; + } + if (! routers_have_same_or_addr(ri, ri_old)) { + /* Address or port changed; launch a reachability test */ + return 1; + } + return 0; +} + +/** Helper function for dirserv_test_reachability(). Start a TLS + * connection to <b>router</b>, and annotate it with when we started + * the test. */ +void +dirserv_single_reachability_test(time_t now, routerinfo_t *router) +{ + tor_addr_t router_addr; + log_debug(LD_OR,"Testing reachability of %s at %s:%u.", + router->nickname, router->address, router->or_port); + /* Remember when we started trying to determine reachability */ + if (!router->testing_since) + router->testing_since = now; + tor_addr_from_ipv4h(&router_addr, router->addr); + connection_or_connect(&router_addr, router->or_port, + router->cache_info.identity_digest); +} + +/** Auth dir server only: load balance such that we only * try a few connections per call. * * The load balancing is such that if we get called once every ten - * seconds, we will cycle through all the tests in 1280 seconds (a - * bit over 20 minutes). + * seconds, we will cycle through all the tests in + * REACHABILITY_TEST_CYCLE_PERIOD seconds (a bit over 20 minutes). */ void -dirserv_test_reachability(time_t now, int try_all) +dirserv_test_reachability(time_t now) { /* XXX decide what to do here; see or-talk thread "purging old router * information, revocation." -NM @@ -2885,38 +3244,33 @@ dirserv_test_reachability(time_t now, int try_all) SMARTLIST_FOREACH_BEGIN(rl->routers, routerinfo_t *, router) { const char *id_digest = router->cache_info.identity_digest; - tor_addr_t router_addr; if (router_is_me(router)) continue; if (bridge_auth && router->purpose != ROUTER_PURPOSE_BRIDGE) continue; /* bridge authorities only test reachability on bridges */ // if (router->cache_info.published_on > cutoff) // continue; - if (try_all || (((uint8_t)id_digest[0]) % 128) == ctr) { - log_debug(LD_OR,"Testing reachability of %s at %s:%u.", - router->nickname, router->address, router->or_port); - /* Remember when we started trying to determine reachability */ - if (!router->testing_since) - router->testing_since = now; - tor_addr_from_ipv4h(&router_addr, router->addr); - connection_or_connect(&router_addr, router->or_port, id_digest); + if ((((uint8_t)id_digest[0]) % REACHABILITY_MODULO_PER_TEST) == ctr) { + dirserv_single_reachability_test(now, router); } } SMARTLIST_FOREACH_END(router); - if (!try_all) /* increment ctr */ - ctr = (ctr + 1) % 128; + ctr = (ctr + 1) % REACHABILITY_MODULO_PER_TEST; /* increment ctr */ } -/** Given a fingerprint <b>fp</b> which is either set if we're looking - * for a v2 status, or zeroes if we're looking for a v3 status, return - * a pointer to the appropriate cached dir object, or NULL if there isn't - * one available. */ +/** Given a fingerprint <b>fp</b> which is either set if we're looking for a + * v2 status, or zeroes if we're looking for a v3 status, or a NUL-padded + * flavor name if we want a flavored v3 status, return a pointer to the + * appropriate cached dir object, or NULL if there isn't one available. */ static cached_dir_t * lookup_cached_dir_by_fp(const char *fp) { cached_dir_t *d = NULL; - if (tor_digest_is_zero(fp) && cached_v3_networkstatus) - d = cached_v3_networkstatus; - else if (router_digest_is_me(fp) && the_v2_networkstatus) + if (tor_digest_is_zero(fp) && cached_consensuses) + d = strmap_get(cached_consensuses, "ns"); + else if (memchr(fp, '\0', DIGEST_LEN) && cached_consensuses && + (d = strmap_get(cached_consensuses, fp))) { + /* this here interface is a nasty hack XXXX023 */; + } else if (router_digest_is_me(fp) && the_v2_networkstatus) d = the_v2_networkstatus; else if (cached_v2_networkstatus) d = digestmap_get(cached_v2_networkstatus, fp); @@ -3002,6 +3356,18 @@ dirserv_have_any_serverdesc(smartlist_t *fps, int spool_src) return 0; } +/** Return true iff any of the 256-bit elements in <b>fps</b> is the digest of + * a microdescriptor we have. */ +int +dirserv_have_any_microdesc(const smartlist_t *fps) +{ + microdesc_cache_t *cache = get_microdesc_cache(); + SMARTLIST_FOREACH(fps, const char *, fp, + if (microdesc_cache_lookup_by_digest256(cache, fp)) + return 1); + return 0; +} + /** Return an approximate estimate of the number of bytes that will * be needed to transmit the server descriptors (if is_serverdescs -- * they can be either d/ or fp/ queries) or networkstatus objects (if @@ -3033,6 +3399,17 @@ dirserv_estimate_data_size(smartlist_t *fps, int is_serverdescs, return result; } +/** Given a list of microdescriptor hashes, guess how many bytes will be + * needed to transmit them, and return the guess. */ +size_t +dirserv_estimate_microdesc_size(const smartlist_t *fps, int compressed) +{ + size_t result = smartlist_len(fps) * microdesc_average_size(NULL); + if (compressed) + result /= 2; + return result; +} + /** When we're spooling data onto our outbuf, add more whenever we dip * below this threshold. */ #define DIRSERV_BUFFER_MIN 16384 @@ -3096,6 +3473,8 @@ connection_dirserv_add_servers_to_outbuf(dir_connection_t *conn) #endif body = signed_descriptor_get_body(sd); if (conn->zlib_state) { + /* XXXX022 This 'last' business should actually happen on the last + * routerinfo, not on the last fingerprint. */ int last = ! smartlist_len(conn->fingerprint_stack); connection_write_to_buf_zlib(body, sd->signed_descriptor_len, conn, last); @@ -3119,6 +3498,44 @@ connection_dirserv_add_servers_to_outbuf(dir_connection_t *conn) return 0; } +/** Spooling helper: called when we're sending a bunch of microdescriptors, + * and the outbuf has become too empty. Pulls some entries from + * fingerprint_stack, and writes the corresponding microdescs onto outbuf. If + * we run out of entries, flushes the zlib state and sets the spool source to + * NONE. Returns 0 on success, negative on failure. + */ +static int +connection_dirserv_add_microdescs_to_outbuf(dir_connection_t *conn) +{ + microdesc_cache_t *cache = get_microdesc_cache(); + while (smartlist_len(conn->fingerprint_stack) && + buf_datalen(conn->_base.outbuf) < DIRSERV_BUFFER_MIN) { + char *fp256 = smartlist_pop_last(conn->fingerprint_stack); + microdesc_t *md = microdesc_cache_lookup_by_digest256(cache, fp256); + tor_free(fp256); + if (!md) + continue; + if (conn->zlib_state) { + /* XXXX022 This 'last' business should actually happen on the last + * routerinfo, not on the last fingerprint. */ + int last = !smartlist_len(conn->fingerprint_stack); + connection_write_to_buf_zlib(md->body, md->bodylen, conn, last); + if (last) { + tor_zlib_free(conn->zlib_state); + conn->zlib_state = NULL; + } + } else { + connection_write_to_buf(md->body, md->bodylen, TO_CONN(conn)); + } + } + if (!smartlist_len(conn->fingerprint_stack)) { + conn->dir_spool_src = DIR_SPOOL_NONE; + smartlist_free(conn->fingerprint_stack); + conn->fingerprint_stack = NULL; + } + return 0; +} + /** Spooling helper: Called when we're sending a directory or networkstatus, * and the outbuf has become too empty. Pulls some bytes from * <b>conn</b>-\>cached_dir-\>dir_z, uncompresses them if appropriate, and @@ -3201,8 +3618,7 @@ connection_dirserv_add_networkstatus_bytes_to_outbuf(dir_connection_t *conn) } } else { connection_dirserv_finish_spooling(conn); - if (conn->fingerprint_stack) - smartlist_free(conn->fingerprint_stack); + smartlist_free(conn->fingerprint_stack); conn->fingerprint_stack = NULL; return 0; } @@ -3226,6 +3642,8 @@ connection_dirserv_flushed_some(dir_connection_t *conn) case DIR_SPOOL_SERVER_BY_DIGEST: case DIR_SPOOL_SERVER_BY_FP: return connection_dirserv_add_servers_to_outbuf(conn); + case DIR_SPOOL_MICRODESC: + return connection_dirserv_add_microdescs_to_outbuf(conn); case DIR_SPOOL_CACHED_DIR: return connection_dirserv_add_dir_bytes_to_outbuf(conn); case DIR_SPOOL_NETWORKSTATUS: @@ -3247,10 +3665,10 @@ dirserv_free_all(void) cached_dir_decref(the_v2_networkstatus); cached_dir_decref(cached_directory); clear_cached_dir(&cached_runningrouters); - if (cached_v2_networkstatus) { - digestmap_free(cached_v2_networkstatus, _free_cached_dir); - cached_v2_networkstatus = NULL; - } - cached_dir_decref(cached_v3_networkstatus); + + digestmap_free(cached_v2_networkstatus, _free_cached_dir); + cached_v2_networkstatus = NULL; + strmap_free(cached_consensuses, _free_cached_dir); + cached_consensuses = NULL; } diff --git a/src/or/dirserv.h b/src/or/dirserv.h new file mode 100644 index 0000000000..569abfca2e --- /dev/null +++ b/src/or/dirserv.h @@ -0,0 +1,146 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file dirserv.h + * \brief Header file for dirserv.c. + **/ + +#ifndef _TOR_DIRSERV_H +#define _TOR_DIRSERV_H + +/** What fraction (1 over this number) of the relay ID space do we + * (as a directory authority) launch connections to at each reachability + * test? */ +#define REACHABILITY_MODULO_PER_TEST 128 + +/** How often (in seconds) do we launch reachability tests? */ +#define REACHABILITY_TEST_INTERVAL 10 + +/** How many seconds apart are the reachability tests for a given relay? */ +#define REACHABILITY_TEST_CYCLE_PERIOD \ + (REACHABILITY_TEST_INTERVAL*REACHABILITY_MODULO_PER_TEST) + +/** Maximum length of an exit policy summary. */ +#define MAX_EXITPOLICY_SUMMARY_LEN 1000 + +/** Maximum allowable length of a version line in a networkstatus. */ +#define MAX_V_LINE_LEN 128 +/** Length of "r Authority BadDirectory BadExit Exit Fast Guard HSDir Named + * Running Stable Unnamed V2Dir Valid\n". */ +#define MAX_FLAG_LINE_LEN 96 +/** Length of "w" line for weighting. Currently at most + * "w Bandwidth=<uint32t> Measured=<uint32t>\n" */ +#define MAX_WEIGHT_LINE_LEN (12+10+10+10+1) +/** Maximum length of an exit policy summary line. */ +#define MAX_POLICY_LINE_LEN (3+MAX_EXITPOLICY_SUMMARY_LEN) +/** Amount of space to allocate for each entry: r, s, and v lines. */ +#define RS_ENTRY_LEN \ + ( /* first line */ \ + MAX_NICKNAME_LEN+BASE64_DIGEST_LEN*2+ISO_TIME_LEN+INET_NTOA_BUF_LEN+ \ + 5*2 /* ports */ + 10 /* punctuation */ + \ + /* second line */ \ + MAX_FLAG_LINE_LEN + \ + /* weight line */ \ + MAX_WEIGHT_LINE_LEN + \ + /* p line. */ \ + MAX_POLICY_LINE_LEN + \ + /* v line. */ \ + MAX_V_LINE_LEN \ + ) + +#define UNNAMED_ROUTER_NICKNAME "Unnamed" + +int connection_dirserv_flushed_some(dir_connection_t *conn); + +int dirserv_add_own_fingerprint(const char *nickname, crypto_pk_env_t *pk); +int dirserv_load_fingerprint_file(void); +void dirserv_free_fingerprint_list(void); +const char *dirserv_get_nickname_by_digest(const char *digest); +enum was_router_added_t dirserv_add_multiple_descriptors( + const char *desc, uint8_t purpose, + const char *source, + const char **msg); +enum was_router_added_t dirserv_add_descriptor(routerinfo_t *ri, + const char **msg, + const char *source); +void dirserv_set_router_is_running(routerinfo_t *router, time_t now); +int list_server_status_v1(smartlist_t *routers, char **router_status_out, + int for_controller); +int dirserv_dump_directory_to_string(char **dir_out, + crypto_pk_env_t *private_key); + +int directory_fetches_from_authorities(or_options_t *options); +int directory_fetches_dir_info_early(or_options_t *options); +int directory_fetches_dir_info_later(or_options_t *options); +int directory_caches_v2_dir_info(or_options_t *options); +#define directory_caches_v1_dir_info(o) directory_caches_v2_dir_info(o) +int directory_caches_dir_info(or_options_t *options); +int directory_permits_begindir_requests(or_options_t *options); +int directory_permits_controller_requests(or_options_t *options); +int directory_too_idle_to_fetch_descriptors(or_options_t *options, time_t now); + +void directory_set_dirty(void); +cached_dir_t *dirserv_get_directory(void); +cached_dir_t *dirserv_get_runningrouters(void); +cached_dir_t *dirserv_get_consensus(const char *flavor_name); +void dirserv_set_cached_directory(const char *directory, time_t when, + int is_running_routers); +void dirserv_set_cached_networkstatus_v2(const char *directory, + const char *identity, + time_t published); +void dirserv_set_cached_consensus_networkstatus(const char *consensus, + const char *flavor_name, + const digests_t *digests, + time_t published); +void dirserv_clear_old_networkstatuses(time_t cutoff); +void dirserv_clear_old_v1_info(time_t now); +void dirserv_get_networkstatus_v2(smartlist_t *result, const char *key); +void dirserv_get_networkstatus_v2_fingerprints(smartlist_t *result, + const char *key); +int dirserv_get_routerdesc_fingerprints(smartlist_t *fps_out, const char *key, + const char **msg, + int for_unencrypted_conn, + int is_extrainfo); +int dirserv_get_routerdescs(smartlist_t *descs_out, const char *key, + const char **msg); +void dirserv_orconn_tls_done(const char *address, + uint16_t or_port, + const char *digest_rcvd, + int as_advertised); +int dirserv_should_launch_reachability_test(routerinfo_t *ri, + routerinfo_t *ri_old); +void dirserv_single_reachability_test(time_t now, routerinfo_t *router); +void dirserv_test_reachability(time_t now); +int authdir_wants_to_reject_router(routerinfo_t *ri, const char **msg, + int complain); +int dirserv_would_reject_router(routerstatus_t *rs); +int dirserv_remove_old_statuses(smartlist_t *fps, time_t cutoff); +int dirserv_have_any_serverdesc(smartlist_t *fps, int spool_src); +int dirserv_have_any_microdesc(const smartlist_t *fps); +size_t dirserv_estimate_data_size(smartlist_t *fps, int is_serverdescs, + int compressed); +size_t dirserv_estimate_microdesc_size(const smartlist_t *fps, int compressed); + +int routerstatus_format_entry(char *buf, size_t buf_len, + routerstatus_t *rs, const char *platform, + routerstatus_format_type_t format); +void dirserv_free_all(void); +void cached_dir_decref(cached_dir_t *d); +cached_dir_t *new_cached_dir(char *s, time_t published); + +#ifdef DIRSERV_PRIVATE +int measured_bw_line_parse(measured_bw_line_t *out, const char *line); + +int measured_bw_line_apply(measured_bw_line_t *parsed_line, + smartlist_t *routerstatuses); +#endif + +int dirserv_read_measured_bandwidths(const char *from_file, + smartlist_t *routerstatuses); + +#endif + diff --git a/src/or/dirvote.c b/src/or/dirvote.c index f1381988c9..c6ce9f6776 100644 --- a/src/or/dirvote.c +++ b/src/or/dirvote.c @@ -5,27 +5,72 @@ #define DIRVOTE_PRIVATE #include "or.h" +#include "config.h" +#include "directory.h" +#include "dirserv.h" +#include "dirvote.h" +#include "microdesc.h" +#include "networkstatus.h" +#include "policies.h" +#include "rephist.h" +#include "router.h" +#include "routerlist.h" +#include "routerparse.h" /** * \file dirvote.c * \brief Functions to compute directory consensus, and schedule voting. **/ -static int dirvote_add_signatures_to_pending_consensus( +/** A consensus that we have built and are appending signatures to. Once it's + * time to publish it, it will become an active consensus if it accumulates + * enough signatures. */ +typedef struct pending_consensus_t { + /** The body of the consensus that we're currently building. Once we + * have it built, it goes into dirserv.c */ + char *body; + /** The parsed in-progress consensus document. */ + networkstatus_t *consensus; +} pending_consensus_t; + +static int dirvote_add_signatures_to_all_pending_consensuses( const char *detached_signatures_body, const char **msg_out); +static int dirvote_add_signatures_to_pending_consensus( + pending_consensus_t *pc, + ns_detached_signatures_t *sigs, + const char **msg_out); static char *list_v3_auth_ids(void); static void dirvote_fetch_missing_votes(void); static void dirvote_fetch_missing_signatures(void); static int dirvote_perform_vote(void); static void dirvote_clear_votes(int all_votes); -static int dirvote_compute_consensus(void); +static int dirvote_compute_consensuses(void); static int dirvote_publish_consensus(void); +static char *make_consensus_method_list(int low, int high, const char *sep); + +/** The highest consensus method that we currently support. */ +#define MAX_SUPPORTED_CONSENSUS_METHOD 11 + +/** Lowest consensus method that contains a 'directory-footer' marker */ +#define MIN_METHOD_FOR_FOOTER 9 + +/** Lowest consensus method that contains bandwidth weights */ +#define MIN_METHOD_FOR_BW_WEIGHTS 9 + +/** Lowest consensus method that contains consensus params */ +#define MIN_METHOD_FOR_PARAMS 7 + +/** Lowest consensus method that generates microdescriptors */ +#define MIN_METHOD_FOR_MICRODESC 8 /* ===== * Voting * =====*/ +/* Overestimated. */ +#define MICRODESC_LINE_LEN 80 + /** Return a new string containing the string representation of the vote in * <b>v3_ns</b>, signed with our v3 signing key <b>private_signing_key</b>. * For v3 authorities. */ @@ -44,6 +89,7 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key, uint32_t addr; routerlist_t *rl = router_get_routerlist(); char *version_lines = NULL; + int r; networkstatus_voter_info_t *voter; tor_assert(private_signing_key); @@ -70,20 +116,30 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key, version_lines = tor_malloc(v_len); cp = version_lines; if (client_versions) { - tor_snprintf(cp, v_len-(cp-version_lines), + r = tor_snprintf(cp, v_len-(cp-version_lines), "client-versions %s\n", client_versions); + if (r < 0) { + log_err(LD_BUG, "Insufficient memory for client-versions line"); + tor_assert(0); + } cp += strlen(cp); } - if (server_versions) - tor_snprintf(cp, v_len-(cp-version_lines), + if (server_versions) { + r = tor_snprintf(cp, v_len-(cp-version_lines), "server-versions %s\n", server_versions); + if (r < 0) { + log_err(LD_BUG, "Insufficient memory for server-versions line"); + tor_assert(0); + } + } } else { version_lines = tor_strdup(""); } len = 8192; len += strlen(version_lines); - len += (RS_ENTRY_LEN)*smartlist_len(rl->routers); + len += (RS_ENTRY_LEN+MICRODESC_LINE_LEN)*smartlist_len(rl->routers); + len += strlen("\ndirectory-footer\n"); len += v3_ns->cert->cache_info.signed_descriptor_len; status = tor_malloc(len); @@ -93,17 +149,25 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key, char fu[ISO_TIME_LEN+1]; char vu[ISO_TIME_LEN+1]; char *flags = smartlist_join_strings(v3_ns->known_flags, " ", 0, NULL); + char *params; authority_cert_t *cert = v3_ns->cert; + char *methods = + make_consensus_method_list(1, MAX_SUPPORTED_CONSENSUS_METHOD, " "); format_iso_time(published, v3_ns->published); format_iso_time(va, v3_ns->valid_after); format_iso_time(fu, v3_ns->fresh_until); format_iso_time(vu, v3_ns->valid_until); + if (v3_ns->net_params) + params = smartlist_join_strings(v3_ns->net_params, " ", 0, NULL); + else + params = tor_strdup(""); + tor_assert(cert); - tor_snprintf(status, len, + r = tor_snprintf(status, len, "network-status-version 3\n" "vote-status %s\n" - "consensus-methods 1 2 3 4 5\n" + "consensus-methods %s\n" "published %s\n" "valid-after %s\n" "fresh-until %s\n" @@ -111,24 +175,38 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key, "voting-delay %d %d\n" "%s" /* versions */ "known-flags %s\n" + "params %s\n" "dir-source %s %s %s %s %d %d\n" "contact %s\n", v3_ns->type == NS_TYPE_VOTE ? "vote" : "opinion", + methods, published, va, fu, vu, v3_ns->vote_seconds, v3_ns->dist_seconds, version_lines, flags, + params, voter->nickname, fingerprint, voter->address, - ipaddr, voter->dir_port, voter->or_port, voter->contact); + ipaddr, voter->dir_port, voter->or_port, voter->contact); + if (r < 0) { + log_err(LD_BUG, "Insufficient memory for network status line"); + tor_assert(0); + } + + tor_free(params); tor_free(flags); + tor_free(methods); outp = status + strlen(status); endp = status + len; if (!tor_digest_is_zero(voter->legacy_id_digest)) { char fpbuf[HEX_DIGEST_LEN+1]; base16_encode(fpbuf, sizeof(fpbuf), voter->legacy_id_digest, DIGEST_LEN); - tor_snprintf(outp, endp-outp, "legacy-dir-key %s\n", fpbuf); + r = tor_snprintf(outp, endp-outp, "legacy-dir-key %s\n", fpbuf); + if (r < 0) { + log_err(LD_BUG, "Insufficient memory for legacy-dir-key line"); + tor_assert(0); + } outp += strlen(outp); } @@ -139,15 +217,32 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key, outp += cert->cache_info.signed_descriptor_len; } - SMARTLIST_FOREACH(v3_ns->routerstatus_list, vote_routerstatus_t *, vrs, - { + SMARTLIST_FOREACH_BEGIN(v3_ns->routerstatus_list, vote_routerstatus_t *, + vrs) { + vote_microdesc_hash_t *h; if (routerstatus_format_entry(outp, endp-outp, &vrs->status, - vrs->version, 0, 0) < 0) { + vrs->version, NS_V3_VOTE) < 0) { log_warn(LD_BUG, "Unable to print router status."); goto err; } outp += strlen(outp); - }); + + for (h = vrs->microdesc; h; h = h->next) { + size_t mlen = strlen(h->microdesc_hash_line); + if (outp+mlen >= endp) { + log_warn(LD_BUG, "Can't fit microdesc line in vote."); + } + memcpy(outp, h->microdesc_hash_line, mlen+1); + outp += strlen(outp); + } + } SMARTLIST_FOREACH_END(vrs); + + r = tor_snprintf(outp, endp-outp, "directory-footer\n"); + if (r < 0) { + log_err(LD_BUG, "Insufficient memory for directory-footer line"); + tor_assert(0); + } + outp += strlen(outp); { char signing_key_fingerprint[FINGERPRINT_LEN+1]; @@ -170,10 +265,10 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key, outp += strlen(outp); } - if (router_get_networkstatus_v3_hash(status, digest)<0) + if (router_get_networkstatus_v3_hash(status, digest, DIGEST_SHA1)<0) goto err; note_crypto_pk_op(SIGN_DIR); - if (router_append_dirobj_signature(outp,endp-outp,digest, + if (router_append_dirobj_signature(outp,endp-outp,digest, DIGEST_LEN, private_signing_key)<0) { log_warn(LD_BUG, "Unable to sign networkstatus vote."); goto err; @@ -216,6 +311,20 @@ get_voter(const networkstatus_t *vote) return smartlist_get(vote->voters, 0); } +/** Return the signature made by <b>voter</b> using the algorithm + * <b>alg</b>, or NULL if none is found. */ +document_signature_t * +voter_get_sig_by_algorithm(const networkstatus_voter_info_t *voter, + digest_algorithm_t alg) +{ + if (!voter->sigs) + return NULL; + SMARTLIST_FOREACH(voter->sigs, document_signature_t *, sig, + if (sig->alg == alg) + return sig); + return NULL; +} + /** Temporary structure used in constructing a list of dir-source entries * for a consensus. One of these is generated for every vote, and one more * for every legacy key in each vote. */ @@ -275,34 +384,8 @@ get_frequent_members(smartlist_t *out, smartlist_t *in, int min) /** Given a sorted list of strings <b>lst</b>, return the member that appears * most. Break ties in favor of later-occurring members. */ -static const char * -get_most_frequent_member(smartlist_t *lst) -{ - const char *most_frequent = NULL; - int most_frequent_count = 0; - - const char *cur = NULL; - int count = 0; - - SMARTLIST_FOREACH(lst, const char *, s, - { - if (cur && !strcmp(s, cur)) { - ++count; - } else { - if (count >= most_frequent_count) { - most_frequent = cur; - most_frequent_count = count; - } - cur = s; - count = 1; - } - }); - if (count >= most_frequent_count) { - most_frequent = cur; - most_frequent_count = count; - } - return most_frequent; -} +#define get_most_frequent_member(lst) \ + smartlist_get_most_frequent_string(lst) /** Return 0 if and only if <b>a</b> and <b>b</b> are routerstatuses * that come from the same routerinfo, with the same derived elements. @@ -345,7 +428,8 @@ _compare_vote_rs(const void **_a, const void **_b) * in favor of smaller descriptor digest. */ static vote_routerstatus_t * -compute_routerstatus_consensus(smartlist_t *votes) +compute_routerstatus_consensus(smartlist_t *votes, int consensus_method, + char *microdesc_digest256_out) { vote_routerstatus_t *most = NULL, *cur = NULL; int most_n = 0, cur_n = 0; @@ -361,9 +445,9 @@ compute_routerstatus_consensus(smartlist_t *votes) if (cur && !compare_vote_rs(cur, rs)) { ++cur_n; } else { - if (cur_n > most_n || - (cur && cur_n == most_n && - cur->status.published_on > most_published)) { + if (cur && (cur_n > most_n || + (cur_n == most_n && + cur->status.published_on > most_published))) { most = cur; most_n = cur_n; most_published = cur->status.published_on; @@ -381,18 +465,45 @@ compute_routerstatus_consensus(smartlist_t *votes) } tor_assert(most); + + if (consensus_method >= MIN_METHOD_FOR_MICRODESC && + microdesc_digest256_out) { + smartlist_t *digests = smartlist_create(); + const char *best_microdesc_digest; + SMARTLIST_FOREACH_BEGIN(votes, vote_routerstatus_t *, rs) { + char d[DIGEST256_LEN]; + if (compare_vote_rs(rs, most)) + continue; + if (!vote_routerstatus_find_microdesc_hash(d, rs, consensus_method, + DIGEST_SHA256)) + smartlist_add(digests, tor_memdup(d, sizeof(d))); + } SMARTLIST_FOREACH_END(rs); + smartlist_sort_digests256(digests); + best_microdesc_digest = smartlist_get_most_frequent_digest256(digests); + if (best_microdesc_digest) + memcpy(microdesc_digest256_out, best_microdesc_digest, DIGEST256_LEN); + SMARTLIST_FOREACH(digests, char *, cp, tor_free(cp)); + smartlist_free(digests); + } + return most; } -/** Given a list of strings in <b>lst</b>, set the DIGEST_LEN-byte digest at - * <b>digest_out</b> to the hash of the concatenation of those strings. */ +/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest + * at <b>digest_out</b> to the hash of the concatenation of those strings, + * computed with the algorithm <b>alg</b>. */ static void -hash_list_members(char *digest_out, smartlist_t *lst) +hash_list_members(char *digest_out, size_t len_out, + smartlist_t *lst, digest_algorithm_t alg) { - crypto_digest_env_t *d = crypto_new_digest_env(); + crypto_digest_env_t *d; + if (alg == DIGEST_SHA1) + d = crypto_new_digest_env(); + else + d = crypto_new_digest256_env(alg); SMARTLIST_FOREACH(lst, const char *, cp, crypto_digest_add_bytes(d, cp, strlen(cp))); - crypto_digest_get_digest(d, digest_out, DIGEST_LEN); + crypto_digest_get_digest(d, digest_out, len_out); crypto_free_digest_env(d); } @@ -456,7 +567,31 @@ compute_consensus_method(smartlist_t *votes) static int consensus_method_is_supported(int method) { - return (method >= 1) && (method <= 5); + return (method >= 1) && (method <= MAX_SUPPORTED_CONSENSUS_METHOD); +} + +/** Return a newly allocated string holding the numbers between low and high + * (inclusive) that are supported consensus methods. */ +static char * +make_consensus_method_list(int low, int high, const char *separator) +{ + char *list; + + char b[32]; + int i; + smartlist_t *lst; + lst = smartlist_create(); + for (i = low; i <= high; ++i) { + if (!consensus_method_is_supported(i)) + continue; + tor_snprintf(b, sizeof(b), "%d", i); + smartlist_add(lst, tor_strdup(b)); + } + list = smartlist_join_strings(lst, separator, 0, NULL); + tor_assert(list); + SMARTLIST_FOREACH(lst, char *, cp, tor_free(cp)); + smartlist_free(lst); + return list; } /** Helper: given <b>lst</b>, a list of version strings such that every @@ -476,6 +611,728 @@ compute_consensus_versions_list(smartlist_t *lst, int n_versioning) return result; } +/** Helper: given a list of valid networkstatus_t, return a new string + * containing the contents of the consensus network parameter set. + */ +/* private */ char * +dirvote_compute_params(smartlist_t *votes) +{ + int i; + int32_t *vals; + + int cur_param_len; + const char *cur_param; + const char *eq; + char *result; + + const int n_votes = smartlist_len(votes); + smartlist_t *output; + smartlist_t *param_list = smartlist_create(); + + /* We require that the parameter lists in the votes are well-formed: that + is, that their keywords are unique and sorted, and that their values are + between INT32_MIN and INT32_MAX inclusive. This should be guaranteed by + the parsing code. */ + + vals = tor_malloc(sizeof(int)*n_votes); + + SMARTLIST_FOREACH_BEGIN(votes, networkstatus_t *, v) { + if (!v->net_params) + continue; + smartlist_add_all(param_list, v->net_params); + } SMARTLIST_FOREACH_END(v); + + if (smartlist_len(param_list) == 0) { + tor_free(vals); + smartlist_free(param_list); + return NULL; + } + + smartlist_sort_strings(param_list); + i = 0; + cur_param = smartlist_get(param_list, 0); + eq = strchr(cur_param, '='); + tor_assert(eq); + cur_param_len = (int)(eq+1 - cur_param); + + output = smartlist_create(); + + SMARTLIST_FOREACH_BEGIN(param_list, const char *, param) { + const char *next_param; + int ok=0; + eq = strchr(param, '='); + tor_assert(i<n_votes); + vals[i++] = (int32_t) + tor_parse_long(eq+1, 10, INT32_MIN, INT32_MAX, &ok, NULL); + tor_assert(ok); + + if (param_sl_idx+1 == smartlist_len(param_list)) + next_param = NULL; + else + next_param = smartlist_get(param_list, param_sl_idx+1); + if (!next_param || strncmp(next_param, param, cur_param_len)) { + /* We've reached the end of a series. */ + int32_t median = median_int32(vals, i); + char *out_string = tor_malloc(64+cur_param_len); + memcpy(out_string, param, cur_param_len); + tor_snprintf(out_string+cur_param_len,64, "%ld", (long)median); + smartlist_add(output, out_string); + + i = 0; + if (next_param) { + eq = strchr(next_param, '='); + cur_param_len = (int)(eq+1 - next_param); + } + } + } SMARTLIST_FOREACH_END(param); + + result = smartlist_join_strings(output, " ", 0, NULL); + SMARTLIST_FOREACH(output, char *, cp, tor_free(cp)); + smartlist_free(output); + smartlist_free(param_list); + tor_free(vals); + return result; +} + +#define RANGE_CHECK(a,b,c,d,e,f,g,mx) \ + ((a) >= 0 && (a) <= (mx) && (b) >= 0 && (b) <= (mx) && \ + (c) >= 0 && (c) <= (mx) && (d) >= 0 && (d) <= (mx) && \ + (e) >= 0 && (e) <= (mx) && (f) >= 0 && (f) <= (mx) && \ + (g) >= 0 && (g) <= (mx)) + +#define CHECK_EQ(a, b, margin) \ + ((a)-(b) >= 0 ? (a)-(b) <= (margin) : (b)-(a) <= (margin)) + +typedef enum { + BW_WEIGHTS_NO_ERROR = 0, + BW_WEIGHTS_RANGE_ERROR = 1, + BW_WEIGHTS_SUMG_ERROR = 2, + BW_WEIGHTS_SUME_ERROR = 3, + BW_WEIGHTS_SUMD_ERROR = 4, + BW_WEIGHTS_BALANCE_MID_ERROR = 5, + BW_WEIGHTS_BALANCE_EG_ERROR = 6 +} bw_weights_error_t; + +/** + * Verify that any weightings satisfy the balanced formulas. + */ +static bw_weights_error_t +networkstatus_check_weights(int64_t Wgg, int64_t Wgd, int64_t Wmg, + int64_t Wme, int64_t Wmd, int64_t Wee, + int64_t Wed, int64_t scale, int64_t G, + int64_t M, int64_t E, int64_t D, int64_t T, + int64_t margin, int do_balance) { + bw_weights_error_t berr = BW_WEIGHTS_NO_ERROR; + + // Wed + Wmd + Wgd == 1 + if (!CHECK_EQ(Wed + Wmd + Wgd, scale, margin)) { + berr = BW_WEIGHTS_SUMD_ERROR; + goto out; + } + + // Wmg + Wgg == 1 + if (!CHECK_EQ(Wmg + Wgg, scale, margin)) { + berr = BW_WEIGHTS_SUMG_ERROR; + goto out; + } + + // Wme + Wee == 1 + if (!CHECK_EQ(Wme + Wee, scale, margin)) { + berr = BW_WEIGHTS_SUME_ERROR; + goto out; + } + + // Verify weights within range 0->1 + if (!RANGE_CHECK(Wgg, Wgd, Wmg, Wme, Wmd, Wed, Wee, scale)) { + berr = BW_WEIGHTS_RANGE_ERROR; + goto out; + } + + if (do_balance) { + // Wgg*G + Wgd*D == Wee*E + Wed*D, already scaled + if (!CHECK_EQ(Wgg*G + Wgd*D, Wee*E + Wed*D, (margin*T)/3)) { + berr = BW_WEIGHTS_BALANCE_EG_ERROR; + goto out; + } + + // Wgg*G + Wgd*D == M*scale + Wmd*D + Wme*E + Wmg*G, already scaled + if (!CHECK_EQ(Wgg*G + Wgd*D, M*scale + Wmd*D + Wme*E + Wmg*G, + (margin*T)/3)) { + berr = BW_WEIGHTS_BALANCE_MID_ERROR; + goto out; + } + } + + out: + if (berr) { + log_info(LD_DIR, + "Bw weight mismatch %d. G="I64_FORMAT" M="I64_FORMAT + " E="I64_FORMAT" D="I64_FORMAT" T="I64_FORMAT + " Wmd=%d Wme=%d Wmg=%d Wed=%d Wee=%d" + " Wgd=%d Wgg=%d Wme=%d Wmg=%d", + berr, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + (int)Wmd, (int)Wme, (int)Wmg, (int)Wed, (int)Wee, + (int)Wgd, (int)Wgg, (int)Wme, (int)Wmg); + } + + return berr; +} + +/** + * This function computes the bandwidth weights for consensus method 10. + * + * It returns true if weights could be computed, false otherwise. + */ +static int +networkstatus_compute_bw_weights_v10(smartlist_t *chunks, int64_t G, + int64_t M, int64_t E, int64_t D, + int64_t T, int64_t weight_scale) +{ + bw_weights_error_t berr = 0; + int64_t Wgg = -1, Wgd = -1; + int64_t Wmg = -1, Wme = -1, Wmd = -1; + int64_t Wed = -1, Wee = -1; + const char *casename; + char buf[512]; + int r; + + if (G <= 0 || M <= 0 || E <= 0 || D <= 0) { + log_warn(LD_DIR, "Consensus with empty bandwidth: " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT + " D="I64_FORMAT" T="I64_FORMAT, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + return 0; + } + + /* + * Computed from cases in 3.4.3 of dir-spec.txt + * + * 1. Neither are scarce + * 2. Both Guard and Exit are scarce + * a. R+D <= S + * b. R+D > S + * 3. One of Guard or Exit is scarce + * a. S+D < T/3 + * b. S+D >= T/3 + */ + if (3*E >= T && 3*G >= T) { // E >= T/3 && G >= T/3 + /* Case 1: Neither are scarce. */ + casename = "Case 1 (Wgd=Wmd=Wed)"; + Wgd = weight_scale/3; + Wed = weight_scale/3; + Wmd = weight_scale/3; + Wee = (weight_scale*(E+G+M))/(3*E); + Wme = weight_scale - Wee; + Wmg = (weight_scale*(2*G-E-M))/(3*G); + Wgg = weight_scale - Wmg; + + berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed, + weight_scale, G, M, E, D, T, 10, 1); + + if (berr) { + log_warn(LD_DIR, + "Bw Weights error %d for %s v10. G="I64_FORMAT" M="I64_FORMAT + " E="I64_FORMAT" D="I64_FORMAT" T="I64_FORMAT + " Wmd=%d Wme=%d Wmg=%d Wed=%d Wee=%d" + " Wgd=%d Wgg=%d Wme=%d Wmg=%d weight_scale=%d", + berr, casename, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + (int)Wmd, (int)Wme, (int)Wmg, (int)Wed, (int)Wee, + (int)Wgd, (int)Wgg, (int)Wme, (int)Wmg, (int)weight_scale); + return 0; + } + } else if (3*E < T && 3*G < T) { // E < T/3 && G < T/3 + int64_t R = MIN(E, G); + int64_t S = MAX(E, G); + /* + * Case 2: Both Guards and Exits are scarce + * Balance D between E and G, depending upon + * D capacity and scarcity. + */ + if (R+D < S) { // Subcase a + Wgg = weight_scale; + Wee = weight_scale; + Wmg = 0; + Wme = 0; + Wmd = 0; + if (E < G) { + casename = "Case 2a (E scarce)"; + Wed = weight_scale; + Wgd = 0; + } else { /* E >= G */ + casename = "Case 2a (G scarce)"; + Wed = 0; + Wgd = weight_scale; + } + } else { // Subcase b: R+D >= S + casename = "Case 2b1 (Wgg=1, Wmd=Wgd)"; + Wee = (weight_scale*(E - G + M))/E; + Wed = (weight_scale*(D - 2*E + 4*G - 2*M))/(3*D); + Wme = (weight_scale*(G-M))/E; + Wmg = 0; + Wgg = weight_scale; + Wmd = (weight_scale - Wed)/2; + Wgd = (weight_scale - Wed)/2; + + berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed, + weight_scale, G, M, E, D, T, 10, 1); + + if (berr) { + casename = "Case 2b2 (Wgg=1, Wee=1)"; + Wgg = weight_scale; + Wee = weight_scale; + Wed = (weight_scale*(D - 2*E + G + M))/(3*D); + Wmd = (weight_scale*(D - 2*M + G + E))/(3*D); + Wme = 0; + Wmg = 0; + + if (Wmd < 0) { // Can happen if M > T/3 + casename = "Case 2b3 (Wmd=0)"; + Wmd = 0; + log_warn(LD_DIR, + "Too much Middle bandwidth on the network to calculate " + "balanced bandwidth-weights. Consider increasing the " + "number of Guard nodes by lowering the requirements."); + } + Wgd = weight_scale - Wed - Wmd; + berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee, + Wed, weight_scale, G, M, E, D, T, 10, 1); + } + if (berr != BW_WEIGHTS_NO_ERROR && + berr != BW_WEIGHTS_BALANCE_MID_ERROR) { + log_warn(LD_DIR, + "Bw Weights error %d for %s v10. G="I64_FORMAT" M="I64_FORMAT + " E="I64_FORMAT" D="I64_FORMAT" T="I64_FORMAT + " Wmd=%d Wme=%d Wmg=%d Wed=%d Wee=%d" + " Wgd=%d Wgg=%d Wme=%d Wmg=%d weight_scale=%d", + berr, casename, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + (int)Wmd, (int)Wme, (int)Wmg, (int)Wed, (int)Wee, + (int)Wgd, (int)Wgg, (int)Wme, (int)Wmg, (int)weight_scale); + return 0; + } + } + } else { // if (E < T/3 || G < T/3) { + int64_t S = MIN(E, G); + // Case 3: Exactly one of Guard or Exit is scarce + if (!(3*E < T || 3*G < T) || !(3*G >= T || 3*E >= T)) { + log_warn(LD_BUG, + "Bw-Weights Case 3 v10 but with G="I64_FORMAT" M=" + I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT" T="I64_FORMAT, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + } + + if (3*(S+D) < T) { // Subcase a: S+D < T/3 + if (G < E) { + casename = "Case 3a (G scarce)"; + Wgg = Wgd = weight_scale; + Wmd = Wed = Wmg = 0; + // Minor subcase, if E is more scarce than M, + // keep its bandwidth in place. + if (E < M) Wme = 0; + else Wme = (weight_scale*(E-M))/(2*E); + Wee = weight_scale-Wme; + } else { // G >= E + casename = "Case 3a (E scarce)"; + Wee = Wed = weight_scale; + Wmd = Wgd = Wme = 0; + // Minor subcase, if G is more scarce than M, + // keep its bandwidth in place. + if (G < M) Wmg = 0; + else Wmg = (weight_scale*(G-M))/(2*G); + Wgg = weight_scale-Wmg; + } + } else { // Subcase b: S+D >= T/3 + // D != 0 because S+D >= T/3 + if (G < E) { + casename = "Case 3bg (G scarce, Wgg=1, Wmd == Wed)"; + Wgg = weight_scale; + Wgd = (weight_scale*(D - 2*G + E + M))/(3*D); + Wmg = 0; + Wee = (weight_scale*(E+M))/(2*E); + Wme = weight_scale - Wee; + Wmd = (weight_scale - Wgd)/2; + Wed = (weight_scale - Wgd)/2; + + berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee, + Wed, weight_scale, G, M, E, D, T, 10, 1); + } else { // G >= E + casename = "Case 3be (E scarce, Wee=1, Wmd == Wgd)"; + Wee = weight_scale; + Wed = (weight_scale*(D - 2*E + G + M))/(3*D); + Wme = 0; + Wgg = (weight_scale*(G+M))/(2*G); + Wmg = weight_scale - Wgg; + Wmd = (weight_scale - Wed)/2; + Wgd = (weight_scale - Wed)/2; + + berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee, + Wed, weight_scale, G, M, E, D, T, 10, 1); + } + if (berr) { + log_warn(LD_DIR, + "Bw Weights error %d for %s v10. G="I64_FORMAT" M="I64_FORMAT + " E="I64_FORMAT" D="I64_FORMAT" T="I64_FORMAT + " Wmd=%d Wme=%d Wmg=%d Wed=%d Wee=%d" + " Wgd=%d Wgg=%d Wme=%d Wmg=%d weight_scale=%d", + berr, casename, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + (int)Wmd, (int)Wme, (int)Wmg, (int)Wed, (int)Wee, + (int)Wgd, (int)Wgg, (int)Wme, (int)Wmg, (int)weight_scale); + return 0; + } + } + } + + /* We cast down the weights to 32 bit ints on the assumption that + * weight_scale is ~= 10000. We need to ensure a rogue authority + * doesn't break this assumption to rig our weights */ + tor_assert(0 < weight_scale && weight_scale < INT32_MAX); + + /* + * Provide Wgm=Wgg, Wmm=1, Wem=Wee, Weg=Wed. May later determine + * that middle nodes need different bandwidth weights for dirport traffic, + * or that weird exit policies need special weight, or that bridges + * need special weight. + * + * NOTE: This list is sorted. + */ + r = tor_snprintf(buf, sizeof(buf), + "bandwidth-weights Wbd=%d Wbe=%d Wbg=%d Wbm=%d " + "Wdb=%d " + "Web=%d Wed=%d Wee=%d Weg=%d Wem=%d " + "Wgb=%d Wgd=%d Wgg=%d Wgm=%d " + "Wmb=%d Wmd=%d Wme=%d Wmg=%d Wmm=%d\n", + (int)Wmd, (int)Wme, (int)Wmg, (int)weight_scale, + (int)weight_scale, + (int)weight_scale, (int)Wed, (int)Wee, (int)Wed, (int)Wee, + (int)weight_scale, (int)Wgd, (int)Wgg, (int)Wgg, + (int)weight_scale, (int)Wmd, (int)Wme, (int)Wmg, (int)weight_scale); + if (r<0) { + log_warn(LD_BUG, + "Not enough space in buffer for bandwidth-weights line."); + *buf = '\0'; + return 0; + } + smartlist_add(chunks, tor_strdup(buf)); + + log_notice(LD_CIRC, "Computed bandwidth weights for %s with v10: " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT, + casename, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + return 1; +} +/** + * This function computes the bandwidth weights for consensus method 9. + * + * It has been obsoleted in favor of consensus method 10. + */ +static void +networkstatus_compute_bw_weights_v9(smartlist_t *chunks, int64_t G, int64_t M, + int64_t E, int64_t D, int64_t T, + int64_t weight_scale) +{ + int64_t Wgg = -1, Wgd = -1; + int64_t Wmg = -1, Wme = -1, Wmd = -1; + int64_t Wed = -1, Wee = -1; + const char *casename; + char buf[512]; + int r; + + if (G <= 0 || M <= 0 || E <= 0 || D <= 0) { + log_warn(LD_DIR, "Consensus with empty bandwidth: " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT + " D="I64_FORMAT" T="I64_FORMAT, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + return; + } + + /* + * Computed from cases in 3.4.3 of dir-spec.txt + * + * 1. Neither are scarce + * 2. Both Guard and Exit are scarce + * a. R+D <= S + * b. R+D > S + * 3. One of Guard or Exit is scarce + * a. S+D < T/3 + * b. S+D >= T/3 + */ + if (3*E >= T && 3*G >= T) { // E >= T/3 && G >= T/3 + bw_weights_error_t berr = 0; + /* Case 1: Neither are scarce. + * + * Attempt to ensure that we have a large amount of exit bandwidth + * in the middle position. + */ + casename = "Case 1 (Wme*E = Wmd*D)"; + Wgg = (weight_scale*(D+E+G+M))/(3*G); + if (D==0) Wmd = 0; + else Wmd = (weight_scale*(2*D + 2*E - G - M))/(6*D); + Wme = (weight_scale*(2*D + 2*E - G - M))/(6*E); + Wee = (weight_scale*(-2*D + 4*E + G + M))/(6*E); + Wgd = 0; + Wmg = weight_scale - Wgg; + Wed = weight_scale - Wmd; + + berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed, + weight_scale, G, M, E, D, T, 10, 1); + + if (berr) { + log_warn(LD_DIR, "Bw Weights error %d for case %s. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT + " D="I64_FORMAT" T="I64_FORMAT, + berr, casename, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + } + } else if (3*E < T && 3*G < T) { // E < T/3 && G < T/3 + int64_t R = MIN(E, G); + int64_t S = MAX(E, G); + /* + * Case 2: Both Guards and Exits are scarce + * Balance D between E and G, depending upon + * D capacity and scarcity. + */ + if (R+D < S) { // Subcase a + Wgg = weight_scale; + Wee = weight_scale; + Wmg = 0; + Wme = 0; + Wmd = 0; + if (E < G) { + casename = "Case 2a (E scarce)"; + Wed = weight_scale; + Wgd = 0; + } else { /* E >= G */ + casename = "Case 2a (G scarce)"; + Wed = 0; + Wgd = weight_scale; + } + } else { // Subcase b: R+D > S + bw_weights_error_t berr = 0; + casename = "Case 2b (Wme*E == Wmd*D)"; + if (D != 0) { + Wgg = weight_scale; + Wgd = (weight_scale*(D + E - 2*G + M))/(3*D); // T/3 >= G (Ok) + Wmd = (weight_scale*(D + E + G - 2*M))/(6*D); // T/3 >= M + Wme = (weight_scale*(D + E + G - 2*M))/(6*E); + Wee = (weight_scale*(-D + 5*E - G + 2*M))/(6*E); // 2E+M >= T/3 + Wmg = 0; + Wed = weight_scale - Wgd - Wmd; + + berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed, + weight_scale, G, M, E, D, T, 10, 1); + } + + if (D == 0 || berr) { // Can happen if M > T/3 + casename = "Case 2b (E=G)"; + Wgg = weight_scale; + Wee = weight_scale; + Wmg = 0; + Wme = 0; + Wmd = 0; + if (D == 0) Wgd = 0; + else Wgd = (weight_scale*(D+E-G))/(2*D); + Wed = weight_scale - Wgd; + berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee, + Wed, weight_scale, G, M, E, D, T, 10, 1); + } + if (berr != BW_WEIGHTS_NO_ERROR && + berr != BW_WEIGHTS_BALANCE_MID_ERROR) { + log_warn(LD_DIR, "Bw Weights error %d for case %s. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT + " D="I64_FORMAT" T="I64_FORMAT, + berr, casename, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + } + } + } else { // if (E < T/3 || G < T/3) { + int64_t S = MIN(E, G); + // Case 3: Exactly one of Guard or Exit is scarce + if (!(3*E < T || 3*G < T) || !(3*G >= T || 3*E >= T)) { + log_warn(LD_BUG, + "Bw-Weights Case 3 but with G="I64_FORMAT" M=" + I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT" T="I64_FORMAT, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + } + + if (3*(S+D) < T) { // Subcase a: S+D < T/3 + if (G < E) { + casename = "Case 3a (G scarce)"; + Wgg = Wgd = weight_scale; + Wmd = Wed = Wmg = 0; + // Minor subcase, if E is more scarce than M, + // keep its bandwidth in place. + if (E < M) Wme = 0; + else Wme = (weight_scale*(E-M))/(2*E); + Wee = weight_scale-Wme; + } else { // G >= E + casename = "Case 3a (E scarce)"; + Wee = Wed = weight_scale; + Wmd = Wgd = Wme = 0; + // Minor subcase, if G is more scarce than M, + // keep its bandwidth in place. + if (G < M) Wmg = 0; + else Wmg = (weight_scale*(G-M))/(2*G); + Wgg = weight_scale-Wmg; + } + } else { // Subcase b: S+D >= T/3 + bw_weights_error_t berr = 0; + // D != 0 because S+D >= T/3 + if (G < E) { + casename = "Case 3b (G scarce, Wme*E == Wmd*D)"; + Wgd = (weight_scale*(D + E - 2*G + M))/(3*D); + Wmd = (weight_scale*(D + E + G - 2*M))/(6*D); + Wme = (weight_scale*(D + E + G - 2*M))/(6*E); + Wee = (weight_scale*(-D + 5*E - G + 2*M))/(6*E); + Wgg = weight_scale; + Wmg = 0; + Wed = weight_scale - Wgd - Wmd; + + berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee, + Wed, weight_scale, G, M, E, D, T, 10, 1); + } else { // G >= E + casename = "Case 3b (E scarce, Wme*E == Wmd*D)"; + Wgg = (weight_scale*(D + E + G + M))/(3*G); + Wmd = (weight_scale*(2*D + 2*E - G - M))/(6*D); + Wme = (weight_scale*(2*D + 2*E - G - M))/(6*E); + Wee = (weight_scale*(-2*D + 4*E + G + M))/(6*E); + Wgd = 0; + Wmg = weight_scale - Wgg; + Wed = weight_scale - Wmd; + + berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee, + Wed, weight_scale, G, M, E, D, T, 10, 1); + } + if (berr) { + log_warn(LD_DIR, "Bw Weights error %d for case %s. " + "G="I64_FORMAT" M="I64_FORMAT + " E="I64_FORMAT" D="I64_FORMAT" T="I64_FORMAT, + berr, casename, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + } + } + } + + /* We cast down the weights to 32 bit ints on the assumption that + * weight_scale is ~= 10000. We need to ensure a rogue authority + * doesn't break this assumption to rig our weights */ + tor_assert(0 < weight_scale && weight_scale < INT32_MAX); + + if (Wgg < 0 || Wgg > weight_scale) { + log_warn(LD_DIR, "Bw %s: Wgg="I64_FORMAT"! G="I64_FORMAT + " M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT, + casename, I64_PRINTF_ARG(Wgg), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + + Wgg = MAX(MIN(Wgg, weight_scale), 0); + } + if (Wgd < 0 || Wgd > weight_scale) { + log_warn(LD_DIR, "Bw %s: Wgd="I64_FORMAT"! G="I64_FORMAT + " M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT, + casename, I64_PRINTF_ARG(Wgd), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + Wgd = MAX(MIN(Wgd, weight_scale), 0); + } + if (Wmg < 0 || Wmg > weight_scale) { + log_warn(LD_DIR, "Bw %s: Wmg="I64_FORMAT"! G="I64_FORMAT + " M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT, + casename, I64_PRINTF_ARG(Wmg), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + Wmg = MAX(MIN(Wmg, weight_scale), 0); + } + if (Wme < 0 || Wme > weight_scale) { + log_warn(LD_DIR, "Bw %s: Wme="I64_FORMAT"! G="I64_FORMAT + " M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT, + casename, I64_PRINTF_ARG(Wme), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + Wme = MAX(MIN(Wme, weight_scale), 0); + } + if (Wmd < 0 || Wmd > weight_scale) { + log_warn(LD_DIR, "Bw %s: Wmd="I64_FORMAT"! G="I64_FORMAT + " M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT, + casename, I64_PRINTF_ARG(Wmd), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + Wmd = MAX(MIN(Wmd, weight_scale), 0); + } + if (Wee < 0 || Wee > weight_scale) { + log_warn(LD_DIR, "Bw %s: Wee="I64_FORMAT"! G="I64_FORMAT + " M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT, + casename, I64_PRINTF_ARG(Wee), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + Wee = MAX(MIN(Wee, weight_scale), 0); + } + if (Wed < 0 || Wed > weight_scale) { + log_warn(LD_DIR, "Bw %s: Wed="I64_FORMAT"! G="I64_FORMAT + " M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT, + casename, I64_PRINTF_ARG(Wed), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + Wed = MAX(MIN(Wed, weight_scale), 0); + } + + // Add consensus weight keywords + smartlist_add(chunks, tor_strdup("bandwidth-weights ")); + /* + * Provide Wgm=Wgg, Wmm=1, Wem=Wee, Weg=Wed. May later determine + * that middle nodes need different bandwidth weights for dirport traffic, + * or that weird exit policies need special weight, or that bridges + * need special weight. + * + * NOTE: This list is sorted. + */ + r = tor_snprintf(buf, sizeof(buf), + "Wbd=%d Wbe=%d Wbg=%d Wbm=%d " + "Wdb=%d " + "Web=%d Wed=%d Wee=%d Weg=%d Wem=%d " + "Wgb=%d Wgd=%d Wgg=%d Wgm=%d " + "Wmb=%d Wmd=%d Wme=%d Wmg=%d Wmm=%d\n", + (int)Wmd, (int)Wme, (int)Wmg, (int)weight_scale, + (int)weight_scale, + (int)weight_scale, (int)Wed, (int)Wee, (int)Wed, (int)Wee, + (int)weight_scale, (int)Wgd, (int)Wgg, (int)Wgg, + (int)weight_scale, (int)Wmd, (int)Wme, (int)Wmg, (int)weight_scale); + if (r<0) { + log_warn(LD_BUG, + "Not enough space in buffer for bandwidth-weights line."); + *buf = '\0'; + } + smartlist_add(chunks, tor_strdup(buf)); + log_notice(LD_CIRC, "Computed bandwidth weights for %s with v9: " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT, + casename, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); +} + /** Given a list of vote networkstatus_t in <b>votes</b>, our public * authority <b>identity_key</b>, our private authority <b>signing_key</b>, * and the number of <b>total_authorities</b> that we believe exist in our @@ -490,18 +1347,27 @@ networkstatus_compute_consensus(smartlist_t *votes, crypto_pk_env_t *identity_key, crypto_pk_env_t *signing_key, const char *legacy_id_key_digest, - crypto_pk_env_t *legacy_signing_key) + crypto_pk_env_t *legacy_signing_key, + consensus_flavor_t flavor) { smartlist_t *chunks; char *result = NULL; int consensus_method; - time_t valid_after, fresh_until, valid_until; int vote_seconds, dist_seconds; char *client_versions = NULL, *server_versions = NULL; smartlist_t *flags; + const char *flavor_name; + int64_t G=0, M=0, E=0, D=0, T=0; /* For bandwidth weights */ + const routerstatus_format_type_t rs_format = + flavor == FLAV_NS ? NS_V3_CONSENSUS : NS_V3_CONSENSUS_MICRODESC; + char *params = NULL; + int added_weights = 0; + tor_assert(flavor == FLAV_NS || flavor == FLAV_MICRODESC); tor_assert(total_authorities >= smartlist_len(votes)); + flavor_name = networkstatus_get_flavor_name(flavor); + if (!smartlist_len(votes)) { log_warn(LD_DIR, "Can't compute a consensus from no votes."); return NULL; @@ -594,7 +1460,7 @@ networkstatus_compute_consensus(smartlist_t *votes, chunks = smartlist_create(); { - char buf[1024]; + char *buf=NULL; char va_buf[ISO_TIME_LEN+1], fu_buf[ISO_TIME_LEN+1], vu_buf[ISO_TIME_LEN+1]; char *flaglist; @@ -603,16 +1469,20 @@ networkstatus_compute_consensus(smartlist_t *votes, format_iso_time(vu_buf, valid_until); flaglist = smartlist_join_strings(flags, " ", 0, NULL); - smartlist_add(chunks, tor_strdup("network-status-version 3\n" - "vote-status consensus\n")); + tor_asprintf(&buf, "network-status-version 3%s%s\n" + "vote-status consensus\n", + flavor == FLAV_NS ? "" : " ", + flavor == FLAV_NS ? "" : flavor_name); + + smartlist_add(chunks, buf); if (consensus_method >= 2) { - tor_snprintf(buf, sizeof(buf), "consensus-method %d\n", + tor_asprintf(&buf, "consensus-method %d\n", consensus_method); - smartlist_add(chunks, tor_strdup(buf)); + smartlist_add(chunks, buf); } - tor_snprintf(buf, sizeof(buf), + tor_asprintf(&buf, "valid-after %s\n" "fresh-until %s\n" "valid-until %s\n" @@ -623,18 +1493,26 @@ networkstatus_compute_consensus(smartlist_t *votes, va_buf, fu_buf, vu_buf, vote_seconds, dist_seconds, client_versions, server_versions, flaglist); - smartlist_add(chunks, tor_strdup(buf)); + smartlist_add(chunks, buf); tor_free(flaglist); } + if (consensus_method >= MIN_METHOD_FOR_PARAMS) { + params = dirvote_compute_params(votes); + if (params) { + smartlist_add(chunks, tor_strdup("params ")); + smartlist_add(chunks, params); + smartlist_add(chunks, tor_strdup("\n")); + } + } + /* Sort the votes. */ smartlist_sort(votes, _compare_votes_by_authority_id); /* Add the authority sections. */ { smartlist_t *dir_sources = smartlist_create(); - SMARTLIST_FOREACH(votes, networkstatus_t *, v, - { + SMARTLIST_FOREACH_BEGIN(votes, networkstatus_t *, v) { dir_src_ent_t *e = tor_malloc_zero(sizeof(dir_src_ent_t)); e->v = v; e->digest = get_voter(v)->identity_digest; @@ -648,18 +1526,17 @@ networkstatus_compute_consensus(smartlist_t *votes, e_legacy->is_legacy = 1; smartlist_add(dir_sources, e_legacy); } - }); + } SMARTLIST_FOREACH_END(v); smartlist_sort(dir_sources, _compare_dir_src_ents_by_authority_id); - SMARTLIST_FOREACH(dir_sources, const dir_src_ent_t *, e, - { - char buf[1024]; + SMARTLIST_FOREACH_BEGIN(dir_sources, const dir_src_ent_t *, e) { struct in_addr in; char ip[INET_NTOA_BUF_LEN]; char fingerprint[HEX_DIGEST_LEN+1]; char votedigest[HEX_DIGEST_LEN+1]; networkstatus_t *v = e->v; networkstatus_voter_info_t *voter = get_voter(v); + char *buf = NULL; if (e->is_legacy) tor_assert(consensus_method >= 2); @@ -670,22 +1547,22 @@ networkstatus_compute_consensus(smartlist_t *votes, base16_encode(votedigest, sizeof(votedigest), voter->vote_digest, DIGEST_LEN); - tor_snprintf(buf, sizeof(buf), + tor_asprintf(&buf, "dir-source %s%s %s %s %s %d %d\n", voter->nickname, e->is_legacy ? "-legacy" : "", fingerprint, voter->address, ip, voter->dir_port, voter->or_port); - smartlist_add(chunks, tor_strdup(buf)); + smartlist_add(chunks, buf); if (! e->is_legacy) { - tor_snprintf(buf, sizeof(buf), + tor_asprintf(&buf, "contact %s\n" "vote-digest %s\n", voter->contact, votedigest); - smartlist_add(chunks, tor_strdup(buf)); + smartlist_add(chunks, buf); } - }); + } SMARTLIST_FOREACH_END(e); SMARTLIST_FOREACH(dir_sources, dir_src_ent_t *, e, tor_free(e)); smartlist_free(dir_sources); } @@ -702,7 +1579,10 @@ networkstatus_compute_consensus(smartlist_t *votes, smartlist_t *versions = smartlist_create(); smartlist_t *exitsummaries = smartlist_create(); uint32_t *bandwidths = tor_malloc(sizeof(uint32_t) * smartlist_len(votes)); + uint32_t *measured_bws = tor_malloc(sizeof(uint32_t) * + smartlist_len(votes)); int num_bandwidths; + int num_mbws; int *n_voter_flags; /* n_voter_flags[j] is the number of flags that * votes[j] knows about. */ @@ -712,7 +1592,7 @@ networkstatus_compute_consensus(smartlist_t *votes, * is the same flag as votes[j]->known_flags[b]. */ int *named_flag; /* Index of the flag "Named" for votes[j] */ int *unnamed_flag; /* Index of the flag "Unnamed" for votes[j] */ - int chosen_named_idx, chosen_unnamed_idx; + int chosen_named_idx; strmap_t *name_to_id_map = strmap_new(); char conflict[DIGEST_LEN]; @@ -730,7 +1610,6 @@ networkstatus_compute_consensus(smartlist_t *votes, for (i = 0; i < smartlist_len(votes); ++i) unnamed_flag[i] = named_flag[i] = -1; chosen_named_idx = smartlist_string_pos(flags, "Named"); - chosen_unnamed_idx = smartlist_string_pos(flags, "Unnamed"); /* Build the flag index. */ SMARTLIST_FOREACH(votes, networkstatus_t *, v, @@ -814,10 +1693,12 @@ networkstatus_compute_consensus(smartlist_t *votes, const char *chosen_name = NULL; int exitsummary_disagreement = 0; int is_named = 0, is_unnamed = 0, is_running = 0; + int is_guard = 0, is_exit = 0, is_bad_exit = 0; int naming_conflict = 0; int n_listing = 0; int i; - char buf[256]; + char *buf=NULL; + char microdesc_digest[DIGEST256_LEN]; /* Of the next-to-be-considered digest in each voter, which is first? */ SMARTLIST_FOREACH(votes, networkstatus_t *, v, { @@ -837,6 +1718,7 @@ networkstatus_compute_consensus(smartlist_t *votes, smartlist_clear(chosen_flags); smartlist_clear(versions); num_bandwidths = 0; + num_mbws = 0; /* Okay, go through all the entries for this digest. */ SMARTLIST_FOREACH_BEGIN(votes, networkstatus_t *, v) { @@ -870,6 +1752,9 @@ networkstatus_compute_consensus(smartlist_t *votes, } /* count bandwidths */ + if (rs->status.has_measured_bw) + measured_bws[num_mbws++] = rs->status.measured_bw; + if (rs->status.has_bandwidth) bandwidths[num_bandwidths++] = rs->status.bandwidth; } SMARTLIST_FOREACH_END(v); @@ -881,7 +1766,9 @@ networkstatus_compute_consensus(smartlist_t *votes, /* Figure out the most popular opinion of what the most recent * routerinfo and its contents are. */ - rs = compute_routerstatus_consensus(matching_descs); + memset(microdesc_digest, 0, sizeof(microdesc_digest)); + rs = compute_routerstatus_consensus(matching_descs, consensus_method, + microdesc_digest); /* Copy bits of that into rs_out. */ tor_assert(fast_memeq(lowest_id, rs->status.identity_digest,DIGEST_LEN)); memcpy(rs_out.identity_digest, lowest_id, DIGEST_LEN); @@ -927,8 +1814,14 @@ networkstatus_compute_consensus(smartlist_t *votes, } else { if (flag_counts[fl_sl_idx] > n_flag_voters[fl_sl_idx]/2) { smartlist_add(chosen_flags, (char*)fl); - if (!strcmp(fl, "Running")) + if (!strcmp(fl, "Exit")) + is_exit = 1; + else if (!strcmp(fl, "Guard")) + is_guard = 1; + else if (!strcmp(fl, "Running")) is_running = 1; + else if (!strcmp(fl, "BadExit")) + is_bad_exit = 1; } } }); @@ -947,11 +1840,36 @@ networkstatus_compute_consensus(smartlist_t *votes, } /* Pick a bandwidth */ - if (consensus_method >= 5 && num_bandwidths > 0) { + if (consensus_method >= 6 && num_mbws > 2) { + rs_out.has_bandwidth = 1; + rs_out.bandwidth = median_uint32(measured_bws, num_mbws); + } else if (consensus_method >= 5 && num_bandwidths > 0) { rs_out.has_bandwidth = 1; rs_out.bandwidth = median_uint32(bandwidths, num_bandwidths); } + /* Fix bug 2203: Do not count BadExit nodes as Exits for bw weights */ + if (consensus_method >= 11) { + is_exit = is_exit && !is_bad_exit; + } + + if (consensus_method >= MIN_METHOD_FOR_BW_WEIGHTS) { + if (rs_out.has_bandwidth) { + T += rs_out.bandwidth; + if (is_exit && is_guard) + D += rs_out.bandwidth; + else if (is_exit) + E += rs_out.bandwidth; + else if (is_guard) + G += rs_out.bandwidth; + else + M += rs_out.bandwidth; + } else { + log_warn(LD_BUG, "Missing consensus bandwidth for router %s", + rs_out.nickname); + } + } + /* Ok, we already picked a descriptor digest we want to list * previously. Now we want to use the exit policy summary from * that descriptor. If everybody plays nice all the voters who @@ -1036,11 +1954,23 @@ networkstatus_compute_consensus(smartlist_t *votes, } } - /* Okay!! Now we can write the descriptor... */ - /* First line goes into "buf". */ - routerstatus_format_entry(buf, sizeof(buf), &rs_out, NULL, 1, 0); - smartlist_add(chunks, tor_strdup(buf)); - /* Second line is all flags. The "\n" is missing. */ + { + char buf[4096]; + /* Okay!! Now we can write the descriptor... */ + /* First line goes into "buf". */ + routerstatus_format_entry(buf, sizeof(buf), &rs_out, NULL, + rs_format); + smartlist_add(chunks, tor_strdup(buf)); + } + /* Now an m line, if applicable. */ + if (flavor == FLAV_MICRODESC && + !tor_digest256_is_zero(microdesc_digest)) { + char m[BASE64_DIGEST256_LEN+1], *cp; + digest256_to_base64(m, microdesc_digest); + tor_asprintf(&cp, "m %s\n", m); + smartlist_add(chunks, cp); + } + /* Next line is all flags. The "\n" is missing. */ smartlist_add(chunks, smartlist_join_strings(chosen_flags, " ", 0, NULL)); /* Now the version line. */ @@ -1051,24 +1981,16 @@ networkstatus_compute_consensus(smartlist_t *votes, smartlist_add(chunks, tor_strdup("\n")); /* Now the weight line. */ if (rs_out.has_bandwidth) { - int r = tor_snprintf(buf, sizeof(buf), - "w Bandwidth=%d\n", rs_out.bandwidth); - if (r<0) { - log_warn(LD_BUG, "Not enough space in buffer for weight line."); - *buf = '\0'; - } - smartlist_add(chunks, tor_strdup(buf)); - }; + char *cp=NULL; + tor_asprintf(&cp, "w Bandwidth=%d\n", rs_out.bandwidth); + smartlist_add(chunks, cp); + } + /* Now the exitpolicy summary line. */ - if (rs_out.has_exitsummary) { - char buf[MAX_POLICY_LINE_LEN+1]; - int r = tor_snprintf(buf, sizeof(buf), "p %s\n", rs_out.exitsummary); - if (r<0) { - log_warn(LD_BUG, "Not enough space in buffer for exitpolicy line."); - *buf = '\0'; - } - smartlist_add(chunks, tor_strdup(buf)); - }; + if (rs_out.has_exitsummary && flavor == FLAV_NS) { + tor_asprintf(&buf, "p %s\n", rs_out.exitsummary); + smartlist_add(chunks, buf); + } /* And the loop is over and we move on to the next router */ } @@ -1089,34 +2011,97 @@ networkstatus_compute_consensus(smartlist_t *votes, smartlist_free(versions); smartlist_free(exitsummaries); tor_free(bandwidths); + tor_free(measured_bws); + } + + if (consensus_method >= MIN_METHOD_FOR_FOOTER) { + /* Starting with consensus method 9, we clearly mark the directory + * footer region */ + smartlist_add(chunks, tor_strdup("directory-footer\n")); + } + + if (consensus_method >= MIN_METHOD_FOR_BW_WEIGHTS) { + int64_t weight_scale = BW_WEIGHT_SCALE; + char *bw_weight_param = NULL; + + // Parse params, extract BW_WEIGHT_SCALE if present + // DO NOT use consensus_param_bw_weight_scale() in this code! + // The consensus is not formed yet! + if (params) { + if (strcmpstart(params, "bwweightscale=") == 0) + bw_weight_param = params; + else + bw_weight_param = strstr(params, " bwweightscale="); + } + + if (bw_weight_param) { + int ok=0; + char *eq = strchr(bw_weight_param, '='); + if (eq) { + weight_scale = tor_parse_long(eq+1, 10, INT32_MIN, INT32_MAX, &ok, + NULL); + if (!ok) { + log_warn(LD_DIR, "Bad element '%s' in bw weight param", + escaped(bw_weight_param)); + weight_scale = BW_WEIGHT_SCALE; + } + } else { + log_warn(LD_DIR, "Bad element '%s' in bw weight param", + escaped(bw_weight_param)); + weight_scale = BW_WEIGHT_SCALE; + } + } + + if (consensus_method < 10) { + networkstatus_compute_bw_weights_v9(chunks, G, M, E, D, T, weight_scale); + added_weights = 1; + } else { + added_weights = networkstatus_compute_bw_weights_v10(chunks, G, M, E, D, + T, weight_scale); + } } /* Add a signature. */ { - char digest[DIGEST_LEN]; + char digest[DIGEST256_LEN]; char fingerprint[HEX_DIGEST_LEN+1]; char signing_key_fingerprint[HEX_DIGEST_LEN+1]; + digest_algorithm_t digest_alg = + flavor == FLAV_NS ? DIGEST_SHA1 : DIGEST_SHA256; + size_t digest_len = + flavor == FLAV_NS ? DIGEST_LEN : DIGEST256_LEN; + const char *algname = crypto_digest_algorithm_get_name(digest_alg); + char *buf = NULL; + char sigbuf[4096]; - char buf[4096]; smartlist_add(chunks, tor_strdup("directory-signature ")); /* Compute the hash of the chunks. */ - hash_list_members(digest, chunks); + hash_list_members(digest, digest_len, chunks, digest_alg); /* Get the fingerprints */ crypto_pk_get_fingerprint(identity_key, fingerprint, 0); crypto_pk_get_fingerprint(signing_key, signing_key_fingerprint, 0); /* add the junk that will go at the end of the line. */ - tor_snprintf(buf, sizeof(buf), "%s %s\n", fingerprint, - signing_key_fingerprint); + if (flavor == FLAV_NS) { + tor_asprintf(&buf, "%s %s\n", fingerprint, + signing_key_fingerprint); + } else { + tor_asprintf(&buf, "%s %s %s\n", + algname, fingerprint, + signing_key_fingerprint); + } + smartlist_add(chunks, buf); /* And the signature. */ - if (router_append_dirobj_signature(buf, sizeof(buf), digest, + sigbuf[0] = '\0'; + if (router_append_dirobj_signature(sigbuf, sizeof(sigbuf), + digest, digest_len, signing_key)) { log_warn(LD_BUG, "Couldn't sign consensus networkstatus."); return NULL; /* This leaks, but it should never happen. */ } - smartlist_add(chunks, tor_strdup(buf)); + smartlist_add(chunks, tor_strdup(sigbuf)); if (legacy_id_key_digest && legacy_signing_key && consensus_method >= 3) { smartlist_add(chunks, tor_strdup("directory-signature ")); @@ -1124,14 +2109,23 @@ networkstatus_compute_consensus(smartlist_t *votes, legacy_id_key_digest, DIGEST_LEN); crypto_pk_get_fingerprint(legacy_signing_key, signing_key_fingerprint, 0); - tor_snprintf(buf, sizeof(buf), "%s %s\n", fingerprint, - signing_key_fingerprint); - if (router_append_dirobj_signature(buf, sizeof(buf), digest, + if (flavor == FLAV_NS) { + tor_asprintf(&buf, "%s %s\n", fingerprint, + signing_key_fingerprint); + } else { + tor_asprintf(&buf, "%s %s %s\n", + algname, fingerprint, + signing_key_fingerprint); + } + smartlist_add(chunks, buf); + sigbuf[0] = '\0'; + if (router_append_dirobj_signature(sigbuf, sizeof(sigbuf), + digest, digest_len, legacy_signing_key)) { log_warn(LD_BUG, "Couldn't sign consensus networkstatus."); return NULL; /* This leaks, but it should never happen. */ } - smartlist_add(chunks, tor_strdup(buf)); + smartlist_add(chunks, tor_strdup(sigbuf)); } } @@ -1148,11 +2142,15 @@ networkstatus_compute_consensus(smartlist_t *votes, networkstatus_t *c; if (!(c = networkstatus_parse_vote_from_string(result, NULL, NS_TYPE_CONSENSUS))) { - log_err(LD_BUG,"Generated a networkstatus consensus we couldn't " + log_err(LD_BUG, "Generated a networkstatus consensus we couldn't " "parse."); tor_free(result); return NULL; } + // Verify balancing parameters + if (consensus_method >= MIN_METHOD_FOR_BW_WEIGHTS && added_weights) { + networkstatus_verify_bw_weights(c); + } networkstatus_vote_free(c); } @@ -1173,10 +2171,14 @@ networkstatus_add_detached_signatures(networkstatus_t *target, const char **msg_out) { int r = 0; + const char *flavor; + smartlist_t *siglist; tor_assert(sigs); tor_assert(target); tor_assert(target->type == NS_TYPE_CONSENSUS); + flavor = networkstatus_get_flavor_name(target->flavor); + /* Do the times seem right? */ if (target->valid_after != sigs->valid_after) { *msg_out = "Valid-After times do not match " @@ -1193,79 +2195,180 @@ networkstatus_add_detached_signatures(networkstatus_t *target, "when adding detached signatures to consensus"; return -1; } - /* Are they the same consensus? */ - if (fast_memcmp(target->networkstatus_digest, sigs->networkstatus_digest, - DIGEST_LEN)) { - *msg_out = "Digest mismatch when adding detached signatures to consensus"; + siglist = strmap_get(sigs->signatures, flavor); + if (!siglist) { + *msg_out = "No signatures for given consensus flavor"; return -1; } - /* For each voter in src... */ - SMARTLIST_FOREACH_BEGIN(sigs->signatures, networkstatus_voter_info_t *, - src_voter) { - char voter_identity[HEX_DIGEST_LEN+1]; - networkstatus_voter_info_t *target_voter = - networkstatus_get_voter_by_id(target, src_voter->identity_digest); - authority_cert_t *cert = NULL; - - base16_encode(voter_identity, sizeof(voter_identity), - src_voter->identity_digest, DIGEST_LEN); - log_info(LD_DIR, "Looking at signature from %s", voter_identity); - /* If the target doesn't know about this voter, then forget it. */ - if (!target_voter) { - log_info(LD_DIR, "We do not know about %s", voter_identity); - continue; + /** Make sure all the digests we know match, and at least one matches. */ + { + digests_t *digests = strmap_get(sigs->digests, flavor); + int n_matches = 0; + digest_algorithm_t alg; + if (!digests) { + *msg_out = "No digests for given consensus flavor"; + return -1; + } + for (alg = DIGEST_SHA1; alg < N_DIGEST_ALGORITHMS; ++alg) { + if (!tor_mem_is_zero(digests->d[alg], DIGEST256_LEN)) { + if (fast_memeq(target->digests.d[alg], digests->d[alg], + DIGEST256_LEN)) { + ++n_matches; + } else { + *msg_out = "Mismatched digest."; + return -1; + } } + } + if (!n_matches) { + *msg_out = "No regognized digests for given consensus flavor"; + } + } - /* If the target already has a good signature from this voter, then skip - * this one. */ - if (target_voter->good_signature) { - log_info(LD_DIR, "We already have a good signature from %s", - voter_identity); - continue; - } + /* For each voter in src... */ + SMARTLIST_FOREACH_BEGIN(siglist, document_signature_t *, sig) { + char voter_identity[HEX_DIGEST_LEN+1]; + networkstatus_voter_info_t *target_voter = + networkstatus_get_voter_by_id(target, sig->identity_digest); + authority_cert_t *cert = NULL; + const char *algorithm; + document_signature_t *old_sig = NULL; + + algorithm = crypto_digest_algorithm_get_name(sig->alg); + + base16_encode(voter_identity, sizeof(voter_identity), + sig->identity_digest, DIGEST_LEN); + log_info(LD_DIR, "Looking at signature from %s using %s", voter_identity, + algorithm); + /* If the target doesn't know about this voter, then forget it. */ + if (!target_voter) { + log_info(LD_DIR, "We do not know any voter with ID %s", voter_identity); + continue; + } - /* Try checking the signature if we haven't already. */ - if (!src_voter->good_signature && !src_voter->bad_signature) { - cert = authority_cert_get_by_digests(src_voter->identity_digest, - src_voter->signing_key_digest); - if (cert) { - networkstatus_check_voter_signature(target, src_voter, cert); - } + old_sig = voter_get_sig_by_algorithm(target_voter, sig->alg); + + /* If the target already has a good signature from this voter, then skip + * this one. */ + if (old_sig && old_sig->good_signature) { + log_info(LD_DIR, "We already have a good signature from %s using %s", + voter_identity, algorithm); + continue; + } + + /* Try checking the signature if we haven't already. */ + if (!sig->good_signature && !sig->bad_signature) { + cert = authority_cert_get_by_digests(sig->identity_digest, + sig->signing_key_digest); + if (cert) + networkstatus_check_document_signature(target, sig, cert); + } + + /* If this signature is good, or we don't have any signature yet, + * then maybe add it. */ + if (sig->good_signature || !old_sig || old_sig->bad_signature) { + log_info(LD_DIR, "Adding signature from %s with %s", voter_identity, + algorithm); + ++r; + if (old_sig) { + smartlist_remove(target_voter->sigs, old_sig); + document_signature_free(old_sig); } + smartlist_add(target_voter->sigs, document_signature_dup(sig)); + } else { + log_info(LD_DIR, "Not adding signature from %s", voter_identity); + } + } SMARTLIST_FOREACH_END(sig); + + return r; +} + +/** Return a newly allocated string containing all the signatures on + * <b>consensus</b> by all voters. If <b>for_detached_signatures</b> is true, + * then the signatures will be put in a detached signatures document, so + * prefix any non-NS-flavored signatures with "additional-signature" rather + * than "directory-signature". */ +static char * +networkstatus_format_signatures(networkstatus_t *consensus, + int for_detached_signatures) +{ + smartlist_t *elements; + char buf[4096]; + char *result = NULL; + int n_sigs = 0; + const consensus_flavor_t flavor = consensus->flavor; + const char *flavor_name = networkstatus_get_flavor_name(flavor); + const char *keyword; + + if (for_detached_signatures && flavor != FLAV_NS) + keyword = "additional-signature"; + else + keyword = "directory-signature"; - /* If this signature is good, or we don't have any signature yet, - * then add it. */ - if (src_voter->good_signature || !target_voter->signature) { - log_info(LD_DIR, "Adding signature from %s", voter_identity); - ++r; - tor_free(target_voter->signature); - target_voter->signature = - tor_memdup(src_voter->signature, src_voter->signature_len); - memcpy(target_voter->signing_key_digest, src_voter->signing_key_digest, - DIGEST_LEN); - target_voter->signature_len = src_voter->signature_len; - target_voter->good_signature = src_voter->good_signature; - target_voter->bad_signature = src_voter->bad_signature; + elements = smartlist_create(); + + SMARTLIST_FOREACH_BEGIN(consensus->voters, networkstatus_voter_info_t *, v) { + SMARTLIST_FOREACH_BEGIN(v->sigs, document_signature_t *, sig) { + char sk[HEX_DIGEST_LEN+1]; + char id[HEX_DIGEST_LEN+1]; + if (!sig->signature || sig->bad_signature) + continue; + ++n_sigs; + base16_encode(sk, sizeof(sk), sig->signing_key_digest, DIGEST_LEN); + base16_encode(id, sizeof(id), sig->identity_digest, DIGEST_LEN); + if (flavor == FLAV_NS) { + tor_snprintf(buf, sizeof(buf), + "%s %s %s\n-----BEGIN SIGNATURE-----\n", + keyword, id, sk); } else { - log_info(LD_DIR, "Not adding signature from %s", voter_identity); + const char *digest_name = + crypto_digest_algorithm_get_name(sig->alg); + tor_snprintf(buf, sizeof(buf), + "%s%s%s %s %s %s\n-----BEGIN SIGNATURE-----\n", + keyword, + for_detached_signatures ? " " : "", + for_detached_signatures ? flavor_name : "", + digest_name, id, sk); } - } SMARTLIST_FOREACH_END(src_voter); + smartlist_add(elements, tor_strdup(buf)); + base64_encode(buf, sizeof(buf), sig->signature, sig->signature_len); + strlcat(buf, "-----END SIGNATURE-----\n", sizeof(buf)); + smartlist_add(elements, tor_strdup(buf)); + } SMARTLIST_FOREACH_END(sig); + } SMARTLIST_FOREACH_END(v); - return r; + result = smartlist_join_strings(elements, "", 0, NULL); + SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp)); + smartlist_free(elements); + if (!n_sigs) + tor_free(result); + return result; } /** Return a newly allocated string holding the detached-signatures document - * corresponding to the signatures on <b>consensus</b>. */ + * corresponding to the signatures on <b>consensuses</b>, which must contain + * exactly one FLAV_NS consensus, and no more than one consensus for each + * other flavor. */ char * -networkstatus_get_detached_signatures(networkstatus_t *consensus) +networkstatus_get_detached_signatures(smartlist_t *consensuses) { smartlist_t *elements; char buf[4096]; - char *result = NULL; - int n_sigs = 0; - tor_assert(consensus); - tor_assert(consensus->type == NS_TYPE_CONSENSUS); + char *result = NULL, *sigs = NULL; + networkstatus_t *consensus_ns = NULL; + tor_assert(consensuses); + + SMARTLIST_FOREACH(consensuses, networkstatus_t *, ns, { + tor_assert(ns); + tor_assert(ns->type == NS_TYPE_CONSENSUS); + if (ns && ns->flavor == FLAV_NS) + consensus_ns = ns; + }); + if (!consensus_ns) { + log_warn(LD_BUG, "No NS consensus given."); + return NULL; + } elements = smartlist_create(); @@ -1274,10 +2377,11 @@ networkstatus_get_detached_signatures(networkstatus_t *consensus) vu_buf[ISO_TIME_LEN+1]; char d[HEX_DIGEST_LEN+1]; - base16_encode(d, sizeof(d), consensus->networkstatus_digest, DIGEST_LEN); - format_iso_time(va_buf, consensus->valid_after); - format_iso_time(fu_buf, consensus->fresh_until); - format_iso_time(vu_buf, consensus->valid_until); + base16_encode(d, sizeof(d), + consensus_ns->digests.d[DIGEST_SHA1], DIGEST_LEN); + format_iso_time(va_buf, consensus_ns->valid_after); + format_iso_time(fu_buf, consensus_ns->fresh_until); + format_iso_time(vu_buf, consensus_ns->valid_until); tor_snprintf(buf, sizeof(buf), "consensus-digest %s\n" @@ -1287,45 +2391,89 @@ networkstatus_get_detached_signatures(networkstatus_t *consensus) smartlist_add(elements, tor_strdup(buf)); } - SMARTLIST_FOREACH(consensus->voters, networkstatus_voter_info_t *, v, - { - char sk[HEX_DIGEST_LEN+1]; - char id[HEX_DIGEST_LEN+1]; - if (!v->signature || v->bad_signature) + /* Get all the digests for the non-FLAV_NS consensuses */ + SMARTLIST_FOREACH_BEGIN(consensuses, networkstatus_t *, ns) { + const char *flavor_name = networkstatus_get_flavor_name(ns->flavor); + int alg; + if (ns->flavor == FLAV_NS) + continue; + + /* start with SHA256; we don't include SHA1 for anything but the basic + * consensus. */ + for (alg = DIGEST_SHA256; alg < N_DIGEST_ALGORITHMS; ++alg) { + char d[HEX_DIGEST256_LEN+1]; + const char *alg_name = + crypto_digest_algorithm_get_name(alg); + if (tor_mem_is_zero(ns->digests.d[alg], DIGEST256_LEN)) continue; - ++n_sigs; - base16_encode(sk, sizeof(sk), v->signing_key_digest, DIGEST_LEN); - base16_encode(id, sizeof(id), v->identity_digest, DIGEST_LEN); - tor_snprintf(buf, sizeof(buf), - "directory-signature %s %s\n-----BEGIN SIGNATURE-----\n", - id, sk); + base16_encode(d, sizeof(d), ns->digests.d[alg], DIGEST256_LEN); + tor_snprintf(buf, sizeof(buf), "additional-digest %s %s %s\n", + flavor_name, alg_name, d); smartlist_add(elements, tor_strdup(buf)); - base64_encode(buf, sizeof(buf), v->signature, v->signature_len); - strlcat(buf, "-----END SIGNATURE-----\n", sizeof(buf)); - smartlist_add(elements, tor_strdup(buf)); - }); + } + } SMARTLIST_FOREACH_END(ns); + + /* Now get all the sigs for non-FLAV_NS consensuses */ + SMARTLIST_FOREACH_BEGIN(consensuses, networkstatus_t *, ns) { + char *sigs; + if (ns->flavor == FLAV_NS) + continue; + sigs = networkstatus_format_signatures(ns, 1); + if (!sigs) { + log_warn(LD_DIR, "Couldn't format signatures"); + goto err; + } + smartlist_add(elements, sigs); + } SMARTLIST_FOREACH_END(ns); - result = smartlist_join_strings(elements, "", 0, NULL); + /* Now add the FLAV_NS consensus signatrures. */ + sigs = networkstatus_format_signatures(consensus_ns, 1); + if (!sigs) + goto err; + smartlist_add(elements, sigs); + result = smartlist_join_strings(elements, "", 0, NULL); + err: SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp)); smartlist_free(elements); - if (!n_sigs) - tor_free(result); return result; } +/** Return a newly allocated string holding a detached-signatures document for + * all of the in-progress consensuses in the <b>n_flavors</b>-element array at + * <b>pending</b>. */ +static char * +get_detached_signatures_from_pending_consensuses(pending_consensus_t *pending, + int n_flavors) +{ + int flav; + char *signatures; + smartlist_t *c = smartlist_create(); + for (flav = 0; flav < n_flavors; ++flav) { + if (pending[flav].consensus) + smartlist_add(c, pending[flav].consensus); + } + signatures = networkstatus_get_detached_signatures(c); + smartlist_free(c); + return signatures; +} + /** Release all storage held in <b>s</b>. */ void ns_detached_signatures_free(ns_detached_signatures_t *s) { + if (!s) + return; if (s->signatures) { - SMARTLIST_FOREACH(s->signatures, networkstatus_voter_info_t *, v, - { - tor_free(v->signature); - tor_free(v); - }); - smartlist_free(s->signatures); + STRMAP_FOREACH(s->signatures, flavor, smartlist_t *, sigs) { + SMARTLIST_FOREACH(sigs, document_signature_t *, sig, + document_signature_free(sig)); + smartlist_free(sigs); + } STRMAP_FOREACH_END; + strmap_free(s->signatures, NULL); + strmap_free(s->digests, _tor_free); } + tor_free(s); } @@ -1515,7 +2663,7 @@ dirvote_act(or_options_t *options, time_t now) if (voting_schedule.voting_ends < now && !voting_schedule.have_built_consensus) { log_notice(LD_DIR, "Time to compute a consensus."); - dirvote_compute_consensus(); + dirvote_compute_consensuses(); /* XXXX We will want to try again later if we haven't got enough * votes yet. Implement this if it turns out to ever happen. */ voting_schedule.have_built_consensus = 1; @@ -1552,14 +2700,13 @@ static smartlist_t *pending_vote_list = NULL; /** List of pending_vote_t for the previous vote. After we've used them to * build a consensus, the votes go here for the next period. */ static smartlist_t *previous_vote_list = NULL; -/** The body of the consensus that we're currently building. Once we - * have it built, it goes into dirserv.c */ -static char *pending_consensus_body = NULL; + +static pending_consensus_t pending_consensuses[N_CONSENSUS_FLAVORS]; + /** The detached signatures for the consensus that we're currently * building. */ static char *pending_consensus_signatures = NULL; -/** The parsed in-progress consensus document. */ -static networkstatus_t *pending_consensus = NULL; + /** List of ns_detached_signatures_t: hold signatures that get posted to us * before we have generated the consensus on our own. */ static smartlist_t *pending_consensus_signature_list = NULL; @@ -1653,15 +2800,39 @@ dirvote_fetch_missing_votes(void) static void dirvote_fetch_missing_signatures(void) { - if (!pending_consensus) + int need_any = 0; + int i; + for (i=0; i < N_CONSENSUS_FLAVORS; ++i) { + networkstatus_t *consensus = pending_consensuses[i].consensus; + if (!consensus || + networkstatus_check_consensus_signature(consensus, -1) == 1) { + /* We have no consensus, or we have one that's signed by everybody. */ + continue; + } + need_any = 1; + } + if (!need_any) return; - if (networkstatus_check_consensus_signature(pending_consensus, -1) == 1) - return; /* we have a signature from everybody. */ directory_get_from_all_authorities(DIR_PURPOSE_FETCH_DETACHED_SIGNATURES, 0, NULL); } +/** Release all storage held by pending consensuses (those waiting for + * signatures). */ +static void +dirvote_clear_pending_consensuses(void) +{ + int i; + for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) { + pending_consensus_t *pc = &pending_consensuses[i]; + tor_free(pc->body); + + networkstatus_vote_free(pc->consensus); + pc->consensus = NULL; + } +} + /** Drop all currently pending votes, consensus, and detached signatures. */ static void dirvote_clear_votes(int all_votes) @@ -1699,12 +2870,8 @@ dirvote_clear_votes(int all_votes) tor_free(cp)); smartlist_clear(pending_consensus_signature_list); } - tor_free(pending_consensus_body); tor_free(pending_consensus_signatures); - if (pending_consensus) { - networkstatus_vote_free(pending_consensus); - pending_consensus = NULL; - } + dirvote_clear_pending_consensuses(); } /** Return a newly allocated string containing the hex-encoded v3 authority @@ -1762,7 +2929,13 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out) } tor_assert(smartlist_len(vote->voters) == 1); vi = get_voter(vote); - tor_assert(vi->good_signature == 1); + { + int any_sig_good = 0; + SMARTLIST_FOREACH(vi->sigs, document_signature_t *, sig, + if (sig->good_signature) + any_sig_good = 1); + tor_assert(any_sig_good); + } ds = trusteddirserver_get_by_v3_auth_digest(vi->identity_digest); if (!ds) { char *keys = list_v3_auth_ids(); @@ -1799,6 +2972,9 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out) goto err; } + /* Fetch any new router descriptors we just learned about */ + update_consensus_router_descriptor_downloads(time(NULL), 1, vote); + /* Now see whether we already have a vote from this authority. */ SMARTLIST_FOREACH(pending_vote_list, pending_vote_t *, v, { if (fast_memeq(v->vote->cert->cache_info.identity_digest, @@ -1807,7 +2983,8 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out) networkstatus_voter_info_t *vi_old = get_voter(v->vote); if (fast_memeq(vi_old->vote_digest, vi->vote_digest, DIGEST_LEN)) { /* Ah, it's the same vote. Not a problem. */ - log_info(LD_DIR, "Discarding a vote we already have."); + log_info(LD_DIR, "Discarding a vote we already have (from %s).", + vi->address); if (*status_out < 200) *status_out = 200; goto discard; @@ -1834,7 +3011,7 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out) goto err; } } - }); + }); pending_vote = tor_malloc_zero(sizeof(pending_vote_t)); pending_vote->vote_body = new_cached_dir(tor_strndup(vote_body, @@ -1858,8 +3035,7 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out) *status_out = 400; discard: - if (vote) - networkstatus_vote_free(vote); + networkstatus_vote_free(vote); if (end_of_vote && !strcmpstart(end_of_vote, "network-status-version ")) { vote_body = end_of_vote; @@ -1886,14 +3062,18 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out) * pending_consensus: it won't be ready to be published until we have * everybody else's signatures collected too. (V3 Authority only) */ static int -dirvote_compute_consensus(void) +dirvote_compute_consensuses(void) { /* Have we got enough votes to try? */ - int n_votes, n_voters; + int n_votes, n_voters, n_vote_running = 0; smartlist_t *votes = NULL, *votestrings = NULL; char *consensus_body = NULL, *signatures = NULL, *votefile; networkstatus_t *consensus = NULL; authority_cert_t *my_cert; + pending_consensus_t pending[N_CONSENSUS_FLAVORS]; + int flav; + + memset(pending, 0, sizeof(pending)); if (!pending_vote_list) pending_vote_list = smartlist_create(); @@ -1902,7 +3082,20 @@ dirvote_compute_consensus(void) n_votes = smartlist_len(pending_vote_list); if (n_votes <= n_voters/2) { log_warn(LD_DIR, "We don't have enough votes to generate a consensus: " - "%d of %d", n_votes, n_voters/2); + "%d of %d", n_votes, n_voters/2+1); + goto err; + } + tor_assert(pending_vote_list); + SMARTLIST_FOREACH(pending_vote_list, pending_vote_t *, v, { + if (smartlist_string_isin(v->vote->known_flags, "Running")) + n_vote_running++; + }); + if (!n_vote_running) { + /* See task 1066. */ + log_warn(LD_DIR, "Nobody has voted on the Running flag. Generating " + "and publishing a consensus without Running nodes " + "would make many clients stop working. Not " + "generating a consensus!"); goto err; } @@ -1933,47 +3126,71 @@ dirvote_compute_consensus(void) char legacy_dbuf[DIGEST_LEN]; crypto_pk_env_t *legacy_sign=NULL; char *legacy_id_digest = NULL; + int n_generated = 0; if (get_options()->V3AuthUseLegacyKey) { authority_cert_t *cert = get_my_v3_legacy_cert(); legacy_sign = get_my_v3_legacy_signing_key(); if (cert) { - crypto_pk_get_digest(cert->identity_key, legacy_dbuf); - legacy_id_digest = legacy_dbuf; + if (crypto_pk_get_digest(cert->identity_key, legacy_dbuf)) { + log_warn(LD_BUG, + "Unable to compute digest of legacy v3 identity key"); + } else { + legacy_id_digest = legacy_dbuf; + } } } - consensus_body = networkstatus_compute_consensus( + + for (flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) { + const char *flavor_name = networkstatus_get_flavor_name(flav); + consensus_body = networkstatus_compute_consensus( votes, n_voters, my_cert->identity_key, - get_my_v3_authority_signing_key(), legacy_id_digest, legacy_sign); - } - if (!consensus_body) { - log_warn(LD_DIR, "Couldn't generate a consensus at all!"); - goto err; - } - consensus = networkstatus_parse_vote_from_string(consensus_body, NULL, - NS_TYPE_CONSENSUS); - if (!consensus) { - log_warn(LD_DIR, "Couldn't parse consensus we generated!"); - goto err; + get_my_v3_authority_signing_key(), legacy_id_digest, legacy_sign, + flav); + + if (!consensus_body) { + log_warn(LD_DIR, "Couldn't generate a %s consensus at all!", + flavor_name); + continue; + } + consensus = networkstatus_parse_vote_from_string(consensus_body, NULL, + NS_TYPE_CONSENSUS); + if (!consensus) { + log_warn(LD_DIR, "Couldn't parse %s consensus we generated!", + flavor_name); + tor_free(consensus_body); + continue; + } + + /* 'Check' our own signature, to mark it valid. */ + networkstatus_check_consensus_signature(consensus, -1); + + pending[flav].body = consensus_body; + pending[flav].consensus = consensus; + n_generated++; + consensus_body = NULL; + consensus = NULL; + } + if (!n_generated) { + log_warn(LD_DIR, "Couldn't generate any consensus flavors at all."); + goto err; + } } - /* 'Check' our own signature, to mark it valid. */ - networkstatus_check_consensus_signature(consensus, -1); - signatures = networkstatus_get_detached_signatures(consensus); + signatures = get_detached_signatures_from_pending_consensuses( + pending, N_CONSENSUS_FLAVORS); + if (!signatures) { log_warn(LD_DIR, "Couldn't extract signatures."); goto err; } - tor_free(pending_consensus_body); - pending_consensus_body = consensus_body; + dirvote_clear_pending_consensuses(); + memcpy(pending_consensuses, pending, sizeof(pending)); + tor_free(pending_consensus_signatures); pending_consensus_signatures = signatures; - if (pending_consensus) - networkstatus_vote_free(pending_consensus); - pending_consensus = consensus; - if (pending_consensus_signature_list) { int n_sigs = 0; /* we may have gotten signatures for this consensus before we built @@ -1981,7 +3198,7 @@ dirvote_compute_consensus(void) SMARTLIST_FOREACH(pending_consensus_signature_list, char *, sig, { const char *msg = NULL; - int r = dirvote_add_signatures_to_pending_consensus(sig, &msg); + int r = dirvote_add_signatures_to_all_pending_consensuses(sig, &msg); if (r >= 0) n_sigs += r; else @@ -2005,10 +3222,10 @@ dirvote_compute_consensus(void) strlen(pending_consensus_signatures), 0); log_notice(LD_DIR, "Signature(s) posted."); + smartlist_free(votes); return 0; err: - if (votes) - smartlist_free(votes); + smartlist_free(votes); tor_free(consensus_body); tor_free(signatures); networkstatus_vote_free(consensus); @@ -2017,76 +3234,62 @@ dirvote_compute_consensus(void) } /** Helper: we just got the <b>detached_signatures_body</b> sent to us as - * signatures on the currently pending consensus. Add them to the consensus + * signatures on the currently pending consensus. Add them to <b>pc</b> * as appropriate. Return the number of signatures added. (?) */ static int dirvote_add_signatures_to_pending_consensus( - const char *detached_signatures_body, + pending_consensus_t *pc, + ns_detached_signatures_t *sigs, const char **msg_out) { - ns_detached_signatures_t *sigs = NULL; + const char *flavor_name; int r = -1; - tor_assert(detached_signatures_body); - tor_assert(msg_out); - /* Only call if we have a pending consensus right now. */ - tor_assert(pending_consensus); - tor_assert(pending_consensus_body); + tor_assert(pc->consensus); + tor_assert(pc->body); tor_assert(pending_consensus_signatures); + flavor_name = networkstatus_get_flavor_name(pc->consensus->flavor); *msg_out = NULL; - if (!(sigs = networkstatus_parse_detached_signatures( - detached_signatures_body, NULL))) { - *msg_out = "Couldn't parse detached signatures."; - goto err; + { + smartlist_t *sig_list = strmap_get(sigs->signatures, flavor_name); + log_info(LD_DIR, "Have %d signatures for adding to %s consensus.", + sig_list ? smartlist_len(sig_list) : 0, flavor_name); } - - log_info(LD_DIR, "Have %d signatures for adding to consensus.", - smartlist_len(sigs->signatures)); - r = networkstatus_add_detached_signatures(pending_consensus, - sigs, msg_out); + r = networkstatus_add_detached_signatures(pc->consensus, sigs, msg_out); log_info(LD_DIR,"Added %d signatures to consensus.", r); if (r >= 1) { - char *new_detached = - networkstatus_get_detached_signatures(pending_consensus); - const char *src; + char *new_signatures = + networkstatus_format_signatures(pc->consensus, 0); char *dst, *dst_end; size_t new_consensus_len; - if (!new_detached) { + if (!new_signatures) { *msg_out = "No signatures to add"; goto err; } new_consensus_len = - strlen(pending_consensus_body) + strlen(new_detached) + 1; - pending_consensus_body = tor_realloc(pending_consensus_body, - new_consensus_len); - dst_end = pending_consensus_body + new_consensus_len; - dst = strstr(pending_consensus_body, "directory-signature "); + strlen(pc->body) + strlen(new_signatures) + 1; + pc->body = tor_realloc(pc->body, new_consensus_len); + dst_end = pc->body + new_consensus_len; + dst = strstr(pc->body, "directory-signature "); tor_assert(dst); - src = strstr(new_detached, "directory-signature "); - tor_assert(src); - strlcpy(dst, src, dst_end-dst); + strlcpy(dst, new_signatures, dst_end-dst); /* We remove this block once it has failed to crash for a while. But * unless it shows up in profiles, we're probably better leaving it in, * just in case we break detached signature processing at some point. */ { - ns_detached_signatures_t *sigs = - networkstatus_parse_detached_signatures(new_detached, NULL); networkstatus_t *v = networkstatus_parse_vote_from_string( - pending_consensus_body, NULL, + pc->body, NULL, NS_TYPE_CONSENSUS); - tor_assert(sigs); - ns_detached_signatures_free(sigs); tor_assert(v); networkstatus_vote_free(v); } - tor_free(pending_consensus_signatures); - pending_consensus_signatures = new_detached; *msg_out = "Signatures added"; + tor_free(new_signatures); } else if (r == 0) { *msg_out = "Signatures ignored"; } else { @@ -2098,8 +3301,62 @@ dirvote_add_signatures_to_pending_consensus( if (!*msg_out) *msg_out = "Unrecognized error while adding detached signatures."; done: - if (sigs) - ns_detached_signatures_free(sigs); + return r; +} + +static int +dirvote_add_signatures_to_all_pending_consensuses( + const char *detached_signatures_body, + const char **msg_out) +{ + int r=0, i, n_added = 0, errors = 0; + ns_detached_signatures_t *sigs; + tor_assert(detached_signatures_body); + tor_assert(msg_out); + tor_assert(pending_consensus_signatures); + + if (!(sigs = networkstatus_parse_detached_signatures( + detached_signatures_body, NULL))) { + *msg_out = "Couldn't parse detached signatures."; + goto err; + } + + for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) { + int res; + pending_consensus_t *pc = &pending_consensuses[i]; + if (!pc->consensus) + continue; + res = dirvote_add_signatures_to_pending_consensus(pc, sigs, msg_out); + if (res < 0) + errors++; + else + n_added += res; + } + + if (errors && !n_added) { + r = -1; + goto err; + } + + if (n_added && pending_consensuses[FLAV_NS].consensus) { + char *new_detached = + get_detached_signatures_from_pending_consensuses( + pending_consensuses, N_CONSENSUS_FLAVORS); + if (new_detached) { + tor_free(pending_consensus_signatures); + pending_consensus_signatures = new_detached; + } + } + + r = n_added; + goto done; + err: + if (!*msg_out) + *msg_out = "Unrecognized error while adding detached signatures."; + done: + ns_detached_signatures_free(sigs); + /* XXXX NM Check how return is used. We can now have an error *and* + signatures added. */ return r; } @@ -2112,10 +3369,10 @@ dirvote_add_signatures(const char *detached_signatures_body, const char *source, const char **msg) { - if (pending_consensus) { + if (pending_consensuses[FLAV_NS].consensus) { log_notice(LD_DIR, "Got a signature from %s. " "Adding it to the pending consensus.", source); - return dirvote_add_signatures_to_pending_consensus( + return dirvote_add_signatures_to_all_pending_consensuses( detached_signatures_body, msg); } else { log_notice(LD_DIR, "Got a signature from %s. " @@ -2134,17 +3391,25 @@ dirvote_add_signatures(const char *detached_signatures_body, static int dirvote_publish_consensus(void) { - /* Can we actually publish it yet? */ - if (!pending_consensus || - networkstatus_check_consensus_signature(pending_consensus, 1)<0) { - log_warn(LD_DIR, "Not enough info to publish pending consensus"); - return -1; - } + int i; + + /* Now remember all the other consensuses as if we were a directory cache. */ + for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) { + pending_consensus_t *pending = &pending_consensuses[i]; + const char *name; + name = networkstatus_get_flavor_name(i); + tor_assert(name); + if (!pending->consensus || + networkstatus_check_consensus_signature(pending->consensus, 1)<0) { + log_warn(LD_DIR, "Not enough info to publish pending %s consensus",name); + continue; + } - if (networkstatus_set_current_consensus(pending_consensus_body, 0)) - log_warn(LD_DIR, "Error publishing consensus"); - else - log_notice(LD_DIR, "Consensus published."); + if (networkstatus_set_current_consensus(pending->body, name, 0)) + log_warn(LD_DIR, "Error publishing %s consensus", name); + else + log_notice(LD_DIR, "Published %s consensus", name); + } return 0; } @@ -2154,20 +3419,16 @@ void dirvote_free_all(void) { dirvote_clear_votes(1); - /* now empty as a result of clear_pending_votes. */ + /* now empty as a result of dirvote_clear_votes(). */ smartlist_free(pending_vote_list); pending_vote_list = NULL; smartlist_free(previous_vote_list); previous_vote_list = NULL; - tor_free(pending_consensus_body); + dirvote_clear_pending_consensuses(); tor_free(pending_consensus_signatures); - if (pending_consensus) { - networkstatus_vote_free(pending_consensus); - pending_consensus = NULL; - } if (pending_consensus_signature_list) { - /* now empty as a result of clear_pending_votes. */ + /* now empty as a result of dirvote_clear_votes(). */ smartlist_free(pending_consensus_signature_list); pending_consensus_signature_list = NULL; } @@ -2179,13 +3440,14 @@ dirvote_free_all(void) /** Return the body of the consensus that we're currently trying to build. */ const char * -dirvote_get_pending_consensus(void) +dirvote_get_pending_consensus(consensus_flavor_t flav) { - return pending_consensus_body; + tor_assert(((int)flav) >= 0 && flav < N_CONSENSUS_FLAVORS); + return pending_consensuses[flav].body; } /** Return the signatures that we know for the consensus that we're currently - * trying to build */ + * trying to build. */ const char * dirvote_get_pending_detached_signatures(void) { @@ -2231,15 +3493,147 @@ dirvote_get_vote(const char *fp, int flags) } else { if (pending_vote_list && include_pending) { SMARTLIST_FOREACH(pending_vote_list, pending_vote_t *, pv, - if (fast_memeq(pv->vote->networkstatus_digest, fp, DIGEST_LEN)) + if (fast_memeq(pv->vote->digests.d[DIGEST_SHA1], fp, DIGEST_LEN)) return pv->vote_body); } if (previous_vote_list && include_previous) { SMARTLIST_FOREACH(previous_vote_list, pending_vote_t *, pv, - if (fast_memeq(pv->vote->networkstatus_digest, fp, DIGEST_LEN)) + if (fast_memeq(pv->vote->digests.d[DIGEST_SHA1], fp, DIGEST_LEN)) return pv->vote_body); } } return NULL; } +/** Construct and return a new microdescriptor from a routerinfo <b>ri</b>. + * + * XXX Right now, there is only one way to generate microdescriptors from + * router descriptors. This may change in future consensus methods. If so, + * we'll need an internal way to remember which method we used, and ask for a + * particular method. + **/ +microdesc_t * +dirvote_create_microdescriptor(const routerinfo_t *ri) +{ + microdesc_t *result = NULL; + char *key = NULL, *summary = NULL, *family = NULL; + char buf[1024]; + size_t keylen; + char *out = buf, *end = buf+sizeof(buf); + + if (crypto_pk_write_public_key_to_string(ri->onion_pkey, &key, &keylen)<0) + goto done; + summary = policy_summarize(ri->exit_policy); + if (ri->declared_family) + family = smartlist_join_strings(ri->declared_family, " ", 0, NULL); + + if (tor_snprintf(out, end-out, "onion-key\n%s", key)<0) + goto done; + out += strlen(out); + if (family) { + if (tor_snprintf(out, end-out, "family %s\n", family)<0) + goto done; + out += strlen(out); + } + if (summary && strcmp(summary, "reject 1-65535")) { + if (tor_snprintf(out, end-out, "p %s\n", summary)<0) + goto done; + out += strlen(out); + } + *out = '\0'; /* Make sure it's nul-terminated. This should be a no-op */ + + { + smartlist_t *lst = microdescs_parse_from_string(buf, out, 0, 1); + if (smartlist_len(lst) != 1) { + log_warn(LD_DIR, "We generated a microdescriptor we couldn't parse."); + SMARTLIST_FOREACH(lst, microdesc_t *, md, microdesc_free(md)); + smartlist_free(lst); + goto done; + } + result = smartlist_get(lst, 0); + smartlist_free(lst); + } + + done: + tor_free(key); + tor_free(summary); + tor_free(family); + return result; +} + +/** Cached space-separated string to hold */ +static char *microdesc_consensus_methods = NULL; + +/** Format the appropriate vote line to describe the microdescriptor <b>md</b> + * in a consensus vote document. Write it into the <b>out_len</b>-byte buffer + * in <b>out</b>. Return -1 on failure and the number of characters written + * on success. */ +ssize_t +dirvote_format_microdesc_vote_line(char *out, size_t out_len, + const microdesc_t *md) +{ + char d64[BASE64_DIGEST256_LEN+1]; + if (!microdesc_consensus_methods) { + microdesc_consensus_methods = + make_consensus_method_list(MIN_METHOD_FOR_MICRODESC, + MAX_SUPPORTED_CONSENSUS_METHOD, + ","); + tor_assert(microdesc_consensus_methods); + } + if (digest256_to_base64(d64, md->digest)<0) + return -1; + + if (tor_snprintf(out, out_len, "m %s sha256=%s\n", + microdesc_consensus_methods, d64)<0) + return -1; + + return strlen(out); +} + +/** If <b>vrs</b> has a hash made for the consensus method <b>method</b> with + * the digest algorithm <b>alg</b>, decode it and copy it into + * <b>digest256_out</b> and return 0. Otherwise return -1. */ +int +vote_routerstatus_find_microdesc_hash(char *digest256_out, + const vote_routerstatus_t *vrs, + int method, + digest_algorithm_t alg) +{ + /* XXXX only returns the sha256 method. */ + const vote_microdesc_hash_t *h; + char mstr[64]; + size_t mlen; + char dstr[64]; + + tor_snprintf(mstr, sizeof(mstr), "%d", method); + mlen = strlen(mstr); + tor_snprintf(dstr, sizeof(dstr), " %s=", + crypto_digest_algorithm_get_name(alg)); + + for (h = vrs->microdesc; h; h = h->next) { + const char *cp = h->microdesc_hash_line; + size_t num_len; + /* cp looks like \d+(,\d+)* (digesttype=val )+ . Let's hunt for mstr in + * the first part. */ + while (1) { + num_len = strspn(cp, "1234567890"); + if (num_len == mlen && fast_memeq(mstr, cp, mlen)) { + /* This is the line. */ + char buf[BASE64_DIGEST256_LEN+1]; + /* XXXX ignores extraneous stuff if the digest is too long. This + * seems harmless enough, right? */ + cp = strstr(cp, dstr); + if (!cp) + return -1; + cp += strlen(dstr); + strlcpy(buf, cp, sizeof(buf)); + return digest256_from_base64(digest256_out, buf); + } + if (num_len == 0 || cp[num_len] != ',') + break; + cp += num_len + 1; + } + } + return -1; +} + diff --git a/src/or/dirvote.h b/src/or/dirvote.h new file mode 100644 index 0000000000..67540a37fb --- /dev/null +++ b/src/or/dirvote.h @@ -0,0 +1,90 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file dirvote.h + * \brief Header file for dirvote.c. + **/ + +#ifndef _TOR_DIRVOTE_H +#define _TOR_DIRVOTE_H + +/** Lowest allowable value for VoteSeconds. */ +#define MIN_VOTE_SECONDS 20 +/** Lowest allowable value for DistSeconds. */ +#define MIN_DIST_SECONDS 20 +/** Smallest allowable voting interval. */ +#define MIN_VOTE_INTERVAL 300 + +void dirvote_free_all(void); + +/* vote manipulation */ +char *networkstatus_compute_consensus(smartlist_t *votes, + int total_authorities, + crypto_pk_env_t *identity_key, + crypto_pk_env_t *signing_key, + const char *legacy_identity_key_digest, + crypto_pk_env_t *legacy_signing_key, + consensus_flavor_t flavor); +int networkstatus_add_detached_signatures(networkstatus_t *target, + ns_detached_signatures_t *sigs, + const char **msg_out); +char *networkstatus_get_detached_signatures(smartlist_t *consensuses); +void ns_detached_signatures_free(ns_detached_signatures_t *s); + +/* cert manipulation */ +authority_cert_t *authority_cert_dup(authority_cert_t *cert); + +/* vote scheduling */ +void dirvote_get_preferred_voting_intervals(vote_timing_t *timing_out); +time_t dirvote_get_start_of_next_interval(time_t now, int interval); +void dirvote_recalculate_timing(or_options_t *options, time_t now); +void dirvote_act(or_options_t *options, time_t now); + +/* invoked on timers and by outside triggers. */ +struct pending_vote_t * dirvote_add_vote(const char *vote_body, + const char **msg_out, + int *status_out); +int dirvote_add_signatures(const char *detached_signatures_body, + const char *source, + const char **msg_out); + +/* Item access */ +const char *dirvote_get_pending_consensus(consensus_flavor_t flav); +const char *dirvote_get_pending_detached_signatures(void); +#define DGV_BY_ID 1 +#define DGV_INCLUDE_PENDING 2 +#define DGV_INCLUDE_PREVIOUS 4 +const cached_dir_t *dirvote_get_vote(const char *fp, int flags); +void set_routerstatus_from_routerinfo(routerstatus_t *rs, + routerinfo_t *ri, time_t now, + int naming, int listbadexits, + int listbaddirs); +void router_clear_status_flags(routerinfo_t *ri); +networkstatus_t * +dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key, + authority_cert_t *cert); + +microdesc_t *dirvote_create_microdescriptor(const routerinfo_t *ri); +ssize_t dirvote_format_microdesc_vote_line(char *out, size_t out_len, + const microdesc_t *md); + +int vote_routerstatus_find_microdesc_hash(char *digest256_out, + const vote_routerstatus_t *vrs, + int method, + digest_algorithm_t alg); +document_signature_t *voter_get_sig_by_algorithm( + const networkstatus_voter_info_t *voter, + digest_algorithm_t alg); + +#ifdef DIRVOTE_PRIVATE +char *format_networkstatus_vote(crypto_pk_env_t *private_key, + networkstatus_t *v3_ns); +char *dirvote_compute_params(smartlist_t *votes); +#endif + +#endif + diff --git a/src/or/dns.c b/src/or/dns.c index 3bb9c34892..9b6b98afaf 100644 --- a/src/or/dns.c +++ b/src/or/dns.c @@ -12,8 +12,70 @@ **/ #include "or.h" +#include "circuitlist.h" +#include "circuituse.h" +#include "config.h" +#include "connection.h" +#include "connection_edge.h" +#include "control.h" +#include "dns.h" +#include "main.h" +#include "policies.h" +#include "relay.h" +#include "router.h" #include "ht.h" +#ifdef HAVE_EVENT2_DNS_H +#include <event2/event.h> +#include <event2/dns.h> +#else +#include <event.h> #include "eventdns.h" +#ifndef HAVE_EVDNS_SET_DEFAULT_OUTGOING_BIND_ADDRESS +#define HAVE_EVDNS_SET_DEFAULT_OUTGOING_BIND_ADDRESS +#endif +#endif + +#ifndef HAVE_EVENT2_DNS_H +struct evdns_base; +struct evdns_request; +#define evdns_base_new(x,y) tor_malloc(1) +#define evdns_base_clear_nameservers_and_suspend(base) \ + evdns_clear_nameservers_and_suspend() +#define evdns_base_search_clear(base) evdns_search_clear() +#define evdns_base_set_default_outgoing_bind_address(base, a, len) \ + evdns_set_default_outgoing_bind_address((a),(len)) +#define evdns_base_resolv_conf_parse(base, options, fname) \ + evdns_resolv_conf_parse((options), (fname)) +#define evdns_base_count_nameservers(base) \ + evdns_count_nameservers() +#define evdns_base_resume(base) \ + evdns_resume() +#define evdns_base_config_windows_nameservers(base) \ + evdns_config_windows_nameservers() +#define evdns_base_set_option_(base, opt, val) \ + evdns_set_option((opt),(val),DNS_OPTIONS_ALL) +/* Note: our internal eventdns.c, plus Libevent 1.4, used a 1 return to + * signify failure to launch a resolve. Libevent 2.0 uses a -1 return to + * signify a failure on a resolve, though if we're on Libevent 2.0, we should + * have event2/dns.h and never hit these macros. Regardless, 0 is success. */ +#define evdns_base_resolve_ipv4(base, addr, options, cb, ptr) \ + ((evdns_resolve_ipv4((addr), (options), (cb), (ptr))!=0) \ + ? NULL : ((void*)1)) +#define evdns_base_resolve_reverse(base, addr, options, cb, ptr) \ + ((evdns_resolve_reverse((addr), (options), (cb), (ptr))!=0) \ + ? NULL : ((void*)1)) +#define evdns_base_resolve_reverse_ipv6(base, addr, options, cb, ptr) \ + ((evdns_resolve_reverse_ipv6((addr), (options), (cb), (ptr))!=0) \ + ? NULL : ((void*)1)) + +#elif defined(LIBEVENT_VERSION_NUMBER) && LIBEVENT_VERSION_NUMBER < 0x02000303 +#define evdns_base_set_option_(base, opt, val) \ + evdns_base_set_option((base), (opt),(val),DNS_OPTIONS_ALL) + +#else +#define evdns_base_set_option_ evdns_base_set_option + +#endif /** Longest hostname we're willing to resolve. */ #define MAX_ADDRESSLEN 256 @@ -28,6 +90,9 @@ #define DNS_RESOLVE_FAILED_PERMANENT 2 #define DNS_RESOLVE_SUCCEEDED 3 +/** Our evdns_base; this structure handles all our name lookups. */ +static struct evdns_base *the_evdns_base = NULL; + /** Have we currently configured nameservers with eventdns? */ static int nameservers_configured = 0; /** Did our most recent attempt to configure nameservers with eventdns fail? */ @@ -89,6 +154,8 @@ typedef struct cached_resolve_t { uint32_t ttl; /**< What TTL did the nameserver tell us? */ /** Connections that want to know when we get an answer for this resolve. */ pending_connection_t *pending_connections; + /** Position of this element in the heap*/ + int minheap_idx; } cached_resolve_t; static void purge_expired_resolves(time_t now); @@ -211,8 +278,16 @@ dns_reset(void) { or_options_t *options = get_options(); if (! server_mode(options)) { - evdns_clear_nameservers_and_suspend(); - evdns_search_clear(); + + if (!the_evdns_base) { + if (!(the_evdns_base = evdns_base_new(tor_libevent_get_base(), 0))) { + log_err(LD_BUG, "Couldn't create an evdns_base"); + return -1; + } + } + + evdns_base_clear_nameservers_and_suspend(the_evdns_base); + evdns_base_search_clear(the_evdns_base); nameservers_configured = 0; tor_free(resolv_conf_fname); resolv_conf_mtime = 0; @@ -262,6 +337,8 @@ dns_get_expiry_ttl(uint32_t ttl) static void _free_cached_resolve(cached_resolve_t *r) { + if (!r) + return; while (r->pending_connections) { pending_connection_t *victim = r->pending_connections; r->pending_connections = victim->next; @@ -303,6 +380,7 @@ set_expiry(cached_resolve_t *resolve, time_t expires) resolve->expire = expires; smartlist_pqueue_add(cached_resolve_pqueue, _compare_cached_resolves_by_expiry, + STRUCT_OFFSET(cached_resolve_t, minheap_idx), resolve); } @@ -325,8 +403,7 @@ dns_free_all(void) _free_cached_resolve(item); } HT_CLEAR(cache_map, &cache_root); - if (cached_resolve_pqueue) - smartlist_free(cached_resolve_pqueue); + smartlist_free(cached_resolve_pqueue); cached_resolve_pqueue = NULL; tor_free(resolv_conf_fname); } @@ -349,7 +426,8 @@ purge_expired_resolves(time_t now) if (resolve->expire > now) break; smartlist_pqueue_pop(cached_resolve_pqueue, - _compare_cached_resolves_by_expiry); + _compare_cached_resolves_by_expiry, + STRUCT_OFFSET(cached_resolve_t, minheap_idx)); if (resolve->state == CACHE_STATE_PENDING) { log_debug(LD_EXIT, @@ -393,7 +471,7 @@ purge_expired_resolves(time_t now) log_err(LD_BUG, "The expired resolve we purged didn't match any in" " the cache. Tried to purge %s (%p); instead got %s (%p).", resolve->address, (void*)resolve, - removed ? removed->address : "NULL", (void*)remove); + removed ? removed->address : "NULL", (void*)removed); } tor_assert(removed == resolve); } else { @@ -611,7 +689,7 @@ dns_resolve_impl(edge_connection_t *exitconn, int is_resolve, * know the answer. */ if (tor_addr_from_str(&addr, exitconn->_base.address) >= 0) { if (tor_addr_family(&addr) == AF_INET) { - tor_addr_assign(&exitconn->_base.addr, &addr); + tor_addr_copy(&exitconn->_base.addr, &addr); exitconn->address_ttl = DEFAULT_DNS_TTL; return 1; } else { @@ -711,6 +789,7 @@ dns_resolve_impl(edge_connection_t *exitconn, int is_resolve, resolve = tor_malloc_zero(sizeof(cached_resolve_t)); resolve->magic = CACHED_RESOLVE_MAGIC; resolve->state = CACHE_STATE_PENDING; + resolve->minheap_idx = -1; resolve->is_reverse = is_reverse; strlcpy(resolve->address, exitconn->_base.address, sizeof(resolve->address)); @@ -807,7 +886,8 @@ connection_dns_remove(edge_connection_t *conn) tor_free(pend); log_debug(LD_EXIT, "First connection (fd %d) no longer waiting " "for resolve of %s", - conn->_base.s, escaped_safe_str(conn->_base.address)); + conn->_base.s, + escaped_safe_str(conn->_base.address)); return; } else { for ( ; pend->next; pend = pend->next) { @@ -1108,6 +1188,14 @@ configure_nameservers(int force) conf_fname = "/etc/resolv.conf"; #endif + if (!the_evdns_base) { + if (!(the_evdns_base = evdns_base_new(tor_libevent_get_base(), 0))) { + log_err(LD_BUG, "Couldn't create an evdns_base"); + return -1; + } + } + +#ifdef HAVE_EVDNS_SET_DEFAULT_OUTGOING_BIND_ADDRESS if (options->OutboundBindAddress) { tor_addr_t addr; if (tor_addr_from_str(&addr, options->OutboundBindAddress) < 0) { @@ -1118,20 +1206,17 @@ configure_nameservers(int force) struct sockaddr_storage ss; socklen = tor_addr_to_sockaddr(&addr, 0, (struct sockaddr *)&ss, sizeof(ss)); - if (socklen < 0) { + if (socklen <= 0) { log_warn(LD_BUG, "Couldn't convert outbound bind address to sockaddr." " Ignoring."); } else { - evdns_set_default_outgoing_bind_address((struct sockaddr *)&ss, - socklen); + evdns_base_set_default_outgoing_bind_address(the_evdns_base, + (struct sockaddr *)&ss, + socklen); } } } - - if (options->ServerDNSRandomizeCase) - evdns_set_option("randomize-case:", "1", DNS_OPTIONS_ALL); - else - evdns_set_option("randomize-case:", "0", DNS_OPTIONS_ALL); +#endif evdns_set_log_fn(evdns_log_cb); if (conf_fname) { @@ -1146,16 +1231,17 @@ configure_nameservers(int force) return 0; } if (nameservers_configured) { - evdns_search_clear(); - evdns_clear_nameservers_and_suspend(); + evdns_base_search_clear(the_evdns_base); + evdns_base_clear_nameservers_and_suspend(the_evdns_base); } log_info(LD_EXIT, "Parsing resolver configuration in '%s'", conf_fname); - if ((r = evdns_resolv_conf_parse(DNS_OPTIONS_ALL, conf_fname))) { + if ((r = evdns_base_resolv_conf_parse(the_evdns_base, + DNS_OPTIONS_ALL, conf_fname))) { log_warn(LD_EXIT, "Unable to parse '%s', or no nameservers in '%s' (%d)", conf_fname, conf_fname, r); goto err; } - if (evdns_count_nameservers() == 0) { + if (evdns_base_count_nameservers(the_evdns_base) == 0) { log_warn(LD_EXIT, "Unable to find any nameservers in '%s'.", conf_fname); goto err; } @@ -1163,51 +1249,63 @@ configure_nameservers(int force) resolv_conf_fname = tor_strdup(conf_fname); resolv_conf_mtime = st.st_mtime; if (nameservers_configured) - evdns_resume(); + evdns_base_resume(the_evdns_base); } #ifdef MS_WINDOWS else { if (nameservers_configured) { - evdns_search_clear(); - evdns_clear_nameservers_and_suspend(); + evdns_base_search_clear(the_evdns_base); + evdns_base_clear_nameservers_and_suspend(the_evdns_base); } - if (evdns_config_windows_nameservers()) { + if (evdns_base_config_windows_nameservers(the_evdns_base)) { log_warn(LD_EXIT,"Could not config nameservers."); goto err; } - if (evdns_count_nameservers() == 0) { + if (evdns_base_count_nameservers(the_evdns_base) == 0) { log_warn(LD_EXIT, "Unable to find any platform nameservers in " "your Windows configuration."); goto err; } if (nameservers_configured) - evdns_resume(); + evdns_base_resume(the_evdns_base); tor_free(resolv_conf_fname); resolv_conf_mtime = 0; } #endif - if (evdns_count_nameservers() == 1) { - evdns_set_option("max-timeouts:", "16", DNS_OPTIONS_ALL); - evdns_set_option("timeout:", "10", DNS_OPTIONS_ALL); +#define SET(k,v) evdns_base_set_option_(the_evdns_base, (k), (v)) + + if (evdns_base_count_nameservers(the_evdns_base) == 1) { + SET("max-timeouts:", "16"); + SET("timeout:", "10"); } else { - evdns_set_option("max-timeouts:", "3", DNS_OPTIONS_ALL); - evdns_set_option("timeout:", "5", DNS_OPTIONS_ALL); + SET("max-timeouts:", "3"); + SET("timeout:", "5"); } + if (options->ServerDNSRandomizeCase) + SET("randomize-case:", "1"); + else + SET("randomize-case:", "0"); + +#undef SET + dns_servers_relaunch_checks(); nameservers_configured = 1; if (nameserver_config_failed) { nameserver_config_failed = 0; - mark_my_descriptor_dirty(); + /* XXX the three calls to republish the descriptor might be producing + * descriptors that are only cosmetically different, especially on + * non-exit relays! -RD */ + mark_my_descriptor_dirty("dns resolvers back"); } return 0; err: nameservers_configured = 0; if (! nameserver_config_failed) { nameserver_config_failed = 1; - mark_my_descriptor_dirty(); + mark_my_descriptor_dirty("dns resolvers failed"); } return -1; } @@ -1292,6 +1390,7 @@ static int launch_resolve(edge_connection_t *exitconn) { char *addr = tor_strdup(exitconn->_base.address); + struct evdns_request *req = NULL; tor_addr_t a; int r; int options = get_options()->ServerDNSSearchDomains ? 0 @@ -1307,28 +1406,34 @@ launch_resolve(edge_connection_t *exitconn) r = tor_addr_parse_reverse_lookup_name( &a, exitconn->_base.address, AF_UNSPEC, 0); + + tor_assert(the_evdns_base); if (r == 0) { log_info(LD_EXIT, "Launching eventdns request for %s", escaped_safe_str(exitconn->_base.address)); - r = evdns_resolve_ipv4(exitconn->_base.address, options, - evdns_callback, addr); + req = evdns_base_resolve_ipv4(the_evdns_base, + exitconn->_base.address, options, + evdns_callback, addr); } else if (r == 1) { log_info(LD_EXIT, "Launching eventdns reverse request for %s", escaped_safe_str(exitconn->_base.address)); if (tor_addr_family(&a) == AF_INET) - r = evdns_resolve_reverse(tor_addr_to_in(&a), DNS_QUERY_NO_SEARCH, + req = evdns_base_resolve_reverse(the_evdns_base, + tor_addr_to_in(&a), DNS_QUERY_NO_SEARCH, evdns_callback, addr); else - r = evdns_resolve_reverse_ipv6(tor_addr_to_in6(&a), DNS_QUERY_NO_SEARCH, + req = evdns_base_resolve_reverse_ipv6(the_evdns_base, + tor_addr_to_in6(&a), DNS_QUERY_NO_SEARCH, evdns_callback, addr); } else if (r == -1) { log_warn(LD_BUG, "Somehow a malformed in-addr.arpa address reached here."); } - if (r) { - log_warn(LD_EXIT, "eventdns rejected address %s: error %d.", - escaped_safe_str(addr), r); - r = evdns_err_is_transient(r) ? -2 : -1; + r = 0; + if (!req) { + log_warn(LD_EXIT, "eventdns rejected address %s.", + escaped_safe_str(addr)); + r = -1; tor_free(addr); /* There is no evdns request in progress; stop * addr from getting leaked. */ } @@ -1420,7 +1525,7 @@ add_wildcarded_test_address(const char *address) "broken.", address, n); if (!dns_is_completely_invalid) { dns_is_completely_invalid = 1; - mark_my_descriptor_dirty(); + mark_my_descriptor_dirty("dns hijacking confirmed"); } if (!dns_wildcarded_test_address_notice_given) control_event_server_status(LOG_WARN, "DNS_USELESS"); @@ -1449,8 +1554,8 @@ evdns_wildcard_check_callback(int result, char type, int count, int ttl, } log(dns_wildcard_one_notice_given ? LOG_INFO : LOG_NOTICE, LD_EXIT, "Your DNS provider gave an answer for \"%s\", which " - "is not supposed to exist. Apparently they are hijacking " - "DNS failures. Trying to correct for this. We've noticed %d " + "is not supposed to exist. Apparently they are hijacking " + "DNS failures. Trying to correct for this. We've noticed %d " "possibly bad address%s so far.", string_address, strmap_size(dns_wildcard_response_count), (strmap_size(dns_wildcard_response_count) == 1) ? "" : "es"); @@ -1466,17 +1571,20 @@ static void launch_wildcard_check(int min_len, int max_len, const char *suffix) { char *addr; - int r; + struct evdns_request *req; addr = crypto_random_hostname(min_len, max_len, "", suffix); log_info(LD_EXIT, "Testing whether our DNS server is hijacking nonexistent " "domains with request for bogus hostname \"%s\"", addr); - r = evdns_resolve_ipv4(/* This "addr" tells us which address to resolve */ + tor_assert(the_evdns_base); + req = evdns_base_resolve_ipv4( + the_evdns_base, + /* This "addr" tells us which address to resolve */ addr, DNS_QUERY_NO_SEARCH, evdns_wildcard_check_callback, /* This "addr" is an argument to the callback*/ addr); - if (r) { + if (!req) { /* There is no evdns request in progress; stop addr from getting leaked */ tor_free(addr); } @@ -1488,6 +1596,7 @@ static void launch_test_addresses(int fd, short event, void *args) { or_options_t *options = get_options(); + struct evdns_request *req; (void)fd; (void)event; (void)args; @@ -1499,14 +1608,19 @@ launch_test_addresses(int fd, short event, void *args) * be an exit server.*/ if (!options->ServerDNSTestAddresses) return; - SMARTLIST_FOREACH(options->ServerDNSTestAddresses, const char *, address, - { - int r = evdns_resolve_ipv4(address, DNS_QUERY_NO_SEARCH, evdns_callback, - tor_strdup(address)); - if (r) - log_info(LD_EXIT, "eventdns rejected test address %s: error %d", - escaped_safe_str(address), r); - }); + tor_assert(the_evdns_base); + SMARTLIST_FOREACH_BEGIN(options->ServerDNSTestAddresses, + const char *, address) { + char *a = tor_strdup(address); + req = evdns_base_resolve_ipv4(the_evdns_base, + address, DNS_QUERY_NO_SEARCH, evdns_callback, a); + + if (!req) { + log_info(LD_EXIT, "eventdns rejected test address %s", + escaped_safe_str(address)); + tor_free(a); + } + } SMARTLIST_FOREACH_END(address); } #define N_WILDCARD_CHECKS 2 @@ -1547,7 +1661,7 @@ dns_launch_wildcard_checks(void) void dns_launch_correctness_checks(void) { - static struct event launch_event; + static struct event *launch_event = NULL; struct timeval timeout; if (!get_options()->ServerDNSDetectHijacking) return; @@ -1555,10 +1669,12 @@ dns_launch_correctness_checks(void) /* Wait a while before launching requests for test addresses, so we can * get the results from checking for wildcarding. */ - evtimer_set(&launch_event, launch_test_addresses, NULL); + if (! launch_event) + launch_event = tor_evtimer_new(tor_libevent_get_base(), + launch_test_addresses, NULL); timeout.tv_sec = 30; timeout.tv_usec = 0; - if (evtimer_add(&launch_event, &timeout)<0) { + if (evtimer_add(launch_event, &timeout)<0) { log_warn(LD_BUG, "Couldn't add timer for checking for dns hijacking"); } } @@ -1574,10 +1690,9 @@ dns_seems_to_be_broken(void) void dns_reset_correctness_checks(void) { - if (dns_wildcard_response_count) { - strmap_free(dns_wildcard_response_count, _tor_free); - dns_wildcard_response_count = NULL; - } + strmap_free(dns_wildcard_response_count, _tor_free); + dns_wildcard_response_count = NULL; + n_wildcard_requests = 0; if (dns_wildcard_list) { @@ -1622,6 +1737,30 @@ assert_resolve_ok(cached_resolve_t *resolve) } } +/** Return the number of DNS cache entries as an int */ +static int +dns_cache_entry_count(void) +{ + return HT_SIZE(&cache_root); +} + +/** Log memory information about our internal DNS cache at level 'severity'. */ +void +dump_dns_mem_usage(int severity) +{ + /* This should never be larger than INT_MAX. */ + int hash_count = dns_cache_entry_count(); + size_t hash_mem = sizeof(struct cached_resolve_t) * hash_count; + hash_mem += HT_MEM_USAGE(&cache_root); + + /* Print out the count and estimated size of our &cache_root. It undercounts + hostnames in cached reverse resolves. + */ + log(severity, LD_MM, "Our DNS cache has %d entries.", hash_count); + log(severity, LD_MM, "Our DNS cache size is approximately %u bytes.", + (unsigned)hash_mem); +} + #ifdef DEBUG_DNS_CACHE /** Exit with an assertion if the DNS cache is corrupt. */ static void @@ -1642,7 +1781,8 @@ _assert_cache_ok(void) return; smartlist_pqueue_assert_ok(cached_resolve_pqueue, - _compare_cached_resolves_by_expiry); + _compare_cached_resolves_by_expiry, + STRUCT_OFFSET(cached_resolve_t, minheap_idx)); SMARTLIST_FOREACH(cached_resolve_pqueue, cached_resolve_t *, res, { diff --git a/src/or/dns.h b/src/or/dns.h new file mode 100644 index 0000000000..25ff86e2c6 --- /dev/null +++ b/src/or/dns.h @@ -0,0 +1,31 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file dns.h + * \brief Header file for dns.c. + **/ + +#ifndef _TOR_DNS_H +#define _TOR_DNS_H + +int dns_init(void); +int has_dns_init_failed(void); +void dns_free_all(void); +uint32_t dns_clip_ttl(uint32_t ttl); +int dns_reset(void); +void connection_dns_remove(edge_connection_t *conn); +void assert_connection_edge_not_dns_pending(edge_connection_t *conn); +void assert_all_pending_dns_resolves_ok(void); +void dns_cancel_pending_resolve(const char *question); +int dns_resolve(edge_connection_t *exitconn); +void dns_launch_correctness_checks(void); +int dns_seems_to_be_broken(void); +void dns_reset_correctness_checks(void); +void dump_dns_mem_usage(int severity); + +#endif + diff --git a/src/or/dnsserv.c b/src/or/dnsserv.c index f8b5c3cdd2..243b730cbf 100644 --- a/src/or/dnsserv.c +++ b/src/or/dnsserv.c @@ -9,7 +9,21 @@ **/ #include "or.h" +#include "dnsserv.h" +#include "config.h" +#include "connection.h" +#include "connection_edge.h" +#include "control.h" +#include "main.h" +#include "policies.h" +#ifdef HAVE_EVENT2_DNS_H +#include <event2/dns.h> +#include <event2/dns_compat.h> +/* XXXX023 this implies we want an improved evdns */ +#include <event2/dns_struct.h> +#else #include "eventdns.h" +#endif /** Helper function: called by evdns whenever the client sends a request to our * DNSPort. We need to eventually answer the request <b>req</b>. @@ -85,12 +99,7 @@ evdns_server_callback(struct evdns_server_request *req, void *_data) evdns_server_request_respond(req, DNS_ERR_NONE); return; } - if (q->type == EVDNS_TYPE_A) { - /* Refuse any attempt to resolve a noconnect address, right now. */ - if (hostname_is_noconnect_address(q->name)) { - err = DNS_ERR_REFUSED; - } - } else { + if (q->type != EVDNS_TYPE_A) { tor_assert(q->type == EVDNS_TYPE_PTR); } @@ -132,17 +141,18 @@ evdns_server_callback(struct evdns_server_request *req, void *_data) control_event_stream_status(conn, STREAM_EVENT_NEW, 0); - /* Now, throw the connection over to get rewritten (which will answer it - * immediately if it's in the cache, or completely bogus, or automapped), - * and then attached to a circuit. */ + /* Now, unless a controller asked us to leave streams unattached, + * throw the connection over to get rewritten (which will + * answer it immediately if it's in the cache, or completely bogus, or + * automapped), and then attached to a circuit. */ log_info(LD_APP, "Passing request for %s to rewrite_and_attach.", - escaped_safe_str(q->name)); + escaped_safe_str_client(q->name)); q_name = tor_strdup(q->name); /* q could be freed in rewrite_and_attach */ - connection_ap_handshake_rewrite_and_attach(conn, NULL, NULL); + connection_ap_rewrite_and_attach_if_allowed(conn, NULL, NULL); /* Now, the connection is marked if it was bad. */ - log_info(LD_APP, "Passed request for %s to rewrite_and_attach.", - escaped_safe_str(q_name)); + log_info(LD_APP, "Passed request for %s to rewrite_and_attach_if_allowed.", + escaped_safe_str_client(q_name)); tor_free(q_name); } @@ -177,17 +187,18 @@ dnsserv_launch_request(const char *name, int reverse) return -1; } - /* Now, throw the connection over to get rewritten (which will answer it - * immediately if it's in the cache, or completely bogus, or automapped), - * and then attached to a circuit. */ + /* Now, unless a controller asked us to leave streams unattached, + * throw the connection over to get rewritten (which will + * answer it immediately if it's in the cache, or completely bogus, or + * automapped), and then attached to a circuit. */ log_info(LD_APP, "Passing request for %s to rewrite_and_attach.", - escaped_safe_str(name)); + escaped_safe_str_client(name)); q_name = tor_strdup(name); /* q could be freed in rewrite_and_attach */ - connection_ap_handshake_rewrite_and_attach(conn, NULL, NULL); + connection_ap_rewrite_and_attach_if_allowed(conn, NULL, NULL); /* Now, the connection is marked if it was bad. */ - log_info(LD_APP, "Passed request for %s to rewrite_and_attach.", - escaped_safe_str(q_name)); + log_info(LD_APP, "Passed request for %s to rewrite_and_attach_if_allowed.", + escaped_safe_str_client(q_name)); tor_free(q_name); return 0; } @@ -269,7 +280,7 @@ dnsserv_resolved(edge_connection_t *conn, conn->socks_request->command == SOCKS_COMMAND_RESOLVE) { evdns_server_request_add_a_reply(req, name, - 1, (char*)answer, ttl); + 1, answer, ttl); } else if (answer_type == RESOLVED_TYPE_HOSTNAME && answer_len < 256 && conn->socks_request->command == SOCKS_COMMAND_RESOLVE_PTR) { @@ -298,8 +309,8 @@ dnsserv_configure_listener(connection_t *conn) tor_assert(conn->s >= 0); tor_assert(conn->type == CONN_TYPE_AP_DNS_LISTENER); - conn->dns_server_port = evdns_add_server_port(conn->s, 0, - evdns_server_callback, NULL); + conn->dns_server_port = + tor_evdns_add_server_port(conn->s, 0, evdns_server_callback, NULL); } /** Free the evdns server port for <b>conn</b>, which must be an diff --git a/src/or/dnsserv.h b/src/or/dnsserv.h new file mode 100644 index 0000000000..fcca868885 --- /dev/null +++ b/src/or/dnsserv.h @@ -0,0 +1,26 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file dnsserv.h + * \brief Header file for dnsserv.c. + **/ + +#ifndef _TOR_DNSSERV_H +#define _TOR_DNSSERV_H + +void dnsserv_configure_listener(connection_t *conn); +void dnsserv_close_listener(connection_t *conn); +void dnsserv_resolved(edge_connection_t *conn, + int answer_type, + size_t answer_len, + const char *answer, + int ttl); +void dnsserv_reject_request(edge_connection_t *conn); +int dnsserv_launch_request(const char *name, int is_reverse); + +#endif + diff --git a/src/or/eventdns.c b/src/or/eventdns.c index 1d93ae048e..4753429b1f 100644 --- a/src/or/eventdns.c +++ b/src/or/eventdns.c @@ -31,6 +31,7 @@ */ #include "eventdns_tor.h" +#include "../common/util.h" #include <sys/types.h> /* #define NDEBUG */ @@ -89,6 +90,7 @@ #include <stdarg.h> #include "eventdns.h" + #ifdef WIN32 #include <windows.h> #include <winsock2.h> @@ -173,8 +175,6 @@ struct evdns_request { /* these objects are kept in a circular list */ struct evdns_request *next, *prev; - u16 timeout_event_deleted; /**< Debugging: where was timeout_event - * deleted? 0 for "it's added." */ struct event timeout_event; u16 trans_id; /* the transaction id */ @@ -214,8 +214,6 @@ struct nameserver { struct event event; /* these objects are kept in a circular list */ struct nameserver *next, *prev; - u16 timeout_event_deleted; /**< Debugging: where was timeout_event - * deleted? 0 for "it's added." */ struct event timeout_event; /* used to keep the timeout for */ /* when we next probe this server. */ /* Valid if state == 0 */ @@ -474,51 +472,10 @@ sockaddr_eq(const struct sockaddr *sa1, const struct sockaddr *sa2, return 1; } -/* for debugging bug 929. XXXX022 */ -static int -_add_timeout_event(u16 *lineno, struct event *ev, struct timeval *to) -{ - *lineno = 0; - return evtimer_add(ev, to); -} -#define add_timeout_event(s, to) \ - (_add_timeout_event(&(s)->timeout_event_deleted, &(s)->timeout_event, (to))) - -/* for debugging bug 929. XXXX022 */ -static int -_del_timeout_event(u16 *lineno, struct event *ev, int line) -{ - if (*lineno) { - log(EVDNS_LOG_DEBUG, - "Duplicate timeout event_del from line %d: first call " - "was at %d.", line, (int)*lineno); - return 0; - } else { - *lineno = (u16)line; - return event_del(ev); - } -} -#define del_timeout_event(s) \ - (_del_timeout_event(&(s)->timeout_event_deleted, &(s)->timeout_event, \ - __LINE__)) -/* For debugging bug 929/957. XXXX022 */ -static int -_del_timeout_event_if_set(u16 *lineno, struct event *ev, int line) -{ - if (*lineno == 0) { - log(EVDNS_LOG_DEBUG, - "Event that I thought was non-added as of line %d " - "was actually added on line %d", - line, (int)*lineno); - *lineno = line; - return event_del(ev); - } - return 0; -} -#define del_timeout_event_if_set(s) \ - _del_timeout_event_if_set(&(s)->timeout_event_deleted, \ - &(s)->timeout_event, \ - __LINE__) +#define add_timeout_event(s, to) \ + (event_add(&(s)->timeout_event, (to))) +#define del_timeout_event(s) \ + (event_del(&(s)->timeout_event)) /* This walks the list of inflight requests to find the */ /* one with a matching transaction id. Returns NULL on */ @@ -555,7 +512,7 @@ static void nameserver_probe_failed(struct nameserver *const ns) { const struct timeval * timeout; del_timeout_event(ns); - CLEAR(&ns->timeout_event); + if (ns->state == 1) { /* This can happen if the nameserver acts in a way which makes us mark */ /* it as bad and then starts sending good replies. */ @@ -567,8 +524,6 @@ nameserver_probe_failed(struct nameserver *const ns) { global_nameserver_timeouts_length - 1)]; ns->failed_times++; - del_timeout_event_if_set(ns); - evtimer_set(&ns->timeout_event, nameserver_prod_callback, ns); if (add_timeout_event(ns, (struct timeval *) timeout) < 0) { log(EVDNS_LOG_WARN, "Error from libevent when adding timer event for %s", @@ -597,8 +552,6 @@ nameserver_failed(struct nameserver *const ns, const char *msg) { ns->state = 0; ns->failed_times = 1; - del_timeout_event_if_set(ns); - evtimer_set(&ns->timeout_event, nameserver_prod_callback, ns); if (add_timeout_event(ns, (struct timeval *) &global_nameserver_timeouts[0]) < 0) { log(EVDNS_LOG_WARN, "Error from libevent when adding timer event for %s", @@ -634,7 +587,6 @@ nameserver_up(struct nameserver *const ns) { log(EVDNS_LOG_WARN, "Nameserver %s is back up", debug_ntop((struct sockaddr *)&ns->address)); del_timeout_event(ns); - CLEAR(&ns->timeout_event); ns->state = 1; ns->failed_times = 0; ns->timedout = 0; @@ -666,7 +618,6 @@ request_finished(struct evdns_request *const req, struct evdns_request **head) { log(EVDNS_LOG_DEBUG, "Removing timeout for request %lx", (unsigned long) req); del_timeout_event(req); - CLEAR(&req->timeout_event); search_request_finished(req); global_requests_inflight--; @@ -1077,6 +1028,9 @@ request_parse(u8 *packet, ssize_t length, struct evdns_server_port *port, struct GET16(answers); GET16(authority); GET16(additional); + (void)additional; + (void)authority; + (void)answers; if (flags & 0x8000) return -1; /* Must not be an answer. */ flags &= 0x0110; /* Only RD and CD get preserved. */ @@ -1294,7 +1248,8 @@ nameserver_read(struct nameserver *ns) { for (;;) { const int r = - (int)recvfrom(ns->socket, packet, (socklen_t)sizeof(packet), 0, + (int)recvfrom(ns->socket, (void*)packet, + (socklen_t)sizeof(packet), 0, sa, &addrlen); if (r < 0) { int err = last_error(ns->socket); @@ -1325,7 +1280,7 @@ server_port_read(struct evdns_server_port *s) { for (;;) { addrlen = (socklen_t)sizeof(struct sockaddr_storage); - r = recvfrom(s->socket, packet, sizeof(packet), 0, + r = recvfrom(s->socket, (void*)packet, sizeof(packet), 0, (struct sockaddr*) &addr, &addrlen); if (r < 0) { int err = last_error(s->socket); @@ -1342,8 +1297,8 @@ server_port_read(struct evdns_server_port *s) { static void server_port_flush(struct evdns_server_port *port) { - while (port->pending_replies) { - struct server_request *req = port->pending_replies; + struct server_request *req = port->pending_replies; + while (req) { ssize_t r = sendto(port->socket, req->response, req->response_len, 0, (struct sockaddr*) &req->addr, (socklen_t)req->addrlen); if (r < 0) { @@ -1355,6 +1310,9 @@ server_port_flush(struct evdns_server_port *port) if (server_request_free(req)) { /* we released the last reference to req->port. */ return; + } else { + assert(port->pending_replies != req); + req = port->pending_replies; } } @@ -1713,7 +1671,7 @@ evdns_server_request_add_reply(struct evdns_server_request *_req, int section, c /* exported function */ int -evdns_server_request_add_a_reply(struct evdns_server_request *req, const char *name, int n, void *addrs, int ttl) +evdns_server_request_add_a_reply(struct evdns_server_request *req, const char *name, int n, const void *addrs, int ttl) { return evdns_server_request_add_reply( req, EVDNS_ANSWER_SECTION, name, TYPE_A, CLASS_INET, @@ -1722,7 +1680,7 @@ evdns_server_request_add_a_reply(struct evdns_server_request *req, const char *n /* exported function */ int -evdns_server_request_add_aaaa_reply(struct evdns_server_request *req, const char *name, int n, void *addrs, int ttl) +evdns_server_request_add_aaaa_reply(struct evdns_server_request *req, const char *name, int n, const void *addrs, int ttl) { return evdns_server_request_add_reply( req, EVDNS_ANSWER_SECTION, name, TYPE_AAAA, CLASS_INET, @@ -2044,9 +2002,8 @@ evdns_request_timeout_callback(int fd, short events, void *arg) { /* retransmit it */ /* Stop waiting for the timeout. No need to do this in * request_finished; that one already deletes the timeout event. - * XXXX021 port this change to libevent. */ + * XXXX023 port this change to libevent. */ del_timeout_event(req); - CLEAR(&req->timeout_event); evdns_request_transmit(req); } } @@ -2059,7 +2016,8 @@ evdns_request_timeout_callback(int fd, short events, void *arg) { /* 2 other failure */ static int evdns_request_transmit_to(struct evdns_request *req, struct nameserver *server) { - const ssize_t r = send(server->socket, req->request, req->request_len, 0); + const ssize_t r = send(server->socket, (void*)req->request, + req->request_len, 0); if (r < 0) { int err = last_error(server->socket); if (error_is_eagain(err)) return 1; @@ -2109,8 +2067,7 @@ evdns_request_transmit(struct evdns_request *req) { /* transmitted; we need to check for timeout. */ log(EVDNS_LOG_DEBUG, "Setting timeout for request %lx", (unsigned long) req); - del_timeout_event_if_set(req); - evtimer_set(&req->timeout_event, evdns_request_timeout_callback, req); + if (add_timeout_event(req, &global_timeout) < 0) { log(EVDNS_LOG_WARN, "Error from libevent when adding timer for request %lx", @@ -2225,7 +2182,6 @@ evdns_clear_nameservers_and_suspend(void) (void) event_del(&server->event); CLEAR(&server->event); del_timeout_event(server); - CLEAR(&server->timeout_event); if (server->socket >= 0) CLOSE_SOCKET(server->socket); CLEAR(server); @@ -2243,7 +2199,6 @@ evdns_clear_nameservers_and_suspend(void) req->ns = NULL; /* ???? What to do about searches? */ del_timeout_event(req); - CLEAR(&req->timeout_event); req->trans_id = 0; req->transmit_me = 0; @@ -2292,6 +2247,21 @@ evdns_resume(void) } static int +sockaddr_is_loopback(const struct sockaddr *addr) +{ + static const char LOOPBACK_S6[16] = + "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\1"; + if (addr->sa_family == AF_INET) { + struct sockaddr_in *sin = (struct sockaddr_in *)addr; + return (ntohl(sin->sin_addr.s_addr) & 0xff000000) == 0x7f000000; + } else if (addr->sa_family == AF_INET6) { + struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)addr; + return fast_memeq(sin6->sin6_addr.s6_addr, LOOPBACK_S6, 16); + } + return 0; +} + +static int _evdns_nameserver_add_impl(const struct sockaddr *address, socklen_t addrlen) { /* first check to see if we already have this nameserver */ @@ -2318,7 +2288,8 @@ _evdns_nameserver_add_impl(const struct sockaddr *address, if (!ns) return -1; memset(ns, 0, sizeof(struct nameserver)); - ns->timeout_event_deleted = __LINE__; + + evtimer_set(&ns->timeout_event, nameserver_prod_callback, ns); ns->socket = socket(address->sa_family, SOCK_DGRAM, 0); if (ns->socket < 0) { err = 1; goto out1; } @@ -2331,7 +2302,8 @@ _evdns_nameserver_add_impl(const struct sockaddr *address, fcntl(ns->socket, F_SETFL, O_NONBLOCK); #endif - if (global_bind_addr_is_set) { + if (global_bind_addr_is_set && + !sockaddr_is_loopback((struct sockaddr*)&global_bind_address)) { if (bind(ns->socket, (struct sockaddr *)&global_bind_address, global_bind_addrlen) < 0) { log(EVDNS_LOG_DEBUG, "Couldn't bind to outgoing address."); @@ -2553,7 +2525,8 @@ request_new(int type, const char *name, int flags, } memset(req, 0, sizeof(struct evdns_request)); - req->timeout_event_deleted = __LINE__; + + evtimer_set(&req->timeout_event, evdns_request_timeout_callback, req); if (global_randomize_case) { unsigned i; @@ -2941,14 +2914,6 @@ evdns_resolv_set_defaults(int flags) { if (flags & DNS_OPTION_NAMESERVERS) evdns_nameserver_ip_add("127.0.0.1"); } -#ifndef HAVE_STRTOK_R -static char * -strtok_r(char *s, const char *delim, char **state) { - (void)state; - return strtok(s, delim); -} -#endif - /* helper version of atoi which returns -1 on error */ static int strtoint(const char *const str) { @@ -3025,9 +2990,9 @@ static void resolv_conf_parse_line(char *const start, int flags) { char *strtok_state; static const char *const delims = " \t"; -#define NEXT_TOKEN strtok_r(NULL, delims, &strtok_state) +#define NEXT_TOKEN tor_strtok_r(NULL, delims, &strtok_state) - char *const first_token = strtok_r(start, delims, &strtok_state); + char *const first_token = tor_strtok_r(start, delims, &strtok_state); if (!first_token) return; if (!strcmp(first_token, "nameserver") && (flags & DNS_OPTION_NAMESERVERS)) { @@ -3171,14 +3136,13 @@ load_nameservers_with_getnetworkparams(void) IP_ADDR_STRING *ns; GetNetworkParams_fn_t fn; - /* XXXX Possibly, we should hardcode the location of this DLL. */ - if (!(handle = LoadLibrary("iphlpapi.dll"))) { + if (!(handle = load_windows_system_library(TEXT("iphlpapi.dll")))) { log(EVDNS_LOG_WARN, "Could not open iphlpapi.dll"); /* right now status = 0, doesn't that mean "good" - mikec */ status = -1; goto done; } - if (!(fn = (GetNetworkParams_fn_t) GetProcAddress(handle, "GetNetworkParams"))) { + if (!(fn = (GetNetworkParams_fn_t) GetProcAddress(handle, TEXT("GetNetworkParams")))) { log(EVDNS_LOG_WARN, "Could not get address of function."); /* same as above */ status = -1; @@ -3241,9 +3205,10 @@ load_nameservers_with_getnetworkparams(void) } static int -config_nameserver_from_reg_key(HKEY key, const char *subkey) +config_nameserver_from_reg_key(HKEY key, const TCHAR *subkey) { char *buf; + char ansibuf[MAX_PATH] = {0}; DWORD bufsz = 0, type = 0; int status = 0; @@ -3255,24 +3220,30 @@ config_nameserver_from_reg_key(HKEY key, const char *subkey) if (RegQueryValueEx(key, subkey, 0, &type, (LPBYTE)buf, &bufsz) == ERROR_SUCCESS && bufsz > 1) { - status = evdns_nameserver_ip_add_line(buf); + wcstombs(ansibuf,(wchar_t*)buf,MAX_PATH);/*XXXX UNICODE */ + status = evdns_nameserver_ip_add_line(ansibuf); } mm_free(buf); return status; } -#define SERVICES_KEY "System\\CurrentControlSet\\Services\\" -#define WIN_NS_9X_KEY SERVICES_KEY "VxD\\MSTCP" -#define WIN_NS_NT_KEY SERVICES_KEY "Tcpip\\Parameters" +#define SERVICES_KEY TEXT("System\\CurrentControlSet\\Services\\") +#define WIN_NS_9X_KEY SERVICES_KEY TEXT("VxD\\MSTCP") +#define WIN_NS_NT_KEY SERVICES_KEY TEXT("Tcpip\\Parameters") static int load_nameservers_from_registry(void) { int found = 0; int r; + OSVERSIONINFO info; + memset(&info, 0, sizeof(info)); + info.dwOSVersionInfoSize = sizeof (info); + GetVersionEx(&info); + #define TRY(k, name) \ - if (!found && config_nameserver_from_reg_key(k,name) == 0) { \ + if (!found && config_nameserver_from_reg_key(k,TEXT(name)) == 0) { \ log(EVDNS_LOG_DEBUG,"Found nameservers in %s/%s",#k,name); \ found = 1; \ } else if (!found) { \ @@ -3280,7 +3251,7 @@ load_nameservers_from_registry(void) #k,#name); \ } - if (((int)GetVersion()) > 0) { /* NT */ + if (info.dwMajorVersion >= 5) { /* NT */ HKEY nt_key = 0, interfaces_key = 0; if (RegOpenKeyEx(HKEY_LOCAL_MACHINE, WIN_NS_NT_KEY, 0, @@ -3288,7 +3259,7 @@ load_nameservers_from_registry(void) log(EVDNS_LOG_DEBUG,"Couldn't open nt key, %d",(int)GetLastError()); return -1; } - r = RegOpenKeyEx(nt_key, "Interfaces", 0, + r = RegOpenKeyEx(nt_key, TEXT("Interfaces"), 0, KEY_QUERY_VALUE|KEY_ENUMERATE_SUB_KEYS, &interfaces_key); if (r != ERROR_SUCCESS) { @@ -3384,8 +3355,7 @@ evdns_shutdown(int fail_requests) if (server->socket >= 0) CLOSE_SOCKET(server->socket); (void) event_del(&server->event); - if (server->state == 0) - del_timeout_event(server); + del_timeout_event(server); CLEAR(server); mm_free(server); if (server_next == server_head) diff --git a/src/or/eventdns.h b/src/or/eventdns.h index bf3b64d08a..2fe4ac9371 100644 --- a/src/or/eventdns.h +++ b/src/or/eventdns.h @@ -323,8 +323,8 @@ struct evdns_server_port *evdns_add_server_port(int socket, int is_tcp, evdns_re void evdns_close_server_port(struct evdns_server_port *port); int evdns_server_request_add_reply(struct evdns_server_request *req, int section, const char *name, int type, int class, int ttl, int datalen, int is_name, const char *data); -int evdns_server_request_add_a_reply(struct evdns_server_request *req, const char *name, int n, void *addrs, int ttl); -int evdns_server_request_add_aaaa_reply(struct evdns_server_request *req, const char *name, int n, void *addrs, int ttl); +int evdns_server_request_add_a_reply(struct evdns_server_request *req, const char *name, int n, const void *addrs, int ttl); +int evdns_server_request_add_aaaa_reply(struct evdns_server_request *req, const char *name, int n, const void *addrs, int ttl); int evdns_server_request_add_ptr_reply(struct evdns_server_request *req, struct in_addr *in, const char *inaddr_name, const char *hostname, int ttl); int evdns_server_request_add_cname_reply(struct evdns_server_request *req, const char *name, const char *cname, int ttl); diff --git a/src/or/geoip.c b/src/or/geoip.c index 360b65f41c..5bb2410a75 100644 --- a/src/or/geoip.c +++ b/src/or/geoip.c @@ -3,15 +3,23 @@ /** * \file geoip.c - * \brief Functions related to maintaining an IP-to-country database and to - * summarizing client connections by country. + * \brief Functions related to maintaining an IP-to-country database; + * to summarizing client connections by country to entry guards, bridges, + * and directory servers; and for statistics on answering network status + * requests. */ #define GEOIP_PRIVATE #include "or.h" #include "ht.h" +#include "config.h" +#include "control.h" +#include "dnsserv.h" +#include "geoip.h" +#include "routerlist.h" static void clear_geoip_db(void); +static void init_geoip_countries(void); /** An entry from the GeoIP file: maps an IP range to a country. */ typedef struct geoip_entry_t { @@ -20,16 +28,11 @@ typedef struct geoip_entry_t { intptr_t country; /**< An index into geoip_countries */ } geoip_entry_t; -/** For how many periods should we remember per-country request history? */ -#define REQUEST_HIST_LEN 3 -/** How long are the periods for which we should remember request history? */ -#define REQUEST_HIST_PERIOD (8*60*60) - /** A per-country record for GeoIP request history. */ typedef struct geoip_country_t { char countrycode[3]; - uint32_t n_v2_ns_requests[REQUEST_HIST_LEN]; - uint32_t n_v3_ns_requests[REQUEST_HIST_LEN]; + uint32_t n_v2_ns_requests; + uint32_t n_v3_ns_requests; } geoip_country_t; /** A list of geoip_country_t */ @@ -42,7 +45,7 @@ static strmap_t *country_idxplus1_by_lc_code = NULL; static smartlist_t *geoip_entries = NULL; /** Return the index of the <b>country</b>'s entry in the GeoIP DB - * if it is a valid 2-letter country code, otherwise return zero. + * if it is a valid 2-letter country code, otherwise return -1. */ country_t geoip_get_country(const char *country) @@ -101,11 +104,11 @@ geoip_parse_entry(const char *line) { unsigned int low, high; char b[3]; - if (!geoip_countries) { - geoip_countries = smartlist_create(); + if (!geoip_countries) + init_geoip_countries(); + if (!geoip_entries) geoip_entries = smartlist_create(); - country_idxplus1_by_lc_code = strmap_new(); - } + while (TOR_ISSPACE(*line)) ++line; if (*line == '#') @@ -142,6 +145,7 @@ _geoip_compare_entries(const void **_a, const void **_b) static int _geoip_compare_key_to_entry(const void *_key, const void **_member) { + /* No alignment issue here, since _key really is a pointer to uint32_t */ const uint32_t addr = *(uint32_t *)_key; const geoip_entry_t *entry = *_member; if (addr < entry->ip_low) @@ -160,6 +164,24 @@ should_record_bridge_info(or_options_t *options) return options->BridgeRelay && options->BridgeRecordUsageByCountry; } +/** Set up a new list of geoip countries with no countries (yet) set in it, + * except for the unknown country. + */ +static void +init_geoip_countries(void) +{ + geoip_country_t *geoip_unresolved; + geoip_countries = smartlist_create(); + /* Add a geoip_country_t for requests that could not be resolved to a + * country as first element (index 0) to geoip_countries. */ + geoip_unresolved = tor_malloc_zero(sizeof(geoip_country_t)); + strlcpy(geoip_unresolved->countrycode, "??", + sizeof(geoip_unresolved->countrycode)); + smartlist_add(geoip_countries, geoip_unresolved); + country_idxplus1_by_lc_code = strmap_new(); + strmap_set_lc(country_idxplus1_by_lc_code, "??", (void*)(1)); +} + /** Clear the GeoIP database and reload it from the file * <b>filename</b>. Return 0 on success, -1 on failure. * @@ -185,16 +207,14 @@ geoip_load_file(const char *filename, or_options_t *options) filename, msg); return -1; } - if (!geoip_countries) { - geoip_countries = smartlist_create(); - country_idxplus1_by_lc_code = strmap_new(); - } + if (!geoip_countries) + init_geoip_countries(); if (geoip_entries) { SMARTLIST_FOREACH(geoip_entries, geoip_entry_t *, e, tor_free(e)); smartlist_free(geoip_entries); } geoip_entries = smartlist_create(); - log_notice(LD_GENERAL, "Parsing GEOIP file."); + log_notice(LD_GENERAL, "Parsing GEOIP file %s.", filename); while (!feof(f)) { char buf[512]; if (fgets(buf, (int)sizeof(buf), f) == NULL) @@ -215,9 +235,10 @@ geoip_load_file(const char *filename, or_options_t *options) } /** Given an IP address in host order, return a number representing the - * country to which that address belongs, or -1 for unknown. The return value - * will always be less than geoip_get_n_countries(). To decode it, - * call geoip_get_country_name(). + * country to which that address belongs, -1 for "No geoip information + * available", or 0 for the 'unknown country'. The return value will always + * be less than geoip_get_n_countries(). To decode it, call + * geoip_get_country_name(). */ int geoip_get_country_by_ip(uint32_t ipaddr) @@ -226,13 +247,15 @@ geoip_get_country_by_ip(uint32_t ipaddr) if (!geoip_entries) return -1; ent = smartlist_bsearch(geoip_entries, &ipaddr, _geoip_compare_key_to_entry); - return ent ? (int)ent->country : -1; + return ent ? (int)ent->country : 0; } /** Return the number of countries recognized by the GeoIP database. */ int geoip_get_n_countries(void) { + if (!geoip_countries) + init_geoip_countries(); return (int) smartlist_len(geoip_countries); } @@ -261,23 +284,22 @@ geoip_is_loaded(void) typedef struct clientmap_entry_t { HT_ENTRY(clientmap_entry_t) node; uint32_t ipaddr; - time_t last_seen; /* The last 2 bits of this value hold the client - * operation. */ + /** Time when we last saw this IP address, in MINUTES since the epoch. + * + * (This will run out of space around 4011 CE. If Tor is still in use around + * 4000 CE, please remember to add more bits to last_seen_in_minutes.) */ + unsigned int last_seen_in_minutes:30; + unsigned int action:2; } clientmap_entry_t; -#define ACTION_MASK 3 +/** Largest allowable value for last_seen_in_minutes. (It's a 30-bit field, + * so it can hold up to (1u<<30)-1, or 0x3fffffffu. + */ +#define MAX_LAST_SEEN_IN_MINUTES 0X3FFFFFFFu /** Map from client IP address to last time seen. */ static HT_HEAD(clientmap, clientmap_entry_t) client_history = HT_INITIALIZER(); -/** Time at which we started tracking client IP history. */ -static time_t client_history_starts = 0; - -/** When did the current period of checking per-country request history - * start? */ -static time_t current_request_period_starts = 0; -/** How many older request periods are we remembering? */ -static int n_old_request_periods = 0; /** Hashtable helper: compute a hash of a clientmap_entry_t. */ static INLINE unsigned @@ -289,7 +311,7 @@ clientmap_entry_hash(const clientmap_entry_t *a) static INLINE int clientmap_entries_eq(const clientmap_entry_t *a, const clientmap_entry_t *b) { - return a->ipaddr == b->ipaddr; + return a->ipaddr == b->ipaddr && a->action == b->action; } HT_PROTOTYPE(clientmap, clientmap_entry_t, node, clientmap_entry_hash, @@ -297,8 +319,87 @@ HT_PROTOTYPE(clientmap, clientmap_entry_t, node, clientmap_entry_hash, HT_GENERATE(clientmap, clientmap_entry_t, node, clientmap_entry_hash, clientmap_entries_eq, 0.6, malloc, realloc, free); +/** Clear history of connecting clients used by entry and bridge stats. */ +static void +client_history_clear(void) +{ + clientmap_entry_t **ent, **next, *this; + for (ent = HT_START(clientmap, &client_history); ent != NULL; + ent = next) { + if ((*ent)->action == GEOIP_CLIENT_CONNECT) { + this = *ent; + next = HT_NEXT_RMV(clientmap, &client_history, ent); + tor_free(this); + } else { + next = HT_NEXT(clientmap, &client_history, ent); + } + } +} + +/** How often do we update our estimate which share of v2 and v3 directory + * requests is sent to us? We could as well trigger updates of shares from + * network status updates, but that means adding a lot of calls into code + * that is independent from geoip stats (and keeping them up-to-date). We + * are perfectly fine with an approximation of 15-minute granularity. */ +#define REQUEST_SHARE_INTERVAL (15 * 60) + +/** When did we last determine which share of v2 and v3 directory requests + * is sent to us? */ +static time_t last_time_determined_shares = 0; + +/** Sum of products of v2 shares times the number of seconds for which we + * consider these shares as valid. */ +static double v2_share_times_seconds; + +/** Sum of products of v3 shares times the number of seconds for which we + * consider these shares as valid. */ +static double v3_share_times_seconds; + +/** Number of seconds we are determining v2 and v3 shares. */ +static int share_seconds; + +/** Try to determine which fraction of v2 and v3 directory requests aimed at + * caches will be sent to us at time <b>now</b> and store that value in + * order to take a mean value later on. */ +static void +geoip_determine_shares(time_t now) +{ + double v2_share = 0.0, v3_share = 0.0; + if (router_get_my_share_of_directory_requests(&v2_share, &v3_share) < 0) + return; + if (last_time_determined_shares) { + v2_share_times_seconds += v2_share * + ((double) (now - last_time_determined_shares)); + v3_share_times_seconds += v3_share * + ((double) (now - last_time_determined_shares)); + share_seconds += (int)(now - last_time_determined_shares); + } + last_time_determined_shares = now; +} + +/** Calculate which fraction of v2 and v3 directory requests aimed at caches + * have been sent to us since the last call of this function up to time + * <b>now</b>. Set *<b>v2_share_out</b> and *<b>v3_share_out</b> to the + * fractions of v2 and v3 protocol shares we expect to have seen. Reset + * counters afterwards. Return 0 on success, -1 on failure (e.g. when zero + * seconds have passed since the last call).*/ +static int +geoip_get_mean_shares(time_t now, double *v2_share_out, + double *v3_share_out) +{ + geoip_determine_shares(now); + if (!share_seconds) + return -1; + *v2_share_out = v2_share_times_seconds / ((double) share_seconds); + *v3_share_out = v3_share_times_seconds / ((double) share_seconds); + v2_share_times_seconds = v3_share_times_seconds = 0.0; + share_seconds = 0; + return 0; +} + /** Note that we've seen a client connect from the IP <b>addr</b> (host order) - * at time <b>now</b>. Ignored by all but bridges. */ + * at time <b>now</b>. Ignored by all but bridges and directories if + * configured accordingly. */ void geoip_note_client_seen(geoip_client_action_t action, uint32_t addr, time_t now) @@ -306,72 +407,46 @@ geoip_note_client_seen(geoip_client_action_t action, or_options_t *options = get_options(); clientmap_entry_t lookup, *ent; if (action == GEOIP_CLIENT_CONNECT) { - if (!(options->BridgeRelay && options->BridgeRecordUsageByCountry)) - return; - /* Did we recently switch from bridge to relay or back? */ - if (client_history_starts > now) + /* Only remember statistics as entry guard or as bridge. */ + if (!options->EntryStatistics && + (!(options->BridgeRelay && options->BridgeRecordUsageByCountry))) return; } else { -#ifndef ENABLE_GEOIP_STATS - return; -#else if (options->BridgeRelay || options->BridgeAuthoritativeDir || - !options->DirRecordUsageByCountry) + !options->DirReqStatistics) return; -#endif } - /* Rotate the current request period. */ - while (current_request_period_starts + REQUEST_HIST_PERIOD < now) { - if (!geoip_countries) - geoip_countries = smartlist_create(); - if (!current_request_period_starts) { - current_request_period_starts = now; - break; - } - SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, { - memmove(&c->n_v2_ns_requests[0], &c->n_v2_ns_requests[1], - sizeof(uint32_t)*(REQUEST_HIST_LEN-1)); - memmove(&c->n_v3_ns_requests[0], &c->n_v3_ns_requests[1], - sizeof(uint32_t)*(REQUEST_HIST_LEN-1)); - c->n_v2_ns_requests[REQUEST_HIST_LEN-1] = 0; - c->n_v3_ns_requests[REQUEST_HIST_LEN-1] = 0; - }); - current_request_period_starts += REQUEST_HIST_PERIOD; - if (n_old_request_periods < REQUEST_HIST_LEN-1) - ++n_old_request_periods; - } - - /* We use the low 3 bits of the time to encode the action. Since we're - * potentially remembering tons of clients, we don't want to make - * clientmap_entry_t larger than it has to be. */ - now = (now & ~ACTION_MASK) | (((int)action) & ACTION_MASK); lookup.ipaddr = addr; + lookup.action = (int)action; ent = HT_FIND(clientmap, &client_history, &lookup); - if (ent) { - ent->last_seen = now; - } else { + if (! ent) { ent = tor_malloc_zero(sizeof(clientmap_entry_t)); ent->ipaddr = addr; - ent->last_seen = now; + ent->action = (int)action; HT_INSERT(clientmap, &client_history, ent); } + if (now / 60 <= (int)MAX_LAST_SEEN_IN_MINUTES && now >= 0) + ent->last_seen_in_minutes = (unsigned)(now/60); + else + ent->last_seen_in_minutes = 0; if (action == GEOIP_CLIENT_NETWORKSTATUS || action == GEOIP_CLIENT_NETWORKSTATUS_V2) { int country_idx = geoip_get_country_by_ip(addr); + if (country_idx < 0) + country_idx = 0; /** unresolved requests are stored at index 0. */ if (country_idx >= 0 && country_idx < smartlist_len(geoip_countries)) { geoip_country_t *country = smartlist_get(geoip_countries, country_idx); if (action == GEOIP_CLIENT_NETWORKSTATUS) - ++country->n_v3_ns_requests[REQUEST_HIST_LEN-1]; + ++country->n_v3_ns_requests; else - ++country->n_v2_ns_requests[REQUEST_HIST_LEN-1]; + ++country->n_v2_ns_requests; } - } - if (!client_history_starts) { - client_history_starts = now; - current_request_period_starts = now; + /* Periodically determine share of requests that we should see */ + if (last_time_determined_shares + REQUEST_SHARE_INTERVAL < now) + geoip_determine_shares(now); } } @@ -380,8 +455,8 @@ geoip_note_client_seen(geoip_client_action_t action, static int _remove_old_client_helper(struct clientmap_entry_t *ent, void *_cutoff) { - time_t cutoff = *(time_t*)_cutoff; - if (ent->last_seen < cutoff) { + time_t cutoff = *(time_t*)_cutoff / 60; + if (ent->last_seen_in_minutes < cutoff) { tor_free(ent); return 1; } else { @@ -389,18 +464,45 @@ _remove_old_client_helper(struct clientmap_entry_t *ent, void *_cutoff) } } -/** Forget about all clients that haven't connected since <b>cutoff</b>. - * If <b>cutoff</b> is in the future, clients won't be added to the history - * until this time is reached. This is useful to prevent relays that switch - * to bridges from reporting unbelievable numbers of clients. */ +/** Forget about all clients that haven't connected since <b>cutoff</b>. */ void geoip_remove_old_clients(time_t cutoff) { clientmap_HT_FOREACH_FN(&client_history, _remove_old_client_helper, &cutoff); - if (client_history_starts < cutoff) - client_history_starts = cutoff; +} + +/** How many responses are we giving to clients requesting v2 network + * statuses? */ +static uint32_t ns_v2_responses[GEOIP_NS_RESPONSE_NUM]; + +/** How many responses are we giving to clients requesting v3 network + * statuses? */ +static uint32_t ns_v3_responses[GEOIP_NS_RESPONSE_NUM]; + +/** Note that we've rejected a client's request for a v2 or v3 network + * status, encoded in <b>action</b> for reason <b>reason</b> at time + * <b>now</b>. */ +void +geoip_note_ns_response(geoip_client_action_t action, + geoip_ns_response_t response) +{ + static int arrays_initialized = 0; + if (!get_options()->DirReqStatistics) + return; + if (!arrays_initialized) { + memset(ns_v2_responses, 0, sizeof(ns_v2_responses)); + memset(ns_v3_responses, 0, sizeof(ns_v3_responses)); + arrays_initialized = 1; + } + tor_assert(action == GEOIP_CLIENT_NETWORKSTATUS || + action == GEOIP_CLIENT_NETWORKSTATUS_V2); + tor_assert(response < GEOIP_NS_RESPONSE_NUM); + if (action == GEOIP_CLIENT_NETWORKSTATUS) + ns_v3_responses[response]++; + else + ns_v2_responses[response]++; } /** Do not mention any country from which fewer than this number of IPs have @@ -414,13 +516,6 @@ geoip_remove_old_clients(time_t cutoff) * multiple of this value. */ #define IP_GRANULARITY 8 -/** Return the time at which we started recording geoip data. */ -time_t -geoip_get_history_start(void) -{ - return client_history_starts; -} - /** Helper type: used to sort per-country totals by value. */ typedef struct c_hist_t { char country[3]; /**< Two-letter country code. */ @@ -442,97 +537,320 @@ _c_hist_compare(const void **_a, const void **_b) return strcmp(a->country, b->country); } -/** How long do we have to have observed per-country request history before we - * are willing to talk about it? */ -#define GEOIP_MIN_OBSERVATION_TIME (12*60*60) +/** When there are incomplete directory requests at the end of a 24-hour + * period, consider those requests running for longer than this timeout as + * failed, the others as still running. */ +#define DIRREQ_TIMEOUT (10*60) -/** Return the lowest x such that x is at least <b>number</b>, and x modulo - * <b>divisor</b> == 0. */ -static INLINE unsigned -round_to_next_multiple_of(unsigned number, unsigned divisor) +/** Entry in a map from either conn->global_identifier for direct requests + * or a unique circuit identifier for tunneled requests to request time, + * response size, and completion time of a network status request. Used to + * measure download times of requests to derive average client + * bandwidths. */ +typedef struct dirreq_map_entry_t { + HT_ENTRY(dirreq_map_entry_t) node; + /** Unique identifier for this network status request; this is either the + * conn->global_identifier of the dir conn (direct request) or a new + * locally unique identifier of a circuit (tunneled request). This ID is + * only unique among other direct or tunneled requests, respectively. */ + uint64_t dirreq_id; + unsigned int state:3; /**< State of this directory request. */ + unsigned int type:1; /**< Is this a direct or a tunneled request? */ + unsigned int completed:1; /**< Is this request complete? */ + unsigned int action:2; /**< Is this a v2 or v3 request? */ + /** When did we receive the request and started sending the response? */ + struct timeval request_time; + size_t response_size; /**< What is the size of the response in bytes? */ + struct timeval completion_time; /**< When did the request succeed? */ +} dirreq_map_entry_t; + +/** Map of all directory requests asking for v2 or v3 network statuses in + * the current geoip-stats interval. Values are + * of type *<b>dirreq_map_entry_t</b>. */ +static HT_HEAD(dirreqmap, dirreq_map_entry_t) dirreq_map = + HT_INITIALIZER(); + +static int +dirreq_map_ent_eq(const dirreq_map_entry_t *a, + const dirreq_map_entry_t *b) { - number += divisor - 1; - number -= number % divisor; - return number; + return a->dirreq_id == b->dirreq_id && a->type == b->type; } -/** Return a newly allocated comma-separated string containing entries for all - * the countries from which we've seen enough clients connect. The entry - * format is cc=num where num is the number of IPs we've seen connecting from - * that country, and cc is a lowercased country code. Returns NULL if we don't - * want to export geoip data yet. */ -char * -geoip_get_client_history(time_t now, geoip_client_action_t action) +static unsigned +dirreq_map_ent_hash(const dirreq_map_entry_t *entry) +{ + unsigned u = (unsigned) entry->dirreq_id; + u += entry->type << 20; + return u; +} + +HT_PROTOTYPE(dirreqmap, dirreq_map_entry_t, node, dirreq_map_ent_hash, + dirreq_map_ent_eq); +HT_GENERATE(dirreqmap, dirreq_map_entry_t, node, dirreq_map_ent_hash, + dirreq_map_ent_eq, 0.6, malloc, realloc, free); + +/** Helper: Put <b>entry</b> into map of directory requests using + * <b>type</b> and <b>dirreq_id</b> as key parts. If there is + * already an entry for that key, print out a BUG warning and return. */ +static void +_dirreq_map_put(dirreq_map_entry_t *entry, dirreq_type_t type, + uint64_t dirreq_id) +{ + dirreq_map_entry_t *old_ent; + tor_assert(entry->type == type); + tor_assert(entry->dirreq_id == dirreq_id); + + /* XXXX we could switch this to HT_INSERT some time, since it seems that + * this bug doesn't happen. But since this function doesn't seem to be + * critical-path, it's sane to leave it alone. */ + old_ent = HT_REPLACE(dirreqmap, &dirreq_map, entry); + if (old_ent && old_ent != entry) { + log_warn(LD_BUG, "Error when putting directory request into local " + "map. There was already an entry for the same identifier."); + return; + } +} + +/** Helper: Look up and return an entry in the map of directory requests + * using <b>type</b> and <b>dirreq_id</b> as key parts. If there + * is no such entry, return NULL. */ +static dirreq_map_entry_t * +_dirreq_map_get(dirreq_type_t type, uint64_t dirreq_id) +{ + dirreq_map_entry_t lookup; + lookup.type = type; + lookup.dirreq_id = dirreq_id; + return HT_FIND(dirreqmap, &dirreq_map, &lookup); +} + +/** Note that an either direct or tunneled (see <b>type</b>) directory + * request for a network status with unique ID <b>dirreq_id</b> of size + * <b>response_size</b> and action <b>action</b> (either v2 or v3) has + * started. */ +void +geoip_start_dirreq(uint64_t dirreq_id, size_t response_size, + geoip_client_action_t action, dirreq_type_t type) +{ + dirreq_map_entry_t *ent; + if (!get_options()->DirReqStatistics) + return; + ent = tor_malloc_zero(sizeof(dirreq_map_entry_t)); + ent->dirreq_id = dirreq_id; + tor_gettimeofday(&ent->request_time); + ent->response_size = response_size; + ent->action = action; + ent->type = type; + _dirreq_map_put(ent, type, dirreq_id); +} + +/** Change the state of the either direct or tunneled (see <b>type</b>) + * directory request with <b>dirreq_id</b> to <b>new_state</b> and + * possibly mark it as completed. If no entry can be found for the given + * key parts (e.g., if this is a directory request that we are not + * measuring, or one that was started in the previous measurement period), + * or if the state cannot be advanced to <b>new_state</b>, do nothing. */ +void +geoip_change_dirreq_state(uint64_t dirreq_id, dirreq_type_t type, + dirreq_state_t new_state) +{ + dirreq_map_entry_t *ent; + if (!get_options()->DirReqStatistics) + return; + ent = _dirreq_map_get(type, dirreq_id); + if (!ent) + return; + if (new_state == DIRREQ_IS_FOR_NETWORK_STATUS) + return; + if (new_state - 1 != ent->state) + return; + ent->state = new_state; + if ((type == DIRREQ_DIRECT && + new_state == DIRREQ_FLUSHING_DIR_CONN_FINISHED) || + (type == DIRREQ_TUNNELED && + new_state == DIRREQ_OR_CONN_BUFFER_FLUSHED)) { + tor_gettimeofday(&ent->completion_time); + ent->completed = 1; + } +} + +/** Return a newly allocated comma-separated string containing statistics + * on network status downloads. The string contains the number of completed + * requests, timeouts, and still running requests as well as the download + * times by deciles and quartiles. Return NULL if we have not observed + * requests for long enough. */ +static char * +geoip_get_dirreq_history(geoip_client_action_t action, + dirreq_type_t type) { char *result = NULL; - if (!geoip_is_loaded()) + smartlist_t *dirreq_completed = NULL; + uint32_t complete = 0, timeouts = 0, running = 0; + int bufsize = 1024, written; + dirreq_map_entry_t **ptr, **next, *ent; + struct timeval now; + + tor_gettimeofday(&now); + if (action != GEOIP_CLIENT_NETWORKSTATUS && + action != GEOIP_CLIENT_NETWORKSTATUS_V2) return NULL; - if (client_history_starts < (now - GEOIP_MIN_OBSERVATION_TIME)) { - char buf[32]; - smartlist_t *chunks = NULL; - smartlist_t *entries = NULL; - int n_countries = geoip_get_n_countries(); - int i; - clientmap_entry_t **ent; - unsigned *counts = tor_malloc_zero(sizeof(unsigned)*n_countries); - unsigned total = 0; - unsigned granularity = IP_GRANULARITY; -#ifdef ENABLE_GEOIP_STATS - if (get_options()->DirRecordUsageByCountry) - granularity = get_options()->DirRecordUsageGranularity; -#endif - HT_FOREACH(ent, clientmap, &client_history) { - int country; - if (((*ent)->last_seen & ACTION_MASK) != (int)action) - continue; - country = geoip_get_country_by_ip((*ent)->ipaddr); - if (country < 0) - continue; - tor_assert(0 <= country && country < n_countries); - ++counts[country]; - ++total; - } - /* Don't record anything if we haven't seen enough IPs. */ - if (total < MIN_IPS_TO_NOTE_ANYTHING) - goto done; - /* Make a list of c_hist_t */ - entries = smartlist_create(); - for (i = 0; i < n_countries; ++i) { - unsigned c = counts[i]; - const char *countrycode; - c_hist_t *ent; - /* Only report a country if it has a minimum number of IPs. */ - if (c >= MIN_IPS_TO_NOTE_COUNTRY) { - c = round_to_next_multiple_of(c, granularity); - countrycode = geoip_get_country_name(i); - ent = tor_malloc(sizeof(c_hist_t)); - strlcpy(ent->country, countrycode, sizeof(ent->country)); - ent->total = c; - smartlist_add(entries, ent); + dirreq_completed = smartlist_create(); + for (ptr = HT_START(dirreqmap, &dirreq_map); ptr; ptr = next) { + ent = *ptr; + if (ent->action != action || ent->type != type) { + next = HT_NEXT(dirreqmap, &dirreq_map, ptr); + continue; + } else { + if (ent->completed) { + smartlist_add(dirreq_completed, ent); + complete++; + next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ptr); + } else { + if (tv_mdiff(&ent->request_time, &now) / 1000 > DIRREQ_TIMEOUT) + timeouts++; + else + running++; + next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ptr); + tor_free(ent); } } - /* Sort entries. Note that we must do this _AFTER_ rounding, or else - * the sort order could leak info. */ - smartlist_sort(entries, _c_hist_compare); - - /* Build the result. */ - chunks = smartlist_create(); - SMARTLIST_FOREACH(entries, c_hist_t *, ch, { - tor_snprintf(buf, sizeof(buf), "%s=%u", ch->country, ch->total); - smartlist_add(chunks, tor_strdup(buf)); - }); - result = smartlist_join_strings(chunks, ",", 0, NULL); - done: - tor_free(counts); - if (chunks) { - SMARTLIST_FOREACH(chunks, char *, c, tor_free(c)); - smartlist_free(chunks); - } - if (entries) { - SMARTLIST_FOREACH(entries, c_hist_t *, c, tor_free(c)); - smartlist_free(entries); + } +#define DIR_REQ_GRANULARITY 4 + complete = round_uint32_to_next_multiple_of(complete, + DIR_REQ_GRANULARITY); + timeouts = round_uint32_to_next_multiple_of(timeouts, + DIR_REQ_GRANULARITY); + running = round_uint32_to_next_multiple_of(running, + DIR_REQ_GRANULARITY); + result = tor_malloc_zero(bufsize); + written = tor_snprintf(result, bufsize, "complete=%u,timeout=%u," + "running=%u", complete, timeouts, running); + if (written < 0) { + tor_free(result); + goto done; + } + +#define MIN_DIR_REQ_RESPONSES 16 + if (complete >= MIN_DIR_REQ_RESPONSES) { + uint32_t *dltimes; + /* We may have rounded 'completed' up. Here we want to use the + * real value. */ + complete = smartlist_len(dirreq_completed); + dltimes = tor_malloc_zero(sizeof(uint32_t) * complete); + SMARTLIST_FOREACH_BEGIN(dirreq_completed, dirreq_map_entry_t *, ent) { + uint32_t bytes_per_second; + uint32_t time_diff = (uint32_t) tv_mdiff(&ent->request_time, + &ent->completion_time); + if (time_diff == 0) + time_diff = 1; /* Avoid DIV/0; "instant" answers are impossible + * by law of nature or something, but a milisecond + * is a bit greater than "instantly" */ + bytes_per_second = (uint32_t)(1000 * ent->response_size / time_diff); + dltimes[ent_sl_idx] = bytes_per_second; + } SMARTLIST_FOREACH_END(ent); + median_uint32(dltimes, complete); /* sorts as a side effect. */ + written = tor_snprintf(result + written, bufsize - written, + ",min=%u,d1=%u,d2=%u,q1=%u,d3=%u,d4=%u,md=%u," + "d6=%u,d7=%u,q3=%u,d8=%u,d9=%u,max=%u", + dltimes[0], + dltimes[1*complete/10-1], + dltimes[2*complete/10-1], + dltimes[1*complete/4-1], + dltimes[3*complete/10-1], + dltimes[4*complete/10-1], + dltimes[5*complete/10-1], + dltimes[6*complete/10-1], + dltimes[7*complete/10-1], + dltimes[3*complete/4-1], + dltimes[8*complete/10-1], + dltimes[9*complete/10-1], + dltimes[complete-1]); + if (written<0) + tor_free(result); + tor_free(dltimes); + } + done: + SMARTLIST_FOREACH(dirreq_completed, dirreq_map_entry_t *, ent, + tor_free(ent)); + smartlist_free(dirreq_completed); + return result; +} + +/** Return a newly allocated comma-separated string containing entries for + * all the countries from which we've seen enough clients connect as a + * bridge, directory server, or entry guard. The entry format is cc=num + * where num is the number of IPs we've seen connecting from that country, + * and cc is a lowercased country code. Returns NULL if we don't want + * to export geoip data yet. */ +char * +geoip_get_client_history(geoip_client_action_t action) +{ + char *result = NULL; + unsigned granularity = IP_GRANULARITY; + smartlist_t *chunks = NULL; + smartlist_t *entries = NULL; + int n_countries = geoip_get_n_countries(); + int i; + clientmap_entry_t **ent; + unsigned *counts = NULL; + unsigned total = 0; + + if (!geoip_is_loaded()) + return NULL; + + counts = tor_malloc_zero(sizeof(unsigned)*n_countries); + HT_FOREACH(ent, clientmap, &client_history) { + int country; + if ((*ent)->action != (int)action) + continue; + country = geoip_get_country_by_ip((*ent)->ipaddr); + if (country < 0) + country = 0; /** unresolved requests are stored at index 0. */ + tor_assert(0 <= country && country < n_countries); + ++counts[country]; + ++total; + } + /* Don't record anything if we haven't seen enough IPs. */ + if (total < MIN_IPS_TO_NOTE_ANYTHING) + goto done; + /* Make a list of c_hist_t */ + entries = smartlist_create(); + for (i = 0; i < n_countries; ++i) { + unsigned c = counts[i]; + const char *countrycode; + c_hist_t *ent; + /* Only report a country if it has a minimum number of IPs. */ + if (c >= MIN_IPS_TO_NOTE_COUNTRY) { + c = round_to_next_multiple_of(c, granularity); + countrycode = geoip_get_country_name(i); + ent = tor_malloc(sizeof(c_hist_t)); + strlcpy(ent->country, countrycode, sizeof(ent->country)); + ent->total = c; + smartlist_add(entries, ent); } } + /* Sort entries. Note that we must do this _AFTER_ rounding, or else + * the sort order could leak info. */ + smartlist_sort(entries, _c_hist_compare); + + /* Build the result. */ + chunks = smartlist_create(); + SMARTLIST_FOREACH(entries, c_hist_t *, ch, { + char *buf=NULL; + tor_asprintf(&buf, "%s=%u", ch->country, ch->total); + smartlist_add(chunks, buf); + }); + result = smartlist_join_strings(chunks, ",", 0, NULL); + done: + tor_free(counts); + if (chunks) { + SMARTLIST_FOREACH(chunks, char *, c, tor_free(c)); + smartlist_free(chunks); + } + if (entries) { + SMARTLIST_FOREACH(entries, c_hist_t *, c, tor_free(c)); + smartlist_free(entries); + } return result; } @@ -540,18 +858,12 @@ geoip_get_client_history(time_t now, geoip_client_action_t action) * for <b>action</b> in a format suitable for an extra-info document, or NULL * on failure. */ char * -geoip_get_request_history(time_t now, geoip_client_action_t action) +geoip_get_request_history(geoip_client_action_t action) { smartlist_t *entries, *strings; char *result; unsigned granularity = IP_GRANULARITY; -#ifdef ENABLE_GEOIP_STATS - if (get_options()->DirRecordUsageByCountry) - granularity = get_options()->DirRecordUsageGranularity; -#endif - if (client_history_starts >= (now - GEOIP_MIN_OBSERVATION_TIME)) - return NULL; if (action != GEOIP_CLIENT_NETWORKSTATUS && action != GEOIP_CLIENT_NETWORKSTATUS_V2) return NULL; @@ -560,13 +872,10 @@ geoip_get_request_history(time_t now, geoip_client_action_t action) entries = smartlist_create(); SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, { - uint32_t *n = (action == GEOIP_CLIENT_NETWORKSTATUS) - ? c->n_v3_ns_requests : c->n_v2_ns_requests; uint32_t tot = 0; - int i; c_hist_t *ent; - for (i=0; i < REQUEST_HIST_LEN; ++i) - tot += n[i]; + tot = (action == GEOIP_CLIENT_NETWORKSTATUS) ? + c->n_v3_ns_requests : c->n_v2_ns_requests; if (!tot) continue; ent = tor_malloc_zero(sizeof(c_hist_t)); @@ -578,9 +887,9 @@ geoip_get_request_history(time_t now, geoip_client_action_t action) strings = smartlist_create(); SMARTLIST_FOREACH(entries, c_hist_t *, ent, { - char buf[32]; - tor_snprintf(buf, sizeof(buf), "%s=%u", ent->country, ent->total); - smartlist_add(strings, tor_strdup(buf)); + char *buf = NULL; + tor_asprintf(&buf, "%s=%u", ent->country, ent->total); + smartlist_add(strings, buf); }); result = smartlist_join_strings(strings, ",", 0, NULL); SMARTLIST_FOREACH(strings, char *, cp, tor_free(cp)); @@ -590,69 +899,450 @@ geoip_get_request_history(time_t now, geoip_client_action_t action) return result; } -/** Store all our geoip statistics into $DATADIR/geoip-stats. */ +/** Start time of directory request stats or 0 if we're not collecting + * directory request statistics. */ +static time_t start_of_dirreq_stats_interval; + +/** Initialize directory request stats. */ void -dump_geoip_stats(void) +geoip_dirreq_stats_init(time_t now) +{ + start_of_dirreq_stats_interval = now; +} + +/** Stop collecting directory request stats in a way that we can re-start + * doing so in geoip_dirreq_stats_init(). */ +void +geoip_dirreq_stats_term(void) +{ + SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, { + c->n_v2_ns_requests = c->n_v3_ns_requests = 0; + }); + { + clientmap_entry_t **ent, **next, *this; + for (ent = HT_START(clientmap, &client_history); ent != NULL; + ent = next) { + if ((*ent)->action == GEOIP_CLIENT_NETWORKSTATUS || + (*ent)->action == GEOIP_CLIENT_NETWORKSTATUS_V2) { + this = *ent; + next = HT_NEXT_RMV(clientmap, &client_history, ent); + tor_free(this); + } else { + next = HT_NEXT(clientmap, &client_history, ent); + } + } + } + v2_share_times_seconds = v3_share_times_seconds = 0.0; + last_time_determined_shares = 0; + share_seconds = 0; + memset(ns_v2_responses, 0, sizeof(ns_v2_responses)); + memset(ns_v3_responses, 0, sizeof(ns_v3_responses)); + { + dirreq_map_entry_t **ent, **next, *this; + for (ent = HT_START(dirreqmap, &dirreq_map); ent != NULL; ent = next) { + this = *ent; + next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ent); + tor_free(this); + } + } + start_of_dirreq_stats_interval = 0; +} + +/** Write dirreq statistics to $DATADIR/stats/dirreq-stats and return when + * we would next want to write. */ +time_t +geoip_dirreq_stats_write(time_t now) { -#ifdef ENABLE_GEOIP_STATS - time_t now = time(NULL); - time_t request_start; - char *filename = get_datadir_fname("geoip-stats"); + char *statsdir = NULL, *filename = NULL; char *data_v2 = NULL, *data_v3 = NULL; - char since[ISO_TIME_LEN+1], written[ISO_TIME_LEN+1]; + char written[ISO_TIME_LEN+1]; open_file_t *open_file = NULL; double v2_share = 0.0, v3_share = 0.0; FILE *out; + int i; - data_v2 = geoip_get_client_history(now, GEOIP_CLIENT_NETWORKSTATUS_V2); - data_v3 = geoip_get_client_history(now, GEOIP_CLIENT_NETWORKSTATUS); - format_iso_time(since, geoip_get_history_start()); + if (!start_of_dirreq_stats_interval) + return 0; /* Not initialized. */ + if (start_of_dirreq_stats_interval + WRITE_STATS_INTERVAL > now) + goto done; /* Not ready to write. */ + + /* Discard all items in the client history that are too old. */ + geoip_remove_old_clients(start_of_dirreq_stats_interval); + + statsdir = get_datadir_fname("stats"); + if (check_private_dir(statsdir, CPD_CREATE) < 0) + goto done; + filename = get_datadir_fname2("stats", "dirreq-stats"); + data_v2 = geoip_get_client_history(GEOIP_CLIENT_NETWORKSTATUS_V2); + data_v3 = geoip_get_client_history(GEOIP_CLIENT_NETWORKSTATUS); format_iso_time(written, now); - out = start_writing_to_stdio_file(filename, OPEN_FLAGS_REPLACE, + out = start_writing_to_stdio_file(filename, OPEN_FLAGS_APPEND, 0600, &open_file); if (!out) goto done; - if (fprintf(out, "written %s\nstarted-at %s\nns-ips %s\nns-v2-ips %s\n", - written, since, + if (fprintf(out, "dirreq-stats-end %s (%d s)\ndirreq-v3-ips %s\n" + "dirreq-v2-ips %s\n", written, + (unsigned) (now - start_of_dirreq_stats_interval), data_v3 ? data_v3 : "", data_v2 ? data_v2 : "") < 0) goto done; tor_free(data_v2); tor_free(data_v3); - request_start = current_request_period_starts - - (n_old_request_periods * REQUEST_HIST_PERIOD); - format_iso_time(since, request_start); - data_v2 = geoip_get_request_history(now, GEOIP_CLIENT_NETWORKSTATUS_V2); - data_v3 = geoip_get_request_history(now, GEOIP_CLIENT_NETWORKSTATUS); - if (fprintf(out, "requests-start %s\nn-ns-reqs %s\nn-v2-ns-reqs %s\n", - since, + data_v2 = geoip_get_request_history(GEOIP_CLIENT_NETWORKSTATUS_V2); + data_v3 = geoip_get_request_history(GEOIP_CLIENT_NETWORKSTATUS); + if (fprintf(out, "dirreq-v3-reqs %s\ndirreq-v2-reqs %s\n", data_v3 ? data_v3 : "", data_v2 ? data_v2 : "") < 0) goto done; - if (!router_get_my_share_of_directory_requests(&v2_share, &v3_share)) { - if (fprintf(out, "v2-ns-share %0.2lf%%\n", v2_share*100) < 0) + tor_free(data_v2); + tor_free(data_v3); + SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, { + c->n_v2_ns_requests = c->n_v3_ns_requests = 0; + }); +#define RESPONSE_GRANULARITY 8 + for (i = 0; i < GEOIP_NS_RESPONSE_NUM; i++) { + ns_v2_responses[i] = round_uint32_to_next_multiple_of( + ns_v2_responses[i], RESPONSE_GRANULARITY); + ns_v3_responses[i] = round_uint32_to_next_multiple_of( + ns_v3_responses[i], RESPONSE_GRANULARITY); + } +#undef RESPONSE_GRANULARITY + if (fprintf(out, "dirreq-v3-resp ok=%u,not-enough-sigs=%u,unavailable=%u," + "not-found=%u,not-modified=%u,busy=%u\n", + ns_v3_responses[GEOIP_SUCCESS], + ns_v3_responses[GEOIP_REJECT_NOT_ENOUGH_SIGS], + ns_v3_responses[GEOIP_REJECT_UNAVAILABLE], + ns_v3_responses[GEOIP_REJECT_NOT_FOUND], + ns_v3_responses[GEOIP_REJECT_NOT_MODIFIED], + ns_v3_responses[GEOIP_REJECT_BUSY]) < 0) + goto done; + if (fprintf(out, "dirreq-v2-resp ok=%u,unavailable=%u," + "not-found=%u,not-modified=%u,busy=%u\n", + ns_v2_responses[GEOIP_SUCCESS], + ns_v2_responses[GEOIP_REJECT_UNAVAILABLE], + ns_v2_responses[GEOIP_REJECT_NOT_FOUND], + ns_v2_responses[GEOIP_REJECT_NOT_MODIFIED], + ns_v2_responses[GEOIP_REJECT_BUSY]) < 0) + goto done; + memset(ns_v2_responses, 0, sizeof(ns_v2_responses)); + memset(ns_v3_responses, 0, sizeof(ns_v3_responses)); + if (!geoip_get_mean_shares(now, &v2_share, &v3_share)) { + if (fprintf(out, "dirreq-v2-share %0.2lf%%\n", v2_share*100) < 0) goto done; - if (fprintf(out, "v3-ns-share %0.2lf%%\n", v3_share*100) < 0) + if (fprintf(out, "dirreq-v3-share %0.2lf%%\n", v3_share*100) < 0) goto done; } + data_v2 = geoip_get_dirreq_history(GEOIP_CLIENT_NETWORKSTATUS_V2, + DIRREQ_DIRECT); + data_v3 = geoip_get_dirreq_history(GEOIP_CLIENT_NETWORKSTATUS, + DIRREQ_DIRECT); + if (fprintf(out, "dirreq-v3-direct-dl %s\ndirreq-v2-direct-dl %s\n", + data_v3 ? data_v3 : "", data_v2 ? data_v2 : "") < 0) + goto done; + tor_free(data_v2); + tor_free(data_v3); + data_v2 = geoip_get_dirreq_history(GEOIP_CLIENT_NETWORKSTATUS_V2, + DIRREQ_TUNNELED); + data_v3 = geoip_get_dirreq_history(GEOIP_CLIENT_NETWORKSTATUS, + DIRREQ_TUNNELED); + if (fprintf(out, "dirreq-v3-tunneled-dl %s\ndirreq-v2-tunneled-dl %s\n", + data_v3 ? data_v3 : "", data_v2 ? data_v2 : "") < 0) + goto done; + finish_writing_to_file(open_file); open_file = NULL; + + start_of_dirreq_stats_interval = now; + done: if (open_file) abort_writing_to_file(open_file); tor_free(filename); + tor_free(statsdir); tor_free(data_v2); tor_free(data_v3); -#endif + return start_of_dirreq_stats_interval + WRITE_STATS_INTERVAL; +} + +/** Start time of bridge stats or 0 if we're not collecting bridge + * statistics. */ +static time_t start_of_bridge_stats_interval; + +/** Initialize bridge stats. */ +void +geoip_bridge_stats_init(time_t now) +{ + start_of_bridge_stats_interval = now; +} + +/** Stop collecting bridge stats in a way that we can re-start doing so in + * geoip_bridge_stats_init(). */ +void +geoip_bridge_stats_term(void) +{ + client_history_clear(); + start_of_bridge_stats_interval = 0; +} + +/** Validate a bridge statistics string as it would be written to a + * current extra-info descriptor. Return 1 if the string is valid and + * recent enough, or 0 otherwise. */ +static int +validate_bridge_stats(const char *stats_str, time_t now) +{ + char stats_end_str[ISO_TIME_LEN+1], stats_start_str[ISO_TIME_LEN+1], + *eos; + + const char *BRIDGE_STATS_END = "bridge-stats-end "; + const char *BRIDGE_IPS = "bridge-ips "; + const char *BRIDGE_IPS_EMPTY_LINE = "bridge-ips\n"; + const char *tmp; + time_t stats_end_time; + int seconds; + tor_assert(stats_str); + + /* Parse timestamp and number of seconds from + "bridge-stats-end YYYY-MM-DD HH:MM:SS (N s)" */ + tmp = find_str_at_start_of_line(stats_str, BRIDGE_STATS_END); + if (!tmp) + return 0; + tmp += strlen(BRIDGE_STATS_END); + + if (strlen(tmp) < ISO_TIME_LEN + 6) + return 0; + strlcpy(stats_end_str, tmp, sizeof(stats_end_str)); + if (parse_iso_time(stats_end_str, &stats_end_time) < 0) + return 0; + if (stats_end_time < now - (25*60*60) || + stats_end_time > now + (1*60*60)) + return 0; + seconds = (int)strtol(tmp + ISO_TIME_LEN + 2, &eos, 10); + if (!eos || seconds < 23*60*60) + return 0; + format_iso_time(stats_start_str, stats_end_time - seconds); + + /* Parse: "bridge-ips CC=N,CC=N,..." */ + tmp = find_str_at_start_of_line(stats_str, BRIDGE_IPS); + if (!tmp) { + /* Look if there is an empty "bridge-ips" line */ + tmp = find_str_at_start_of_line(stats_str, BRIDGE_IPS_EMPTY_LINE); + if (!tmp) + return 0; + } + + return 1; +} + +/** Most recent bridge statistics formatted to be written to extra-info + * descriptors. */ +static char *bridge_stats_extrainfo = NULL; + +/** Return a newly allocated string holding our bridge usage stats by country + * in a format suitable for inclusion in an extrainfo document. Return NULL on + * failure. */ +static char * +format_bridge_stats_extrainfo(time_t now) +{ + char *out = NULL, *data = NULL; + long duration = now - start_of_bridge_stats_interval; + char written[ISO_TIME_LEN+1]; + + if (duration < 0) + return NULL; + + format_iso_time(written, now); + data = geoip_get_client_history(GEOIP_CLIENT_CONNECT); + + tor_asprintf(&out, + "bridge-stats-end %s (%ld s)\n" + "bridge-ips %s\n", + written, duration, + data ? data : ""); + tor_free(data); + + return out; +} + +/** Return a newly allocated string holding our bridge usage stats by country + * in a format suitable for the answer to a controller request. Return NULL on + * failure. */ +static char * +format_bridge_stats_controller(time_t now) +{ + char *out = NULL, *data = NULL; + char started[ISO_TIME_LEN+1]; + (void) now; + + format_iso_time(started, start_of_bridge_stats_interval); + data = geoip_get_client_history(GEOIP_CLIENT_CONNECT); + + tor_asprintf(&out, + "TimeStarted=\"%s\" CountrySummary=%s", + started, data ? data : ""); + tor_free(data); + return out; +} + +/** Write bridge statistics to $DATADIR/stats/bridge-stats and return + * when we should next try to write statistics. */ +time_t +geoip_bridge_stats_write(time_t now) +{ + char *filename = NULL, *val = NULL, *statsdir = NULL; + + /* Check if 24 hours have passed since starting measurements. */ + if (now < start_of_bridge_stats_interval + WRITE_STATS_INTERVAL) + return start_of_bridge_stats_interval + WRITE_STATS_INTERVAL; + + /* Discard all items in the client history that are too old. */ + geoip_remove_old_clients(start_of_bridge_stats_interval); + + /* Generate formatted string */ + val = format_bridge_stats_extrainfo(now); + if (val == NULL) + goto done; + + /* Update the stored value. */ + tor_free(bridge_stats_extrainfo); + bridge_stats_extrainfo = val; + start_of_bridge_stats_interval = now; + + /* Write it to disk. */ + statsdir = get_datadir_fname("stats"); + if (check_private_dir(statsdir, CPD_CREATE) < 0) + goto done; + filename = get_datadir_fname2("stats", "bridge-stats"); + + write_str_to_file(filename, bridge_stats_extrainfo, 0); + + /* Tell the controller, "hey, there are clients!" */ + { + char *controller_str = format_bridge_stats_controller(now); + if (controller_str) + control_event_clients_seen(controller_str); + tor_free(controller_str); + } + done: + tor_free(filename); + tor_free(statsdir); + + return start_of_bridge_stats_interval + WRITE_STATS_INTERVAL; +} + +/** Try to load the most recent bridge statistics from disk, unless we + * have finished a measurement interval lately, and check whether they + * are still recent enough. */ +static void +load_bridge_stats(time_t now) +{ + char *fname, *contents; + if (bridge_stats_extrainfo) + return; + + fname = get_datadir_fname2("stats", "bridge-stats"); + contents = read_file_to_str(fname, RFTS_IGNORE_MISSING, NULL); + if (contents && validate_bridge_stats(contents, now)) + bridge_stats_extrainfo = contents; + + tor_free(fname); +} + +/** Return most recent bridge statistics for inclusion in extra-info + * descriptors, or NULL if we don't have recent bridge statistics. */ +const char * +geoip_get_bridge_stats_extrainfo(time_t now) +{ + load_bridge_stats(now); + return bridge_stats_extrainfo; +} + +/** Return a new string containing the recent bridge statistics to be returned + * to controller clients, or NULL if we don't have any bridge statistics. */ +char * +geoip_get_bridge_stats_controller(time_t now) +{ + return format_bridge_stats_controller(now); +} + +/** Start time of entry stats or 0 if we're not collecting entry + * statistics. */ +static time_t start_of_entry_stats_interval; + +/** Initialize entry stats. */ +void +geoip_entry_stats_init(time_t now) +{ + start_of_entry_stats_interval = now; +} + +/** Stop collecting entry stats in a way that we can re-start doing so in + * geoip_entry_stats_init(). */ +void +geoip_entry_stats_term(void) +{ + client_history_clear(); + start_of_entry_stats_interval = 0; +} + +/** Write entry statistics to $DATADIR/stats/entry-stats and return time + * when we would next want to write. */ +time_t +geoip_entry_stats_write(time_t now) +{ + char *statsdir = NULL, *filename = NULL; + char *data = NULL; + char written[ISO_TIME_LEN+1]; + open_file_t *open_file = NULL; + FILE *out; + + if (!start_of_entry_stats_interval) + return 0; /* Not initialized. */ + if (start_of_entry_stats_interval + WRITE_STATS_INTERVAL > now) + goto done; /* Not ready to write. */ + + /* Discard all items in the client history that are too old. */ + geoip_remove_old_clients(start_of_entry_stats_interval); + + statsdir = get_datadir_fname("stats"); + if (check_private_dir(statsdir, CPD_CREATE) < 0) + goto done; + filename = get_datadir_fname2("stats", "entry-stats"); + data = geoip_get_client_history(GEOIP_CLIENT_CONNECT); + format_iso_time(written, now); + out = start_writing_to_stdio_file(filename, OPEN_FLAGS_APPEND, + 0600, &open_file); + if (!out) + goto done; + if (fprintf(out, "entry-stats-end %s (%u s)\nentry-ips %s\n", + written, (unsigned) (now - start_of_entry_stats_interval), + data ? data : "") < 0) + goto done; + + start_of_entry_stats_interval = now; + + finish_writing_to_file(open_file); + open_file = NULL; + done: + if (open_file) + abort_writing_to_file(open_file); + tor_free(filename); + tor_free(statsdir); + tor_free(data); + return start_of_entry_stats_interval + WRITE_STATS_INTERVAL; } /** Helper used to implement GETINFO ip-to-country/... controller command. */ int getinfo_helper_geoip(control_connection_t *control_conn, - const char *question, char **answer) + const char *question, char **answer, + const char **errmsg) { (void)control_conn; - if (geoip_is_loaded() && !strcmpstart(question, "ip-to-country/")) { + if (!geoip_is_loaded()) { + *errmsg = "GeoIP data not loaded"; + return -1; + } + if (!strcmpstart(question, "ip-to-country/")) { int c; uint32_t ip; struct in_addr in; @@ -674,8 +1364,8 @@ clear_geoip_db(void) SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, tor_free(c)); smartlist_free(geoip_countries); } - if (country_idxplus1_by_lc_code) - strmap_free(country_idxplus1_by_lc_code, NULL); + + strmap_free(country_idxplus1_by_lc_code, NULL); if (geoip_entries) { SMARTLIST_FOREACH(geoip_entries, geoip_entry_t *, ent, tor_free(ent)); smartlist_free(geoip_entries); @@ -689,13 +1379,24 @@ clear_geoip_db(void) void geoip_free_all(void) { - clientmap_entry_t **ent, **next, *this; - for (ent = HT_START(clientmap, &client_history); ent != NULL; ent = next) { - this = *ent; - next = HT_NEXT_RMV(clientmap, &client_history, ent); - tor_free(this); + { + clientmap_entry_t **ent, **next, *this; + for (ent = HT_START(clientmap, &client_history); ent != NULL; ent = next) { + this = *ent; + next = HT_NEXT_RMV(clientmap, &client_history, ent); + tor_free(this); + } + HT_CLEAR(clientmap, &client_history); + } + { + dirreq_map_entry_t **ent, **next, *this; + for (ent = HT_START(dirreqmap, &dirreq_map); ent != NULL; ent = next) { + this = *ent; + next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ent); + tor_free(this); + } + HT_CLEAR(dirreqmap, &dirreq_map); } - HT_CLEAR(clientmap, &client_history); clear_geoip_db(); } diff --git a/src/or/geoip.h b/src/or/geoip.h new file mode 100644 index 0000000000..24f7c5b931 --- /dev/null +++ b/src/or/geoip.h @@ -0,0 +1,57 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file geoip.h + * \brief Header file for geoip.c. + **/ + +#ifndef _TOR_GEOIP_H +#define _TOR_GEOIP_H + +#ifdef GEOIP_PRIVATE +int geoip_parse_entry(const char *line); +#endif +int should_record_bridge_info(or_options_t *options); +int geoip_load_file(const char *filename, or_options_t *options); +int geoip_get_country_by_ip(uint32_t ipaddr); +int geoip_get_n_countries(void); +const char *geoip_get_country_name(country_t num); +int geoip_is_loaded(void); +country_t geoip_get_country(const char *countrycode); + +void geoip_note_client_seen(geoip_client_action_t action, + uint32_t addr, time_t now); +void geoip_remove_old_clients(time_t cutoff); + +void geoip_note_ns_response(geoip_client_action_t action, + geoip_ns_response_t response); +char *geoip_get_client_history(geoip_client_action_t action); +char *geoip_get_request_history(geoip_client_action_t action); +int getinfo_helper_geoip(control_connection_t *control_conn, + const char *question, char **answer, + const char **errmsg); +void geoip_free_all(void); + +void geoip_start_dirreq(uint64_t dirreq_id, size_t response_size, + geoip_client_action_t action, dirreq_type_t type); +void geoip_change_dirreq_state(uint64_t dirreq_id, dirreq_type_t type, + dirreq_state_t new_state); + +void geoip_dirreq_stats_init(time_t now); +time_t geoip_dirreq_stats_write(time_t now); +void geoip_dirreq_stats_term(void); +void geoip_entry_stats_init(time_t now); +time_t geoip_entry_stats_write(time_t now); +void geoip_entry_stats_term(void); +void geoip_bridge_stats_init(time_t now); +time_t geoip_bridge_stats_write(time_t now); +void geoip_bridge_stats_term(void); +const char *geoip_get_bridge_stats_extrainfo(time_t); +char *geoip_get_bridge_stats_controller(time_t); + +#endif + diff --git a/src/or/hibernate.c b/src/or/hibernate.c index 9ec63756fa..aebce4cc88 100644 --- a/src/or/hibernate.c +++ b/src/or/hibernate.c @@ -22,6 +22,12 @@ hibernating, phase 2: */ #include "or.h" +#include "config.h" +#include "connection.h" +#include "connection_edge.h" +#include "hibernate.h" +#include "main.h" +#include "router.h" /** Possible values of hibernate_state */ typedef enum { @@ -89,6 +95,13 @@ static uint64_t n_bytes_read_in_interval = 0; static uint64_t n_bytes_written_in_interval = 0; /** How many seconds have we been running this interval? */ static uint32_t n_seconds_active_in_interval = 0; +/** How many seconds were we active in this interval before we hit our soft + * limit? */ +static int n_seconds_to_hit_soft_limit = 0; +/** When in this interval was the soft limit hit. */ +static time_t soft_limit_hit_at = 0; +/** How many bytes had we read/written when we hit the soft limit? */ +static uint64_t n_bytes_at_soft_limit = 0; /** When did this accounting interval start? */ static time_t interval_start_time = 0; /** When will this accounting interval end? */ @@ -182,6 +195,9 @@ accounting_parse_options(or_options_t *options, int validate_only) case UNIT_DAY: d = 0; break; + /* Coverity dislikes unreachable default cases; some compilers warn on + * switch statements missing a case. Tell Coverity not to worry. */ + /* coverity[dead_error_begin] */ default: tor_assert(0); } @@ -332,29 +348,58 @@ start_of_accounting_period_after(time_t now) return edge_of_accounting_period_containing(now, 1); } +/** Return the length of the accounting period containing the time + * <b>now</b>. */ +static long +length_of_accounting_period_containing(time_t now) +{ + return edge_of_accounting_period_containing(now, 1) - + edge_of_accounting_period_containing(now, 0); +} + /** Initialize the accounting subsystem. */ void configure_accounting(time_t now) { + time_t s_now; /* Try to remember our recorded usage. */ if (!interval_start_time) read_bandwidth_usage(); /* If we fail, we'll leave values at zero, and * reset below.*/ - if (!interval_start_time || - start_of_accounting_period_after(interval_start_time) <= now) { - /* We didn't have recorded usage, or we don't have recorded usage - * for this interval. Start a new interval. */ + + s_now = start_of_accounting_period_containing(now); + + if (!interval_start_time) { + /* We didn't have recorded usage; Start a new interval. */ log_info(LD_ACCT, "Starting new accounting interval."); reset_accounting(now); - } else if (interval_start_time == - start_of_accounting_period_containing(interval_start_time)) { + } else if (s_now == interval_start_time) { log_info(LD_ACCT, "Continuing accounting interval."); /* We are in the interval we thought we were in. Do nothing.*/ interval_end_time = start_of_accounting_period_after(interval_start_time); } else { - log_warn(LD_ACCT, - "Mismatched accounting interval; starting a fresh one."); - reset_accounting(now); + long duration = + length_of_accounting_period_containing(interval_start_time); + double delta = ((double)(s_now - interval_start_time)) / duration; + if (-0.50 <= delta && delta <= 0.50) { + /* The start of the period is now a little later or earlier than we + * remembered. That's fine; we might lose some bytes we could otherwise + * have written, but better to err on the side of obeying people's + * accounting settings. */ + log_info(LD_ACCT, "Accounting interval moved by %.02f%%; " + "that's fine.", delta*100); + interval_end_time = start_of_accounting_period_after(now); + } else if (delta >= 0.99) { + /* This is the regular time-moved-forward case; don't be too noisy + * about it or people will complain */ + log_info(LD_ACCT, "Accounting interval elapsed; starting a new one"); + reset_accounting(now); + } else { + log_warn(LD_ACCT, + "Mismatched accounting interval: moved by %.02f%%. " + "Starting a fresh one.", delta*100); + reset_accounting(now); + } } accounting_set_wakeup_time(); } @@ -365,23 +410,42 @@ configure_accounting(time_t now) static void update_expected_bandwidth(void) { - uint64_t used, expected; - uint64_t max_configured = (get_options()->BandwidthRate * 60); - - if (n_seconds_active_in_interval < 1800) { + uint64_t expected; + or_options_t *options= get_options(); + uint64_t max_configured = (options->RelayBandwidthRate > 0 ? + options->RelayBandwidthRate : + options->BandwidthRate) * 60; + +#define MIN_TIME_FOR_MEASUREMENT (1800) + + if (soft_limit_hit_at > interval_start_time && n_bytes_at_soft_limit && + (soft_limit_hit_at - interval_start_time) > MIN_TIME_FOR_MEASUREMENT) { + /* If we hit our soft limit last time, only count the bytes up to that + * time. This is a better predictor of our actual bandwidth than + * considering the entirety of the last interval, since we likely started + * using bytes very slowly once we hit our soft limit. */ + expected = n_bytes_at_soft_limit / + (soft_limit_hit_at - interval_start_time); + expected /= 60; + } else if (n_seconds_active_in_interval >= MIN_TIME_FOR_MEASUREMENT) { + /* Otherwise, we either measured enough time in the last interval but + * never hit our soft limit, or we're using a state file from a Tor that + * doesn't know to store soft-limit info. Just take rate at which + * we were reading/writing in the last interval as our expected rate. + */ + uint64_t used = MAX(n_bytes_written_in_interval, + n_bytes_read_in_interval); + expected = used / (n_seconds_active_in_interval / 60); + } else { /* If we haven't gotten enough data last interval, set 'expected' * to 0. This will set our wakeup to the start of the interval. * Next interval, we'll choose our starting time based on how much * we sent this interval. */ expected = 0; - } else { - used = n_bytes_written_in_interval < n_bytes_read_in_interval ? - n_bytes_read_in_interval : n_bytes_written_in_interval; - expected = used / (n_seconds_active_in_interval / 60); - if (expected > max_configured) - expected = max_configured; } + if (expected > max_configured) + expected = max_configured; expected_bandwidth_usage = expected; } @@ -399,6 +463,9 @@ reset_accounting(time_t now) n_bytes_read_in_interval = 0; n_bytes_written_in_interval = 0; n_seconds_active_in_interval = 0; + n_bytes_at_soft_limit = 0; + soft_limit_hit_at = 0; + n_seconds_to_hit_soft_limit = 0; } /** Return true iff we should save our bandwidth usage to disk. */ @@ -449,35 +516,39 @@ accounting_run_housekeeping(time_t now) static void accounting_set_wakeup_time(void) { - char buf[ISO_TIME_LEN+1]; char digest[DIGEST_LEN]; crypto_digest_env_t *d_env; int time_in_interval; uint64_t time_to_exhaust_bw; int time_to_consider; - if (! identity_key_is_set()) { + if (! server_identity_key_is_set()) { if (init_keys() < 0) { log_err(LD_BUG, "Error initializing keys"); tor_assert(0); } } - format_iso_time(buf, interval_start_time); - crypto_pk_get_digest(get_identity_key(), digest); + if (server_identity_key_is_set()) { + char buf[ISO_TIME_LEN+1]; + format_iso_time(buf, interval_start_time); - d_env = crypto_new_digest_env(); - crypto_digest_add_bytes(d_env, buf, ISO_TIME_LEN); - crypto_digest_add_bytes(d_env, digest, DIGEST_LEN); - crypto_digest_get_digest(d_env, digest, DIGEST_LEN); - crypto_free_digest_env(d_env); + crypto_pk_get_digest(get_server_identity_key(), digest); + + d_env = crypto_new_digest_env(); + crypto_digest_add_bytes(d_env, buf, ISO_TIME_LEN); + crypto_digest_add_bytes(d_env, digest, DIGEST_LEN); + crypto_digest_get_digest(d_env, digest, DIGEST_LEN); + crypto_free_digest_env(d_env); + } else { + crypto_rand(digest, DIGEST_LEN); + } if (!expected_bandwidth_usage) { char buf1[ISO_TIME_LEN+1]; char buf2[ISO_TIME_LEN+1]; format_local_iso_time(buf1, interval_start_time); format_local_iso_time(buf2, interval_end_time); - time_to_exhaust_bw = GUESS_TIME_TO_USE_BANDWIDTH; interval_wakeup_time = interval_start_time; log_notice(LD_ACCT, @@ -492,8 +563,8 @@ accounting_set_wakeup_time(void) time_to_exhaust_bw = (get_options()->AccountingMax/expected_bandwidth_usage)*60; - if (time_to_exhaust_bw > TIME_MAX) { - time_to_exhaust_bw = TIME_MAX; + if (time_to_exhaust_bw > INT_MAX) { + time_to_exhaust_bw = INT_MAX; time_to_consider = 0; } else { time_to_consider = time_in_interval - (int)time_to_exhaust_bw; @@ -511,8 +582,6 @@ accounting_set_wakeup_time(void) * to be chosen than the last half. */ interval_wakeup_time = interval_start_time + (get_uint32(digest) % time_to_consider); - - format_iso_time(buf, interval_wakeup_time); } { @@ -559,6 +628,10 @@ accounting_record_bandwidth_usage(time_t now, or_state_t *state) state->AccountingSecondsActive = n_seconds_active_in_interval; state->AccountingExpectedUsage = expected_bandwidth_usage; + state->AccountingSecondsToReachSoftLimit = n_seconds_to_hit_soft_limit; + state->AccountingSoftLimitHitAt = soft_limit_hit_at; + state->AccountingBytesAtSoftLimit = n_bytes_at_soft_limit; + or_state_mark_dirty(state, now+(get_options()->AvoidDiskWrites ? 7200 : 60)); @@ -582,10 +655,6 @@ read_bandwidth_usage(void) if (!state) return -1; - /* Okay; it looks like the state file is more up-to-date than the - * bw_accounting file, or the bw_accounting file is nonexistent, - * or the bw_accounting file is corrupt. - */ log_info(LD_ACCT, "Reading bandwidth accounting data from state file"); n_bytes_read_in_interval = state->AccountingBytesReadInInterval; n_bytes_written_in_interval = state->AccountingBytesWrittenInInterval; @@ -593,6 +662,21 @@ read_bandwidth_usage(void) interval_start_time = state->AccountingIntervalStart; expected_bandwidth_usage = state->AccountingExpectedUsage; + /* Older versions of Tor (before 0.2.2.17-alpha or so) didn't generate these + * fields. If you switch back and forth, you might get an + * AccountingSoftLimitHitAt value from long before the most recent + * interval_start_time. If that's so, then ignore the softlimit-related + * values. */ + if (state->AccountingSoftLimitHitAt > interval_start_time) { + soft_limit_hit_at = state->AccountingSoftLimitHitAt; + n_bytes_at_soft_limit = state->AccountingBytesAtSoftLimit; + n_seconds_to_hit_soft_limit = state->AccountingSecondsToReachSoftLimit; + } else { + soft_limit_hit_at = 0; + n_bytes_at_soft_limit = 0; + n_seconds_to_hit_soft_limit = 0; + } + { char tbuf1[ISO_TIME_LEN+1]; char tbuf2[ISO_TIME_LEN+1]; @@ -632,8 +716,27 @@ hibernate_hard_limit_reached(void) static int hibernate_soft_limit_reached(void) { - uint64_t soft_limit = DBL_TO_U64(U64_TO_DBL(get_options()->AccountingMax) - * .95); + const uint64_t acct_max = get_options()->AccountingMax; +#define SOFT_LIM_PCT (.95) +#define SOFT_LIM_BYTES (500*1024*1024) +#define SOFT_LIM_MINUTES (3*60) + /* The 'soft limit' is a fair bit more complicated now than once it was. + * We want to stop accepting connections when ALL of the following are true: + * - We expect to use up the remaining bytes in under 3 hours + * - We have used up 95% of our bytes. + * - We have less than 500MB of bytes left. + */ + uint64_t soft_limit = DBL_TO_U64(U64_TO_DBL(acct_max) * SOFT_LIM_PCT); + if (acct_max > SOFT_LIM_BYTES && acct_max - SOFT_LIM_BYTES > soft_limit) { + soft_limit = acct_max - SOFT_LIM_BYTES; + } + if (expected_bandwidth_usage) { + const uint64_t expected_usage = + expected_bandwidth_usage * SOFT_LIM_MINUTES; + if (acct_max > expected_usage && acct_max - expected_usage > soft_limit) + soft_limit = acct_max - expected_usage; + } + if (!soft_limit) return 0; return n_bytes_read_in_interval >= soft_limit @@ -658,6 +761,14 @@ hibernate_begin(hibernate_state_t new_state, time_t now) exit(0); } + if (new_state == HIBERNATE_STATE_LOWBANDWIDTH && + hibernate_state == HIBERNATE_STATE_LIVE) { + soft_limit_hit_at = now; + n_seconds_to_hit_soft_limit = n_seconds_active_in_interval; + n_bytes_at_soft_limit = MAX(n_bytes_read_in_interval, + n_bytes_written_in_interval); + } + /* close listeners. leave control listener(s). */ while ((conn = connection_get_by_type(CONN_TYPE_OR_LISTENER)) || (conn = connection_get_by_type(CONN_TYPE_AP_LISTENER)) || @@ -673,7 +784,8 @@ hibernate_begin(hibernate_state_t new_state, time_t now) /* XXX upload rendezvous service descriptors with no intro points */ if (new_state == HIBERNATE_STATE_EXITING) { - log_notice(LD_GENERAL,"Interrupt: will shut down in %d seconds. Interrupt " + log_notice(LD_GENERAL,"Interrupt: we have stopped accepting new " + "connections, and will shut down in %d seconds. Interrupt " "again to exit now.", options->ShutdownWaitLength); shutdown_time = time(NULL) + options->ShutdownWaitLength; } else { /* soft limit reached */ @@ -830,7 +942,8 @@ consider_hibernation(time_t now) if (hibernate_state == HIBERNATE_STATE_LIVE) { if (hibernate_soft_limit_reached()) { log_notice(LD_ACCT, - "Bandwidth soft limit reached; commencing hibernation."); + "Bandwidth soft limit reached; commencing hibernation. " + "No new conncetions will be accepted"); hibernate_begin(HIBERNATE_STATE_LOWBANDWIDTH, now); } else if (accounting_enabled && now < interval_wakeup_time) { format_local_iso_time(buf,interval_wakeup_time); @@ -860,9 +973,11 @@ consider_hibernation(time_t now) * NULL. */ int getinfo_helper_accounting(control_connection_t *conn, - const char *question, char **answer) + const char *question, char **answer, + const char **errmsg) { (void) conn; + (void) errmsg; if (!strcmp(question, "accounting/enabled")) { *answer = tor_strdup(accounting_is_enabled(get_options()) ? "1" : "0"); } else if (!strcmp(question, "accounting/hibernating")) { diff --git a/src/or/hibernate.h b/src/or/hibernate.h new file mode 100644 index 0000000000..2aea0fab0c --- /dev/null +++ b/src/or/hibernate.h @@ -0,0 +1,29 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file hibernate.h + * \brief Header file for hibernate.c. + **/ + +#ifndef _TOR_HIBERNATE_H +#define _TOR_HIBERNATE_H + +int accounting_parse_options(or_options_t *options, int validate_only); +int accounting_is_enabled(or_options_t *options); +void configure_accounting(time_t now); +void accounting_run_housekeeping(time_t now); +void accounting_add_bytes(size_t n_read, size_t n_written, int seconds); +int accounting_record_bandwidth_usage(time_t now, or_state_t *state); +void hibernate_begin_shutdown(void); +int we_are_hibernating(void); +void consider_hibernation(time_t now); +int getinfo_helper_accounting(control_connection_t *conn, + const char *question, char **answer, + const char **errmsg); + +#endif + diff --git a/src/or/main.c b/src/or/main.c index e44fd49462..d1ceeec347 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -12,12 +12,50 @@ #define MAIN_PRIVATE #include "or.h" +#include "buffers.h" +#include "circuitbuild.h" +#include "circuitlist.h" +#include "circuituse.h" +#include "command.h" +#include "config.h" +#include "connection.h" +#include "connection_edge.h" +#include "connection_or.h" +#include "control.h" +#include "cpuworker.h" +#include "directory.h" +#include "dirserv.h" +#include "dirvote.h" +#include "dns.h" +#include "dnsserv.h" +#include "geoip.h" +#include "hibernate.h" +#include "main.h" +#include "microdesc.h" +#include "networkstatus.h" +#include "ntmain.h" +#include "onion.h" +#include "policies.h" +#include "relay.h" +#include "rendclient.h" +#include "rendcommon.h" +#include "rendservice.h" +#include "rephist.h" +#include "router.h" +#include "routerlist.h" +#include "routerparse.h" #ifdef USE_DMALLOC #include <dmalloc.h> #include <openssl/crypto.h> #endif #include "memarea.h" +#ifdef HAVE_EVENT2_EVENT_H +#include <event2/event.h> +#else +#include <event.h> +#endif + void evdns_shutdown(int); /********* PROTOTYPES **********/ @@ -26,8 +64,7 @@ static void dumpmemusage(int severity); static void dumpstats(int severity); /* log stats */ static void conn_read_callback(int fd, short event, void *_conn); static void conn_write_callback(int fd, short event, void *_conn); -static void signal_callback(int fd, short events, void *arg); -static void second_elapsed_callback(int fd, short event, void *args); +static void second_elapsed_callback(periodic_timer_t *timer, void *args); static int conn_close_if_marked(int i); static void connection_start_reading_from_linked_conn(connection_t *conn); static int connection_should_read_from_linked_conn(connection_t *conn); @@ -81,8 +118,12 @@ static smartlist_t *active_linked_connection_lst = NULL; static int called_loop_once = 0; /** We set this to 1 when we've opened a circuit, so we can print a log - * entry to inform the user that Tor is working. */ -int has_completed_circuit=0; + * entry to inform the user that Tor is working. We set it to 0 when + * we think the fact that we once opened a circuit doesn't mean we can do so + * any longer (a big time jump happened, when we notice our directory is + * heinously out-of-date, etc. + */ +int can_complete_circuit=0; /** How often do we check for router descriptors that we should download * when we have too little directory info? */ @@ -127,12 +168,10 @@ connection_add(connection_t *conn) smartlist_add(connection_array, conn); if (conn->s >= 0 || conn->linked) { - conn->read_event = tor_malloc_zero(sizeof(struct event)); - conn->write_event = tor_malloc_zero(sizeof(struct event)); - event_set(conn->read_event, conn->s, EV_READ|EV_PERSIST, - conn_read_callback, conn); - event_set(conn->write_event, conn->s, EV_WRITE|EV_PERSIST, - conn_write_callback, conn); + conn->read_event = tor_event_new(tor_libevent_get_base(), + conn->s, EV_READ|EV_PERSIST, conn_read_callback, conn); + conn->write_event = tor_event_new(tor_libevent_get_base(), + conn->s, EV_WRITE|EV_PERSIST, conn_write_callback, conn); } log_debug(LD_NET,"new conn type %s, socket %d, address %s, n_conns %d.", @@ -142,6 +181,25 @@ connection_add(connection_t *conn) return 0; } +/** Tell libevent that we don't care about <b>conn</b> any more. */ +void +connection_unregister_events(connection_t *conn) +{ + if (conn->read_event) { + if (event_del(conn->read_event)) + log_warn(LD_BUG, "Error removing read event for %d", conn->s); + tor_free(conn->read_event); + } + if (conn->write_event) { + if (event_del(conn->write_event)) + log_warn(LD_BUG, "Error removing write event for %d", conn->s); + tor_free(conn->write_event); + } + if (conn->dns_server_port) { + dnsserv_close_listener(conn); + } +} + /** Remove the connection from the global list, and remove the * corresponding poll entry. Calling this function will shift the last * connection (if any) into the position occupied by conn. @@ -246,17 +304,17 @@ get_connection_array(void) } /** Set the event mask on <b>conn</b> to <b>events</b>. (The event - * mask is a bitmask whose bits are EV_READ and EV_WRITE.) + * mask is a bitmask whose bits are READ_EVENT and WRITE_EVENT) */ void -connection_watch_events(connection_t *conn, short events) +connection_watch_events(connection_t *conn, watchable_events_t events) { - if (events & EV_READ) + if (events & READ_EVENT) connection_start_reading(conn); else connection_stop_reading(conn); - if (events & EV_WRITE) + if (events & WRITE_EVENT) connection_start_writing(conn); else connection_stop_writing(conn); @@ -393,11 +451,11 @@ connection_start_reading_from_linked_conn(connection_t *conn) smartlist_add(active_linked_connection_lst, conn); if (!called_loop_once) { /* This is the first event on the list; we won't be in LOOP_ONCE mode, - * so we need to make sure that the event_loop() actually exits at the - * end of its run through the current connections and - * lets us activate read events for linked connections. */ + * so we need to make sure that the event_base_loop() actually exits at + * the end of its run through the current connections and lets us + * activate read events for linked connections. */ struct timeval tv = { 0, 0 }; - event_loopexit(&tv); + tor_event_base_loopexit(tor_libevent_get_base(), &tv); } } else { tor_assert(smartlist_isin(active_linked_connection_lst, conn)); @@ -540,7 +598,7 @@ conn_close_if_marked(int i) log_info(LD_NET, "Conn (addr %s, fd %d, type %s, state %d) marked, but wants " "to flush %d bytes. (Marked at %s:%d)", - escaped_safe_str(conn->address), + escaped_safe_str_client(conn->address), conn->s, conn_type_to_string(conn->type), conn->state, (int)conn->outbuf_flushlen, conn->marked_for_close_file, conn->marked_for_close); @@ -593,8 +651,8 @@ conn_close_if_marked(int i) "something is wrong with theirs. " "(fd %d, type %s, state %d, marked at %s:%d).", (int)buf_datalen(conn->outbuf), - escaped_safe_str(conn->address), conn->s, - conn_type_to_string(conn->type), conn->state, + escaped_safe_str_client(conn->address), + conn->s, conn_type_to_string(conn->type), conn->state, conn->marked_for_close_file, conn->marked_for_close); } @@ -623,7 +681,7 @@ directory_all_unreachable(time_t now) log_notice(LD_NET, "Is your network connection down? " "Failing connection to '%s:%d'.", - safe_str(edge_conn->socks_request->address), + safe_str_client(edge_conn->socks_request->address), edge_conn->socks_request->port); connection_mark_unattached_ap(edge_conn, END_STREAM_REASON_NET_UNREACHABLE); @@ -651,7 +709,7 @@ directory_info_has_arrived(time_t now, int from_cache) /* if we have enough dir info, then update our guard status with * whatever we just learned. */ - entry_guards_compute_status(); + entry_guards_compute_status(options, now); /* Don't even bother trying to get extrainfo until the rest of our * directory info is up-to-date */ if (options->DownloadExtraInfo) @@ -659,7 +717,7 @@ directory_info_has_arrived(time_t now, int from_cache) } if (server_mode(options) && !we_are_hibernating() && !from_cache && - (has_completed_circuit || !any_predicted_circuits(now))) + (can_complete_circuit || !any_predicted_circuits(now))) consider_testing_reachability(1, 1); } @@ -722,6 +780,7 @@ run_connection_housekeeping(int i, time_t now) the connection or send a keepalive, depending. */ or_conn = TO_OR_CONN(conn); + tor_assert(conn->outbuf); if (or_conn->is_bad_for_new_circs && !or_conn->n_circuits) { /* It's bad for new circuits, and has no unmarked circuits on it: @@ -778,11 +837,18 @@ run_connection_housekeeping(int i, time_t now) } } -/** Honor a NEWNYM request: make future requests unlinkability to past +/** Honor a NEWNYM request: make future requests unlinkable to past * requests. */ static void signewnym_impl(time_t now) { + or_options_t *options = get_options(); + if (!proxy_mode(options)) { + log_info(LD_CONTROL, "Ignoring SIGNAL NEWNYM because client functionality " + "is disabled."); + return; + } + circuit_expire_all_dirty_circs(); addressmap_clear_transient(); rend_cache_purge(); @@ -806,16 +872,19 @@ run_scheduled_events(time_t now) static time_t time_to_try_getting_descriptors = 0; static time_t time_to_reset_descriptor_failures = 0; static time_t time_to_add_entropy = 0; - static time_t time_to_write_hs_statistics = 0; static time_t time_to_write_bridge_status_file = 0; static time_t time_to_downrate_stability = 0; static time_t time_to_save_stability = 0; static time_t time_to_clean_caches = 0; static time_t time_to_recheck_bandwidth = 0; static time_t time_to_check_for_expired_networkstatus = 0; - static time_t time_to_dump_geoip_stats = 0; + static time_t time_to_write_stats_files = 0; + static time_t time_to_write_bridge_stats = 0; + static time_t time_to_launch_reachability_tests = 0; + static int should_init_bridge_stats = 1; static time_t time_to_retry_dns_init = 0; or_options_t *options = get_options(); + int is_server = server_mode(options); int i; int have_dir_info; @@ -833,11 +902,14 @@ run_scheduled_events(time_t now) signewnym_impl(now); } + /* 0c. If we've deferred log messages for the controller, handle them now */ + flush_pending_log_callbacks(); + /** 1a. Every MIN_ONION_KEY_LIFETIME seconds, rotate the onion keys, * shut down and restart all cpuworkers, and update the directory if * necessary. */ - if (server_mode(options) && + if (is_server && get_onion_key_set_at()+MIN_ONION_KEY_LIFETIME < now) { log_info(LD_GENERAL,"Rotating onion key."); rotate_onion_key(); @@ -852,8 +924,6 @@ run_scheduled_events(time_t now) if (time_to_try_getting_descriptors < now) { update_router_descriptor_downloads(now); update_extrainfo_downloads(now); - if (options->UseBridges) - fetch_bridge_descriptors(now); if (router_have_minimum_dir_info()) time_to_try_getting_descriptors = now + LAZY_DESCRIPTOR_RETRY_INTERVAL; else @@ -866,12 +936,18 @@ run_scheduled_events(time_t now) now + DESCRIPTOR_FAILURE_RESET_INTERVAL; } + if (options->UseBridges) + fetch_bridge_descriptors(options, now); + /** 1b. Every MAX_SSL_KEY_LIFETIME seconds, we change our TLS context. */ if (!last_rotated_x509_certificate) last_rotated_x509_certificate = now; if (last_rotated_x509_certificate+MAX_SSL_KEY_LIFETIME < now) { log_info(LD_GENERAL,"Rotating tls context."); - if (tor_tls_context_new(get_identity_key(), MAX_SSL_KEY_LIFETIME) < 0) { + if (tor_tls_context_init(public_server_mode(options), + get_tlsclient_identity_key(), + is_server ? get_server_identity_key() : NULL, + MAX_SSL_KEY_LIFETIME) < 0) { log_warn(LD_BUG, "Error reinitializing TLS context"); /* XXX is it a bug here, that we just keep going? -RD */ } @@ -896,10 +972,12 @@ run_scheduled_events(time_t now) if (accounting_is_enabled(options)) accounting_run_housekeeping(now); - if (now % 10 == 0 && (authdir_mode_tests_reachability(options)) && - !we_are_hibernating()) { + if (time_to_launch_reachability_tests < now && + (authdir_mode_tests_reachability(options)) && + !we_are_hibernating()) { + time_to_launch_reachability_tests = now + REACHABILITY_TEST_INTERVAL; /* try to determine reachability of the other Tor relays */ - dirserv_test_reachability(now, 0); + dirserv_test_reachability(now); } /** 1d. Periodically, we discount older stability information so that new @@ -941,11 +1019,56 @@ run_scheduled_events(time_t now) time_to_check_for_expired_networkstatus = now + CHECK_EXPIRED_NS_INTERVAL; } - if (time_to_dump_geoip_stats < now) { -#define DUMP_GEOIP_STATS_INTERVAL (60*60); - if (time_to_dump_geoip_stats) - dump_geoip_stats(); - time_to_dump_geoip_stats = now + DUMP_GEOIP_STATS_INTERVAL; + /* 1g. Check whether we should write statistics to disk. + */ + if (time_to_write_stats_files < now) { +#define CHECK_WRITE_STATS_INTERVAL (60*60) + time_t next_time_to_write_stats_files = (time_to_write_stats_files > 0 ? + time_to_write_stats_files : now) + CHECK_WRITE_STATS_INTERVAL; + if (options->CellStatistics) { + time_t next_write = + rep_hist_buffer_stats_write(time_to_write_stats_files); + if (next_write && next_write < next_time_to_write_stats_files) + next_time_to_write_stats_files = next_write; + } + if (options->DirReqStatistics) { + time_t next_write = geoip_dirreq_stats_write(time_to_write_stats_files); + if (next_write && next_write < next_time_to_write_stats_files) + next_time_to_write_stats_files = next_write; + } + if (options->EntryStatistics) { + time_t next_write = geoip_entry_stats_write(time_to_write_stats_files); + if (next_write && next_write < next_time_to_write_stats_files) + next_time_to_write_stats_files = next_write; + } + if (options->ExitPortStatistics) { + time_t next_write = rep_hist_exit_stats_write(time_to_write_stats_files); + if (next_write && next_write < next_time_to_write_stats_files) + next_time_to_write_stats_files = next_write; + } + time_to_write_stats_files = next_time_to_write_stats_files; + } + + /* 1h. Check whether we should write bridge statistics to disk. + */ + if (should_record_bridge_info(options)) { + if (time_to_write_bridge_stats < now) { + if (should_init_bridge_stats) { + /* (Re-)initialize bridge statistics. */ + geoip_bridge_stats_init(now); + time_to_write_bridge_stats = now + WRITE_STATS_INTERVAL; + should_init_bridge_stats = 0; + } else { + /* Possibly write bridge statistics to disk and ask when to write + * them next time. */ + time_to_write_bridge_stats = geoip_bridge_stats_write( + time_to_write_bridge_stats); + } + } + } else if (!should_init_bridge_stats) { + /* Bridge mode was turned off. Ensure that stats are re-initialized + * next time bridge mode is turned on. */ + should_init_bridge_stats = 1; } /* Remove old information from rephist and the rend cache. */ @@ -953,6 +1076,8 @@ run_scheduled_events(time_t now) rep_history_clean(now - options->RephistTrackTime); rend_cache_clean(); rend_cache_clean_v2_descs_as_dir(); + if (authdir_mode_v3(options)) + microdesc_cache_rebuild(NULL, 0); #define CLEAN_CACHES_INTERVAL (30*60) time_to_clean_caches = now + CLEAN_CACHES_INTERVAL; } @@ -993,7 +1118,7 @@ run_scheduled_events(time_t now) /* also, check religiously for reachability, if it's within the first * 20 minutes of our uptime. */ if (server_mode(options) && - (has_completed_circuit || !any_predicted_circuits(now)) && + (can_complete_circuit || !any_predicted_circuits(now)) && !we_are_hibernating()) { if (stats_n_seconds_working < TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT) { consider_testing_reachability(1, dirport_reachability_count==0); @@ -1036,7 +1161,9 @@ run_scheduled_events(time_t now) * We do this before step 4, so it can try building more if * it's not comfortable with the number of available circuits. */ - circuit_expire_building(now); + /* XXXX022 If our circuit build timeout is much lower than a second, maybe + we should do this more often? */ + circuit_expire_building(); /** 3b. Also look at pending streams and prune the ones that 'began' * a long time ago but haven't gotten a 'connected' yet. @@ -1069,7 +1196,7 @@ run_scheduled_events(time_t now) circuit_expire_old_circuits_serverside(now); /** 5. We do housekeeping for each connection... */ - connection_or_set_bad_connections(); + connection_or_set_bad_connections(NULL, 0); for (i=0;i<smartlist_len(connection_array);i++) { run_connection_housekeeping(i, now); } @@ -1092,7 +1219,7 @@ run_scheduled_events(time_t now) circuit_close_all_marked(); /** 7. And upload service descriptors if necessary. */ - if (has_completed_circuit && !we_are_hibernating()) { + if (can_complete_circuit && !we_are_hibernating()) { rend_consider_services_upload(now); rend_consider_descriptor_republication(); } @@ -1109,7 +1236,7 @@ run_scheduled_events(time_t now) /** 9. and if we're a server, check whether our DNS is telling stories to * us. */ - if (server_mode(options) && time_to_check_for_correct_dns < now) { + if (is_server && time_to_check_for_correct_dns < now) { if (!time_to_check_for_correct_dns) { time_to_check_for_correct_dns = now + 60 + crypto_rand_int(120); } else { @@ -1119,12 +1246,6 @@ run_scheduled_events(time_t now) } } - /** 10. write hidden service usage statistic to disk */ - if (options->HSAuthorityRecordStats && time_to_write_hs_statistics < now) { - hs_usage_write_statistics_to_file(now); -#define WRITE_HSUSAGE_INTERVAL (30*60) - time_to_write_hs_statistics = now+WRITE_HSUSAGE_INTERVAL; - } /** 10b. write bridge networkstatus file to disk */ if (options->BridgeAuthoritativeDir && time_to_write_bridge_status_file < now) { @@ -1134,39 +1255,30 @@ run_scheduled_events(time_t now) } } -/** Libevent timer: used to invoke second_elapsed_callback() once per - * second. */ -static struct event *timeout_event = NULL; +/** Timer: used to invoke second_elapsed_callback() once per second. */ +static periodic_timer_t *second_timer = NULL; /** Number of libevent errors in the last second: we die if we get too many. */ static int n_libevent_errors = 0; /** Libevent callback: invoked once every second. */ static void -second_elapsed_callback(int fd, short event, void *args) +second_elapsed_callback(periodic_timer_t *timer, void *arg) { /* XXXX This could be sensibly refactored into multiple callbacks, and we * could use Libevent's timers for this rather than checking the current * time against a bunch of timeouts every second. */ - static struct timeval one_second; static time_t current_second = 0; time_t now; size_t bytes_written; size_t bytes_read; int seconds_elapsed; or_options_t *options = get_options(); - (void)fd; - (void)event; - (void)args; - if (!timeout_event) { - timeout_event = tor_malloc_zero(sizeof(struct event)); - evtimer_set(timeout_event, second_elapsed_callback, NULL); - one_second.tv_sec = 1; - one_second.tv_usec = 0; - } + (void)timer; + (void)arg; n_libevent_errors = 0; - /* log_fn(LOG_NOTICE, "Tick."); */ + /* log_notice(LD_GENERAL, "Tick."); */ now = time(NULL); update_approx_time(now); @@ -1189,7 +1301,7 @@ second_elapsed_callback(int fd, short event, void *args) if (server_mode(options) && !we_are_hibernating() && seconds_elapsed > 0 && - has_completed_circuit && + can_complete_circuit && stats_n_seconds_working / TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT != (stats_n_seconds_working+seconds_elapsed) / TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT) { @@ -1231,18 +1343,6 @@ second_elapsed_callback(int fd, short event, void *args) run_scheduled_events(now); current_second = now; /* remember which second it is, for next time */ - -#if 0 - if (current_second % 300 == 0) { - rep_history_clean(current_second - options->RephistTrackTime); - dumpmemusage(get_min_log_level()<LOG_INFO ? - get_min_log_level() : LOG_INFO); - } -#endif - - if (evtimer_add(timeout_event, &one_second)) - log_err(LD_NET, - "Error from libevent when setting one-second timeout event"); } #ifndef MS_WINDOWS @@ -1281,7 +1381,7 @@ ip_address_changed(int at_interface) reset_bandwidth_test(); stats_n_seconds_working = 0; router_reset_reachability(); - mark_my_descriptor_dirty(); + mark_my_descriptor_dirty("IP address changed"); } } @@ -1318,7 +1418,6 @@ do_hup(void) router_reset_warnings(); routerlist_reset_warnings(); - addressmap_clear_transient(); /* first, reload config variables, in case they've changed */ if (options->ReloadTorrcOnSIGHUP) { /* no need to provide argc/v, they've been cached in init_from_config */ @@ -1385,7 +1484,7 @@ do_main_loop(void) /* load the private keys, if we're supposed to have them, and set up the * TLS context. */ - if (! identity_key_is_set()) { + if (! client_identity_key_is_set()) { if (init_keys() < 0) { log_err(LD_BUG,"Error initializing keys; exiting"); return -1; @@ -1403,8 +1502,10 @@ do_main_loop(void) /* initialize the bootstrap status events to know we're starting up */ control_event_bootstrap(BOOTSTRAP_STATUS_STARTING, 0); - if (trusted_dirs_reload_certs()) - return -1; + if (trusted_dirs_reload_certs()) { + log_warn(LD_DIR, + "Couldn't load all cached v3 certificates. Starting anyway."); + } if (router_reload_v2_networkstatus()) { return -1; } @@ -1421,18 +1522,23 @@ do_main_loop(void) now = time(NULL); directory_info_has_arrived(now, 1); - if (authdir_mode_tests_reachability(get_options())) { - /* the directory is already here, run startup things */ - dirserv_test_reachability(now, 1); - } - if (server_mode(get_options())) { /* launch cpuworkers. Need to do this *after* we've read the onion key. */ cpu_init(); } /* set up once-a-second callback. */ - second_elapsed_callback(0,0,NULL); + if (! second_timer) { + struct timeval one_second; + one_second.tv_sec = 1; + one_second.tv_usec = 0; + + second_timer = periodic_timer_new(tor_libevent_get_base(), + &one_second, + second_elapsed_callback, + NULL); + tor_assert(second_timer); + } for (;;) { if (nt_service_is_stopping()) @@ -1451,20 +1557,16 @@ do_main_loop(void) /* poll until we have an event, or the second ends, or until we have * some active linked connections to trigger events for. */ - loop_result = event_loop(called_loop_once ? EVLOOP_ONCE : 0); + loop_result = event_base_loop(tor_libevent_get_base(), + called_loop_once ? EVLOOP_ONCE : 0); /* let catch() handle things like ^c, and otherwise don't worry about it */ if (loop_result < 0) { int e = tor_socket_errno(-1); /* let the program survive things like ^z */ if (e != EINTR && !ERRNO_IS_EINPROGRESS(e)) { -#ifdef HAVE_EVENT_GET_METHOD log_err(LD_NET,"libevent call with %s failed: %s [%d]", - event_get_method(), tor_socket_strerror(e), e); -#else - log_err(LD_NET,"libevent call failed: %s [%d]", - tor_socket_strerror(e), e); -#endif + tor_libevent_get_method(), tor_socket_strerror(e), e); return -1; #ifndef MS_WINDOWS } else if (e == EINVAL) { @@ -1485,46 +1587,7 @@ do_main_loop(void) } } -/** Used to implement the SIGNAL control command: if we accept - * <b>the_signal</b> as a remote pseudo-signal, act on it. */ -/* We don't re-use catch() here because: - * 1. We handle a different set of signals than those allowed in catch. - * 2. Platforms without signal() are unlikely to define SIGfoo. - * 3. The control spec is defined to use fixed numeric signal values - * which just happen to match the Unix values. - */ -void -control_signal_act(int the_signal) -{ - switch (the_signal) - { - case 1: - signal_callback(0,0,(void*)(uintptr_t)SIGHUP); - break; - case 2: - signal_callback(0,0,(void*)(uintptr_t)SIGINT); - break; - case 10: - signal_callback(0,0,(void*)(uintptr_t)SIGUSR1); - break; - case 12: - signal_callback(0,0,(void*)(uintptr_t)SIGUSR2); - break; - case 15: - signal_callback(0,0,(void*)(uintptr_t)SIGTERM); - break; - case SIGNEWNYM: - signal_callback(0,0,(void*)(uintptr_t)SIGNEWNYM); - break; - case SIGCLEARDNSCACHE: - signal_callback(0,0,(void*)(uintptr_t)SIGCLEARDNSCACHE); - break; - default: - log_warn(LD_BUG, "Unrecognized signal number %d.", the_signal); - break; - } -} - +#ifndef MS_WINDOWS /* Only called when we're willing to use signals */ /** Libevent callback: invoked when we get a signal. */ static void @@ -1533,6 +1596,15 @@ signal_callback(int fd, short events, void *arg) uintptr_t sig = (uintptr_t)arg; (void)fd; (void)events; + + process_signal(sig); +} +#endif + +/** Do the work of acting on a signal received in <b>sig</b> */ +void +process_signal(uintptr_t sig) +{ switch (sig) { case SIGTERM: @@ -1607,6 +1679,7 @@ dumpmemusage(int severity) U64_PRINTF_ARG(rephist_total_alloc), rephist_total_num); dump_routerlist_mem_usage(severity); dump_cell_pool_usage(severity); + dump_dns_mem_usage(severity); buf_dump_freelist_sizes(severity); tor_log_mallinfo(severity); } @@ -1633,7 +1706,8 @@ dumpstats(int severity) if (!connection_is_listener(conn)) { log(severity,LD_GENERAL, "Conn %d is to %s:%d.", i, - safe_str(conn->address), conn->port); + safe_str_client(conn->address), + conn->port); log(severity,LD_GENERAL, "Conn %d: %d bytes waiting on inbuf (len %d, last read %d secs ago)", i, @@ -1730,7 +1804,7 @@ handle_signals(int is_parent) { #ifndef MS_WINDOWS /* do signal stuff only on Unix */ int i; - static int signals[] = { + static const int signals[] = { SIGINT, /* do a controlled slow shutdown */ SIGTERM, /* to terminate now */ SIGPIPE, /* otherwise SIGPIPE kills us */ @@ -1742,12 +1816,13 @@ handle_signals(int is_parent) #endif SIGCHLD, /* handle dns/cpu workers that exit */ -1 }; - static struct event signal_events[16]; /* bigger than it has to be. */ + static struct event *signal_events[16]; /* bigger than it has to be. */ if (is_parent) { for (i = 0; signals[i] >= 0; ++i) { - signal_set(&signal_events[i], signals[i], signal_callback, - (void*)(uintptr_t)signals[i]); - if (signal_add(&signal_events[i], NULL)) + signal_events[i] = tor_evsignal_new( + tor_libevent_get_base(), signals[i], signal_callback, + (void*)(uintptr_t)signals[i]); + if (event_add(signal_events[i], NULL)) log_warn(LD_BUG, "Error from libevent when adding event for signal %d", signals[i]); } @@ -1836,7 +1911,9 @@ tor_init(int argc, char *argv[]) "and you probably shouldn't."); #endif - if (crypto_global_init(get_options()->HardwareAccel)) { + if (crypto_global_init(get_options()->HardwareAccel, + get_options()->AccelName, + get_options()->AccelDir)) { log_err(LD_BUG, "Unable to initialize OpenSSL. Exiting."); return -1; } @@ -1930,7 +2007,6 @@ tor_free_all(int postfork) rend_cache_free_all(); rend_service_authorization_free_all(); rep_hist_free_all(); - hs_usage_free_all(); dns_free_all(); clear_pending_onions(); circuit_free_all(); @@ -1938,6 +2014,7 @@ tor_free_all(int postfork) connection_free_all(); buf_shrink_freelists(1); memarea_clear_freelist(); + microdesc_free_all(); if (!postfork) { config_free_all(); router_free_all(); @@ -1948,13 +2025,11 @@ tor_free_all(int postfork) tor_tls_free_all(); } /* stuff in main.c */ - if (connection_array) - smartlist_free(connection_array); - if (closeable_connection_lst) - smartlist_free(closeable_connection_lst); - if (active_linked_connection_lst) - smartlist_free(active_linked_connection_lst); - tor_free(timeout_event); + + smartlist_free(connection_array); + smartlist_free(closeable_connection_lst); + smartlist_free(active_linked_connection_lst); + periodic_timer_free(second_timer); if (!postfork) { release_lockfile(); } @@ -1971,12 +2046,14 @@ void tor_cleanup(void) { or_options_t *options = get_options(); - /* Remove our pid file. We don't care if there was an error when we - * unlink, nothing we could do about it anyways. */ if (options->command == CMD_RUN_TOR) { time_t now = time(NULL); + /* Remove our pid file. We don't care if there was an error when we + * unlink, nothing we could do about it anyways. */ if (options->PidFile) unlink(options->PidFile); + if (options->ControlPortWriteToFile) + unlink(options->ControlPortWriteToFile); if (accounting_is_enabled(options)) accounting_record_bandwidth_usage(now, get_or_state()); or_state_mark_dirty(get_or_state(), 0); /* force an immediate save. */ @@ -2015,7 +2092,7 @@ do_list_fingerprint(void) log_err(LD_BUG,"Error initializing keys; can't display fingerprint"); return -1; } - if (!(k = get_identity_key())) { + if (!(k = get_server_identity_key())) { log_err(LD_GENERAL,"Error: missing identity key."); return -1; } @@ -2045,6 +2122,31 @@ do_hash_password(void) printf("16:%s\n",output); } +#if defined (WINCE) +int +find_flashcard_path(PWCHAR path, size_t size) +{ + WIN32_FIND_DATA d = {0}; + HANDLE h = NULL; + + if (!path) + return -1; + + h = FindFirstFlashCard(&d); + if (h == INVALID_HANDLE_VALUE) + return -1; + + if (wcslen(d.cFileName) == 0) { + FindClose(h); + return -1; + } + + wcsncpy(path,d.cFileName,size); + FindClose(h); + return 0; +} +#endif + /** Main entry point for the Tor process. Called from main(). */ /* This function is distinct from main() only so we can link main.c into * the unittest binary without conflicting with the unittests' main. */ @@ -2052,6 +2154,46 @@ int tor_main(int argc, char *argv[]) { int result = 0; +#if defined (WINCE) + WCHAR path [MAX_PATH] = {0}; + WCHAR fullpath [MAX_PATH] = {0}; + PWCHAR p = NULL; + FILE* redir = NULL; + FILE* redirdbg = NULL; + + // this is to facilitate debugging by opening + // a file on a folder shared by the wm emulator. + // if no flashcard (real or emulated) is present, + // log files will be written in the root folder + if (find_flashcard_path(path,MAX_PATH) == -1) + { + redir = _wfreopen( L"\\stdout.log", L"w", stdout ); + redirdbg = _wfreopen( L"\\stderr.log", L"w", stderr ); + } else { + swprintf(fullpath,L"\\%s\\tor",path); + CreateDirectory(fullpath,NULL); + + swprintf(fullpath,L"\\%s\\tor\\stdout.log",path); + redir = _wfreopen( fullpath, L"w", stdout ); + + swprintf(fullpath,L"\\%s\\tor\\stderr.log",path); + redirdbg = _wfreopen( fullpath, L"w", stderr ); + } +#endif + +#ifdef MS_WINDOWS + /* Call SetProcessDEPPolicy to permanently enable DEP. + The function will not resolve on earlier versions of Windows, + and failure is not dangerous. */ + HMODULE hMod = GetModuleHandleA("Kernel32.dll"); + if (hMod) { + typedef BOOL (WINAPI *PSETDEP)(DWORD); + PSETDEP setdeppolicy = (PSETDEP)GetProcAddress(hMod, + "SetProcessDEPPolicy"); + if (setdeppolicy) setdeppolicy(1); /* PROCESS_DEP_ENABLE */ + } +#endif + update_approx_time(time(NULL)); tor_threads_init(); init_logging(); diff --git a/src/or/main.h b/src/or/main.h new file mode 100644 index 0000000000..0551f7aaf9 --- /dev/null +++ b/src/or/main.h @@ -0,0 +1,69 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file main.h + * \brief Header file for main.c. + **/ + +#ifndef _TOR_MAIN_H +#define _TOR_MAIN_H + +extern int can_complete_circuit; + +int connection_add(connection_t *conn); +int connection_remove(connection_t *conn); +void connection_unregister_events(connection_t *conn); +int connection_in_array(connection_t *conn); +void add_connection_to_closeable_list(connection_t *conn); +int connection_is_on_closeable_list(connection_t *conn); + +smartlist_t *get_connection_array(void); + +/** Bitmask for events that we can turn on and off with + * connection_watch_events. */ +typedef enum watchable_events { + READ_EVENT=0x02, /**< We want to know when a connection is readable */ + WRITE_EVENT=0x04 /**< We want to know when a connection is writable */ +} watchable_events_t; +void connection_watch_events(connection_t *conn, watchable_events_t events); +int connection_is_reading(connection_t *conn); +void connection_stop_reading(connection_t *conn); +void connection_start_reading(connection_t *conn); + +int connection_is_writing(connection_t *conn); +void connection_stop_writing(connection_t *conn); +void connection_start_writing(connection_t *conn); + +void connection_stop_reading_from_linked_conn(connection_t *conn); + +void directory_all_unreachable(time_t now); +void directory_info_has_arrived(time_t now, int from_cache); + +void ip_address_changed(int at_interface); +void dns_servers_relaunch_checks(void); + +void handle_signals(int is_parent); +void process_signal(uintptr_t sig); + +int try_locking(or_options_t *options, int err_if_locked); +int have_lockfile(void); +void release_lockfile(void); + +void tor_cleanup(void); +void tor_free_all(int postfork); + +int tor_main(int argc, char *argv[]); + +#ifdef MAIN_PRIVATE +int do_main_loop(void); +int do_list_fingerprint(void); +void do_hash_password(void); +int tor_init(int argc, char **argv); +#endif + +#endif + diff --git a/src/or/microdesc.c b/src/or/microdesc.c new file mode 100644 index 0000000000..7c67d51448 --- /dev/null +++ b/src/or/microdesc.c @@ -0,0 +1,493 @@ +/* Copyright (c) 2009-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "or.h" +#include "config.h" +#include "microdesc.h" +#include "routerparse.h" + +/** A data structure to hold a bunch of cached microdescriptors. There are + * two active files in the cache: a "cache file" that we mmap, and a "journal + * file" that we append to. Periodically, we rebuild the cache file to hold + * only the microdescriptors that we want to keep */ +struct microdesc_cache_t { + /** Map from sha256-digest to microdesc_t for every microdesc_t in the + * cache. */ + HT_HEAD(microdesc_map, microdesc_t) map; + + /** Name of the cache file. */ + char *cache_fname; + /** Name of the journal file. */ + char *journal_fname; + /** Mmap'd contents of the cache file, or NULL if there is none. */ + tor_mmap_t *cache_content; + /** Number of bytes used in the journal file. */ + size_t journal_len; + /** Number of bytes in descriptors removed as too old. */ + size_t bytes_dropped; + + /** Total bytes of microdescriptor bodies we have added to this cache */ + uint64_t total_len_seen; + /** Total number of microdescriptors we have added to this cache */ + unsigned n_seen; +}; + +/** Helper: computes a hash of <b>md</b> to place it in a hash table. */ +static INLINE unsigned int +_microdesc_hash(microdesc_t *md) +{ + unsigned *d = (unsigned*)md->digest; +#if SIZEOF_INT == 4 + return d[0] ^ d[1] ^ d[2] ^ d[3] ^ d[4] ^ d[5] ^ d[6] ^ d[7]; +#else + return d[0] ^ d[1] ^ d[2] ^ d[3]; +#endif +} + +/** Helper: compares <b>a</b> and </b> for equality for hash-table purposes. */ +static INLINE int +_microdesc_eq(microdesc_t *a, microdesc_t *b) +{ + return tor_memeq(a->digest, b->digest, DIGEST256_LEN); +} + +HT_PROTOTYPE(microdesc_map, microdesc_t, node, + _microdesc_hash, _microdesc_eq); +HT_GENERATE(microdesc_map, microdesc_t, node, + _microdesc_hash, _microdesc_eq, 0.6, + malloc, realloc, free); + +/** Write the body of <b>md</b> into <b>f</b>, with appropriate annotations. + * On success, return the total number of bytes written, and set + * *<b>annotation_len_out</b> to the number of bytes written as + * annotations. */ +static ssize_t +dump_microdescriptor(FILE *f, microdesc_t *md, size_t *annotation_len_out) +{ + ssize_t r = 0; + size_t written; + /* XXXX drops unkown annotations. */ + if (md->last_listed) { + char buf[ISO_TIME_LEN+1]; + char annotation[ISO_TIME_LEN+32]; + format_iso_time(buf, md->last_listed); + tor_snprintf(annotation, sizeof(annotation), "@last-listed %s\n", buf); + fputs(annotation, f); + r += strlen(annotation); + *annotation_len_out = r; + } else { + *annotation_len_out = 0; + } + + md->off = (off_t) ftell(f); + written = fwrite(md->body, 1, md->bodylen, f); + if (written != md->bodylen) { + log_warn(LD_DIR, + "Couldn't dump microdescriptor (wrote %lu out of %lu): %s", + (unsigned long)written, (unsigned long)md->bodylen, + strerror(ferror(f))); + return -1; + } + r += md->bodylen; + return r; +} + +/** Holds a pointer to the current microdesc_cache_t object, or NULL if no + * such object has been allocated. */ +static microdesc_cache_t *the_microdesc_cache = NULL; + +/** Return a pointer to the microdescriptor cache, loading it if necessary. */ +microdesc_cache_t * +get_microdesc_cache(void) +{ + if (PREDICT_UNLIKELY(the_microdesc_cache==NULL)) { + microdesc_cache_t *cache = tor_malloc_zero(sizeof(microdesc_cache_t)); + HT_INIT(microdesc_map, &cache->map); + cache->cache_fname = get_datadir_fname("cached-microdescs"); + cache->journal_fname = get_datadir_fname("cached-microdescs.new"); + microdesc_cache_reload(cache); + the_microdesc_cache = cache; + } + return the_microdesc_cache; +} + +/* There are three sources of microdescriptors: + 1) Generated by us while acting as a directory authority. + 2) Loaded from the cache on disk. + 3) Downloaded. +*/ + +/** Decode the microdescriptors from the string starting at <b>s</b> and + * ending at <b>eos</b>, and store them in <b>cache</b>. If <b>no-save</b>, + * mark them as non-writable to disk. If <b>where</b> is SAVED_IN_CACHE, + * leave their bodies as pointers to the mmap'd cache. If where is + * <b>SAVED_NOWHERE</b>, do not allow annotations. Return a list of the added + * microdescriptors. */ +smartlist_t * +microdescs_add_to_cache(microdesc_cache_t *cache, + const char *s, const char *eos, saved_location_t where, + int no_save) +{ + /*XXXX need an argument that sets last_listed as appropriate. */ + + smartlist_t *descriptors, *added; + const int allow_annotations = (where != SAVED_NOWHERE); + const int copy_body = (where != SAVED_IN_CACHE); + + descriptors = microdescs_parse_from_string(s, eos, + allow_annotations, + copy_body); + + added = microdescs_add_list_to_cache(cache, descriptors, where, no_save); + smartlist_free(descriptors); + return added; +} + +/* As microdescs_add_to_cache, but takes a list of micrdescriptors instead of + * a string to encode. Frees any members of <b>descriptors</b> that it does + * not add. */ +smartlist_t * +microdescs_add_list_to_cache(microdesc_cache_t *cache, + smartlist_t *descriptors, saved_location_t where, + int no_save) +{ + smartlist_t *added; + open_file_t *open_file = NULL; + FILE *f = NULL; + // int n_added = 0; + ssize_t size = 0; + + if (where == SAVED_NOWHERE && !no_save) { + f = start_writing_to_stdio_file(cache->journal_fname, + OPEN_FLAGS_APPEND|O_BINARY, + 0600, &open_file); + if (!f) { + log_warn(LD_DIR, "Couldn't append to journal in %s: %s", + cache->journal_fname, strerror(errno)); + return NULL; + } + } + + added = smartlist_create(); + SMARTLIST_FOREACH_BEGIN(descriptors, microdesc_t *, md) { + microdesc_t *md2; + md2 = HT_FIND(microdesc_map, &cache->map, md); + if (md2) { + /* We already had this one. */ + if (md2->last_listed < md->last_listed) + md2->last_listed = md->last_listed; + microdesc_free(md); + if (where != SAVED_NOWHERE) + cache->bytes_dropped += size; + continue; + } + + /* Okay, it's a new one. */ + if (f) { + size_t annotation_len; + size = dump_microdescriptor(f, md, &annotation_len); + if (size < 0) { + /* XXX handle errors from dump_microdescriptor() */ + /* log? return -1? die? coredump the universe? */ + continue; + } + md->saved_location = SAVED_IN_JOURNAL; + cache->journal_len += size; + } else { + md->saved_location = where; + } + + md->no_save = no_save; + + HT_INSERT(microdesc_map, &cache->map, md); + smartlist_add(added, md); + ++cache->n_seen; + cache->total_len_seen += md->bodylen; + } SMARTLIST_FOREACH_END(md); + + if (f) + finish_writing_to_file(open_file); /*XXX Check me.*/ + + return added; +} + +/** Remove every microdescriptor in <b>cache</b>. */ +void +microdesc_cache_clear(microdesc_cache_t *cache) +{ + microdesc_t **entry, **next; + for (entry = HT_START(microdesc_map, &cache->map); entry; entry = next) { + microdesc_t *md = *entry; + next = HT_NEXT_RMV(microdesc_map, &cache->map, entry); + microdesc_free(md); + } + HT_CLEAR(microdesc_map, &cache->map); + if (cache->cache_content) { + tor_munmap_file(cache->cache_content); + cache->cache_content = NULL; + } + cache->total_len_seen = 0; + cache->n_seen = 0; + cache->bytes_dropped = 0; +} + +/** Reload the contents of <b>cache</b> from disk. If it is empty, load it + * for the first time. Return 0 on success, -1 on failure. */ +int +microdesc_cache_reload(microdesc_cache_t *cache) +{ + struct stat st; + char *journal_content; + smartlist_t *added; + tor_mmap_t *mm; + int total = 0; + + microdesc_cache_clear(cache); + + mm = cache->cache_content = tor_mmap_file(cache->cache_fname); + if (mm) { + added = microdescs_add_to_cache(cache, mm->data, mm->data+mm->size, + SAVED_IN_CACHE, 0); + if (added) { + total += smartlist_len(added); + smartlist_free(added); + } + } + + journal_content = read_file_to_str(cache->journal_fname, + RFTS_IGNORE_MISSING, &st); + if (journal_content) { + cache->journal_len = (size_t) st.st_size; + added = microdescs_add_to_cache(cache, journal_content, + journal_content+st.st_size, + SAVED_IN_JOURNAL, 0); + if (added) { + total += smartlist_len(added); + smartlist_free(added); + } + tor_free(journal_content); + } + log_notice(LD_DIR, "Reloaded microdescriptor cache. Found %d descriptors.", + total); + + microdesc_cache_rebuild(cache, 0 /* don't force */); + + return 0; +} + +/** By default, we remove any microdescriptors that have gone at least this + * long without appearing in a current consensus. */ +#define TOLERATE_MICRODESC_AGE (7*24*60*60) + +/** Remove all microdescriptors from <b>cache</b> that haven't been listed for + * a long time. Does not rebuild the cache on disk. If <b>cutoff</b> is + * positive, specifically remove microdescriptors that have been unlisted + * since <b>cutoff</b>. If <b>force</b> is true, remove microdescriptors even + * if we have no current live microdescriptor consensus. + */ +void +microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force) +{ + microdesc_t **mdp, *victim; + int dropped=0, kept=0; + size_t bytes_dropped = 0; + time_t now = time(NULL); + + (void) force; + /* In 0.2.2, we let this proceed unconditionally: only authorities have + * microdesc caches. */ + + if (cutoff <= 0) + cutoff = now - TOLERATE_MICRODESC_AGE; + + for (mdp = HT_START(microdesc_map, &cache->map); mdp != NULL; ) { + if ((*mdp)->last_listed < cutoff) { + ++dropped; + victim = *mdp; + mdp = HT_NEXT_RMV(microdesc_map, &cache->map, mdp); + bytes_dropped += victim->bodylen; + microdesc_free(victim); + } else { + ++kept; + mdp = HT_NEXT(microdesc_map, &cache->map, mdp); + } + } + + if (dropped) { + log_notice(LD_DIR, "Removed %d/%d microdescriptors as old.", + dropped,dropped+kept); + cache->bytes_dropped += bytes_dropped; + } +} + +static int +should_rebuild_md_cache(microdesc_cache_t *cache) +{ + const size_t old_len = + cache->cache_content ? cache->cache_content->size : 0; + const size_t journal_len = cache->journal_len; + const size_t dropped = cache->bytes_dropped; + + if (journal_len < 16384) + return 0; /* Don't bother, not enough has happened yet. */ + if (dropped > (journal_len + old_len) / 3) + return 1; /* We could save 1/3 or more of the currently used space. */ + if (journal_len > old_len / 2) + return 1; /* We should append to the regular file */ + + return 0; +} + +/** Regenerate the main cache file for <b>cache</b>, clear the journal file, + * and update every microdesc_t in the cache with pointers to its new + * location. If <b>force</b> is true, do this unconditionally. If + * <b>force</b> is false, do it only if we expect to save space on disk. */ +int +microdesc_cache_rebuild(microdesc_cache_t *cache, int force) +{ + open_file_t *open_file; + FILE *f; + microdesc_t **mdp; + smartlist_t *wrote; + ssize_t size; + off_t off = 0; + int orig_size, new_size; + + if (cache == NULL) { + cache = the_microdesc_cache; + if (cache == NULL) + return 0; + } + + /* Remove dead descriptors */ + microdesc_cache_clean(cache, 0/*cutoff*/, 0/*force*/); + + if (!force && !should_rebuild_md_cache(cache)) + return 0; + + log_info(LD_DIR, "Rebuilding the microdescriptor cache..."); + + orig_size = (int)(cache->cache_content ? cache->cache_content->size : 0); + orig_size += (int)cache->journal_len; + + f = start_writing_to_stdio_file(cache->cache_fname, + OPEN_FLAGS_REPLACE|O_BINARY, + 0600, &open_file); + if (!f) + return -1; + + wrote = smartlist_create(); + + HT_FOREACH(mdp, microdesc_map, &cache->map) { + microdesc_t *md = *mdp; + size_t annotation_len; + if (md->no_save) + continue; + + size = dump_microdescriptor(f, md, &annotation_len); + if (size < 0) { + /* XXX handle errors from dump_microdescriptor() */ + /* log? return -1? die? coredump the universe? */ + continue; + } + md->off = off + annotation_len; + off += size; + if (md->saved_location != SAVED_IN_CACHE) { + tor_free(md->body); + md->saved_location = SAVED_IN_CACHE; + } + smartlist_add(wrote, md); + } + + if (cache->cache_content) + tor_munmap_file(cache->cache_content); + + finish_writing_to_file(open_file); /*XXX Check me.*/ + + cache->cache_content = tor_mmap_file(cache->cache_fname); + + if (!cache->cache_content && smartlist_len(wrote)) { + log_err(LD_DIR, "Couldn't map file that we just wrote to %s!", + cache->cache_fname); + smartlist_free(wrote); + return -1; + } + SMARTLIST_FOREACH_BEGIN(wrote, microdesc_t *, md) { + tor_assert(md->saved_location == SAVED_IN_CACHE); + md->body = (char*)cache->cache_content->data + md->off; + tor_assert(fast_memeq(md->body, "onion-key", 9)); + } SMARTLIST_FOREACH_END(md); + + smartlist_free(wrote); + + write_str_to_file(cache->journal_fname, "", 1); + cache->journal_len = 0; + cache->bytes_dropped = 0; + + new_size = cache->cache_content ? (int)cache->cache_content->size : 0; + log_info(LD_DIR, "Done rebuilding microdesc cache. " + "Saved %d bytes; %d still used.", + orig_size-new_size, new_size); + + return 0; +} + +/** Deallocate a single microdescriptor. Note: the microdescriptor MUST have + * previously been removed from the cache if it had ever been inserted. */ +void +microdesc_free(microdesc_t *md) +{ + if (!md) + return; + /* Must be removed from hash table! */ + if (md->onion_pkey) + crypto_free_pk_env(md->onion_pkey); + if (md->body && md->saved_location != SAVED_IN_CACHE) + tor_free(md->body); + + if (md->family) { + SMARTLIST_FOREACH(md->family, char *, cp, tor_free(cp)); + smartlist_free(md->family); + } + tor_free(md->exitsummary); + + tor_free(md); +} + +/** Free all storage held in the microdesc.c module. */ +void +microdesc_free_all(void) +{ + if (the_microdesc_cache) { + microdesc_cache_clear(the_microdesc_cache); + tor_free(the_microdesc_cache->cache_fname); + tor_free(the_microdesc_cache->journal_fname); + tor_free(the_microdesc_cache); + } +} + +/** If there is a microdescriptor in <b>cache</b> whose sha256 digest is + * <b>d</b>, return it. Otherwise return NULL. */ +microdesc_t * +microdesc_cache_lookup_by_digest256(microdesc_cache_t *cache, const char *d) +{ + microdesc_t *md, search; + if (!cache) + cache = get_microdesc_cache(); + memcpy(search.digest, d, DIGEST256_LEN); + md = HT_FIND(microdesc_map, &cache->map, &search); + return md; +} + +/** Return the mean size of decriptors added to <b>cache</b> since it was last + * cleared. Used to estimate the size of large downloads. */ +size_t +microdesc_average_size(microdesc_cache_t *cache) +{ + if (!cache) + cache = get_microdesc_cache(); + if (!cache->n_seen) + return 512; + return (size_t)(cache->total_len_seen / cache->n_seen); +} + diff --git a/src/or/microdesc.h b/src/or/microdesc.h new file mode 100644 index 0000000000..77ce8536bc --- /dev/null +++ b/src/or/microdesc.h @@ -0,0 +1,38 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file microdesc.h + * \brief Header file for microdesc.c. + **/ + +#ifndef _TOR_MICRODESC_H +#define _TOR_MICRODESC_H + +microdesc_cache_t *get_microdesc_cache(void); + +smartlist_t *microdescs_add_to_cache(microdesc_cache_t *cache, + const char *s, const char *eos, saved_location_t where, + int no_save); +smartlist_t *microdescs_add_list_to_cache(microdesc_cache_t *cache, + smartlist_t *descriptors, saved_location_t where, + int no_save); + +void microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force); +int microdesc_cache_rebuild(microdesc_cache_t *cache, int force); +int microdesc_cache_reload(microdesc_cache_t *cache); +void microdesc_cache_clear(microdesc_cache_t *cache); + +microdesc_t *microdesc_cache_lookup_by_digest256(microdesc_cache_t *cache, + const char *d); + +size_t microdesc_average_size(microdesc_cache_t *cache); + +void microdesc_free(microdesc_t *md); +void microdesc_free_all(void); + +#endif + diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c index 34d1e6ec98..1aa4e4a239 100644 --- a/src/or/networkstatus.c +++ b/src/or/networkstatus.c @@ -11,6 +11,20 @@ */ #include "or.h" +#include "circuitbuild.h" +#include "config.h" +#include "connection.h" +#include "connection_or.h" +#include "control.h" +#include "directory.h" +#include "dirserv.h" +#include "dirvote.h" +#include "main.h" +#include "networkstatus.h" +#include "relay.h" +#include "router.h" +#include "routerlist.h" +#include "routerparse.h" /* For tracking v2 networkstatus documents. Only caches do this now. */ @@ -35,16 +49,22 @@ static networkstatus_t *current_consensus = NULL; /** A v3 consensus networkstatus that we've received, but which we don't * have enough certificates to be happy about. */ -static networkstatus_t *consensus_waiting_for_certs = NULL; -/** The encoded version of consensus_waiting_for_certs. */ -static char *consensus_waiting_for_certs_body = NULL; -/** When did we set the current value of consensus_waiting_for_certs? If this - * is too recent, we shouldn't try to fetch a new consensus for a little while, - * to give ourselves time to get certificates for this one. */ -static time_t consensus_waiting_for_certs_set_at = 0; -/** Set to 1 if we've been holding on to consensus_waiting_for_certs so long - * that we should treat it as maybe being bad. */ -static int consensus_waiting_for_certs_dl_failed = 0; +typedef struct consensus_waiting_for_certs_t { + /** The consensus itself. */ + networkstatus_t *consensus; + /** The encoded version of the consensus, nul-terminated. */ + char *body; + /** When did we set the current value of consensus_waiting_for_certs? If + * this is too recent, we shouldn't try to fetch a new consensus for a + * little while, to give ourselves time to get certificates for this one. */ + time_t set_at; + /** Set to 1 if we've been holding on to it for so long we should maybe + * treat it as being bad. */ + int dl_failed; +} consensus_waiting_for_certs_t; + +static consensus_waiting_for_certs_t + consensus_waiting_for_certs[N_CONSENSUS_FLAVORS]; /** The last time we tried to download a networkstatus, or 0 for "never". We * use this to rate-limit download attempts for directory caches (including @@ -56,7 +76,7 @@ static time_t last_networkstatus_download_attempted = 0; * before the current consensus becomes invalid. */ static time_t time_to_download_next_consensus = 0; /** Download status for the current consensus networkstatus. */ -static download_status_t consensus_dl_status = { 0, 0, DL_SCHED_CONSENSUS }; +static download_status_t consensus_dl_status[N_CONSENSUS_FLAVORS]; /** True iff we have logged a warning about this OR's version being older than * listed by the authorities. */ @@ -89,6 +109,7 @@ networkstatus_reset_warnings(void) void networkstatus_reset_download_failures(void) { + int i; const smartlist_t *networkstatus_v2_list = networkstatus_get_v2_list(); SMARTLIST_FOREACH(networkstatus_v2_list, networkstatus_v2_t *, ns, SMARTLIST_FOREACH(ns->entries, routerstatus_t *, rs, @@ -97,7 +118,8 @@ networkstatus_reset_download_failures(void) rs->need_to_mirror = 1; }));; - download_status_reset(&consensus_dl_status); + for (i=0; i < N_CONSENSUS_FLAVORS; ++i) + download_status_reset(&consensus_dl_status[i]); if (v2_download_status_map) { digestmap_iter_t *iter; digestmap_t *map = v2_download_status_map; @@ -170,7 +192,7 @@ router_reload_v2_networkstatus(void) return 0; } -/** Read the cached v3 consensus networkstatus from the disk. */ +/** Read every cached v3 consensus networkstatus from the disk. */ int router_reload_consensus_networkstatus(void) { @@ -179,31 +201,46 @@ router_reload_consensus_networkstatus(void) struct stat st; or_options_t *options = get_options(); const unsigned int flags = NSSET_FROM_CACHE | NSSET_DONT_DOWNLOAD_CERTS; + int flav; /* FFFF Suppress warnings if cached consensus is bad? */ + for (flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) { + char buf[128]; + const char *flavor = networkstatus_get_flavor_name(flav); + if (flav == FLAV_NS) { + filename = get_datadir_fname("cached-consensus"); + } else { + tor_snprintf(buf, sizeof(buf), "cached-%s-consensus", flavor); + filename = get_datadir_fname(buf); + } + s = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL); + if (s) { + if (networkstatus_set_current_consensus(s, flavor, flags) < -1) { + log_warn(LD_FS, "Couldn't load consensus %s networkstatus from \"%s\"", + flavor, filename); + } + tor_free(s); + } + tor_free(filename); - filename = get_datadir_fname("cached-consensus"); - s = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL); - if (s) { - if (networkstatus_set_current_consensus(s, flags) < -1) { - log_warn(LD_FS, "Couldn't load consensus networkstatus from \"%s\"", - filename); + if (flav == FLAV_NS) { + filename = get_datadir_fname("unverified-consensus"); + } else { + tor_snprintf(buf, sizeof(buf), "unverified-%s-consensus", flavor); + filename = get_datadir_fname(buf); } - tor_free(s); - } - tor_free(filename); - filename = get_datadir_fname("unverified-consensus"); - s = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL); - if (s) { - if (networkstatus_set_current_consensus(s, + s = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL); + if (s) { + if (networkstatus_set_current_consensus(s, flavor, flags|NSSET_WAS_WAITING_FOR_CERTS)) { - log_info(LD_FS, "Couldn't load consensus networkstatus from \"%s\"", - filename); + log_info(LD_FS, "Couldn't load consensus %s networkstatus from \"%s\"", + flavor, filename); } - tor_free(s); + tor_free(s); + } + tor_free(filename); } - tor_free(filename); if (!current_consensus || (stat(options->FallbackNetworkstatusFile, &st)==0 && @@ -211,7 +248,7 @@ router_reload_consensus_networkstatus(void) s = read_file_to_str(options->FallbackNetworkstatusFile, RFTS_IGNORE_MISSING, NULL); if (s) { - if (networkstatus_set_current_consensus(s, + if (networkstatus_set_current_consensus(s, "ns", flags|NSSET_ACCEPT_OBSOLETE)) { log_info(LD_FS, "Couldn't load consensus networkstatus from \"%s\"", options->FallbackNetworkstatusFile); @@ -242,8 +279,16 @@ router_reload_consensus_networkstatus(void) static void vote_routerstatus_free(vote_routerstatus_t *rs) { + vote_microdesc_hash_t *h, *next; + if (!rs) + return; tor_free(rs->version); tor_free(rs->status.exitsummary); + for (h = rs->microdesc; h; h = next) { + tor_free(h->microdesc_hash_line); + next = h->next; + tor_free(h); + } tor_free(rs); } @@ -251,6 +296,8 @@ vote_routerstatus_free(vote_routerstatus_t *rs) void routerstatus_free(routerstatus_t *rs) { + if (!rs) + return; tor_free(rs->exitsummary); tor_free(rs); } @@ -259,6 +306,8 @@ routerstatus_free(routerstatus_t *rs) void networkstatus_v2_free(networkstatus_v2_t *ns) { + if (!ns) + return; tor_free(ns->source_address); tor_free(ns->contact); if (ns->signing_key) @@ -273,7 +322,25 @@ networkstatus_v2_free(networkstatus_v2_t *ns) tor_free(ns); } -/** Clear all storage held in <b>ns</b>. */ +/** Free all storage held in <b>sig</b> */ +void +document_signature_free(document_signature_t *sig) +{ + tor_free(sig->signature); + tor_free(sig); +} + +/** Return a newly allocated copy of <b>sig</b> */ +document_signature_t * +document_signature_dup(const document_signature_t *sig) +{ + document_signature_t *r = tor_memdup(sig, sizeof(document_signature_t)); + if (r->signature) + r->signature = tor_memdup(sig->signature, sig->signature_len); + return r; +} + +/** Free all storage held in <b>ns</b>. */ void networkstatus_vote_free(networkstatus_t *ns) { @@ -286,6 +353,10 @@ networkstatus_vote_free(networkstatus_t *ns) SMARTLIST_FOREACH(ns->known_flags, char *, c, tor_free(c)); smartlist_free(ns->known_flags); } + if (ns->weight_params) { + SMARTLIST_FOREACH(ns->weight_params, char *, c, tor_free(c)); + smartlist_free(ns->weight_params); + } if (ns->net_params) { SMARTLIST_FOREACH(ns->net_params, char *, c, tor_free(c)); smartlist_free(ns->net_params); @@ -295,18 +366,20 @@ networkstatus_vote_free(networkstatus_t *ns) smartlist_free(ns->supported_methods); } if (ns->voters) { - SMARTLIST_FOREACH(ns->voters, networkstatus_voter_info_t *, voter, - { + SMARTLIST_FOREACH_BEGIN(ns->voters, networkstatus_voter_info_t *, voter) { tor_free(voter->nickname); tor_free(voter->address); tor_free(voter->contact); - tor_free(voter->signature); + if (voter->sigs) { + SMARTLIST_FOREACH(voter->sigs, document_signature_t *, sig, + document_signature_free(sig)); + smartlist_free(voter->sigs); + } tor_free(voter); - }); + } SMARTLIST_FOREACH_END(voter); smartlist_free(ns->voters); } - if (ns->cert) - authority_cert_free(ns->cert); + authority_cert_free(ns->cert); if (ns->routerstatus_list) { if (ns->type == NS_TYPE_VOTE || ns->type == NS_TYPE_OPINION) { @@ -319,8 +392,8 @@ networkstatus_vote_free(networkstatus_t *ns) smartlist_free(ns->routerstatus_list); } - if (ns->desc_digest_map) - digestmap_free(ns->desc_digest_map, NULL); + + digestmap_free(ns->desc_digest_map, NULL); memset(ns, 11, sizeof(*ns)); tor_free(ns); @@ -341,35 +414,39 @@ networkstatus_get_voter_by_id(networkstatus_t *vote, return NULL; } -/** Check whether the signature on <b>voter</b> is correctly signed by - * the signing key of <b>cert</b>. Return -1 if <b>cert</b> doesn't match the +/** Check whether the signature <b>sig</b> is correctly signed with the + * signing key in <b>cert</b>. Return -1 if <b>cert</b> doesn't match the * signing key; otherwise set the good_signature or bad_signature flag on * <b>voter</b>, and return 0. */ -/* (private; exposed for testing.) */ int -networkstatus_check_voter_signature(networkstatus_t *consensus, - networkstatus_voter_info_t *voter, - authority_cert_t *cert) +networkstatus_check_document_signature(const networkstatus_t *consensus, + document_signature_t *sig, + const authority_cert_t *cert) { - char d[DIGEST_LEN]; + char key_digest[DIGEST_LEN]; + const int dlen = sig->alg == DIGEST_SHA1 ? DIGEST_LEN : DIGEST256_LEN; char *signed_digest; size_t signed_digest_len; - if (crypto_pk_get_digest(cert->signing_key, d)<0) + + if (crypto_pk_get_digest(cert->signing_key, key_digest)<0) return -1; - if (tor_memneq(voter->signing_key_digest, d, DIGEST_LEN)) + if (tor_memneq(sig->signing_key_digest, key_digest, DIGEST_LEN) || + tor_memneq(sig->identity_digest, cert->cache_info.identity_digest, + DIGEST_LEN)) return -1; + signed_digest_len = crypto_pk_keysize(cert->signing_key); signed_digest = tor_malloc(signed_digest_len); if (crypto_pk_public_checksig(cert->signing_key, signed_digest, signed_digest_len, - voter->signature, - voter->signature_len) != DIGEST_LEN || - tor_memneq(signed_digest, consensus->networkstatus_digest, DIGEST_LEN)) { + sig->signature, + sig->signature_len) < dlen || + tor_memneq(signed_digest, consensus->digests.d[sig->alg], dlen)) { log_warn(LD_DIR, "Got a bad signature on a networkstatus vote"); - voter->bad_signature = 1; + sig->bad_signature = 1; } else { - voter->good_signature = 1; + sig->good_signature = 1; } tor_free(signed_digest); return 0; @@ -388,7 +465,7 @@ networkstatus_check_consensus_signature(networkstatus_t *consensus, int warn) { int n_good = 0; - int n_missing_key = 0; + int n_missing_key = 0, n_dl_failed_key = 0; int n_bad = 0; int n_unknown = 0; int n_no_signature = 0; @@ -402,37 +479,62 @@ networkstatus_check_consensus_signature(networkstatus_t *consensus, tor_assert(consensus->type == NS_TYPE_CONSENSUS); - SMARTLIST_FOREACH(consensus->voters, networkstatus_voter_info_t *, voter, - { - if (!voter->good_signature && !voter->bad_signature && voter->signature) { - /* we can try to check the signature. */ - int is_v3_auth = trusteddirserver_get_by_v3_auth_digest( - voter->identity_digest) != NULL; - authority_cert_t *cert = - authority_cert_get_by_digests(voter->identity_digest, - voter->signing_key_digest); - if (!is_v3_auth) { - smartlist_add(unrecognized, voter); - ++n_unknown; - continue; - } else if (!cert || cert->expires < now) { - smartlist_add(need_certs_from, voter); - ++n_missing_key; - continue; - } - if (networkstatus_check_voter_signature(consensus, voter, cert) < 0) { - smartlist_add(need_certs_from, voter); - ++n_missing_key; - continue; + SMARTLIST_FOREACH_BEGIN(consensus->voters, networkstatus_voter_info_t *, + voter) { + int good_here = 0; + int bad_here = 0; + int unknown_here = 0; + int missing_key_here = 0, dl_failed_key_here = 0; + SMARTLIST_FOREACH_BEGIN(voter->sigs, document_signature_t *, sig) { + if (!sig->good_signature && !sig->bad_signature && + sig->signature) { + /* we can try to check the signature. */ + int is_v3_auth = trusteddirserver_get_by_v3_auth_digest( + sig->identity_digest) != NULL; + authority_cert_t *cert = + authority_cert_get_by_digests(sig->identity_digest, + sig->signing_key_digest); + tor_assert(tor_memeq(sig->identity_digest, voter->identity_digest, + DIGEST_LEN)); + + if (!is_v3_auth) { + smartlist_add(unrecognized, voter); + ++unknown_here; + continue; + } else if (!cert || cert->expires < now) { + smartlist_add(need_certs_from, voter); + ++missing_key_here; + if (authority_cert_dl_looks_uncertain(sig->identity_digest)) + ++dl_failed_key_here; + continue; + } + if (networkstatus_check_document_signature(consensus, sig, cert) < 0) { + smartlist_add(need_certs_from, voter); + ++missing_key_here; + if (authority_cert_dl_looks_uncertain(sig->identity_digest)) + ++dl_failed_key_here; + continue; + } } - } - if (voter->good_signature) + if (sig->good_signature) + ++good_here; + else if (sig->bad_signature) + ++bad_here; + } SMARTLIST_FOREACH_END(sig); + if (good_here) ++n_good; - else if (voter->bad_signature) + else if (bad_here) ++n_bad; - else + else if (missing_key_here) { + ++n_missing_key; + if (dl_failed_key_here) + ++n_dl_failed_key; + } else if (unknown_here) { + ++n_unknown; + } else { ++n_no_signature; - }); + } + } SMARTLIST_FOREACH_END(voter); /* Now see whether we're missing any voters entirely. */ SMARTLIST_FOREACH(router_get_trusted_dir_servers(), @@ -443,39 +545,71 @@ networkstatus_check_consensus_signature(networkstatus_t *consensus, smartlist_add(missing_authorities, ds); }); - if (warn > 1 || (warn >= 0 && n_good < n_required)) + if (warn > 1 || (warn >= 0 && + (n_good + n_missing_key - n_dl_failed_key < n_required))) { severity = LOG_WARN; - else + } else { severity = LOG_INFO; + } if (warn >= 0) { SMARTLIST_FOREACH(unrecognized, networkstatus_voter_info_t *, voter, { - log_info(LD_DIR, "Consensus includes unrecognized authority '%s' " - "at %s:%d (contact %s; identity %s)", + log(severity, LD_DIR, "Consensus includes unrecognized authority " + "'%s' at %s:%d (contact %s; identity %s)", voter->nickname, voter->address, (int)voter->dir_port, voter->contact?voter->contact:"n/a", hex_str(voter->identity_digest, DIGEST_LEN)); }); SMARTLIST_FOREACH(need_certs_from, networkstatus_voter_info_t *, voter, { - log_info(LD_DIR, "Looks like we need to download a new certificate " - "from authority '%s' at %s:%d (contact %s; identity %s)", + log(severity, LD_DIR, "Looks like we need to download a new " + "certificate from authority '%s' at %s:%d (contact %s; " + "identity %s)", voter->nickname, voter->address, (int)voter->dir_port, voter->contact?voter->contact:"n/a", hex_str(voter->identity_digest, DIGEST_LEN)); }); SMARTLIST_FOREACH(missing_authorities, trusted_dir_server_t *, ds, { - log_info(LD_DIR, "Consensus does not include configured " + log(severity, LD_DIR, "Consensus does not include configured " "authority '%s' at %s:%d (identity %s)", ds->nickname, ds->address, (int)ds->dir_port, hex_str(ds->v3_identity_digest, DIGEST_LEN)); }); - log(severity, LD_DIR, - "%d unknown, %d missing key, %d good, %d bad, %d no signature, " - "%d required", n_unknown, n_missing_key, n_good, n_bad, - n_no_signature, n_required); + { + smartlist_t *sl = smartlist_create(); + char *cp; + tor_asprintf(&cp, "A consensus needs %d good signatures from recognized " + "authorities for us to accept it. This one has %d.", + n_required, n_good); + smartlist_add(sl,cp); + if (n_no_signature) { + tor_asprintf(&cp, "%d of the authorities we know didn't sign it.", + n_no_signature); + smartlist_add(sl,cp); + } + if (n_unknown) { + tor_asprintf(&cp, "It has %d signatures from authorities we don't " + "recognize.", n_unknown); + smartlist_add(sl,cp); + } + if (n_bad) { + tor_asprintf(&cp, "%d of the signatures on it didn't verify " + "correctly.", n_bad); + smartlist_add(sl,cp); + } + if (n_missing_key) { + tor_asprintf(&cp, "We were unable to check %d of the signatures, " + "because we were missing the keys.", n_missing_key); + smartlist_add(sl,cp); + } + cp = smartlist_join_strings(sl, " ", 0, NULL); + log(severity, LD_DIR, "%s", cp); + tor_free(cp); + SMARTLIST_FOREACH(sl, char *, c, tor_free(c)); + smartlist_free(sl); + } } smartlist_free(unrecognized); @@ -785,8 +919,8 @@ networkstatus_v2_list_clean(time_t now) /** Helper for bsearching a list of routerstatus_t pointers: compare a * digest in the key to the identity digest of a routerstatus_t. */ -static int -_compare_digest_to_routerstatus_entry(const void *_key, const void **_member) +int +compare_digest_to_routerstatus_entry(const void *_key, const void **_member) { const char *key = _key; const routerstatus_t *rs = *_member; @@ -799,7 +933,7 @@ routerstatus_t * networkstatus_v2_find_entry(networkstatus_v2_t *ns, const char *digest) { return smartlist_bsearch(ns->entries, digest, - _compare_digest_to_routerstatus_entry); + compare_digest_to_routerstatus_entry); } /** Return the entry in <b>ns</b> for the identity digest <b>digest</b>, or @@ -808,7 +942,7 @@ routerstatus_t * networkstatus_vote_find_entry(networkstatus_t *ns, const char *digest) { return smartlist_bsearch(ns->routerstatus_list, digest, - _compare_digest_to_routerstatus_entry); + compare_digest_to_routerstatus_entry); } /*XXXX make this static once functions are moved into this file. */ @@ -820,7 +954,7 @@ networkstatus_vote_find_entry_idx(networkstatus_t *ns, const char *digest, int *found_out) { return smartlist_bsearch_idx(ns->routerstatus_list, digest, - _compare_digest_to_routerstatus_entry, + compare_digest_to_routerstatus_entry, found_out); } @@ -873,7 +1007,7 @@ router_get_consensus_status_by_id(const char *digest) if (!current_consensus) return NULL; return smartlist_bsearch(current_consensus->routerstatus_list, digest, - _compare_digest_to_routerstatus_entry); + compare_digest_to_routerstatus_entry); } /** Given a nickname (possibly verbose, possibly a hexadecimal digest), return @@ -1077,28 +1211,30 @@ update_v2_networkstatus_cache_downloads(time_t now) static void update_consensus_networkstatus_downloads(time_t now) { - or_options_t *options = get_options(); + int i; if (!networkstatus_get_live_consensus(now)) time_to_download_next_consensus = now; /* No live consensus? Get one now!*/ if (time_to_download_next_consensus > now) return; /* Wait until the current consensus is older. */ - if (authdir_mode_v3(options)) - return; /* Authorities never fetch a consensus */ - if (!download_status_is_ready(&consensus_dl_status, now, + /* XXXXNM Microdescs: may need to download more types. */ + if (!download_status_is_ready(&consensus_dl_status[FLAV_NS], now, CONSENSUS_NETWORKSTATUS_MAX_DL_TRIES)) return; /* We failed downloading a consensus too recently. */ if (connection_get_by_type_purpose(CONN_TYPE_DIR, DIR_PURPOSE_FETCH_CONSENSUS)) return; /* There's an in-progress download.*/ - if (consensus_waiting_for_certs) { - /* XXXX make sure this doesn't delay sane downloads. */ - if (consensus_waiting_for_certs_set_at + DELAY_WHILE_FETCHING_CERTS > now) - return; /* We're still getting certs for this one. */ - else { - if (!consensus_waiting_for_certs_dl_failed) { - download_status_failed(&consensus_dl_status, 0); - consensus_waiting_for_certs_dl_failed=1; + for (i=0; i < N_CONSENSUS_FLAVORS; ++i) { + consensus_waiting_for_certs_t *waiting = &consensus_waiting_for_certs[i]; + if (waiting->consensus) { + /* XXXX make sure this doesn't delay sane downloads. */ + if (waiting->set_at + DELAY_WHILE_FETCHING_CERTS > now) + return; /* We're still getting certs for this one. */ + else { + if (!waiting->dl_failed) { + download_status_failed(&consensus_dl_status[FLAV_NS], 0); + waiting->dl_failed=1; + } } } } @@ -1114,7 +1250,8 @@ update_consensus_networkstatus_downloads(time_t now) void networkstatus_consensus_download_failed(int status_code) { - download_status_failed(&consensus_dl_status, status_code); + /* XXXXNM Microdescs: may need to handle more types. */ + download_status_failed(&consensus_dl_status[FLAV_NS], status_code); /* Retry immediately, if appropriate. */ update_consensus_networkstatus_downloads(time(NULL)); } @@ -1148,8 +1285,15 @@ update_consensus_networkstatus_fetch_time(time_t now) /* We want to cache the next one at some point after this one * is no longer fresh... */ start = c->fresh_until + min_sec_before_caching; - /* But only in the first half-interval after that. */ - dl_interval = interval/2; + /* Some clients may need the consensus sooner than others. */ + if (options->FetchDirInfoExtraEarly || authdir_mode_v3(options)) { + dl_interval = 60; + if (min_sec_before_caching + dl_interval > interval) + dl_interval = interval/2; + } else { + /* But only in the first half-interval after that. */ + dl_interval = interval/2; + } } else { /* We're an ordinary client or a bridge. Give all the caches enough * time to download the consensus. */ @@ -1168,7 +1312,7 @@ update_consensus_networkstatus_fetch_time(time_t now) } if (dl_interval < 1) dl_interval = 1; - /* We must not try to replace c while it's still the most valid: */ + /* We must not try to replace c while it's still fresh: */ tor_assert(c->fresh_until < start); /* We must download the next one before c is invalid: */ tor_assert(start+dl_interval < c->valid_until); @@ -1189,7 +1333,6 @@ update_consensus_networkstatus_fetch_time(time_t now) time_to_download_next_consensus = now; log_info(LD_DIR, "No live consensus; we should fetch one immediately."); } - } /** Return 1 if there's a reason we shouldn't try any directory @@ -1213,7 +1356,7 @@ update_networkstatus_downloads(time_t now) or_options_t *options = get_options(); if (should_delay_dir_fetches(options)) return; - if (directory_fetches_dir_info_early(options)) + if (authdir_mode_any_main(options) || options->FetchV2Networkstatus) update_v2_networkstatus_cache_downloads(now); update_consensus_networkstatus_downloads(now); update_certificate_downloads(now); @@ -1224,10 +1367,14 @@ update_networkstatus_downloads(time_t now) void update_certificate_downloads(time_t now) { - if (consensus_waiting_for_certs) - authority_certs_fetch_missing(consensus_waiting_for_certs, now); - else - authority_certs_fetch_missing(current_consensus, now); + int i; + for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) { + if (consensus_waiting_for_certs[i].consensus) + authority_certs_fetch_missing(consensus_waiting_for_certs[i].consensus, + now); + } + + authority_certs_fetch_missing(current_consensus, now); } /** Return 1 if we have a consensus but we don't have enough certificates @@ -1235,7 +1382,8 @@ update_certificate_downloads(time_t now) int consensus_is_waiting_for_certs(void) { - return consensus_waiting_for_certs ? 1 : 0; + return consensus_waiting_for_certs[USABLE_CONSENSUS_FLAVOR].consensus + ? 1 : 0; } /** Return the network status with a given identity digest. */ @@ -1404,16 +1552,31 @@ networkstatus_copy_old_consensus_info(networkstatus_t *new_c, * user, and -2 for more serious problems. */ int -networkstatus_set_current_consensus(const char *consensus, unsigned flags) +networkstatus_set_current_consensus(const char *consensus, + const char *flavor, + unsigned flags) { - networkstatus_t *c; + networkstatus_t *c=NULL; int r, result = -1; time_t now = time(NULL); + or_options_t *options = get_options(); char *unverified_fname = NULL, *consensus_fname = NULL; + int flav = networkstatus_parse_flavor_name(flavor); const unsigned from_cache = flags & NSSET_FROM_CACHE; const unsigned was_waiting_for_certs = flags & NSSET_WAS_WAITING_FOR_CERTS; const unsigned dl_certs = !(flags & NSSET_DONT_DOWNLOAD_CERTS); const unsigned accept_obsolete = flags & NSSET_ACCEPT_OBSOLETE; + const unsigned require_flavor = flags & NSSET_REQUIRE_FLAVOR; + const digests_t *current_digests = NULL; + consensus_waiting_for_certs_t *waiting = NULL; + time_t current_valid_after = 0; + int free_consensus = 1; /* Free 'c' at the end of the function */ + + if (flav < 0) { + /* XXXX we don't handle unrecognized flavors yet. */ + log_warn(LD_BUG, "Unrecognized consensus flavor %s", flavor); + return -2; + } /* Make sure it's parseable. */ c = networkstatus_parse_vote_from_string(consensus, NULL, NS_TYPE_CONSENSUS); @@ -1423,34 +1586,69 @@ networkstatus_set_current_consensus(const char *consensus, unsigned flags) goto done; } + if ((int)c->flavor != flav) { + /* This wasn't the flavor we thought we were getting. */ + if (require_flavor) { + log_warn(LD_DIR, "Got consensus with unexpected flavor %s (wanted %s)", + networkstatus_get_flavor_name(c->flavor), flavor); + goto done; + } + flav = c->flavor; + flavor = networkstatus_get_flavor_name(flav); + } + + if (flav != USABLE_CONSENSUS_FLAVOR && + !directory_caches_dir_info(options)) { + /* This consensus is totally boring to us: we won't use it, and we won't + * serve it. Drop it. */ + goto done; + } + if (from_cache && !accept_obsolete && c->valid_until < now-OLD_ROUTER_DESC_MAX_AGE) { - /* XXX022 when we try to make fallbackconsensus work again, we should + /* XXXX If we try to make fallbackconsensus work again, we should * consider taking this out. Until then, believing obsolete consensuses * is causing more harm than good. See also bug 887. */ - log_info(LD_DIR, "Loaded an obsolete consensus. Discarding."); + log_info(LD_DIR, "Loaded an expired consensus. Discarding."); goto done; } - if (current_consensus && - tor_memeq(c->networkstatus_digest, - current_consensus->networkstatus_digest, - DIGEST_LEN)) { + if (!strcmp(flavor, "ns")) { + consensus_fname = get_datadir_fname("cached-consensus"); + unverified_fname = get_datadir_fname("unverified-consensus"); + if (current_consensus) { + current_digests = ¤t_consensus->digests; + current_valid_after = current_consensus->valid_after; + } + } else { + cached_dir_t *cur; + char buf[128]; + tor_snprintf(buf, sizeof(buf), "cached-%s-consensus", flavor); + consensus_fname = get_datadir_fname(buf); + tor_snprintf(buf, sizeof(buf), "unverified-%s-consensus", flavor); + unverified_fname = get_datadir_fname(buf); + cur = dirserv_get_consensus(flavor); + if (cur) { + current_digests = &cur->digests; + current_valid_after = cur->published; + } + } + + if (current_digests && + tor_memeq(&c->digests, current_digests, sizeof(c->digests))) { /* We already have this one. That's a failure. */ - log_info(LD_DIR, "Got a consensus we already have"); + log_info(LD_DIR, "Got a %s consensus we already have", flavor); goto done; } - if (current_consensus && c->valid_after <= current_consensus->valid_after) { + if (current_valid_after && c->valid_after <= current_valid_after) { /* We have a newer one. There's no point in accepting this one, * even if it's great. */ - log_info(LD_DIR, "Got a consensus at least as old as the one we have"); + log_info(LD_DIR, "Got a %s consensus at least as old as the one we have", + flavor); goto done; } - consensus_fname = get_datadir_fname("cached-consensus"); - unverified_fname = get_datadir_fname("unverified-consensus"); - /* Make sure it's signed enough. */ if ((r=networkstatus_check_consensus_signature(c, 1))<0) { if (r == -1) { @@ -1459,16 +1657,16 @@ networkstatus_set_current_consensus(const char *consensus, unsigned flags) log_info(LD_DIR, "Not enough certificates to check networkstatus consensus"); } - if (!current_consensus || - c->valid_after > current_consensus->valid_after) { - if (consensus_waiting_for_certs) - networkstatus_vote_free(consensus_waiting_for_certs); - tor_free(consensus_waiting_for_certs_body); - consensus_waiting_for_certs = c; - c = NULL; /* Prevent free. */ - consensus_waiting_for_certs_body = tor_strdup(consensus); - consensus_waiting_for_certs_set_at = now; - consensus_waiting_for_certs_dl_failed = 0; + if (!current_valid_after || + c->valid_after > current_valid_after) { + waiting = &consensus_waiting_for_certs[flav]; + networkstatus_vote_free(waiting->consensus); + tor_free(waiting->body); + waiting->consensus = c; + free_consensus = 0; + waiting->body = tor_strdup(consensus); + waiting->set_at = now; + waiting->dl_failed = 0; if (!from_cache) { write_str_to_file(unverified_fname, consensus, 0); } @@ -1497,57 +1695,81 @@ networkstatus_set_current_consensus(const char *consensus, unsigned flags) } } - if (!from_cache) + if (!from_cache && flav == USABLE_CONSENSUS_FLAVOR) control_event_client_status(LOG_NOTICE, "CONSENSUS_ARRIVED"); /* Are we missing any certificates at all? */ if (r != 1 && dl_certs) authority_certs_fetch_missing(c, now); - notify_control_networkstatus_changed(current_consensus, c); + if (flav == USABLE_CONSENSUS_FLAVOR) { + notify_control_networkstatus_changed(current_consensus, c); - if (current_consensus) { - networkstatus_copy_old_consensus_info(c, current_consensus); - networkstatus_vote_free(current_consensus); + if (current_consensus) { + networkstatus_copy_old_consensus_info(c, current_consensus); + networkstatus_vote_free(current_consensus); + /* Defensive programming : we should set current_consensus very soon, + * but we're about to call some stuff in the meantime, and leaving this + * dangling pointer around has proven to be trouble. */ + current_consensus = NULL; + } } - if (consensus_waiting_for_certs && - consensus_waiting_for_certs->valid_after <= c->valid_after) { - networkstatus_vote_free(consensus_waiting_for_certs); - consensus_waiting_for_certs = NULL; - if (consensus != consensus_waiting_for_certs_body) - tor_free(consensus_waiting_for_certs_body); + waiting = &consensus_waiting_for_certs[flav]; + if (waiting->consensus && + waiting->consensus->valid_after <= c->valid_after) { + networkstatus_vote_free(waiting->consensus); + waiting->consensus = NULL; + if (consensus != waiting->body) + tor_free(waiting->body); else - consensus_waiting_for_certs_body = NULL; - consensus_waiting_for_certs_set_at = 0; - consensus_waiting_for_certs_dl_failed = 0; + waiting->body = NULL; + waiting->set_at = 0; + waiting->dl_failed = 0; unlink(unverified_fname); } /* Reset the failure count only if this consensus is actually valid. */ if (c->valid_after <= now && now <= c->valid_until) { - download_status_reset(&consensus_dl_status); + download_status_reset(&consensus_dl_status[flav]); } else { if (!from_cache) - download_status_failed(&consensus_dl_status, 0); + download_status_failed(&consensus_dl_status[flav], 0); } - current_consensus = c; - c = NULL; /* Prevent free. */ + if (flav == USABLE_CONSENSUS_FLAVOR) { + current_consensus = c; + free_consensus = 0; /* Prevent free. */ - update_consensus_networkstatus_fetch_time(now); - dirvote_recalculate_timing(get_options(), now); - routerstatus_list_update_named_server_map(); + /* XXXXNM Microdescs: needs a non-ns variant. */ + update_consensus_networkstatus_fetch_time(now); + dirvote_recalculate_timing(options, now); + routerstatus_list_update_named_server_map(); + cell_ewma_set_scale_factor(options, current_consensus); + + /* XXXX023 this call might be unnecessary here: can changing the + * current consensus really alter our view of any OR's rate limits? */ + connection_or_update_token_buckets(get_connection_array(), options); + + circuit_build_times_new_consensus_params(&circ_times, current_consensus); + } + + if (directory_caches_dir_info(options)) { + dirserv_set_cached_consensus_networkstatus(consensus, + flavor, + &c->digests, + c->valid_after); + } if (!from_cache) { write_str_to_file(consensus_fname, consensus, 0); } - if (directory_caches_dir_info(get_options())) - dirserv_set_cached_networkstatus_v3(consensus, - current_consensus->valid_after); +/** If a consensus appears more than this many seconds before its declared + * valid-after time, declare that our clock is skewed. */ +#define EARLY_CONSENSUS_NOTICE_SKEW 60 - if (ftime_definitely_before(now, current_consensus->valid_after)) { + if (now < current_consensus->valid_after - EARLY_CONSENSUS_NOTICE_SKEW) { char tbuf[ISO_TIME_LEN+1]; char dbuf[64]; long delta = now - current_consensus->valid_after; @@ -1565,7 +1787,7 @@ networkstatus_set_current_consensus(const char *consensus, unsigned flags) result = 0; done: - if (c) + if (free_consensus) networkstatus_vote_free(c); tor_free(consensus_fname); tor_free(unverified_fname); @@ -1577,13 +1799,17 @@ networkstatus_set_current_consensus(const char *consensus, unsigned flags) void networkstatus_note_certs_arrived(void) { - if (consensus_waiting_for_certs) { - if (networkstatus_check_consensus_signature( - consensus_waiting_for_certs, 0)>=0) { + int i; + for (i=0; i<N_CONSENSUS_FLAVORS; ++i) { + consensus_waiting_for_certs_t *waiting = &consensus_waiting_for_certs[i]; + if (!waiting->consensus) + continue; + if (networkstatus_check_consensus_signature(waiting->consensus, 0)>=0) { if (!networkstatus_set_current_consensus( - consensus_waiting_for_certs_body, + waiting->body, + networkstatus_get_flavor_name(i), NSSET_WAS_WAITING_FOR_CERTS)) { - tor_free(consensus_waiting_for_certs_body); + tor_free(waiting->body); } } } @@ -1669,10 +1895,8 @@ download_status_map_update_from_v2_networkstatus(void) v2_download_status_map = digestmap_new(); dl_status = digestmap_new(); - SMARTLIST_FOREACH(networkstatus_v2_list, networkstatus_v2_t *, ns, - { - SMARTLIST_FOREACH(ns->entries, routerstatus_t *, rs, - { + SMARTLIST_FOREACH_BEGIN(networkstatus_v2_list, networkstatus_v2_t *, ns) { + SMARTLIST_FOREACH_BEGIN(ns->entries, routerstatus_t *, rs) { const char *d = rs->descriptor_digest; download_status_t *s; if (digestmap_get(dl_status, d)) @@ -1681,8 +1905,8 @@ download_status_map_update_from_v2_networkstatus(void) s = tor_malloc_zero(sizeof(download_status_t)); } digestmap_set(dl_status, d, s); - }); - }); + } SMARTLIST_FOREACH_END(rs); + } SMARTLIST_FOREACH_END(ns); digestmap_free(v2_download_status_map, _tor_free); v2_download_status_map = dl_status; networkstatus_v2_list_has_changed = 0; @@ -1696,11 +1920,9 @@ routerstatus_list_update_named_server_map(void) if (!current_consensus) return; - if (named_server_map) - strmap_free(named_server_map, _tor_free); + strmap_free(named_server_map, _tor_free); named_server_map = strmap_new(); - if (unnamed_server_map) - strmap_free(unnamed_server_map, NULL); + strmap_free(unnamed_server_map, NULL); unnamed_server_map = strmap_new(); SMARTLIST_FOREACH(current_consensus->routerstatus_list, routerstatus_t *, rs, { @@ -1775,6 +1997,15 @@ routers_update_status_from_consensus_networkstatus(smartlist_t *routers, router->is_bad_directory = rs->is_bad_directory; router->is_bad_exit = rs->is_bad_exit; router->is_hs_dir = rs->is_hs_dir; + } else { + /* If we _are_ an authority, we should check whether this router + * is one that will cause us to need a reachability test. */ + routerinfo_t *old_router = + router_get_by_digest(router->cache_info.identity_digest); + if (old_router != router) { + router->needs_retest_if_added = + dirserv_should_launch_reachability_test(router, old_router); + } } if (router->is_running && ds) { download_status_reset(&ds->v2_ns_dl_status); @@ -1837,7 +2068,7 @@ char * networkstatus_getinfo_helper_single(routerstatus_t *rs) { char buf[RS_ENTRY_LEN+1]; - routerstatus_format_entry(buf, sizeof(buf), rs, NULL, 0, 1); + routerstatus_format_entry(buf, sizeof(buf), rs, NULL, NS_CONTROL_PORT); return tor_strdup(buf); } @@ -1874,7 +2105,7 @@ networkstatus_getinfo_by_purpose(const char *purpose_string, time_t now) if (bridge_auth && ri->purpose == ROUTER_PURPOSE_BRIDGE) dirserv_set_router_is_running(ri, now); /* then generate and write out status lines for each of them */ - set_routerstatus_from_routerinfo(&rs, ri, now, 0, 0, 0, 0); + set_routerstatus_from_routerinfo(&rs, ri, now, 0, 0, 0); smartlist_add(statuses, networkstatus_getinfo_helper_single(&rs)); }); @@ -1899,35 +2130,118 @@ networkstatus_dump_bridge_status_to_file(time_t now) tor_free(status); } +static int32_t +get_net_param_from_list(smartlist_t *net_params, const char *param_name, + int32_t default_val, int32_t min_val, int32_t max_val) +{ + int32_t res = default_val; + size_t name_len = strlen(param_name); + + tor_assert(max_val > min_val); + tor_assert(min_val <= default_val); + tor_assert(max_val >= default_val); + + SMARTLIST_FOREACH_BEGIN(net_params, const char *, p) { + if (!strcmpstart(p, param_name) && p[name_len] == '=') { + int ok=0; + long v = tor_parse_long(p+name_len+1, 10, INT32_MIN, + INT32_MAX, &ok, NULL); + if (ok) { + res = (int32_t) v; + break; + } + } + } SMARTLIST_FOREACH_END(p); + + if (res < min_val) { + log_warn(LD_DIR, "Consensus parameter %s is too small. Got %d, raising to " + "%d.", param_name, res, min_val); + res = min_val; + } else if (res > max_val) { + log_warn(LD_DIR, "Consensus parameter %s is too large. Got %d, capping to " + "%d.", param_name, res, max_val); + res = max_val; + } + + return res; +} + /** Return the value of a integer parameter from the networkstatus <b>ns</b> * whose name is <b>param_name</b>. If <b>ns</b> is NULL, try loading the * latest consensus ourselves. Return <b>default_val</b> if no latest - * consensus, or if it has no parameter called <b>param_name</b>. */ + * consensus, or if it has no parameter called <b>param_name</b>. + * Make sure the value parsed from the consensus is at least + * <b>min_val</b> and at most <b>max_val</b> and raise/cap the parsed value + * if necessary. */ int32_t networkstatus_get_param(networkstatus_t *ns, const char *param_name, - int32_t default_val) + int32_t default_val, int32_t min_val, int32_t max_val) { - size_t name_len; - if (!ns) /* if they pass in null, go find it ourselves */ ns = networkstatus_get_latest_consensus(); if (!ns || !ns->net_params) return default_val; - name_len = strlen(param_name); + return get_net_param_from_list(ns->net_params, param_name, + default_val, min_val, max_val); +} - SMARTLIST_FOREACH_BEGIN(ns->net_params, const char *, p) { - if (!strcmpstart(p, param_name) && p[name_len] == '=') { - int ok=0; - long v = tor_parse_long(p+name_len+1, 10, INT32_MIN, INT32_MAX, &ok, - NULL); - if (ok) - return (int32_t) v; - } - } SMARTLIST_FOREACH_END(p); +/** Return the value of a integer bw weight parameter from the networkstatus + * <b>ns</b> whose name is <b>weight_name</b>. If <b>ns</b> is NULL, try + * loading the latest consensus ourselves. Return <b>default_val</b> if no + * latest consensus, or if it has no parameter called <b>weight_name</b>. */ +int32_t +networkstatus_get_bw_weight(networkstatus_t *ns, const char *weight_name, + int32_t default_val) +{ + int32_t param; + int max; + if (!ns) /* if they pass in null, go find it ourselves */ + ns = networkstatus_get_latest_consensus(); + + if (!ns || !ns->weight_params) + return default_val; + + max = circuit_build_times_get_bw_scale(ns); + param = get_net_param_from_list(ns->weight_params, weight_name, + default_val, -1, + BW_MAX_WEIGHT_SCALE); + if (param > max) { + log_warn(LD_DIR, "Value of consensus weight %s was too large, capping " + "to %d", weight_name, max); + param = max; + } + return param; +} - return default_val; +/** Return the name of the consensus flavor <b>flav</b> as used to identify + * the flavor in directory documents. */ +const char * +networkstatus_get_flavor_name(consensus_flavor_t flav) +{ + switch (flav) { + case FLAV_NS: + return "ns"; + case FLAV_MICRODESC: + return "microdesc"; + default: + tor_fragile_assert(); + return "??"; + } +} + +/** Return the consensus_flavor_t value for the flavor called <b>flavname</b>, + * or -1 if the flavor is not recognized. */ +int +networkstatus_parse_flavor_name(const char *flavname) +{ + if (!strcmp(flavname, "ns")) + return FLAV_NS; + else if (!strcmp(flavname, "microdesc")) + return FLAV_MICRODESC; + else + return -1; } /** If <b>question</b> is a string beginning with "ns/" in a format the @@ -1936,7 +2250,8 @@ networkstatus_get_param(networkstatus_t *ns, const char *param_name, * ORs. Return 0 on success, -1 on unrecognized question format. */ int getinfo_helper_networkstatus(control_connection_t *conn, - const char *question, char **answer) + const char *question, char **answer, + const char **errmsg) { routerstatus_t *status; (void) conn; @@ -1960,8 +2275,10 @@ getinfo_helper_networkstatus(control_connection_t *conn, } else if (!strcmpstart(question, "ns/id/")) { char d[DIGEST_LEN]; - if (base16_decode(d, DIGEST_LEN, question+6, strlen(question+6))) + if (base16_decode(d, DIGEST_LEN, question+6, strlen(question+6))) { + *errmsg = "Data not decodeable as hex"; return -1; + } status = router_get_consensus_status_by_id(d); } else if (!strcmpstart(question, "ns/name/")) { status = router_get_consensus_status_by_nickname(question+8, 0); @@ -1969,7 +2286,7 @@ getinfo_helper_networkstatus(control_connection_t *conn, *answer = networkstatus_getinfo_by_purpose(question+11, time(NULL)); return *answer ? 0 : -1; } else { - return -1; + return 0; } if (status) @@ -1981,30 +2298,29 @@ getinfo_helper_networkstatus(control_connection_t *conn, void networkstatus_free_all(void) { + int i; if (networkstatus_v2_list) { SMARTLIST_FOREACH(networkstatus_v2_list, networkstatus_v2_t *, ns, networkstatus_v2_free(ns)); smartlist_free(networkstatus_v2_list); networkstatus_v2_list = NULL; } - if (v2_download_status_map) { - digestmap_free(v2_download_status_map, _tor_free); - v2_download_status_map = NULL; - } - if (current_consensus) { - networkstatus_vote_free(current_consensus); - current_consensus = NULL; - } - if (consensus_waiting_for_certs) { - networkstatus_vote_free(consensus_waiting_for_certs); - consensus_waiting_for_certs = NULL; - } - tor_free(consensus_waiting_for_certs_body); - if (named_server_map) { - strmap_free(named_server_map, _tor_free); - } - if (unnamed_server_map) { - strmap_free(unnamed_server_map, NULL); + + digestmap_free(v2_download_status_map, _tor_free); + v2_download_status_map = NULL; + networkstatus_vote_free(current_consensus); + current_consensus = NULL; + + for (i=0; i < N_CONSENSUS_FLAVORS; ++i) { + consensus_waiting_for_certs_t *waiting = &consensus_waiting_for_certs[i]; + if (waiting->consensus) { + networkstatus_vote_free(waiting->consensus); + waiting->consensus = NULL; + } + tor_free(waiting->body); } + + strmap_free(named_server_map, _tor_free); + strmap_free(unnamed_server_map, NULL); } diff --git a/src/or/networkstatus.h b/src/or/networkstatus.h new file mode 100644 index 0000000000..ec2e8f884d --- /dev/null +++ b/src/or/networkstatus.h @@ -0,0 +1,99 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file networkstatus.h + * \brief Header file for networkstatus.c. + **/ + +#ifndef _TOR_NETWORKSTATUS_H +#define _TOR_NETWORKSTATUS_H + +/** How old do we allow a v2 network-status to get before removing it + * completely? */ +#define MAX_NETWORKSTATUS_AGE (10*24*60*60) + +void networkstatus_reset_warnings(void); +void networkstatus_reset_download_failures(void); +int router_reload_v2_networkstatus(void); +int router_reload_consensus_networkstatus(void); +void routerstatus_free(routerstatus_t *rs); +void networkstatus_v2_free(networkstatus_v2_t *ns); +void networkstatus_vote_free(networkstatus_t *ns); +networkstatus_voter_info_t *networkstatus_get_voter_by_id( + networkstatus_t *vote, + const char *identity); +int networkstatus_check_consensus_signature(networkstatus_t *consensus, + int warn); +int networkstatus_check_document_signature(const networkstatus_t *consensus, + document_signature_t *sig, + const authority_cert_t *cert); +char *networkstatus_get_cache_filename(const char *identity_digest); +int router_set_networkstatus_v2(const char *s, time_t arrived_at, + v2_networkstatus_source_t source, + smartlist_t *requested_fingerprints); +void networkstatus_v2_list_clean(time_t now); +int compare_digest_to_routerstatus_entry(const void *_key, + const void **_member); +routerstatus_t *networkstatus_v2_find_entry(networkstatus_v2_t *ns, + const char *digest); +routerstatus_t *networkstatus_vote_find_entry(networkstatus_t *ns, + const char *digest); +int networkstatus_vote_find_entry_idx(networkstatus_t *ns, + const char *digest, int *found_out); +const smartlist_t *networkstatus_get_v2_list(void); +download_status_t *router_get_dl_status_by_descriptor_digest(const char *d); +routerstatus_t *router_get_consensus_status_by_id(const char *digest); +routerstatus_t *router_get_consensus_status_by_descriptor_digest( + const char *digest); +routerstatus_t *router_get_consensus_status_by_nickname(const char *nickname, + int warn_if_unnamed); +const char *networkstatus_get_router_digest_by_nickname(const char *nickname); +int networkstatus_nickname_is_unnamed(const char *nickname); +void networkstatus_consensus_download_failed(int status_code); +void update_consensus_networkstatus_fetch_time(time_t now); +int should_delay_dir_fetches(or_options_t *options); +void update_networkstatus_downloads(time_t now); +void update_certificate_downloads(time_t now); +int consensus_is_waiting_for_certs(void); +networkstatus_v2_t *networkstatus_v2_get_by_digest(const char *digest); +networkstatus_t *networkstatus_get_latest_consensus(void); +networkstatus_t *networkstatus_get_live_consensus(time_t now); +networkstatus_t *networkstatus_get_reasonably_live_consensus(time_t now); +#define NSSET_FROM_CACHE 1 +#define NSSET_WAS_WAITING_FOR_CERTS 2 +#define NSSET_DONT_DOWNLOAD_CERTS 4 +#define NSSET_ACCEPT_OBSOLETE 8 +#define NSSET_REQUIRE_FLAVOR 16 +int networkstatus_set_current_consensus(const char *consensus, + const char *flavor, + unsigned flags); +void networkstatus_note_certs_arrived(void); +void routers_update_all_from_networkstatus(time_t now, int dir_version); +void routers_update_status_from_consensus_networkstatus(smartlist_t *routers, + int reset_failures); +void signed_descs_update_status_from_consensus_networkstatus( + smartlist_t *descs); + +char *networkstatus_getinfo_helper_single(routerstatus_t *rs); +char *networkstatus_getinfo_by_purpose(const char *purpose_string, time_t now); +void networkstatus_dump_bridge_status_to_file(time_t now); +int32_t networkstatus_get_param(networkstatus_t *ns, const char *param_name, + int32_t default_val, int32_t min_val, + int32_t max_val); +int getinfo_helper_networkstatus(control_connection_t *conn, + const char *question, char **answer, + const char **errmsg); +int32_t networkstatus_get_bw_weight(networkstatus_t *ns, const char *weight, + int32_t default_val); +const char *networkstatus_get_flavor_name(consensus_flavor_t flav); +int networkstatus_parse_flavor_name(const char *flavname); +void document_signature_free(document_signature_t *sig); +document_signature_t *document_signature_dup(const document_signature_t *sig); +void networkstatus_free_all(void); + +#endif + diff --git a/src/or/ntmain.c b/src/or/ntmain.c index 4dcf9b3763..b2fee648cc 100644 --- a/src/or/ntmain.c +++ b/src/or/ntmain.c @@ -5,13 +5,22 @@ #define MAIN_PRIVATE #include "or.h" - -#include <tchar.h> -#define GENSRV_SERVICENAME TEXT("tor") -#define GENSRV_DISPLAYNAME TEXT("Tor Win32 Service") +#include "config.h" +#include "main.h" +#include "ntmain.h" + +#ifdef HAVE_EVENT2_EVENT_H +#include <event2/event.h> +#else +#include <event.h> +#endif + +#include <windows.h> +#define GENSRV_SERVICENAME "tor" +#define GENSRV_DISPLAYNAME "Tor Win32 Service" #define GENSRV_DESCRIPTION \ - TEXT("Provides an anonymous Internet communication system") -#define GENSRV_USERACCT TEXT("NT AUTHORITY\\LocalService") + "Provides an anonymous Internet communication system" +#define GENSRV_USERACCT "NT AUTHORITY\\LocalService" // Cheating: using the pre-defined error codes, tricks Windows into displaying // a semi-related human-readable error message if startup fails as @@ -27,7 +36,6 @@ static SERVICE_STATUS_HANDLE hStatus; * to the NT service functions. */ static char **backup_argv; static int backup_argc; -static char* nt_strerror(uint32_t errnum); static void nt_service_control(DWORD request); static void nt_service_body(int argc, char **argv); @@ -46,6 +54,11 @@ static int nt_service_cmd_stop(void); struct service_fns { int loaded; + /** @{ */ + /** Function pointers for Windows API functions related to service + * management. These are NULL, or they point to the . They're set by + * calling the LOAD macro below. */ + BOOL (WINAPI *ChangeServiceConfig2A_fn)( SC_HANDLE hService, DWORD dwInfoLevel, @@ -61,30 +74,30 @@ struct service_fns { SC_HANDLE (WINAPI *CreateServiceA_fn)( SC_HANDLE hSCManager, - LPCTSTR lpServiceName, - LPCTSTR lpDisplayName, + LPCSTR lpServiceName, + LPCSTR lpDisplayName, DWORD dwDesiredAccess, DWORD dwServiceType, DWORD dwStartType, DWORD dwErrorControl, - LPCTSTR lpBinaryPathName, - LPCTSTR lpLoadOrderGroup, + LPCSTR lpBinaryPathName, + LPCSTR lpLoadOrderGroup, LPDWORD lpdwTagId, - LPCTSTR lpDependencies, - LPCTSTR lpServiceStartName, - LPCTSTR lpPassword); + LPCSTR lpDependencies, + LPCSTR lpServiceStartName, + LPCSTR lpPassword); BOOL (WINAPI *DeleteService_fn)( SC_HANDLE hService); SC_HANDLE (WINAPI *OpenSCManagerA_fn)( - LPCTSTR lpMachineName, - LPCTSTR lpDatabaseName, + LPCSTR lpMachineName, + LPCSTR lpDatabaseName, DWORD dwDesiredAccess); SC_HANDLE (WINAPI *OpenServiceA_fn)( SC_HANDLE hSCManager, - LPCTSTR lpServiceName, + LPCSTR lpServiceName, DWORD dwDesiredAccess); BOOL (WINAPI *QueryServiceStatus_fn)( @@ -92,28 +105,29 @@ struct service_fns { LPSERVICE_STATUS lpServiceStatus); SERVICE_STATUS_HANDLE (WINAPI *RegisterServiceCtrlHandlerA_fn)( - LPCTSTR lpServiceName, + LPCSTR lpServiceName, LPHANDLER_FUNCTION lpHandlerProc); BOOL (WINAPI *SetServiceStatus_fn)(SERVICE_STATUS_HANDLE, LPSERVICE_STATUS); BOOL (WINAPI *StartServiceCtrlDispatcherA_fn)( - const SERVICE_TABLE_ENTRY* lpServiceTable); + const SERVICE_TABLE_ENTRYA* lpServiceTable); BOOL (WINAPI *StartServiceA_fn)( SC_HANDLE hService, DWORD dwNumServiceArgs, - LPCTSTR* lpServiceArgVectors); + LPCSTR* lpServiceArgVectors); BOOL (WINAPI *LookupAccountNameA_fn)( - LPCTSTR lpSystemName, - LPCTSTR lpAccountName, + LPCSTR lpSystemName, + LPCSTR lpAccountName, PSID Sid, LPDWORD cbSid, LPTSTR ReferencedDomainName, LPDWORD cchReferencedDomainName, PSID_NAME_USE peUse); + /** @} */ } service_fns = { 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, @@ -130,13 +144,16 @@ nt_service_loadlibrary(void) if (service_fns.loaded) return; - /* XXXX Possibly, we should hardcode the location of this DLL. */ - if (!(library = LoadLibrary("advapi32.dll"))) { + if (!(library = load_windows_system_library(TEXT("advapi32.dll")))) { log_err(LD_GENERAL, "Couldn't open advapi32.dll. Are you trying to use " "NT services on Windows 98? That doesn't work."); goto err; } +/* Helper macro: try to load a function named <b>f</b> from "library" into + * service_functions.<b>f</b>_fn. On failure, log an error message, and goto + * err. + */ #define LOAD(f) STMT_BEGIN \ if (!(fn = GetProcAddress(library, #f))) { \ log_err(LD_BUG, \ @@ -218,7 +235,7 @@ nt_service_control(DWORD request) log_notice(LD_GENERAL, "Got stop/shutdown request; shutting down cleanly."); service_status.dwCurrentState = SERVICE_STOP_PENDING; - event_loopexit(&exit_now); + event_base_loopexit(tor_libevent_get_base(), &exit_now); return; } service_fns.SetServiceStatus_fn(hStatus, &service_status); @@ -275,20 +292,20 @@ nt_service_body(int argc, char **argv) static void nt_service_main(void) { - SERVICE_TABLE_ENTRY table[2]; + SERVICE_TABLE_ENTRYA table[2]; DWORD result = 0; char *errmsg; nt_service_loadlibrary(); table[0].lpServiceName = (char*)GENSRV_SERVICENAME; - table[0].lpServiceProc = (LPSERVICE_MAIN_FUNCTION)nt_service_body; + table[0].lpServiceProc = (LPSERVICE_MAIN_FUNCTIONA)nt_service_body; table[1].lpServiceName = NULL; table[1].lpServiceProc = NULL; if (!service_fns.StartServiceCtrlDispatcherA_fn(table)) { result = GetLastError(); - errmsg = nt_strerror(result); + errmsg = format_win32_error(result); printf("Service error %d : %s\n", (int) result, errmsg); - LocalFree(errmsg); + tor_free(errmsg); if (result == ERROR_FAILED_SERVICE_CONTROLLER_CONNECT) { if (tor_init(backup_argc, backup_argv) < 0) return; @@ -323,9 +340,9 @@ nt_service_open_scm(void) nt_service_loadlibrary(); if ((hSCManager = service_fns.OpenSCManagerA_fn( NULL, NULL, SC_MANAGER_CREATE_SERVICE)) == NULL) { - errmsg = nt_strerror(GetLastError()); + errmsg = format_win32_error(GetLastError()); printf("OpenSCManager() failed : %s\n", errmsg); - LocalFree(errmsg); + tor_free(errmsg); } return hSCManager; } @@ -340,9 +357,9 @@ nt_service_open(SC_HANDLE hSCManager) nt_service_loadlibrary(); if ((hService = service_fns.OpenServiceA_fn(hSCManager, GENSRV_SERVICENAME, SERVICE_ALL_ACCESS)) == NULL) { - errmsg = nt_strerror(GetLastError()); + errmsg = format_win32_error(GetLastError()); printf("OpenService() failed : %s\n", errmsg); - LocalFree(errmsg); + tor_free(errmsg); } return hService; } @@ -374,14 +391,14 @@ nt_service_start(SC_HANDLE hService) printf("Service started successfully\n"); return 0; } else { - errmsg = nt_strerror(service_status.dwWin32ExitCode); + errmsg = format_win32_error(service_status.dwWin32ExitCode); printf("Service failed to start : %s\n", errmsg); - LocalFree(errmsg); + tor_free(errmsg); } } else { - errmsg = nt_strerror(GetLastError()); + errmsg = format_win32_error(GetLastError()); printf("StartService() failed : %s\n", errmsg); - LocalFree(errmsg); + tor_free(errmsg); } return -1; } @@ -418,14 +435,14 @@ nt_service_stop(SC_HANDLE hService) } else if (wait_time == MAX_SERVICE_WAIT_TIME) { printf("Service did not stop within %d seconds.\n", wait_time); } else { - errmsg = nt_strerror(GetLastError()); + errmsg = format_win32_error(GetLastError()); printf("QueryServiceStatus() failed : %s\n",errmsg); - LocalFree(errmsg); + tor_free(errmsg); } } else { - errmsg = nt_strerror(GetLastError()); + errmsg = format_win32_error(GetLastError()); printf("ControlService() failed : %s\n", errmsg); - LocalFree(errmsg); + tor_free(errmsg); } return -1; } @@ -439,6 +456,7 @@ static char * nt_service_command_line(int *using_default_torrc) { TCHAR tor_exe[MAX_PATH+1]; + char tor_exe_ascii[MAX_PATH+1]; char *command, *options=NULL; smartlist_t *sl; int i, cmdlen; @@ -464,18 +482,25 @@ nt_service_command_line(int *using_default_torrc) options = smartlist_join_strings(sl,"\" \"",0,NULL); smartlist_free(sl); +#ifdef UNICODE + wcstombs(tor_exe_ascii, tor_exe, sizeof(tor_exe_ascii)); +#else + strlcpy(tor_exe_ascii, tor_exe, sizeof(tor_exe_ascii)); +#endif + /* Allocate a string for the NT service command line */ - cmdlen = strlen(tor_exe) + (options?strlen(options):0) + 32; + cmdlen = strlen(tor_exe_ascii) + (options?strlen(options):0) + 32; command = tor_malloc(cmdlen); /* Format the service command */ if (options) { if (tor_snprintf(command, cmdlen, "\"%s\" --nt-service \"%s\"", - tor_exe, options)<0) { + tor_exe_ascii, options)<0) { tor_free(command); /* sets command to NULL. */ } } else { /* ! options */ - if (tor_snprintf(command, cmdlen, "\"%s\" --nt-service", tor_exe)<0) { + if (tor_snprintf(command, cmdlen, "\"%s\" --nt-service", + tor_exe_ascii)<0) { tor_free(command); /* sets command to NULL. */ } } @@ -500,10 +525,10 @@ nt_service_install(int argc, char **argv) SC_HANDLE hSCManager = NULL; SC_HANDLE hService = NULL; - SERVICE_DESCRIPTION sdBuff; + SERVICE_DESCRIPTIONA sdBuff; char *command; char *errmsg; - const char *user_acct = GENSRV_USERACCT; + const char *user_acct = NULL; const char *password = ""; int i; OSVERSIONINFOEX info; @@ -547,13 +572,12 @@ nt_service_install(int argc, char **argv) is_win2k_or_worse = 1; } - if (user_acct == GENSRV_USERACCT) { + if (!user_acct) { if (is_win2k_or_worse) { /* On Win2k, there is no LocalService account, so we actually need to * fall back on NULL (the system account). */ printf("Running on Win2K or earlier, so the LocalService account " "doesn't exist. Falling back to SYSTEM account.\n"); - user_acct = NULL; } else { /* Genericity is apparently _so_ last year in Redmond, where some * accounts are accounts that you can look up, and some accounts @@ -562,6 +586,7 @@ nt_service_install(int argc, char **argv) */ printf("Running on a Post-Win2K OS, so we'll assume that the " "LocalService account exists.\n"); + user_acct = GENSRV_USERACCT; } } else if (0 && service_fns.LookupAccountNameA_fn(NULL, // On this system user_acct, @@ -590,10 +615,10 @@ nt_service_install(int argc, char **argv) SERVICE_AUTO_START, SERVICE_ERROR_IGNORE, command, NULL, NULL, NULL, user_acct, password)) == NULL) { - errmsg = nt_strerror(GetLastError()); + errmsg = format_win32_error(GetLastError()); printf("CreateService() failed : %s\n", errmsg); service_fns.CloseServiceHandle_fn(hSCManager); - LocalFree(errmsg); + tor_free(errmsg); tor_free(command); return -1; } @@ -634,9 +659,9 @@ nt_service_remove(void) nt_service_stop(hService); if (service_fns.DeleteService_fn(hService) == FALSE) { - errmsg = nt_strerror(GetLastError()); + errmsg = format_win32_error(GetLastError()); printf("DeleteService() failed : %s\n", errmsg); - LocalFree(errmsg); + tor_free(errmsg); service_fns.CloseServiceHandle_fn(hService); service_fns.CloseServiceHandle_fn(hSCManager); return -1; @@ -693,20 +718,6 @@ nt_service_cmd_stop(void) return stop; } -/** Given a Win32 error code, this attempts to make Windows - * return a human-readable error message. The char* returned - * is allocated by Windows, but should be freed with LocalFree() - * when finished with it. */ -static char* -nt_strerror(uint32_t errnum) -{ - char *msgbuf; - FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM, - NULL, errnum, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), - (LPSTR)&msgbuf, 0, NULL); - return msgbuf; -} - int nt_service_parse_options(int argc, char **argv, int *should_exit) { diff --git a/src/or/ntmain.h b/src/or/ntmain.h new file mode 100644 index 0000000000..acd0e1d7eb --- /dev/null +++ b/src/or/ntmain.h @@ -0,0 +1,30 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file ntmain.h + * \brief Header file for ntmain.c. + **/ + +#ifndef _TOR_NTMAIN_H +#define _TOR_NTMAIN_H + +#ifdef MS_WINDOWS +#if !defined (WINCE) +#define NT_SERVICE +#endif +#endif + +#ifdef NT_SERVICE +int nt_service_parse_options(int argc, char **argv, int *should_exit); +int nt_service_is_stopping(void); +void nt_service_set_state(DWORD state); +#else +#define nt_service_is_stopping() 0 +#endif + +#endif + diff --git a/src/or/onion.c b/src/or/onion.c index 4c26debf1c..211d14c1e1 100644 --- a/src/or/onion.c +++ b/src/or/onion.c @@ -11,6 +11,10 @@ **/ #include "or.h" +#include "circuitlist.h" +#include "config.h" +#include "onion.h" +#include "rephist.h" /** Type for a linked list of circuits that are waiting for a free CPU worker * to process a waiting onion handshake. */ @@ -58,11 +62,18 @@ onion_pending_add(or_circuit_t *circ, char *onionskin) tor_assert(!ol_tail->next); if (ol_length >= get_options()->MaxOnionsPending) { - log_warn(LD_GENERAL, - "Your computer is too slow to handle this many circuit " - "creation requests! Please consider using the " - "MaxAdvertisedBandwidth config option or choosing a more " - "restricted exit policy."); +#define WARN_TOO_MANY_CIRC_CREATIONS_INTERVAL (60) + static ratelim_t last_warned = + RATELIM_INIT(WARN_TOO_MANY_CIRC_CREATIONS_INTERVAL); + char *m; + if ((m = rate_limit_log(&last_warned, approx_time()))) { + log_warn(LD_GENERAL, + "Your computer is too slow to handle this many circuit " + "creation requests! Please consider using the " + "MaxAdvertisedBandwidth config option or choosing a more " + "restricted exit policy.%s",m); + tor_free(m); + } tor_free(tmp); return -1; } @@ -248,6 +259,10 @@ onion_skin_server_handshake(const char *onion_skin, /*ONIONSKIN_CHALLENGE_LEN*/ } dh = crypto_dh_new(DH_TYPE_CIRCUIT); + if (!dh) { + log_warn(LD_BUG, "Couldn't allocate DH key"); + goto err; + } if (crypto_dh_get_public(dh, handshake_reply_out, DH_KEY_LEN)) { log_info(LD_GENERAL, "crypto_dh_get_public failed."); goto err; @@ -255,8 +270,9 @@ onion_skin_server_handshake(const char *onion_skin, /*ONIONSKIN_CHALLENGE_LEN*/ key_material_len = DIGEST_LEN+key_out_len; key_material = tor_malloc(key_material_len); - len = crypto_dh_compute_secret(dh, challenge, DH_KEY_LEN, - key_material, key_material_len); + len = crypto_dh_compute_secret(LOG_PROTOCOL_WARN, dh, challenge, + DH_KEY_LEN, key_material, + key_material_len); if (len < 0) { log_info(LD_GENERAL, "crypto_dh_compute_secret failed."); goto err; @@ -306,8 +322,9 @@ onion_skin_client_handshake(crypto_dh_env_t *handshake_state, key_material_len = DIGEST_LEN + key_out_len; key_material = tor_malloc(key_material_len); - len = crypto_dh_compute_secret(handshake_state, handshake_reply, DH_KEY_LEN, - key_material, key_material_len); + len = crypto_dh_compute_secret(LOG_PROTOCOL_WARN, handshake_state, + handshake_reply, DH_KEY_LEN, key_material, + key_material_len); if (len < 0) goto err; diff --git a/src/or/onion.h b/src/or/onion.h new file mode 100644 index 0000000000..7f603b8147 --- /dev/null +++ b/src/or/onion.h @@ -0,0 +1,48 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file onion.h + * \brief Header file for onion.c. + **/ + +#ifndef _TOR_ONION_H +#define _TOR_ONION_H + +int onion_pending_add(or_circuit_t *circ, char *onionskin); +or_circuit_t *onion_next_task(char **onionskin_out); +void onion_pending_remove(or_circuit_t *circ); + +int onion_skin_create(crypto_pk_env_t *router_key, + crypto_dh_env_t **handshake_state_out, + char *onion_skin_out); + +int onion_skin_server_handshake(const char *onion_skin, + crypto_pk_env_t *private_key, + crypto_pk_env_t *prev_private_key, + char *handshake_reply_out, + char *key_out, + size_t key_out_len); + +int onion_skin_client_handshake(crypto_dh_env_t *handshake_state, + const char *handshake_reply, + char *key_out, + size_t key_out_len); + +int fast_server_handshake(const uint8_t *key_in, + uint8_t *handshake_reply_out, + uint8_t *key_out, + size_t key_out_len); + +int fast_client_handshake(const uint8_t *handshake_state, + const uint8_t *handshake_reply_out, + uint8_t *key_out, + size_t key_out_len); + +void clear_pending_onions(void); + +#endif + diff --git a/src/or/or.h b/src/or/or.h index 976ba9f8e5..9cac5f3548 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -20,9 +20,6 @@ #ifndef INSTRUMENT_DOWNLOADS #define INSTRUMENT_DOWNLOADS 1 #endif -#ifndef ENABLE_GEOIP_STATS -#define ENABLE_GEOIP_STATS 1 -#endif #endif #ifdef MS_WINDOWS @@ -62,6 +59,9 @@ #ifdef HAVE_SYS_STAT_H #include <sys/stat.h> #endif +#ifdef HAVE_NETINET_IN_H +#include <netinet/in.h> +#endif #ifdef HAVE_ARPA_INET_H #include <arpa/inet.h> #endif @@ -83,18 +83,15 @@ #define snprintf _snprintf #endif -#include "crypto.h" #include "tortls.h" -#include "log.h" -#include "compat.h" +#include "../common/torlog.h" #include "container.h" -#include "util.h" #include "torgzip.h" #include "address.h" +#include "compat_libevent.h" +#include "ht.h" -#include <event.h> - -/* These signals are defined to help control_signal_act work. +/* These signals are defined to help handle_control_signal work. */ #ifndef SIGHUP #define SIGHUP 1 @@ -161,7 +158,7 @@ #define MAX_DNS_TTL (3*60*60) /** How small can a TTL be before we stop believing it? Provides rudimentary * pinning. */ -#define MIN_DNS_TTL (60) +#define MIN_DNS_TTL 60 /** How often do we rotate onion keys? */ #define MIN_ONION_KEY_LIFETIME (7*24*60*60) @@ -221,6 +218,21 @@ typedef enum { /* !!!! If _CONN_TYPE_MAX is ever over 15, we must grow the type field in * connection_t. */ +/* Proxy client types */ +#define PROXY_NONE 0 +#define PROXY_CONNECT 1 +#define PROXY_SOCKS4 2 +#define PROXY_SOCKS5 3 + +/* Proxy client handshake states */ +#define PROXY_HTTPS_WANT_CONNECT_OK 1 +#define PROXY_SOCKS4_WANT_CONNECT_OK 2 +#define PROXY_SOCKS5_WANT_AUTH_METHOD_NONE 3 +#define PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929 4 +#define PROXY_SOCKS5_WANT_AUTH_RFC1929_OK 5 +#define PROXY_SOCKS5_WANT_CONNECT_OK 6 +#define PROXY_CONNECTED 7 + /** True iff <b>x</b> is an edge connection. */ #define CONN_IS_EDGE(x) \ ((x)->type == CONN_TYPE_EXIT || (x)->type == CONN_TYPE_AP) @@ -241,26 +253,24 @@ typedef enum { #define _OR_CONN_STATE_MIN 1 /** State for a connection to an OR: waiting for connect() to finish. */ #define OR_CONN_STATE_CONNECTING 1 -/** State for a connection to an OR: waiting for proxy command to flush. */ -#define OR_CONN_STATE_PROXY_FLUSHING 2 -/** State for a connection to an OR: waiting for proxy response. */ -#define OR_CONN_STATE_PROXY_READING 3 +/** State for a connection to an OR: waiting for proxy handshake to complete */ +#define OR_CONN_STATE_PROXY_HANDSHAKING 2 /** State for a connection to an OR or client: SSL is handshaking, not done * yet. */ -#define OR_CONN_STATE_TLS_HANDSHAKING 4 +#define OR_CONN_STATE_TLS_HANDSHAKING 3 /** State for a connection to an OR: We're doing a second SSL handshake for * renegotiation purposes. */ -#define OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING 5 +#define OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING 4 /** State for a connection at an OR: We're waiting for the client to * renegotiate. */ -#define OR_CONN_STATE_TLS_SERVER_RENEGOTIATING 6 +#define OR_CONN_STATE_TLS_SERVER_RENEGOTIATING 5 /** State for a connection to an OR: We're done with our SSL handshake, but we * haven't yet negotiated link protocol versions and sent a netinfo cell. */ -#define OR_CONN_STATE_OR_HANDSHAKING 7 +#define OR_CONN_STATE_OR_HANDSHAKING 6 /** State for a connection to an OR: Ready to send/receive cells. */ -#define OR_CONN_STATE_OPEN 8 -#define _OR_CONN_STATE_MAX 8 +#define OR_CONN_STATE_OPEN 7 +#define _OR_CONN_STATE_MAX 7 #define _EXIT_CONN_STATE_MIN 1 /** State for an exit connection: waiting for response from DNS farm. */ @@ -457,23 +467,23 @@ typedef enum { #define CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED 11 /** Client-side circuit purpose: at Alice, rendezvous established. */ #define CIRCUIT_PURPOSE_C_REND_JOINED 12 - -#define _CIRCUIT_PURPOSE_C_MAX 12 - +/** This circuit is used for build time measurement only */ +#define CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT 13 +#define _CIRCUIT_PURPOSE_C_MAX 13 /** Hidden-service-side circuit purpose: at Bob, waiting for introductions. */ -#define CIRCUIT_PURPOSE_S_ESTABLISH_INTRO 13 +#define CIRCUIT_PURPOSE_S_ESTABLISH_INTRO 14 /** Hidden-service-side circuit purpose: at Bob, successfully established * intro. */ -#define CIRCUIT_PURPOSE_S_INTRO 14 +#define CIRCUIT_PURPOSE_S_INTRO 15 /** Hidden-service-side circuit purpose: at Bob, connecting to rend point. */ -#define CIRCUIT_PURPOSE_S_CONNECT_REND 15 +#define CIRCUIT_PURPOSE_S_CONNECT_REND 16 /** Hidden-service-side circuit purpose: at Bob, rendezvous established. */ -#define CIRCUIT_PURPOSE_S_REND_JOINED 16 +#define CIRCUIT_PURPOSE_S_REND_JOINED 17 /** A testing circuit; not meant to be used for actual traffic. */ -#define CIRCUIT_PURPOSE_TESTING 17 +#define CIRCUIT_PURPOSE_TESTING 18 /** A controller made this circuit and Tor should not use it. */ -#define CIRCUIT_PURPOSE_CONTROLLER 18 -#define _CIRCUIT_PURPOSE_MAX 18 +#define CIRCUIT_PURPOSE_CONTROLLER 19 +#define _CIRCUIT_PURPOSE_MAX 19 /** A catch-all for unrecognized purposes. Currently we don't expect * to make or see any circuits with this purpose. */ #define CIRCUIT_PURPOSE_UNKNOWN 255 @@ -483,7 +493,7 @@ typedef enum { #define CIRCUIT_PURPOSE_IS_ORIGIN(p) ((p)>_CIRCUIT_PURPOSE_OR_MAX) /** True iff the circuit purpose <b>p</b> is for a circuit that originated * here to serve as a client. (Hidden services don't count here.) */ -#define CIRCUIT_PURPOSE_IS_CLIENT(p) \ +#define CIRCUIT_PURPOSE_IS_CLIENT(p) \ ((p)> _CIRCUIT_PURPOSE_OR_MAX && \ (p)<=_CIRCUIT_PURPOSE_C_MAX) /** True iff the circuit_t <b>c</b> is actually an origin_circuit_t. */ @@ -573,6 +583,9 @@ typedef enum { /** This is a connection on the NATD port, and the destination IP:Port was * either ill-formed or out-of-range. */ #define END_STREAM_REASON_INVALID_NATD_DEST 261 +/** The target address is in a private network (like 127.0.0.1 or 10.0.0.1); + * you don't want to do that over a randomly chosen exit */ +#define END_STREAM_REASON_PRIVATE_ADDR 262 /** Bitwise-and this value with endreason to mask out all flags. */ #define END_STREAM_REASON_MASK 511 @@ -604,6 +617,10 @@ typedef enum { /* Negative reasons are internal: we never send them in a DESTROY or TRUNCATE * call; they only go to the controller for tracking */ +/** Our post-timeout circuit time measurement period expired. + * We must give up now */ +#define END_CIRC_REASON_MEASUREMENT_EXPIRED -3 + /** We couldn't build a path for this circuit. */ #define END_CIRC_REASON_NOPATH -2 /** Catch-all "other" reason for closing origin circuits. */ @@ -641,10 +658,6 @@ typedef enum { /** Length of a binary-encoded rendezvous service ID. */ #define REND_SERVICE_ID_LEN 10 -/** How long after we receive a hidden service descriptor do we consider - * it fresh? */ -#define NUM_SECONDS_BEFORE_HS_REFETCH (60*15) - /** Time period for which a v2 descriptor will be valid. */ #define REND_TIME_PERIOD_V2_DESC_VALIDITY (24*60*60) @@ -739,12 +752,6 @@ typedef struct rend_data_t { /** Rendezvous cookie used by both, client and service. */ char rend_cookie[REND_COOKIE_LEN]; - - /** Rendezvous descriptor version that is used by a service. Used to - * distinguish introduction and rendezvous points belonging to the same - * rendezvous service ID, but different descriptor versions. - */ - uint8_t rend_desc_version; } rend_data_t; /** Time interval for tracking possible replays of INTRODUCE2 cells. @@ -761,6 +768,8 @@ typedef enum { /** Initial value for both sides of a circuit transmission window when the * circuit is initialized. Measured in cells. */ #define CIRCWINDOW_START 1000 +#define CIRCWINDOW_START_MIN 100 +#define CIRCWINDOW_START_MAX 1000 /** Amount to increment a circuit window when we get a circuit SENDME. */ #define CIRCWINDOW_INCREMENT 100 /** Initial value on both sides of a stream transmission window when the @@ -839,9 +848,13 @@ typedef struct cell_t { /** Parsed variable-length onion routing cell. */ typedef struct var_cell_t { + /** Type of the cell: CELL_VERSIONS, etc. */ uint8_t command; + /** Circuit thich received the cell */ circid_t circ_id; + /** Number of bytes actually stored in <b>payload</b> */ uint16_t payload_len; + /** Payload of this cell */ uint8_t payload[1]; } var_cell_t; @@ -851,12 +864,28 @@ typedef struct packed_cell_t { char body[CELL_NETWORK_SIZE]; /**< Cell as packed for network. */ } packed_cell_t; +/** Number of cells added to a circuit queue including their insertion + * time on 10 millisecond detail; used for buffer statistics. */ +typedef struct insertion_time_elem_t { + struct insertion_time_elem_t *next; /**< Next element in queue. */ + uint32_t insertion_time; /**< When were cells inserted (in 10 ms steps + * starting at 0:00 of the current day)? */ + unsigned counter; /**< How many cells were inserted? */ +} insertion_time_elem_t; + +/** Queue of insertion times. */ +typedef struct insertion_time_queue_t { + struct insertion_time_elem_t *first; /**< First element in queue. */ + struct insertion_time_elem_t *last; /**< Last element in queue. */ +} insertion_time_queue_t; + /** A queue of cells on a circuit, waiting to be added to the * or_connection_t's outbuf. */ typedef struct cell_queue_t { packed_cell_t *head; /**< The first cell, or NULL if the queue is empty. */ packed_cell_t *tail; /**< The last cell, or NULL if the queue is empty. */ int n; /**< The number of cells in the queue. */ + insertion_time_queue_t *insertion_times; /**< Insertion times of cells. */ } cell_queue_t; /** Beginning of a RELAY cell payload. */ @@ -912,7 +941,7 @@ typedef struct connection_t { * again once the bandwidth throttler allows it? */ unsigned int write_blocked_on_bw:1; /**< Boolean: should we start writing * again once the bandwidth throttler allows - * reads? */ + * writes? */ unsigned int hold_open_until_flushed:1; /**< Despite this connection's being * marked for close, do we flush it * before closing it? */ @@ -937,8 +966,11 @@ typedef struct connection_t { * connection. */ unsigned int linked_conn_is_closed:1; - int s; /**< Our socket; -1 if this connection is closed, or has no - * socket. */ + /** CONNECT/SOCKS proxy client handshake state (for outgoing connections). */ + unsigned int proxy_state:4; + + /** Our socket; -1 if this connection is closed, or has no socket. */ + evutil_socket_t s; int conn_array_index; /**< Index into the global connection array. */ struct event *read_event; /**< Libevent event structure. */ struct event *write_event; /**< Libevent event structure. */ @@ -974,12 +1006,14 @@ typedef struct connection_t { /** Unique identifier for this connection on this Tor instance. */ uint64_t global_identifier; - /* XXXX022 move this field, and all the listener-only fields (just + /* XXXX023 move this field, and all the listener-only fields (just socket_family, I think), into a new listener_connection_t subtype. */ /** If the connection is a CONN_TYPE_AP_DNS_LISTENER, this field points - * to the evdns_server_port is uses to listen to and answer connections. */ + * to the evdns_server_port it uses to listen to and answer connections. */ struct evdns_server_port *dns_server_port; + /** Unique ID for measuring tunneled network status requests. */ + uint64_t dirreq_id; } connection_t; /** Stores flags and information related to the portion of a v2 Tor OR @@ -1026,7 +1060,10 @@ typedef struct or_connection_t { * NETINFO cell listed the address we're connected to as recognized. */ unsigned int is_canonical:1; /** True iff this connection shouldn't get any new circs attached to it, - * because the connection is too old, or because there's a better one, etc. + * because the connection is too old, or because there's a better one. + * More generally, this flag is used to note an unhealthy connection; + * for example, if a bad connection fails we shouldn't assume that the + * router itself has a problem. */ unsigned int is_bad_for_new_circs:1; uint8_t link_proto; /**< What protocol version are we using? 0 for @@ -1041,12 +1078,13 @@ typedef struct or_connection_t { time_t timestamp_last_added_nonpadding; /** When did we last add a * non-padding cell to the outbuf? */ - /* bandwidth* and read_bucket only used by ORs in OPEN state: */ + /* bandwidth* and *_bucket only used by ORs in OPEN state: */ int bandwidthrate; /**< Bytes/s added to the bucket. (OPEN ORs only.) */ int bandwidthburst; /**< Max bucket size for this conn. (OPEN ORs only.) */ int read_bucket; /**< When this hits 0, stop receiving. Every second we * add 'bandwidthrate' to this, capping it at * bandwidthburst. (OPEN ORs only) */ + int write_bucket; /**< When this hits 0, stop writing. Like read_bucket. */ int n_circuits; /**< How many circuits use this connection as p_conn or * n_conn ? */ @@ -1054,6 +1092,17 @@ typedef struct or_connection_t { * free up on this connection's outbuf. Every time we pull cells from a * circuit, we advance this pointer to the next circuit in the ring. */ struct circuit_t *active_circuits; + /** Priority queue of cell_ewma_t for circuits with queued cells waiting for + * room to free up on this connection's outbuf. Kept in heap order + * according to EWMA. + * + * This is redundant with active_circuits; if we ever decide only to use the + * cell_ewma algorithm for choosing circuits, we can remove active_circuits. + */ + smartlist_t *active_circuit_pqueue; + /** The tick on which the cell_ewma_ts in active_circuit_pqueue last had + * their ewma values rescaled. */ + unsigned active_circuit_pqueue_last_recalibrated; struct or_connection_t *next_with_same_id; /**< Next connection with same * identity digest as this one. */ } or_connection_t; @@ -1102,6 +1151,13 @@ typedef struct edge_connection_t { * already retried several times. */ uint8_t num_socks_retries; +#define NUM_CIRCUITS_LAUNCHED_THRESHOLD 10 + /** Number of times we've launched a circuit to handle this stream. If + * it gets too high, that could indicate an inconsistency between our + * "launch a circuit to handle this stream" logic and our "attach our + * stream to one of the available circuits" logic. */ + unsigned int num_circuits_launched:4; + /** True iff this connection is for a DNS request only. */ unsigned int is_dns_request:1; @@ -1128,6 +1184,10 @@ typedef struct edge_connection_t { * zero, abandon the associated mapaddress. */ unsigned int chosen_exit_retries:3; + /** True iff this is an AP connection that came from a transparent or + * NATd connection */ + unsigned int is_transparent_ap:1; + /** If this is a DNSPort connection, this field holds the pending DNS * request that we're going to try to answer. */ struct evdns_server_request *dns_server_request; @@ -1150,7 +1210,8 @@ typedef struct dir_connection_t { enum { DIR_SPOOL_NONE=0, DIR_SPOOL_SERVER_BY_DIGEST, DIR_SPOOL_SERVER_BY_FP, DIR_SPOOL_EXTRA_BY_DIGEST, DIR_SPOOL_EXTRA_BY_FP, - DIR_SPOOL_CACHED_DIR, DIR_SPOOL_NETWORKSTATUS + DIR_SPOOL_CACHED_DIR, DIR_SPOOL_NETWORKSTATUS, + DIR_SPOOL_MICRODESC, /* NOTE: if we add another entry, add another bit. */ } dir_spool_src : 3; /** If we're fetching descriptors, what router purpose shall we assign * to them? */ @@ -1178,15 +1239,12 @@ typedef struct control_connection_t { uint32_t event_mask; /**< Bitfield: which events does this controller * care about? */ - unsigned int use_long_names:1; /**< True if we should use long nicknames - * on this (v1) connection. Only settable - * via v1 controllers. */ - /** For control connections only. If set, we send extended info with control - * events as appropriate. */ - unsigned int use_extended_events:1; /** True if we have sent a protocolinfo reply on this connection. */ unsigned int have_sent_protocolinfo:1; + /** True if we have received a takeownership command on this + * connection. */ + unsigned int is_owning_control_connection:1; /** Amount of space allocated in incoming_cmd. */ uint32_t incoming_cmd_len; @@ -1267,6 +1325,7 @@ typedef struct cached_dir_t { size_t dir_len; /**< Length of <b>dir</b> (not counting its NUL). */ size_t dir_z_len; /**< Length of <b>dir_z</b>. */ time_t published; /**< When was this object published. */ + digests_t digests; /**< Digests of this object (networkstatus only) */ int refcnt; /**< Reference count for this cached_dir_t. */ } cached_dir_t; @@ -1427,6 +1486,9 @@ typedef struct { * directory according to the authorities. */ unsigned int policy_is_reject_star:1; /**< True iff the exit policy for this * router rejects everything. */ + /** True if, after we have added this router, we should re-launch + * tests for it. */ + unsigned int needs_retest_if_added:1; /** Tor can use this router for general positions in circuits. */ #define ROUTER_PURPOSE_GENERAL 0 @@ -1516,6 +1578,9 @@ typedef struct routerstatus_t { unsigned int has_bandwidth:1; /**< The vote/consensus had bw info */ unsigned int has_exitsummary:1; /**< The vote/consensus had exit summaries */ + unsigned int has_measured_bw:1; /**< The vote/consensus had a measured bw */ + + uint32_t measured_bw; /**< Measured bandwidth (capacity) of the router */ uint32_t bandwidth; /**< Bandwidth (capacity) of the router as reported in * the vote/consensus, in kilobytes/sec. */ @@ -1540,6 +1605,52 @@ typedef struct routerstatus_t { } routerstatus_t; +/** A microdescriptor is the smallest amount of information needed to build a + * circuit through a router. They are generated by the directory authorities, + * using information from the uploaded routerinfo documents. They are not + * self-signed, but are rather authenticated by having their hash in a signed + * networkstatus document. */ +typedef struct microdesc_t { + /** Hashtable node, used to look up the microdesc by its digest. */ + HT_ENTRY(microdesc_t) node; + + /* Cache information */ + + /** When was this microdescriptor last listed in a consensus document? + * Once a microdesc has been unlisted long enough, we can drop it. + */ + time_t last_listed; + /** Where is this microdescriptor currently stored? */ + saved_location_t saved_location : 3; + /** If true, do not attempt to cache this microdescriptor on disk. */ + unsigned int no_save : 1; + /** If saved_location == SAVED_IN_CACHE, this field holds the offset of the + * microdescriptor in the cache. */ + off_t off; + + /* The string containing the microdesc. */ + + /** A pointer to the encoded body of the microdescriptor. If the + * saved_location is SAVED_IN_CACHE, then the body is a pointer into an + * mmap'd region. Otherwise, it is a malloc'd string. The string might not + * be NUL-terminated; take the length from <b>bodylen</b>. */ + char *body; + /** The length of the microdescriptor in <b>body</b>. */ + size_t bodylen; + /** A SHA256-digest of the microdescriptor. */ + char digest[DIGEST256_LEN]; + + /* Fields in the microdescriptor. */ + + /** As routerinfo_t.onion_pkey */ + crypto_pk_env_t *onion_pkey; + /** As routerinfo_t.family */ + smartlist_t *family; + /** Encoded exit policy summary */ + char *exitsummary; /**< exit policy summary - + * XXX this probably should not stay a string. */ +} microdesc_t; + /** How many times will we try to download a router's descriptor before giving * up? */ #define MAX_ROUTERDESC_DOWNLOAD_FAILURES 8 @@ -1582,6 +1693,16 @@ typedef struct networkstatus_v2_t { * sorted by identity_digest. */ } networkstatus_v2_t; +/** Linked list of microdesc hash lines for a single router in a directory + * vote. + */ +typedef struct vote_microdesc_hash_t { + /** Next element in the list, or NULL. */ + struct vote_microdesc_hash_t *next; + /** The raw contents of the microdesc hash line, excluding the "m". */ + char *microdesc_hash_line; +} vote_microdesc_hash_t; + /** The claim about a single router, made in a vote. */ typedef struct vote_routerstatus_t { routerstatus_t status; /**< Underlying 'status' object for this router. @@ -1590,31 +1711,46 @@ typedef struct vote_routerstatus_t { * networkstatus_t.known_flags. */ char *version; /**< The version that the authority says this router is * running. */ + /** The hash or hashes that the authority claims this microdesc has. */ + vote_microdesc_hash_t *microdesc; } vote_routerstatus_t; +/** A signature of some document by an authority. */ +typedef struct document_signature_t { + /** Declared SHA-1 digest of this voter's identity key */ + char identity_digest[DIGEST_LEN]; + /** Declared SHA-1 digest of signing key used by this voter. */ + char signing_key_digest[DIGEST_LEN]; + /** Algorithm used to compute the digest of the document. */ + digest_algorithm_t alg; + /** Signature of the signed thing. */ + char *signature; + /** Length of <b>signature</b> */ + int signature_len; + unsigned int bad_signature : 1; /**< Set to true if we've tried to verify + * the sig, and we know it's bad. */ + unsigned int good_signature : 1; /**< Set to true if we've verified the sig + * as good. */ +} document_signature_t; + /** Information about a single voter in a vote or a consensus. */ typedef struct networkstatus_voter_info_t { + /** Declared SHA-1 digest of this voter's identity key */ + char identity_digest[DIGEST_LEN]; char *nickname; /**< Nickname of this voter */ - char identity_digest[DIGEST_LEN]; /**< Digest of this voter's identity key */ + /** Digest of this voter's "legacy" identity key, if any. In vote only; for + * consensuses, we treat legacy keys as additional signers. */ + char legacy_id_digest[DIGEST_LEN]; char *address; /**< Address of this voter, in string format. */ uint32_t addr; /**< Address of this voter, in IPv4, in host order. */ uint16_t dir_port; /**< Directory port of this voter */ uint16_t or_port; /**< OR port of this voter */ char *contact; /**< Contact information for this voter. */ char vote_digest[DIGEST_LEN]; /**< Digest of this voter's vote, as signed. */ - /** Digest of this voter's "legacy" identity key, if any. In vote only; for - * consensuses, we treat legacy keys as additional signers. */ - char legacy_id_digest[DIGEST_LEN]; /* Nothing from here on is signed. */ - char signing_key_digest[DIGEST_LEN]; /**< Declared digest of signing key - * used by this voter. */ - char *signature; /**< Signature from this voter. */ - int signature_len; /**< Length of <b>signature</b> */ - unsigned int bad_signature : 1; /**< Set to true if we've tried to verify - * the sig, and we know it's bad. */ - unsigned int good_signature : 1; /**< Set to true if we've verified the sig - * as good. */ + /** The signature of the document and the signature's status. */ + smartlist_t *sigs; } networkstatus_voter_info_t; /** Enumerates the possible seriousness values of a networkstatus document. */ @@ -1624,10 +1760,25 @@ typedef enum { NS_TYPE_OPINION, } networkstatus_type_t; +/** Enumerates recognized flavors of a consensus networkstatus document. All + * flavors of a consensus are generated from the same set of votes, but they + * present different types information to different versions of Tor. */ +typedef enum { + FLAV_NS = 0, + FLAV_MICRODESC = 1, +} consensus_flavor_t; + +/** Which consensus flavor do we actually want to use to build circuits? */ +#define USABLE_CONSENSUS_FLAVOR FLAV_NS + +/** How many different consensus flavors are there? */ +#define N_CONSENSUS_FLAVORS ((int)(FLAV_MICRODESC)+1) + /** A common structure to hold a v3 network status vote, or a v3 network * status consensus. */ typedef struct networkstatus_t { - networkstatus_type_t type; /**< Vote, consensus, or opinion? */ + networkstatus_type_t type : 8; /**< Vote, consensus, or opinion? */ + consensus_flavor_t flavor : 8; /**< If a consensus, what kind? */ time_t published; /**< Vote only: Time when vote was written. */ time_t valid_after; /**< Time after which this vote or consensus applies. */ time_t fresh_until; /**< Time before which this is the most recent vote or @@ -1659,6 +1810,10 @@ typedef struct networkstatus_t { * consensus, sorted by key. */ smartlist_t *net_params; + /** List of key=value strings for the bw weight parameters in the + * consensus. */ + smartlist_t *weight_params; + /** List of networkstatus_voter_info_t. For a vote, only one element * is included. For a consensus, one element is included for every voter * whose vote contributed to the consensus. */ @@ -1666,8 +1821,8 @@ typedef struct networkstatus_t { struct authority_cert_t *cert; /**< Vote only: the voter's certificate. */ - /** Digest of this document, as signed. */ - char networkstatus_digest[DIGEST_LEN]; + /** Digests of this document, as signed. */ + digests_t digests; /** List of router statuses, sorted by identity digest. For a vote, * the elements are vote_routerstatus_t; for a consensus, the elements @@ -1679,14 +1834,15 @@ typedef struct networkstatus_t { digestmap_t *desc_digest_map; } networkstatus_t; -/** A set of signatures for a networkstatus consensus. All fields are as for - * networkstatus_t. */ +/** A set of signatures for a networkstatus consensus. Unless otherwise + * noted, all fields are as for networkstatus_t. */ typedef struct ns_detached_signatures_t { time_t valid_after; time_t fresh_until; time_t valid_until; - char networkstatus_digest[DIGEST_LEN]; - smartlist_t *signatures; /* list of networkstatus_voter_info_t */ + strmap_t *digests; /**< Map from flavor name to digestset_t */ + strmap_t *signatures; /**< Map from flavor name to list of + * document_signature_t */ } ns_detached_signatures_t; /** Allowable types of desc_store_t. */ @@ -1891,6 +2047,29 @@ typedef struct { time_t expiry_time; } cpath_build_state_t; +/** + * The cell_ewma_t structure keeps track of how many cells a circuit has + * transferred recently. It keeps an EWMA (exponentially weighted moving + * average) of the number of cells flushed from the circuit queue onto a + * connection in connection_or_flush_from_first_active_circuit(). + */ +typedef struct { + /** The last 'tick' at which we recalibrated cell_count. + * + * A cell sent at exactly the start of this tick has weight 1.0. Cells sent + * since the start of this tick have weight greater than 1.0; ones sent + * earlier have less weight. */ + unsigned last_adjusted_tick; + /** The EWMA of the cell count. */ + double cell_count; + /** True iff this is the cell count for a circuit's previous + * connection. */ + unsigned int is_for_p_conn : 1; + /** The position of the circuit within the OR connection's priority + * queue. */ + int heap_index; +} cell_ewma_t; + #define ORIGIN_CIRCUIT_MAGIC 0x35315243u #define OR_CIRCUIT_MAGIC 0x98ABC04Fu @@ -1957,9 +2136,15 @@ typedef struct circuit_t { * length ONIONSKIN_CHALLENGE_LEN. */ char *n_conn_onionskin; - time_t timestamp_created; /**< When was this circuit created? */ - time_t timestamp_dirty; /**< When the circuit was first used, or 0 if the - * circuit is clean. */ + struct timeval timestamp_created; /**< When was the circuit created? */ + /** When the circuit was first used, or 0 if the circuit is clean. + * + * XXXX023 Note that some code will artifically adjust this value backward + * in time in order to indicate that a circuit shouldn't be used for new + * streams, but that it can stay alive as long as it has streams on it. + * That's a kludge we should fix. + */ + time_t timestamp_dirty; uint16_t marked_for_close; /**< Should we close this circuit at the end of * the main loop? (If true, holds the line number @@ -1976,6 +2161,14 @@ typedef struct circuit_t { * linked to an OR connection. */ struct circuit_t *prev_active_on_n_conn; struct circuit_t *next; /**< Next circuit in linked list of all circuits. */ + + /** Unique ID for measuring tunneled network status requests. */ + uint64_t dirreq_id; + + /** The EWMA count for the number of cells flushed from the + * n_conn_cells queue. Used to determine which circuit to flush from next. + */ + cell_ewma_t n_cell_ewma; } circuit_t; /** Largest number of relay_early cells that we can send on a given @@ -2008,6 +2201,13 @@ typedef struct origin_circuit_t { * to the specification? */ unsigned int remaining_relay_early_cells : 4; + /** Set if this circuit is insanely old and we already informed the user */ + unsigned int is_ancient : 1; + + /** Set if this circuit has already been opened. Used to detect + * cannibalized circuits. */ + unsigned int has_opened : 1; + /** What commands were sent over this circuit that decremented the * RELAY_EARLY counter? This is for debugging task 878. */ uint8_t relay_early_commands[MAX_RELAY_EARLY_CELLS_PER_CIRCUIT]; @@ -2098,6 +2298,19 @@ typedef struct or_circuit_t { /** True iff this circuit was made with a CREATE_FAST cell. */ unsigned int is_first_hop : 1; + + /** Number of cells that were removed from circuit queue; reset every + * time when writing buffer stats to disk. */ + uint32_t processed_cells; + + /** Total time in milliseconds that cells spent in both app-ward and + * exit-ward queues of this circuit; reset every time when writing + * buffer stats to disk. */ + uint64_t total_cell_waiting_time; + + /** The EWMA count for the number of cells flushed from the + * p_conn_cells queue. */ + cell_ewma_t p_cell_ewma; } or_circuit_t; /** Convert a circuit subtype to a circuit_t.*/ @@ -2143,6 +2356,10 @@ typedef struct config_line_t { typedef struct routerset_t routerset_t; +/** A magic value for the (Socks|OR|...)Port options below, telling Tor + * to pick its own port. */ +#define CFG_AUTO_PORT 0xc4005e + /** Configuration options for a Tor process. */ typedef struct { uint32_t _magic; @@ -2157,6 +2374,9 @@ typedef struct { config_line_t *Logs; /**< New-style list of configuration lines * for logs */ + int LogMessageDomains; /**< Boolean: Should we log the domain(s) in which + * each log message occurs? */ + char *DebugLogFile; /**< Where to send verbose log messages. */ char *DataDirectory; /**< OR only: where to store long-term data. */ char *Nickname; /**< OR only: nickname of this onion router. */ @@ -2169,19 +2389,22 @@ typedef struct { routerset_t *EntryNodes;/**< Structure containing nicknames, digests, * country codes and IP address patterns of ORs to * consider as entry points. */ - int StrictExitNodes; /**< Boolean: When none of our ExitNodes are up, do we - * stop building circuits? */ - int StrictEntryNodes; /**< Boolean: When none of our EntryNodes are up, do we - * stop building circuits? */ + int StrictNodes; /**< Boolean: When none of our EntryNodes or ExitNodes + * are up, or we need to access a node in ExcludeNodes, + * do we just fail instead? */ routerset_t *ExcludeNodes;/**< Structure containing nicknames, digests, * country codes and IP address patterns of ORs - * not to use in circuits. */ + * not to use in circuits. But see StrictNodes + * above. */ routerset_t *ExcludeExitNodes;/**< Structure containing nicknames, digests, * country codes and IP address patterns of * ORs not to consider as exits. */ /** Union of ExcludeNodes and ExcludeExitNodes */ - struct routerset_t *_ExcludeExitNodesUnion; + routerset_t *_ExcludeExitNodesUnion; + + int DisableAllSwap; /**< Boolean: Attempt to call mlockall() on our + * process for all current and future memory. */ /** List of "entry", "middle", "exit", "introduction", "rendezvous". */ smartlist_t *AllowInvalidNodes; @@ -2197,7 +2420,7 @@ typedef struct { * connections. */ config_line_t *TransListenAddress; /** Addresses to bind for listening for transparent natd connections */ - config_line_t *NatdListenAddress; + config_line_t *NATDListenAddress; /** Addresses to bind for listening for SOCKS connections. */ config_line_t *DNSListenAddress; /** Addresses to bind for listening for OR connections. */ @@ -2221,10 +2444,11 @@ typedef struct { int SocksPort; /**< Port to listen on for SOCKS connections. */ /** Port to listen on for transparent pf/netfilter connections. */ int TransPort; - int NatdPort; /**< Port to listen on for transparent natd connections. */ + int NATDPort; /**< Port to listen on for transparent natd connections. */ int ControlPort; /**< Port to listen on for control connections. */ config_line_t *ControlSocket; /**< List of Unix Domain Sockets to listen on * for control connections. */ + int ControlSocketsGroupWritable; /**< Boolean: Are control sockets g+rw? */ int DirPort; /**< Port to listen on for directory connections. */ int DNSPort; /**< Port to listen on for DNS requests. */ int AssumeReachable; /**< Whether to publish our descriptor regardless. */ @@ -2237,8 +2461,6 @@ typedef struct { * for version 3 directories? */ int HSAuthoritativeDir; /**< Boolean: does this an authoritative directory * handle hidden service requests? */ - int HSAuthorityRecordStats; /**< Boolean: does this HS authoritative - * directory record statistics? */ int NamingAuthoritativeDir; /**< Boolean: is this an authoritative directory * that's willing to bind names? */ int VersioningAuthoritativeDir; /**< Boolean: is this an authoritative @@ -2267,8 +2489,6 @@ typedef struct { int AvoidDiskWrites; /**< Boolean: should we never cache things to disk? * Not used yet. */ int ClientOnly; /**< Boolean: should we never evolve into a server role? */ - /** Boolean: should we never publish a descriptor? Deprecated. */ - int NoPublish; /** To what authority types do we publish our descriptor? Choices are * "v1", "v2", "v3", "bridge", or "". */ smartlist_t *PublishServerDescriptor; @@ -2277,7 +2497,9 @@ typedef struct { /** Boolean: do we publish hidden service descriptors to the HS auths? */ int PublishHidServDescriptors; int FetchServerDescriptors; /**< Do we fetch server descriptors as normal? */ - int FetchHidServDescriptors; /** and hidden service descriptors? */ + int FetchHidServDescriptors; /**< and hidden service descriptors? */ + int FetchV2Networkstatus; /**< Do we fetch v2 networkstatus documents when + * we don't need to? */ int HidServDirectoryV2; /**< Do we participate in the HS DHT? */ int MinUptimeHidServDirectoryV2; /**< As directory authority, accept hidden @@ -2299,6 +2521,14 @@ typedef struct { int ConstrainedSockets; /**< Shrink xmit and recv socket buffers. */ uint64_t ConstrainedSockSize; /**< Size of constrained buffers. */ + /** Whether we should drop exit streams from Tors that we don't know are + * relays. One of "0" (never refuse), "1" (always refuse), or "auto" (do + * what the consensus says, defaulting to 'refuse' if the consensus says + * nothing). */ + char *RefuseUnknownExits; + /** Parsed version of RefuseUnknownExits. -1 for auto. */ + int RefuseUnknownExits_; + /** Application ports that require all nodes in circ to have sufficient * uptime. */ smartlist_t *LongLivedPorts; @@ -2328,10 +2558,18 @@ typedef struct { * connections alive? */ int SocksTimeout; /**< How long do we let a socks connection wait * unattached before we fail it? */ - int CircuitBuildTimeout; /**< Cull non-open circuits that were born - * at least this many seconds ago. */ + int LearnCircuitBuildTimeout; /**< If non-zero, we attempt to learn a value + * for CircuitBuildTimeout based on timeout + * history */ + int CircuitBuildTimeout; /**< Cull non-open circuits that were born at + * least this many seconds ago. Used until + * adaptive algorithm learns a new value. */ int CircuitIdleTimeout; /**< Cull open clean circuits that were born * at least this many seconds ago. */ + int CircuitStreamTimeout; /**< If non-zero, detach streams from circuits + * and try a new circuit if the stream has been + * waiting for this many seconds. If zero, use + * our default internal timeout schedule. */ int MaxOnionsPending; /**< How many circuit CREATE requests do we allow * to wait simultaneously before we start dropping * them? */ @@ -2349,24 +2587,36 @@ typedef struct { * willing to use for all relayed conns? */ uint64_t RelayBandwidthBurst; /**< How much bandwidth, at maximum, will we * use in a second for all relayed conns? */ - int NumCpus; /**< How many CPUs should we try to use? */ - int RunTesting; /**< If true, create testing circuits to measure how well the - * other ORs are running. */ + uint64_t PerConnBWRate; /**< Long-term bw on a single TLS conn, if set. */ + uint64_t PerConnBWBurst; /**< Allowed burst on a single TLS conn, if set. */ + int NumCPUs; /**< How many CPUs should we try to use? */ +//int RunTesting; /**< If true, create testing circuits to measure how well the +// * other ORs are running. */ config_line_t *RendConfigLines; /**< List of configuration lines * for rendezvous services. */ config_line_t *HidServAuth; /**< List of configuration lines for client-side * authorizations for hidden services */ char *ContactInfo; /**< Contact info to be published in the directory. */ - char *HttpProxy; /**< hostname[:port] to use as http proxy, if any. */ - uint32_t HttpProxyAddr; /**< Parsed IPv4 addr for http proxy, if any. */ - uint16_t HttpProxyPort; /**< Parsed port for http proxy, if any. */ - char *HttpProxyAuthenticator; /**< username:password string, if any. */ + char *HTTPProxy; /**< hostname[:port] to use as http proxy, if any. */ + tor_addr_t HTTPProxyAddr; /**< Parsed IPv4 addr for http proxy, if any. */ + uint16_t HTTPProxyPort; /**< Parsed port for http proxy, if any. */ + char *HTTPProxyAuthenticator; /**< username:password string, if any. */ - char *HttpsProxy; /**< hostname[:port] to use as https proxy, if any. */ - uint32_t HttpsProxyAddr; /**< Parsed IPv4 addr for https proxy, if any. */ - uint16_t HttpsProxyPort; /**< Parsed port for https proxy, if any. */ - char *HttpsProxyAuthenticator; /**< username:password string, if any. */ + char *HTTPSProxy; /**< hostname[:port] to use as https proxy, if any. */ + tor_addr_t HTTPSProxyAddr; /**< Parsed addr for https proxy, if any. */ + uint16_t HTTPSProxyPort; /**< Parsed port for https proxy, if any. */ + char *HTTPSProxyAuthenticator; /**< username:password string, if any. */ + + char *Socks4Proxy; /**< hostname:port to use as a SOCKS4 proxy, if any. */ + tor_addr_t Socks4ProxyAddr; /**< Derived from Socks4Proxy. */ + uint16_t Socks4ProxyPort; /**< Derived from Socks4Proxy. */ + + char *Socks5Proxy; /**< hostname:port to use as a SOCKS5 proxy, if any. */ + tor_addr_t Socks5ProxyAddr; /**< Derived from Sock5Proxy. */ + uint16_t Socks5ProxyPort; /**< Derived from Socks5Proxy. */ + char *Socks5ProxyUsername; /**< Username for SOCKS5 authentication, if any */ + char *Socks5ProxyPassword; /**< Password for SOCKS5 authentication, if any */ /** List of configuration lines for replacement directory authorities. * If you just want to replace one class of authority at a time, @@ -2427,10 +2677,20 @@ typedef struct { int DisablePredictedCircuits; /**< Boolean: does Tor preemptively * make circuits in the background (0), * or not (1)? */ + + /** Process specifier for a controller that ‘owns’ this Tor + * instance. Tor will terminate if its owning controller does. */ + char *OwningControllerProcess; + int ShutdownWaitLength; /**< When we get a SIGINT and we're a server, how * long do we wait before exiting? */ - int SafeLogging; /**< Boolean: are we allowed to log sensitive strings - * such as addresses (0), or do we scrub them first (1)? */ + char *SafeLogging; /**< Contains "relay", "1", "0" (meaning no scrubbing). */ + + /* Derived from SafeLogging */ + enum { + SAFELOG_SCRUB_ALL, SAFELOG_SCRUB_RELAY, SAFELOG_SCRUB_NONE + } _SafeLogging; + int SafeSocks; /**< Boolean: should we outright refuse application * connections that use socks4 or socks5-with-local-dns? */ #define LOG_PROTOCOL_WARN (get_options()->ProtocolWarnings ? \ @@ -2441,6 +2701,8 @@ typedef struct { * log whether it was DNS-leaking or not? */ int HardwareAccel; /**< Boolean: Should we enable OpenSSL hardware * acceleration where available? */ + char *AccelName; /**< Optional hardware acceleration engine name. */ + char *AccelDir; /**< Optional hardware acceleration engine search dir. */ int UseEntryGuards; /**< Boolean: Do we try to enter from a smallish number * of fixed nodes? */ int NumEntryGuards; /**< How many entry guards do we try to establish? */ @@ -2451,6 +2713,9 @@ typedef struct { * means directly from the authorities) no matter our other config? */ int FetchDirInfoEarly; + /** Should we fetch our dir info at the start of the consensus period? */ + int FetchDirInfoExtraEarly; + char *VirtualAddrNetwork; /**< Address and mask to hand out for virtual * MAPADDRESS requests. */ int ServerDNSSearchDomains; /**< Boolean: If set, we don't force exit @@ -2499,11 +2764,42 @@ typedef struct { * exit allows it, we use it. */ int AllowSingleHopCircuits; + /** If true, we convert "www.google.com.foo.exit" addresses on the + * socks/trans/natd ports into "www.google.com" addresses that + * exit from the node "foo". Disabled by default since attacking + * websites and exit relays can use it to manipulate your path + * selection. */ + int AllowDotExit; + + /** If true, we will warn if a user gives us only an IP address + * instead of a hostname. */ + int WarnUnsafeSocks; + + /** If true, the user wants us to collect statistics on clients + * requesting network statuses from us as directory. */ + int DirReqStatistics; + + /** If true, the user wants us to collect statistics on port usage. */ + int ExitPortStatistics; + + /** If true, the user wants us to collect cell statistics. */ + int CellStatistics; + + /** If true, the user wants us to collect statistics as entry node. */ + int EntryStatistics; + + /** If true, include statistics file contents in extra-info documents. */ + int ExtraInfoStatistics; + /** If true, do not believe anybody who tells us that a domain resolves * to an internal address, or that an internal address has a PTR mapping. * Helps avoid some cross-site attacks. */ int ClientDNSRejectInternalAddresses; + /** If true, do not accept any requests to connect to internal addresses + * over randomly chosen exits. */ + int ClientRejectInternalAddresses; + /** The length of time that we think a consensus should be fresh. */ int V3AuthVotingInterval; /** The length of time we think it will take to distribute votes. */ @@ -2517,6 +2813,13 @@ typedef struct { * migration purposes? */ int V3AuthUseLegacyKey; + /** Location of bandwidth measurement file */ + char *V3BandwidthsFile; + + /** Authority only: key=value pairs that we add to our networkstatus + * consensus vote on the 'params' line. */ + char *ConsensusParams; + /** The length of time that we think an initial consensus should be fresh. * Only altered on testing networks. */ int TestingV3AuthInitialVotingInterval; @@ -2553,19 +2856,6 @@ typedef struct { * the bridge authority guess which countries have blocked access to us. */ int BridgeRecordUsageByCountry; -#ifdef ENABLE_GEOIP_STATS - /** If true, and Tor is built with GEOIP_STATS support, and we're a - * directory, record how many directory requests we get from each country. */ - int DirRecordUsageByCountry; - /** Round all GeoIP results to the next multiple of this value, to avoid - * leaking information. */ - int DirRecordUsageGranularity; - /** Time interval: purge geoip stats after this long. */ - int DirRecordUsageRetainIPs; - /** Time interval: Flush geoip data to disk this often. */ - int DirRecordUsageSaveInterval; -#endif - /** Optionally, a file with GeoIP data. */ char *GeoIPFile; @@ -2573,6 +2863,31 @@ typedef struct { * to make this false. */ int ReloadTorrcOnSIGHUP; + /* The main parameter for picking circuits within a connection. + * + * If this value is positive, when picking a cell to relay on a connection, + * we always relay from the circuit whose weighted cell count is lowest. + * Cells are weighted exponentially such that if one cell is sent + * 'CircuitPriorityHalflife' seconds before another, it counts for half as + * much. + * + * If this value is zero, we're disabling the cell-EWMA algorithm. + * + * If this value is negative, we're using the default approach + * according to either Tor or a parameter set in the consensus. + */ + double CircuitPriorityHalflife; + + /** Set to true if the TestingTorNetwork configuration option is set. + * This is used so that options_validate() has a chance to realize that + * the defaults have changed. */ + int _UsingTestNetworkDefaults; + + /** File where we should write the ControlPort. */ + char *ControlPortWriteToFile; + /** Should that file be group-readable? */ + int ControlPortFileGroupReadable; + } or_options_t; /** Persistent state for an onion router, as saved to disk. */ @@ -2591,6 +2906,9 @@ typedef struct { uint64_t AccountingBytesReadInInterval; uint64_t AccountingBytesWrittenInInterval; int AccountingSecondsActive; + int AccountingSecondsToReachSoftLimit; + time_t AccountingSoftLimitHitAt; + uint64_t AccountingBytesAtSoftLimit; uint64_t AccountingExpectedUsage; /** A list of Entry Guard-related configuration lines. */ @@ -2601,13 +2919,30 @@ typedef struct { * bandwidth usage. The "Interval" fields hold the granularity, in seconds, * of the entries of Values. The "Values" lists hold decimal string * representations of the number of bytes read or written in each - * interval. */ + * interval. The "Maxima" list holds decimal strings describing the highest + * rate achieved during the interval. + */ time_t BWHistoryReadEnds; int BWHistoryReadInterval; smartlist_t *BWHistoryReadValues; + smartlist_t *BWHistoryReadMaxima; time_t BWHistoryWriteEnds; int BWHistoryWriteInterval; smartlist_t *BWHistoryWriteValues; + smartlist_t *BWHistoryWriteMaxima; + time_t BWHistoryDirReadEnds; + int BWHistoryDirReadInterval; + smartlist_t *BWHistoryDirReadValues; + smartlist_t *BWHistoryDirReadMaxima; + time_t BWHistoryDirWriteEnds; + int BWHistoryDirWriteInterval; + smartlist_t *BWHistoryDirWriteValues; + smartlist_t *BWHistoryDirWriteMaxima; + + /** Build time histogram */ + config_line_t * BuildtimeHistogram; + unsigned int TotalBuildTimes; + unsigned int CircuitBuildAbandonedCount; /** What version of Tor wrote this state file? */ char *TorVersion; @@ -2669,209 +3004,143 @@ struct socks_request_t { /* all the function prototypes go here */ -/********************************* buffers.c ***************************/ - -buf_t *buf_new(void); -buf_t *buf_new_with_capacity(size_t size); -void buf_free(buf_t *buf); -void buf_clear(buf_t *buf); -void buf_shrink(buf_t *buf); -void buf_shrink_freelists(int free_all); -void buf_dump_freelist_sizes(int severity); - -size_t buf_datalen(const buf_t *buf); -size_t buf_allocation(const buf_t *buf); -size_t buf_slack(const buf_t *buf); -const char *_buf_peek_raw_buffer(const buf_t *buf); - -int read_to_buf(int s, size_t at_most, buf_t *buf, int *reached_eof, - int *socket_error); -int read_to_buf_tls(tor_tls_t *tls, size_t at_most, buf_t *buf); - -int flush_buf(int s, buf_t *buf, size_t sz, size_t *buf_flushlen); -int flush_buf_tls(tor_tls_t *tls, buf_t *buf, size_t sz, size_t *buf_flushlen); - -int write_to_buf(const char *string, size_t string_len, buf_t *buf); -int write_to_buf_zlib(buf_t *buf, tor_zlib_state_t *state, - const char *data, size_t data_len, int done); -int move_buf_to_buf(buf_t *buf_out, buf_t *buf_in, size_t *buf_flushlen); -int fetch_from_buf(char *string, size_t string_len, buf_t *buf); -int fetch_var_cell_from_buf(buf_t *buf, var_cell_t **out, int linkproto); -int fetch_from_buf_http(buf_t *buf, - char **headers_out, size_t max_headerlen, - char **body_out, size_t *body_used, size_t max_bodylen, - int force_complete); -int fetch_from_buf_socks(buf_t *buf, socks_request_t *req, - int log_sockstype, int safe_socks); -int fetch_from_buf_line(buf_t *buf, char *data_out, size_t *data_len); - -int peek_buf_has_control0_command(buf_t *buf); - -void assert_buf_ok(buf_t *buf); - -#ifdef BUFFERS_PRIVATE -int buf_find_string_offset(const buf_t *buf, const char *s, size_t n); -#endif - /********************************* circuitbuild.c **********************/ -char *circuit_list_path(origin_circuit_t *circ, int verbose); -char *circuit_list_path_for_controller(origin_circuit_t *circ); -void circuit_log_path(int severity, unsigned int domain, - origin_circuit_t *circ); -void circuit_rep_hist_note_result(origin_circuit_t *circ); -origin_circuit_t *origin_circuit_init(uint8_t purpose, int flags); -origin_circuit_t *circuit_establish_circuit(uint8_t purpose, - extend_info_t *exit, - int flags); -int circuit_handle_first_hop(origin_circuit_t *circ); -void circuit_n_conn_done(or_connection_t *or_conn, int status); -int inform_testing_reachability(void); -int circuit_send_next_onion_skin(origin_circuit_t *circ); -void circuit_note_clock_jumped(int seconds_elapsed); -int circuit_extend(cell_t *cell, circuit_t *circ); -int circuit_init_cpath_crypto(crypt_path_t *cpath, const char *key_data, - int reverse); -int circuit_finish_handshake(origin_circuit_t *circ, uint8_t cell_type, - const uint8_t *reply); -int circuit_truncated(origin_circuit_t *circ, crypt_path_t *layer); -int onionskin_answer(or_circuit_t *circ, uint8_t cell_type, - const char *payload, const char *keys); -int circuit_all_predicted_ports_handled(time_t now, int *need_uptime, - int *need_capacity); - -int circuit_append_new_exit(origin_circuit_t *circ, extend_info_t *info); -int circuit_extend_to_new_exit(origin_circuit_t *circ, extend_info_t *info); -void onion_append_to_cpath(crypt_path_t **head_ptr, crypt_path_t *new_hop); -extend_info_t *extend_info_alloc(const char *nickname, const char *digest, - crypto_pk_env_t *onion_key, - const tor_addr_t *addr, uint16_t port); -extend_info_t *extend_info_from_router(routerinfo_t *r); -extend_info_t *extend_info_dup(extend_info_t *info); -void extend_info_free(extend_info_t *info); -routerinfo_t *build_state_get_exit_router(cpath_build_state_t *state); -const char *build_state_get_exit_nickname(cpath_build_state_t *state); - -void entry_guards_compute_status(void); -int entry_guard_register_connect_status(const char *digest, int succeeded, - int mark_relay_status, time_t now); -void entry_nodes_should_be_added(void); -int entry_list_can_grow(or_options_t *options); -routerinfo_t *choose_random_entry(cpath_build_state_t *state); -int entry_guards_parse_state(or_state_t *state, int set, char **msg); -void entry_guards_update_state(or_state_t *state); -int getinfo_helper_entry_guards(control_connection_t *conn, - const char *question, char **answer); - -void clear_bridge_list(void); -int routerinfo_is_a_configured_bridge(routerinfo_t *ri); -void bridge_add_from_config(const tor_addr_t *addr, uint16_t port, - char *digest); -void retry_bridge_descriptor_fetch_directly(const char *digest); -void fetch_bridge_descriptors(time_t now); -void learned_bridge_descriptor(routerinfo_t *ri, int from_cache); -int any_bridge_descriptors_known(void); -int any_pending_bridge_descriptor_fetches(void); -int bridges_known_but_down(void); -void bridges_retry_all(void); - -void entry_guards_free_all(void); - -/********************************* circuitlist.c ***********************/ - -circuit_t * _circuit_get_global_list(void); -const char *circuit_state_to_string(int state); -const char *circuit_purpose_to_controller_string(uint8_t purpose); -void circuit_dump_by_conn(connection_t *conn, int severity); -void circuit_set_p_circid_orconn(or_circuit_t *circ, circid_t id, - or_connection_t *conn); -void circuit_set_n_circid_orconn(circuit_t *circ, circid_t id, - or_connection_t *conn); -void circuit_set_state(circuit_t *circ, uint8_t state); -void circuit_close_all_marked(void); -int32_t circuit_initial_package_window(void); -origin_circuit_t *origin_circuit_new(void); -or_circuit_t *or_circuit_new(circid_t p_circ_id, or_connection_t *p_conn); -circuit_t *circuit_get_by_circid_orconn(circid_t circ_id, - or_connection_t *conn); -int circuit_id_in_use_on_orconn(circid_t circ_id, or_connection_t *conn); -circuit_t *circuit_get_by_edge_conn(edge_connection_t *conn); -void circuit_unlink_all_from_or_conn(or_connection_t *conn, int reason); -origin_circuit_t *circuit_get_by_global_id(uint32_t id); -origin_circuit_t *circuit_get_by_rend_query_and_purpose(const char *rend_query, - uint8_t purpose); -origin_circuit_t *circuit_get_next_by_pk_and_purpose(origin_circuit_t *start, - const char *digest, uint8_t purpose); -or_circuit_t *circuit_get_rendezvous(const char *cookie); -or_circuit_t *circuit_get_intro_point(const char *digest); -origin_circuit_t *circuit_find_to_cannibalize(uint8_t purpose, - extend_info_t *info, int flags); -void circuit_mark_all_unused_circs(void); -void circuit_expire_all_dirty_circs(void); -void _circuit_mark_for_close(circuit_t *circ, int reason, - int line, const char *file); -int circuit_get_cpath_len(origin_circuit_t *circ); -crypt_path_t *circuit_get_cpath_hop(origin_circuit_t *circ, int hopnum); -void circuit_get_all_pending_on_or_conn(smartlist_t *out, - or_connection_t *or_conn); -int circuit_count_pending_on_or_conn(or_connection_t *or_conn); - -#define circuit_mark_for_close(c, reason) \ - _circuit_mark_for_close((c), (reason), __LINE__, _SHORT_FILE_) - -void assert_cpath_layer_ok(const crypt_path_t *cp); -void assert_circuit_ok(const circuit_t *c); -void circuit_free_all(void); - -/********************************* circuituse.c ************************/ - -void circuit_expire_building(time_t now); -void circuit_remove_handled_ports(smartlist_t *needed_ports); -int circuit_stream_is_being_handled(edge_connection_t *conn, uint16_t port, - int min); -int circuit_conforms_to_options(const origin_circuit_t *circ, - const or_options_t *options); -void circuit_build_needed_circs(time_t now); -void circuit_detach_stream(circuit_t *circ, edge_connection_t *conn); - -void circuit_expire_old_circuits_serverside(time_t now); - -void reset_bandwidth_test(void); -int circuit_enough_testing_circs(void); - -void circuit_has_opened(origin_circuit_t *circ); -void circuit_build_failed(origin_circuit_t *circ); - -/** Flag to set when a circuit should have only a single hop. */ -#define CIRCLAUNCH_ONEHOP_TUNNEL (1<<0) -/** Flag to set when a circuit needs to be built of high-uptime nodes */ -#define CIRCLAUNCH_NEED_UPTIME (1<<1) -/** Flag to set when a circuit needs to be built of high-capacity nodes */ -#define CIRCLAUNCH_NEED_CAPACITY (1<<2) -/** Flag to set when the last hop of a circuit doesn't need to be an - * exit node. */ -#define CIRCLAUNCH_IS_INTERNAL (1<<3) -origin_circuit_t *circuit_launch_by_extend_info(uint8_t purpose, - extend_info_t *info, - int flags); -origin_circuit_t *circuit_launch_by_router(uint8_t purpose, - routerinfo_t *exit, int flags); -void circuit_reset_failure_count(int timeout); -int connection_ap_handshake_attach_chosen_circuit(edge_connection_t *conn, - origin_circuit_t *circ, - crypt_path_t *cpath); -int connection_ap_handshake_attach_circuit(edge_connection_t *conn); - -/********************************* command.c ***************************/ - -void command_process_cell(cell_t *cell, or_connection_t *conn); -void command_process_var_cell(var_cell_t *cell, or_connection_t *conn); - -extern uint64_t stats_n_padding_cells_processed; -extern uint64_t stats_n_create_cells_processed; -extern uint64_t stats_n_created_cells_processed; -extern uint64_t stats_n_relay_cells_processed; -extern uint64_t stats_n_destroy_cells_processed; +/** How many hops does a general-purpose circuit have by default? */ +#define DEFAULT_ROUTE_LEN 3 + +/* Circuit Build Timeout "public" structures. */ + +/** Precision multiplier for the Bw weights */ +#define BW_WEIGHT_SCALE 10000 +#define BW_MIN_WEIGHT_SCALE 1 +#define BW_MAX_WEIGHT_SCALE INT32_MAX + +/** Total size of the circuit timeout history to accumulate. + * 1000 is approx 2.5 days worth of continual-use circuits. */ +#define CBT_NCIRCUITS_TO_OBSERVE 1000 + +/** Width of the histogram bins in milliseconds */ +#define CBT_BIN_WIDTH ((build_time_t)50) + +/** Number of modes to use in the weighted-avg computation of Xm */ +#define CBT_DEFAULT_NUM_XM_MODES 3 +#define CBT_MIN_NUM_XM_MODES 1 +#define CBT_MAX_NUM_XM_MODES 20 + +/** A build_time_t is milliseconds */ +typedef uint32_t build_time_t; + +/** + * CBT_BUILD_ABANDONED is our flag value to represent a force-closed + * circuit (Aka a 'right-censored' pareto value). + */ +#define CBT_BUILD_ABANDONED ((build_time_t)(INT32_MAX-1)) +#define CBT_BUILD_TIME_MAX ((build_time_t)(INT32_MAX)) + +/** Save state every 10 circuits */ +#define CBT_SAVE_STATE_EVERY 10 + +/* Circuit build times consensus parameters */ + +/** + * How long to wait before actually closing circuits that take too long to + * build in terms of CDF quantile. + */ +#define CBT_DEFAULT_CLOSE_QUANTILE 95 +#define CBT_MIN_CLOSE_QUANTILE CBT_MIN_QUANTILE_CUTOFF +#define CBT_MAX_CLOSE_QUANTILE CBT_MAX_QUANTILE_CUTOFF + +/** + * How many circuits count as recent when considering if the + * connection has gone gimpy or changed. + */ +#define CBT_DEFAULT_RECENT_CIRCUITS 20 +#define CBT_MIN_RECENT_CIRCUITS 3 +#define CBT_MAX_RECENT_CIRCUITS 1000 + +/** + * Maximum count of timeouts that finish the first hop in the past + * RECENT_CIRCUITS before calculating a new timeout. + * + * This tells us whether to abandon timeout history and set + * the timeout back to whatever circuit_build_times_get_initial_timeout() + * gives us. + */ +#define CBT_DEFAULT_MAX_RECENT_TIMEOUT_COUNT (CBT_DEFAULT_RECENT_CIRCUITS*9/10) +#define CBT_MIN_MAX_RECENT_TIMEOUT_COUNT 3 +#define CBT_MAX_MAX_RECENT_TIMEOUT_COUNT 10000 + +/** Minimum circuits before estimating a timeout */ +#define CBT_DEFAULT_MIN_CIRCUITS_TO_OBSERVE 100 +#define CBT_MIN_MIN_CIRCUITS_TO_OBSERVE 1 +#define CBT_MAX_MIN_CIRCUITS_TO_OBSERVE 10000 + +/** Cutoff percentile on the CDF for our timeout estimation. */ +#define CBT_DEFAULT_QUANTILE_CUTOFF 80 +#define CBT_MIN_QUANTILE_CUTOFF 10 +#define CBT_MAX_QUANTILE_CUTOFF 99 +double circuit_build_times_quantile_cutoff(void); + +/** How often in seconds should we build a test circuit */ +#define CBT_DEFAULT_TEST_FREQUENCY 60 +#define CBT_MIN_TEST_FREQUENCY 1 +#define CBT_MAX_TEST_FREQUENCY INT32_MAX + +/** Lowest allowable value for CircuitBuildTimeout in milliseconds */ +#define CBT_DEFAULT_TIMEOUT_MIN_VALUE (1500) +#define CBT_MIN_TIMEOUT_MIN_VALUE 500 +#define CBT_MAX_TIMEOUT_MIN_VALUE INT32_MAX + +/** Initial circuit build timeout in milliseconds */ +#define CBT_DEFAULT_TIMEOUT_INITIAL_VALUE (60*1000) +#define CBT_MIN_TIMEOUT_INITIAL_VALUE CBT_MIN_TIMEOUT_MIN_VALUE +#define CBT_MAX_TIMEOUT_INITIAL_VALUE INT32_MAX +int32_t circuit_build_times_initial_timeout(void); + +#if CBT_DEFAULT_MAX_RECENT_TIMEOUT_COUNT < CBT_MIN_MAX_RECENT_TIMEOUT_COUNT +#error "RECENT_CIRCUITS is set too low." +#endif + +/** Information about the state of our local network connection */ +typedef struct { + /** The timestamp we last completed a TLS handshake or received a cell */ + time_t network_last_live; + /** If the network is not live, how many timeouts has this caused? */ + int nonlive_timeouts; + /** Circular array of circuits that have made it to the first hop. Slot is + * 1 if circuit timed out, 0 if circuit succeeded */ + int8_t *timeouts_after_firsthop; + /** Number of elements allocated for the above array */ + int num_recent_circs; + /** Index into circular array. */ + int after_firsthop_idx; +} network_liveness_t; + +/** Structure for circuit build times history */ +typedef struct { + /** The circular array of recorded build times in milliseconds */ + build_time_t circuit_build_times[CBT_NCIRCUITS_TO_OBSERVE]; + /** Current index in the circuit_build_times circular array */ + int build_times_idx; + /** Total number of build times accumulated. Max CBT_NCIRCUITS_TO_OBSERVE */ + int total_build_times; + /** Information about the state of our local network connection */ + network_liveness_t liveness; + /** Last time we built a circuit. Used to decide to build new test circs */ + time_t last_circ_at; + /** "Minimum" value of our pareto distribution (actually mode) */ + build_time_t Xm; + /** alpha exponent for pareto dist. */ + double alpha; + /** Have we computed a timeout? */ + int have_computed_timeout; + /** The exact value for that timeout in milliseconds. Stored as a double + * to maintain precision from calculations to and from quantile value. */ + double timeout_ms; + /** How long we wait before actually closing the circuit. */ + double close_ms; +} circuit_build_times_t; /********************************* config.c ***************************/ @@ -2884,207 +3153,15 @@ typedef enum setopt_err_t { SETOPT_ERR_SETTING = -4, } setopt_err_t; -const char *get_dirportfrontpage(void); -or_options_t *get_options(void); -int set_options(or_options_t *new_val, char **msg); -void config_free_all(void); -const char *safe_str(const char *address); -const char *escaped_safe_str(const char *address); -const char *get_version(void); - -int config_get_lines(const char *string, config_line_t **result); -void config_free_lines(config_line_t *front); -setopt_err_t options_trial_assign(config_line_t *list, int use_defaults, - int clear_first, char **msg); -int resolve_my_address(int warn_severity, or_options_t *options, - uint32_t *addr, char **hostname_out); -int is_local_addr(const tor_addr_t *addr) ATTR_PURE; -void options_init(or_options_t *options); -int options_init_from_torrc(int argc, char **argv); -setopt_err_t options_init_from_string(const char *cf, - int command, const char *command_arg, char **msg); -int option_is_recognized(const char *key); -const char *option_get_canonical_name(const char *key); -config_line_t *option_get_assignment(or_options_t *options, - const char *key); -int options_save_current(void); -const char *get_torrc_fname(void); -char *options_get_datadir_fname2_suffix(or_options_t *options, - const char *sub1, const char *sub2, - const char *suffix); -#define get_datadir_fname2_suffix(sub1, sub2, suffix) \ - options_get_datadir_fname2_suffix(get_options(), (sub1), (sub2), (suffix)) -/** Return a newly allocated string containing datadir/sub1. See - * get_datadir_fname2_suffix. */ -#define get_datadir_fname(sub1) get_datadir_fname2_suffix((sub1), NULL, NULL) -/** Return a newly allocated string containing datadir/sub1/sub2. See - * get_datadir_fname2_suffix. */ -#define get_datadir_fname2(sub1,sub2) \ - get_datadir_fname2_suffix((sub1), (sub2), NULL) -/** Return a newly allocated string containing datadir/sub1suffix. See - * get_datadir_fname2_suffix. */ -#define get_datadir_fname_suffix(sub1, suffix) \ - get_datadir_fname2_suffix((sub1), NULL, (suffix)) - -or_state_t *get_or_state(void); -int or_state_save(time_t now); - -int options_need_geoip_info(or_options_t *options, const char **reason_out); -int getinfo_helper_config(control_connection_t *conn, - const char *question, char **answer); - -uint32_t get_effective_bwrate(or_options_t *options); -uint32_t get_effective_bwburst(or_options_t *options); - -#ifdef CONFIG_PRIVATE -/* Used only by config.c and test.c */ -or_options_t *options_new(void); -#endif - -/********************************* connection.c ***************************/ - -const char *conn_type_to_string(int type); -const char *conn_state_to_string(int type, int state); - -dir_connection_t *dir_connection_new(int socket_family); -or_connection_t *or_connection_new(int socket_family); -edge_connection_t *edge_connection_new(int type, int socket_family); -control_connection_t *control_connection_new(int socket_family); -connection_t *connection_new(int type, int socket_family); - -void connection_link_connections(connection_t *conn_a, connection_t *conn_b); -void connection_unregister_events(connection_t *conn); -void connection_free(connection_t *conn); -void connection_free_all(void); -void connection_about_to_close_connection(connection_t *conn); -void connection_close_immediate(connection_t *conn); -void _connection_mark_for_close(connection_t *conn,int line, const char *file); - -#define connection_mark_for_close(c) \ - _connection_mark_for_close((c), __LINE__, _SHORT_FILE_) - -void connection_expire_held_open(void); - -int connection_connect(connection_t *conn, const char *address, - const tor_addr_t *addr, - uint16_t port, int *socket_error); -int retry_all_listeners(smartlist_t *replaced_conns, - smartlist_t *new_conns); - -ssize_t connection_bucket_write_limit(connection_t *conn, time_t now); -int global_write_bucket_low(connection_t *conn, size_t attempt, int priority); -void connection_bucket_init(void); -void connection_bucket_refill(int seconds_elapsed, time_t now); - -int connection_handle_read(connection_t *conn); - -int connection_fetch_from_buf(char *string, size_t len, connection_t *conn); - -int connection_wants_to_flush(connection_t *conn); -int connection_outbuf_too_full(connection_t *conn); -int connection_handle_write(connection_t *conn, int force); -void _connection_write_to_buf_impl(const char *string, size_t len, - connection_t *conn, int zlib); -static void connection_write_to_buf(const char *string, size_t len, - connection_t *conn); -static void connection_write_to_buf_zlib(const char *string, size_t len, - dir_connection_t *conn, int done); -static INLINE void -connection_write_to_buf(const char *string, size_t len, connection_t *conn) -{ - _connection_write_to_buf_impl(string, len, conn, 0); -} -static INLINE void -connection_write_to_buf_zlib(const char *string, size_t len, - dir_connection_t *conn, int done) -{ - _connection_write_to_buf_impl(string, len, TO_CONN(conn), done ? -1 : 1); -} - -connection_t *connection_get_by_global_id(uint64_t id); - -connection_t *connection_get_by_type(int type); -connection_t *connection_get_by_type_purpose(int type, int purpose); -connection_t *connection_get_by_type_addr_port_purpose(int type, - const tor_addr_t *addr, - uint16_t port, int purpose); -connection_t *connection_get_by_type_state(int type, int state); -connection_t *connection_get_by_type_state_rendquery(int type, int state, - const char *rendquery, - int rendversion); - -#define connection_speaks_cells(conn) ((conn)->type == CONN_TYPE_OR) -int connection_is_listener(connection_t *conn); -int connection_state_is_open(connection_t *conn); -int connection_state_is_connecting(connection_t *conn); - -char *alloc_http_authenticator(const char *authenticator); - -void assert_connection_ok(connection_t *conn, time_t now); -int connection_or_nonopen_was_started_here(or_connection_t *conn); -void connection_dump_buffer_mem_stats(int severity); -void remove_file_if_very_old(const char *fname, time_t now); - /********************************* connection_edge.c *************************/ -#define connection_mark_unattached_ap(conn, endreason) \ - _connection_mark_unattached_ap((conn), (endreason), __LINE__, _SHORT_FILE_) - -void _connection_mark_unattached_ap(edge_connection_t *conn, int endreason, - int line, const char *file); -int connection_edge_reached_eof(edge_connection_t *conn); -int connection_edge_process_inbuf(edge_connection_t *conn, - int package_partial); -int connection_edge_destroy(circid_t circ_id, edge_connection_t *conn); -int connection_edge_end(edge_connection_t *conn, uint8_t reason); -int connection_edge_end_errno(edge_connection_t *conn); -int connection_edge_finished_flushing(edge_connection_t *conn); -int connection_edge_finished_connecting(edge_connection_t *conn); - -int connection_ap_handshake_send_begin(edge_connection_t *ap_conn); -int connection_ap_handshake_send_resolve(edge_connection_t *ap_conn); - -edge_connection_t *connection_ap_make_link(char *address, uint16_t port, - const char *digest, - int use_begindir, int want_onehop); -void connection_ap_handshake_socks_reply(edge_connection_t *conn, char *reply, - size_t replylen, - int endreason); -void connection_ap_handshake_socks_resolved(edge_connection_t *conn, - int answer_type, - size_t answer_len, - const uint8_t *answer, - int ttl, - time_t expires); - -int connection_exit_begin_conn(cell_t *cell, circuit_t *circ); -int connection_exit_begin_resolve(cell_t *cell, or_circuit_t *circ); -void connection_exit_connect(edge_connection_t *conn); -int connection_edge_is_rendezvous_stream(edge_connection_t *conn); -int connection_ap_can_use_exit(edge_connection_t *conn, routerinfo_t *exit); -void connection_ap_expire_beginning(void); -void connection_ap_attach_pending(void); -void connection_ap_fail_onehop(const char *failed_digest, - cpath_build_state_t *build_state); -void circuit_discard_optional_exit_enclaves(extend_info_t *info); -int connection_ap_detach_retriable(edge_connection_t *conn, - origin_circuit_t *circ, - int reason); -int connection_ap_process_transparent(edge_connection_t *conn); - -int address_is_invalid_destination(const char *address, int client); - -void addressmap_init(void); -void addressmap_clean(time_t now); -void addressmap_clear_configured(void); -void addressmap_clear_transient(void); -void addressmap_free_all(void); -int addressmap_rewrite(char *address, size_t maxlen, time_t *expires_out); -int addressmap_have_mapping(const char *address, int update_timeout); /** Enumerates possible origins of a client-side address mapping. */ typedef enum { /** We're remapping this address because the controller told us to. */ ADDRMAPSRC_CONTROLLER, + /** We're remapping this address because of an AutomapHostsOnResolve + * configuration. */ + ADDRMAPSRC_AUTOMAP, /** We're remapping this address because our configuration (via torrc, the * command line, or a SETCONF command) told us to. */ ADDRMAPSRC_TORRC, @@ -3095,75 +3172,6 @@ typedef enum { * Tor server that told us what its value was. */ ADDRMAPSRC_DNS, } addressmap_entry_source_t; -void addressmap_register(const char *address, char *new_address, - time_t expires, addressmap_entry_source_t source); -int parse_virtual_addr_network(const char *val, int validate_only, - char **msg); -int client_dns_incr_failures(const char *address); -void client_dns_clear_failures(const char *address); -void client_dns_set_addressmap(const char *address, uint32_t val, - const char *exitname, int ttl); -const char *addressmap_register_virtual_address(int type, char *new_address); -void addressmap_get_mappings(smartlist_t *sl, time_t min_expires, - time_t max_expires, int want_expiry); -int connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, - origin_circuit_t *circ, - crypt_path_t *cpath); -int hostname_is_noconnect_address(const char *address); - -/** Possible return values for parse_extended_hostname. */ -typedef enum hostname_type_t { - NORMAL_HOSTNAME, ONION_HOSTNAME, EXIT_HOSTNAME, BAD_HOSTNAME -} hostname_type_t; -hostname_type_t parse_extended_hostname(char *address); - -#if defined(HAVE_NET_IF_H) && defined(HAVE_NET_PFVAR_H) -int get_pf_socket(void); -#endif - -/********************************* connection_or.c ***************************/ - -void connection_or_remove_from_identity_map(or_connection_t *conn); -void connection_or_clear_identity_map(void); -or_connection_t *connection_or_get_for_extend(const char *digest, - const tor_addr_t *target_addr, - const char **msg_out, - int *launch_out); -void connection_or_set_bad_connections(void); - -int connection_or_reached_eof(or_connection_t *conn); -int connection_or_process_inbuf(or_connection_t *conn); -int connection_or_flushed_some(or_connection_t *conn); -int connection_or_finished_flushing(or_connection_t *conn); -int connection_or_finished_connecting(or_connection_t *conn); - -void connection_or_connect_failed(or_connection_t *conn, - int reason, const char *msg); -or_connection_t *connection_or_connect(const tor_addr_t *addr, uint16_t port, - const char *id_digest); - -int connection_tls_start_handshake(or_connection_t *conn, int receiving); -int connection_tls_continue_handshake(or_connection_t *conn); - -void or_handshake_state_free(or_handshake_state_t *state); -int connection_or_set_state_open(or_connection_t *conn); -void connection_or_write_cell_to_buf(const cell_t *cell, - or_connection_t *conn); -void connection_or_write_var_cell_to_buf(const var_cell_t *cell, - or_connection_t *conn); -int connection_or_send_destroy(circid_t circ_id, or_connection_t *conn, - int reason); -int connection_or_send_netinfo(or_connection_t *conn); -int connection_or_send_cert(or_connection_t *conn); -int connection_or_send_link_auth(or_connection_t *conn); -int connection_or_compute_link_auth_hmac(or_connection_t *conn, - char *hmac_out); -int is_or_protocol_version_known(uint16_t version); - -void cell_pack(packed_cell_t *dest, const cell_t *src); -void var_cell_pack_header(const var_cell_t *cell, char *hdr_out); -var_cell_t *var_cell_new(uint16_t payload_len); -void var_cell_free(var_cell_t *cell); /********************************* control.c ***************************/ @@ -3201,8 +3209,14 @@ typedef enum or_conn_status_event_t { OR_CONN_EVENT_NEW = 4, } or_conn_status_event_t; -void control_update_global_event_mask(void); -void control_adjust_event_log_severity(void); +/** Used to indicate the type of a buildtime event */ +typedef enum buildtimeout_set_event_t { + BUILDTIMEOUT_SET_EVENT_COMPUTED = 0, + BUILDTIMEOUT_SET_EVENT_RESET = 1, + BUILDTIMEOUT_SET_EVENT_SUSPENDED = 2, + BUILDTIMEOUT_SET_EVENT_DISCARD = 3, + BUILDTIMEOUT_SET_EVENT_RESUME = 4 +} buildtimeout_set_event_t; /** Execute the statement <b>stmt</b>, which may log events concerning the * connection <b>conn</b>. To prevent infinite loops, disable log messages @@ -3212,7 +3226,9 @@ void control_adjust_event_log_severity(void); */ #define CONN_LOG_PROTECT(conn, stmt) \ STMT_BEGIN \ - int _log_conn_is_control = (conn && conn->type == CONN_TYPE_CONTROL); \ + int _log_conn_is_control; \ + tor_assert(conn); \ + _log_conn_is_control = (conn->type == CONN_TYPE_CONTROL); \ if (_log_conn_is_control) \ disable_control_logging(); \ STMT_BEGIN stmt; STMT_END; \ @@ -3220,58 +3236,6 @@ void control_adjust_event_log_severity(void); enable_control_logging(); \ STMT_END -/** Log information about the connection <b>conn</b>, protecting it as with - * CONN_LOG_PROTECT. Example: - * - * LOG_FN_CONN(conn, (LOG_DEBUG, "Socket %d wants to write", conn->s)); - **/ -#define LOG_FN_CONN(conn, args) \ - CONN_LOG_PROTECT(conn, log_fn args) - -int connection_control_finished_flushing(control_connection_t *conn); -int connection_control_reached_eof(control_connection_t *conn); -int connection_control_process_inbuf(control_connection_t *conn); - -#define EVENT_AUTHDIR_NEWDESCS 0x000D -#define EVENT_NS 0x000F -int control_event_is_interesting(int event); - -int control_event_circuit_status(origin_circuit_t *circ, - circuit_status_event_t e, int reason); -int control_event_stream_status(edge_connection_t *conn, - stream_status_event_t e, - int reason); -int control_event_or_conn_status(or_connection_t *conn, - or_conn_status_event_t e, int reason); -int control_event_bandwidth_used(uint32_t n_read, uint32_t n_written); -int control_event_stream_bandwidth(edge_connection_t *edge_conn); -int control_event_stream_bandwidth_used(void); -void control_event_logmsg(int severity, unsigned int domain, const char *msg); -int control_event_descriptors_changed(smartlist_t *routers); -int control_event_address_mapped(const char *from, const char *to, - time_t expires, const char *error); -int control_event_or_authdir_new_descriptor(const char *action, - const char *desc, - size_t desclen, - const char *msg); -int control_event_my_descriptor_changed(void); -int control_event_networkstatus_changed(smartlist_t *statuses); -int control_event_newconsensus(const networkstatus_t *consensus); -int control_event_networkstatus_changed_single(routerstatus_t *rs); -int control_event_general_status(int severity, const char *format, ...) - CHECK_PRINTF(2,3); -int control_event_client_status(int severity, const char *format, ...) - CHECK_PRINTF(2,3); -int control_event_server_status(int severity, const char *format, ...) - CHECK_PRINTF(2,3); -int control_event_guard(const char *nickname, const char *digest, - const char *status); - -int init_cookie_authentication(int enabled); -smartlist_t *decode_hashed_passwords(config_line_t *passwords); -void disable_control_logging(void); -void enable_control_logging(void); - /** Enum describing various stages of bootstrapping, for use with controller * bootstrap status events. The values range from 0 to 100. */ typedef enum { @@ -3292,427 +3256,133 @@ typedef enum { BOOTSTRAP_STATUS_DONE=100 } bootstrap_status_t; -void control_event_bootstrap(bootstrap_status_t status, int progress); -void control_event_bootstrap_problem(const char *warn, int reason); - -void control_event_clients_seen(const char *timestarted, - const char *countries); - -#ifdef CONTROL_PRIVATE -/* Used only by control.c and test.c */ -size_t write_escaped_data(const char *data, size_t len, char **out); -size_t read_escaped_data(const char *data, size_t len, char **out); -#endif - -/********************************* cpuworker.c *****************************/ - -void cpu_init(void); -void cpuworkers_rotate(void); -int connection_cpu_finished_flushing(connection_t *conn); -int connection_cpu_reached_eof(connection_t *conn); -int connection_cpu_process_inbuf(connection_t *conn); -int assign_onionskin_to_cpuworker(connection_t *cpuworker, - or_circuit_t *circ, - char *onionskin); - /********************************* directory.c ***************************/ -int directories_have_accepted_server_descriptor(void); -char *authority_type_to_string(authority_type_t auth); -void directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose, - authority_type_t type, const char *payload, - size_t payload_len, size_t extrainfo_len); -void directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose, - const char *resource, - int pds_flags); -void directory_get_from_all_authorities(uint8_t dir_purpose, - uint8_t router_purpose, - const char *resource); -void directory_initiate_command_routerstatus(routerstatus_t *status, - uint8_t dir_purpose, - uint8_t router_purpose, - int anonymized_connection, - const char *resource, - const char *payload, - size_t payload_len, - time_t if_modified_since); -void directory_initiate_command_routerstatus_rend(routerstatus_t *status, - uint8_t dir_purpose, - uint8_t router_purpose, - int anonymized_connection, - const char *resource, - const char *payload, - size_t payload_len, - time_t if_modified_since, - const rend_data_t *rend_query); - -int parse_http_response(const char *headers, int *code, time_t *date, - compress_method_t *compression, char **response); - -int connection_dir_is_encrypted(dir_connection_t *conn); -int connection_dir_reached_eof(dir_connection_t *conn); -int connection_dir_process_inbuf(dir_connection_t *conn); -int connection_dir_finished_flushing(dir_connection_t *conn); -int connection_dir_finished_connecting(dir_connection_t *conn); -void connection_dir_request_failed(dir_connection_t *conn); -void directory_initiate_command(const char *address, const tor_addr_t *addr, - uint16_t or_port, uint16_t dir_port, - int supports_conditional_consensus, - int supports_begindir, const char *digest, - uint8_t dir_purpose, uint8_t router_purpose, - int anonymized_connection, - const char *resource, - const char *payload, size_t payload_len, - time_t if_modified_since); - -int dir_split_resource_into_fingerprints(const char *resource, - smartlist_t *fp_out, int *compresseed_out, - int decode_hex, int sort_uniq); /** A pair of digests created by dir_split_resource_info_fingerprint_pairs() */ typedef struct { char first[DIGEST_LEN]; char second[DIGEST_LEN]; } fp_pair_t; -int dir_split_resource_into_fingerprint_pairs(const char *res, - smartlist_t *pairs_out); -char *directory_dump_request_log(void); -void note_request(const char *key, size_t bytes); -int router_supports_extrainfo(const char *identity_digest, int is_authority); - -time_t download_status_increment_failure(download_status_t *dls, - int status_code, const char *item, - int server, time_t now); -/** Increment the failure count of the download_status_t <b>dls</b>, with - * the optional status code <b>sc</b>. */ -#define download_status_failed(dls, sc) \ - download_status_increment_failure((dls), (sc), NULL, \ - get_options()->DirPort, time(NULL)) - -void download_status_reset(download_status_t *dls); -static int download_status_is_ready(download_status_t *dls, time_t now, - int max_failures); -/** Return true iff, as of <b>now</b>, the resource tracked by <b>dls</b> is - * ready to get its download reattempted. */ -static INLINE int -download_status_is_ready(download_status_t *dls, time_t now, - int max_failures) -{ - return (dls->n_download_failures <= max_failures - && dls->next_attempt_at <= now); -} - -static void download_status_mark_impossible(download_status_t *dl); -/** Mark <b>dl</b> as never downloadable. */ -static INLINE void -download_status_mark_impossible(download_status_t *dl) -{ - dl->n_download_failures = IMPOSSIBLE_TO_DOWNLOAD; -} /********************************* dirserv.c ***************************/ -/** Maximum length of an exit policy summary. */ -#define MAX_EXITPOLICY_SUMMARY_LEN (1000) - -/** Maximum allowable length of a version line in a networkstatus. */ -#define MAX_V_LINE_LEN 128 -/** Length of "r Authority BadDirectory BadExit Exit Fast Guard HSDir Named - * Running Stable Unnamed V2Dir Valid\n". */ -#define MAX_FLAG_LINE_LEN 96 -/** Length of "w" line for weighting. Currently at most - * "w Bandwidth=<uint32t>\n" */ -#define MAX_WEIGHT_LINE_LEN (13+10) -/** Maximum length of an exit policy summary line. */ -#define MAX_POLICY_LINE_LEN (3+MAX_EXITPOLICY_SUMMARY_LEN) -/** Amount of space to allocate for each entry: r, s, and v lines. */ -#define RS_ENTRY_LEN \ - ( /* first line */ \ - MAX_NICKNAME_LEN+BASE64_DIGEST_LEN*2+ISO_TIME_LEN+INET_NTOA_BUF_LEN+ \ - 5*2 /* ports */ + 10 /* punctuation */ + \ - /* second line */ \ - MAX_FLAG_LINE_LEN + \ - /* weight line */ \ - MAX_WEIGHT_LINE_LEN + \ - /* p line. */ \ - MAX_POLICY_LINE_LEN + \ - /* v line. */ \ - MAX_V_LINE_LEN \ - ) -#define UNNAMED_ROUTER_NICKNAME "Unnamed" - -int connection_dirserv_flushed_some(dir_connection_t *conn); - -int dirserv_add_own_fingerprint(const char *nickname, crypto_pk_env_t *pk); -int dirserv_load_fingerprint_file(void); -void dirserv_free_fingerprint_list(void); -const char *dirserv_get_nickname_by_digest(const char *digest); -enum was_router_added_t dirserv_add_multiple_descriptors( - const char *desc, uint8_t purpose, - const char *source, - const char **msg); -enum was_router_added_t dirserv_add_descriptor(routerinfo_t *ri, - const char **msg, - const char *source); -int getinfo_helper_dirserv_unregistered(control_connection_t *conn, - const char *question, char **answer); -void dirserv_free_descriptors(void); -void dirserv_set_router_is_running(routerinfo_t *router, time_t now); -int list_server_status_v1(smartlist_t *routers, char **router_status_out, - int for_controller); -int dirserv_dump_directory_to_string(char **dir_out, - crypto_pk_env_t *private_key); - -int directory_fetches_from_authorities(or_options_t *options); -int directory_fetches_dir_info_early(or_options_t *options); -int directory_fetches_dir_info_later(or_options_t *options); -int directory_caches_v2_dir_info(or_options_t *options); -#define directory_caches_v1_dir_info(o) directory_caches_v2_dir_info(o) -int directory_caches_dir_info(or_options_t *options); -int directory_permits_begindir_requests(or_options_t *options); -int directory_permits_controller_requests(or_options_t *options); -int directory_too_idle_to_fetch_descriptors(or_options_t *options, time_t now); - -void directory_set_dirty(void); -cached_dir_t *dirserv_get_directory(void); -cached_dir_t *dirserv_get_runningrouters(void); -cached_dir_t *dirserv_get_consensus(void); -void dirserv_set_cached_directory(const char *directory, time_t when, - int is_running_routers); -void dirserv_set_cached_networkstatus_v2(const char *directory, - const char *identity, - time_t published); -void dirserv_set_cached_networkstatus_v3(const char *consensus, - time_t published); -void dirserv_clear_old_networkstatuses(time_t cutoff); -void dirserv_clear_old_v1_info(time_t now); -void dirserv_get_networkstatus_v2(smartlist_t *result, const char *key); -void dirserv_get_networkstatus_v2_fingerprints(smartlist_t *result, - const char *key); -int dirserv_get_routerdesc_fingerprints(smartlist_t *fps_out, const char *key, - const char **msg, - int for_unencrypted_conn, - int is_extrainfo); -int dirserv_get_routerdescs(smartlist_t *descs_out, const char *key, - const char **msg); -void dirserv_orconn_tls_done(const char *address, - uint16_t or_port, - const char *digest_rcvd, - int as_advertised); -void dirserv_test_reachability(time_t now, int try_all); -int authdir_wants_to_reject_router(routerinfo_t *ri, const char **msg, - int complain); -int dirserv_would_reject_router(routerstatus_t *rs); -int dirserv_remove_old_statuses(smartlist_t *fps, time_t cutoff); -int dirserv_have_any_serverdesc(smartlist_t *fps, int spool_src); -size_t dirserv_estimate_data_size(smartlist_t *fps, int is_serverdescs, - int compressed); -int routerstatus_format_entry(char *buf, size_t buf_len, - routerstatus_t *rs, const char *platform, - int first_line_only, int v2_format); -void dirserv_free_all(void); -void cached_dir_decref(cached_dir_t *d); -cached_dir_t *new_cached_dir(char *s, time_t published); -/********************************* dirvote.c ************************/ +/** An enum to describe what format we're generating a routerstatus line in. + */ +typedef enum { + /** For use in a v2 opinion */ + NS_V2, + /** For use in a consensus networkstatus document (ns flavor) */ + NS_V3_CONSENSUS, + /** For use in a vote networkstatus document */ + NS_V3_VOTE, + /** For passing to the controlport in response to a GETINFO request */ + NS_CONTROL_PORT, + /** For use in a consensus networkstatus document (microdesc flavor) */ + NS_V3_CONSENSUS_MICRODESC +} routerstatus_format_type_t; + +#ifdef DIRSERV_PRIVATE +typedef struct measured_bw_line_t { + char node_id[DIGEST_LEN]; + char node_hex[MAX_HEX_NICKNAME_LEN+1]; + long int bw; +} measured_bw_line_t; -/** Lowest allowable value for VoteSeconds. */ -#define MIN_VOTE_SECONDS 20 -/** Lowest allowable value for DistSeconds. */ -#define MIN_DIST_SECONDS 20 -/** Smallest allowable voting interval. */ -#define MIN_VOTE_INTERVAL 300 - -void dirvote_free_all(void); - -/* vote manipulation */ -char *networkstatus_compute_consensus(smartlist_t *votes, - int total_authorities, - crypto_pk_env_t *identity_key, - crypto_pk_env_t *signing_key, - const char *legacy_identity_key_digest, - crypto_pk_env_t *legacy_signing_key); -int networkstatus_add_detached_signatures(networkstatus_t *target, - ns_detached_signatures_t *sigs, - const char **msg_out); -char *networkstatus_get_detached_signatures(networkstatus_t *consensus); -void ns_detached_signatures_free(ns_detached_signatures_t *s); - -/* cert manipulation */ -authority_cert_t *authority_cert_dup(authority_cert_t *cert); +#endif + +/********************************* dirvote.c ************************/ /** Describes the schedule by which votes should be generated. */ typedef struct vote_timing_t { + /** Length in seconds between one consensus becoming valid and the next + * becoming valid. */ int vote_interval; + /** For how many intervals is a consensus valid? */ int n_intervals_valid; + /** Time in seconds allowed to propagate votes */ int vote_delay; + /** Time in seconds allowed to propagate signatures */ int dist_delay; } vote_timing_t; -/* vote scheduling */ -void dirvote_get_preferred_voting_intervals(vote_timing_t *timing_out); -time_t dirvote_get_start_of_next_interval(time_t now, int interval); -void dirvote_recalculate_timing(or_options_t *options, time_t now); -void dirvote_act(or_options_t *options, time_t now); - -/* invoked on timers and by outside triggers. */ -struct pending_vote_t * dirvote_add_vote(const char *vote_body, - const char **msg_out, - int *status_out); -int dirvote_add_signatures(const char *detached_signatures_body, - const char *source, - const char **msg_out); - -/* Item access */ -const char *dirvote_get_pending_consensus(void); -const char *dirvote_get_pending_detached_signatures(void); -#define DGV_BY_ID 1 -#define DGV_INCLUDE_PENDING 2 -#define DGV_INCLUDE_PREVIOUS 4 -const cached_dir_t *dirvote_get_vote(const char *fp, int flags); -void set_routerstatus_from_routerinfo(routerstatus_t *rs, - routerinfo_t *ri, time_t now, - int naming, int exits_can_be_guards, - int listbadexits, int listbaddirs); -void router_clear_status_flags(routerinfo_t *ri); -networkstatus_t * -dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key, - authority_cert_t *cert); - -#ifdef DIRVOTE_PRIVATE -char *format_networkstatus_vote(crypto_pk_env_t *private_key, - networkstatus_t *v3_ns); -char *dirvote_compute_params(smartlist_t *votes); -#endif - -/********************************* dns.c ***************************/ - -int dns_init(void); -int has_dns_init_failed(void); -void dns_free_all(void); -uint32_t dns_clip_ttl(uint32_t ttl); -int dns_reset(void); -void connection_dns_remove(edge_connection_t *conn); -void assert_connection_edge_not_dns_pending(edge_connection_t *conn); -void assert_all_pending_dns_resolves_ok(void); -void dns_cancel_pending_resolve(const char *question); -int dns_resolve(edge_connection_t *exitconn); -void dns_launch_correctness_checks(void); -int dns_seems_to_be_broken(void); -void dns_reset_correctness_checks(void); - -/********************************* dnsserv.c ************************/ - -void dnsserv_configure_listener(connection_t *conn); -void dnsserv_close_listener(connection_t *conn); -void dnsserv_resolved(edge_connection_t *conn, - int answer_type, - size_t answer_len, - const char *answer, - int ttl); -void dnsserv_reject_request(edge_connection_t *conn); -int dnsserv_launch_request(const char *name, int is_reverse); /********************************* geoip.c **************************/ -#ifdef GEOIP_PRIVATE -int geoip_parse_entry(const char *line); -#endif -int should_record_bridge_info(or_options_t *options); -int geoip_load_file(const char *filename, or_options_t *options); -int geoip_get_country_by_ip(uint32_t ipaddr); -int geoip_get_n_countries(void); -const char *geoip_get_country_name(country_t num); -int geoip_is_loaded(void); -country_t geoip_get_country(const char *countrycode); +/** Round all GeoIP results to the next multiple of this value, to avoid + * leaking information. */ +#define DIR_RECORD_USAGE_GRANULARITY 8 +/** Time interval: Flush geoip data to disk this often. */ +#define DIR_ENTRY_RECORD_USAGE_RETAIN_IPS (24*60*60) +/** How long do we have to have observed per-country request history before + * we are willing to talk about it? */ +#define DIR_RECORD_USAGE_MIN_OBSERVATION_TIME (12*60*60) + /** Indicates an action that we might be noting geoip statistics on. * Note that if we're noticing CONNECT, we're a bridge, and if we're noticing * the others, we're not. */ typedef enum { - /** We've noticed a connection as a bridge relay. */ + /** We've noticed a connection as a bridge relay or entry guard. */ GEOIP_CLIENT_CONNECT = 0, /** We've served a networkstatus consensus as a directory server. */ GEOIP_CLIENT_NETWORKSTATUS = 1, /** We've served a v2 networkstatus consensus as a directory server. */ GEOIP_CLIENT_NETWORKSTATUS_V2 = 2, } geoip_client_action_t; -void geoip_note_client_seen(geoip_client_action_t action, - uint32_t addr, time_t now); -void geoip_remove_old_clients(time_t cutoff); -time_t geoip_get_history_start(void); -char *geoip_get_client_history(time_t now, geoip_client_action_t action); -char *geoip_get_request_history(time_t now, geoip_client_action_t action); -int getinfo_helper_geoip(control_connection_t *control_conn, - const char *question, char **answer); -void geoip_free_all(void); -void dump_geoip_stats(void); - -/********************************* hibernate.c **********************/ - -int accounting_parse_options(or_options_t *options, int validate_only); -int accounting_is_enabled(or_options_t *options); -void configure_accounting(time_t now); -void accounting_run_housekeeping(time_t now); -void accounting_add_bytes(size_t n_read, size_t n_written, int seconds); -int accounting_record_bandwidth_usage(time_t now, or_state_t *state); -void hibernate_begin_shutdown(void); -int we_are_hibernating(void); -void consider_hibernation(time_t now); -int getinfo_helper_accounting(control_connection_t *conn, - const char *question, char **answer); -void accounting_set_bandwidth_usage_from_state(or_state_t *state); - -/********************************* main.c ***************************/ - -extern int has_completed_circuit; - -int connection_add(connection_t *conn); -int connection_remove(connection_t *conn); -int connection_in_array(connection_t *conn); -void add_connection_to_closeable_list(connection_t *conn); -int connection_is_on_closeable_list(connection_t *conn); - -smartlist_t *get_connection_array(void); - -void connection_watch_events(connection_t *conn, short events); -int connection_is_reading(connection_t *conn); -void connection_stop_reading(connection_t *conn); -void connection_start_reading(connection_t *conn); - -int connection_is_writing(connection_t *conn); -void connection_stop_writing(connection_t *conn); -void connection_start_writing(connection_t *conn); - -void connection_stop_reading_from_linked_conn(connection_t *conn); - -void directory_all_unreachable(time_t now); -void directory_info_has_arrived(time_t now, int from_cache); - -void ip_address_changed(int at_interface); -void dns_servers_relaunch_checks(void); - -void control_signal_act(int the_signal); -void handle_signals(int is_parent); - -int try_locking(or_options_t *options, int err_if_locked); -int have_lockfile(void); -void release_lockfile(void); - -void tor_cleanup(void); -void tor_free_all(int postfork); - -int tor_main(int argc, char *argv[]); - -#ifdef MAIN_PRIVATE -int do_main_loop(void); -int do_list_fingerprint(void); -void do_hash_password(void); -int tor_init(int argc, char **argv); -#endif +/** Indicates either a positive reply or a reason for rejectng a network + * status request that will be included in geoip statistics. */ +typedef enum { + /** Request is answered successfully. */ + GEOIP_SUCCESS = 0, + /** V3 network status is not signed by a sufficient number of requested + * authorities. */ + GEOIP_REJECT_NOT_ENOUGH_SIGS = 1, + /** Requested network status object is unavailable. */ + GEOIP_REJECT_UNAVAILABLE = 2, + /** Requested network status not found. */ + GEOIP_REJECT_NOT_FOUND = 3, + /** Network status has not been modified since If-Modified-Since time. */ + GEOIP_REJECT_NOT_MODIFIED = 4, + /** Directory is busy. */ + GEOIP_REJECT_BUSY = 5, +} geoip_ns_response_t; +#define GEOIP_NS_RESPONSE_NUM 6 + +/** Directory requests that we are measuring can be either direct or + * tunneled. */ +typedef enum { + DIRREQ_DIRECT = 0, + DIRREQ_TUNNELED = 1, +} dirreq_type_t; -/********************************* networkstatus.c *********************/ +/** Possible states for either direct or tunneled directory requests that + * are relevant for determining network status download times. */ +typedef enum { + /** Found that the client requests a network status; applies to both + * direct and tunneled requests; initial state of a request that we are + * measuring. */ + DIRREQ_IS_FOR_NETWORK_STATUS = 0, + /** Finished writing a network status to the directory connection; + * applies to both direct and tunneled requests; completes a direct + * request. */ + DIRREQ_FLUSHING_DIR_CONN_FINISHED = 1, + /** END cell sent to circuit that initiated a tunneled request. */ + DIRREQ_END_CELL_SENT = 2, + /** Flushed last cell from queue of the circuit that initiated a + * tunneled request to the outbuf of the OR connection. */ + DIRREQ_CIRC_QUEUE_FLUSHED = 3, + /** Flushed last byte from buffer of the OR connection belonging to the + * circuit that initiated a tunneled request; completes a tunneled + * request. */ + DIRREQ_OR_CONN_BUFFER_FLUSHED = 4 +} dirreq_state_t; + +#define WRITE_STATS_INTERVAL (24*60*60) + +/********************************* microdesc.c *************************/ + +typedef struct microdesc_cache_t microdesc_cache_t; -/** How old do we allow a v2 network-status to get before removing it - * completely? */ -#define MAX_NETWORKSTATUS_AGE (10*24*60*60) +/********************************* networkstatus.c *********************/ /** Location where we found a v2 networkstatus. */ typedef enum { @@ -3733,127 +3403,8 @@ typedef enum version_status_t { VS_UNKNOWN, /**< We have no idea. */ } version_status_t; -void networkstatus_reset_warnings(void); -void networkstatus_reset_download_failures(void); -int router_reload_v2_networkstatus(void); -int router_reload_consensus_networkstatus(void); -void routerstatus_free(routerstatus_t *rs); -void networkstatus_v2_free(networkstatus_v2_t *ns); -void networkstatus_vote_free(networkstatus_t *ns); -networkstatus_voter_info_t *networkstatus_get_voter_by_id( - networkstatus_t *vote, - const char *identity); -int networkstatus_check_consensus_signature(networkstatus_t *consensus, - int warn); -int networkstatus_check_voter_signature(networkstatus_t *consensus, - networkstatus_voter_info_t *voter, - authority_cert_t *cert); -char *networkstatus_get_cache_filename(const char *identity_digest); -int router_set_networkstatus_v2(const char *s, time_t arrived_at, - v2_networkstatus_source_t source, - smartlist_t *requested_fingerprints); -void networkstatus_v2_list_clean(time_t now); -routerstatus_t *networkstatus_v2_find_entry(networkstatus_v2_t *ns, - const char *digest); -routerstatus_t *networkstatus_vote_find_entry(networkstatus_t *ns, - const char *digest); -int networkstatus_vote_find_entry_idx(networkstatus_t *ns, - const char *digest, int *found_out); -const smartlist_t *networkstatus_get_v2_list(void); -download_status_t *router_get_dl_status_by_descriptor_digest(const char *d); -routerstatus_t *router_get_consensus_status_by_id(const char *digest); -routerstatus_t *router_get_consensus_status_by_descriptor_digest( - const char *digest); -routerstatus_t *router_get_consensus_status_by_nickname(const char *nickname, - int warn_if_unnamed); -const char *networkstatus_get_router_digest_by_nickname(const char *nickname); -int networkstatus_nickname_is_unnamed(const char *nickname); -void networkstatus_consensus_download_failed(int status_code); -void update_consensus_networkstatus_fetch_time(time_t now); -int should_delay_dir_fetches(or_options_t *options); -void update_networkstatus_downloads(time_t now); -void update_certificate_downloads(time_t now); -int consensus_is_waiting_for_certs(void); -networkstatus_v2_t *networkstatus_v2_get_by_digest(const char *digest); -networkstatus_t *networkstatus_get_latest_consensus(void); -networkstatus_t *networkstatus_get_live_consensus(time_t now); -networkstatus_t *networkstatus_get_reasonably_live_consensus(time_t now); -#define NSSET_FROM_CACHE 1 -#define NSSET_WAS_WAITING_FOR_CERTS 2 -#define NSSET_DONT_DOWNLOAD_CERTS 4 -#define NSSET_ACCEPT_OBSOLETE 8 -int networkstatus_set_current_consensus(const char *consensus, unsigned flags); -void networkstatus_note_certs_arrived(void); -void routers_update_all_from_networkstatus(time_t now, int dir_version); -void routerstatus_list_update_from_consensus_networkstatus(time_t now); -void routers_update_status_from_consensus_networkstatus(smartlist_t *routers, - int reset_failures); -void signed_descs_update_status_from_consensus_networkstatus( - smartlist_t *descs); - -char *networkstatus_getinfo_helper_single(routerstatus_t *rs); -char *networkstatus_getinfo_by_purpose(const char *purpose_string, time_t now); -void networkstatus_dump_bridge_status_to_file(time_t now); -int32_t networkstatus_get_param(networkstatus_t *ns, const char *param_name, - int32_t default_val); -int getinfo_helper_networkstatus(control_connection_t *conn, - const char *question, char **answer); -void networkstatus_free_all(void); - -/********************************* ntmain.c ***************************/ -#ifdef MS_WINDOWS -#define NT_SERVICE -#endif - -#ifdef NT_SERVICE -int nt_service_parse_options(int argc, char **argv, int *should_exit); -int nt_service_is_stopping(void); -void nt_service_set_state(DWORD state); -#else -#define nt_service_is_stopping() (0) -#endif - -/********************************* onion.c ***************************/ - -int onion_pending_add(or_circuit_t *circ, char *onionskin); -or_circuit_t *onion_next_task(char **onionskin_out); -void onion_pending_remove(or_circuit_t *circ); - -int onion_skin_create(crypto_pk_env_t *router_key, - crypto_dh_env_t **handshake_state_out, - char *onion_skin_out); - -int onion_skin_server_handshake(const char *onion_skin, - crypto_pk_env_t *private_key, - crypto_pk_env_t *prev_private_key, - char *handshake_reply_out, - char *key_out, - size_t key_out_len); - -int onion_skin_client_handshake(crypto_dh_env_t *handshake_state, - const char *handshake_reply, - char *key_out, - size_t key_out_len); - -int fast_server_handshake(const uint8_t *key_in, - uint8_t *handshake_reply_out, - uint8_t *key_out, - size_t key_out_len); - -int fast_client_handshake(const uint8_t *handshake_state, - const uint8_t *handshake_reply_out, - uint8_t *key_out, - size_t key_out_len); - -void clear_pending_onions(void); - /********************************* policies.c ************************/ -/* (length of "accept 255.255.255.255/255.255.255.255:65535-65535\n" plus a - * NUL.) - */ -#define POLICY_BUF_LEN 52 - /** Outcome of applying an address policy to an address. */ typedef enum { /** The address was accepted */ @@ -3868,146 +3419,8 @@ typedef enum { ADDR_POLICY_PROBABLY_REJECTED=2 } addr_policy_result_t; -int firewall_is_fascist_or(void); -int fascist_firewall_allows_address_or(const tor_addr_t *addr, uint16_t port); -int fascist_firewall_allows_or(routerinfo_t *ri); -int fascist_firewall_allows_address_dir(const tor_addr_t *addr, uint16_t port); -int dir_policy_permits_address(const tor_addr_t *addr); -int socks_policy_permits_address(const tor_addr_t *addr); -int authdir_policy_permits_address(uint32_t addr, uint16_t port); -int authdir_policy_valid_address(uint32_t addr, uint16_t port); -int authdir_policy_baddir_address(uint32_t addr, uint16_t port); -int authdir_policy_badexit_address(uint32_t addr, uint16_t port); - -int validate_addr_policies(or_options_t *options, char **msg); -void policy_expand_private(smartlist_t **policy); -int policies_parse_from_options(or_options_t *options); - -addr_policy_t *addr_policy_get_canonical_entry(addr_policy_t *ent); -int cmp_addr_policies(smartlist_t *a, smartlist_t *b); -addr_policy_result_t compare_tor_addr_to_addr_policy(const tor_addr_t *addr, - uint16_t port, const smartlist_t *policy); -addr_policy_result_t compare_addr_to_addr_policy(uint32_t addr, - uint16_t port, const smartlist_t *policy); -int policies_parse_exit_policy(config_line_t *cfg, smartlist_t **dest, - int rejectprivate, const char *local_address); -void policies_set_router_exitpolicy_to_reject_all(routerinfo_t *exitrouter); -int exit_policy_is_general_exit(smartlist_t *policy); -int policy_is_reject_star(const smartlist_t *policy); -int getinfo_helper_policies(control_connection_t *conn, - const char *question, char **answer); -int policy_write_item(char *buf, size_t buflen, addr_policy_t *item, - int format_for_desc); - -void addr_policy_list_free(smartlist_t *p); -void addr_policy_free(addr_policy_t *p); -void policies_free_all(void); - -char *policy_summarize(smartlist_t *policy); - -/********************************* reasons.c ***************************/ - -const char *stream_end_reason_to_control_string(int reason); -const char *stream_end_reason_to_string(int reason); -socks5_reply_status_t stream_end_reason_to_socks5_response(int reason); -uint8_t errno_to_stream_end_reason(int e); - -const char *orconn_end_reason_to_control_string(int r); -int tls_error_to_orconn_end_reason(int e); -int errno_to_orconn_end_reason(int e); - -const char *circuit_end_reason_to_control_string(int reason); - -/********************************* relay.c ***************************/ - -extern uint64_t stats_n_relay_cells_relayed; -extern uint64_t stats_n_relay_cells_delivered; - -int circuit_receive_relay_cell(cell_t *cell, circuit_t *circ, - cell_direction_t cell_direction); - -void relay_header_pack(uint8_t *dest, const relay_header_t *src); -void relay_header_unpack(relay_header_t *dest, const uint8_t *src); -int relay_send_command_from_edge(streamid_t stream_id, circuit_t *circ, - uint8_t relay_command, const char *payload, - size_t payload_len, crypt_path_t *cpath_layer); -int connection_edge_send_command(edge_connection_t *fromconn, - uint8_t relay_command, const char *payload, - size_t payload_len); -int connection_edge_package_raw_inbuf(edge_connection_t *conn, - int package_partial); -void connection_edge_consider_sending_sendme(edge_connection_t *conn); - -extern uint64_t stats_n_data_cells_packaged; -extern uint64_t stats_n_data_bytes_packaged; -extern uint64_t stats_n_data_cells_received; -extern uint64_t stats_n_data_bytes_received; - -void init_cell_pool(void); -void free_cell_pool(void); -void clean_cell_pool(void); -void dump_cell_pool_usage(int severity); - -void cell_queue_clear(cell_queue_t *queue); -void cell_queue_append(cell_queue_t *queue, packed_cell_t *cell); -void cell_queue_append_packed_copy(cell_queue_t *queue, const cell_t *cell); - -void append_cell_to_circuit_queue(circuit_t *circ, or_connection_t *orconn, - cell_t *cell, cell_direction_t direction); -void connection_or_unlink_all_active_circs(or_connection_t *conn); -int connection_or_flush_from_first_active_circuit(or_connection_t *conn, - int max, time_t now); -void assert_active_circuits_ok(or_connection_t *orconn); -void make_circuit_inactive_on_conn(circuit_t *circ, or_connection_t *conn); -void make_circuit_active_on_conn(circuit_t *circ, or_connection_t *conn); - -int append_address_to_payload(uint8_t *payload_out, const tor_addr_t *addr); -const uint8_t *decode_address_from_payload(tor_addr_t *addr_out, - const uint8_t *payload, - int payload_len); - /********************************* rephist.c ***************************/ -void rep_hist_init(void); -void rep_hist_note_connect_failed(const char* nickname, time_t when); -void rep_hist_note_connect_succeeded(const char* nickname, time_t when); -void rep_hist_note_disconnect(const char* nickname, time_t when); -void rep_hist_note_connection_died(const char* nickname, time_t when); -void rep_hist_note_extend_succeeded(const char *from_name, - const char *to_name); -void rep_hist_note_extend_failed(const char *from_name, const char *to_name); -void rep_hist_dump_stats(time_t now, int severity); -void rep_hist_note_bytes_read(size_t num_bytes, time_t when); -void rep_hist_note_bytes_written(size_t num_bytes, time_t when); -int rep_hist_bandwidth_assess(void); -char *rep_hist_get_bandwidth_lines(int for_extrainfo); -void rep_hist_update_state(or_state_t *state); -int rep_hist_load_state(or_state_t *state, char **err); -void rep_history_clean(time_t before); - -void rep_hist_note_router_reachable(const char *id, time_t when); -void rep_hist_note_router_unreachable(const char *id, time_t when); -int rep_hist_record_mtbf_data(time_t now, int missing_means_down); -int rep_hist_load_mtbf_data(time_t now); - -time_t rep_hist_downrate_old_runs(time_t now); -double rep_hist_get_stability(const char *id, time_t when); -double rep_hist_get_weighted_fractional_uptime(const char *id, time_t when); -long rep_hist_get_weighted_time_known(const char *id, time_t when); -int rep_hist_have_measured_enough_stability(void); -const char *rep_hist_get_router_stability_doc(time_t now); - -void rep_hist_note_used_port(time_t now, uint16_t port); -smartlist_t *rep_hist_get_predicted_ports(time_t now); -void rep_hist_note_used_resolve(time_t now); -void rep_hist_note_used_internal(time_t now, int need_uptime, - int need_capacity); -int rep_hist_get_predicted_internal(time_t now, int *need_uptime, - int *need_capacity); - -int any_predicted_circuits(time_t now); -int rep_hist_circbuilding_dormant(time_t now); - /** Possible public/private key operations in Tor: used to keep track of where * we're spending our time. */ typedef enum { @@ -4017,49 +3430,6 @@ typedef enum { TLS_HANDSHAKE_C, TLS_HANDSHAKE_S, REND_CLIENT, REND_MID, REND_SERVER, } pk_op_t; -void note_crypto_pk_op(pk_op_t operation); -void dump_pk_ops(int severity); - -void rep_hist_free_all(void); - -/* for hidden service usage statistics */ -void hs_usage_note_publish_total(const char *service_id, time_t now); -void hs_usage_note_publish_novel(const char *service_id, time_t now); -void hs_usage_note_fetch_total(const char *service_id, time_t now); -void hs_usage_note_fetch_successful(const char *service_id, time_t now); -void hs_usage_write_statistics_to_file(time_t now); -void hs_usage_free_all(void); - -/********************************* rendclient.c ***************************/ - -void rend_client_introcirc_has_opened(origin_circuit_t *circ); -void rend_client_rendcirc_has_opened(origin_circuit_t *circ); -int rend_client_introduction_acked(origin_circuit_t *circ, - const uint8_t *request, - size_t request_len); -void rend_client_refetch_renddesc(const char *query); -void rend_client_refetch_v2_renddesc(const rend_data_t *rend_query); -void rend_client_cancel_descriptor_fetches(void); -int rend_client_remove_intro_point(extend_info_t *failed_intro, - const rend_data_t *rend_query); -int rend_client_rendezvous_acked(origin_circuit_t *circ, - const uint8_t *request, - size_t request_len); -int rend_client_receive_rendezvous(origin_circuit_t *circ, - const uint8_t *request, - size_t request_len); -void rend_client_desc_trynow(const char *query, int rend_version); - -extend_info_t *rend_client_get_random_intro(const rend_data_t *rend_query); - -int rend_client_send_introduction(origin_circuit_t *introcirc, - origin_circuit_t *rendcirc); -int rend_parse_service_authorization(or_options_t *options, - int validate_only); -rend_service_authorization_t *rend_client_lookup_service_authorization( - const char *onion_address); -void rend_service_authorization_free_all(void); -rend_data_t *rend_data_dup(const rend_data_t *request); /********************************* rendcommon.c ***************************/ @@ -4102,31 +3472,6 @@ typedef struct rend_service_descriptor_t { smartlist_t *successful_uploads; } rend_service_descriptor_t; -/** Free all storage associated with <b>data</b> */ -static INLINE void -rend_data_free(rend_data_t *data) -{ - tor_free(data); -} - -int rend_cmp_service_ids(const char *one, const char *two); - -void rend_process_relay_cell(circuit_t *circ, const crypt_path_t *layer_hint, - int command, size_t length, - const uint8_t *payload); - -void rend_service_descriptor_free(rend_service_descriptor_t *desc); -int rend_encode_service_descriptor(rend_service_descriptor_t *desc, - crypto_pk_env_t *key, - char **str_out, - size_t *len_out); -rend_service_descriptor_t *rend_parse_service_descriptor(const char *str, - size_t len); -int rend_get_service_id(crypto_pk_env_t *pk, char *out); -void rend_encoded_v2_service_descriptor_free( - rend_encoded_v2_service_descriptor_t *desc); -void rend_intro_point_free(rend_intro_point_t *intro); - /** A cached rendezvous descriptor. */ typedef struct rend_cache_entry_t { size_t len; /**< Length of <b>desc</b> */ @@ -4135,152 +3480,6 @@ typedef struct rend_cache_entry_t { rend_service_descriptor_t *parsed; /**< Parsed value of 'desc' */ } rend_cache_entry_t; -void rend_cache_init(void); -void rend_cache_clean(void); -void rend_cache_clean_v2_descs_as_dir(void); -void rend_cache_purge(void); -void rend_cache_free_all(void); -int rend_valid_service_id(const char *query); -int rend_cache_lookup_desc(const char *query, int version, const char **desc, - size_t *desc_len); -int rend_cache_lookup_entry(const char *query, int version, - rend_cache_entry_t **entry_out); -int rend_cache_lookup_v2_desc_as_dir(const char *query, const char **desc); -int rend_cache_store(const char *desc, size_t desc_len, int published, - const char *service_id); -int rend_cache_store_v2_desc_as_client(const char *desc, - const rend_data_t *rend_query); -int rend_cache_store_v2_desc_as_dir(const char *desc); -int rend_cache_size(void); -int rend_encode_v2_descriptors(smartlist_t *descs_out, - rend_service_descriptor_t *desc, time_t now, - uint8_t period, rend_auth_type_t auth_type, - crypto_pk_env_t *client_key, - smartlist_t *client_cookies); -int rend_compute_v2_desc_id(char *desc_id_out, const char *service_id, - const char *descriptor_cookie, - time_t now, uint8_t replica); -int rend_id_is_in_interval(const char *a, const char *b, const char *c); -void rend_get_descriptor_id_bytes(char *descriptor_id_out, - const char *service_id, - const char *secret_id_part); - -/********************************* rendservice.c ***************************/ - -int num_rend_services(void); -int rend_config_services(or_options_t *options, int validate_only); -int rend_service_load_keys(void); -void rend_services_init(void); -void rend_services_introduce(void); -void rend_consider_services_upload(time_t now); -void rend_hsdir_routers_changed(void); -void rend_consider_descriptor_republication(void); - -void rend_service_intro_has_opened(origin_circuit_t *circuit); -int rend_service_intro_established(origin_circuit_t *circuit, - const uint8_t *request, - size_t request_len); -void rend_service_rendezvous_has_opened(origin_circuit_t *circuit); -int rend_service_introduce(origin_circuit_t *circuit, const uint8_t *request, - size_t request_len); -void rend_service_relaunch_rendezvous(origin_circuit_t *oldcirc); -int rend_service_set_connection_addr_port(edge_connection_t *conn, - origin_circuit_t *circ); -void rend_service_dump_stats(int severity); -void rend_service_free_all(void); - -/********************************* rendmid.c *******************************/ -int rend_mid_establish_intro(or_circuit_t *circ, const uint8_t *request, - size_t request_len); -int rend_mid_introduce(or_circuit_t *circ, const uint8_t *request, - size_t request_len); -int rend_mid_establish_rendezvous(or_circuit_t *circ, const uint8_t *request, - size_t request_len); -int rend_mid_rendezvous(or_circuit_t *circ, const uint8_t *request, - size_t request_len); - -/********************************* router.c ***************************/ - -crypto_pk_env_t *get_onion_key(void); -time_t get_onion_key_set_at(void); -void set_identity_key(crypto_pk_env_t *k); -crypto_pk_env_t *get_identity_key(void); -int identity_key_is_set(void); -authority_cert_t *get_my_v3_authority_cert(void); -crypto_pk_env_t *get_my_v3_authority_signing_key(void); -authority_cert_t *get_my_v3_legacy_cert(void); -crypto_pk_env_t *get_my_v3_legacy_signing_key(void); -void dup_onion_keys(crypto_pk_env_t **key, crypto_pk_env_t **last); -void rotate_onion_key(void); -crypto_pk_env_t *init_key_from_file(const char *fname, int generate, - int severity); -void v3_authority_check_key_expiry(void); - -int init_keys(void); - -int check_whether_orport_reachable(void); -int check_whether_dirport_reachable(void); -void consider_testing_reachability(int test_or, int test_dir); -void router_orport_found_reachable(void); -void router_dirport_found_reachable(void); -void router_perform_bandwidth_test(int num_circs, time_t now); - -int authdir_mode(or_options_t *options); -int authdir_mode_v1(or_options_t *options); -int authdir_mode_v2(or_options_t *options); -int authdir_mode_v3(or_options_t *options); -int authdir_mode_any_main(or_options_t *options); -int authdir_mode_any_nonhidserv(or_options_t *options); -int authdir_mode_handles_descs(or_options_t *options, int purpose); -int authdir_mode_publishes_statuses(or_options_t *options); -int authdir_mode_tests_reachability(or_options_t *options); -int authdir_mode_bridge(or_options_t *options); - -int server_mode(or_options_t *options); -int advertised_server_mode(void); -int proxy_mode(or_options_t *options); -void consider_publishable_server(int force); - -void router_upload_dir_desc_to_dirservers(int force); -void mark_my_descriptor_dirty_if_older_than(time_t when); -void mark_my_descriptor_dirty(void); -void check_descriptor_bandwidth_changed(time_t now); -void check_descriptor_ipaddress_changed(time_t now); -void router_new_address_suggestion(const char *suggestion, - const dir_connection_t *d_conn); -int router_compare_to_my_exit_policy(edge_connection_t *conn); -routerinfo_t *router_get_my_routerinfo(void); -extrainfo_t *router_get_my_extrainfo(void); -const char *router_get_my_descriptor(void); -int router_digest_is_me(const char *digest); -int router_extrainfo_digest_is_me(const char *digest); -int router_is_me(routerinfo_t *router); -int router_fingerprint_is_me(const char *fp); -int router_pick_published_address(or_options_t *options, uint32_t *addr); -int router_rebuild_descriptor(int force); -int router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router, - crypto_pk_env_t *ident_key); -int extrainfo_dump_to_string(char *s, size_t maxlen, extrainfo_t *extrainfo, - crypto_pk_env_t *ident_key); -char *extrainfo_get_client_geoip_summary(time_t); -int is_legal_nickname(const char *s); -int is_legal_nickname_or_hexdigest(const char *s); -int is_legal_hexdigest(const char *s); -void router_get_verbose_nickname(char *buf, const routerinfo_t *router); -void routerstatus_get_verbose_nickname(char *buf, - const routerstatus_t *router); -void router_reset_warnings(void); -void router_reset_reachability(void); -void router_free_all(void); - -const char *router_purpose_to_string(uint8_t p); -uint8_t router_purpose_from_string(const char *s); - -#ifdef ROUTER_PRIVATE -/* Used only by router.c and test.c */ -void get_platform_str(char *platform, size_t len); -#endif - /********************************* routerlist.c ***************************/ /** Represents information about a single trusted directory server. */ @@ -4320,22 +3519,7 @@ typedef struct trusted_dir_server_t { #define ROUTER_MAX_DECLARED_BANDWIDTH INT32_MAX -int get_n_authorities(authority_type_t type); -int trusted_dirs_reload_certs(void); -int trusted_dirs_load_certs_from_string(const char *contents, int from_store, - int flush); -void trusted_dirs_flush_certs_to_disk(void); -authority_cert_t *authority_cert_get_newest_by_id(const char *id_digest); -authority_cert_t *authority_cert_get_by_sk_digest(const char *sk_digest); -authority_cert_t *authority_cert_get_by_digests(const char *id_digest, - const char *sk_digest); -void authority_cert_get_all(smartlist_t *certs_out); -void authority_cert_dl_failed(const char *id_digest, int status); -void authority_certs_fetch_missing(networkstatus_t *status, time_t now); -int router_reload_router_list(void); -smartlist_t *router_get_trusted_dir_servers(void); - -/* Flags for pick_directory_server and pick_trusteddirserver. */ +/* Flags for pick_directory_server() and pick_trusteddirserver(). */ /** Flag to indicate that we should not automatically be willing to use * ourself to answer a directory request. * Passed to router_pick_directory_server (et al).*/ @@ -4364,34 +3548,13 @@ smartlist_t *router_get_trusted_dir_servers(void); */ #define PDS_NO_EXISTING_SERVERDESC_FETCH (1<<3) #define _PDS_PREFER_TUNNELED_DIR_CONNS (1<<16) -routerstatus_t *router_pick_directory_server(authority_type_t type, int flags); -trusted_dir_server_t *router_get_trusteddirserver_by_digest(const char *d); -trusted_dir_server_t *trusteddirserver_get_by_v3_auth_digest(const char *d); -routerstatus_t *router_pick_trusteddirserver(authority_type_t type, int flags); -int router_get_my_share_of_directory_requests(double *v2_share_out, - double *v3_share_out); -void router_reset_status_download_failures(void); -void routerlist_add_family(smartlist_t *sl, routerinfo_t *router); -int routers_in_same_family(routerinfo_t *r1, routerinfo_t *r2); -void add_nickname_list_to_smartlist(smartlist_t *sl, const char *list, - int must_be_running); -int router_nickname_is_in_list(routerinfo_t *router, const char *list); -routerinfo_t *routerlist_find_my_routerinfo(void); -routerinfo_t *router_find_exact_exit_enclave(const char *address, - uint16_t port); -int router_is_unreliable(routerinfo_t *router, int need_uptime, - int need_capacity, int need_guard); -uint32_t router_get_advertised_bandwidth(routerinfo_t *router); -uint32_t router_get_advertised_bandwidth_capped(routerinfo_t *router); /** Possible ways to weight routers when choosing one randomly. See * routerlist_sl_choose_by_bandwidth() for more information.*/ -typedef enum { - NO_WEIGHTING, WEIGHT_FOR_EXIT, WEIGHT_FOR_GUARD +typedef enum bandwidth_weight_rule_t { + NO_WEIGHTING, WEIGHT_FOR_EXIT, WEIGHT_FOR_MID, WEIGHT_FOR_GUARD, + WEIGHT_FOR_DIR } bandwidth_weight_rule_t; -routerinfo_t *routerlist_sl_choose_by_bandwidth(smartlist_t *sl, - bandwidth_weight_rule_t rule); -routerstatus_t *routerstatus_sl_choose_by_bandwidth(smartlist_t *sl); /** Flags to be passed to control router_choose_random_node() to indicate what * kind of nodes to pick according to what algorithm. */ @@ -4401,44 +3564,9 @@ typedef enum { CRN_NEED_GUARD = 1<<2, CRN_ALLOW_INVALID = 1<<3, /* XXXX not used, apparently. */ - CRN_STRICT_PREFERRED = 1<<4, - /* XXXX not used, apparently. */ CRN_WEIGHT_AS_EXIT = 1<<5 } router_crn_flags_t; -routerinfo_t *router_choose_random_node(const char *preferred, - smartlist_t *excludedsmartlist, - struct routerset_t *excludedset, - router_crn_flags_t flags); - -routerinfo_t *router_get_by_nickname(const char *nickname, - int warn_if_unnamed); -int router_digest_version_as_new_as(const char *digest, const char *cutoff); -int router_digest_is_trusted_dir_type(const char *digest, - authority_type_t type); -#define router_digest_is_trusted_dir(d) \ - router_digest_is_trusted_dir_type((d), NO_AUTHORITY) - -int router_addr_is_trusted_dir(uint32_t addr); -int hexdigest_to_digest(const char *hexdigest, char *digest); -routerinfo_t *router_get_by_hexdigest(const char *hexdigest); -routerinfo_t *router_get_by_digest(const char *digest); -signed_descriptor_t *router_get_by_descriptor_digest(const char *digest); -signed_descriptor_t *router_get_by_extrainfo_digest(const char *digest); -signed_descriptor_t *extrainfo_get_by_descriptor_digest(const char *digest); -const char *signed_descriptor_get_body(signed_descriptor_t *desc); -const char *signed_descriptor_get_annotations(signed_descriptor_t *desc); -routerlist_t *router_get_routerlist(void); -void routerinfo_free(routerinfo_t *router); -void extrainfo_free(extrainfo_t *extrainfo); -void routerlist_free(routerlist_t *rl); -void dump_routerlist_mem_usage(int severity); -void routerlist_remove(routerlist_t *rl, routerinfo_t *ri, int make_old, - time_t now); -void routerlist_free_all(void); -void routerlist_reset_warnings(void); -void router_set_status(const char *digest, int up); - /** Return value for router_add_to_routerlist() and dirserv_add_descriptor() */ typedef enum was_router_added_t { ROUTER_ADDED_SUCCESSFULLY = 1, @@ -4448,111 +3576,9 @@ typedef enum was_router_added_t { ROUTER_NOT_IN_CONSENSUS = -3, ROUTER_NOT_IN_CONSENSUS_OR_NETWORKSTATUS = -4, ROUTER_AUTHDIR_REJECTS = -5, + ROUTER_WAS_NOT_WANTED = -6 } was_router_added_t; -static int WRA_WAS_ADDED(was_router_added_t s); -static int WRA_WAS_OUTDATED(was_router_added_t s); -static int WRA_WAS_REJECTED(was_router_added_t s); -/** Return true iff the descriptor was added. It might still be necessary to - * check whether the descriptor generator should be notified. - */ -static INLINE int -WRA_WAS_ADDED(was_router_added_t s) { - return s == ROUTER_ADDED_SUCCESSFULLY || s == ROUTER_ADDED_NOTIFY_GENERATOR; -} -/** Return true iff the descriptor was not added because it was either: - * - not in the consensus - * - neither in the consensus nor in any networkstatus document - * - it was outdated. - */ -static INLINE int WRA_WAS_OUTDATED(was_router_added_t s) -{ - return (s == ROUTER_WAS_NOT_NEW || - s == ROUTER_NOT_IN_CONSENSUS || - s == ROUTER_NOT_IN_CONSENSUS_OR_NETWORKSTATUS); -} -/** Return true iff the descriptor rejected because it was malformed. */ -static INLINE int WRA_WAS_REJECTED(was_router_added_t s) -{ - return (s == ROUTER_AUTHDIR_REJECTS); -} -was_router_added_t router_add_to_routerlist(routerinfo_t *router, - const char **msg, - int from_cache, - int from_fetch); -was_router_added_t router_add_extrainfo_to_routerlist( - extrainfo_t *ei, const char **msg, - int from_cache, int from_fetch); -void routerlist_remove_old_routers(void); -int router_load_single_router(const char *s, uint8_t purpose, int cache, - const char **msg); -int router_load_routers_from_string(const char *s, const char *eos, - saved_location_t saved_location, - smartlist_t *requested_fingerprints, - int descriptor_digests, - const char *prepend_annotations); -void router_load_extrainfo_from_string(const char *s, const char *eos, - saved_location_t saved_location, - smartlist_t *requested_fingerprints, - int descriptor_digests); -void routerlist_retry_directory_downloads(time_t now); -int router_exit_policy_all_routers_reject(uint32_t addr, uint16_t port, - int need_uptime); -int router_exit_policy_rejects_all(routerinfo_t *router); -trusted_dir_server_t *add_trusted_dir_server(const char *nickname, - const char *address, - uint16_t dir_port, uint16_t or_port, - const char *digest, const char *v3_auth_digest, - authority_type_t type); -void authority_cert_free(authority_cert_t *cert); -void clear_trusted_dir_servers(void); -int any_trusted_dir_is_v1_authority(void); -void update_router_descriptor_downloads(time_t now); -void update_extrainfo_downloads(time_t now); -int router_have_minimum_dir_info(void); -void router_dir_info_changed(void); -const char *get_dir_info_status_string(void); -int count_loading_descriptors_progress(void); -void router_reset_descriptor_download_failures(void); -int router_differences_are_cosmetic(routerinfo_t *r1, routerinfo_t *r2); -int routerinfo_incompatible_with_extrainfo(routerinfo_t *ri, extrainfo_t *ei, - signed_descriptor_t *sd, - const char **msg); -void routerlist_assert_ok(routerlist_t *rl); -const char *esc_router_info(routerinfo_t *router); -void routers_sort_by_identity(smartlist_t *routers); - -routerset_t *routerset_new(void); -int routerset_parse(routerset_t *target, const char *s, - const char *description); -void routerset_union(routerset_t *target, const routerset_t *source); -int routerset_is_list(const routerset_t *set); -int routerset_needs_geoip(const routerset_t *set); -int routerset_contains_router(const routerset_t *set, routerinfo_t *ri); -int routerset_contains_routerstatus(const routerset_t *set, - routerstatus_t *rs); -int routerset_contains_extendinfo(const routerset_t *set, - const extend_info_t *ei); -void routerset_get_all_routers(smartlist_t *out, const routerset_t *routerset, - int running_only); -void routersets_get_disjunction(smartlist_t *target, const smartlist_t *source, - const routerset_t *include, - const routerset_t *exclude, int running_only); -void routerset_subtract_routers(smartlist_t *out, - const routerset_t *routerset); -char *routerset_to_string(const routerset_t *routerset); -void routerset_refresh_countries(routerset_t *target); -int routerset_equal(const routerset_t *old, const routerset_t *new); -void routerset_free(routerset_t *routerset); -void routerinfo_set_country(routerinfo_t *ri); -void routerlist_refresh_countries(void); -void refresh_all_country_info(void); - -int hid_serv_get_responsible_directories(smartlist_t *responsible_dirs, - const char *id); -int hid_serv_acting_as_directory(void); -int hid_serv_responsible_for_desc_id(const char *id); - /********************************* routerparse.c ************************/ #define MAX_STATUS_TAG_LEN 32 @@ -4571,71 +3597,10 @@ typedef struct tor_version_t { int patchlevel; char status_tag[MAX_STATUS_TAG_LEN]; int svn_revision; -} tor_version_t; -int router_get_router_hash(const char *s, size_t s_len, char *digest); -int router_get_dir_hash(const char *s, char *digest); -int router_get_runningrouters_hash(const char *s, char *digest); -int router_get_networkstatus_v2_hash(const char *s, char *digest); -int router_get_networkstatus_v3_hash(const char *s, char *digest); -int router_get_extrainfo_hash(const char *s, char *digest); -int router_append_dirobj_signature(char *buf, size_t buf_len, - const char *digest, - crypto_pk_env_t *private_key); -int router_parse_list_from_string(const char **s, const char *eos, - smartlist_t *dest, - saved_location_t saved_location, - int is_extrainfo, - int allow_annotations, - const char *prepend_annotations); -int router_parse_routerlist_from_directory(const char *s, - routerlist_t **dest, - crypto_pk_env_t *pkey, - int check_version, - int write_to_cache); -int router_parse_runningrouters(const char *str); -int router_parse_directory(const char *str); -routerinfo_t *router_parse_entry_from_string(const char *s, const char *end, - int cache_copy, - int allow_annotations, - const char *prepend_annotations); -extrainfo_t *extrainfo_parse_entry_from_string(const char *s, const char *end, - int cache_copy, struct digest_ri_map_t *routermap); -addr_policy_t *router_parse_addr_policy_item_from_string(const char *s, - int assume_action); -version_status_t tor_version_is_obsolete(const char *myversion, - const char *versionlist); -int tor_version_parse(const char *s, tor_version_t *out); -int tor_version_as_new_as(const char *platform, const char *cutoff); -int tor_version_compare(tor_version_t *a, tor_version_t *b); -void sort_version_list(smartlist_t *lst, int remove_duplicates); -void assert_addr_policy_ok(smartlist_t *t); -void dump_distinct_digest_count(int severity); - -networkstatus_v2_t *networkstatus_v2_parse_from_string(const char *s); -networkstatus_t *networkstatus_parse_vote_from_string(const char *s, - const char **eos_out, - networkstatus_type_t ns_type); -ns_detached_signatures_t *networkstatus_parse_detached_signatures( - const char *s, const char *eos); - -authority_cert_t *authority_cert_parse_from_string(const char *s, - const char **end_of_string); -int rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out, - char *desc_id_out, - char **intro_points_encrypted_out, - size_t *intro_points_encrypted_size_out, - size_t *encoded_size_out, - const char **next_out, const char *desc); -int rend_decrypt_introduction_points(char **ipos_decrypted, - size_t *ipos_decrypted_size, - const char *descriptor_cookie, - const char *ipos_encrypted, - size_t ipos_encrypted_size); -int rend_parse_introduction_points(rend_service_descriptor_t *parsed, - const char *intro_points_encoded, - size_t intro_points_encoded_size); -int rend_parse_client_keys(strmap_t *parsed_clients, const char *str); + int git_tag_len; + char git_tag[DIGEST_LEN]; +} tor_version_t; #endif diff --git a/src/or/policies.c b/src/or/policies.c index f8c36c784b..e48f42058f 100644 --- a/src/or/policies.c +++ b/src/or/policies.c @@ -9,6 +9,10 @@ **/ #include "or.h" +#include "config.h" +#include "dirserv.h" +#include "policies.h" +#include "routerparse.h" #include "ht.h" /** Policy that addresses for incoming SOCKS connections must match. */ @@ -344,7 +348,8 @@ validate_addr_policies(or_options_t *options, char **msg) *msg = NULL; if (policies_parse_exit_policy(options->ExitPolicy, &addr_policy, - options->ExitPolicyRejectPrivate, NULL)) + options->ExitPolicyRejectPrivate, NULL, + !options->BridgeRelay)) REJECT("Error in ExitPolicy entry."); /* The rest of these calls *append* to addr_policy. So don't actually @@ -375,14 +380,8 @@ validate_addr_policies(or_options_t *options, char **msg) if (parse_addr_policy(options->ReachableDirAddresses, &addr_policy, ADDR_POLICY_ACCEPT)) REJECT("Error in ReachableDirAddresses entry."); - if (parse_addr_policy(options->AuthDirReject, &addr_policy, - ADDR_POLICY_REJECT)) - REJECT("Error in AuthDirReject entry."); - if (parse_addr_policy(options->AuthDirInvalid, &addr_policy, - ADDR_POLICY_REJECT)) - REJECT("Error in AuthDirInvalid entry."); -err: + err: addr_policy_list_free(addr_policy); return *msg ? -1 : 0; #undef REJECT @@ -829,14 +828,16 @@ exit_policy_remove_redundancies(smartlist_t *dest) "reject *:6346-6429,reject *:6699,reject *:6881-6999,accept *:*" /** Parse the exit policy <b>cfg</b> into the linked list *<b>dest</b>. If - * cfg doesn't end in an absolute accept or reject, add the default exit + * cfg doesn't end in an absolute accept or reject and if + * <b>add_default_policy</b> is true, add the default exit * policy afterwards. If <b>rejectprivate</b> is true, prepend * "reject private:*" to the policy. Return -1 if we can't parse cfg, * else return 0. */ int policies_parse_exit_policy(config_line_t *cfg, smartlist_t **dest, - int rejectprivate, const char *local_address) + int rejectprivate, const char *local_address, + int add_default_policy) { if (rejectprivate) { append_exit_policy_string(dest, "reject private:*"); @@ -848,13 +849,23 @@ policies_parse_exit_policy(config_line_t *cfg, smartlist_t **dest, } if (parse_addr_policy(cfg, dest, -1)) return -1; - append_exit_policy_string(dest, DEFAULT_EXIT_POLICY); - + if (add_default_policy) + append_exit_policy_string(dest, DEFAULT_EXIT_POLICY); + else + append_exit_policy_string(dest, "reject *:*"); exit_policy_remove_redundancies(*dest); return 0; } +/** Add "reject *:*" to the end of the policy in *<b>dest</b>, allocating + * *<b>dest</b> as needed. */ +void +policies_exit_policy_append_reject_star(smartlist_t **dest) +{ + append_exit_policy_string(dest, "reject *:*"); +} + /** Replace the exit policy of <b>r</b> with reject *:*. */ void policies_set_router_exitpolicy_to_reject_all(routerinfo_t *r) @@ -866,6 +877,51 @@ policies_set_router_exitpolicy_to_reject_all(routerinfo_t *r) smartlist_add(r->exit_policy, item); } +/** Return 1 if there is at least one /8 subnet in <b>policy</b> that + * allows exiting to <b>port</b>. Otherwise, return 0. */ +static int +exit_policy_is_general_exit_helper(smartlist_t *policy, int port) +{ + uint32_t mask, ip, i; + /* Is this /8 rejected (1), or undecided (0)? */ + char subnet_status[256]; + + memset(subnet_status, 0, sizeof(subnet_status)); + SMARTLIST_FOREACH(policy, addr_policy_t *, p, { + if (tor_addr_family(&p->addr) != AF_INET) + continue; /* IPv4 only for now */ + if (p->prt_min > port || p->prt_max < port) + continue; /* Doesn't cover our port. */ + mask = 0; + tor_assert(p->maskbits <= 32); + + if (p->maskbits) + mask = UINT32_MAX<<(32-p->maskbits); + ip = tor_addr_to_ipv4h(&p->addr); + + /* Calculate the first and last subnet that this exit policy touches + * and set it as loop boundaries. */ + for (i = ((mask & ip)>>24); i <= (~((mask & ip) ^ mask)>>24); ++i) { + tor_addr_t addr; + if (subnet_status[i] != 0) + continue; /* We already reject some part of this /8 */ + tor_addr_from_ipv4h(&addr, i<<24); + if (tor_addr_is_internal(&addr, 0)) + continue; /* Local or non-routable addresses */ + if (p->policy_type == ADDR_POLICY_ACCEPT) { + if (p->maskbits > 8) + continue; /* Narrower than a /8. */ + /* We found an allowed subnet of at least size /8. Done + * for this port! */ + return 1; + } else if (p->policy_type == ADDR_POLICY_REJECT) { + subnet_status[i] = 1; + } + } + }); + return 0; +} + /** Return true iff <b>ri</b> is "useful as an exit node", meaning * it allows exit to at least one /8 address space for at least * two of ports 80, 443, and 6667. */ @@ -879,21 +935,7 @@ exit_policy_is_general_exit(smartlist_t *policy) return 0; for (i = 0; i < 3; ++i) { - SMARTLIST_FOREACH(policy, addr_policy_t *, p, { - if (tor_addr_family(&p->addr) != AF_INET) - continue; /* IPv4 only for now */ - if (p->prt_min > ports[i] || p->prt_max < ports[i]) - continue; /* Doesn't cover our port. */ - if (p->maskbits > 8) - continue; /* Narrower than a /8. */ - if (tor_addr_is_loopback(&p->addr)) - continue; /* 127.x or ::1. */ - /* We have a match that is at least a /8. */ - if (p->policy_type == ADDR_POLICY_ACCEPT) { - ++n_allowed; - break; /* stop considering this port */ - } - }); + n_allowed += exit_policy_is_general_exit_helper(policy, ports[i]); } return n_allowed >= 2; } @@ -1240,7 +1282,7 @@ policy_summarize(smartlist_t *policy) result = tor_malloc(final_size); tor_snprintf(result, final_size, "%s %s", prefix, shorter_str); -cleanup: + cleanup: /* cleanup */ SMARTLIST_FOREACH(summary, policy_summary_item_t *, s, tor_free(s)); smartlist_free(summary); @@ -1260,9 +1302,11 @@ cleanup: * about "exit-policy/..." */ int getinfo_helper_policies(control_connection_t *conn, - const char *question, char **answer) + const char *question, char **answer, + const char **errmsg) { (void) conn; + (void) errmsg; if (!strcmp(question, "exit-policy/default")) { *answer = tor_strdup(DEFAULT_EXIT_POLICY); } @@ -1273,7 +1317,8 @@ getinfo_helper_policies(control_connection_t *conn, void addr_policy_list_free(smartlist_t *lst) { - if (!lst) return; + if (!lst) + return; SMARTLIST_FOREACH(lst, addr_policy_t *, policy, addr_policy_free(policy)); smartlist_free(lst); } @@ -1282,19 +1327,20 @@ addr_policy_list_free(smartlist_t *lst) void addr_policy_free(addr_policy_t *p) { - if (p) { - if (--p->refcnt <= 0) { - if (p->is_canonical) { - policy_map_ent_t search, *found; - search.policy = p; - found = HT_REMOVE(policy_map, &policy_root, &search); - if (found) { - tor_assert(p == found->policy); - tor_free(found); - } + if (!p) + return; + + if (--p->refcnt <= 0) { + if (p->is_canonical) { + policy_map_ent_t search, *found; + search.policy = p; + found = HT_REMOVE(policy_map, &policy_root, &search); + if (found) { + tor_assert(p == found->policy); + tor_free(found); } - tor_free(p); } + tor_free(p); } } diff --git a/src/or/policies.h b/src/or/policies.h new file mode 100644 index 0000000000..b2947c67e7 --- /dev/null +++ b/src/or/policies.h @@ -0,0 +1,61 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file policies.h + * \brief Header file for policies.c. + **/ + +#ifndef _TOR_POLICIES_H +#define _TOR_POLICIES_H + +/* (length of "accept 255.255.255.255/255.255.255.255:65535-65535\n" plus a + * NUL.) + */ +#define POLICY_BUF_LEN 52 + +int firewall_is_fascist_or(void); +int fascist_firewall_allows_address_or(const tor_addr_t *addr, uint16_t port); +int fascist_firewall_allows_or(routerinfo_t *ri); +int fascist_firewall_allows_address_dir(const tor_addr_t *addr, uint16_t port); +int dir_policy_permits_address(const tor_addr_t *addr); +int socks_policy_permits_address(const tor_addr_t *addr); +int authdir_policy_permits_address(uint32_t addr, uint16_t port); +int authdir_policy_valid_address(uint32_t addr, uint16_t port); +int authdir_policy_baddir_address(uint32_t addr, uint16_t port); +int authdir_policy_badexit_address(uint32_t addr, uint16_t port); + +int validate_addr_policies(or_options_t *options, char **msg); +void policy_expand_private(smartlist_t **policy); +int policies_parse_from_options(or_options_t *options); + +addr_policy_t *addr_policy_get_canonical_entry(addr_policy_t *ent); +int cmp_addr_policies(smartlist_t *a, smartlist_t *b); +addr_policy_result_t compare_tor_addr_to_addr_policy(const tor_addr_t *addr, + uint16_t port, const smartlist_t *policy); +addr_policy_result_t compare_addr_to_addr_policy(uint32_t addr, + uint16_t port, const smartlist_t *policy); +int policies_parse_exit_policy(config_line_t *cfg, smartlist_t **dest, + int rejectprivate, const char *local_address, + int add_default_policy); +void policies_exit_policy_append_reject_star(smartlist_t **dest); +void policies_set_router_exitpolicy_to_reject_all(routerinfo_t *exitrouter); +int exit_policy_is_general_exit(smartlist_t *policy); +int policy_is_reject_star(const smartlist_t *policy); +int getinfo_helper_policies(control_connection_t *conn, + const char *question, char **answer, + const char **errmsg); +int policy_write_item(char *buf, size_t buflen, addr_policy_t *item, + int format_for_desc); + +void addr_policy_list_free(smartlist_t *p); +void addr_policy_free(addr_policy_t *p); +void policies_free_all(void); + +char *policy_summarize(smartlist_t *policy); + +#endif + diff --git a/src/or/reasons.c b/src/or/reasons.c index 38eb4078ca..319e6c055a 100644 --- a/src/or/reasons.c +++ b/src/or/reasons.c @@ -9,6 +9,8 @@ **/ #include "or.h" +#include "config.h" +#include "reasons.h" /***************************** Edge (stream) reasons **********************/ @@ -38,6 +40,8 @@ stream_end_reason_to_control_string(int reason) case END_STREAM_REASON_NET_UNREACHABLE: return "NET_UNREACHABLE"; case END_STREAM_REASON_SOCKSPROTOCOL: return "SOCKS_PROTOCOL"; + case END_STREAM_REASON_PRIVATE_ADDR: return "PRIVATE_ADDR"; + default: return NULL; } } @@ -123,6 +127,9 @@ stream_end_reason_to_socks5_response(int reason) return SOCKS5_NET_UNREACHABLE; case END_STREAM_REASON_SOCKSPROTOCOL: return SOCKS5_GENERAL_ERROR; + case END_STREAM_REASON_PRIVATE_ADDR: + return SOCKS5_GENERAL_ERROR; + default: log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "Reason for ending (%d) not recognized; " @@ -167,13 +174,7 @@ errno_to_stream_end_reason(int e) S_CASE(ENETUNREACH): return END_STREAM_REASON_INTERNAL; S_CASE(EHOSTUNREACH): - /* XXXX022 - * The correct behavior is END_STREAM_REASON_NOROUTE, but older - * clients don't recognize it. So we're going to continue sending - * "MISC" until 0.2.1.27 or later is "well established". - */ - /* return END_STREAM_REASON_NOROUTE; */ - return END_STREAM_REASON_MISC; + return END_STREAM_REASON_NOROUTE; S_CASE(ECONNREFUSED): return END_STREAM_REASON_CONNECTREFUSED; S_CASE(ECONNRESET): @@ -332,9 +333,78 @@ circuit_end_reason_to_control_string(int reason) return "NOPATH"; case END_CIRC_REASON_NOSUCHSERVICE: return "NOSUCHSERVICE"; + case END_CIRC_REASON_MEASUREMENT_EXPIRED: + return "MEASUREMENT_EXPIRED"; default: log_warn(LD_BUG, "Unrecognized reason code %d", (int)reason); return NULL; } } +/** Return a string corresponding to a SOCKS4 reponse code. */ +const char * +socks4_response_code_to_string(uint8_t code) +{ + switch (code) { + case 0x5a: + return "connection accepted"; + case 0x5b: + return "server rejected connection"; + case 0x5c: + return "server cannot connect to identd on this client"; + case 0x5d: + return "user id does not match identd"; + default: + return "invalid SOCKS 4 response code"; + } +} + +/** Return a string corresponding to a SOCKS5 reponse code. */ +const char * +socks5_response_code_to_string(uint8_t code) +{ + switch (code) { + case 0x00: + return "connection accepted"; + case 0x01: + return "general SOCKS server failure"; + case 0x02: + return "connection not allowed by ruleset"; + case 0x03: + return "Network unreachable"; + case 0x04: + return "Host unreachable"; + case 0x05: + return "Connection refused"; + case 0x06: + return "TTL expired"; + case 0x07: + return "Command not supported"; + case 0x08: + return "Address type not supported"; + default: + return "unknown reason"; + } +} + +/** Return a string corresponding to a bandwidht_weight_rule_t */ +const char * +bandwidth_weight_rule_to_string(bandwidth_weight_rule_t rule) +{ + switch (rule) + { + case NO_WEIGHTING: + return "no weighting"; + case WEIGHT_FOR_EXIT: + return "weight as exit"; + case WEIGHT_FOR_MID: + return "weight as middle node"; + case WEIGHT_FOR_GUARD: + return "weight as guard"; + case WEIGHT_FOR_DIR: + return "weight as directory"; + default: + return "unknown rule"; + } +} + diff --git a/src/or/reasons.h b/src/or/reasons.h new file mode 100644 index 0000000000..01f9717948 --- /dev/null +++ b/src/or/reasons.h @@ -0,0 +1,31 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file reasons.h + * \brief Header file for reasons.c. + **/ + +#ifndef _TOR_REASONS_H +#define _TOR_REASONS_H + +const char *stream_end_reason_to_control_string(int reason); +const char *stream_end_reason_to_string(int reason); +socks5_reply_status_t stream_end_reason_to_socks5_response(int reason); +uint8_t errno_to_stream_end_reason(int e); + +const char *orconn_end_reason_to_control_string(int r); +int tls_error_to_orconn_end_reason(int e); +int errno_to_orconn_end_reason(int e); + +const char *circuit_end_reason_to_control_string(int reason); +const char *socks4_response_code_to_string(uint8_t code); +const char *socks5_response_code_to_string(uint8_t code); + +const char *bandwidth_weight_rule_to_string(enum bandwidth_weight_rule_t rule); + +#endif + diff --git a/src/or/relay.c b/src/or/relay.c index 59e2c5c969..9effae3036 100644 --- a/src/or/relay.c +++ b/src/or/relay.c @@ -10,8 +10,26 @@ * receiving from circuits, plus queuing on circuits. **/ +#include <math.h> #include "or.h" +#include "buffers.h" +#include "circuitbuild.h" +#include "circuitlist.h" +#include "config.h" +#include "connection.h" +#include "connection_edge.h" +#include "connection_or.h" +#include "control.h" +#include "geoip.h" +#include "main.h" #include "mempool.h" +#include "networkstatus.h" +#include "policies.h" +#include "reasons.h" +#include "relay.h" +#include "rendcommon.h" +#include "routerlist.h" +#include "routerparse.h" static int relay_crypt(circuit_t *circ, cell_t *cell, cell_direction_t cell_direction, @@ -20,20 +38,46 @@ static edge_connection_t *relay_lookup_conn(circuit_t *circ, cell_t *cell, cell_direction_t cell_direction, crypt_path_t *layer_hint); -static int -connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, - edge_connection_t *conn, - crypt_path_t *layer_hint); -static void -circuit_consider_sending_sendme(circuit_t *circ, crypt_path_t *layer_hint); +static int connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, + edge_connection_t *conn, + crypt_path_t *layer_hint); +static void circuit_consider_sending_sendme(circuit_t *circ, + crypt_path_t *layer_hint); +static void circuit_resume_edge_reading(circuit_t *circ, + crypt_path_t *layer_hint); +static int circuit_resume_edge_reading_helper(edge_connection_t *conn, + circuit_t *circ, + crypt_path_t *layer_hint); +static int circuit_consider_stop_edge_reading(circuit_t *circ, + crypt_path_t *layer_hint); +static int circuit_queue_streams_are_blocked(circuit_t *circ); + +/* XXXX023 move this all to compat_libevent */ +/** Cache the current hi-res time; the cache gets reset when libevent + * calls us. */ +static struct timeval cached_time_hires = {0, 0}; + +/** Stop reading on edge connections when we have this many cells + * waiting on the appropriate queue. */ +#define CELL_QUEUE_HIGHWATER_SIZE 256 +/** Start reading from edge connections again when we get down to this many + * cells. */ +#define CELL_QUEUE_LOWWATER_SIZE 64 + static void -circuit_resume_edge_reading(circuit_t *circ, crypt_path_t *layer_hint); -static int -circuit_resume_edge_reading_helper(edge_connection_t *conn, - circuit_t *circ, - crypt_path_t *layer_hint); -static int -circuit_consider_stop_edge_reading(circuit_t *circ, crypt_path_t *layer_hint); +tor_gettimeofday_cached(struct timeval *tv) +{ + if (cached_time_hires.tv_sec == 0) { + tor_gettimeofday(&cached_time_hires); + } + *tv = cached_time_hires; +} + +void +tor_gettimeofday_cache_clear(void) +{ + cached_time_hires.tv_sec = 0; +} /** Stats: how many relay cells have originated at this hop, or have * been relayed onward (not recognized at this hop)? @@ -230,7 +274,7 @@ circuit_receive_relay_cell(cell_t *cell, circuit_t *circ, * we might kill the circ before we relay * the cells. */ - append_cell_to_circuit_queue(circ, or_conn, cell, cell_direction); + append_cell_to_circuit_queue(circ, or_conn, cell, cell_direction, 0); return 0; } @@ -327,7 +371,7 @@ relay_crypt(circuit_t *circ, cell_t *cell, cell_direction_t cell_direction, static int circuit_package_relay_cell(cell_t *cell, circuit_t *circ, cell_direction_t cell_direction, - crypt_path_t *layer_hint) + crypt_path_t *layer_hint, streamid_t on_stream) { or_connection_t *conn; /* where to send the cell */ @@ -371,7 +415,7 @@ circuit_package_relay_cell(cell_t *cell, circuit_t *circ, } ++stats_n_relay_cells_relayed; - append_cell_to_circuit_queue(circ, conn, cell, cell_direction); + append_cell_to_circuit_queue(circ, conn, cell, cell_direction, on_stream); return 0; } @@ -496,7 +540,7 @@ relay_command_to_string(uint8_t command) * return 0. */ int -relay_send_command_from_edge(uint16_t stream_id, circuit_t *circ, +relay_send_command_from_edge(streamid_t stream_id, circuit_t *circ, uint8_t relay_command, const char *payload, size_t payload_len, crypt_path_t *cpath_layer) { @@ -531,6 +575,12 @@ relay_send_command_from_edge(uint16_t stream_id, circuit_t *circ, log_debug(LD_OR,"delivering %d cell %s.", relay_command, cell_direction == CELL_DIRECTION_OUT ? "forward" : "backward"); + /* If we are sending an END cell and this circuit is used for a tunneled + * directory request, advance its state. */ + if (relay_command == RELAY_COMMAND_END && circ->dirreq_id) + geoip_change_dirreq_state(circ->dirreq_id, DIRREQ_TUNNELED, + DIRREQ_END_CELL_SENT); + if (cell_direction == CELL_DIRECTION_OUT && circ->n_conn) { /* if we're using relaybandwidthrate, this conn wants priority */ circ->n_conn->client_used = approx_time(); @@ -540,17 +590,11 @@ relay_send_command_from_edge(uint16_t stream_id, circuit_t *circ, origin_circuit_t *origin_circ = TO_ORIGIN_CIRCUIT(circ); if (origin_circ->remaining_relay_early_cells > 0 && (relay_command == RELAY_COMMAND_EXTEND || - (cpath_layer != origin_circ->cpath && - !CIRCUIT_PURPOSE_IS_ESTABLISHED_REND(circ->purpose)))) { - /* If we've got any relay_early cells left, and we're sending - * an extend cell or (we're not talking to the first hop and we're - * not talking to a rendezvous circuit), use one of them. - * Don't worry about the conn protocol version: + cpath_layer != origin_circ->cpath)) { + /* If we've got any relay_early cells left and (we're sending + * an extend cell or we're not talking to the first hop), use + * one of them. Don't worry about the conn protocol version: * append_cell_to_circuit_queue will fix it up. */ - /* XXX For now, clients don't use RELAY_EARLY cells when sending - * relay cells on rendezvous circuits. See bug 1038. Eventually, - * we can take this behavior away in favor of having clients avoid - * rendezvous points running 0.2.1.3-alpha through 0.2.1.18. -RD */ cell.command = CELL_RELAY_EARLY; --origin_circ->remaining_relay_early_cells; log_debug(LD_OR, "Sending a RELAY_EARLY cell; %d remaining.", @@ -578,8 +622,8 @@ relay_send_command_from_edge(uint16_t stream_id, circuit_t *circ, } } - if (circuit_package_relay_cell(&cell, circ, cell_direction, cpath_layer) - < 0) { + if (circuit_package_relay_cell(&cell, circ, cell_direction, cpath_layer, + stream_id) < 0) { log_warn(LD_BUG,"circuit_package_relay_cell failed. Closing."); circuit_mark_for_close(circ, END_CIRC_REASON_INTERNAL); return -1; @@ -747,6 +791,8 @@ connection_ap_process_end_not_open( < MAX_RESOLVE_FAILURES) { /* We haven't retried too many times; reattach the connection. */ circuit_log_path(LOG_INFO,LD_APP,circ); + /* Mark this circuit "unusable for new streams". */ + /* XXXX023 this is a kludgy way to do this. */ tor_assert(circ->_base.timestamp_dirty); circ->_base.timestamp_dirty -= get_options()->MaxCircuitDirtiness; @@ -901,7 +947,7 @@ connection_edge_process_relay_cell_not_open( } /* handle anything that might have queued */ - if (connection_edge_package_raw_inbuf(conn, 1) < 0) { + if (connection_edge_package_raw_inbuf(conn, 1, NULL) < 0) { /* (We already sent an end cell if possible) */ connection_mark_for_close(TO_CONN(conn)); return 0; @@ -999,7 +1045,8 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, relay_header_unpack(&rh, cell->payload); // log_fn(LOG_DEBUG,"command %d stream %d", rh.command, rh.stream_id); num_seen++; - log_debug(domain, "Now seen %d relay cells here.", num_seen); + log_debug(domain, "Now seen %d relay cells here (command %d, stream %d).", + num_seen, rh.command, rh.stream_id); if (rh.length > RELAY_PAYLOAD_SIZE) { log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, @@ -1038,6 +1085,16 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, "Begin cell for known stream. Dropping."); return 0; } + if (rh.command == RELAY_COMMAND_BEGIN_DIR) { + /* Assign this circuit and its app-ward OR connection a unique ID, + * so that we can measure download times. The local edge and dir + * connection will be assigned the same ID when they are created + * and linked. */ + static uint64_t next_id = 0; + circ->dirreq_id = ++next_id; + TO_CONN(TO_OR_CIRCUIT(circ)->p_conn)->dirreq_id = circ->dirreq_id; + } + return connection_exit_begin_conn(cell, circ); case RELAY_COMMAND_DATA: ++stats_n_data_cells_received; @@ -1131,6 +1188,7 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, } if (circ->n_conn) { uint8_t trunc_reason = *(uint8_t*)(cell->payload + RELAY_HEADER_SIZE); + circuit_clear_cell_queue(circ, circ->n_conn); connection_or_send_destroy(circ->n_circ_id, circ->n_conn, trunc_reason); circuit_set_n_circid_orconn(circ, 0, NULL); @@ -1179,9 +1237,13 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, conn->package_window += STREAMWINDOW_INCREMENT; log_debug(domain,"stream-level sendme, packagewindow now %d.", conn->package_window); + if (circuit_queue_streams_are_blocked(circ)) { + /* Still waiting for queue to flush; don't touch conn */ + return 0; + } connection_start_reading(TO_CONN(conn)); /* handle whatever might still be on the inbuf */ - if (connection_edge_package_raw_inbuf(conn, 1) < 0) { + if (connection_edge_package_raw_inbuf(conn, 1, NULL) < 0) { /* (We already sent an end cell if possible) */ connection_mark_for_close(TO_CONN(conn)); return 0; @@ -1247,15 +1309,19 @@ uint64_t stats_n_data_cells_received = 0; * ever received were completely full of data. */ uint64_t stats_n_data_bytes_received = 0; -/** While conn->inbuf has an entire relay payload of bytes on it, - * and the appropriate package windows aren't empty, grab a cell - * and send it down the circuit. +/** If <b>conn</b> has an entire relay payload of bytes on its inbuf (or + * <b>package_partial</b> is true), and the appropriate package windows aren't + * empty, grab a cell and send it down the circuit. + * + * If *<b>max_cells</b> is given, package no more than max_cells. Decrement + * *<b>max_cells</b> by the number of cells packaged. * * Return -1 (and send a RELAY_COMMAND_END cell if necessary) if conn should * be marked for close, else return 0. */ int -connection_edge_package_raw_inbuf(edge_connection_t *conn, int package_partial) +connection_edge_package_raw_inbuf(edge_connection_t *conn, int package_partial, + int *max_cells) { size_t amount_to_process, length; char payload[CELL_PAYLOAD_SIZE]; @@ -1271,7 +1337,10 @@ connection_edge_package_raw_inbuf(edge_connection_t *conn, int package_partial) return 0; } -repeat_connection_edge_package_raw_inbuf: + if (max_cells && *max_cells <= 0) + return 0; + + repeat_connection_edge_package_raw_inbuf: circ = circuit_get_by_edge_conn(conn); if (!circ) { @@ -1332,12 +1401,19 @@ repeat_connection_edge_package_raw_inbuf: } log_debug(domain,"conn->package_window is now %d",conn->package_window); + if (max_cells) { + *max_cells -= 1; + if (*max_cells <= 0) + return 0; + } + /* handle more if there's more, or return 0 if there isn't */ goto repeat_connection_edge_package_raw_inbuf; } -/** Called when we've just received a relay data cell, or when - * we've just finished flushing all bytes to stream <b>conn</b>. +/** Called when we've just received a relay data cell, when + * we've just finished flushing all bytes to stream <b>conn</b>, + * or when we've flushed *some* bytes to the stream <b>conn</b>. * * If conn->outbuf is not too full, and our deliver window is * low, send back a suitable number of stream-level sendme cells. @@ -1379,7 +1455,10 @@ connection_edge_consider_sending_sendme(edge_connection_t *conn) static void circuit_resume_edge_reading(circuit_t *circ, crypt_path_t *layer_hint) { - + if (circuit_queue_streams_are_blocked(circ)) { + log_debug(layer_hint?LD_APP:LD_EXIT,"Too big queue, no resuming"); + return; + } log_debug(layer_hint?LD_APP:LD_EXIT,"resuming"); if (CIRCUIT_IS_ORIGIN(circ)) @@ -1395,31 +1474,136 @@ circuit_resume_edge_reading(circuit_t *circ, crypt_path_t *layer_hint) * of a linked list of edge streams that should each be considered. */ static int -circuit_resume_edge_reading_helper(edge_connection_t *conn, +circuit_resume_edge_reading_helper(edge_connection_t *first_conn, circuit_t *circ, crypt_path_t *layer_hint) { - for ( ; conn; conn=conn->next_stream) { - if (conn->_base.marked_for_close) + edge_connection_t *conn; + int n_packaging_streams, n_streams_left; + int packaged_this_round; + int cells_on_queue; + int cells_per_conn; + edge_connection_t *chosen_stream = NULL; + + /* How many cells do we have space for? It will be the minimum of + * the number needed to exhaust the package window, and the minimum + * needed to fill the cell queue. */ + int max_to_package = circ->package_window; + if (CIRCUIT_IS_ORIGIN(circ)) { + cells_on_queue = circ->n_conn_cells.n; + } else { + or_circuit_t *or_circ = TO_OR_CIRCUIT(circ); + cells_on_queue = or_circ->p_conn_cells.n; + } + if (CELL_QUEUE_HIGHWATER_SIZE - cells_on_queue < max_to_package) + max_to_package = CELL_QUEUE_HIGHWATER_SIZE - cells_on_queue; + + /* Once we used to start listening on the streams in the order they + * appeared in the linked list. That leads to starvation on the + * streams that appeared later on the list, since the first streams + * would always get to read first. Instead, we just pick a random + * stream on the list, and enable reading for streams starting at that + * point (and wrapping around as if the list were circular). It would + * probably be better to actually remember which streams we've + * serviced in the past, but this is simple and effective. */ + + /* Select a stream uniformly at random from the linked list. We + * don't need cryptographic randomness here. */ + { + int num_streams = 0; + for (conn = first_conn; conn; conn = conn->next_stream) { + num_streams++; + if ((tor_weak_random() % num_streams)==0) + chosen_stream = conn; + /* Invariant: chosen_stream has been chosen uniformly at random from + * among the first num_streams streams on first_conn. */ + } + } + + /* Count how many non-marked streams there are that have anything on + * their inbuf, and enable reading on all of the connections. */ + n_packaging_streams = 0; + /* Activate reading starting from the chosen stream */ + for (conn=chosen_stream; conn; conn = conn->next_stream) { + /* Start reading for the streams starting from here */ + if (conn->_base.marked_for_close || conn->package_window <= 0) continue; - if ((!layer_hint && conn->package_window > 0) || - (layer_hint && conn->package_window > 0 && - conn->cpath_layer == layer_hint)) { + if (!layer_hint || conn->cpath_layer == layer_hint) { connection_start_reading(TO_CONN(conn)); + + if (buf_datalen(conn->_base.inbuf) > 0) + ++n_packaging_streams; + } + } + /* Go back and do the ones we skipped, circular-style */ + for (conn = first_conn; conn != chosen_stream; conn = conn->next_stream) { + if (conn->_base.marked_for_close || conn->package_window <= 0) + continue; + if (!layer_hint || conn->cpath_layer == layer_hint) { + connection_start_reading(TO_CONN(conn)); + + if (buf_datalen(conn->_base.inbuf) > 0) + ++n_packaging_streams; + } + } + + if (n_packaging_streams == 0) /* avoid divide-by-zero */ + return 0; + + again: + + cells_per_conn = CEIL_DIV(max_to_package, n_packaging_streams); + + packaged_this_round = 0; + n_streams_left = 0; + + /* Iterate over all connections. Package up to cells_per_conn cells on + * each. Update packaged_this_round with the total number of cells + * packaged, and n_streams_left with the number that still have data to + * package. + */ + for (conn=first_conn; conn; conn=conn->next_stream) { + if (conn->_base.marked_for_close || conn->package_window <= 0) + continue; + if (!layer_hint || conn->cpath_layer == layer_hint) { + int n = cells_per_conn, r; /* handle whatever might still be on the inbuf */ - if (connection_edge_package_raw_inbuf(conn, 1)<0) { - /* (We already sent an end cell if possible) */ + r = connection_edge_package_raw_inbuf(conn, 1, &n); + + /* Note how many we packaged */ + packaged_this_round += (cells_per_conn-n); + + if (r<0) { + /* Problem while packaging. (We already sent an end cell if + * possible) */ connection_mark_for_close(TO_CONN(conn)); continue; } + /* If there's still data to read, we'll be coming back to this stream. */ + if (buf_datalen(conn->_base.inbuf)) + ++n_streams_left; + /* If the circuit won't accept any more data, return without looking * at any more of the streams. Any connections that should be stopped * have already been stopped by connection_edge_package_raw_inbuf. */ if (circuit_consider_stop_edge_reading(circ, layer_hint)) return -1; + /* XXXX should we also stop immediately if we fill up the cell queue? + * Probably. */ } } + + /* If we made progress, and we are willing to package more, and there are + * any streams left that want to package stuff... try again! + */ + if (packaged_this_round && packaged_this_round < max_to_package && + n_streams_left) { + max_to_package -= packaged_this_round; + n_packaging_streams = n_streams_left; + goto again; + } + return 0; } @@ -1488,13 +1672,6 @@ circuit_consider_sending_sendme(circuit_t *circ, crypt_path_t *layer_hint) } } -/** Stop reading on edge connections when we have this many cells - * waiting on the appropriate queue. */ -#define CELL_QUEUE_HIGHWATER_SIZE 256 -/** Start reading from edge connections again when we get down to this many - * cells. */ -#define CELL_QUEUE_LOWWATER_SIZE 64 - #ifdef ACTIVE_CIRCUITS_PARANOIA #define assert_active_circuits_ok_paranoid(conn) \ assert_active_circuits_ok(conn) @@ -1508,6 +1685,10 @@ static int total_cells_allocated = 0; /** A memory pool to allocate packed_cell_t objects. */ static mp_pool_t *cell_pool = NULL; +/** Memory pool to allocate insertion_time_elem_t objects used for cell + * statistics. */ +static mp_pool_t *it_pool = NULL; + /** Allocate structures to hold cells. */ void init_cell_pool(void) @@ -1516,7 +1697,8 @@ init_cell_pool(void) cell_pool = mp_pool_new(sizeof(packed_cell_t), 128*1024); } -/** Free all storage used to hold cells. */ +/** Free all storage used to hold cells (and insertion times if we measure + * cell statistics). */ void free_cell_pool(void) { @@ -1525,6 +1707,10 @@ free_cell_pool(void) mp_pool_destroy(cell_pool); cell_pool = NULL; } + if (it_pool) { + mp_pool_destroy(it_pool); + it_pool = NULL; + } } /** Free excess storage in cell pool. */ @@ -1537,7 +1723,7 @@ clean_cell_pool(void) /** Release storage held by <b>cell</b>. */ static INLINE void -packed_cell_free(packed_cell_t *cell) +packed_cell_free_unchecked(packed_cell_t *cell) { --total_cells_allocated; mp_pool_release(cell); @@ -1599,7 +1785,38 @@ cell_queue_append(cell_queue_t *queue, packed_cell_t *cell) void cell_queue_append_packed_copy(cell_queue_t *queue, const cell_t *cell) { - cell_queue_append(queue, packed_cell_copy(cell)); + packed_cell_t *copy = packed_cell_copy(cell); + /* Remember the time when this cell was put in the queue. */ + if (get_options()->CellStatistics) { + struct timeval now; + uint32_t added; + insertion_time_queue_t *it_queue = queue->insertion_times; + if (!it_pool) + it_pool = mp_pool_new(sizeof(insertion_time_elem_t), 1024); + tor_gettimeofday_cached(&now); +#define SECONDS_IN_A_DAY 86400L + added = (uint32_t)(((now.tv_sec % SECONDS_IN_A_DAY) * 100L) + + ((uint32_t)now.tv_usec / (uint32_t)10000L)); + if (!it_queue) { + it_queue = tor_malloc_zero(sizeof(insertion_time_queue_t)); + queue->insertion_times = it_queue; + } + if (it_queue->last && it_queue->last->insertion_time == added) { + it_queue->last->counter++; + } else { + insertion_time_elem_t *elem = mp_pool_get(it_pool); + elem->next = NULL; + elem->insertion_time = added; + elem->counter = 1; + if (it_queue->last) { + it_queue->last->next = elem; + it_queue->last = elem; + } else { + it_queue->first = it_queue->last = elem; + } + } + } + cell_queue_append(queue, copy); } /** Remove and free every cell in <b>queue</b>. */ @@ -1610,11 +1827,19 @@ cell_queue_clear(cell_queue_t *queue) cell = queue->head; while (cell) { next = cell->next; - packed_cell_free(cell); + packed_cell_free_unchecked(cell); cell = next; } queue->head = queue->tail = NULL; queue->n = 0; + if (queue->insertion_times) { + while (queue->insertion_times->first) { + insertion_time_elem_t *elem = queue->insertion_times->first; + queue->insertion_times->first = elem->next; + mp_pool_release(elem); + } + tor_free(queue->insertion_times); + } } /** Extract and return the cell at the head of <b>queue</b>; return NULL if @@ -1666,8 +1891,226 @@ prev_circ_on_conn_p(circuit_t *circ, or_connection_t *conn) } } +/** Helper for sorting cell_ewma_t values in their priority queue. */ +static int +compare_cell_ewma_counts(const void *p1, const void *p2) +{ + const cell_ewma_t *e1=p1, *e2=p2; + if (e1->cell_count < e2->cell_count) + return -1; + else if (e1->cell_count > e2->cell_count) + return 1; + else + return 0; +} + +/** Given a cell_ewma_t, return a pointer to the circuit containing it. */ +static circuit_t * +cell_ewma_to_circuit(cell_ewma_t *ewma) +{ + if (ewma->is_for_p_conn) { + /* This is an or_circuit_t's p_cell_ewma. */ + or_circuit_t *orcirc = SUBTYPE_P(ewma, or_circuit_t, p_cell_ewma); + return TO_CIRCUIT(orcirc); + } else { + /* This is some circuit's n_cell_ewma. */ + return SUBTYPE_P(ewma, circuit_t, n_cell_ewma); + } +} + +/* ==== Functions for scaling cell_ewma_t ==== + + When choosing which cells to relay first, we favor circuits that have been + quiet recently. This gives better latency on connections that aren't + pushing lots of data, and makes the network feel more interactive. + + Conceptually, we take an exponentially weighted mean average of the number + of cells a circuit has sent, and allow active circuits (those with cells to + relay) to send cells in reverse order of their exponentially-weighted mean + average (EWMA) cell count. [That is, a cell sent N seconds ago 'counts' + F^N times as much as a cell sent now, for 0<F<1.0, and we favor the + circuit that has sent the fewest cells] + + If 'double' had infinite precision, we could do this simply by counting a + cell sent at startup as having weight 1.0, and a cell sent N seconds later + as having weight F^-N. This way, we would never need to re-scale + any already-sent cells. + + To prevent double from overflowing, we could count a cell sent now as + having weight 1.0 and a cell sent N seconds ago as having weight F^N. + This, however, would mean we'd need to re-scale *ALL* old circuits every + time we wanted to send a cell. + + So as a compromise, we divide time into 'ticks' (currently, 10-second + increments) and say that a cell sent at the start of a current tick is + worth 1.0, a cell sent N seconds before the start of the current tick is + worth F^N, and a cell sent N seconds after the start of the current tick is + worth F^-N. This way we don't overflow, and we don't need to constantly + rescale. + */ + +/** How long does a tick last (seconds)? */ +#define EWMA_TICK_LEN 10 + +/** The default per-tick scale factor, if it hasn't been overridden by a + * consensus or a configuration setting. zero means "disabled". */ +#define EWMA_DEFAULT_HALFLIFE 0.0 + +/** Given a timeval <b>now</b>, compute the cell_ewma tick in which it occurs + * and the fraction of the tick that has elapsed between the start of the tick + * and <b>now</b>. Return the former and store the latter in + * *<b>remainder_out</b>. + * + * These tick values are not meant to be shared between Tor instances, or used + * for other purposes. */ +static unsigned +cell_ewma_tick_from_timeval(const struct timeval *now, + double *remainder_out) +{ + unsigned res = (unsigned) (now->tv_sec / EWMA_TICK_LEN); + /* rem */ + double rem = (now->tv_sec % EWMA_TICK_LEN) + + ((double)(now->tv_usec)) / 1.0e6; + *remainder_out = rem / EWMA_TICK_LEN; + return res; +} + +/** Compute and return the current cell_ewma tick. */ +unsigned +cell_ewma_get_tick(void) +{ + return ((unsigned)approx_time() / EWMA_TICK_LEN); +} + +/** The per-tick scale factor to be used when computing cell-count EWMA + * values. (A cell sent N ticks before the start of the current tick + * has value ewma_scale_factor ** N.) + */ +static double ewma_scale_factor = 0.1; +static int ewma_enabled = 0; + +#define EPSILON 0.00001 +#define LOG_ONEHALF -0.69314718055994529 + +/** Adjust the global cell scale factor based on <b>options</b> */ +void +cell_ewma_set_scale_factor(or_options_t *options, networkstatus_t *consensus) +{ + int32_t halflife_ms; + double halflife; + const char *source; + if (options && options->CircuitPriorityHalflife >= -EPSILON) { + halflife = options->CircuitPriorityHalflife; + source = "CircuitPriorityHalflife in configuration"; + } else if (consensus && (halflife_ms = networkstatus_get_param( + consensus, "CircuitPriorityHalflifeMsec", + -1, -1, INT32_MAX)) >= 0) { + halflife = ((double)halflife_ms)/1000.0; + source = "CircuitPriorityHalflifeMsec in consensus"; + } else { + halflife = EWMA_DEFAULT_HALFLIFE; + source = "Default value"; + } + + if (halflife <= EPSILON) { + /* The cell EWMA algorithm is disabled. */ + ewma_scale_factor = 0.1; + ewma_enabled = 0; + log_info(LD_OR, + "Disabled cell_ewma algorithm because of value in %s", + source); + } else { + /* convert halflife into halflife-per-tick. */ + halflife /= EWMA_TICK_LEN; + /* compute per-tick scale factor. */ + ewma_scale_factor = exp( LOG_ONEHALF / halflife ); + ewma_enabled = 1; + log_info(LD_OR, + "Enabled cell_ewma algorithm because of value in %s; " + "scale factor is %lf per %d seconds", + source, ewma_scale_factor, EWMA_TICK_LEN); + } +} + +/** Return the multiplier necessary to convert the value of a cell sent in + * 'from_tick' to one sent in 'to_tick'. */ +static INLINE double +get_scale_factor(unsigned from_tick, unsigned to_tick) +{ + /* This math can wrap around, but that's okay: unsigned overflow is + well-defined */ + int diff = (int)(to_tick - from_tick); + return pow(ewma_scale_factor, diff); +} + +/** Adjust the cell count of <b>ewma</b> so that it is scaled with respect to + * <b>cur_tick</b> */ +static void +scale_single_cell_ewma(cell_ewma_t *ewma, unsigned cur_tick) +{ + double factor = get_scale_factor(ewma->last_adjusted_tick, cur_tick); + ewma->cell_count *= factor; + ewma->last_adjusted_tick = cur_tick; +} + +/** Adjust the cell count of every active circuit on <b>conn</b> so + * that they are scaled with respect to <b>cur_tick</b> */ +static void +scale_active_circuits(or_connection_t *conn, unsigned cur_tick) +{ + + double factor = get_scale_factor( + conn->active_circuit_pqueue_last_recalibrated, + cur_tick); + /** Ordinarily it isn't okay to change the value of an element in a heap, + * but it's okay here, since we are preserving the order. */ + SMARTLIST_FOREACH(conn->active_circuit_pqueue, cell_ewma_t *, e, { + tor_assert(e->last_adjusted_tick == + conn->active_circuit_pqueue_last_recalibrated); + e->cell_count *= factor; + e->last_adjusted_tick = cur_tick; + }); + conn->active_circuit_pqueue_last_recalibrated = cur_tick; +} + +/** Rescale <b>ewma</b> to the same scale as <b>conn</b>, and add it to + * <b>conn</b>'s priority queue of active circuits */ +static void +add_cell_ewma_to_conn(or_connection_t *conn, cell_ewma_t *ewma) +{ + tor_assert(ewma->heap_index == -1); + scale_single_cell_ewma(ewma, + conn->active_circuit_pqueue_last_recalibrated); + + smartlist_pqueue_add(conn->active_circuit_pqueue, + compare_cell_ewma_counts, + STRUCT_OFFSET(cell_ewma_t, heap_index), + ewma); +} + +/** Remove <b>ewma</b> from <b>conn</b>'s priority queue of active circuits */ +static void +remove_cell_ewma_from_conn(or_connection_t *conn, cell_ewma_t *ewma) +{ + tor_assert(ewma->heap_index != -1); + smartlist_pqueue_remove(conn->active_circuit_pqueue, + compare_cell_ewma_counts, + STRUCT_OFFSET(cell_ewma_t, heap_index), + ewma); +} + +/** Remove and return the first cell_ewma_t from conn's priority queue of + * active circuits. Requires that the priority queue is nonempty. */ +static cell_ewma_t * +pop_first_cell_ewma_from_conn(or_connection_t *conn) +{ + return smartlist_pqueue_pop(conn->active_circuit_pqueue, + compare_cell_ewma_counts, + STRUCT_OFFSET(cell_ewma_t, heap_index)); +} + /** Add <b>circ</b> to the list of circuits with pending cells on - * <b>conn</b>. No effect if <b>circ</b> is already unlinked. */ + * <b>conn</b>. No effect if <b>circ</b> is already linked. */ void make_circuit_active_on_conn(circuit_t *circ, or_connection_t *conn) { @@ -1679,6 +2122,8 @@ make_circuit_active_on_conn(circuit_t *circ, or_connection_t *conn) return; } + assert_active_circuits_ok_paranoid(conn); + if (! conn->active_circuits) { conn->active_circuits = circ; *prevp = *nextp = circ; @@ -1690,10 +2135,19 @@ make_circuit_active_on_conn(circuit_t *circ, or_connection_t *conn) *prev_circ_on_conn_p(head, conn) = circ; *prevp = old_tail; } + + if (circ->n_conn == conn) { + add_cell_ewma_to_conn(conn, &circ->n_cell_ewma); + } else { + or_circuit_t *orcirc = TO_OR_CIRCUIT(circ); + tor_assert(conn == orcirc->p_conn); + add_cell_ewma_to_conn(conn, &orcirc->p_cell_ewma); + } + assert_active_circuits_ok_paranoid(conn); } -/** Remove <b>circ</b> to the list of circuits with pending cells on +/** Remove <b>circ</b> from the list of circuits with pending cells on * <b>conn</b>. No effect if <b>circ</b> is already unlinked. */ void make_circuit_inactive_on_conn(circuit_t *circ, or_connection_t *conn) @@ -1707,6 +2161,8 @@ make_circuit_inactive_on_conn(circuit_t *circ, or_connection_t *conn) return; } + assert_active_circuits_ok_paranoid(conn); + tor_assert(next && prev); tor_assert(*prev_circ_on_conn_p(next, conn) == circ); tor_assert(*next_circ_on_conn_p(prev, conn) == circ); @@ -1720,6 +2176,15 @@ make_circuit_inactive_on_conn(circuit_t *circ, or_connection_t *conn) conn->active_circuits = next; } *prevp = *nextp = NULL; + + if (circ->n_conn == conn) { + remove_cell_ewma_from_conn(conn, &circ->n_cell_ewma); + } else { + or_circuit_t *orcirc = TO_OR_CIRCUIT(circ); + tor_assert(conn == orcirc->p_conn); + remove_cell_ewma_from_conn(conn, &orcirc->p_cell_ewma); + } + assert_active_circuits_ok_paranoid(conn); } @@ -1739,16 +2204,27 @@ connection_or_unlink_all_active_circs(or_connection_t *orconn) cur = next; } while (cur != head); orconn->active_circuits = NULL; + + SMARTLIST_FOREACH(orconn->active_circuit_pqueue, cell_ewma_t *, e, + e->heap_index = -1); + smartlist_clear(orconn->active_circuit_pqueue); } /** Block (if <b>block</b> is true) or unblock (if <b>block</b> is false) * every edge connection that is using <b>circ</b> to write to <b>orconn</b>, - * and start or stop reading as appropriate. */ -static void + * and start or stop reading as appropriate. + * + * If <b>stream_id</b> is nonzero, block only the edge connection whose + * stream_id matches it. + * + * Returns the number of streams whose status we changed. + */ +static int set_streams_blocked_on_circ(circuit_t *circ, or_connection_t *orconn, - int block) + int block, streamid_t stream_id) { edge_connection_t *edge = NULL; + int n = 0; if (circ->n_conn == orconn) { circ->streams_blocked_on_n_conn = block; if (CIRCUIT_IS_ORIGIN(circ)) @@ -1761,7 +2237,13 @@ set_streams_blocked_on_circ(circuit_t *circ, or_connection_t *orconn, for (; edge; edge = edge->next_stream) { connection_t *conn = TO_CONN(edge); - edge->edge_blocked_on_circ = block; + if (stream_id && edge->stream_id != stream_id) + continue; + + if (edge->edge_blocked_on_circ != block) { + ++n; + edge->edge_blocked_on_circ = block; + } if (!conn->read_event) { /* This connection is a placeholder for something; probably a DNS @@ -1778,10 +2260,12 @@ set_streams_blocked_on_circ(circuit_t *circ, or_connection_t *orconn, connection_start_reading(conn); } } + + return n; } /** Pull as many cells as possible (but no more than <b>max</b>) from the - * queue of the first active circuit on <b>conn</b>, and write then to + * queue of the first active circuit on <b>conn</b>, and write them to * <b>conn</b>->outbuf. Return the number of cells written. Advance * the active circuit pointer to the next active circuit in the ring. */ int @@ -1792,9 +2276,35 @@ connection_or_flush_from_first_active_circuit(or_connection_t *conn, int max, cell_queue_t *queue; circuit_t *circ; int streams_blocked; + + /* The current (hi-res) time */ + struct timeval now_hires; + + /* The EWMA cell counter for the circuit we're flushing. */ + cell_ewma_t *cell_ewma = NULL; + double ewma_increment = -1; + circ = conn->active_circuits; if (!circ) return 0; assert_active_circuits_ok_paranoid(conn); + + /* See if we're doing the ewma circuit selection algorithm. */ + if (ewma_enabled) { + unsigned tick; + double fractional_tick; + tor_gettimeofday_cached(&now_hires); + tick = cell_ewma_tick_from_timeval(&now_hires, &fractional_tick); + + if (tick != conn->active_circuit_pqueue_last_recalibrated) { + scale_active_circuits(conn, tick); + } + + ewma_increment = pow(ewma_scale_factor, -fractional_tick); + + cell_ewma = smartlist_get(conn->active_circuit_pqueue, 0); + circ = cell_ewma_to_circuit(cell_ewma); + } + if (circ->n_conn == conn) { queue = &circ->n_conn_cells; streams_blocked = circ->streams_blocked_on_n_conn; @@ -1808,10 +2318,60 @@ connection_or_flush_from_first_active_circuit(or_connection_t *conn, int max, packed_cell_t *cell = cell_queue_pop(queue); tor_assert(*next_circ_on_conn_p(circ,conn)); + /* Calculate the exact time that this cell has spent in the queue. */ + if (get_options()->CellStatistics && !CIRCUIT_IS_ORIGIN(circ)) { + struct timeval now; + uint32_t flushed; + uint32_t cell_waiting_time; + insertion_time_queue_t *it_queue = queue->insertion_times; + tor_gettimeofday_cached(&now); + flushed = (uint32_t)((now.tv_sec % SECONDS_IN_A_DAY) * 100L + + (uint32_t)now.tv_usec / (uint32_t)10000L); + if (!it_queue || !it_queue->first) { + log_info(LD_GENERAL, "Cannot determine insertion time of cell. " + "Looks like the CellStatistics option was " + "recently enabled."); + } else { + or_circuit_t *orcirc = TO_OR_CIRCUIT(circ); + insertion_time_elem_t *elem = it_queue->first; + cell_waiting_time = + (uint32_t)((flushed * 10L + SECONDS_IN_A_DAY * 1000L - + elem->insertion_time * 10L) % + (SECONDS_IN_A_DAY * 1000L)); +#undef SECONDS_IN_A_DAY + elem->counter--; + if (elem->counter < 1) { + it_queue->first = elem->next; + if (elem == it_queue->last) + it_queue->last = NULL; + mp_pool_release(elem); + } + orcirc->total_cell_waiting_time += cell_waiting_time; + orcirc->processed_cells++; + } + } + + /* If we just flushed our queue and this circuit is used for a + * tunneled directory request, possibly advance its state. */ + if (queue->n == 0 && TO_CONN(conn)->dirreq_id) + geoip_change_dirreq_state(TO_CONN(conn)->dirreq_id, + DIRREQ_TUNNELED, + DIRREQ_CIRC_QUEUE_FLUSHED); + connection_write_to_buf(cell->body, CELL_NETWORK_SIZE, TO_CONN(conn)); - packed_cell_free(cell); + packed_cell_free_unchecked(cell); ++n_flushed; + if (cell_ewma) { + cell_ewma_t *tmp; + cell_ewma->cell_count += ewma_increment; + /* We pop and re-add the cell_ewma_t here, not above, since we need to + * re-add it immediately to keep the priority queue consistent with + * the linked-list implementation */ + tmp = pop_first_cell_ewma_from_conn(conn); + tor_assert(tmp == cell_ewma); + add_cell_ewma_to_conn(conn, cell_ewma); + } if (circ != conn->active_circuits) { /* If this happens, the current circuit just got made inactive by * a call in connection_write_to_buf(). That's nothing to worry about: @@ -1829,9 +2389,9 @@ connection_or_flush_from_first_active_circuit(or_connection_t *conn, int max, /* Is the cell queue low enough to unblock all the streams that are waiting * to write to this circuit? */ if (streams_blocked && queue->n <= CELL_QUEUE_LOWWATER_SIZE) - set_streams_blocked_on_circ(circ, conn, 0); /* unblock streams */ + set_streams_blocked_on_circ(circ, conn, 0, 0); /* unblock streams */ - /* Did we just ran out of cells on this queue? */ + /* Did we just run out of cells on this circuit's queue? */ if (queue->n == 0) { log_debug(LD_GENERAL, "Made a circuit inactive."); make_circuit_inactive_on_conn(circ, conn); @@ -1846,10 +2406,14 @@ connection_or_flush_from_first_active_circuit(or_connection_t *conn, int max, * transmitting in <b>direction</b>. */ void append_cell_to_circuit_queue(circuit_t *circ, or_connection_t *orconn, - cell_t *cell, cell_direction_t direction) + cell_t *cell, cell_direction_t direction, + streamid_t fromstream) { cell_queue_t *queue; int streams_blocked; + if (circ->marked_for_close) + return; + if (direction == CELL_DIRECTION_OUT) { queue = &circ->n_conn_cells; streams_blocked = circ->streams_blocked_on_n_conn; @@ -1868,7 +2432,12 @@ append_cell_to_circuit_queue(circuit_t *circ, or_connection_t *orconn, /* If we have too many cells on the circuit, we should stop reading from * the edge streams for a while. */ if (!streams_blocked && queue->n >= CELL_QUEUE_HIGHWATER_SIZE) - set_streams_blocked_on_circ(circ, orconn, 1); /* block streams */ + set_streams_blocked_on_circ(circ, orconn, 1, 0); /* block streams */ + + if (streams_blocked && fromstream) { + /* This edge connection is apparently not blocked; block it. */ + set_streams_blocked_on_circ(circ, orconn, 1, fromstream); + } if (queue->n == 1) { /* This was the first cell added to the queue. We need to make this @@ -1947,6 +2516,25 @@ decode_address_from_payload(tor_addr_t *addr_out, const uint8_t *payload, return payload + 2 + payload[1]; } +/** Remove all the cells queued on <b>circ</b> for <b>orconn</b>. */ +void +circuit_clear_cell_queue(circuit_t *circ, or_connection_t *orconn) +{ + cell_queue_t *queue; + if (circ->n_conn == orconn) { + queue = &circ->n_conn_cells; + } else { + or_circuit_t *orcirc = TO_OR_CIRCUIT(circ); + tor_assert(orcirc->p_conn == orconn); + queue = &orcirc->p_conn_cells; + } + + if (queue->n) + make_circuit_inactive_on_conn(circ,orconn); + + cell_queue_clear(queue); +} + /** Fail with an assert if the active circuits ring on <b>orconn</b> is * corrupt. */ void @@ -1954,16 +2542,44 @@ assert_active_circuits_ok(or_connection_t *orconn) { circuit_t *head = orconn->active_circuits; circuit_t *cur = head; + int n = 0; if (! head) return; do { circuit_t *next = *next_circ_on_conn_p(cur, orconn); circuit_t *prev = *prev_circ_on_conn_p(cur, orconn); + cell_ewma_t *ewma; tor_assert(next); tor_assert(prev); tor_assert(*next_circ_on_conn_p(prev, orconn) == cur); tor_assert(*prev_circ_on_conn_p(next, orconn) == cur); + if (orconn == cur->n_conn) { + ewma = &cur->n_cell_ewma; + tor_assert(!ewma->is_for_p_conn); + } else { + ewma = &TO_OR_CIRCUIT(cur)->p_cell_ewma; + tor_assert(ewma->is_for_p_conn); + } + tor_assert(ewma->heap_index != -1); + tor_assert(ewma == smartlist_get(orconn->active_circuit_pqueue, + ewma->heap_index)); + n++; cur = next; } while (cur != head); + + tor_assert(n == smartlist_len(orconn->active_circuit_pqueue)); +} + +/** Return 1 if we shouldn't restart reading on this circuit, even if + * we get a SENDME. Else return 0. +*/ +static int +circuit_queue_streams_are_blocked(circuit_t *circ) +{ + if (CIRCUIT_IS_ORIGIN(circ)) { + return circ->streams_blocked_on_n_conn; + } else { + return circ->streams_blocked_on_p_conn; + } } diff --git a/src/or/relay.h b/src/or/relay.h new file mode 100644 index 0000000000..f64752da5d --- /dev/null +++ b/src/or/relay.h @@ -0,0 +1,70 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file relay.h + * \brief Header file for relay.c. + **/ + +#ifndef _TOR_RELAY_H +#define _TOR_RELAY_H + +extern uint64_t stats_n_relay_cells_relayed; +extern uint64_t stats_n_relay_cells_delivered; + +int circuit_receive_relay_cell(cell_t *cell, circuit_t *circ, + cell_direction_t cell_direction); + +void relay_header_pack(uint8_t *dest, const relay_header_t *src); +void relay_header_unpack(relay_header_t *dest, const uint8_t *src); +int relay_send_command_from_edge(streamid_t stream_id, circuit_t *circ, + uint8_t relay_command, const char *payload, + size_t payload_len, crypt_path_t *cpath_layer); +int connection_edge_send_command(edge_connection_t *fromconn, + uint8_t relay_command, const char *payload, + size_t payload_len); +int connection_edge_package_raw_inbuf(edge_connection_t *conn, + int package_partial, + int *max_cells); +void connection_edge_consider_sending_sendme(edge_connection_t *conn); + +extern uint64_t stats_n_data_cells_packaged; +extern uint64_t stats_n_data_bytes_packaged; +extern uint64_t stats_n_data_cells_received; +extern uint64_t stats_n_data_bytes_received; + +void init_cell_pool(void); +void free_cell_pool(void); +void clean_cell_pool(void); +void dump_cell_pool_usage(int severity); + +void cell_queue_clear(cell_queue_t *queue); +void cell_queue_append(cell_queue_t *queue, packed_cell_t *cell); +void cell_queue_append_packed_copy(cell_queue_t *queue, const cell_t *cell); + +void append_cell_to_circuit_queue(circuit_t *circ, or_connection_t *orconn, + cell_t *cell, cell_direction_t direction, + streamid_t fromstream); +void connection_or_unlink_all_active_circs(or_connection_t *conn); +int connection_or_flush_from_first_active_circuit(or_connection_t *conn, + int max, time_t now); +void assert_active_circuits_ok(or_connection_t *orconn); +void make_circuit_inactive_on_conn(circuit_t *circ, or_connection_t *conn); +void make_circuit_active_on_conn(circuit_t *circ, or_connection_t *conn); + +int append_address_to_payload(uint8_t *payload_out, const tor_addr_t *addr); +const uint8_t *decode_address_from_payload(tor_addr_t *addr_out, + const uint8_t *payload, + int payload_len); +unsigned cell_ewma_get_tick(void); +void cell_ewma_set_scale_factor(or_options_t *options, + networkstatus_t *consensus); +void circuit_clear_cell_queue(circuit_t *circ, or_connection_t *orconn); + +void tor_gettimeofday_cache_clear(void); + +#endif + diff --git a/src/or/rendclient.c b/src/or/rendclient.c index 7bda70562b..77e11c2a07 100644 --- a/src/or/rendclient.c +++ b/src/or/rendclient.c @@ -8,6 +8,23 @@ **/ #include "or.h" +#include "circuitbuild.h" +#include "circuitlist.h" +#include "circuituse.h" +#include "config.h" +#include "connection.h" +#include "connection_edge.h" +#include "directory.h" +#include "main.h" +#include "relay.h" +#include "rendclient.h" +#include "rendcommon.h" +#include "rephist.h" +#include "routerlist.h" + +static extend_info_t *rend_client_get_random_intro_impl( + const rend_cache_entry_t *rend_query, + const int strict, const int warnings); /** Called when we've established a circuit to an introduction point: * send the introduction request. */ @@ -49,6 +66,50 @@ rend_client_send_establish_rendezvous(origin_circuit_t *circ) return 0; } +/** Extend the introduction circuit <b>circ</b> to another valid + * introduction point for the hidden service it is trying to connect + * to, or mark it and launch a new circuit if we can't extend it. + * Return 0 on success. Return -1 and mark the introduction + * circuit on failure. + * + * On failure, the caller is responsible for marking the associated + * rendezvous circuit for close. */ +static int +rend_client_reextend_intro_circuit(origin_circuit_t *circ) +{ + extend_info_t *extend_info; + int result; + extend_info = rend_client_get_random_intro(circ->rend_data); + if (!extend_info) { + log_warn(LD_REND, + "No usable introduction points left for %s. Closing.", + safe_str_client(circ->rend_data->onion_address)); + circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_INTERNAL); + return -1; + } + if (circ->remaining_relay_early_cells) { + log_info(LD_REND, + "Re-extending circ %d, this time to %s.", + circ->_base.n_circ_id, extend_info->nickname); + result = circuit_extend_to_new_exit(circ, extend_info); + } else { + log_info(LD_REND, + "Building a new introduction circuit, this time to %s.", + extend_info->nickname); + circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_FINISHED); + if (!circuit_launch_by_extend_info(CIRCUIT_PURPOSE_C_INTRODUCING, + extend_info, + CIRCLAUNCH_IS_INTERNAL)) { + log_warn(LD_REND, "Building introduction circuit failed."); + result = -1; + } else { + result = 0; + } + } + extend_info_free(extend_info); + return result; +} + /** Called when we're trying to connect an ap conn; sends an INTRODUCE1 cell * down introcirc if possible. */ @@ -63,7 +124,7 @@ rend_client_send_introduction(origin_circuit_t *introcirc, rend_cache_entry_t *entry; crypt_path_t *cpath; off_t dh_offset; - crypto_pk_env_t *intro_key; /* either Bob's public key or an intro key. */ + crypto_pk_env_t *intro_key = NULL; tor_assert(introcirc->_base.purpose == CIRCUIT_PURPOSE_C_INTRODUCING); tor_assert(rendcirc->_base.purpose == CIRCUIT_PURPOSE_C_REND_READY); @@ -77,19 +138,14 @@ rend_client_send_introduction(origin_circuit_t *introcirc, log_info(LD_REND, "query %s didn't have valid rend desc in cache. " "Refetching descriptor.", - safe_str(introcirc->rend_data->onion_address)); - /* Fetch both v0 and v2 rend descriptors in parallel. Use whichever - * arrives first. Exception: When using client authorization, only - * fetch v2 descriptors.*/ + safe_str_client(introcirc->rend_data->onion_address)); rend_client_refetch_v2_renddesc(introcirc->rend_data); - if (introcirc->rend_data->auth_type == REND_NO_AUTH) - rend_client_refetch_renddesc(introcirc->rend_data->onion_address); { connection_t *conn; while ((conn = connection_get_by_type_state_rendquery(CONN_TYPE_AP, AP_CONN_STATE_CIRCUIT_WAIT, - introcirc->rend_data->onion_address, -1))) { + introcirc->rend_data->onion_address))) { conn->state = AP_CONN_STATE_RENDDESC_WAIT; } } @@ -98,42 +154,27 @@ rend_client_send_introduction(origin_circuit_t *introcirc, } /* first 20 bytes of payload are the hash of Bob's pk */ - if (entry->parsed->version == 0) { /* un-versioned descriptor */ - intro_key = entry->parsed->pk; - } else { /* versioned descriptor */ - intro_key = NULL; - SMARTLIST_FOREACH(entry->parsed->intro_nodes, rend_intro_point_t *, - intro, { - if (tor_memeq(introcirc->build_state->chosen_exit->identity_digest, - intro->extend_info->identity_digest, DIGEST_LEN)) { - intro_key = intro->intro_key; - break; - } - }); - if (!intro_key) { - /** XXX This case probably means that the intro point vanished while - * we were building a circuit to it. In the future, we should find - * out how that happened and whether we should kill the circuits to - * removed intro points immediately. See task 1073. */ - int num_intro_points = smartlist_len(entry->parsed->intro_nodes); - if (rend_cache_lookup_entry(introcirc->rend_data->onion_address, - 0, &entry) > 0) { - log_info(LD_REND, "We have both a v0 and a v2 rend desc for this " - "service. The v2 desc doesn't contain the introduction " - "point (and key) to send an INTRODUCE1/2 cell to this " - "introduction point. Assuming the introduction point " - "is for v0 rend clients and using the service key " - "from the v0 desc instead. (This is probably a bug, " - "because we shouldn't even have both a v0 and a v2 " - "descriptor for the same service.)"); - /* See flyspray task 1024. */ - intro_key = entry->parsed->pk; - } else { - log_info(LD_REND, "Internal error: could not find intro key; we " - "only have a v2 rend desc with %d intro points.", - num_intro_points); - goto perm_err; - } + intro_key = NULL; + SMARTLIST_FOREACH(entry->parsed->intro_nodes, rend_intro_point_t *, + intro, { + if (tor_memeq(introcirc->build_state->chosen_exit->identity_digest, + intro->extend_info->identity_digest, DIGEST_LEN)) { + intro_key = intro->intro_key; + break; + } + }); + if (!intro_key) { + log_info(LD_REND, "Could not find intro key for %s at %s; we " + "have a v2 rend desc with %d intro points. " + "Trying a different intro point...", + safe_str_client(introcirc->rend_data->onion_address), + introcirc->build_state->chosen_exit->nickname, + smartlist_len(entry->parsed->intro_nodes)); + + if (rend_client_reextend_intro_circuit(introcirc)) { + goto perm_err; + } else { + return -1; } } if (crypto_pk_get_digest(intro_key, payload)<0) { @@ -236,8 +277,9 @@ rend_client_send_introduction(origin_circuit_t *introcirc, introcirc->_base.purpose = CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT; return 0; -perm_err: - circuit_mark_for_close(TO_CIRCUIT(introcirc), END_CIRC_REASON_INTERNAL); + perm_err: + if (!introcirc->_base.marked_for_close) + circuit_mark_for_close(TO_CIRCUIT(introcirc), END_CIRC_REASON_INTERNAL); circuit_mark_for_close(TO_CIRCUIT(rendcirc), END_CIRC_REASON_INTERNAL); return -2; } @@ -300,45 +342,16 @@ rend_client_introduction_acked(origin_circuit_t *circ, * points. If any remain, extend to a new one and try again. * If none remain, refetch the service descriptor. */ + log_info(LD_REND, "Got nack for %s from %s...", + safe_str_client(circ->rend_data->onion_address), + circ->build_state->chosen_exit->nickname); if (rend_client_remove_intro_point(circ->build_state->chosen_exit, circ->rend_data) > 0) { /* There are introduction points left. Re-extend the circuit to * another intro point and try again. */ - extend_info_t *extend_info; - int result; - extend_info = rend_client_get_random_intro(circ->rend_data); - if (!extend_info) { - log_warn(LD_REND, "No introduction points left for %s. Closing.", - escaped_safe_str(circ->rend_data->onion_address)); - circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_INTERNAL); - return -1; - } - if (circ->remaining_relay_early_cells) { - log_info(LD_REND, - "Got nack for %s from %s. Re-extending circ %d, " - "this time to %s.", - escaped_safe_str(circ->rend_data->onion_address), - circ->build_state->chosen_exit->nickname, - circ->_base.n_circ_id, extend_info->nickname); - result = circuit_extend_to_new_exit(circ, extend_info); - } else { - log_info(LD_REND, - "Got nack for %s from %s. Building a new introduction " - "circuit, this time to %s.", - escaped_safe_str(circ->rend_data->onion_address), - circ->build_state->chosen_exit->nickname, - extend_info->nickname); - circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_FINISHED); - if (!circuit_launch_by_extend_info(CIRCUIT_PURPOSE_C_INTRODUCING, - extend_info, - CIRCLAUNCH_IS_INTERNAL)) { - log_warn(LD_REND, "Building introduction circuit failed."); - result = -1; - } else { - result = 0; - } - } - extend_info_free(extend_info); + int result = rend_client_reextend_intro_circuit(circ); + /* XXXX If that call failed, should we close the rend circuit, + * too? */ return result; } } @@ -488,45 +501,21 @@ directory_get_from_hs_dir(const char *desc_id, const rend_data_t *rend_query) rend_query->onion_address, desc_id_base32, rend_query->auth_type, (rend_query->auth_type == REND_NO_AUTH ? "[none]" : - escaped_safe_str(descriptor_cookie_base64)), + escaped_safe_str_client(descriptor_cookie_base64)), hs_dir->nickname, hs_dir->dir_port); return 1; } -/** If we are not currently fetching a rendezvous service descriptor - * for the service ID <b>query</b>, start a directory connection to fetch a - * new one. - */ -void -rend_client_refetch_renddesc(const char *query) -{ - if (!get_options()->FetchHidServDescriptors) - return; - log_info(LD_REND, "Fetching rendezvous descriptor for service %s", - escaped_safe_str(query)); - if (connection_get_by_type_state_rendquery(CONN_TYPE_DIR, 0, query, 0)) { - log_info(LD_REND,"Would fetch a new renddesc here (for %s), but one is " - "already in progress.", escaped_safe_str(query)); - } else { - /* not one already; initiate a dir rend desc lookup */ - directory_get_from_dirserver(DIR_PURPOSE_FETCH_RENDDESC, - ROUTER_PURPOSE_GENERAL, query, - PDS_RETRY_IF_NO_SERVERS); - } -} - -/** Start a connection to a hidden service directory to fetch a v2 - * rendezvous service descriptor for the base32-encoded service ID - * <b>query</b>. - */ +/** Unless we already have a descriptor for <b>rend_query</b> with at least + * one (possibly) working introduction point in it, start a connection to a + * hidden service directory to fetch a v2 rendezvous service descriptor. */ void rend_client_refetch_v2_renddesc(const rend_data_t *rend_query) { char descriptor_id[DIGEST_LEN]; int replicas_left_to_try[REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS]; - int i, tries_left, r; + int i, tries_left; rend_cache_entry_t *e = NULL; - time_t now = time(NULL); tor_assert(rend_query); /* Are we configured to fetch descriptors? */ if (!get_options()->FetchHidServDescriptors) { @@ -535,15 +524,13 @@ rend_client_refetch_v2_renddesc(const rend_data_t *rend_query) return; } /* Before fetching, check if we already have the descriptor here. */ - r = rend_cache_lookup_entry(rend_query->onion_address, -1, &e); - if (r > 0 && now - e->received < NUM_SECONDS_BEFORE_HS_REFETCH) { + if (rend_cache_lookup_entry(rend_query->onion_address, -1, &e) > 0) { log_info(LD_REND, "We would fetch a v2 rendezvous descriptor, but we " - "already have a fresh copy of that descriptor here. " - "Not fetching."); + "already have that descriptor here. Not fetching."); return; } log_debug(LD_REND, "Fetching v2 rendezvous descriptor for service %s", - safe_str(rend_query->onion_address)); + safe_str_client(rend_query->onion_address)); /* Randomly iterate over the replicas until a descriptor can be fetched * from one of the consecutive nodes, or no options are left. */ tries_left = REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS; @@ -569,8 +556,8 @@ rend_client_refetch_v2_renddesc(const rend_data_t *rend_query) log_info(LD_REND, "Could not pick one of the responsible hidden " "service directories to fetch descriptors, because " "we already tried them all unsuccessfully."); - /* Close pending connections (unless a v0 request is still going on). */ - rend_client_desc_trynow(rend_query->onion_address, 2); + /* Close pending connections. */ + rend_client_desc_trynow(rend_query->onion_address); return; } @@ -600,9 +587,8 @@ rend_client_cancel_descriptor_fetches(void) "Marking for close dir conn fetching rendezvous " "descriptor for unknown service!"); } else { - log_debug(LD_REND, "Marking for close dir conn fetching v%d " + log_debug(LD_REND, "Marking for close dir conn fetching " "rendezvous descriptor for service %s", - (int)(rd->rend_desc_version), safe_str(rd->onion_address)); } connection_mark_for_close(conn); @@ -628,18 +614,13 @@ rend_client_remove_intro_point(extend_info_t *failed_intro, r = rend_cache_lookup_entry(rend_query->onion_address, -1, &ent); if (r<0) { log_warn(LD_BUG, "Malformed service ID %s.", - escaped_safe_str(rend_query->onion_address)); + escaped_safe_str_client(rend_query->onion_address)); return -1; } if (r==0) { log_info(LD_REND, "Unknown service %s. Re-fetching descriptor.", - escaped_safe_str(rend_query->onion_address)); - /* Fetch both, v0 and v2 rend descriptors in parallel. Use whichever - * arrives first. Exception: When using client authorization, only - * fetch v2 descriptors.*/ + escaped_safe_str_client(rend_query->onion_address)); rend_client_refetch_v2_renddesc(rend_query); - if (rend_query->auth_type == REND_NO_AUTH) - rend_client_refetch_renddesc(rend_query->onion_address); return 0; } @@ -653,21 +634,16 @@ rend_client_remove_intro_point(extend_info_t *failed_intro, } } - if (smartlist_len(ent->parsed->intro_nodes) == 0) { + if (! rend_client_any_intro_points_usable(ent)) { log_info(LD_REND, "No more intro points remain for %s. Re-fetching descriptor.", - escaped_safe_str(rend_query->onion_address)); - /* Fetch both, v0 and v2 rend descriptors in parallel. Use whichever - * arrives first. Exception: When using client authorization, only - * fetch v2 descriptors.*/ + escaped_safe_str_client(rend_query->onion_address)); rend_client_refetch_v2_renddesc(rend_query); - if (rend_query->auth_type == REND_NO_AUTH) - rend_client_refetch_renddesc(rend_query->onion_address); /* move all pending streams back to renddesc_wait */ while ((conn = connection_get_by_type_state_rendquery(CONN_TYPE_AP, AP_CONN_STATE_CIRCUIT_WAIT, - rend_query->onion_address, -1))) { + rend_query->onion_address))) { conn->state = AP_CONN_STATE_RENDDESC_WAIT; } @@ -675,7 +651,7 @@ rend_client_remove_intro_point(extend_info_t *failed_intro, } log_info(LD_REND,"%d options left for %s.", smartlist_len(ent->parsed->intro_nodes), - escaped_safe_str(rend_query->onion_address)); + escaped_safe_str_client(rend_query->onion_address)); return 1; } @@ -698,7 +674,7 @@ rend_client_rendezvous_acked(origin_circuit_t *circ, const uint8_t *request, log_info(LD_REND,"Got rendezvous ack. This circuit is now ready for " "rendezvous."); circ->_base.purpose = CIRCUIT_PURPOSE_C_REND_READY; - /* XXXX022 This is a pretty brute-force approach. It'd be better to + /* XXXX023 This is a pretty brute-force approach. It'd be better to * attach only the connections that are waiting on this circuit, rather * than trying to attach them all. See comments bug 743. */ /* If we already have the introduction circuit built, make sure we send @@ -737,7 +713,8 @@ rend_client_receive_rendezvous(origin_circuit_t *circ, const uint8_t *request, tor_assert(circ->build_state->pending_final_cpath); hop = circ->build_state->pending_final_cpath; tor_assert(hop->dh_handshake_state); - if (crypto_dh_compute_secret(hop->dh_handshake_state, (char*)request, + if (crypto_dh_compute_secret(LOG_PROTOCOL_WARN, + hop->dh_handshake_state, (char*)request, DH_KEY_LEN, keys, DIGEST_LEN+CPATH_KEY_MATERIAL_LEN)<0) { log_warn(LD_GENERAL, "Couldn't complete DH handshake."); @@ -767,7 +744,7 @@ rend_client_receive_rendezvous(origin_circuit_t *circ, const uint8_t *request, onion_append_to_cpath(&circ->cpath, hop); circ->build_state->pending_final_cpath = NULL; /* prevent double-free */ - /* XXXX022 This is a pretty brute-force approach. It'd be better to + /* XXXX023 This is a pretty brute-force approach. It'd be better to * attach only the connections that are waiting on this circuit, rather * than trying to attach them all. See comments bug 743. */ connection_ap_attach_pending(); @@ -779,24 +756,18 @@ rend_client_receive_rendezvous(origin_circuit_t *circ, const uint8_t *request, return -1; } -/** Find all the apconns in state AP_CONN_STATE_RENDDESC_WAIT that - * are waiting on query. If there's a working cache entry here - * with at least one intro point, move them to the next state. If - * <b>rend_version</b> is non-negative, fail connections that have - * requested <b>query</b> unless there are still descriptor fetch - * requests in progress for other descriptor versions than - * <b>rend_version</b>. - */ +/** Find all the apconns in state AP_CONN_STATE_RENDDESC_WAIT that are + * waiting on <b>query</b>. If there's a working cache entry here with at + * least one intro point, move them to the next state. */ void -rend_client_desc_trynow(const char *query, int rend_version) +rend_client_desc_trynow(const char *query) { edge_connection_t *conn; rend_cache_entry_t *entry; time_t now = time(NULL); smartlist_t *conns = get_connection_array(); - SMARTLIST_FOREACH(conns, connection_t *, _conn, - { + SMARTLIST_FOREACH_BEGIN(conns, connection_t *, _conn) { if (_conn->type != CONN_TYPE_AP || _conn->state != AP_CONN_STATE_RENDDESC_WAIT || _conn->marked_for_close) @@ -809,7 +780,7 @@ rend_client_desc_trynow(const char *query, int rend_version) assert_connection_ok(TO_CONN(conn), now); if (rend_cache_lookup_entry(conn->rend_data->onion_address, -1, &entry) == 1 && - smartlist_len(entry->parsed->intro_nodes) > 0) { + rend_client_any_intro_points_usable(entry)) { /* either this fetch worked, or it failed but there was a * valid entry from before which we should reuse */ log_info(LD_REND,"Rend desc is usable. Launching circuits."); @@ -828,17 +799,12 @@ rend_client_desc_trynow(const char *query, int rend_version) connection_mark_unattached_ap(conn, END_STREAM_REASON_CANT_ATTACH); } } else { /* 404, or fetch didn't get that far */ - /* Unless there are requests for another descriptor version pending, - * close the connection. */ - if (rend_version >= 0 && - !connection_get_by_type_state_rendquery(CONN_TYPE_DIR, 0, query, - rend_version == 0 ? 2 : 0)) { - log_notice(LD_REND,"Closing stream for '%s.onion': hidden service is " - "unavailable (try again later).", safe_str(query)); - connection_mark_unattached_ap(conn, END_STREAM_REASON_RESOLVEFAILED); - } + log_notice(LD_REND,"Closing stream for '%s.onion': hidden service is " + "unavailable (try again later).", + safe_str_client(query)); + connection_mark_unattached_ap(conn, END_STREAM_REASON_RESOLVEFAILED); } - }); + } SMARTLIST_FOREACH_END(_conn); } /** Return a newly allocated extend_info_t* for a randomly chosen introduction @@ -848,24 +814,63 @@ rend_client_desc_trynow(const char *query, int rend_version) extend_info_t * rend_client_get_random_intro(const rend_data_t *rend_query) { - int i; + extend_info_t *result; rend_cache_entry_t *entry; - rend_intro_point_t *intro; - routerinfo_t *router; if (rend_cache_lookup_entry(rend_query->onion_address, -1, &entry) < 1) { - log_warn(LD_REND, - "Query '%s' didn't have valid rend desc in cache. Failing.", - safe_str(rend_query->onion_address)); + log_warn(LD_REND, + "Query '%s' didn't have valid rend desc in cache. Failing.", + safe_str_client(rend_query->onion_address)); return NULL; } + /* See if we can get a node that complies with ExcludeNodes */ + if ((result = rend_client_get_random_intro_impl(entry, 1, 1))) + return result; + /* If not, and StrictNodes is not set, see if we can return any old node + */ + if (!get_options()->StrictNodes) + return rend_client_get_random_intro_impl(entry, 0, 1); + return NULL; +} + +/** As rend_client_get_random_intro, except assume that StrictNodes is set + * iff <b>strict</b> is true. If <b>warnings</b> is false, don't complain + * to the user when we're out of nodes, even if StrictNodes is true. + */ +static extend_info_t * +rend_client_get_random_intro_impl(const rend_cache_entry_t *entry, + const int strict, + const int warnings) +{ + int i; + + rend_intro_point_t *intro; + routerinfo_t *router; + or_options_t *options = get_options(); + smartlist_t *usable_nodes; + int n_excluded = 0; + + /* We'll keep a separate list of the usable nodes. If this becomes empty, + * no nodes are usable. */ + usable_nodes = smartlist_create(); + smartlist_add_all(usable_nodes, entry->parsed->intro_nodes); + again: - if (smartlist_len(entry->parsed->intro_nodes) == 0) + if (smartlist_len(usable_nodes) == 0) { + if (n_excluded && get_options()->StrictNodes && warnings) { + /* We only want to warn if StrictNodes is really set. Otherwise + * we're just about to retry anyways. + */ + log_warn(LD_REND, "All introduction points for hidden service are " + "at excluded relays, and StrictNodes is set. Skipping."); + } + smartlist_free(usable_nodes); return NULL; + } - i = crypto_rand_int(smartlist_len(entry->parsed->intro_nodes)); - intro = smartlist_get(entry->parsed->intro_nodes, i); + i = crypto_rand_int(smartlist_len(usable_nodes)); + intro = smartlist_get(usable_nodes, i); /* Do we need to look up the router or is the extend info complete? */ if (!intro->extend_info->onion_key) { if (tor_digest_is_zero(intro->extend_info->identity_digest)) @@ -875,16 +880,34 @@ rend_client_get_random_intro(const rend_data_t *rend_query) if (!router) { log_info(LD_REND, "Unknown router with nickname '%s'; trying another.", intro->extend_info->nickname); - rend_intro_point_free(intro); - smartlist_del(entry->parsed->intro_nodes, i); + smartlist_del(usable_nodes, i); goto again; } extend_info_free(intro->extend_info); intro->extend_info = extend_info_from_router(router); } + /* Check if we should refuse to talk to this router. */ + if (options->ExcludeNodes && strict && + routerset_contains_extendinfo(options->ExcludeNodes, + intro->extend_info)) { + n_excluded++; + smartlist_del(usable_nodes, i); + goto again; + } + + smartlist_free(usable_nodes); return extend_info_dup(intro->extend_info); } +/** Return true iff any introduction points still listed in <b>entry</b> are + * usable. */ +int +rend_client_any_intro_points_usable(const rend_cache_entry_t *entry) +{ + return rend_client_get_random_intro_impl( + entry, get_options()->StrictNodes, 0) != NULL; +} + /** Client-side authorizations for hidden services; map of onion address to * rend_service_authorization_t*. */ static strmap_t *auth_hid_servs = NULL; @@ -1009,8 +1032,7 @@ rend_parse_service_authorization(or_options_t *options, int validate_only) err: res = -1; done: - if (auth) - rend_service_authorization_free(auth); + rend_service_authorization_free(auth); SMARTLIST_FOREACH(sl, char *, c, tor_free(c);); smartlist_free(sl); if (!validate_only && res == 0) { diff --git a/src/or/rendclient.h b/src/or/rendclient.h new file mode 100644 index 0000000000..6910c1a97b --- /dev/null +++ b/src/or/rendclient.h @@ -0,0 +1,45 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file rendclient.h + * \brief Header file for rendclient.c. + **/ + +#ifndef _TOR_RENDCLIENT_H +#define _TOR_RENDCLIENT_H + +void rend_client_introcirc_has_opened(origin_circuit_t *circ); +void rend_client_rendcirc_has_opened(origin_circuit_t *circ); +int rend_client_introduction_acked(origin_circuit_t *circ, + const uint8_t *request, + size_t request_len); +void rend_client_refetch_v2_renddesc(const rend_data_t *rend_query); +void rend_client_cancel_descriptor_fetches(void); +int rend_client_remove_intro_point(extend_info_t *failed_intro, + const rend_data_t *rend_query); +int rend_client_rendezvous_acked(origin_circuit_t *circ, + const uint8_t *request, + size_t request_len); +int rend_client_receive_rendezvous(origin_circuit_t *circ, + const uint8_t *request, + size_t request_len); +void rend_client_desc_trynow(const char *query); + +extend_info_t *rend_client_get_random_intro(const rend_data_t *rend_query); +int rend_client_any_intro_points_usable(const rend_cache_entry_t *entry); + +int rend_client_send_introduction(origin_circuit_t *introcirc, + origin_circuit_t *rendcirc); +int rend_parse_service_authorization(or_options_t *options, + int validate_only); +rend_service_authorization_t *rend_client_lookup_service_authorization( + const char *onion_address); +void rend_service_authorization_free_all(void); +rend_data_t *rend_data_dup(const rend_data_t *request); + +#endif + diff --git a/src/or/rendcommon.c b/src/or/rendcommon.c index 8727a70c2e..4d4a90f61a 100644 --- a/src/or/rendcommon.c +++ b/src/or/rendcommon.c @@ -9,6 +9,15 @@ **/ #include "or.h" +#include "circuitbuild.h" +#include "config.h" +#include "rendclient.h" +#include "rendcommon.h" +#include "rendmid.h" +#include "rendservice.h" +#include "rephist.h" +#include "routerlist.h" +#include "routerparse.h" /** Return 0 if one and two are the same service ids, else -1 or 1 */ int @@ -22,6 +31,8 @@ rend_cmp_service_ids(const char *one, const char *two) void rend_service_descriptor_free(rend_service_descriptor_t *desc) { + if (!desc) + return; if (desc->pk) crypto_free_pk_env(desc->pk); if (desc->intro_nodes) { @@ -125,7 +136,8 @@ rend_compute_v2_desc_id(char *desc_id_out, const char *service_id, if (!service_id || strlen(service_id) != REND_SERVICE_ID_LEN_BASE32) { log_warn(LD_REND, "Could not compute v2 descriptor ID: " - "Illegal service ID: %s", safe_str(service_id)); + "Illegal service ID: %s", + safe_str(service_id)); return -1; } if (replica >= REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS) { @@ -138,7 +150,7 @@ rend_compute_v2_desc_id(char *desc_id_out, const char *service_id, service_id, REND_SERVICE_ID_LEN_BASE32) < 0) { log_warn(LD_REND, "Could not compute v2 descriptor ID: " "Illegal characters in service ID: %s", - safe_str(service_id)); + safe_str_client(service_id)); return -1; } /* Calculate current time-period. */ @@ -403,8 +415,7 @@ rend_desc_v2_is_parsable(rend_encoded_v2_service_descriptor_t *desc) &test_intro_size, &test_encoded_size, &test_next, desc->desc_str); - if (test_parsed) - rend_service_descriptor_free(test_parsed); + rend_service_descriptor_free(test_parsed); tor_free(test_intro_content); return (res >= 0); } @@ -414,6 +425,8 @@ void rend_encoded_v2_service_descriptor_free( rend_encoded_v2_service_descriptor_t *desc) { + if (!desc) + return; tor_free(desc->desc_str); tor_free(desc); } @@ -422,10 +435,11 @@ rend_encoded_v2_service_descriptor_free( void rend_intro_point_free(rend_intro_point_t *intro) { - if (intro->extend_info) - extend_info_free(intro->extend_info); - if (intro->intro_key) - crypto_free_pk_env(intro->intro_key); + if (!intro) + return; + + extend_info_free(intro->extend_info); + crypto_free_pk_env(intro->intro_key); tor_free(intro); } @@ -618,7 +632,8 @@ rend_encode_v2_descriptors(smartlist_t *descs_out, } if (router_append_dirobj_signature(desc_str + written, desc_len - written, - desc_digest, service_key) < 0) { + desc_digest, DIGEST_LEN, + service_key) < 0) { log_warn(LD_BUG, "Couldn't sign desc."); rend_encoded_v2_service_descriptor_free(enc); goto err; @@ -655,63 +670,6 @@ rend_encode_v2_descriptors(smartlist_t *descs_out, return seconds_valid; } -/** Encode a service descriptor for <b>desc</b>, and sign it with - * <b>key</b>. Store the descriptor in *<b>str_out</b>, and set - * *<b>len_out</b> to its length. - */ -int -rend_encode_service_descriptor(rend_service_descriptor_t *desc, - crypto_pk_env_t *key, - char **str_out, size_t *len_out) -{ - char *cp; - char *end; - int i, r; - size_t asn1len; - size_t buflen = - PK_BYTES*2*(smartlist_len(desc->intro_nodes)+2);/*Too long, but ok*/ - cp = *str_out = tor_malloc(buflen); - end = cp + PK_BYTES*2*(smartlist_len(desc->intro_nodes)+1); - r = crypto_pk_asn1_encode(desc->pk, cp+2, end-(cp+2)); - if (r < 0) { - tor_free(*str_out); - return -1; - } - asn1len = r; - set_uint16(cp, htons((uint16_t)asn1len)); - cp += 2+asn1len; - set_uint32(cp, htonl((uint32_t)desc->timestamp)); - cp += 4; - set_uint16(cp, htons((uint16_t)smartlist_len(desc->intro_nodes))); - cp += 2; - for (i=0; i < smartlist_len(desc->intro_nodes); ++i) { - rend_intro_point_t *intro = smartlist_get(desc->intro_nodes, i); - char ipoint[HEX_DIGEST_LEN+2]; - const size_t ipoint_len = HEX_DIGEST_LEN+1; - ipoint[0] = '$'; - base16_encode(ipoint+1, HEX_DIGEST_LEN+1, - intro->extend_info->identity_digest, - DIGEST_LEN); - tor_assert(strlen(ipoint) == ipoint_len); - /* Assert that appending ipoint and its NUL won't over overrun the - * buffer. */ - tor_assert(cp + ipoint_len+1 < *str_out + buflen); - memcpy(cp, ipoint, ipoint_len+1); - cp += ipoint_len+1; - } - note_crypto_pk_op(REND_SERVER); - r = crypto_pk_private_sign_digest(key, - cp, buflen - (cp - *str_out), - *str_out, cp-*str_out); - if (r<0) { - tor_free(*str_out); - return -1; - } - cp += r; - *len_out = (size_t)(cp-*str_out); - return 0; -} - /** Parse a service descriptor at <b>str</b> (<b>len</b> bytes). On * success, return a newly alloced service_descriptor_t. On failure, * return NULL. @@ -828,22 +786,27 @@ rend_cache_init(void) /** Helper: free storage held by a single service descriptor cache entry. */ static void -_rend_cache_entry_free(void *p) +rend_cache_entry_free(rend_cache_entry_t *e) { - rend_cache_entry_t *e = p; + if (!e) + return; rend_service_descriptor_free(e->parsed); tor_free(e->desc); tor_free(e); } +static void +_rend_cache_entry_free(void *p) +{ + rend_cache_entry_free(p); +} + /** Free all storage held by the service descriptor cache. */ void rend_cache_free_all(void) { - if (rend_cache) - strmap_free(rend_cache, _rend_cache_entry_free); - if (rend_cache_v2_dir) - digestmap_free(rend_cache_v2_dir, _rend_cache_entry_free); + strmap_free(rend_cache, _rend_cache_entry_free); + digestmap_free(rend_cache_v2_dir, _rend_cache_entry_free); rend_cache = NULL; rend_cache_v2_dir = NULL; } @@ -864,7 +827,7 @@ rend_cache_clean(void) ent = (rend_cache_entry_t*)val; if (ent->parsed->timestamp < cutoff) { iter = strmap_iter_next_rmv(rend_cache, iter); - _rend_cache_entry_free(ent); + rend_cache_entry_free(ent); } else { iter = strmap_iter_next(rend_cache, iter); } @@ -900,9 +863,9 @@ rend_cache_clean_v2_descs_as_dir(void) char key_base32[REND_DESC_ID_V2_LEN_BASE32 + 1]; base32_encode(key_base32, sizeof(key_base32), key, DIGEST_LEN); log_info(LD_REND, "Removing descriptor with ID '%s' from cache", - safe_str(key_base32)); + safe_str_client(key_base32)); iter = digestmap_iter_next_rmv(rend_cache_v2_dir, iter); - _rend_cache_entry_free(ent); + rend_cache_entry_free(ent); } else { iter = digestmap_iter_next(rend_cache_v2_dir, iter); } @@ -978,6 +941,11 @@ rend_cache_lookup_entry(const char *query, int version, rend_cache_entry_t **e) } if (!*e) return 0; + tor_assert((*e)->parsed && (*e)->parsed->intro_nodes); + /* XXX023 hack for now, to return "not found" if there are no intro + * points remaining. See bug 997. */ + if (! rend_client_any_intro_points_usable(*e)) + return 0; return 1; } @@ -1061,7 +1029,6 @@ rend_cache_store(const char *desc, size_t desc_len, int published, char query[REND_SERVICE_ID_LEN_BASE32+1]; char key[REND_SERVICE_ID_LEN_BASE32+2]; /* 0<query>\0 */ time_t now; - or_options_t *options = get_options(); tor_assert(rend_cache); parsed = rend_parse_service_descriptor(desc,desc_len); if (!parsed) { @@ -1082,13 +1049,15 @@ rend_cache_store(const char *desc, size_t desc_len, int published, now = time(NULL); if (parsed->timestamp < now-REND_CACHE_MAX_AGE-REND_CACHE_MAX_SKEW) { log_fn(LOG_PROTOCOL_WARN, LD_REND, - "Service descriptor %s is too old.", safe_str(query)); + "Service descriptor %s is too old.", + safe_str_client(query)); rend_service_descriptor_free(parsed); return -2; } if (parsed->timestamp > now+REND_CACHE_MAX_SKEW) { log_fn(LOG_PROTOCOL_WARN, LD_REND, - "Service descriptor %s is too far in the future.", safe_str(query)); + "Service descriptor %s is too far in the future.", + safe_str_client(query)); rend_service_descriptor_free(parsed); return -2; } @@ -1096,25 +1065,22 @@ rend_cache_store(const char *desc, size_t desc_len, int published, tor_snprintf(key, sizeof(key), "2%s", query); if (!published && strmap_get_lc(rend_cache, key)) { log_info(LD_REND, "We already have a v2 descriptor for service %s.", - safe_str(query)); + safe_str_client(query)); rend_service_descriptor_free(parsed); return -1; } - /* report novel publication to statistics */ - if (published && options->HSAuthorityRecordStats) { - hs_usage_note_publish_total(query, now); - } tor_snprintf(key, sizeof(key), "0%s", query); e = (rend_cache_entry_t*) strmap_get_lc(rend_cache, key); if (e && e->parsed->timestamp > parsed->timestamp) { log_info(LD_REND,"We already have a newer service descriptor %s with the " - "same ID and version.", safe_str(query)); + "same ID and version.", + safe_str_client(query)); rend_service_descriptor_free(parsed); return 0; } if (e && e->len == desc_len && tor_memeq(desc,e->desc,desc_len)) { log_info(LD_REND,"We already have this service descriptor %s.", - safe_str(query)); + safe_str_client(query)); e->received = time(NULL); rend_service_descriptor_free(parsed); return 0; @@ -1122,10 +1088,6 @@ rend_cache_store(const char *desc, size_t desc_len, int published, if (!e) { e = tor_malloc_zero(sizeof(rend_cache_entry_t)); strmap_set_lc(rend_cache, key, e); - /* report novel publication to statistics */ - if (published && options->HSAuthorityRecordStats) { - hs_usage_note_publish_novel(query, now); - } } else { rend_service_descriptor_free(e->parsed); tor_free(e->desc); @@ -1137,7 +1099,7 @@ rend_cache_store(const char *desc, size_t desc_len, int published, memcpy(e->desc, desc, desc_len); log_debug(LD_REND,"Successfully stored rend desc '%s', len %d.", - safe_str(query), (int)desc_len); + safe_str_client(query), (int)desc_len); return 1; } @@ -1188,7 +1150,7 @@ rend_cache_store_v2_desc_as_dir(const char *desc) if (!hid_serv_responsible_for_desc_id(desc_id)) { log_info(LD_REND, "Service descriptor with desc ID %s is not in " "interval that we are responsible for.", - safe_str(desc_id_base32)); + safe_str_client(desc_id_base32)); goto skip; } /* Is descriptor too old? */ @@ -1326,7 +1288,8 @@ rend_cache_store_v2_desc_as_client(const char *desc, /* Decode/decrypt introduction points. */ if (intro_content) { if (rend_query->auth_type != REND_NO_AUTH && - rend_query->descriptor_cookie) { + !tor_mem_is_zero(rend_query->descriptor_cookie, + sizeof(rend_query->descriptor_cookie))) { char *ipos_decrypted = NULL; size_t ipos_decrypted_size; if (rend_decrypt_introduction_points(&ipos_decrypted, @@ -1361,14 +1324,14 @@ rend_cache_store_v2_desc_as_client(const char *desc, /* Is descriptor too old? */ if (parsed->timestamp < now - REND_CACHE_MAX_AGE-REND_CACHE_MAX_SKEW) { log_warn(LD_REND, "Service descriptor with service ID %s is too old.", - safe_str(service_id)); + safe_str_client(service_id)); retval = -2; goto err; } /* Is descriptor too far in the future? */ if (parsed->timestamp > now + REND_CACHE_MAX_SKEW) { log_warn(LD_REND, "Service descriptor with service ID %s is too far in " - "the future.", safe_str(service_id)); + "the future.", safe_str_client(service_id)); retval = -2; goto err; } @@ -1376,7 +1339,7 @@ rend_cache_store_v2_desc_as_client(const char *desc, tor_snprintf(key, sizeof(key), "0%s", service_id); if (strmap_get_lc(rend_cache, key)) { log_info(LD_REND, "We already have a v0 descriptor for service ID %s.", - safe_str(service_id)); + safe_str_client(service_id)); retval = -1; goto err; } @@ -1386,14 +1349,14 @@ rend_cache_store_v2_desc_as_client(const char *desc, if (e && e->parsed->timestamp > parsed->timestamp) { log_info(LD_REND, "We already have a newer service descriptor for " "service ID %s with the same desc ID and version.", - safe_str(service_id)); + safe_str_client(service_id)); retval = 0; goto err; } /* Do we already have this descriptor? */ if (e && !strcmp(desc, e->desc)) { log_info(LD_REND,"We already have this service descriptor %s.", - safe_str(service_id)); + safe_str_client(service_id)); e->received = time(NULL); retval = 0; goto err; @@ -1411,12 +1374,11 @@ rend_cache_store_v2_desc_as_client(const char *desc, strlcpy(e->desc, desc, encoded_size + 1); e->len = encoded_size; log_debug(LD_REND,"Successfully stored rend desc '%s', len %d.", - safe_str(service_id), (int)encoded_size); + safe_str_client(service_id), (int)encoded_size); return 1; err: - if (parsed) - rend_service_descriptor_free(parsed); + rend_service_descriptor_free(parsed); tor_free(intro_content); return retval; } diff --git a/src/or/rendcommon.h b/src/or/rendcommon.h new file mode 100644 index 0000000000..c51039f1f2 --- /dev/null +++ b/src/or/rendcommon.h @@ -0,0 +1,67 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file rendcommon.h + * \brief Header file for rendcommon.c. + **/ + +#ifndef _TOR_RENDCOMMON_H +#define _TOR_RENDCOMMON_H + +/** Free all storage associated with <b>data</b> */ +static INLINE void +rend_data_free(rend_data_t *data) +{ + tor_free(data); +} + +int rend_cmp_service_ids(const char *one, const char *two); + +void rend_process_relay_cell(circuit_t *circ, const crypt_path_t *layer_hint, + int command, size_t length, + const uint8_t *payload); + +void rend_service_descriptor_free(rend_service_descriptor_t *desc); +rend_service_descriptor_t *rend_parse_service_descriptor(const char *str, + size_t len); +int rend_get_service_id(crypto_pk_env_t *pk, char *out); +void rend_encoded_v2_service_descriptor_free( + rend_encoded_v2_service_descriptor_t *desc); +void rend_intro_point_free(rend_intro_point_t *intro); + +void rend_cache_init(void); +void rend_cache_clean(void); +void rend_cache_clean_v2_descs_as_dir(void); +void rend_cache_purge(void); +void rend_cache_free_all(void); +int rend_valid_service_id(const char *query); +int rend_cache_lookup_desc(const char *query, int version, const char **desc, + size_t *desc_len); +int rend_cache_lookup_entry(const char *query, int version, + rend_cache_entry_t **entry_out); +int rend_cache_lookup_v2_desc_as_dir(const char *query, const char **desc); +int rend_cache_store(const char *desc, size_t desc_len, int published, + const char *service_id); +int rend_cache_store_v2_desc_as_client(const char *desc, + const rend_data_t *rend_query); +int rend_cache_store_v2_desc_as_dir(const char *desc); +int rend_cache_size(void); +int rend_encode_v2_descriptors(smartlist_t *descs_out, + rend_service_descriptor_t *desc, time_t now, + uint8_t period, rend_auth_type_t auth_type, + crypto_pk_env_t *client_key, + smartlist_t *client_cookies); +int rend_compute_v2_desc_id(char *desc_id_out, const char *service_id, + const char *descriptor_cookie, + time_t now, uint8_t replica); +int rend_id_is_in_interval(const char *a, const char *b, const char *c); +void rend_get_descriptor_id_bytes(char *descriptor_id_out, + const char *service_id, + const char *secret_id_part); + +#endif + diff --git a/src/or/rendmid.c b/src/or/rendmid.c index d73f0a1ba9..04edd8e3e2 100644 --- a/src/or/rendmid.c +++ b/src/or/rendmid.c @@ -8,6 +8,11 @@ **/ #include "or.h" +#include "circuitlist.h" +#include "config.h" +#include "relay.h" +#include "rendmid.h" +#include "rephist.h" /** Respond to an ESTABLISH_INTRO cell by checking the signed data and * setting the circuit's purpose and service pk digest. diff --git a/src/or/rendmid.h b/src/or/rendmid.h new file mode 100644 index 0000000000..5ed87fd2b1 --- /dev/null +++ b/src/or/rendmid.h @@ -0,0 +1,25 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file rendmid.h + * \brief Header file for rendmid.c. + **/ + +#ifndef _TOR_RENDMID_H +#define _TOR_RENDMID_H + +int rend_mid_establish_intro(or_circuit_t *circ, const uint8_t *request, + size_t request_len); +int rend_mid_introduce(or_circuit_t *circ, const uint8_t *request, + size_t request_len); +int rend_mid_establish_rendezvous(or_circuit_t *circ, const uint8_t *request, + size_t request_len); +int rend_mid_rendezvous(or_circuit_t *circ, const uint8_t *request, + size_t request_len); + +#endif + diff --git a/src/or/rendservice.c b/src/or/rendservice.c index d1cc7f4f11..35e8b9057a 100644 --- a/src/or/rendservice.c +++ b/src/or/rendservice.c @@ -8,10 +8,23 @@ **/ #include "or.h" +#include "circuitbuild.h" +#include "circuitlist.h" +#include "circuituse.h" +#include "config.h" +#include "directory.h" +#include "networkstatus.h" +#include "rendclient.h" +#include "rendcommon.h" +#include "rendservice.h" +#include "router.h" +#include "relay.h" +#include "rephist.h" +#include "routerlist.h" +#include "routerparse.h" static origin_circuit_t *find_intro_circuit(rend_intro_point_t *intro, - const char *pk_digest, - int desc_version); + const char *pk_digest); /** Represents the mapping from a virtual port of a rendezvous service to * a real port on some IP. @@ -42,8 +55,6 @@ typedef struct rend_service_t { /* Fields specified in config file */ char *directory; /**< where in the filesystem it stores it */ smartlist_t *ports; /**< List of rend_service_port_config_t */ - int descriptor_version; /**< Rendezvous descriptor version that will be - * published. */ rend_auth_type_t auth_type; /**< Client authorization type or 0 if no client * authorization is performed. */ smartlist_t *clients; /**< List of rend_authorized_client_t's of @@ -58,7 +69,7 @@ typedef struct rend_service_t { * or are trying to establish. */ time_t intro_period_started; /**< Start of the current period to build * introduction points. */ - int n_intro_circuits_launched; /**< count of intro circuits we have + int n_intro_circuits_launched; /**< Count of intro circuits we have * established in this period. */ rend_service_descriptor_t *desc; /**< Current hidden service descriptor. */ time_t desc_is_dirty; /**< Time at which changes to the hidden service @@ -90,7 +101,8 @@ num_rend_services(void) static void rend_authorized_client_free(rend_authorized_client_t *client) { - if (!client) return; + if (!client) + return; if (client->client_key) crypto_free_pk_env(client->client_key); tor_free(client->client_name); @@ -109,7 +121,9 @@ rend_authorized_client_strmap_item_free(void *authorized_client) static void rend_service_free(rend_service_t *service) { - if (!service) return; + if (!service) + return; + tor_free(service->directory); SMARTLIST_FOREACH(service->ports, void*, p, tor_free(p)); smartlist_free(service->ports); @@ -120,15 +134,14 @@ rend_service_free(rend_service_t *service) rend_intro_point_free(intro);); smartlist_free(service->intro_nodes); } - if (service->desc) - rend_service_descriptor_free(service->desc); + + rend_service_descriptor_free(service->desc); if (service->clients) { SMARTLIST_FOREACH(service->clients, rend_authorized_client_t *, c, rend_authorized_client_free(c);); smartlist_free(service->clients); } - if (service->accepted_intros) - digestmap_free(service->accepted_intros, _tor_free); + digestmap_free(service->accepted_intros, _tor_free); tor_free(service); } @@ -137,9 +150,9 @@ rend_service_free(rend_service_t *service) void rend_service_free_all(void) { - if (!rend_service_list) { + if (!rend_service_list) return; - } + SMARTLIST_FOREACH(rend_service_list, rend_service_t*, ptr, rend_service_free(ptr)); smartlist_free(rend_service_list); @@ -156,36 +169,6 @@ rend_add_service(rend_service_t *service) service->intro_nodes = smartlist_create(); - /* If the service is configured to publish unversioned (v0) and versioned - * descriptors (v2 or higher), split it up into two separate services - * (unless it is configured to perform client authorization). */ - if (service->descriptor_version == -1) { - if (service->auth_type == REND_NO_AUTH) { - rend_service_t *v0_service = tor_malloc_zero(sizeof(rend_service_t)); - v0_service->directory = tor_strdup(service->directory); - v0_service->ports = smartlist_create(); - SMARTLIST_FOREACH(service->ports, rend_service_port_config_t *, p, { - rend_service_port_config_t *copy = - tor_malloc_zero(sizeof(rend_service_port_config_t)); - memcpy(copy, p, sizeof(rend_service_port_config_t)); - smartlist_add(v0_service->ports, copy); - }); - v0_service->intro_period_started = service->intro_period_started; - v0_service->descriptor_version = 0; /* Unversioned descriptor. */ - v0_service->auth_type = REND_NO_AUTH; - rend_add_service(v0_service); - } - - service->descriptor_version = 2; /* Versioned descriptor. */ - } - - if (service->auth_type != REND_NO_AUTH && !service->descriptor_version) { - log_warn(LD_CONFIG, "Hidden service with client authorization and " - "version 0 descriptors configured; ignoring."); - rend_service_free(service); - return; - } - if (service->auth_type != REND_NO_AUTH && smartlist_len(service->clients) == 0) { log_warn(LD_CONFIG, "Hidden service with client authorization but no " @@ -297,7 +280,7 @@ rend_config_services(or_options_t *options, int validate_only) for (line = options->RendConfigLines; line; line = line->next) { if (!strcasecmp(line->key, "HiddenServiceDir")) { - if (service) { + if (service) { /* register the one we just finished parsing */ if (validate_only) rend_service_free(service); else @@ -307,7 +290,6 @@ rend_config_services(or_options_t *options, int validate_only) service->directory = tor_strdup(line->value); service->ports = smartlist_create(); service->intro_period_started = time(NULL); - service->descriptor_version = -1; /**< All descriptor versions. */ continue; } if (!service) { @@ -433,35 +415,13 @@ rend_config_services(or_options_t *options, int validate_only) return -1; } } else { - smartlist_t *versions; - char *version_str; - int i, version, ver_ok=1, versions_bitmask = 0; tor_assert(!strcasecmp(line->key, "HiddenServiceVersion")); - versions = smartlist_create(); - smartlist_split_string(versions, line->value, ",", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - for (i = 0; i < smartlist_len(versions); i++) { - version_str = smartlist_get(versions, i); - if (strlen(version_str) != 1 || strspn(version_str, "02") != 1) { - log_warn(LD_CONFIG, - "HiddenServiceVersion can only be 0 and/or 2."); - SMARTLIST_FOREACH(versions, char *, cp, tor_free(cp)); - smartlist_free(versions); - rend_service_free(service); - return -1; - } - version = (int)tor_parse_long(version_str, 10, 0, INT_MAX, &ver_ok, - NULL); - if (!ver_ok) - continue; - versions_bitmask |= 1 << version; + if (strcmp(line->value, "2")) { + log_warn(LD_CONFIG, + "The only supported HiddenServiceVersion is 2."); + rend_service_free(service); + return -1; } - /* If exactly one version is set, change descriptor_version to that - * value; otherwise leave it at -1. */ - if (versions_bitmask == 1 << 0) service->descriptor_version = 0; - if (versions_bitmask == 1 << 2) service->descriptor_version = 2; - SMARTLIST_FOREACH(versions, char *, cp, tor_free(cp)); - smartlist_free(versions); } } if (service) { @@ -483,8 +443,7 @@ rend_config_services(or_options_t *options, int validate_only) * probably ok? */ SMARTLIST_FOREACH(rend_service_list, rend_service_t *, new, { SMARTLIST_FOREACH(old_service_list, rend_service_t *, old, { - if (!strcmp(old->directory, new->directory) && - old->descriptor_version == new->descriptor_version) { + if (!strcmp(old->directory, new->directory)) { smartlist_add_all(new->intro_nodes, old->intro_nodes); smartlist_clear(old->intro_nodes); smartlist_add(surviving_services, old); @@ -507,18 +466,16 @@ rend_config_services(or_options_t *options, int validate_only) tor_assert(oc->rend_data); SMARTLIST_FOREACH(surviving_services, rend_service_t *, ptr, { if (tor_memeq(ptr->pk_digest, oc->rend_data->rend_pk_digest, - DIGEST_LEN) && - ptr->descriptor_version == oc->rend_data->rend_desc_version) { + DIGEST_LEN)) { keep_it = 1; break; } }); if (keep_it) continue; - log_info(LD_REND, "Closing intro point %s for service %s version %d.", - safe_str(oc->build_state->chosen_exit->nickname), - oc->rend_data->onion_address, - oc->rend_data->rend_desc_version); + log_info(LD_REND, "Closing intro point %s for service %s.", + safe_str_client(oc->build_state->chosen_exit->nickname), + oc->rend_data->onion_address); circuit_mark_for_close(circ, END_CIRC_REASON_FINISHED); /* XXXX Is there another reason we should use here? */ } @@ -541,14 +498,13 @@ rend_service_update_descriptor(rend_service_t *service) rend_service_descriptor_t *d; origin_circuit_t *circ; int i; - if (service->desc) { - rend_service_descriptor_free(service->desc); - service->desc = NULL; - } + + rend_service_descriptor_free(service->desc); + service->desc = NULL; + d = service->desc = tor_malloc_zero(sizeof(rend_service_descriptor_t)); d->pk = crypto_pk_dup_key(service->private_key); d->timestamp = time(NULL); - d->version = service->descriptor_version; d->intro_nodes = smartlist_create(); /* Support intro protocols 2 and 3. */ d->protocols = (1 << 2) + (1 << 3); @@ -556,7 +512,7 @@ rend_service_update_descriptor(rend_service_t *service) for (i = 0; i < smartlist_len(service->intro_nodes); ++i) { rend_intro_point_t *intro_svc = smartlist_get(service->intro_nodes, i); rend_intro_point_t *intro_desc; - circ = find_intro_circuit(intro_svc, service->pk_digest, d->version); + circ = find_intro_circuit(intro_svc, service->pk_digest); if (!circ || circ->_base.purpose != CIRCUIT_PURPOSE_S_INTRO) continue; @@ -797,17 +753,15 @@ rend_service_load_keys(void) return r; } -/** Return the service whose public key has a digest of <b>digest</b> and - * which publishes the given descriptor <b>version</b>. Return NULL if no - * such service exists. +/** Return the service whose public key has a digest of <b>digest</b>, or + * NULL if no such service exists. */ static rend_service_t * -rend_service_get_by_pk_digest_and_version(const char* digest, - uint8_t version) +rend_service_get_by_pk_digest(const char* digest) { SMARTLIST_FOREACH(rend_service_list, rend_service_t*, s, - if (tor_memeq(s->pk_digest,digest,DIGEST_LEN) && - s->descriptor_version == version) return s); + if (tor_memeq(s->pk_digest,digest,DIGEST_LEN)) + return s); return NULL; } @@ -894,6 +848,7 @@ clean_accepted_intros(rend_service_t *service, time_t now) /** Respond to an INTRODUCE2 cell by launching a circuit to the chosen * rendezvous point. */ + /* XXX022 this function sure could use some organizing. -RD */ int rend_service_introduce(origin_circuit_t *circuit, const uint8_t *request, size_t request_len) @@ -921,6 +876,8 @@ rend_service_introduce(origin_circuit_t *circuit, const uint8_t *request, time_t now = time(NULL); char diffie_hellman_hash[DIGEST_LEN]; time_t *access_time; + or_options_t *options = get_options(); + tor_assert(circuit->rend_data); base32_encode(serviceid, REND_SERVICE_ID_LEN_BASE32+1, @@ -944,21 +901,16 @@ rend_service_introduce(origin_circuit_t *circuit, const uint8_t *request, } /* look up service depending on circuit. */ - service = rend_service_get_by_pk_digest_and_version( - circuit->rend_data->rend_pk_digest, - circuit->rend_data->rend_desc_version); + service = rend_service_get_by_pk_digest( + circuit->rend_data->rend_pk_digest); if (!service) { log_warn(LD_REND, "Got an INTRODUCE2 cell for an unrecognized service %s.", escaped(serviceid)); return -1; } - /* if descriptor version is 2, use intro key instead of service key. */ - if (circuit->rend_data->rend_desc_version == 0) { - intro_key = service->private_key; - } else { - intro_key = circuit->intro_key; - } + /* use intro key instead of service key. */ + intro_key = circuit->intro_key; /* first DIGEST_LEN bytes of request is intro or service pk digest */ crypto_pk_get_digest(intro_key, intro_key_digest); @@ -989,7 +941,7 @@ rend_service_introduce(origin_circuit_t *circuit, const uint8_t *request, len = r; if (*buf == 3) { /* Version 3 INTRODUCE2 cell. */ - time_t ts = 0, now = time(NULL); + time_t ts = 0; v3_shift = 1; auth_type = buf[1]; switch (auth_type) { @@ -1083,7 +1035,7 @@ rend_service_introduce(origin_circuit_t *circuit, const uint8_t *request, router = router_get_by_nickname(rp_nickname, 0); if (!router) { log_info(LD_REND, "Couldn't find router %s named in introduce2 cell.", - escaped_safe_str(rp_nickname)); + escaped_safe_str_client(rp_nickname)); /* XXXX Add a no-such-router reason? */ reason = END_CIRC_REASON_TORPROTOCOL; goto err; @@ -1098,6 +1050,15 @@ rend_service_introduce(origin_circuit_t *circuit, const uint8_t *request, goto err; } + /* Check if we'd refuse to talk to this router */ + if (options->ExcludeNodes && options->StrictNodes && + routerset_contains_extendinfo(options->ExcludeNodes, extend_info)) { + log_warn(LD_REND, "Client asked to rendezvous at a relay that we " + "exclude, and StrictNodes is set. Refusing service."); + reason = END_CIRC_REASON_INTERNAL; /* XXX might leak why we refused */ + goto err; + } + r_cookie = ptr; base16_encode(hexcookie,9,r_cookie,4); @@ -1158,7 +1119,8 @@ rend_service_introduce(origin_circuit_t *circuit, const uint8_t *request, reason = END_CIRC_REASON_INTERNAL; goto err; } - if (crypto_dh_compute_secret(dh, ptr+REND_COOKIE_LEN, DH_KEY_LEN, keys, + if (crypto_dh_compute_secret(LOG_PROTOCOL_WARN, dh, ptr+REND_COOKIE_LEN, + DH_KEY_LEN, keys, DIGEST_LEN+CPATH_KEY_MATERIAL_LEN)<0) { log_warn(LD_BUG, "Internal error: couldn't complete DH handshake"); reason = END_CIRC_REASON_INTERNAL; @@ -1168,7 +1130,7 @@ rend_service_introduce(origin_circuit_t *circuit, const uint8_t *request, circ_needs_uptime = rend_service_requires_uptime(service); /* help predict this next time */ - rep_hist_note_used_internal(time(NULL), circ_needs_uptime, 1); + rep_hist_note_used_internal(now, circ_needs_uptime, 1); /* Launch a circuit to alice's chosen rendezvous point. */ @@ -1184,14 +1146,16 @@ rend_service_introduce(origin_circuit_t *circuit, const uint8_t *request, if (!launched) { /* give up */ log_warn(LD_REND, "Giving up launching first hop of circuit to rendezvous " "point %s for service %s.", - escaped_safe_str(extend_info->nickname), serviceid); + escaped_safe_str_client(extend_info->nickname), + serviceid); reason = END_CIRC_REASON_CONNECTFAILED; goto err; } log_info(LD_REND, "Accepted intro; launching circuit to %s " "(cookie %s) for service %s.", - escaped_safe_str(extend_info->nickname), hexcookie, serviceid); + escaped_safe_str_client(extend_info->nickname), + hexcookie, serviceid); tor_assert(launched->build_state); /* Fill in the circuit's state. */ launched->rend_data = tor_malloc_zero(sizeof(rend_data_t)); @@ -1201,11 +1165,10 @@ rend_service_introduce(origin_circuit_t *circuit, const uint8_t *request, memcpy(launched->rend_data->rend_cookie, r_cookie, REND_COOKIE_LEN); strlcpy(launched->rend_data->onion_address, service->service_id, sizeof(launched->rend_data->onion_address)); - launched->rend_data->rend_desc_version = service->descriptor_version; launched->build_state->pending_final_cpath = cpath = tor_malloc_zero(sizeof(crypt_path_t)); cpath->magic = CRYPT_PATH_MAGIC; - launched->build_state->expiry_time = time(NULL) + MAX_REND_TIMEOUT; + launched->build_state->expiry_time = now + MAX_REND_TIMEOUT; cpath->dh_handshake_state = dh; dh = NULL; @@ -1289,7 +1252,7 @@ rend_service_launch_establish_intro(rend_service_t *service, log_info(LD_REND, "Launching circuit to introduction point %s for service %s", - escaped_safe_str(intro->extend_info->nickname), + escaped_safe_str_client(intro->extend_info->nickname), service->service_id); rep_hist_note_used_internal(time(NULL), 1, 0); @@ -1302,7 +1265,7 @@ rend_service_launch_establish_intro(rend_service_t *service, if (!launched) { log_info(LD_REND, "Can't launch circuit to establish introduction at %s.", - escaped_safe_str(intro->extend_info->nickname)); + escaped_safe_str_client(intro->extend_info->nickname)); return -1; } @@ -1325,18 +1288,16 @@ rend_service_launch_establish_intro(rend_service_t *service, strlcpy(launched->rend_data->onion_address, service->service_id, sizeof(launched->rend_data->onion_address)); memcpy(launched->rend_data->rend_pk_digest, service->pk_digest, DIGEST_LEN); - launched->rend_data->rend_desc_version = service->descriptor_version; - if (service->descriptor_version == 2) - launched->intro_key = crypto_pk_dup_key(intro->intro_key); + launched->intro_key = crypto_pk_dup_key(intro->intro_key); if (launched->_base.state == CIRCUIT_STATE_OPEN) rend_service_intro_has_opened(launched); return 0; } /** Return the number of introduction points that are or have been - * established for the given service address and rendezvous version. */ + * established for the given service address in <b>query</b>. */ static int -count_established_intro_points(const char *query, int rend_version) +count_established_intro_points(const char *query) { int num_ipos = 0; circuit_t *circ; @@ -1347,7 +1308,6 @@ count_established_intro_points(const char *query, int rend_version) circ->purpose == CIRCUIT_PURPOSE_S_INTRO)) { origin_circuit_t *oc = TO_ORIGIN_CIRCUIT(circ); if (oc->rend_data && - oc->rend_data->rend_desc_version == rend_version && !rend_cmp_service_ids(query, oc->rend_data->onion_address)) num_ipos++; } @@ -1377,9 +1337,8 @@ rend_service_intro_has_opened(origin_circuit_t *circuit) base32_encode(serviceid, REND_SERVICE_ID_LEN_BASE32+1, circuit->rend_data->rend_pk_digest, REND_SERVICE_ID_LEN); - service = rend_service_get_by_pk_digest_and_version( - circuit->rend_data->rend_pk_digest, - circuit->rend_data->rend_desc_version); + service = rend_service_get_by_pk_digest( + circuit->rend_data->rend_pk_digest); if (!service) { log_warn(LD_REND, "Unrecognized service ID %s on introduction circuit %d.", serviceid, circuit->_base.n_circ_id); @@ -1388,28 +1347,34 @@ rend_service_intro_has_opened(origin_circuit_t *circuit) } /* If we already have enough introduction circuits for this service, - * redefine this one as a general circuit. */ - if (count_established_intro_points(serviceid, - circuit->rend_data->rend_desc_version) > NUM_INTRO_POINTS) { - log_info(LD_CIRC|LD_REND, "We have just finished an introduction " - "circuit, but we already have enough. Redefining purpose to " - "general."); - TO_CIRCUIT(circuit)->purpose = CIRCUIT_PURPOSE_C_GENERAL; - circuit_has_opened(circuit); - return; + * redefine this one as a general circuit or close it, depending. */ + if (count_established_intro_points(serviceid) > NUM_INTRO_POINTS) { + or_options_t *options = get_options(); + if (options->ExcludeNodes) { + /* XXXX in some future version, we can test whether the transition is + allowed or not given the actual nodes in the circuit. But for now, + this case, we might as well close the thing. */ + log_info(LD_CIRC|LD_REND, "We have just finished an introduction " + "circuit, but we already have enough. Closing it."); + circuit_mark_for_close(TO_CIRCUIT(circuit), END_CIRC_REASON_NONE); + return; + } else { + tor_assert(circuit->build_state->is_internal); + log_info(LD_CIRC|LD_REND, "We have just finished an introduction " + "circuit, but we already have enough. Redefining purpose to " + "general; leaving as internal."); + TO_CIRCUIT(circuit)->purpose = CIRCUIT_PURPOSE_C_GENERAL; + circuit_has_opened(circuit); + return; + } } log_info(LD_REND, "Established circuit %d as introduction point for service %s", circuit->_base.n_circ_id, serviceid); - /* If the introduction point will not be used in an unversioned - * descriptor, use the intro key instead of the service key in - * ESTABLISH_INTRO. */ - if (service->descriptor_version == 0) - intro_key = service->private_key; - else - intro_key = circuit->intro_key; + /* Use the intro key instead of the service key in ESTABLISH_INTRO. */ + intro_key = circuit->intro_key; /* Build the payload for a RELAY_ESTABLISH_INTRO cell. */ r = crypto_pk_asn1_encode(intro_key, buf+2, RELAY_PAYLOAD_SIZE-2); @@ -1453,7 +1418,7 @@ rend_service_intro_has_opened(origin_circuit_t *circuit) /** Called when we get an INTRO_ESTABLISHED cell; mark the circuit as a * live introduction point, and note that the service descriptor is - * now out-of-date.*/ + * now out-of-date. */ int rend_service_intro_established(origin_circuit_t *circuit, const uint8_t *request, @@ -1470,9 +1435,8 @@ rend_service_intro_established(origin_circuit_t *circuit, goto err; } tor_assert(circuit->rend_data); - service = rend_service_get_by_pk_digest_and_version( - circuit->rend_data->rend_pk_digest, - circuit->rend_data->rend_desc_version); + service = rend_service_get_by_pk_digest( + circuit->rend_data->rend_pk_digest); if (!service) { log_warn(LD_REND, "Unknown service on introduction circuit %d.", circuit->_base.n_circ_id); @@ -1522,9 +1486,8 @@ rend_service_rendezvous_has_opened(origin_circuit_t *circuit) "cookie %s for service %s", circuit->_base.n_circ_id, hexcookie, serviceid); - service = rend_service_get_by_pk_digest_and_version( - circuit->rend_data->rend_pk_digest, - circuit->rend_data->rend_desc_version); + service = rend_service_get_by_pk_digest( + circuit->rend_data->rend_pk_digest); if (!service) { log_warn(LD_GENERAL, "Internal error: unrecognized service ID on " "introduction circuit."); @@ -1580,13 +1543,12 @@ rend_service_rendezvous_has_opened(origin_circuit_t *circuit) */ /** Return the (possibly non-open) introduction circuit ending at - * <b>intro</b> for the service whose public key is <b>pk_digest</b> and - * which publishes descriptor of version <b>desc_version</b>. Return - * NULL if no such service is found. + * <b>intro</b> for the service whose public key is <b>pk_digest</b>. + * (<b>desc_version</b> is ignored). Return NULL if no such service is + * found. */ static origin_circuit_t * -find_intro_circuit(rend_intro_point_t *intro, const char *pk_digest, - int desc_version) +find_intro_circuit(rend_intro_point_t *intro, const char *pk_digest) { origin_circuit_t *circ = NULL; @@ -1595,8 +1557,7 @@ find_intro_circuit(rend_intro_point_t *intro, const char *pk_digest, CIRCUIT_PURPOSE_S_INTRO))) { if (tor_memeq(circ->build_state->chosen_exit->identity_digest, intro->extend_info->identity_digest, DIGEST_LEN) && - circ->rend_data && - circ->rend_data->rend_desc_version == desc_version) { + circ->rend_data) { return circ; } } @@ -1606,8 +1567,7 @@ find_intro_circuit(rend_intro_point_t *intro, const char *pk_digest, CIRCUIT_PURPOSE_S_ESTABLISH_INTRO))) { if (tor_memeq(circ->build_state->chosen_exit->identity_digest, intro->extend_info->identity_digest, DIGEST_LEN) && - circ->rend_data && - circ->rend_data->rend_desc_version == desc_version) { + circ->rend_data) { return circ; } } @@ -1640,6 +1600,7 @@ directory_post_to_hs_dir(rend_service_descriptor_t *renddesc, } for (j = 0; j < smartlist_len(responsible_dirs); j++) { char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1]; + char *hs_dir_ip; hs_dir = smartlist_get(responsible_dirs, j); if (smartlist_digest_isin(renddesc->successful_uploads, hs_dir->identity_digest)) @@ -1661,15 +1622,18 @@ directory_post_to_hs_dir(rend_service_descriptor_t *renddesc, strlen(desc->desc_str), 0); base32_encode(desc_id_base32, sizeof(desc_id_base32), desc->desc_id, DIGEST_LEN); + hs_dir_ip = tor_dup_ip(hs_dir->addr); log_info(LD_REND, "Sending publish request for v2 descriptor for " "service '%s' with descriptor ID '%s' with validity " "of %d seconds to hidden service directory '%s' on " - "port %d.", - safe_str(service_id), - safe_str(desc_id_base32), + "%s:%d.", + safe_str_client(service_id), + safe_str_client(desc_id_base32), seconds_valid, hs_dir->nickname, - hs_dir->dir_port); + hs_dir_ip, + hs_dir->or_port); + tor_free(hs_dir_ip); /* Remember successful upload to this router for next time. */ if (!smartlist_digest_isin(successful_uploads, hs_dir->identity_digest)) smartlist_add(successful_uploads, hs_dir->identity_digest); @@ -1699,9 +1663,8 @@ directory_post_to_hs_dir(rend_service_descriptor_t *renddesc, smartlist_free(successful_uploads); } -/** Encode and sign up-to-date v0 and/or v2 service descriptors for - * <b>service</b>, and upload it/them to all the dirservers/to the - * responsible hidden service directories. +/** Encode and sign an up-to-date service descriptor for <b>service</b>, + * and upload it/them to the responsible hidden service directories. */ static void upload_service_descriptor(rend_service_t *service) @@ -1713,35 +1676,8 @@ upload_service_descriptor(rend_service_t *service) rendpostperiod = get_options()->RendPostPeriod; - /* Upload unversioned (v0) descriptor? */ - if (service->descriptor_version == 0 && - get_options()->PublishHidServDescriptors) { - char *desc; - size_t desc_len; - /* Encode the descriptor. */ - if (rend_encode_service_descriptor(service->desc, - service->private_key, - &desc, &desc_len)<0) { - log_warn(LD_BUG, "Internal error: couldn't encode service descriptor; " - "not uploading."); - return; - } - - /* Post it to the dirservers */ - rend_get_service_id(service->desc->pk, serviceid); - log_info(LD_REND, "Sending publish request for hidden service %s", - serviceid); - directory_post_to_dirservers(DIR_PURPOSE_UPLOAD_RENDDESC, - ROUTER_PURPOSE_GENERAL, - HIDSERV_AUTHORITY, desc, desc_len, 0); - tor_free(desc); - service->next_upload_time = now + rendpostperiod; - uploaded = 1; - } - - /* Upload v2 descriptor? */ - if (service->descriptor_version == 2 && - get_options()->PublishHidServDescriptors) { + /* Upload descriptor? */ + if (get_options()->PublishHidServDescriptors) { networkstatus_t *c = networkstatus_get_latest_consensus(); if (c && smartlist_len(c->routerstatus_list) > 0) { int seconds_valid, i, j, num_descs; @@ -1880,8 +1816,7 @@ rend_services_introduce(void) for (j=0; j < smartlist_len(service->intro_nodes); ++j) { intro = smartlist_get(service->intro_nodes, j); router = router_get_by_digest(intro->extend_info->identity_digest); - if (!router || !find_intro_circuit(intro, service->pk_digest, - service->descriptor_version)) { + if (!router || !find_intro_circuit(intro, service->pk_digest)) { log_info(LD_REND,"Giving up on %s as intro point for %s.", intro->extend_info->nickname, service->service_id); if (service->desc) { @@ -1933,7 +1868,7 @@ rend_services_introduce(void) router_crn_flags_t flags = CRN_NEED_UPTIME; if (get_options()->_AllowInvalid & ALLOW_INVALID_INTRODUCTION) flags |= CRN_ALLOW_INVALID; - router = router_choose_random_node(NULL, intro_routers, + router = router_choose_random_node(intro_routers, options->ExcludeNodes, flags); if (!router) { log_warn(LD_REND, @@ -1945,10 +1880,8 @@ rend_services_introduce(void) smartlist_add(intro_routers, router); intro = tor_malloc_zero(sizeof(rend_intro_point_t)); intro->extend_info = extend_info_from_router(router); - if (service->descriptor_version == 2) { - intro->intro_key = crypto_new_pk_env(); - tor_assert(!crypto_pk_generate_key(intro->intro_key)); - } + intro->intro_key = crypto_new_pk_env(); + tor_assert(!crypto_pk_generate_key(intro->intro_key)); smartlist_add(service->intro_nodes, intro); log_info(LD_REND, "Picked router %s as an intro point for %s.", router->nickname, service->service_id); @@ -2041,8 +1974,7 @@ rend_consider_descriptor_republication(void) for (i=0; i < smartlist_len(rend_service_list); ++i) { service = smartlist_get(rend_service_list, i); - if (service->descriptor_version && service->desc && - !service->desc->all_uploads_performed) { + if (service->desc && !service->desc->all_uploads_performed) { /* If we failed in uploading a descriptor last time, try again *without* * updating the descriptor's contents. */ upload_service_descriptor(service); @@ -2068,10 +2000,9 @@ rend_service_dump_stats(int severity) service->directory); for (j=0; j < smartlist_len(service->intro_nodes); ++j) { intro = smartlist_get(service->intro_nodes, j); - safe_name = safe_str(intro->extend_info->nickname); + safe_name = safe_str_client(intro->extend_info->nickname); - circ = find_intro_circuit(intro, service->pk_digest, - service->descriptor_version); + circ = find_intro_circuit(intro, service->pk_digest); if (!circ) { log(severity, LD_GENERAL, " Intro point %d at %s: no circuit", j, safe_name); @@ -2102,9 +2033,8 @@ rend_service_set_connection_addr_port(edge_connection_t *conn, log_debug(LD_REND,"beginning to hunt for addr/port"); base32_encode(serviceid, REND_SERVICE_ID_LEN_BASE32+1, circ->rend_data->rend_pk_digest, REND_SERVICE_ID_LEN); - service = rend_service_get_by_pk_digest_and_version( - circ->rend_data->rend_pk_digest, - circ->rend_data->rend_desc_version); + service = rend_service_get_by_pk_digest( + circ->rend_data->rend_pk_digest); if (!service) { log_warn(LD_REND, "Couldn't find any service associated with pk %s on " "rendezvous circuit %d; closing.", diff --git a/src/or/rendservice.h b/src/or/rendservice.h new file mode 100644 index 0000000000..70389afe9a --- /dev/null +++ b/src/or/rendservice.h @@ -0,0 +1,37 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file rendservice.h + * \brief Header file for rendservice.c. + **/ + +#ifndef _TOR_RENDSERVICE_H +#define _TOR_RENDSERVICE_H + +int num_rend_services(void); +int rend_config_services(or_options_t *options, int validate_only); +int rend_service_load_keys(void); +void rend_services_introduce(void); +void rend_consider_services_upload(time_t now); +void rend_hsdir_routers_changed(void); +void rend_consider_descriptor_republication(void); + +void rend_service_intro_has_opened(origin_circuit_t *circuit); +int rend_service_intro_established(origin_circuit_t *circuit, + const uint8_t *request, + size_t request_len); +void rend_service_rendezvous_has_opened(origin_circuit_t *circuit); +int rend_service_introduce(origin_circuit_t *circuit, const uint8_t *request, + size_t request_len); +void rend_service_relaunch_rendezvous(origin_circuit_t *oldcirc); +int rend_service_set_connection_addr_port(edge_connection_t *conn, + origin_circuit_t *circ); +void rend_service_dump_stats(int severity); +void rend_service_free_all(void); + +#endif + diff --git a/src/or/rephist.c b/src/or/rephist.c index a3d8bcc986..8cddd2b5eb 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -6,15 +6,22 @@ * \file rephist.c * \brief Basic history and "reputation" functionality to remember * which servers have worked in the past, how much bandwidth we've - * been using, which ports we tend to want, and so on. + * been using, which ports we tend to want, and so on; further, + * exit port statistics and cell statistics. **/ #include "or.h" +#include "circuitlist.h" +#include "circuituse.h" +#include "config.h" +#include "networkstatus.h" +#include "rephist.h" +#include "router.h" +#include "routerlist.h" #include "ht.h" static void bw_arrays_init(void); static void predicted_ports_init(void); -static void hs_usage_init(void); /** Total number of bytes currently allocated in fields used by rephist.c. */ uint64_t rephist_total_alloc=0; @@ -67,6 +74,13 @@ typedef struct or_history_t { /** If nonzero, we have been unable to connect since this time. */ time_t down_since; + /** The address at which we most recently connected to this OR + * successfully. */ + tor_addr_t last_reached_addr; + + /** The port at which we most recently connected to this OR successfully */ + uint16_t last_reached_port; + /* === For MTBF tracking: */ /** Weighted sum total of all times that this router has been online. */ @@ -113,6 +127,7 @@ get_or_history(const char* id) rephist_total_num++; hist->link_history_map = digestmap_new(); hist->since = hist->changed = time(NULL); + tor_addr_make_unspec(&hist->last_reached_addr); digestmap_set(history_map, id, hist); } return hist; @@ -185,7 +200,6 @@ rep_hist_init(void) history_map = digestmap_new(); bw_arrays_init(); predicted_ports_init(); - hs_usage_init(); } /** Helper: note that we are no longer connected to the router with history @@ -284,13 +298,20 @@ rep_hist_note_connection_died(const char* id, time_t when) /** We have just decided that this router with identity digest <b>id</b> is * reachable, meaning we will give it a "Running" flag for the next while. */ void -rep_hist_note_router_reachable(const char *id, time_t when) +rep_hist_note_router_reachable(const char *id, const tor_addr_t *at_addr, + const uint16_t at_port, time_t when) { or_history_t *hist = get_or_history(id); int was_in_run = 1; char tbuf[ISO_TIME_LEN+1]; + int addr_changed, port_changed; tor_assert(hist); + tor_assert((!at_addr && !at_port) || (at_addr && at_port)); + + addr_changed = at_addr && + tor_addr_compare(at_addr, &hist->last_reached_addr, CMP_EXACT) != 0; + port_changed = at_port && at_port != hist->last_reached_port; if (!started_tracking_stability) started_tracking_stability = time(NULL); @@ -310,6 +331,27 @@ rep_hist_note_router_reachable(const char *id, time_t when) down_length = when - hist->start_of_downtime; hist->total_weighted_time += down_length; hist->start_of_downtime = 0; + } else if (addr_changed || port_changed) { + /* If we're reachable, but the address changed, treat this as some + * downtime. */ + int penalty = get_options()->TestingTorNetwork ? 240 : 3600; + networkstatus_t *ns; + + if ((ns = networkstatus_get_latest_consensus())) { + int fresh_interval = (int)(ns->fresh_until - ns->valid_after); + int live_interval = (int)(ns->valid_until - ns->valid_after); + /* on average, a descriptor addr change takes .5 intervals to make it + * into a consensus, and half a liveness period to make it to + * clients. */ + penalty = (int)(fresh_interval + live_interval) / 2; + } + format_local_iso_time(tbuf, hist->start_of_run); + log_info(LD_HIST,"Router %s still seems Running, but its address appears " + "to have changed since the last time it was reachable. I'm " + "going to treat it as having been down for %d seconds", + hex_str(id, DIGEST_LEN), penalty); + rep_hist_note_router_unreachable(id, when-penalty); + rep_hist_note_router_reachable(id, NULL, 0, when); } else { format_local_iso_time(tbuf, hist->start_of_run); if (was_in_run) @@ -319,6 +361,10 @@ rep_hist_note_router_reachable(const char *id, time_t when) log_info(LD_HIST,"Router %s is now Running; it was previously untracked", hex_str(id, DIGEST_LEN)); } + if (at_addr) + tor_addr_copy(&hist->last_reached_addr, at_addr); + if (at_port) + hist->last_reached_port = at_port; } /** We have just decided that this router is unreachable, meaning @@ -339,12 +385,20 @@ rep_hist_note_router_unreachable(const char *id, time_t when) long run_length = when - hist->start_of_run; format_local_iso_time(tbuf, hist->start_of_run); - hist->weighted_run_length += run_length; hist->total_run_weights += 1.0; hist->start_of_run = 0; - hist->weighted_uptime += run_length; - hist->total_weighted_time += run_length; + if (run_length < 0) { + unsigned long penalty = -run_length; +#define SUBTRACT_CLAMPED(var, penalty) \ + do { (var) = (var) < (penalty) ? 0 : (var) - (penalty); } while (0) + SUBTRACT_CLAMPED(hist->weighted_run_length, penalty); + SUBTRACT_CLAMPED(hist->weighted_uptime, penalty); + } else { + hist->weighted_run_length += run_length; + hist->weighted_uptime += run_length; + hist->total_weighted_time += run_length; + } was_running = 1; log_info(LD_HIST, "Router %s is now non-Running: it had previously been " "Running since %s. Its total weighted uptime is %lu/%lu.", @@ -417,7 +471,7 @@ rep_hist_downrate_old_runs(time_t now) static double get_stability(or_history_t *hist, time_t when) { - unsigned long total = hist->weighted_run_length; + long total = hist->weighted_run_length; double total_weights = hist->total_run_weights; if (hist->start_of_run) { @@ -453,8 +507,8 @@ get_total_weighted_time(or_history_t *hist, time_t when) static double get_weighted_fractional_uptime(or_history_t *hist, time_t when) { - unsigned long total = hist->total_weighted_time; - unsigned long up = hist->weighted_uptime; + long total = hist->total_weighted_time; + long up = hist->weighted_uptime; if (hist->start_of_run) { long run_length = (when - hist->start_of_run); @@ -474,6 +528,20 @@ get_weighted_fractional_uptime(or_history_t *hist, time_t when) return ((double) up) / total; } +/** Return how long the router whose identity digest is <b>id</b> has + * been reachable. Return 0 if the router is unknown or currently deemed + * unreachable. */ +long +rep_hist_get_uptime(const char *id, time_t when) +{ + or_history_t *hist = get_or_history(id); + if (!hist) + return 0; + if (!hist->start_of_run || when < hist->start_of_run) + return 0; + return when - hist->start_of_run; +} + /** Return an estimated MTBF for the router whose identity digest is * <b>id</b>. Return 0 if the router is unknown. */ double @@ -519,7 +587,7 @@ rep_hist_get_weighted_time_known(const char *id, time_t when) int rep_hist_have_measured_enough_stability(void) { - /* XXXX021 This doesn't do so well when we change our opinion + /* XXXX022 This doesn't do so well when we change our opinion * as to whether we're tracking router stability. */ return started_tracking_stability < time(NULL) - 4*60*60; } @@ -795,12 +863,12 @@ rep_hist_record_mtbf_data(time_t now, int missing_means_down) static char * rep_hist_format_router_status(or_history_t *hist, time_t now) { - char buf[1024]; char sor_buf[ISO_TIME_LEN+1]; char sod_buf[ISO_TIME_LEN+1]; double wfu; double mtbf; int up = 0, down = 0; + char *cp = NULL; if (hist->start_of_run) { format_iso_time(sor_buf, hist->start_of_run); @@ -813,7 +881,7 @@ rep_hist_format_router_status(or_history_t *hist, time_t now) wfu = get_weighted_fractional_uptime(hist, now); mtbf = get_stability(hist, now); - tor_snprintf(buf, sizeof(buf), + tor_asprintf(&cp, "%s%s%s" "%s%s%s" "wfu %0.3lf\n" @@ -831,8 +899,7 @@ rep_hist_format_router_status(or_history_t *hist, time_t now) hist->weighted_run_length, hist->total_run_weights ); - - return tor_strdup(buf); + return cp; } /** The last stability analysis document that we created, or NULL if we never @@ -925,10 +992,10 @@ find_next_with(smartlist_t *sl, int i, const char *prefix) return -1; } -/** How many bad times has parse_possibly_bad_iso_time parsed? */ +/** How many bad times has parse_possibly_bad_iso_time() parsed? */ static int n_bogus_times = 0; /** Parse the ISO-formatted time in <b>s</b> into *<b>time_out</b>, but - * rounds any pre-1970 date to Jan 1, 1970. */ + * round any pre-1970 date to Jan 1, 1970. */ static int parse_possibly_bad_iso_time(const char *s, time_t *time_out) { @@ -1161,6 +1228,8 @@ rep_hist_load_mtbf_data(time_t now) * totals? */ #define NUM_SECS_ROLLING_MEASURE 10 /** How large are the intervals for which we track and report bandwidth use? */ +/* XXXX Watch out! Before Tor 0.2.2.21-alpha, using any other value here would + * generate an unparseable state file. */ #define NUM_SECS_BW_SUM_INTERVAL (15*60) /** How far in the past do we remember and publish bandwidth use? */ #define NUM_SECS_BW_SUM_IS_VALID (24*60*60) @@ -1219,7 +1288,7 @@ commit_max(bw_array_t *b) b->total_in_period = 0; } -/** Shift the current observation time of 'b' forward by one second. */ +/** Shift the current observation time of <b>b</b> forward by one second. */ static INLINE void advance_obs(bw_array_t *b) { @@ -1249,14 +1318,18 @@ advance_obs(bw_array_t *b) static INLINE void add_obs(bw_array_t *b, time_t when, uint64_t n) { - /* Don't record data in the past. */ - if (when<b->cur_obs_time) - return; + if (when < b->cur_obs_time) + return; /* Don't record data in the past. */ + /* If we're currently adding observations for an earlier second than * 'when', advance b->cur_obs_time and b->cur_obs_idx by an - * appropriate number of seconds, and do all the other housekeeping */ - while (when>b->cur_obs_time) + * appropriate number of seconds, and do all the other housekeeping. */ + while (when > b->cur_obs_time) { + /* Doing this one second at a time is potentially inefficient, if we start + with a state file that is very old. Fortunately, it doesn't seem to + show up in profiles, so we can just ignore it for now. */ advance_obs(b); + } b->obs[b->cur_obs_idx] += n; b->total_in_period += n; @@ -1280,16 +1353,29 @@ bw_array_new(void) static bw_array_t *read_array = NULL; /** Recent history of bandwidth observations for write operations. */ static bw_array_t *write_array = NULL; - -/** Set up read_array and write_array. */ +/** Recent history of bandwidth observations for read operations for the + directory protocol. */ +static bw_array_t *dir_read_array = NULL; +/** Recent history of bandwidth observations for write operations for the + directory protocol. */ +static bw_array_t *dir_write_array = NULL; + +/** Set up [dir-]read_array and [dir-]write_array, freeing them if they + * already exist. */ static void bw_arrays_init(void) { + tor_free(read_array); + tor_free(write_array); + tor_free(dir_read_array); + tor_free(dir_write_array); read_array = bw_array_new(); write_array = bw_array_new(); + dir_read_array = bw_array_new(); + dir_write_array = bw_array_new(); } -/** We read <b>num_bytes</b> more bytes in second <b>when</b>. +/** Remember that we read <b>num_bytes</b> bytes in second <b>when</b>. * * Add num_bytes to the current running total for <b>when</b>. * @@ -1310,7 +1396,7 @@ rep_hist_note_bytes_written(size_t num_bytes, time_t when) add_obs(write_array, when, num_bytes); } -/** We wrote <b>num_bytes</b> more bytes in second <b>when</b>. +/** Remember that we wrote <b>num_bytes</b> bytes in second <b>when</b>. * (like rep_hist_note_bytes_written() above) */ void @@ -1320,6 +1406,24 @@ rep_hist_note_bytes_read(size_t num_bytes, time_t when) add_obs(read_array, when, num_bytes); } +/** Remember that we wrote <b>num_bytes</b> directory bytes in second + * <b>when</b>. (like rep_hist_note_bytes_written() above) + */ +void +rep_hist_note_dir_bytes_written(size_t num_bytes, time_t when) +{ + add_obs(dir_write_array, when, num_bytes); +} + +/** Remember that we read <b>num_bytes</b> directory bytes in second + * <b>when</b>. (like rep_hist_note_bytes_written() above) + */ +void +rep_hist_note_dir_bytes_read(size_t num_bytes, time_t when) +{ + add_obs(dir_read_array, when, num_bytes); +} + /** Helper: Return the largest value in b->maxima. (This is equal to the * most bandwidth used in any NUM_SECS_ROLLING_MEASURE period for the last * NUM_SECS_BW_SUM_IS_VALID seconds.) @@ -1355,14 +1459,14 @@ rep_hist_bandwidth_assess(void) return (int)(U64_TO_DBL(r)/NUM_SECS_ROLLING_MEASURE); } -/** Print the bandwidth history of b (either read_array or write_array) - * into the buffer pointed to by buf. The format is simply comma - * separated numbers, from oldest to newest. +/** Print the bandwidth history of b (either [dir-]read_array or + * [dir-]write_array) into the buffer pointed to by buf. The format is + * simply comma separated numbers, from oldest to newest. * * It returns the number of bytes written. */ static size_t -rep_hist_fill_bandwidth_history(char *buf, size_t len, bw_array_t *b) +rep_hist_fill_bandwidth_history(char *buf, size_t len, const bw_array_t *b) { char *cp = buf; int i, n; @@ -1407,151 +1511,254 @@ rep_hist_fill_bandwidth_history(char *buf, size_t len, bw_array_t *b) } /** Allocate and return lines for representing this server's bandwidth - * history in its descriptor. + * history in its descriptor. We publish these lines in our extra-info + * descriptor. */ char * -rep_hist_get_bandwidth_lines(int for_extrainfo) +rep_hist_get_bandwidth_lines(void) { char *buf, *cp; char t[ISO_TIME_LEN+1]; int r; - bw_array_t *b; + bw_array_t *b = NULL; + const char *desc = NULL; size_t len; - /* opt (read|write)-history yyyy-mm-dd HH:MM:SS (n s) n,n,n,n,n... */ - len = (60+20*NUM_TOTALS)*2; + /* opt [dirreq-](read|write)-history yyyy-mm-dd HH:MM:SS (n s) n,n,n... */ +/* The n,n,n part above. Largest representation of a uint64_t is 20 chars + * long, plus the comma. */ +#define MAX_HIST_VALUE_LEN 21*NUM_TOTALS + len = (67+MAX_HIST_VALUE_LEN)*4; buf = tor_malloc_zero(len); cp = buf; - for (r=0;r<2;++r) { - b = r?read_array:write_array; + for (r=0;r<4;++r) { + char tmp[MAX_HIST_VALUE_LEN]; + size_t slen; + switch (r) { + case 0: + b = write_array; + desc = "write-history"; + break; + case 1: + b = read_array; + desc = "read-history"; + break; + case 2: + b = dir_write_array; + desc = "dirreq-write-history"; + break; + case 3: + b = dir_read_array; + desc = "dirreq-read-history"; + break; + } tor_assert(b); + slen = rep_hist_fill_bandwidth_history(tmp, MAX_HIST_VALUE_LEN, b); + /* If we don't have anything to write, skip to the next entry. */ + if (slen == 0) + continue; format_iso_time(t, b->next_period-NUM_SECS_BW_SUM_INTERVAL); - tor_snprintf(cp, len-(cp-buf), "%s%s %s (%d s) ", - for_extrainfo ? "" : "opt ", - r ? "read-history" : "write-history", t, - NUM_SECS_BW_SUM_INTERVAL); + tor_snprintf(cp, len-(cp-buf), "%s %s (%d s) ", + desc, t, NUM_SECS_BW_SUM_INTERVAL); cp += strlen(cp); - cp += rep_hist_fill_bandwidth_history(cp, len-(cp-buf), b); + strlcat(cp, tmp, len-(cp-buf)); + cp += slen; strlcat(cp, "\n", len-(cp-buf)); ++cp; } return buf; } -/** Update <b>state</b> with the newest bandwidth history. */ +/** Write a single bw_array_t into the Values, Ends, Interval, and Maximum + * entries of an or_state_t. Done before writing out a new state file. */ +static void +rep_hist_update_bwhist_state_section(or_state_t *state, + const bw_array_t *b, + smartlist_t **s_values, + smartlist_t **s_maxima, + time_t *s_begins, + int *s_interval) +{ + char *cp; + int i,j; + uint64_t maxval; + + if (*s_values) { + SMARTLIST_FOREACH(*s_values, char *, val, tor_free(val)); + smartlist_free(*s_values); + } + if (*s_maxima) { + SMARTLIST_FOREACH(*s_maxima, char *, val, tor_free(val)); + smartlist_free(*s_maxima); + } + if (! server_mode(get_options())) { + /* Clients don't need to store bandwidth history persistently; + * force these values to the defaults. */ + /* FFFF we should pull the default out of config.c's state table, + * so we don't have two defaults. */ + if (*s_begins != 0 || *s_interval != 900) { + time_t now = time(NULL); + time_t save_at = get_options()->AvoidDiskWrites ? now+3600 : now+600; + or_state_mark_dirty(state, save_at); + } + *s_begins = 0; + *s_interval = 900; + *s_values = smartlist_create(); + *s_maxima = smartlist_create(); + return; + } + *s_begins = b->next_period; + *s_interval = NUM_SECS_BW_SUM_INTERVAL; + + *s_values = smartlist_create(); + *s_maxima = smartlist_create(); + /* Set i to first position in circular array */ + i = (b->num_maxes_set <= b->next_max_idx) ? 0 : b->next_max_idx; + for (j=0; j < b->num_maxes_set; ++j,++i) { + if (i >= NUM_TOTALS) + i = 0; + tor_asprintf(&cp, U64_FORMAT, U64_PRINTF_ARG(b->totals[i] & ~0x3ff)); + smartlist_add(*s_values, cp); + maxval = b->maxima[i] / NUM_SECS_ROLLING_MEASURE; + tor_asprintf(&cp, U64_FORMAT, U64_PRINTF_ARG(maxval & ~0x3ff)); + smartlist_add(*s_maxima, cp); + } + tor_asprintf(&cp, U64_FORMAT, U64_PRINTF_ARG(b->total_in_period & ~0x3ff)); + smartlist_add(*s_values, cp); + maxval = b->max_total / NUM_SECS_ROLLING_MEASURE; + tor_asprintf(&cp, U64_FORMAT, U64_PRINTF_ARG(maxval & ~0x3ff)); + smartlist_add(*s_maxima, cp); +} + +/** Update <b>state</b> with the newest bandwidth history. Done before + * writing out a new state file. */ void rep_hist_update_state(or_state_t *state) { - int len, r; - char *buf, *cp; - smartlist_t **s_values; - time_t *s_begins; - int *s_interval; - bw_array_t *b; - - len = 20*NUM_TOTALS+1; - buf = tor_malloc_zero(len); +#define UPDATE(arrname,st) \ + rep_hist_update_bwhist_state_section(state,\ + (arrname),\ + &state->BWHistory ## st ## Values, \ + &state->BWHistory ## st ## Maxima, \ + &state->BWHistory ## st ## Ends, \ + &state->BWHistory ## st ## Interval) - for (r=0;r<2;++r) { - b = r?read_array:write_array; - s_begins = r?&state->BWHistoryReadEnds :&state->BWHistoryWriteEnds; - s_interval= r?&state->BWHistoryReadInterval:&state->BWHistoryWriteInterval; - s_values = r?&state->BWHistoryReadValues :&state->BWHistoryWriteValues; + UPDATE(write_array, Write); + UPDATE(read_array, Read); + UPDATE(dir_write_array, DirWrite); + UPDATE(dir_read_array, DirRead); - if (*s_values) { - SMARTLIST_FOREACH(*s_values, char *, val, tor_free(val)); - smartlist_free(*s_values); - } - if (! server_mode(get_options())) { - /* Clients don't need to store bandwidth history persistently; - * force these values to the defaults. */ - /* FFFF we should pull the default out of config.c's state table, - * so we don't have two defaults. */ - if (*s_begins != 0 || *s_interval != 900) { - time_t now = time(NULL); - time_t save_at = get_options()->AvoidDiskWrites ? now+3600 : now+600; - or_state_mark_dirty(state, save_at); - } - *s_begins = 0; - *s_interval = 900; - *s_values = smartlist_create(); - continue; - } - *s_begins = b->next_period; - *s_interval = NUM_SECS_BW_SUM_INTERVAL; - cp = buf; - cp += rep_hist_fill_bandwidth_history(cp, len, b); - tor_snprintf(cp, len-(cp-buf), cp == buf ? U64_FORMAT : ","U64_FORMAT, - U64_PRINTF_ARG(b->total_in_period)); - *s_values = smartlist_create(); - if (server_mode(get_options())) - smartlist_split_string(*s_values, buf, ",", SPLIT_SKIP_SPACE, 0); - } - tor_free(buf); if (server_mode(get_options())) { - or_state_mark_dirty(get_or_state(), time(NULL)+(2*3600)); + or_state_mark_dirty(state, time(NULL)+(2*3600)); } +#undef UPDATE } -/** Set bandwidth history from our saved state. */ -int -rep_hist_load_state(or_state_t *state, char **err) +/** Load a single bw_array_t from its Values, Ends, Maxima, and Interval + * entries in an or_state_t. Done while reading the state file. */ +static int +rep_hist_load_bwhist_state_section(bw_array_t *b, + const smartlist_t *s_values, + const smartlist_t *s_maxima, + const time_t s_begins, + const int s_interval) { - time_t s_begins, start; time_t now = time(NULL); - uint64_t v; - int r,i,ok; - int all_ok = 1; - int s_interval; - smartlist_t *s_values; - bw_array_t *b; - - /* Assert they already have been malloced */ - tor_assert(read_array && write_array); + int retval = 0; + time_t start; - for (r=0;r<2;++r) { - b = r?read_array:write_array; - s_begins = r?state->BWHistoryReadEnds:state->BWHistoryWriteEnds; - s_interval = r?state->BWHistoryReadInterval:state->BWHistoryWriteInterval; - s_values = r?state->BWHistoryReadValues:state->BWHistoryWriteValues; - if (s_values && s_begins >= now - NUM_SECS_BW_SUM_INTERVAL*NUM_TOTALS) { - start = s_begins - s_interval*(smartlist_len(s_values)); - if (start > now) - continue; - b->cur_obs_time = start; - b->next_period = start + NUM_SECS_BW_SUM_INTERVAL; - SMARTLIST_FOREACH(s_values, char *, cp, { + uint64_t v, mv; + int i,ok,ok_m; + int have_maxima = (smartlist_len(s_values) == smartlist_len(s_maxima)); + + if (s_values && s_begins >= now - NUM_SECS_BW_SUM_INTERVAL*NUM_TOTALS) { + start = s_begins - s_interval*(smartlist_len(s_values)); + if (start > now) + return 0; + b->cur_obs_time = start; + b->next_period = start + NUM_SECS_BW_SUM_INTERVAL; + SMARTLIST_FOREACH_BEGIN(s_values, const char *, cp) { + const char *maxstr = NULL; v = tor_parse_uint64(cp, 10, 0, UINT64_MAX, &ok, NULL); + if (have_maxima) { + maxstr = smartlist_get(s_maxima, cp_sl_idx); + mv = tor_parse_uint64(maxstr, 10, 0, UINT64_MAX, &ok_m, NULL); + mv *= NUM_SECS_ROLLING_MEASURE; + } else { + /* No maxima known; guess average rate to be conservative. */ + mv = (v / s_interval) * NUM_SECS_ROLLING_MEASURE; + } if (!ok) { - all_ok=0; - log_notice(LD_HIST, "Could not parse '%s' into a number.'", cp); + retval = -1; + log_notice(LD_HIST, "Could not parse value '%s' into a number.'",cp); } + if (maxstr && !ok_m) { + retval = -1; + log_notice(LD_HIST, "Could not parse maximum '%s' into a number.'", + maxstr); + } + if (start < now) { - add_obs(b, start, v); - start += NUM_SECS_BW_SUM_INTERVAL; + time_t cur_start = start; + time_t actual_interval_len = s_interval; + uint64_t cur_val = 0; + /* Calculate the average per second. This is the best we can do + * because our state file doesn't have per-second resolution. */ + if (start + s_interval > now) + actual_interval_len = now - start; + cur_val = v / actual_interval_len; + /* This is potentially inefficient, but since we don't do it very + * often it should be ok. */ + while (cur_start < start + actual_interval_len) { + add_obs(b, cur_start, cur_val); + ++cur_start; + } + b->max_total = mv; + /* This will result in some fairly choppy history if s_interval + * is not the same as NUM_SECS_BW_SUM_INTERVAL. XXXX */ + start += actual_interval_len; } - }); - } + } SMARTLIST_FOREACH_END(cp); + } - /* Clean up maxima and observed */ - /* Do we really want to zero this for the purpose of max capacity? */ - for (i=0; i<NUM_SECS_ROLLING_MEASURE; ++i) { - b->obs[i] = 0; - } - b->total_obs = 0; - for (i=0; i<NUM_TOTALS; ++i) { - b->maxima[i] = 0; - } - b->max_total = 0; + /* Clean up maxima and observed */ + for (i=0; i<NUM_SECS_ROLLING_MEASURE; ++i) { + b->obs[i] = 0; } + b->total_obs = 0; + + return retval; +} + +/** Set bandwidth history from the state file we just loaded. */ +int +rep_hist_load_state(or_state_t *state, char **err) +{ + int all_ok = 1; + /* Assert they already have been malloced */ + tor_assert(read_array && write_array); + tor_assert(dir_read_array && dir_write_array); + +#define LOAD(arrname,st) \ + if (rep_hist_load_bwhist_state_section( \ + (arrname), \ + state->BWHistory ## st ## Values, \ + state->BWHistory ## st ## Maxima, \ + state->BWHistory ## st ## Ends, \ + state->BWHistory ## st ## Interval)<0) \ + all_ok = 0 + + LOAD(write_array, Write); + LOAD(read_array, Read); + LOAD(dir_write_array, DirWrite); + LOAD(dir_read_array, DirRead); + +#undef LOAD if (!all_ok) { *err = tor_strdup("Parsing of bandwidth history values failed"); /* and create fresh arrays */ - tor_free(read_array); - tor_free(write_array); - read_array = bw_array_new(); - write_array = bw_array_new(); + bw_arrays_init(); return -1; } return 0; @@ -1571,7 +1778,7 @@ static smartlist_t *predicted_ports_times=NULL; static void add_predicted_port(time_t now, uint16_t port) { - /* XXXX we could just use uintptr_t here, I think. */ + /* XXXX we could just use uintptr_t here, I think. -NM */ uint16_t *tmp_port = tor_malloc(sizeof(uint16_t)); time_t *tmp_time = tor_malloc(sizeof(time_t)); *tmp_port = port; @@ -1715,8 +1922,8 @@ rep_hist_get_predicted_internal(time_t now, int *need_uptime, return 0; /* too long ago */ if (predicted_internal_uptime_time + PREDICTED_CIRCS_RELEVANCE_TIME >= now) *need_uptime = 1; - if (predicted_internal_capacity_time + PREDICTED_CIRCS_RELEVANCE_TIME >= now) - *need_capacity = 1; + // Always predict that we need capacity. + *need_capacity = 1; return 1; } @@ -1845,564 +2052,530 @@ dump_pk_ops(int severity) pk_op_counts.n_rend_server_ops); } -/** Free all storage held by the OR/link history caches, by the - * bandwidth history arrays, or by the port history. */ +/*** Exit port statistics ***/ + +/* Some constants */ +/** To what multiple should byte numbers be rounded up? */ +#define EXIT_STATS_ROUND_UP_BYTES 1024 +/** To what multiple should stream counts be rounded up? */ +#define EXIT_STATS_ROUND_UP_STREAMS 4 +/** Number of TCP ports */ +#define EXIT_STATS_NUM_PORTS 65536 +/** Top n ports that will be included in exit stats. */ +#define EXIT_STATS_TOP_N_PORTS 10 + +/* The following data structures are arrays and no fancy smartlists or maps, + * so that all write operations can be done in constant time. This comes at + * the price of some memory (1.25 MB) and linear complexity when writing + * stats for measuring relays. */ +/** Number of bytes read in current period by exit port */ +static uint64_t *exit_bytes_read = NULL; +/** Number of bytes written in current period by exit port */ +static uint64_t *exit_bytes_written = NULL; +/** Number of streams opened in current period by exit port */ +static uint32_t *exit_streams = NULL; + +/** Start time of exit stats or 0 if we're not collecting exit stats. */ +static time_t start_of_exit_stats_interval; + +/** Initialize exit port stats. */ void -rep_hist_free_all(void) +rep_hist_exit_stats_init(time_t now) { - digestmap_free(history_map, free_or_history); - tor_free(read_array); - tor_free(write_array); - tor_free(last_stability_doc); - built_last_stability_doc_at = 0; - predicted_ports_free(); + start_of_exit_stats_interval = now; + exit_bytes_read = tor_malloc_zero(EXIT_STATS_NUM_PORTS * + sizeof(uint64_t)); + exit_bytes_written = tor_malloc_zero(EXIT_STATS_NUM_PORTS * + sizeof(uint64_t)); + exit_streams = tor_malloc_zero(EXIT_STATS_NUM_PORTS * + sizeof(uint32_t)); } -/****************** hidden service usage statistics ******************/ - -/** How large are the intervals for which we track and report hidden service - * use? */ -#define NUM_SECS_HS_USAGE_SUM_INTERVAL (15*60) -/** How far in the past do we remember and publish hidden service use? */ -#define NUM_SECS_HS_USAGE_SUM_IS_VALID (24*60*60) -/** How many hidden service usage intervals do we remember? (derived) */ -#define NUM_TOTALS_HS_USAGE (NUM_SECS_HS_USAGE_SUM_IS_VALID/ \ - NUM_SECS_HS_USAGE_SUM_INTERVAL) - -/** List element containing a service id and the count. */ -typedef struct hs_usage_list_elem_t { - /** Service id of this elem. */ - char service_id[REND_SERVICE_ID_LEN_BASE32+1]; - /** Number of occurrences for the given service id. */ - uint32_t count; - /* Pointer to next list elem */ - struct hs_usage_list_elem_t *next; -} hs_usage_list_elem_t; - -/** Ordered list that stores service ids and the number of observations. It is - * ordered by the number of occurrences in descending order. Its purpose is to - * calculate the frequency distribution when the period is over. */ -typedef struct hs_usage_list_t { - /* Pointer to the first element in the list. */ - hs_usage_list_elem_t *start; - /* Number of total occurrences for all list elements. */ - uint32_t total_count; - /* Number of service ids, i.e. number of list elements. */ - uint32_t total_service_ids; -} hs_usage_list_t; - -/** Tracks service-related observations in the current period and their - * history. */ -typedef struct hs_usage_service_related_observation_t { - /** Ordered list that stores service ids and the number of observations in - * the current period. It is ordered by the number of occurrences in - * descending order. Its purpose is to calculate the frequency distribution - * when the period is over. */ - hs_usage_list_t *list; - /** Circular arrays that store the history of observations. totals stores all - * observations, twenty (ten, five) the number of observations related to a - * service id being accounted for the top 20 (10, 5) percent of all - * observations. */ - uint32_t totals[NUM_TOTALS_HS_USAGE]; - uint32_t five[NUM_TOTALS_HS_USAGE]; - uint32_t ten[NUM_TOTALS_HS_USAGE]; - uint32_t twenty[NUM_TOTALS_HS_USAGE]; -} hs_usage_service_related_observation_t; - -/** Tracks the history of general period-related observations, i.e. those that - * cannot be related to a specific service id. */ -typedef struct hs_usage_general_period_related_observations_t { - /** Circular array that stores the history of observations. */ - uint32_t totals[NUM_TOTALS_HS_USAGE]; -} hs_usage_general_period_related_observations_t; - -/** Keeps information about the current observation period and its relation to - * the histories of observations. */ -typedef struct hs_usage_current_observation_period_t { - /** Where do we write the next history entry? */ - int next_idx; - /** How many values in history have been set ever? (upper bound!) */ - int num_set; - /** When did this period begin? */ - time_t start_of_current_period; - /** When does the next period begin? */ - time_t start_of_next_period; -} hs_usage_current_observation_period_t; - -/** Usage statistics for the current observation period. */ -static hs_usage_current_observation_period_t *current_period = NULL; - -/** Total number of descriptor publish requests in the current observation - * period. */ -static hs_usage_service_related_observation_t *publish_total = NULL; - -/** Number of descriptor publish requests for services that have not been - * seen before in the current observation period. */ -static hs_usage_service_related_observation_t *publish_novel = NULL; - -/** Total number of descriptor fetch requests in the current observation - * period. */ -static hs_usage_service_related_observation_t *fetch_total = NULL; - -/** Number of successful descriptor fetch requests in the current - * observation period. */ -static hs_usage_service_related_observation_t *fetch_successful = NULL; - -/** Number of descriptors stored in the current observation period. */ -static hs_usage_general_period_related_observations_t *descs = NULL; - -/** Creates an empty ordered list element. */ -static hs_usage_list_elem_t * -hs_usage_list_elem_new(void) +/** Reset counters for exit port statistics. */ +void +rep_hist_reset_exit_stats(time_t now) { - hs_usage_list_elem_t *e; - e = tor_malloc_zero(sizeof(hs_usage_list_elem_t)); - rephist_total_alloc += sizeof(hs_usage_list_elem_t); - e->count = 1; - e->next = NULL; - return e; + start_of_exit_stats_interval = now; + memset(exit_bytes_read, 0, EXIT_STATS_NUM_PORTS * sizeof(uint64_t)); + memset(exit_bytes_written, 0, EXIT_STATS_NUM_PORTS * sizeof(uint64_t)); + memset(exit_streams, 0, EXIT_STATS_NUM_PORTS * sizeof(uint32_t)); } -/** Creates an empty ordered list. */ -static hs_usage_list_t * -hs_usage_list_new(void) +/** Stop collecting exit port stats in a way that we can re-start doing + * so in rep_hist_exit_stats_init(). */ +void +rep_hist_exit_stats_term(void) { - hs_usage_list_t *l; - l = tor_malloc_zero(sizeof(hs_usage_list_t)); - rephist_total_alloc += sizeof(hs_usage_list_t); - l->start = NULL; - l->total_count = 0; - l->total_service_ids = 0; - return l; + start_of_exit_stats_interval = 0; + tor_free(exit_bytes_read); + tor_free(exit_bytes_written); + tor_free(exit_streams); } -/** Creates an empty structure for storing service-related observations. */ -static hs_usage_service_related_observation_t * -hs_usage_service_related_observation_new(void) +/** Helper for qsort: compare two ints. */ +static int +_compare_int(const void *x, const void *y) { - hs_usage_service_related_observation_t *h; - h = tor_malloc_zero(sizeof(hs_usage_service_related_observation_t)); - rephist_total_alloc += sizeof(hs_usage_service_related_observation_t); - h->list = hs_usage_list_new(); - return h; + return (*(int*)x - *(int*)y); } -/** Creates an empty structure for storing general period-related - * observations. */ -static hs_usage_general_period_related_observations_t * -hs_usage_general_period_related_observations_new(void) -{ - hs_usage_general_period_related_observations_t *p; - p = tor_malloc_zero(sizeof(hs_usage_general_period_related_observations_t)); - rephist_total_alloc+= sizeof(hs_usage_general_period_related_observations_t); - return p; -} +/** Return a newly allocated string containing the exit port statistics + * until <b>now</b>, or NULL if we're not collecting exit stats. */ +char * +rep_hist_format_exit_stats(time_t now) +{ + int i, j, top_elements = 0, cur_min_idx = 0, cur_port; + uint64_t top_bytes[EXIT_STATS_TOP_N_PORTS]; + int top_ports[EXIT_STATS_TOP_N_PORTS]; + uint64_t cur_bytes = 0, other_read = 0, other_written = 0, + total_read = 0, total_written = 0; + uint32_t total_streams = 0, other_streams = 0; + char *buf; + smartlist_t *written_strings, *read_strings, *streams_strings; + char *written_string, *read_string, *streams_string; + char t[ISO_TIME_LEN+1]; + char *result; -/** Creates an empty structure for storing period-specific information. */ -static hs_usage_current_observation_period_t * -hs_usage_current_observation_period_new(void) -{ - hs_usage_current_observation_period_t *c; - time_t now; - c = tor_malloc_zero(sizeof(hs_usage_current_observation_period_t)); - rephist_total_alloc += sizeof(hs_usage_current_observation_period_t); - now = time(NULL); - c->start_of_current_period = now; - c->start_of_next_period = now + NUM_SECS_HS_USAGE_SUM_INTERVAL; - return c; -} + if (!start_of_exit_stats_interval) + return NULL; /* Not initialized. */ -/** Initializes the structures for collecting hidden service usage data. */ -static void -hs_usage_init(void) -{ - current_period = hs_usage_current_observation_period_new(); - publish_total = hs_usage_service_related_observation_new(); - publish_novel = hs_usage_service_related_observation_new(); - fetch_total = hs_usage_service_related_observation_new(); - fetch_successful = hs_usage_service_related_observation_new(); - descs = hs_usage_general_period_related_observations_new(); -} + /* Go through all ports to find the n ports that saw most written and + * read bytes. + * + * Invariant: at the end of the loop for iteration i, + * total_read is the sum of all exit_bytes_read[0..i] + * total_written is the sum of all exit_bytes_written[0..i] + * total_stream is the sum of all exit_streams[0..i] + * + * top_elements = MAX(EXIT_STATS_TOP_N_PORTS, + * #{j | 0 <= j <= i && volume(i) > 0}) + * + * For all 0 <= j < top_elements, + * top_bytes[j] > 0 + * 0 <= top_ports[j] <= 65535 + * top_bytes[j] = volume(top_ports[j]) + * + * There is no j in 0..i and k in 0..top_elements such that: + * volume(j) > top_bytes[k] AND j is not in top_ports[0..top_elements] + * + * There is no j!=cur_min_idx in 0..top_elements such that: + * top_bytes[j] < top_bytes[cur_min_idx] + * + * where volume(x) == exit_bytes_read[x]+exit_bytes_written[x] + * + * Worst case: O(EXIT_STATS_NUM_PORTS * EXIT_STATS_TOP_N_PORTS) + */ + for (i = 1; i < EXIT_STATS_NUM_PORTS; i++) { + total_read += exit_bytes_read[i]; + total_written += exit_bytes_written[i]; + total_streams += exit_streams[i]; + cur_bytes = exit_bytes_read[i] + exit_bytes_written[i]; + if (cur_bytes == 0) { + continue; + } + if (top_elements < EXIT_STATS_TOP_N_PORTS) { + top_bytes[top_elements] = cur_bytes; + top_ports[top_elements++] = i; + } else if (cur_bytes > top_bytes[cur_min_idx]) { + top_bytes[cur_min_idx] = cur_bytes; + top_ports[cur_min_idx] = i; + } else { + continue; + } + cur_min_idx = 0; + for (j = 1; j < top_elements; j++) { + if (top_bytes[j] < top_bytes[cur_min_idx]) { + cur_min_idx = j; + } + } + } -/** Clears the given ordered list by resetting its attributes and releasing - * the memory allocated by its elements. */ -static void -hs_usage_list_clear(hs_usage_list_t *lst) -{ - /* walk through elements and free memory */ - hs_usage_list_elem_t *current = lst->start; - hs_usage_list_elem_t *tmp; - while (current != NULL) { - tmp = current->next; - rephist_total_alloc -= sizeof(hs_usage_list_elem_t); - tor_free(current); - current = tmp; + /* Add observations of top ports to smartlists. */ + written_strings = smartlist_create(); + read_strings = smartlist_create(); + streams_strings = smartlist_create(); + other_read = total_read; + other_written = total_written; + other_streams = total_streams; + /* Sort the ports; this puts them out of sync with top_bytes, but we + * won't be using top_bytes again anyway */ + qsort(top_ports, top_elements, sizeof(int), _compare_int); + for (j = 0; j < top_elements; j++) { + cur_port = top_ports[j]; + if (exit_bytes_written[cur_port] > 0) { + uint64_t num = round_uint64_to_next_multiple_of( + exit_bytes_written[cur_port], + EXIT_STATS_ROUND_UP_BYTES); + num /= 1024; + buf = NULL; + tor_asprintf(&buf, "%d="U64_FORMAT, cur_port, U64_PRINTF_ARG(num)); + smartlist_add(written_strings, buf); + other_written -= exit_bytes_written[cur_port]; + } + if (exit_bytes_read[cur_port] > 0) { + uint64_t num = round_uint64_to_next_multiple_of( + exit_bytes_read[cur_port], + EXIT_STATS_ROUND_UP_BYTES); + num /= 1024; + buf = NULL; + tor_asprintf(&buf, "%d="U64_FORMAT, cur_port, U64_PRINTF_ARG(num)); + smartlist_add(read_strings, buf); + other_read -= exit_bytes_read[cur_port]; + } + if (exit_streams[cur_port] > 0) { + uint32_t num = round_uint32_to_next_multiple_of( + exit_streams[cur_port], + EXIT_STATS_ROUND_UP_STREAMS); + buf = NULL; + tor_asprintf(&buf, "%d=%u", cur_port, num); + smartlist_add(streams_strings, buf); + other_streams -= exit_streams[cur_port]; + } } - /* reset attributes */ - lst->start = NULL; - lst->total_count = 0; - lst->total_service_ids = 0; - return; -} -/** Frees the memory used by the given list. */ -static void -hs_usage_list_free(hs_usage_list_t *lst) -{ - if (!lst) - return; - hs_usage_list_clear(lst); - rephist_total_alloc -= sizeof(hs_usage_list_t); - tor_free(lst); + /* Add observations of other ports in a single element. */ + other_written = round_uint64_to_next_multiple_of(other_written, + EXIT_STATS_ROUND_UP_BYTES); + other_written /= 1024; + buf = NULL; + tor_asprintf(&buf, "other="U64_FORMAT, U64_PRINTF_ARG(other_written)); + smartlist_add(written_strings, buf); + other_read = round_uint64_to_next_multiple_of(other_read, + EXIT_STATS_ROUND_UP_BYTES); + other_read /= 1024; + buf = NULL; + tor_asprintf(&buf, "other="U64_FORMAT, U64_PRINTF_ARG(other_read)); + smartlist_add(read_strings, buf); + other_streams = round_uint32_to_next_multiple_of(other_streams, + EXIT_STATS_ROUND_UP_STREAMS); + buf = NULL; + tor_asprintf(&buf, "other=%u", other_streams); + smartlist_add(streams_strings, buf); + + /* Join all observations in single strings. */ + written_string = smartlist_join_strings(written_strings, ",", 0, NULL); + read_string = smartlist_join_strings(read_strings, ",", 0, NULL); + streams_string = smartlist_join_strings(streams_strings, ",", 0, NULL); + SMARTLIST_FOREACH(written_strings, char *, cp, tor_free(cp)); + SMARTLIST_FOREACH(read_strings, char *, cp, tor_free(cp)); + SMARTLIST_FOREACH(streams_strings, char *, cp, tor_free(cp)); + smartlist_free(written_strings); + smartlist_free(read_strings); + smartlist_free(streams_strings); + + /* Put everything together. */ + format_iso_time(t, now); + tor_asprintf(&result, "exit-stats-end %s (%d s)\n" + "exit-kibibytes-written %s\n" + "exit-kibibytes-read %s\n" + "exit-streams-opened %s\n", + t, (unsigned) (now - start_of_exit_stats_interval), + written_string, + read_string, + streams_string); + tor_free(written_string); + tor_free(read_string); + tor_free(streams_string); + return result; } -/** Frees the memory used by the given service-related observations. */ -static void -hs_usage_service_related_observation_free( - hs_usage_service_related_observation_t *s) +/** If 24 hours have passed since the beginning of the current exit port + * stats period, write exit stats to $DATADIR/stats/exit-stats (possibly + * overwriting an existing file) and reset counters. Return when we would + * next want to write exit stats or 0 if we never want to write. */ +time_t +rep_hist_exit_stats_write(time_t now) { - if (!s) - return; - hs_usage_list_free(s->list); - rephist_total_alloc -= sizeof(hs_usage_service_related_observation_t); - tor_free(s); -} + char *statsdir = NULL, *filename = NULL, *str = NULL; -/** Frees the memory used by the given period-specific observations. */ -static void -hs_usage_general_period_related_observations_free( - hs_usage_general_period_related_observations_t *s) -{ - rephist_total_alloc-=sizeof(hs_usage_general_period_related_observations_t); - tor_free(s); -} + if (!start_of_exit_stats_interval) + return 0; /* Not initialized. */ + if (start_of_exit_stats_interval + WRITE_STATS_INTERVAL > now) + goto done; /* Not ready to write. */ -/** Frees the memory used by period-specific information. */ -static void -hs_usage_current_observation_period_free( - hs_usage_current_observation_period_t *s) -{ - rephist_total_alloc -= sizeof(hs_usage_current_observation_period_t); - tor_free(s); -} + log_info(LD_HIST, "Writing exit port statistics to disk."); -/** Frees all memory that was used for collecting hidden service usage data. */ -void -hs_usage_free_all(void) -{ - hs_usage_general_period_related_observations_free(descs); - descs = NULL; - hs_usage_service_related_observation_free(fetch_successful); - hs_usage_service_related_observation_free(fetch_total); - hs_usage_service_related_observation_free(publish_novel); - hs_usage_service_related_observation_free(publish_total); - fetch_successful = fetch_total = publish_novel = publish_total = NULL; - hs_usage_current_observation_period_free(current_period); - current_period = NULL; -} + /* Generate history string. */ + str = rep_hist_format_exit_stats(now); -/** Inserts a new occurrence for the given service id to the given ordered - * list. */ -static void -hs_usage_insert_value(hs_usage_list_t *lst, const char *service_id) -{ - /* search if there is already an elem with same service_id in list */ - hs_usage_list_elem_t *current = lst->start; - hs_usage_list_elem_t *previous = NULL; - while (current != NULL && strcasecmp(current->service_id,service_id)) { - previous = current; - current = current->next; - } - /* found an element with same service_id? */ - if (current == NULL) { - /* not found! append to end (which could also be the end of a zero-length - * list), don't need to sort (1 is smallest value). */ - /* create elem */ - hs_usage_list_elem_t *e = hs_usage_list_elem_new(); - /* update list attributes (one new elem, one new occurrence) */ - lst->total_count++; - lst->total_service_ids++; - /* copy service id to elem */ - strlcpy(e->service_id,service_id,sizeof(e->service_id)); - /* let either l->start or previously last elem point to new elem */ - if (lst->start == NULL) { - /* this is the first elem */ - lst->start = e; - } else { - /* there were elems in the list before */ - previous->next = e; - } - } else { - /* found! add occurrence to elem and consider resorting */ - /* update list attributes (no new elem, but one new occurrence) */ - lst->total_count++; - /* add occurrence to elem */ - current->count++; - /* is it another than the first list elem? and has previous elem fewer - * count than current? then we need to resort */ - if (previous != NULL && previous->count < current->count) { - /* yes! we need to resort */ - /* remove current elem first */ - previous->next = current->next; - /* can we prepend elem to all other elements? */ - if (lst->start->count <= current->count) { - /* yes! prepend elem */ - current->next = lst->start; - lst->start = current; - } else { - /* no! walk through list a second time and insert at correct place */ - hs_usage_list_elem_t *insert_current = lst->start->next; - hs_usage_list_elem_t *insert_previous = lst->start; - while (insert_current != NULL && - insert_current->count > current->count) { - insert_previous = insert_current; - insert_current = insert_current->next; - } - /* insert here */ - current->next = insert_current; - insert_previous->next = current; - } - } - } -} + /* Reset counters. */ + rep_hist_reset_exit_stats(now); -/** Writes the current service-related observations to the history array and - * clears the observations of the current period. */ -static void -hs_usage_write_service_related_observations_to_history( - hs_usage_current_observation_period_t *p, - hs_usage_service_related_observation_t *h) -{ - /* walk through the first 20 % of list elements and calculate frequency - * distributions */ - /* maximum indices for the three frequencies */ - int five_percent_idx = h->list->total_service_ids/20; - int ten_percent_idx = h->list->total_service_ids/10; - int twenty_percent_idx = h->list->total_service_ids/5; - /* temp values */ - uint32_t five_percent = 0; - uint32_t ten_percent = 0; - uint32_t twenty_percent = 0; - /* walk through list */ - hs_usage_list_elem_t *current = h->list->start; - int i=0; - while (current != NULL && i <= twenty_percent_idx) { - twenty_percent += current->count; - if (i <= ten_percent_idx) - ten_percent += current->count; - if (i <= five_percent_idx) - five_percent += current->count; - current = current->next; - i++; - } - /* copy frequencies */ - h->twenty[p->next_idx] = twenty_percent; - h->ten[p->next_idx] = ten_percent; - h->five[p->next_idx] = five_percent; - /* copy total number of observations */ - h->totals[p->next_idx] = h->list->total_count; - /* free memory of old list */ - hs_usage_list_clear(h->list); -} - -/** Advances to next observation period. */ -static void -hs_usage_advance_current_observation_period(void) -{ - /* aggregate observations to history, including frequency distribution - * arrays */ - hs_usage_write_service_related_observations_to_history( - current_period, publish_total); - hs_usage_write_service_related_observations_to_history( - current_period, publish_novel); - hs_usage_write_service_related_observations_to_history( - current_period, fetch_total); - hs_usage_write_service_related_observations_to_history( - current_period, fetch_successful); - /* write current number of descriptors to descs history */ - descs->totals[current_period->next_idx] = rend_cache_size(); - /* advance to next period */ - current_period->next_idx++; - if (current_period->next_idx == NUM_TOTALS_HS_USAGE) - current_period->next_idx = 0; - if (current_period->num_set < NUM_TOTALS_HS_USAGE) - ++current_period->num_set; - current_period->start_of_current_period=current_period->start_of_next_period; - current_period->start_of_next_period += NUM_SECS_HS_USAGE_SUM_INTERVAL; -} - -/** Checks if the current period is up to date, and if not, advances it. */ -static void -hs_usage_check_if_current_period_is_up_to_date(time_t now) -{ - while (now > current_period->start_of_next_period) { - hs_usage_advance_current_observation_period(); + /* Try to write to disk. */ + statsdir = get_datadir_fname("stats"); + if (check_private_dir(statsdir, CPD_CREATE) < 0) { + log_warn(LD_HIST, "Unable to create stats/ directory!"); + goto done; } -} + filename = get_datadir_fname2("stats", "exit-stats"); + if (write_str_to_file(filename, str, 0) < 0) + log_warn(LD_HIST, "Unable to write exit port statistics to disk!"); -/** Adds a service-related observation, maybe after advancing to next - * observation period. */ -static void -hs_usage_add_service_related_observation( - hs_usage_service_related_observation_t *h, - time_t now, - const char *service_id) -{ - if (now < current_period->start_of_current_period) { - /* don't record old data */ - return; - } - /* check if we are up-to-date */ - hs_usage_check_if_current_period_is_up_to_date(now); - /* add observation */ - hs_usage_insert_value(h->list, service_id); + done: + tor_free(str); + tor_free(statsdir); + tor_free(filename); + return start_of_exit_stats_interval + WRITE_STATS_INTERVAL; } -/** Adds the observation of storing a rendezvous service descriptor to our - * cache in our role as HS authoritative directory. */ +/** Note that we wrote <b>num_written</b> bytes and read <b>num_read</b> + * bytes to/from an exit connection to <b>port</b>. */ void -hs_usage_note_publish_total(const char *service_id, time_t now) +rep_hist_note_exit_bytes(uint16_t port, size_t num_written, + size_t num_read) { - hs_usage_add_service_related_observation(publish_total, now, service_id); + if (!start_of_exit_stats_interval) + return; /* Not initialized. */ + exit_bytes_written[port] += num_written; + exit_bytes_read[port] += num_read; + log_debug(LD_HIST, "Written %lu bytes and read %lu bytes to/from an " + "exit connection to port %d.", + (unsigned long)num_written, (unsigned long)num_read, port); } -/** Adds the observation of storing a novel rendezvous service descriptor to - * our cache in our role as HS authoritative directory. */ +/** Note that we opened an exit stream to <b>port</b>. */ void -hs_usage_note_publish_novel(const char *service_id, time_t now) +rep_hist_note_exit_stream_opened(uint16_t port) { - hs_usage_add_service_related_observation(publish_novel, now, service_id); + if (!start_of_exit_stats_interval) + return; /* Not initialized. */ + exit_streams[port]++; + log_debug(LD_HIST, "Opened exit stream to port %d", port); } -/** Adds the observation of being requested for a rendezvous service descriptor - * in our role as HS authoritative directory. */ +/*** cell statistics ***/ + +/** Start of the current buffer stats interval or 0 if we're not + * collecting buffer statistics. */ +static time_t start_of_buffer_stats_interval; + +/** Initialize buffer stats. */ void -hs_usage_note_fetch_total(const char *service_id, time_t now) +rep_hist_buffer_stats_init(time_t now) { - hs_usage_add_service_related_observation(fetch_total, now, service_id); + start_of_buffer_stats_interval = now; } -/** Adds the observation of being requested for a rendezvous service descriptor - * in our role as HS authoritative directory and being able to answer that - * request successfully. */ +/** Statistics from a single circuit. Collected when the circuit closes, or + * when we flush statistics to disk. */ +typedef struct circ_buffer_stats_t { + /** Average number of cells in the circuit's queue */ + double mean_num_cells_in_queue; + /** Average time a cell waits in the queue. */ + double mean_time_cells_in_queue; + /** Total number of cells sent over this circuit */ + uint32_t processed_cells; +} circ_buffer_stats_t; + +/** List of circ_buffer_stats_t. */ +static smartlist_t *circuits_for_buffer_stats = NULL; + +/** Remember cell statistics for circuit <b>circ</b> at time + * <b>end_of_interval</b> and reset cell counters in case the circuit + * remains open in the next measurement interval. */ void -hs_usage_note_fetch_successful(const char *service_id, time_t now) +rep_hist_buffer_stats_add_circ(circuit_t *circ, time_t end_of_interval) +{ + circ_buffer_stats_t *stat; + time_t start_of_interval; + int interval_length; + or_circuit_t *orcirc; + if (CIRCUIT_IS_ORIGIN(circ)) + return; + orcirc = TO_OR_CIRCUIT(circ); + if (!orcirc->processed_cells) + return; + if (!circuits_for_buffer_stats) + circuits_for_buffer_stats = smartlist_create(); + start_of_interval = circ->timestamp_created.tv_sec > + start_of_buffer_stats_interval ? + circ->timestamp_created.tv_sec : + start_of_buffer_stats_interval; + interval_length = (int) (end_of_interval - start_of_interval); + if (interval_length <= 0) + return; + stat = tor_malloc_zero(sizeof(circ_buffer_stats_t)); + stat->processed_cells = orcirc->processed_cells; + /* 1000.0 for s -> ms; 2.0 because of app-ward and exit-ward queues */ + stat->mean_num_cells_in_queue = interval_length == 0 ? 0.0 : + (double) orcirc->total_cell_waiting_time / + (double) interval_length / 1000.0 / 2.0; + stat->mean_time_cells_in_queue = + (double) orcirc->total_cell_waiting_time / + (double) orcirc->processed_cells; + smartlist_add(circuits_for_buffer_stats, stat); + orcirc->total_cell_waiting_time = 0; + orcirc->processed_cells = 0; +} + +/** Sorting helper: return -1, 1, or 0 based on comparison of two + * circ_buffer_stats_t */ +static int +_buffer_stats_compare_entries(const void **_a, const void **_b) { - hs_usage_add_service_related_observation(fetch_successful, now, service_id); + const circ_buffer_stats_t *a = *_a, *b = *_b; + if (a->processed_cells < b->processed_cells) + return 1; + else if (a->processed_cells > b->processed_cells) + return -1; + else + return 0; } -/** Writes the given circular array to a string. */ -static size_t -hs_usage_format_history(char *buf, size_t len, uint32_t *data) +/** Stop collecting cell stats in a way that we can re-start doing so in + * rep_hist_buffer_stats_init(). */ +void +rep_hist_buffer_stats_term(void) { - char *cp = buf; /* pointer where we are in the buffer */ - int i, n; - if (current_period->num_set <= current_period->next_idx) { - i = 0; /* not been through circular array */ - } else { - i = current_period->next_idx; - } - for (n = 0; n < current_period->num_set; ++n,++i) { - if (i >= NUM_TOTALS_HS_USAGE) - i -= NUM_TOTALS_HS_USAGE; - tor_assert(i < NUM_TOTALS_HS_USAGE); - if (n == (current_period->num_set-1)) - tor_snprintf(cp, len-(cp-buf), "%d", data[i]); - else - tor_snprintf(cp, len-(cp-buf), "%d,", data[i]); - cp += strlen(cp); - } - return cp-buf; + start_of_buffer_stats_interval = 0; + if (!circuits_for_buffer_stats) + circuits_for_buffer_stats = smartlist_create(); + SMARTLIST_FOREACH(circuits_for_buffer_stats, circ_buffer_stats_t *, + stat, tor_free(stat)); + smartlist_clear(circuits_for_buffer_stats); } -/** Writes the complete usage history as hidden service authoritative directory - * to a string. */ -static char * -hs_usage_format_statistics(void) +/** Write buffer statistics to $DATADIR/stats/buffer-stats and return when + * we would next want to write exit stats. */ +time_t +rep_hist_buffer_stats_write(time_t now) { - char *buf, *cp, *s = NULL; - char t[ISO_TIME_LEN+1]; - int r; - uint32_t *data = NULL; - size_t len; - len = (70+20*NUM_TOTALS_HS_USAGE)*11; - buf = tor_malloc_zero(len); - cp = buf; - for (r = 0; r < 11; ++r) { - switch (r) { - case 0: - s = (char*) "publish-total-history"; - data = publish_total->totals; - break; - case 1: - s = (char*) "publish-novel-history"; - data = publish_novel->totals; - break; - case 2: - s = (char*) "publish-top-5-percent-history"; - data = publish_total->five; - break; - case 3: - s = (char*) "publish-top-10-percent-history"; - data = publish_total->ten; - break; - case 4: - s = (char*) "publish-top-20-percent-history"; - data = publish_total->twenty; - break; - case 5: - s = (char*) "fetch-total-history"; - data = fetch_total->totals; - break; - case 6: - s = (char*) "fetch-successful-history"; - data = fetch_successful->totals; - break; - case 7: - s = (char*) "fetch-top-5-percent-history"; - data = fetch_total->five; - break; - case 8: - s = (char*) "fetch-top-10-percent-history"; - data = fetch_total->ten; - break; - case 9: - s = (char*) "fetch-top-20-percent-history"; - data = fetch_total->twenty; - break; - case 10: - s = (char*) "desc-total-history"; - data = descs->totals; - break; - } - format_iso_time(t, current_period->start_of_current_period); - tor_snprintf(cp, len-(cp-buf), "%s %s (%d s) ", s, t, - NUM_SECS_HS_USAGE_SUM_INTERVAL); - cp += strlen(cp); - cp += hs_usage_format_history(cp, len-(cp-buf), data); - strlcat(cp, "\n", len-(cp-buf)); - ++cp; + char *statsdir = NULL, *filename = NULL; + char written[ISO_TIME_LEN+1]; + open_file_t *open_file = NULL; + FILE *out; +#define SHARES 10 + int processed_cells[SHARES], circs_in_share[SHARES], + number_of_circuits, i; + double queued_cells[SHARES], time_in_queue[SHARES]; + smartlist_t *str_build = smartlist_create(); + char *str = NULL, *buf=NULL; + circuit_t *circ; + + if (!start_of_buffer_stats_interval) + return 0; /* Not initialized. */ + if (start_of_buffer_stats_interval + WRITE_STATS_INTERVAL > now) + goto done; /* Not ready to write */ + + /* add current circuits to stats */ + for (circ = _circuit_get_global_list(); circ; circ = circ->next) + rep_hist_buffer_stats_add_circ(circ, now); + /* calculate deciles */ + memset(processed_cells, 0, SHARES * sizeof(int)); + memset(circs_in_share, 0, SHARES * sizeof(int)); + memset(queued_cells, 0, SHARES * sizeof(double)); + memset(time_in_queue, 0, SHARES * sizeof(double)); + if (!circuits_for_buffer_stats) + circuits_for_buffer_stats = smartlist_create(); + smartlist_sort(circuits_for_buffer_stats, + _buffer_stats_compare_entries); + number_of_circuits = smartlist_len(circuits_for_buffer_stats); + if (number_of_circuits < 1) { + log_info(LD_HIST, "Attempt to write cell statistics to disk failed. " + "We haven't seen a single circuit to report about."); + goto done; } - return buf; + i = 0; + SMARTLIST_FOREACH_BEGIN(circuits_for_buffer_stats, + circ_buffer_stats_t *, stat) + { + int share = i++ * SHARES / number_of_circuits; + processed_cells[share] += stat->processed_cells; + queued_cells[share] += stat->mean_num_cells_in_queue; + time_in_queue[share] += stat->mean_time_cells_in_queue; + circs_in_share[share]++; + } + SMARTLIST_FOREACH_END(stat); + /* clear buffer stats history */ + SMARTLIST_FOREACH(circuits_for_buffer_stats, circ_buffer_stats_t *, + stat, tor_free(stat)); + smartlist_clear(circuits_for_buffer_stats); + /* write to file */ + statsdir = get_datadir_fname("stats"); + if (check_private_dir(statsdir, CPD_CREATE) < 0) + goto done; + filename = get_datadir_fname2("stats", "buffer-stats"); + out = start_writing_to_stdio_file(filename, OPEN_FLAGS_APPEND, + 0600, &open_file); + if (!out) + goto done; + format_iso_time(written, now); + if (fprintf(out, "cell-stats-end %s (%d s)\n", written, + (unsigned) (now - start_of_buffer_stats_interval)) < 0) + goto done; + for (i = 0; i < SHARES; i++) { + tor_asprintf(&buf,"%d", !circs_in_share[i] ? 0 : + processed_cells[i] / circs_in_share[i]); + smartlist_add(str_build, buf); + } + str = smartlist_join_strings(str_build, ",", 0, NULL); + if (fprintf(out, "cell-processed-cells %s\n", str) < 0) + goto done; + tor_free(str); + SMARTLIST_FOREACH(str_build, char *, c, tor_free(c)); + smartlist_clear(str_build); + for (i = 0; i < SHARES; i++) { + tor_asprintf(&buf, "%.2f", circs_in_share[i] == 0 ? 0.0 : + queued_cells[i] / (double) circs_in_share[i]); + smartlist_add(str_build, buf); + } + str = smartlist_join_strings(str_build, ",", 0, NULL); + if (fprintf(out, "cell-queued-cells %s\n", str) < 0) + goto done; + tor_free(str); + SMARTLIST_FOREACH(str_build, char *, c, tor_free(c)); + smartlist_clear(str_build); + for (i = 0; i < SHARES; i++) { + tor_asprintf(&buf, "%.0f", circs_in_share[i] == 0 ? 0.0 : + time_in_queue[i] / (double) circs_in_share[i]); + smartlist_add(str_build, buf); + } + str = smartlist_join_strings(str_build, ",", 0, NULL); + if (fprintf(out, "cell-time-in-queue %s\n", str) < 0) + goto done; + tor_free(str); + SMARTLIST_FOREACH(str_build, char *, c, tor_free(c)); + smartlist_free(str_build); + str_build = NULL; + if (fprintf(out, "cell-circuits-per-decile %d\n", + (number_of_circuits + SHARES - 1) / SHARES) < 0) + goto done; + finish_writing_to_file(open_file); + open_file = NULL; + start_of_buffer_stats_interval = now; + done: + if (open_file) + abort_writing_to_file(open_file); + tor_free(filename); + tor_free(statsdir); + if (str_build) { + SMARTLIST_FOREACH(str_build, char *, c, tor_free(c)); + smartlist_free(str_build); + } + tor_free(str); +#undef SHARES + return start_of_buffer_stats_interval + WRITE_STATS_INTERVAL; } -/** Write current statistics about hidden service usage to file. */ +/** Free all storage held by the OR/link history caches, by the + * bandwidth history arrays, by the port history, or by statistics . */ void -hs_usage_write_statistics_to_file(time_t now) +rep_hist_free_all(void) { - char *buf; - size_t len; - char *fname; - or_options_t *options = get_options(); - /* check if we are up-to-date */ - hs_usage_check_if_current_period_is_up_to_date(now); - buf = hs_usage_format_statistics(); - len = strlen(options->DataDirectory) + 16; - fname = tor_malloc(len); - tor_snprintf(fname, len, "%s"PATH_SEPARATOR"hsusage", - options->DataDirectory); - write_str_to_file(fname,buf,0); - tor_free(buf); - tor_free(fname); + digestmap_free(history_map, free_or_history); + tor_free(read_array); + tor_free(write_array); + tor_free(last_stability_doc); + tor_free(exit_bytes_read); + tor_free(exit_bytes_written); + tor_free(exit_streams); + built_last_stability_doc_at = 0; + predicted_ports_free(); + if (circuits_for_buffer_stats) { + SMARTLIST_FOREACH(circuits_for_buffer_stats, circ_buffer_stats_t *, s, + tor_free(s)); + smartlist_free(circuits_for_buffer_stats); + circuits_for_buffer_stats = NULL; + } } diff --git a/src/or/rephist.h b/src/or/rephist.h new file mode 100644 index 0000000000..b06a39ed59 --- /dev/null +++ b/src/or/rephist.h @@ -0,0 +1,82 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file rephist.h + * \brief Header file for rephist.c. + **/ + +#ifndef _TOR_REPHIST_H +#define _TOR_REPHIST_H + +void rep_hist_init(void); +void rep_hist_note_connect_failed(const char* nickname, time_t when); +void rep_hist_note_connect_succeeded(const char* nickname, time_t when); +void rep_hist_note_disconnect(const char* nickname, time_t when); +void rep_hist_note_connection_died(const char* nickname, time_t when); +void rep_hist_note_extend_succeeded(const char *from_name, + const char *to_name); +void rep_hist_note_extend_failed(const char *from_name, const char *to_name); +void rep_hist_dump_stats(time_t now, int severity); +void rep_hist_note_bytes_read(size_t num_bytes, time_t when); +void rep_hist_note_bytes_written(size_t num_bytes, time_t when); + +void rep_hist_note_dir_bytes_read(size_t num_bytes, time_t when); +void rep_hist_note_dir_bytes_written(size_t num_bytes, time_t when); + +int rep_hist_bandwidth_assess(void); +char *rep_hist_get_bandwidth_lines(void); +void rep_hist_update_state(or_state_t *state); +int rep_hist_load_state(or_state_t *state, char **err); +void rep_history_clean(time_t before); + +void rep_hist_note_router_reachable(const char *id, const tor_addr_t *at_addr, + uint16_t at_port, time_t when); +void rep_hist_note_router_unreachable(const char *id, time_t when); +int rep_hist_record_mtbf_data(time_t now, int missing_means_down); +int rep_hist_load_mtbf_data(time_t now); + +time_t rep_hist_downrate_old_runs(time_t now); +long rep_hist_get_uptime(const char *id, time_t when); +double rep_hist_get_stability(const char *id, time_t when); +double rep_hist_get_weighted_fractional_uptime(const char *id, time_t when); +long rep_hist_get_weighted_time_known(const char *id, time_t when); +int rep_hist_have_measured_enough_stability(void); +const char *rep_hist_get_router_stability_doc(time_t now); + +void rep_hist_note_used_port(time_t now, uint16_t port); +smartlist_t *rep_hist_get_predicted_ports(time_t now); +void rep_hist_note_used_resolve(time_t now); +void rep_hist_note_used_internal(time_t now, int need_uptime, + int need_capacity); +int rep_hist_get_predicted_internal(time_t now, int *need_uptime, + int *need_capacity); + +int any_predicted_circuits(time_t now); +int rep_hist_circbuilding_dormant(time_t now); + +void note_crypto_pk_op(pk_op_t operation); +void dump_pk_ops(int severity); + +void rep_hist_free_all(void); + +void rep_hist_exit_stats_init(time_t now); +void rep_hist_reset_exit_stats(time_t now); +void rep_hist_exit_stats_term(void); +char *rep_hist_format_exit_stats(time_t now); +time_t rep_hist_exit_stats_write(time_t now); +void rep_hist_note_exit_bytes(uint16_t port, size_t num_written, + size_t num_read); +void rep_hist_note_exit_stream_opened(uint16_t port); + +void rep_hist_buffer_stats_init(time_t now); +void rep_hist_buffer_stats_add_circ(circuit_t *circ, + time_t end_of_interval); +time_t rep_hist_buffer_stats_write(time_t now); +void rep_hist_buffer_stats_term(void); + +#endif + diff --git a/src/or/router.c b/src/or/router.c index cc600415f0..184715b750 100644 --- a/src/or/router.c +++ b/src/or/router.c @@ -7,6 +7,24 @@ #define ROUTER_PRIVATE #include "or.h" +#include "circuitlist.h" +#include "circuituse.h" +#include "config.h" +#include "connection.h" +#include "control.h" +#include "directory.h" +#include "dirserv.h" +#include "dns.h" +#include "geoip.h" +#include "hibernate.h" +#include "main.h" +#include "networkstatus.h" +#include "policies.h" +#include "relay.h" +#include "rephist.h" +#include "router.h" +#include "routerlist.h" +#include "routerparse.h" /** * \file router.c @@ -31,11 +49,15 @@ static crypto_pk_env_t *onionkey=NULL; /** Previous private onionskin decryption key: used to decode CREATE cells * generated by clients that have an older version of our descriptor. */ static crypto_pk_env_t *lastonionkey=NULL; -/** Private "identity key": used to sign directory info and TLS +/** Private server "identity key": used to sign directory info and TLS * certificates. Never changes. */ -static crypto_pk_env_t *identitykey=NULL; -/** Digest of identitykey. */ -static char identitykey_digest[DIGEST_LEN]; +static crypto_pk_env_t *server_identitykey=NULL; +/** Digest of server_identitykey. */ +static char server_identitykey_digest[DIGEST_LEN]; +/** Private client "identity key": used to sign bridges' and clients' + * outbound TLS certificates. Regenerated on startup and on IP address + * change. */ +static crypto_pk_env_t *client_identitykey=NULL; /** Signing key used for v3 directory material; only set for authorities. */ static crypto_pk_env_t *authority_signing_key = NULL; /** Key certificate to authenticate v3 directory material; only set for @@ -61,12 +83,10 @@ static void set_onion_key(crypto_pk_env_t *k) { tor_mutex_acquire(key_lock); - if (onionkey) - crypto_free_pk_env(onionkey); + crypto_free_pk_env(onionkey); onionkey = k; - onionkey_set_at = time(NULL); tor_mutex_release(key_lock); - mark_my_descriptor_dirty(); + mark_my_descriptor_dirty("set onion key"); } /** Return the current onion key. Requires that the onion key has been @@ -106,32 +126,78 @@ get_onion_key_set_at(void) return onionkey_set_at; } -/** Set the current identity key to k. +/** Set the current server identity key to <b>k</b>. */ void -set_identity_key(crypto_pk_env_t *k) +set_server_identity_key(crypto_pk_env_t *k) { - if (identitykey) - crypto_free_pk_env(identitykey); - identitykey = k; - crypto_pk_get_digest(identitykey, identitykey_digest); + crypto_free_pk_env(server_identitykey); + server_identitykey = k; + crypto_pk_get_digest(server_identitykey, server_identitykey_digest); } -/** Returns the current identity key; requires that the identity key has been - * set. +/** Make sure that we have set up our identity keys to match or not match as + * appropriate, and die with an assertion if we have not. */ +static void +assert_identity_keys_ok(void) +{ + tor_assert(client_identitykey); + if (public_server_mode(get_options())) { + /* assert that we have set the client and server keys to be equal */ + tor_assert(server_identitykey); + tor_assert(0==crypto_pk_cmp_keys(client_identitykey, server_identitykey)); + } else { + /* assert that we have set the client and server keys to be unequal */ + if (server_identitykey) + tor_assert(0!=crypto_pk_cmp_keys(client_identitykey, + server_identitykey)); + } +} + +/** Returns the current server identity key; requires that the key has + * been set, and that we are running as a Tor server. + */ +crypto_pk_env_t * +get_server_identity_key(void) +{ + tor_assert(server_identitykey); + tor_assert(server_mode(get_options())); + assert_identity_keys_ok(); + return server_identitykey; +} + +/** Return true iff the server identity key has been set. */ +int +server_identity_key_is_set(void) +{ + return server_identitykey != NULL; +} + +/** Set the current client identity key to <b>k</b>. + */ +void +set_client_identity_key(crypto_pk_env_t *k) +{ + crypto_free_pk_env(client_identitykey); + client_identitykey = k; +} + +/** Returns the current client identity key for use on outgoing TLS + * connections; requires that the key has been set. */ crypto_pk_env_t * -get_identity_key(void) +get_tlsclient_identity_key(void) { - tor_assert(identitykey); - return identitykey; + tor_assert(client_identitykey); + assert_identity_keys_ok(); + return client_identitykey; } -/** Return true iff the identity key has been set. */ +/** Return true iff the client identity key has been set. */ int -identity_key_is_set(void) +client_identity_key_is_set(void) { - return identitykey != NULL; + return client_identitykey != NULL; } /** Return the key certificate for this v3 (voting) authority, or NULL @@ -201,14 +267,13 @@ rotate_onion_key(void) } log_info(LD_GENERAL, "Rotating onion key"); tor_mutex_acquire(key_lock); - if (lastonionkey) - crypto_free_pk_env(lastonionkey); + crypto_free_pk_env(lastonionkey); lastonionkey = onionkey; onionkey = prkey; now = time(NULL); state->LastRotatedOnionKey = onionkey_set_at = now; tor_mutex_release(key_lock); - mark_my_descriptor_dirty(); + mark_my_descriptor_dirty("rotated onion key"); or_state_mark_dirty(state, get_options()->AvoidDiskWrites ? now+3600 : 0); goto done; error: @@ -331,10 +396,9 @@ load_authority_keyset(int legacy, crypto_pk_env_t **key_out, goto done; } - if (*key_out) - crypto_free_pk_env(*key_out); - if (*cert_out) - authority_cert_free(*cert_out); + crypto_free_pk_env(*key_out); + authority_cert_free(*cert_out); + *key_out = signing_key; *cert_out = parsed; r = 0; @@ -344,10 +408,8 @@ load_authority_keyset(int legacy, crypto_pk_env_t **key_out, done: tor_free(fname); tor_free(cert); - if (signing_key) - crypto_free_pk_env(signing_key); - if (parsed) - authority_cert_free(parsed); + crypto_free_pk_env(signing_key); + authority_cert_free(parsed); return r; } @@ -428,8 +490,8 @@ init_keys(void) char fingerprint_line[MAX_NICKNAME_LEN+FINGERPRINT_LEN+3]; const char *mydesc; crypto_pk_env_t *prkey; - char digest[20]; - char v3_digest[20]; + char digest[DIGEST_LEN]; + char v3_digest[DIGEST_LEN]; char *cp; or_options_t *options = get_options(); authority_type_t type; @@ -441,8 +503,11 @@ init_keys(void) if (!key_lock) key_lock = tor_mutex_new(); - /* There are a couple of paths that put us here before */ - if (crypto_global_init(get_options()->HardwareAccel)) { + /* There are a couple of paths that put us here before we've asked + * openssl to initialize itself. */ + if (crypto_global_init(get_options()->HardwareAccel, + get_options()->AccelName, + get_options()->AccelDir)) { log_err(LD_BUG, "Unable to initialize OpenSSL. Exiting."); return -1; } @@ -456,9 +521,12 @@ init_keys(void) crypto_free_pk_env(prkey); return -1; } - set_identity_key(prkey); - /* Create a TLS context; default the client nickname to "client". */ - if (tor_tls_context_new(get_identity_key(), MAX_SSL_KEY_LIFETIME) < 0) { + set_client_identity_key(prkey); + /* Create a TLS context. */ + if (tor_tls_context_init(0, + get_tlsclient_identity_key(), + NULL, + MAX_SSL_KEY_LIFETIME) < 0) { log_err(LD_GENERAL,"Error creating TLS context for Tor client."); return -1; } @@ -493,13 +561,28 @@ init_keys(void) } } - /* 1. Read identity key. Make it if none is found. */ + /* 1b. Read identity key. Make it if none is found. */ keydir = get_datadir_fname2("keys", "secret_id_key"); log_info(LD_GENERAL,"Reading/making identity key \"%s\"...",keydir); prkey = init_key_from_file(keydir, 1, LOG_ERR); tor_free(keydir); if (!prkey) return -1; - set_identity_key(prkey); + set_server_identity_key(prkey); + + /* 1c. If we are configured as a bridge, generate a client key; + * otherwise, set the server identity key as our client identity + * key. */ + if (public_server_mode(options)) { + set_client_identity_key(crypto_pk_dup_key(prkey)); /* set above */ + } else { + if (!(prkey = crypto_new_pk_env())) + return -1; + if (crypto_pk_generate_key(prkey)) { + crypto_free_pk_env(prkey); + return -1; + } + set_client_identity_key(prkey); + } /* 2. Read onion key. Make it if none is found. */ keydir = get_datadir_fname2("keys", "secret_onion_key"); @@ -536,18 +619,22 @@ init_keys(void) tor_free(keydir); /* 3. Initialize link key and TLS context. */ - if (tor_tls_context_new(get_identity_key(), MAX_SSL_KEY_LIFETIME) < 0) { + if (tor_tls_context_init(public_server_mode(options), + get_tlsclient_identity_key(), + get_server_identity_key(), + MAX_SSL_KEY_LIFETIME) < 0) { log_err(LD_GENERAL,"Error initializing TLS context"); return -1; } /* 4. Build our router descriptor. */ /* Must be called after keys are initialized. */ mydesc = router_get_my_descriptor(); - if (authdir_mode(options)) { + if (authdir_mode_handles_descs(options, ROUTER_PURPOSE_GENERAL)) { const char *m = NULL; routerinfo_t *ri; /* We need to add our own fingerprint so it gets recognized. */ - if (dirserv_add_own_fingerprint(options->Nickname, get_identity_key())) { + if (dirserv_add_own_fingerprint(options->Nickname, + get_server_identity_key())) { log_err(LD_GENERAL,"Error adding own fingerprint to approved set"); return -1; } @@ -568,7 +655,8 @@ init_keys(void) /* 5. Dump fingerprint to 'fingerprint' */ keydir = get_datadir_fname("fingerprint"); log_info(LD_GENERAL,"Dumping fingerprint to \"%s\"...",keydir); - if (crypto_pk_get_fingerprint(get_identity_key(), fingerprint, 0)<0) { + if (crypto_pk_get_fingerprint(get_server_identity_key(), + fingerprint, 0) < 0) { log_err(LD_GENERAL,"Error computing fingerprint"); tor_free(keydir); return -1; @@ -606,7 +694,7 @@ init_keys(void) return -1; } /* 6b. [authdirserver only] add own key to approved directories. */ - crypto_pk_get_digest(get_identity_key(), digest); + crypto_pk_get_digest(get_server_identity_key(), digest); type = ((options->V1AuthoritativeDir ? V1_AUTHORITY : NO_AUTHORITY) | (options->V2AuthoritativeDir ? V2_AUTHORITY : NO_AUTHORITY) | (options->V3AuthoritativeDir ? V3_AUTHORITY : NO_AUTHORITY) | @@ -616,8 +704,8 @@ init_keys(void) ds = router_get_trusteddirserver_by_digest(digest); if (!ds) { ds = add_trusted_dir_server(options->Nickname, NULL, - (uint16_t)options->DirPort, - (uint16_t)options->ORPort, + router_get_advertised_dir_port(options), + router_get_advertised_or_port(options), digest, v3_digest, type); @@ -762,9 +850,29 @@ consider_testing_reachability(int test_or, int test_dir) routerinfo_t *me = router_get_my_routerinfo(); int orport_reachable = check_whether_orport_reachable(); tor_addr_t addr; + or_options_t *options = get_options(); if (!me) return; + if (routerset_contains_router(options->ExcludeNodes, me) && + options->StrictNodes) { + /* If we've excluded ourself, and StrictNodes is set, we can't test + * ourself. */ + if (test_or || test_dir) { +#define SELF_EXCLUDED_WARN_INTERVAL 3600 + static ratelim_t warning_limit=RATELIM_INIT(SELF_EXCLUDED_WARN_INTERVAL); + char *msg; + if ((msg = rate_limit_log(&warning_limit, approx_time()))) { + log_warn(LD_CIRC, "Can't peform self-tests for this relay: we have " + "listed ourself in ExcludeNodes, and StrictNodes is set. " + "We cannot learn whether we are usable, and will not " + "be able to advertise ourself.%s", msg); + tor_free(msg); + } + } + return; + } + if (test_or && (!orport_reachable || !circuit_enough_testing_circs())) { log_info(LD_CIRC, "Testing %s of my ORPort: %s:%d.", !orport_reachable ? "reachability" : "bandwidth", @@ -800,7 +908,7 @@ router_orport_found_reachable(void) get_options()->_PublishServerDescriptor != NO_AUTHORITY ? " Publishing server descriptor." : ""); can_reach_or_port = 1; - mark_my_descriptor_dirty(); + mark_my_descriptor_dirty("ORPort found reachable"); control_event_server_status(LOG_NOTICE, "REACHABILITY_SUCCEEDED ORADDRESS=%s:%d", me->address, me->or_port); @@ -817,7 +925,7 @@ router_dirport_found_reachable(void) "from the outside. Excellent."); can_reach_dir_port = 1; if (decide_to_advertise_dirport(get_options(), me->dir_port)) - mark_my_descriptor_dirty(); + mark_my_descriptor_dirty("DirPort found reachable"); control_event_server_status(LOG_NOTICE, "REACHABILITY_SUCCEEDED DIRADDRESS=%s:%d", me->address, me->dir_port); @@ -952,6 +1060,28 @@ server_mode(or_options_t *options) return (options->ORPort != 0 || options->ORListenAddress); } +/** Return true iff we are trying to be a non-bridge server. + */ +int +public_server_mode(or_options_t *options) +{ + if (!server_mode(options)) return 0; + return (!options->BridgeRelay); +} + +/** Return true iff the combination of options in <b>options</b> and parameters + * in the consensus mean that we don't want to allow exits from circuits + * we got from addresses not known to be servers. */ +int +should_refuse_unknown_exits(or_options_t *options) +{ + if (options->RefuseUnknownExits_ != -1) { + return options->RefuseUnknownExits_; + } else { + return networkstatus_get_param(NULL, "refuseunknownexits", 1, 0, 1); + } +} + /** Remember if we've advertised ourselves to the dirservers. */ static int server_is_advertised=0; @@ -977,10 +1107,10 @@ set_server_advertised(int s) int proxy_mode(or_options_t *options) { - return (options->SocksPort != 0 || options->SocksListenAddress || - options->TransPort != 0 || options->TransListenAddress || - options->NatdPort != 0 || options->NatdListenAddress || - options->DNSPort != 0 || options->DNSListenAddress); + return (options->SocksPort != 0 || + options->TransPort != 0 || + options->NATDPort != 0 || + options->DNSPort != 0); } /** Decide if we're a publishable server. We are a publishable server if: @@ -1035,6 +1165,36 @@ consider_publishable_server(int force) } } +/** Return the port that we should advertise as our ORPort; this is either + * the one configured in the ORPort option, or the one we actually bound to + * if ORPort is "auto". */ +uint16_t +router_get_advertised_or_port(or_options_t *options) +{ + if (options->ORPort == CFG_AUTO_PORT) { + connection_t *c = connection_get_by_type(CONN_TYPE_OR_LISTENER); + if (c) + return c->port; + return 0; + } + return options->ORPort; +} + +/** Return the port that we should advertise as our DirPort; this is either + * the one configured in the DirPort option, or the one we actually bound to + * if DirPort is "auto". */ +uint16_t +router_get_advertised_dir_port(or_options_t *options) +{ + if (options->DirPort == CFG_AUTO_PORT) { + connection_t *c = connection_get_by_type(CONN_TYPE_DIR_LISTENER); + if (c) + return c->port; + return 0; + } + return options->DirPort; +} + /* * OR descriptor generation. */ @@ -1072,6 +1232,10 @@ router_upload_dir_desc_to_dirservers(int force) return; if (!force && !desc_needs_upload) return; + + log_info(LD_OR, "Uploading relay descriptor to directory authorities%s", + force ? " (forced)" : ""); + desc_needs_upload = 0; desc_len = ri->cache_info.signed_descriptor_len; @@ -1114,12 +1278,24 @@ router_compare_to_my_exit_policy(edge_connection_t *conn) desc_routerinfo->exit_policy) != ADDR_POLICY_ACCEPTED; } +/** Return true iff my exit policy is reject *:*. Return -1 if we don't + * have a descriptor */ +int +router_my_exit_policy_is_reject_star(void) +{ + if (!router_get_my_routerinfo()) /* make sure desc_routerinfo exists */ + return -1; + + return desc_routerinfo->policy_is_reject_star; +} + /** Return true iff I'm a server and <b>digest</b> is equal to - * my identity digest. */ + * my server identity key digest. */ int router_digest_is_me(const char *digest) { - return identitykey && tor_memeq(identitykey_digest, digest, DIGEST_LEN); + return (server_identitykey && + tor_memeq(server_identitykey_digest, digest, DIGEST_LEN)); } /** Return true iff I'm a server and <b>digest</b> is equal to @@ -1209,6 +1385,8 @@ static int router_guess_address_from_dir_headers(uint32_t *guess); int router_pick_published_address(or_options_t *options, uint32_t *addr) { + char buf[INET_NTOA_BUF_LEN]; + struct in_addr a; if (resolve_my_address(LOG_INFO, options, addr, NULL) < 0) { log_info(LD_CONFIG, "Could not determine our address locally. " "Checking if directory headers provide any hints."); @@ -1218,6 +1396,9 @@ router_pick_published_address(or_options_t *options, uint32_t *addr) return -1; } } + a.s_addr = htonl(*addr); + tor_inet_ntoa(&a, buf, sizeof(buf)); + log_info(LD_CONFIG,"Success: chose address '%s'.", buf); return 0; } @@ -1240,23 +1421,25 @@ router_rebuild_descriptor(int force) if (router_pick_published_address(options, &addr) < 0) { /* Stop trying to rebuild our descriptor every second. We'll - * learn that it's time to try again when server_has_changed_ip() + * learn that it's time to try again when ip_address_changed() * marks it dirty. */ desc_clean_since = time(NULL); return -1; } + log_info(LD_OR, "Rebuilding relay descriptor%s", force ? " (forced)" : ""); + ri = tor_malloc_zero(sizeof(routerinfo_t)); ri->cache_info.routerlist_index = -1; ri->address = tor_dup_ip(addr); ri->nickname = tor_strdup(options->Nickname); ri->addr = addr; - ri->or_port = options->ORPort; - ri->dir_port = options->DirPort; + ri->or_port = router_get_advertised_or_port(options); + ri->dir_port = router_get_advertised_dir_port(options); ri->cache_info.published_on = time(NULL); ri->onion_pkey = crypto_pk_dup_key(get_onion_key()); /* must invoke from * main thread */ - ri->identity_pkey = crypto_pk_dup_key(get_identity_key()); + ri->identity_pkey = crypto_pk_dup_key(get_server_identity_key()); if (crypto_pk_get_digest(ri->identity_pkey, ri->cache_info.identity_digest)<0) { routerinfo_free(ri); @@ -1273,9 +1456,16 @@ router_rebuild_descriptor(int force) ri->bandwidthcapacity = hibernating ? 0 : rep_hist_bandwidth_assess(); - policies_parse_exit_policy(options->ExitPolicy, &ri->exit_policy, - options->ExitPolicyRejectPrivate, - ri->address); + if (dns_seems_to_be_broken() || has_dns_init_failed()) { + /* DNS is screwed up; don't claim to be an exit. */ + policies_exit_policy_append_reject_star(&ri->exit_policy); + } else { + policies_parse_exit_policy(options->ExitPolicy, &ri->exit_policy, + options->ExitPolicyRejectPrivate, + ri->address, !options->BridgeRelay); + } + ri->policy_is_reject_star = + policy_is_reject_star(ri->exit_policy); if (desc_routerinfo) { /* inherit values */ ri->is_valid = desc_routerinfo->is_valid; @@ -1346,26 +1536,30 @@ router_rebuild_descriptor(int force) ei->cache_info.published_on = ri->cache_info.published_on; memcpy(ei->cache_info.identity_digest, ri->cache_info.identity_digest, DIGEST_LEN); - ei->cache_info.signed_descriptor_body = tor_malloc(8192); - if (extrainfo_dump_to_string(ei->cache_info.signed_descriptor_body, 8192, - ei, get_identity_key()) < 0) { + if (extrainfo_dump_to_string(&ei->cache_info.signed_descriptor_body, + ei, get_server_identity_key()) < 0) { log_warn(LD_BUG, "Couldn't generate extra-info descriptor."); - routerinfo_free(ri); extrainfo_free(ei); - return -1; + ei = NULL; + } else { + ei->cache_info.signed_descriptor_len = + strlen(ei->cache_info.signed_descriptor_body); + router_get_extrainfo_hash(ei->cache_info.signed_descriptor_body, + ei->cache_info.signed_descriptor_digest); } - ei->cache_info.signed_descriptor_len = - strlen(ei->cache_info.signed_descriptor_body); - router_get_extrainfo_hash(ei->cache_info.signed_descriptor_body, - ei->cache_info.signed_descriptor_digest); /* Now finish the router descriptor. */ - memcpy(ri->cache_info.extra_info_digest, - ei->cache_info.signed_descriptor_digest, - DIGEST_LEN); + if (ei) { + memcpy(ri->cache_info.extra_info_digest, + ei->cache_info.signed_descriptor_digest, + DIGEST_LEN); + } else { + /* ri was allocated with tor_malloc_zero, so there is no need to + * zero ri->cache_info.extra_info_digest here. */ + } ri->cache_info.signed_descriptor_body = tor_malloc(8192); if (router_dump_router_to_string(ri->cache_info.signed_descriptor_body, 8192, - ri, get_identity_key())<0) { + ri, get_server_identity_key()) < 0) { log_warn(LD_BUG, "Couldn't generate router descriptor."); routerinfo_free(ri); extrainfo_free(ei); @@ -1380,7 +1574,7 @@ router_rebuild_descriptor(int force) /* Let bridges serve their own descriptors unencrypted, so they can * pass reachability testing. (If they want to be harder to notice, * they can always leave the DirPort off). */ - if (!options->BridgeRelay) + if (ei && !options->BridgeRelay) ei->cache_info.send_unencrypted = 1; router_get_router_hash(ri->cache_info.signed_descriptor_body, @@ -1389,13 +1583,13 @@ router_rebuild_descriptor(int force) routerinfo_set_country(ri); - tor_assert(! routerinfo_incompatible_with_extrainfo(ri, ei, NULL, NULL)); + if (ei) { + tor_assert(! routerinfo_incompatible_with_extrainfo(ri, ei, NULL, NULL)); + } - if (desc_routerinfo) - routerinfo_free(desc_routerinfo); + routerinfo_free(desc_routerinfo); desc_routerinfo = ri; - if (desc_extrainfo) - extrainfo_free(desc_extrainfo); + extrainfo_free(desc_extrainfo); desc_extrainfo = ei; desc_clean_since = time(NULL); @@ -1409,14 +1603,15 @@ void mark_my_descriptor_dirty_if_older_than(time_t when) { if (desc_clean_since < when) - mark_my_descriptor_dirty(); + mark_my_descriptor_dirty("time for new descriptor"); } /** Call when the current descriptor is out of date. */ void -mark_my_descriptor_dirty(void) +mark_my_descriptor_dirty(const char *reason) { desc_clean_since = 0; + log_info(LD_OR, "Decided to publish new relay descriptor: %s", reason); } /** How frequently will we republish our descriptor because of large (factor @@ -1441,7 +1636,7 @@ check_descriptor_bandwidth_changed(time_t now) if (last_changed+MAX_BANDWIDTH_CHANGE_FREQ < now) { log_info(LD_GENERAL, "Measured bandwidth has changed; rebuilding descriptor."); - mark_my_descriptor_dirty(); + mark_my_descriptor_dirty("bandwidth has changed"); last_changed = now; } } @@ -1606,6 +1801,7 @@ router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router, char digest[DIGEST_LEN]; char published[ISO_TIME_LEN+1]; char fingerprint[FINGERPRINT_LEN+1]; + int has_extra_info_digest; char extra_info_digest[HEX_DIGEST_LEN+1]; size_t onion_pkeylen, identity_pkeylen; size_t written; @@ -1634,7 +1830,7 @@ router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router, return -1; } - /* PEM-encode the identity key key */ + /* PEM-encode the identity key */ if (crypto_pk_write_public_key_to_string(router->identity_pkey, &identity_pkey,&identity_pkeylen)<0) { log_warn(LD_BUG,"write identity_pkey to string failed!"); @@ -1656,8 +1852,13 @@ router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router, family_line = tor_strdup(""); } - base16_encode(extra_info_digest, sizeof(extra_info_digest), - router->cache_info.extra_info_digest, DIGEST_LEN); + has_extra_info_digest = + ! tor_digest_is_zero(router->cache_info.extra_info_digest); + + if (has_extra_info_digest) { + base16_encode(extra_info_digest, sizeof(extra_info_digest), + router->cache_info.extra_info_digest, DIGEST_LEN); + } /* Generate the easy portion of the router descriptor. */ result = tor_snprintf(s, maxlen, @@ -1668,7 +1869,7 @@ router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router, "opt fingerprint %s\n" "uptime %ld\n" "bandwidth %d %d %d\n" - "opt extra-info-digest %s\n%s" + "%s%s%s%s" "onion-key\n%s" "signing-key\n%s" "%s%s%s%s", @@ -1683,7 +1884,9 @@ router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router, (int) router->bandwidthrate, (int) router->bandwidthburst, (int) router->bandwidthcapacity, - extra_info_digest, + has_extra_info_digest ? "opt extra-info-digest " : "", + has_extra_info_digest ? extra_info_digest : "", + has_extra_info_digest ? "\n" : "", options->DownloadExtraInfo ? "opt caches-extra-info\n" : "", onion_pkey, identity_pkey, family_line, @@ -1715,9 +1918,7 @@ router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router, } /* Write the exit policy to the end of 's'. */ - if (dns_seems_to_be_broken() || has_dns_init_failed() || - !router->exit_policy || !smartlist_len(router->exit_policy)) { - /* DNS is screwed up; don't claim to be an exit. */ + if (!router->exit_policy || !smartlist_len(router->exit_policy)) { strlcat(s+written, "reject *:*\n", maxlen-written); written += strlen("reject *:*\n"); tmpe = NULL; @@ -1740,7 +1941,8 @@ router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router, } } - if (written+256 > maxlen) { /* Not enough room for signature. */ + if (written + DIROBJ_MAX_SIG_LEN > maxlen) { + /* Not enough room for signature. */ log_warn(LD_BUG,"not enough room left in descriptor for signature!"); return -1; } @@ -1755,7 +1957,7 @@ router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router, note_crypto_pk_op(SIGN_RTR); if (router_append_dirobj_signature(s+written,maxlen-written, - digest,ident_key)<0) { + digest,DIGEST_LEN,ident_key)<0) { log_warn(LD_BUG, "Couldn't sign router descriptor"); return -1; } @@ -1790,11 +1992,62 @@ router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router, return (int)written+1; } -/** Write the contents of <b>extrainfo</b> to the <b>maxlen</b>-byte string - * <b>s</b>, signing them with <b>ident_key</b>. Return 0 on success, - * negative on failure. */ +/** Load the contents of <b>filename</b>, find the last line starting with + * <b>end_line</b>, ensure that its timestamp is not more than 25 hours in + * the past or more than 1 hour in the future with respect to <b>now</b>, + * and write the file contents starting with that line to *<b>out</b>. + * Return 1 for success, 0 if the file does not exist, or -1 if the file + * does not contain a line matching these criteria or other failure. */ +static int +load_stats_file(const char *filename, const char *end_line, time_t now, + char **out) +{ + int r = -1; + char *fname = get_datadir_fname(filename); + char *contents, *start = NULL, *tmp, timestr[ISO_TIME_LEN+1]; + time_t written; + switch (file_status(fname)) { + case FN_FILE: + /* X022 Find an alternative to reading the whole file to memory. */ + if ((contents = read_file_to_str(fname, 0, NULL))) { + tmp = strstr(contents, end_line); + /* Find last block starting with end_line */ + while (tmp) { + start = tmp; + tmp = strstr(tmp + 1, end_line); + } + if (!start) + goto notfound; + if (strlen(start) < strlen(end_line) + 1 + sizeof(timestr)) + goto notfound; + strlcpy(timestr, start + 1 + strlen(end_line), sizeof(timestr)); + if (parse_iso_time(timestr, &written) < 0) + goto notfound; + if (written < now - (25*60*60) || written > now + (1*60*60)) + goto notfound; + *out = tor_strdup(start); + r = 1; + } + notfound: + tor_free(contents); + break; + case FN_NOENT: + r = 0; + break; + case FN_ERROR: + case FN_DIR: + default: + break; + } + tor_free(fname); + return r; +} + +/** Write the contents of <b>extrainfo</b> and aggregated statistics to + * *<b>s_out</b>, signing them with <b>ident_key</b>. Return 0 on + * success, negative on failure. */ int -extrainfo_dump_to_string(char *s, size_t maxlen, extrainfo_t *extrainfo, +extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo, crypto_pk_env_t *ident_key) { or_options_t *options = get_options(); @@ -1803,87 +2056,128 @@ extrainfo_dump_to_string(char *s, size_t maxlen, extrainfo_t *extrainfo, char digest[DIGEST_LEN]; char *bandwidth_usage; int result; - size_t len; + static int write_stats_to_extrainfo = 1; + char sig[DIROBJ_MAX_SIG_LEN+1]; + char *s, *pre, *contents, *cp, *s_dup = NULL; + time_t now = time(NULL); + smartlist_t *chunks = smartlist_create(); + extrainfo_t *ei_tmp = NULL; base16_encode(identity, sizeof(identity), extrainfo->cache_info.identity_digest, DIGEST_LEN); format_iso_time(published, extrainfo->cache_info.published_on); - bandwidth_usage = rep_hist_get_bandwidth_lines(1); + bandwidth_usage = rep_hist_get_bandwidth_lines(); - result = tor_snprintf(s, maxlen, - "extra-info %s %s\n" - "published %s\n%s", - extrainfo->nickname, identity, - published, bandwidth_usage); + tor_asprintf(&pre, "extra-info %s %s\npublished %s\n%s", + extrainfo->nickname, identity, + published, bandwidth_usage); tor_free(bandwidth_usage); - if (result<0) - return -1; + smartlist_add(chunks, pre); + + if (options->ExtraInfoStatistics && write_stats_to_extrainfo) { + log_info(LD_GENERAL, "Adding stats to extra-info descriptor."); + if (options->DirReqStatistics && + load_stats_file("stats"PATH_SEPARATOR"dirreq-stats", + "dirreq-stats-end", now, &contents) > 0) { + smartlist_add(chunks, contents); + } + if (options->EntryStatistics && + load_stats_file("stats"PATH_SEPARATOR"entry-stats", + "entry-stats-end", now, &contents) > 0) { + smartlist_add(chunks, contents); + } + if (options->CellStatistics && + load_stats_file("stats"PATH_SEPARATOR"buffer-stats", + "cell-stats-end", now, &contents) > 0) { + smartlist_add(chunks, contents); + } + if (options->ExitPortStatistics && + load_stats_file("stats"PATH_SEPARATOR"exit-stats", + "exit-stats-end", now, &contents) > 0) { + smartlist_add(chunks, contents); + } + } - if (should_record_bridge_info(options)) { - char *geoip_summary = extrainfo_get_client_geoip_summary(time(NULL)); - if (geoip_summary) { - char geoip_start[ISO_TIME_LEN+1]; - format_iso_time(geoip_start, geoip_get_history_start()); - result = tor_snprintf(s+strlen(s), maxlen-strlen(s), - "geoip-start-time %s\n" - "geoip-client-origins %s\n", - geoip_start, geoip_summary); - control_event_clients_seen(geoip_start, geoip_summary); - tor_free(geoip_summary); - if (result<0) - return -1; + if (should_record_bridge_info(options) && write_stats_to_extrainfo) { + const char *bridge_stats = geoip_get_bridge_stats_extrainfo(now); + if (bridge_stats) { + smartlist_add(chunks, tor_strdup(bridge_stats)); } } - len = strlen(s); - strlcat(s+len, "router-signature\n", maxlen-len); - len += strlen(s+len); - if (router_get_extrainfo_hash(s, digest)<0) - return -1; - if (router_append_dirobj_signature(s+len, maxlen-len, digest, ident_key)<0) - return -1; + smartlist_add(chunks, tor_strdup("router-signature\n")); + s = smartlist_join_strings(chunks, "", 0, NULL); + + while (strlen(s) > MAX_EXTRAINFO_UPLOAD_SIZE - DIROBJ_MAX_SIG_LEN) { + /* So long as there are at least two chunks (one for the initial + * extra-info line and one for the router-signature), we can keep removing + * things. */ + if (smartlist_len(chunks) > 2) { + /* We remove the next-to-last element (remember, len-1 is the last + element), since we need to keep the router-signature element. */ + int idx = smartlist_len(chunks) - 2; + char *e = smartlist_get(chunks, idx); + smartlist_del_keeporder(chunks, idx); + log_warn(LD_GENERAL, "We just generated an extra-info descriptor " + "with statistics that exceeds the 50 KB " + "upload limit. Removing last added " + "statistics."); + tor_free(e); + tor_free(s); + s = smartlist_join_strings(chunks, "", 0, NULL); + } else { + log_warn(LD_BUG, "We just generated an extra-info descriptors that " + "exceeds the 50 KB upload limit."); + goto err; + } + } -#ifdef DEBUG_ROUTER_DUMP_ROUTER_TO_STRING - { - char *cp, *s_dup; - extrainfo_t *ei_tmp; - cp = s_dup = tor_strdup(s); - ei_tmp = extrainfo_parse_entry_from_string(cp, NULL, 1, NULL); - if (!ei_tmp) { - log_err(LD_BUG, - "We just generated an extrainfo descriptor we can't parse."); - log_err(LD_BUG, "Descriptor was: <<%s>>", s); - tor_free(s_dup); - return -1; + memset(sig, 0, sizeof(sig)); + if (router_get_extrainfo_hash(s, digest) < 0 || + router_append_dirobj_signature(sig, sizeof(sig), digest, DIGEST_LEN, + ident_key) < 0) { + log_warn(LD_BUG, "Could not append signature to extra-info " + "descriptor."); + goto err; + } + smartlist_add(chunks, tor_strdup(sig)); + tor_free(s); + s = smartlist_join_strings(chunks, "", 0, NULL); + + cp = s_dup = tor_strdup(s); + ei_tmp = extrainfo_parse_entry_from_string(cp, NULL, 1, NULL); + if (!ei_tmp) { + if (write_stats_to_extrainfo) { + log_warn(LD_GENERAL, "We just generated an extra-info descriptor " + "with statistics that we can't parse. Not " + "adding statistics to this or any future " + "extra-info descriptors."); + write_stats_to_extrainfo = 0; + result = extrainfo_dump_to_string(s_out, extrainfo, ident_key); + goto done; + } else { + log_warn(LD_BUG, "We just generated an extrainfo descriptor we " + "can't parse."); + goto err; } - tor_free(s_dup); - extrainfo_free(ei_tmp); } -#endif - return (int)strlen(s)+1; -} + *s_out = s; + s = NULL; /* prevent free */ + result = 0; + goto done; -/** Wrapper function for geoip_get_client_history(). It first discards - * any items in the client history that are too old -- it dumps anything - * more than 48 hours old, but it only considers whether to dump at most - * once per 48 hours, so we aren't too precise to an observer (see also - * r14780). - */ -char * -extrainfo_get_client_geoip_summary(time_t now) -{ - static time_t last_purged_at = 0; - int geoip_purge_interval = 48*60*60; -#ifdef ENABLE_GEOIP_STATS - if (get_options()->DirRecordUsageByCountry) - geoip_purge_interval = get_options()->DirRecordUsageRetainIPs; -#endif - if (now > last_purged_at+geoip_purge_interval) { - geoip_remove_old_clients(now-geoip_purge_interval); - last_purged_at = now; - } - return geoip_get_client_history(now, GEOIP_CLIENT_CONNECT); + err: + result = -1; + + done: + tor_free(s); + SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp)); + smartlist_free(chunks); + tor_free(s_dup); + extrainfo_free(ei_tmp); + + return result; } /** Return true iff <b>s</b> is a legally valid server nickname. */ @@ -2010,26 +2304,17 @@ router_purpose_from_string(const char *s) void router_free_all(void) { - if (onionkey) - crypto_free_pk_env(onionkey); - if (lastonionkey) - crypto_free_pk_env(lastonionkey); - if (identitykey) - crypto_free_pk_env(identitykey); - if (key_lock) - tor_mutex_free(key_lock); - if (desc_routerinfo) - routerinfo_free(desc_routerinfo); - if (desc_extrainfo) - extrainfo_free(desc_extrainfo); - if (authority_signing_key) - crypto_free_pk_env(authority_signing_key); - if (authority_key_certificate) - authority_cert_free(authority_key_certificate); - if (legacy_signing_key) - crypto_free_pk_env(legacy_signing_key); - if (legacy_key_certificate) - authority_cert_free(legacy_key_certificate); + crypto_free_pk_env(onionkey); + crypto_free_pk_env(lastonionkey); + crypto_free_pk_env(server_identitykey); + crypto_free_pk_env(client_identitykey); + tor_mutex_free(key_lock); + routerinfo_free(desc_routerinfo); + extrainfo_free(desc_extrainfo); + crypto_free_pk_env(authority_signing_key); + authority_cert_free(authority_key_certificate); + crypto_free_pk_env(legacy_signing_key); + authority_cert_free(legacy_key_certificate); if (warned_nonexistent_family) { SMARTLIST_FOREACH(warned_nonexistent_family, char *, cp, tor_free(cp)); diff --git a/src/or/router.h b/src/or/router.h new file mode 100644 index 0000000000..95c0a34e2e --- /dev/null +++ b/src/or/router.h @@ -0,0 +1,104 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file router.h + * \brief Header file for router.c. + **/ + +#ifndef _TOR_ROUTER_H +#define _TOR_ROUTER_H + +crypto_pk_env_t *get_onion_key(void); +time_t get_onion_key_set_at(void); +void set_server_identity_key(crypto_pk_env_t *k); +crypto_pk_env_t *get_server_identity_key(void); +int server_identity_key_is_set(void); +void set_client_identity_key(crypto_pk_env_t *k); +crypto_pk_env_t *get_tlsclient_identity_key(void); +int client_identity_key_is_set(void); +authority_cert_t *get_my_v3_authority_cert(void); +crypto_pk_env_t *get_my_v3_authority_signing_key(void); +authority_cert_t *get_my_v3_legacy_cert(void); +crypto_pk_env_t *get_my_v3_legacy_signing_key(void); +void dup_onion_keys(crypto_pk_env_t **key, crypto_pk_env_t **last); +void rotate_onion_key(void); +crypto_pk_env_t *init_key_from_file(const char *fname, int generate, + int severity); +void v3_authority_check_key_expiry(void); + +int init_keys(void); + +int check_whether_orport_reachable(void); +int check_whether_dirport_reachable(void); +void consider_testing_reachability(int test_or, int test_dir); +void router_orport_found_reachable(void); +void router_dirport_found_reachable(void); +void router_perform_bandwidth_test(int num_circs, time_t now); + +int authdir_mode(or_options_t *options); +int authdir_mode_v1(or_options_t *options); +int authdir_mode_v2(or_options_t *options); +int authdir_mode_v3(or_options_t *options); +int authdir_mode_any_main(or_options_t *options); +int authdir_mode_any_nonhidserv(or_options_t *options); +int authdir_mode_handles_descs(or_options_t *options, int purpose); +int authdir_mode_publishes_statuses(or_options_t *options); +int authdir_mode_tests_reachability(or_options_t *options); +int authdir_mode_bridge(or_options_t *options); + +uint16_t router_get_advertised_or_port(or_options_t *options); +uint16_t router_get_advertised_dir_port(or_options_t *options); + +int server_mode(or_options_t *options); +int public_server_mode(or_options_t *options); +int advertised_server_mode(void); +int proxy_mode(or_options_t *options); +void consider_publishable_server(int force); +int should_refuse_unknown_exits(or_options_t *options); + +void router_upload_dir_desc_to_dirservers(int force); +void mark_my_descriptor_dirty_if_older_than(time_t when); +void mark_my_descriptor_dirty(const char *reason); +void check_descriptor_bandwidth_changed(time_t now); +void check_descriptor_ipaddress_changed(time_t now); +void router_new_address_suggestion(const char *suggestion, + const dir_connection_t *d_conn); +int router_compare_to_my_exit_policy(edge_connection_t *conn); +int router_my_exit_policy_is_reject_star(void); +routerinfo_t *router_get_my_routerinfo(void); +extrainfo_t *router_get_my_extrainfo(void); +const char *router_get_my_descriptor(void); +int router_digest_is_me(const char *digest); +int router_extrainfo_digest_is_me(const char *digest); +int router_is_me(routerinfo_t *router); +int router_fingerprint_is_me(const char *fp); +int router_pick_published_address(or_options_t *options, uint32_t *addr); +int router_rebuild_descriptor(int force); +int router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router, + crypto_pk_env_t *ident_key); +int extrainfo_dump_to_string(char **s, extrainfo_t *extrainfo, + crypto_pk_env_t *ident_key); +int is_legal_nickname(const char *s); +int is_legal_nickname_or_hexdigest(const char *s); +int is_legal_hexdigest(const char *s); +void router_get_verbose_nickname(char *buf, const routerinfo_t *router); +void routerstatus_get_verbose_nickname(char *buf, + const routerstatus_t *router); +void router_reset_warnings(void); +void router_reset_reachability(void); +void router_free_all(void); + +const char *router_purpose_to_string(uint8_t p); +uint8_t router_purpose_from_string(const char *s); + +#ifdef ROUTER_PRIVATE +/* Used only by router.c and test.c */ +void get_platform_str(char *platform, size_t len); +#endif + +#endif + diff --git a/src/or/routerlist.c b/src/or/routerlist.c index be7ba0e965..a3d9b78ee7 100644 --- a/src/or/routerlist.c +++ b/src/or/routerlist.c @@ -12,6 +12,25 @@ **/ #include "or.h" +#include "circuitbuild.h" +#include "config.h" +#include "connection.h" +#include "control.h" +#include "directory.h" +#include "dirserv.h" +#include "dirvote.h" +#include "geoip.h" +#include "hibernate.h" +#include "main.h" +#include "networkstatus.h" +#include "policies.h" +#include "reasons.h" +#include "rendcommon.h" +#include "rendservice.h" +#include "rephist.h" +#include "router.h" +#include "routerlist.h" +#include "routerparse.h" // #define DEBUG_ROUTERLIST @@ -26,8 +45,8 @@ static void mark_all_trusteddirservers_up(void); static int router_nickname_matches(routerinfo_t *router, const char *nickname); static void trusted_dir_server_free(trusted_dir_server_t *ds); static void launch_router_descriptor_downloads(smartlist_t *downloadable, + routerstatus_t *source, time_t now); -static void update_consensus_router_descriptor_downloads(time_t now); static int signed_desc_digest_is_recognized(signed_descriptor_t *desc); static void update_router_have_minimum_dir_info(void); static const char *signed_descriptor_get_body_impl(signed_descriptor_t *desc, @@ -156,21 +175,24 @@ already_have_cert(authority_cert_t *cert) /** Load a bunch of new key certificates from the string <b>contents</b>. If * <b>from_store</b> is true, the certificates are from the cache, and we - * don't need to flush them to disk. If <b>from_store</b> is false, we need - * to flush any changed certificates to disk. Return 0 on success, -1 on - * failure. */ + * don't need to flush them to disk. If <b>flush</b> is true, we need + * to flush any changed certificates to disk now. Return 0 on success, -1 + * if any certs fail to parse. */ int trusted_dirs_load_certs_from_string(const char *contents, int from_store, int flush) { trusted_dir_server_t *ds; const char *s, *eos; + int failure_code = 0; for (s = contents; *s; s = eos) { authority_cert_t *cert = authority_cert_parse_from_string(s, &eos); cert_list_t *cl; - if (!cert) + if (!cert) { + failure_code = -1; break; + } ds = trusteddirserver_get_by_v3_auth_digest( cert->cache_info.identity_digest); log_debug(LD_DIR, "Parsed certificate for %s", @@ -181,15 +203,15 @@ trusted_dirs_load_certs_from_string(const char *contents, int from_store, log_info(LD_DIR, "Skipping %s certificate for %s that we " "already have.", from_store ? "cached" : "downloaded", - ds ? ds->nickname : "??"); + ds ? ds->nickname : "an old or new authority"); /* a duplicate on a download should be treated as a failure, since it * probably means we wanted a different secret key or we are trying to * replace an expired cert that has not in fact been updated. */ if (!from_store) { - log_warn(LD_DIR, "Got a certificate for %s that we already have. " - "Maybe they haven't updated it. Waiting for a while.", - ds ? ds->nickname : "??"); + log_warn(LD_DIR, "Got a certificate for %s, but we already have it. " + "Maybe they haven't updated it. Waiting for a while.", + ds ? ds->nickname : "an old or new authority"); authority_cert_dl_failed(cert->cache_info.identity_digest, 404); } @@ -224,7 +246,7 @@ trusted_dirs_load_certs_from_string(const char *contents, int from_store, ds->dir_port != cert->dir_port)) { char *a = tor_dup_ip(cert->addr); log_notice(LD_DIR, "Updating address for directory authority %s " - "from %s:%d to %s:%d based on in certificate.", + "from %s:%d to %s:%d based on certificate.", ds->nickname, ds->address, (int)ds->dir_port, a, cert->dir_port); tor_free(a); @@ -241,8 +263,11 @@ trusted_dirs_load_certs_from_string(const char *contents, int from_store, if (flush) trusted_dirs_flush_certs_to_disk(); + /* call this even if failure_code is <0, since some certs might have + * succeeded. */ networkstatus_note_certs_arrived(); - return 0; + + return failure_code; } /** Save all v3 key certificates to the cached-certs file. */ @@ -303,7 +328,7 @@ trusted_dirs_remove_old_certs(void) time_t cert_published; if (newest == cert) continue; - expired = ftime_definitely_after(now, cert->expires); + expired = now > cert->expires; cert_published = cert->cache_info.published_on; /* Store expired certs for 48 hours after a newer arrives; */ @@ -415,6 +440,23 @@ authority_cert_dl_failed(const char *id_digest, int status) download_status_failed(&cl->dl_status, status); } +/** Return true iff when we've been getting enough failures when trying to + * download the certificate with ID digest <b>id_digest</b> that we're willing + * to start bugging the user about it. */ +int +authority_cert_dl_looks_uncertain(const char *id_digest) +{ +#define N_AUTH_CERT_DL_FAILURES_TO_BUG_USER 2 + cert_list_t *cl; + int n_failures; + if (!trusted_dir_certs || + !(cl = digestmap_get(trusted_dir_certs, id_digest))) + return 0; + + n_failures = download_status_get_n_failures(&cl->dl_status); + return n_failures >= N_AUTH_CERT_DL_FAILURES_TO_BUG_USER; +} + /** How many times will we try to fetch a certificate before giving up? */ #define MAX_CERT_DL_FAILURES 8 @@ -442,17 +484,18 @@ authority_certs_fetch_missing(networkstatus_t *status, time_t now) list_pending_downloads(pending, DIR_PURPOSE_FETCH_CERTIFICATE, "fp/"); if (status) { - SMARTLIST_FOREACH(status->voters, networkstatus_voter_info_t *, voter, - { - if (tor_digest_is_zero(voter->signing_key_digest)) - continue; /* This authority never signed this consensus, so don't - * go looking for a cert with key digest 0000000000. */ - if (!cache && - !trusteddirserver_get_by_v3_auth_digest(voter->identity_digest)) - continue; /* We are not a cache, and we don't know this authority.*/ - cl = get_cert_list(voter->identity_digest); + SMARTLIST_FOREACH_BEGIN(status->voters, networkstatus_voter_info_t *, + voter) { + if (!smartlist_len(voter->sigs)) + continue; /* This authority never signed this consensus, so don't + * go looking for a cert with key digest 0000000000. */ + if (!cache && + !trusteddirserver_get_by_v3_auth_digest(voter->identity_digest)) + continue; /* We are not a cache, and we don't know this authority.*/ + cl = get_cert_list(voter->identity_digest); + SMARTLIST_FOREACH_BEGIN(voter->sigs, document_signature_t *, sig) { cert = authority_cert_get_by_digests(voter->identity_digest, - voter->signing_key_digest); + sig->signing_key_digest); if (cert) { if (now < cert->expires) download_status_reset(&cl->dl_status); @@ -463,37 +506,36 @@ authority_certs_fetch_missing(networkstatus_t *status, time_t now) !digestmap_get(pending, voter->identity_digest)) { log_notice(LD_DIR, "We're missing a certificate from authority " "with signing key %s: launching request.", - hex_str(voter->signing_key_digest, DIGEST_LEN)); - smartlist_add(missing_digests, voter->identity_digest); + hex_str(sig->signing_key_digest, DIGEST_LEN)); + smartlist_add(missing_digests, sig->identity_digest); } - }); + } SMARTLIST_FOREACH_END(sig); + } SMARTLIST_FOREACH_END(voter); } - SMARTLIST_FOREACH(trusted_dir_servers, trusted_dir_server_t *, ds, - { - int found = 0; - if (!(ds->type & V3_AUTHORITY)) - continue; - if (smartlist_digest_isin(missing_digests, ds->v3_identity_digest)) - continue; - cl = get_cert_list(ds->v3_identity_digest); - SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert, - { - if (!ftime_definitely_after(now, cert->expires)) { - /* It's not expired, and we weren't looking for something to - * verify a consensus with. Call it done. */ - download_status_reset(&cl->dl_status); - found = 1; - break; - } - }); - if (!found && - download_status_is_ready(&cl->dl_status, now,MAX_CERT_DL_FAILURES) && - !digestmap_get(pending, ds->v3_identity_digest)) { - log_notice(LD_DIR, "No current certificate known for authority %s; " - "launching request.", ds->nickname); - smartlist_add(missing_digests, ds->v3_identity_digest); + SMARTLIST_FOREACH_BEGIN(trusted_dir_servers, trusted_dir_server_t *, ds) { + int found = 0; + if (!(ds->type & V3_AUTHORITY)) + continue; + if (smartlist_digest_isin(missing_digests, ds->v3_identity_digest)) + continue; + cl = get_cert_list(ds->v3_identity_digest); + SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert, { + if (now < cert->expires) { + /* It's not expired, and we weren't looking for something to + * verify a consensus with. Call it done. */ + download_status_reset(&cl->dl_status); + found = 1; + break; } }); + if (!found && + download_status_is_ready(&cl->dl_status, now,MAX_CERT_DL_FAILURES) && + !digestmap_get(pending, ds->v3_identity_digest)) { + log_info(LD_DIR, "No current certificate known for authority %s; " + "launching request.", ds->nickname); + smartlist_add(missing_digests, ds->v3_identity_digest); + } + } SMARTLIST_FOREACH_END(ds); if (!smartlist_len(missing_digests)) { goto done; @@ -749,8 +791,7 @@ router_rebuild_store(int flags, desc_store_t *store) store->journal_len = 0; store->bytes_dropped = 0; done: - if (signed_descriptors) - smartlist_free(signed_descriptors); + smartlist_free(signed_descriptors); tor_free(fname); tor_free(fname_tmp); if (chunk_list) { @@ -968,8 +1009,9 @@ router_get_trusteddirserver_by_digest(const char *digest) return NULL; } -/** Return the trusted_dir_server_t for the directory authority whose identity - * key hashes to <b>digest</b>, or NULL if no such authority is known. +/** Return the trusted_dir_server_t for the directory authority whose + * v3 identity key hashes to <b>digest</b>, or NULL if no such authority + * is known. */ trusted_dir_server_t * trusteddirserver_get_by_v3_auth_digest(const char *digest) @@ -1028,6 +1070,7 @@ router_pick_trusteddirserver(authority_type_t type, int flags) static routerstatus_t * router_pick_directory_server_impl(authority_type_t type, int flags) { + or_options_t *options = get_options(); routerstatus_t *result; smartlist_t *direct, *tunnel; smartlist_t *trusted_direct, *trusted_tunnel; @@ -1037,10 +1080,13 @@ router_pick_directory_server_impl(authority_type_t type, int flags) int requireother = ! (flags & PDS_ALLOW_SELF); int fascistfirewall = ! (flags & PDS_IGNORE_FASCISTFIREWALL); int prefer_tunnel = (flags & _PDS_PREFER_TUNNELED_DIR_CONNS); + int try_excluding = 1, n_excluded = 0; if (!consensus) return NULL; + retry_without_exclude: + direct = smartlist_create(); tunnel = smartlist_create(); trusted_direct = smartlist_create(); @@ -1072,6 +1118,11 @@ router_pick_directory_server_impl(authority_type_t type, int flags) if ((type & EXTRAINFO_CACHE) && !router_supports_extrainfo(status->identity_digest, 0)) continue; + if (try_excluding && options->ExcludeNodes && + routerset_contains_routerstatus(options->ExcludeNodes, status)) { + ++n_excluded; + continue; + } /* XXXX IP6 proposal 118 */ tor_addr_from_ipv4h(&addr, status->addr); @@ -1089,9 +1140,10 @@ router_pick_directory_server_impl(authority_type_t type, int flags) } SMARTLIST_FOREACH_END(status); if (smartlist_len(tunnel)) { - result = routerstatus_sl_choose_by_bandwidth(tunnel); + result = routerstatus_sl_choose_by_bandwidth(tunnel, WEIGHT_FOR_DIR); } else if (smartlist_len(overloaded_tunnel)) { - result = routerstatus_sl_choose_by_bandwidth(overloaded_tunnel); + result = routerstatus_sl_choose_by_bandwidth(overloaded_tunnel, + WEIGHT_FOR_DIR); } else if (smartlist_len(trusted_tunnel)) { /* FFFF We don't distinguish between trusteds and overloaded trusteds * yet. Maybe one day we should. */ @@ -1099,9 +1151,10 @@ router_pick_directory_server_impl(authority_type_t type, int flags) * is a feature, but it could easily be a bug. -RD */ result = smartlist_choose(trusted_tunnel); } else if (smartlist_len(direct)) { - result = routerstatus_sl_choose_by_bandwidth(direct); + result = routerstatus_sl_choose_by_bandwidth(direct, WEIGHT_FOR_DIR); } else if (smartlist_len(overloaded_direct)) { - result = routerstatus_sl_choose_by_bandwidth(overloaded_direct); + result = routerstatus_sl_choose_by_bandwidth(overloaded_direct, + WEIGHT_FOR_DIR); } else { result = smartlist_choose(trusted_direct); } @@ -1111,6 +1164,15 @@ router_pick_directory_server_impl(authority_type_t type, int flags) smartlist_free(trusted_tunnel); smartlist_free(overloaded_direct); smartlist_free(overloaded_tunnel); + + if (result == NULL && try_excluding && !options->StrictNodes && n_excluded) { + /* If we got no result, and we are excluding nodes, and StrictNodes is + * not set, try again without excluding nodes. */ + try_excluding = 0; + n_excluded = 0; + goto retry_without_exclude; + } + return result; } @@ -1121,6 +1183,7 @@ static routerstatus_t * router_pick_trusteddirserver_impl(authority_type_t type, int flags, int *n_busy_out) { + or_options_t *options = get_options(); smartlist_t *direct, *tunnel; smartlist_t *overloaded_direct, *overloaded_tunnel; routerinfo_t *me = router_get_my_routerinfo(); @@ -1131,10 +1194,13 @@ router_pick_trusteddirserver_impl(authority_type_t type, int flags, const int prefer_tunnel = (flags & _PDS_PREFER_TUNNELED_DIR_CONNS); const int no_serverdesc_fetching =(flags & PDS_NO_EXISTING_SERVERDESC_FETCH); int n_busy = 0; + int try_excluding = 1, n_excluded = 0; if (!trusted_dir_servers) return NULL; + retry_without_exclude: + direct = smartlist_create(); tunnel = smartlist_create(); overloaded_direct = smartlist_create(); @@ -1153,6 +1219,12 @@ router_pick_trusteddirserver_impl(authority_type_t type, int flags, continue; if (requireother && me && router_digest_is_me(d->digest)) continue; + if (try_excluding && options->ExcludeNodes && + routerset_contains_routerstatus(options->ExcludeNodes, + &d->fake_status)) { + ++n_excluded; + continue; + } /* XXXX IP6 proposal 118 */ tor_addr_from_ipv4h(&addr, d->addr); @@ -1199,6 +1271,15 @@ router_pick_trusteddirserver_impl(authority_type_t type, int flags, smartlist_free(tunnel); smartlist_free(overloaded_direct); smartlist_free(overloaded_tunnel); + + if (result == NULL && try_excluding && !options->StrictNodes && n_excluded) { + /* If we got no result, and we are excluding nodes, and StrictNodes is + * not set, try again without excluding nodes. */ + try_excluding = 0; + n_excluded = 0; + goto retry_without_exclude; + } + return result; } @@ -1230,6 +1311,13 @@ mark_all_trusteddirservers_up(void) router_dir_info_changed(); } +/** Return true iff r1 and r2 have the same address and OR port. */ +int +routers_have_same_or_addr(const routerinfo_t *r1, const routerinfo_t *r2) +{ + return r1->addr == r2->addr && r1->or_port == r2->or_port; +} + /** Reset all internal variables used to count failed downloads of network * status objects. */ void @@ -1450,6 +1538,8 @@ routerlist_find_my_routerinfo(void) /** Find a router that's up, that has this IP address, and * that allows exit to this address:port, or return NULL if there * isn't a good one. + * Don't exit enclave to excluded relays -- it wouldn't actually + * hurt anything, but this way there are fewer confused users. */ routerinfo_t * router_find_exact_exit_enclave(const char *address, uint16_t port) @@ -1457,6 +1547,7 @@ router_find_exact_exit_enclave(const char *address, uint16_t port) uint32_t addr; struct in_addr in; tor_addr_t a; + or_options_t *options = get_options(); if (!tor_inet_aton(address, &in)) return NULL; /* it's not an IP already */ @@ -1469,7 +1560,8 @@ router_find_exact_exit_enclave(const char *address, uint16_t port) if (router->addr == addr && router->is_running && compare_tor_addr_to_addr_policy(&a, port, router->exit_policy) == - ADDR_POLICY_ACCEPTED) + ADDR_POLICY_ACCEPTED && + !routerset_contains_router(options->_ExcludeExitNodesUnion, router)) return router; }); return NULL; @@ -1521,6 +1613,29 @@ router_get_advertised_bandwidth_capped(routerinfo_t *router) return result; } +/** When weighting bridges, enforce these values as lower and upper + * bound for believable bandwidth, because there is no way for us + * to verify a bridge's bandwidth currently. */ +#define BRIDGE_MIN_BELIEVABLE_BANDWIDTH 20000 /* 20 kB/sec */ +#define BRIDGE_MAX_BELIEVABLE_BANDWIDTH 100000 /* 100 kB/sec */ + +/** Return the smaller of the router's configured BandwidthRate + * and its advertised capacity, making sure to stay within the + * interval between bridge-min-believe-bw and + * bridge-max-believe-bw. */ +static uint32_t +bridge_get_advertised_bandwidth_bounded(routerinfo_t *router) +{ + uint32_t result = router->bandwidthcapacity; + if (result > router->bandwidthrate) + result = router->bandwidthrate; + if (result > BRIDGE_MAX_BELIEVABLE_BANDWIDTH) + result = BRIDGE_MAX_BELIEVABLE_BANDWIDTH; + else if (result < BRIDGE_MIN_BELIEVABLE_BANDWIDTH) + result = BRIDGE_MIN_BELIEVABLE_BANDWIDTH; + return result; +} + /** Return bw*1000, unless bw*1000 would overflow, in which case return * INT32_MAX. */ static INLINE int32_t @@ -1531,6 +1646,218 @@ kb_to_bytes(uint32_t bw) /** Helper function: * choose a random element of smartlist <b>sl</b>, weighted by + * the advertised bandwidth of each element using the consensus + * bandwidth weights. + * + * If <b>statuses</b> is zero, then <b>sl</b> is a list of + * routerinfo_t's. Otherwise it's a list of routerstatus_t's. + * + * If <b>rule</b>==WEIGHT_FOR_EXIT. we're picking an exit node: consider all + * nodes' bandwidth equally regardless of their Exit status, since there may + * be some in the list because they exit to obscure ports. If + * <b>rule</b>==NO_WEIGHTING, we're picking a non-exit node: weight + * exit-node's bandwidth less depending on the smallness of the fraction of + * Exit-to-total bandwidth. If <b>rule</b>==WEIGHT_FOR_GUARD, we're picking a + * guard node: consider all guard's bandwidth equally. Otherwise, weight + * guards proportionally less. + */ +static void * +smartlist_choose_by_bandwidth_weights(smartlist_t *sl, + bandwidth_weight_rule_t rule, + int statuses) +{ + int64_t weight_scale; + int64_t rand_bw; + double Wg = -1, Wm = -1, We = -1, Wd = -1; + double Wgb = -1, Wmb = -1, Web = -1, Wdb = -1; + double weighted_bw = 0; + double *bandwidths; + double tmp = 0; + unsigned int i; + int have_unknown = 0; /* true iff sl contains element not in consensus. */ + + /* Can't choose exit and guard at same time */ + tor_assert(rule == NO_WEIGHTING || + rule == WEIGHT_FOR_EXIT || + rule == WEIGHT_FOR_GUARD || + rule == WEIGHT_FOR_MID || + rule == WEIGHT_FOR_DIR); + + if (smartlist_len(sl) == 0) { + log_info(LD_CIRC, + "Empty routerlist passed in to consensus weight node " + "selection for rule %s", + bandwidth_weight_rule_to_string(rule)); + return NULL; + } + + weight_scale = circuit_build_times_get_bw_scale(NULL); + + if (rule == WEIGHT_FOR_GUARD) { + Wg = networkstatus_get_bw_weight(NULL, "Wgg", -1); + Wm = networkstatus_get_bw_weight(NULL, "Wgm", -1); /* Bridges */ + We = 0; + Wd = networkstatus_get_bw_weight(NULL, "Wgd", -1); + + Wgb = networkstatus_get_bw_weight(NULL, "Wgb", -1); + Wmb = networkstatus_get_bw_weight(NULL, "Wmb", -1); + Web = networkstatus_get_bw_weight(NULL, "Web", -1); + Wdb = networkstatus_get_bw_weight(NULL, "Wdb", -1); + } else if (rule == WEIGHT_FOR_MID) { + Wg = networkstatus_get_bw_weight(NULL, "Wmg", -1); + Wm = networkstatus_get_bw_weight(NULL, "Wmm", -1); + We = networkstatus_get_bw_weight(NULL, "Wme", -1); + Wd = networkstatus_get_bw_weight(NULL, "Wmd", -1); + + Wgb = networkstatus_get_bw_weight(NULL, "Wgb", -1); + Wmb = networkstatus_get_bw_weight(NULL, "Wmb", -1); + Web = networkstatus_get_bw_weight(NULL, "Web", -1); + Wdb = networkstatus_get_bw_weight(NULL, "Wdb", -1); + } else if (rule == WEIGHT_FOR_EXIT) { + // Guards CAN be exits if they have weird exit policies + // They are d then I guess... + We = networkstatus_get_bw_weight(NULL, "Wee", -1); + Wm = networkstatus_get_bw_weight(NULL, "Wem", -1); /* Odd exit policies */ + Wd = networkstatus_get_bw_weight(NULL, "Wed", -1); + Wg = networkstatus_get_bw_weight(NULL, "Weg", -1); /* Odd exit policies */ + + Wgb = networkstatus_get_bw_weight(NULL, "Wgb", -1); + Wmb = networkstatus_get_bw_weight(NULL, "Wmb", -1); + Web = networkstatus_get_bw_weight(NULL, "Web", -1); + Wdb = networkstatus_get_bw_weight(NULL, "Wdb", -1); + } else if (rule == WEIGHT_FOR_DIR) { + We = networkstatus_get_bw_weight(NULL, "Wbe", -1); + Wm = networkstatus_get_bw_weight(NULL, "Wbm", -1); + Wd = networkstatus_get_bw_weight(NULL, "Wbd", -1); + Wg = networkstatus_get_bw_weight(NULL, "Wbg", -1); + + Wgb = Wmb = Web = Wdb = weight_scale; + } else if (rule == NO_WEIGHTING) { + Wg = Wm = We = Wd = weight_scale; + Wgb = Wmb = Web = Wdb = weight_scale; + } + + if (Wg < 0 || Wm < 0 || We < 0 || Wd < 0 || Wgb < 0 || Wmb < 0 || Wdb < 0 + || Web < 0) { + log_debug(LD_CIRC, + "Got negative bandwidth weights. Defaulting to old selection" + " algorithm."); + return NULL; // Use old algorithm. + } + + Wg /= weight_scale; + Wm /= weight_scale; + We /= weight_scale; + Wd /= weight_scale; + + Wgb /= weight_scale; + Wmb /= weight_scale; + Web /= weight_scale; + Wdb /= weight_scale; + + bandwidths = tor_malloc_zero(sizeof(double)*smartlist_len(sl)); + + // Cycle through smartlist and total the bandwidth. + for (i = 0; i < (unsigned)smartlist_len(sl); ++i) { + int is_exit = 0, is_guard = 0, is_dir = 0, this_bw = 0, is_me = 0; + double weight = 1; + if (statuses) { + routerstatus_t *status = smartlist_get(sl, i); + is_exit = status->is_exit && !status->is_bad_exit; + is_guard = status->is_possible_guard; + is_dir = (status->dir_port != 0); + if (!status->has_bandwidth) { + tor_free(bandwidths); + /* This should never happen, unless all the authorites downgrade + * to 0.2.0 or rogue routerstatuses get inserted into our consensus. */ + log_warn(LD_BUG, + "Consensus is not listing bandwidths. Defaulting back to " + "old router selection algorithm."); + return NULL; + } + this_bw = kb_to_bytes(status->bandwidth); + if (router_digest_is_me(status->identity_digest)) + is_me = 1; + } else { + routerstatus_t *rs; + routerinfo_t *router = smartlist_get(sl, i); + rs = router_get_consensus_status_by_id( + router->cache_info.identity_digest); + is_exit = router->is_exit && !router->is_bad_exit; + is_guard = router->is_possible_guard; + is_dir = (router->dir_port != 0); + if (rs && rs->has_bandwidth) { + this_bw = kb_to_bytes(rs->bandwidth); + } else { /* bridge or other descriptor not in our consensus */ + this_bw = bridge_get_advertised_bandwidth_bounded(router); + have_unknown = 1; + } + if (router_digest_is_me(router->cache_info.identity_digest)) + is_me = 1; + } + if (is_guard && is_exit) { + weight = (is_dir ? Wdb*Wd : Wd); + } else if (is_guard) { + weight = (is_dir ? Wgb*Wg : Wg); + } else if (is_exit) { + weight = (is_dir ? Web*We : We); + } else { // middle + weight = (is_dir ? Wmb*Wm : Wm); + } + + bandwidths[i] = weight*this_bw; + weighted_bw += weight*this_bw; + if (is_me) + sl_last_weighted_bw_of_me = weight*this_bw; + } + + /* XXXX023 this is a kludge to expose these values. */ + sl_last_total_weighted_bw = weighted_bw; + + log_debug(LD_CIRC, "Choosing node for rule %s based on weights " + "Wg=%lf Wm=%lf We=%lf Wd=%lf with total bw %lf", + bandwidth_weight_rule_to_string(rule), + Wg, Wm, We, Wd, weighted_bw); + + /* If there is no bandwidth, choose at random */ + if (DBL_TO_U64(weighted_bw) == 0) { + /* Don't warn when using bridges/relays not in the consensus */ + if (!have_unknown) + log_warn(LD_CIRC, + "Weighted bandwidth is %lf in node selection for rule %s", + weighted_bw, bandwidth_weight_rule_to_string(rule)); + tor_free(bandwidths); + return smartlist_choose(sl); + } + + rand_bw = crypto_rand_uint64(DBL_TO_U64(weighted_bw)); + rand_bw++; /* crypto_rand_uint64() counts from 0, and we need to count + * from 1 below. See bug 1203 for details. */ + + /* Last, count through sl until we get to the element we picked */ + tmp = 0.0; + for (i=0; i < (unsigned)smartlist_len(sl); i++) { + tmp += bandwidths[i]; + if (tmp >= rand_bw) + break; + } + + if (i == (unsigned)smartlist_len(sl)) { + /* This was once possible due to round-off error, but shouldn't be able + * to occur any longer. */ + tor_fragile_assert(); + --i; + log_warn(LD_BUG, "Round-off error in computing bandwidth had an effect on " + " which router we chose. Please tell the developers. " + "%lf " U64_FORMAT " %lf", tmp, U64_PRINTF_ARG(rand_bw), + weighted_bw); + } + tor_free(bandwidths); + return smartlist_get(sl, i); +} + +/** Helper function: + * choose a random element of smartlist <b>sl</b>, weighted by * the advertised bandwidth of each element. * * If <b>statuses</b> is zero, then <b>sl</b> is a list of @@ -1565,11 +1892,24 @@ smartlist_choose_by_bandwidth(smartlist_t *sl, bandwidth_weight_rule_t rule, bitarray_t *guard_bits; int me_idx = -1; + // This function does not support WEIGHT_FOR_DIR + // or WEIGHT_FOR_MID + if (rule == WEIGHT_FOR_DIR || rule == WEIGHT_FOR_MID) { + rule = NO_WEIGHTING; + } + /* Can't choose exit and guard at same time */ tor_assert(rule == NO_WEIGHTING || rule == WEIGHT_FOR_EXIT || rule == WEIGHT_FOR_GUARD); + if (smartlist_len(sl) == 0) { + log_info(LD_CIRC, + "Empty routerlist passed in to old node selection for rule %s", + bandwidth_weight_rule_to_string(rule)); + return NULL; + } + /* First count the total bandwidth weight, and make a list * of each value. <0 means "unknown; no routerinfo." We use the * bits of negative values to remember whether the router was fast (-x)&1 @@ -1594,7 +1934,7 @@ smartlist_choose_by_bandwidth(smartlist_t *sl, bandwidth_weight_rule_t rule, if (status->has_bandwidth) { this_bw = kb_to_bytes(status->bandwidth); } else { /* guess */ - /* XXX022 once consensuses always list bandwidths, we can take + /* XXX023 once consensuses always list bandwidths, we can take * this guessing business out. -RD */ is_known = 0; flags = status->is_fast ? 1 : 0; @@ -1613,14 +1953,14 @@ smartlist_choose_by_bandwidth(smartlist_t *sl, bandwidth_weight_rule_t rule, if (rs && rs->has_bandwidth) { this_bw = kb_to_bytes(rs->bandwidth); } else if (rs) { /* guess; don't trust the descriptor */ - /* XXX022 once consensuses always list bandwidths, we can take + /* XXX023 once consensuses always list bandwidths, we can take * this guessing business out. -RD */ is_known = 0; flags = router->is_fast ? 1 : 0; flags |= is_exit ? 2 : 0; flags |= is_guard ? 4 : 0; } else /* bridge or other descriptor not in our consensus */ - this_bw = router_get_advertised_bandwidth_capped(router); + this_bw = bridge_get_advertised_bandwidth_bounded(router); } if (is_exit) bitarray_set(exit_bits, i); @@ -1628,6 +1968,8 @@ smartlist_choose_by_bandwidth(smartlist_t *sl, bandwidth_weight_rule_t rule, bitarray_set(guard_bits, i); if (is_known) { bandwidths[i] = (int32_t) this_bw; // safe since MAX_BELIEVABLE<INT32_MAX + // XXX this is no longer true! We don't always cap the bw anymore. Can + // a consensus make us overflow?-sh tor_assert(bandwidths[i] >= 0); if (is_guard) total_guard_bw += this_bw; @@ -1691,12 +2033,12 @@ smartlist_choose_by_bandwidth(smartlist_t *sl, bandwidth_weight_rule_t rule, * For detailed derivation of this formula, see * http://archives.seul.org/or/dev/Jul-2007/msg00056.html */ - if (rule == WEIGHT_FOR_EXIT) + if (rule == WEIGHT_FOR_EXIT || !total_exit_bw) exit_weight = 1.0; else exit_weight = 1.0 - all_bw/(3.0*exit_bw); - if (rule == WEIGHT_FOR_GUARD) + if (rule == WEIGHT_FOR_GUARD || !total_guard_bw) guard_weight = 1.0; else guard_weight = 1.0 - all_bw/(3.0*guard_bw); @@ -1727,7 +2069,7 @@ smartlist_choose_by_bandwidth(smartlist_t *sl, bandwidth_weight_rule_t rule, } } - /* XXXX022 this is a kludge to expose these values. */ + /* XXXX023 this is a kludge to expose these values. */ sl_last_total_weighted_bw = total_bw; log_debug(LD_CIRC, "Total weighted bw = "U64_FORMAT @@ -1745,6 +2087,8 @@ smartlist_choose_by_bandwidth(smartlist_t *sl, bandwidth_weight_rule_t rule, /* Almost done: choose a random value from the bandwidth weights. */ rand_bw = crypto_rand_uint64(total_bw); + rand_bw++; /* crypto_rand_uint64() counts from 0, and we need to count + * from 1 below. See bug 1203 for details. */ /* Last, count through sl until we get to the element we picked */ tmp = 0; @@ -1788,26 +2132,34 @@ routerinfo_t * routerlist_sl_choose_by_bandwidth(smartlist_t *sl, bandwidth_weight_rule_t rule) { - return smartlist_choose_by_bandwidth(sl, rule, 0); + routerinfo_t *ret; + if ((ret = smartlist_choose_by_bandwidth_weights(sl, rule, 0))) { + return ret; + } else { + return smartlist_choose_by_bandwidth(sl, rule, 0); + } } /** Choose a random element of status list <b>sl</b>, weighted by * the advertised bandwidth of each status. */ routerstatus_t * -routerstatus_sl_choose_by_bandwidth(smartlist_t *sl) +routerstatus_sl_choose_by_bandwidth(smartlist_t *sl, + bandwidth_weight_rule_t rule) { /* We are choosing neither exit nor guard here. Weight accordingly. */ - return smartlist_choose_by_bandwidth(sl, NO_WEIGHTING, 1); + routerstatus_t *ret; + if ((ret = smartlist_choose_by_bandwidth_weights(sl, rule, 1))) { + return ret; + } else { + return smartlist_choose_by_bandwidth(sl, rule, 1); + } } -/** Return a random running router from the routerlist. If any node - * named in <b>preferred</b> is available, pick one of those. Never +/** Return a random running router from the routerlist. Never * pick a node whose routerinfo is in * <b>excludedsmartlist</b>, or whose routerinfo matches <b>excludedset</b>, - * even if they are the only nodes - * available. If <b>CRN_STRICT_PREFERRED</b> is set in flags, never pick - * any node besides those in <b>preferred</b>. + * even if they are the only nodes available. * If <b>CRN_NEED_UPTIME</b> is set in flags and any router has more than * a minimum uptime, return one of those. * If <b>CRN_NEED_CAPACITY</b> is set in flags, weight your choice by the @@ -1820,8 +2172,7 @@ routerstatus_sl_choose_by_bandwidth(smartlist_t *sl) * node (that is, possibly discounting exit nodes). */ routerinfo_t * -router_choose_random_node(const char *preferred, - smartlist_t *excludedsmartlist, +router_choose_random_node(smartlist_t *excludedsmartlist, routerset_t *excludedset, router_crn_flags_t flags) { @@ -1829,18 +2180,16 @@ router_choose_random_node(const char *preferred, const int need_capacity = (flags & CRN_NEED_CAPACITY) != 0; const int need_guard = (flags & CRN_NEED_GUARD) != 0; const int allow_invalid = (flags & CRN_ALLOW_INVALID) != 0; - const int strict = (flags & CRN_STRICT_PREFERRED) != 0; const int weight_for_exit = (flags & CRN_WEIGHT_AS_EXIT) != 0; - smartlist_t *sl, *excludednodes; + smartlist_t *sl=smartlist_create(), + *excludednodes=smartlist_create(); routerinfo_t *choice = NULL, *r; bandwidth_weight_rule_t rule; tor_assert(!(weight_for_exit && need_guard)); rule = weight_for_exit ? WEIGHT_FOR_EXIT : - (need_guard ? WEIGHT_FOR_GUARD : NO_WEIGHTING); - - excludednodes = smartlist_create(); + (need_guard ? WEIGHT_FOR_GUARD : WEIGHT_FOR_MID); /* Exclude relays that allow single hop exit circuits, if the user * wants to (such relays might be risky) */ @@ -1857,60 +2206,35 @@ router_choose_random_node(const char *preferred, routerlist_add_family(excludednodes, r); } - /* Try the preferred nodes first. Ignore need_uptime and need_capacity - * and need_guard, since the user explicitly asked for these nodes. */ - if (preferred) { - sl = smartlist_create(); - add_nickname_list_to_smartlist(sl,preferred,1); - smartlist_subtract(sl,excludednodes); - if (excludedsmartlist) - smartlist_subtract(sl,excludedsmartlist); - if (excludedset) - routerset_subtract_routers(sl,excludedset); - choice = smartlist_choose(sl); - smartlist_free(sl); - } - if (!choice && !strict) { - /* Then give up on our preferred choices: any node - * will do that has the required attributes. */ - sl = smartlist_create(); - router_add_running_routers_to_smartlist(sl, allow_invalid, - need_uptime, need_capacity, - need_guard); - smartlist_subtract(sl,excludednodes); - if (excludedsmartlist) - smartlist_subtract(sl,excludedsmartlist); - if (excludedset) - routerset_subtract_routers(sl,excludedset); - - if (need_capacity || need_guard) - choice = routerlist_sl_choose_by_bandwidth(sl, rule); - else - choice = smartlist_choose(sl); - - smartlist_free(sl); - if (!choice && (need_uptime || need_capacity || need_guard)) { - /* try once more -- recurse but with fewer restrictions. */ - log_info(LD_CIRC, - "We couldn't find any live%s%s%s routers; falling back " - "to list of all routers.", - need_capacity?", fast":"", - need_uptime?", stable":"", - need_guard?", guard":""); - flags &= ~ (CRN_NEED_UPTIME|CRN_NEED_CAPACITY|CRN_NEED_GUARD); - choice = router_choose_random_node( - NULL, excludedsmartlist, excludedset, flags); - } + router_add_running_routers_to_smartlist(sl, allow_invalid, + need_uptime, need_capacity, + need_guard); + smartlist_subtract(sl,excludednodes); + if (excludedsmartlist) + smartlist_subtract(sl,excludedsmartlist); + if (excludedset) + routerset_subtract_routers(sl,excludedset); + + // Always weight by bandwidth + choice = routerlist_sl_choose_by_bandwidth(sl, rule); + + smartlist_free(sl); + if (!choice && (need_uptime || need_capacity || need_guard)) { + /* try once more -- recurse but with fewer restrictions. */ + log_info(LD_CIRC, + "We couldn't find any live%s%s%s routers; falling back " + "to list of all routers.", + need_capacity?", fast":"", + need_uptime?", stable":"", + need_guard?", guard":""); + flags &= ~ (CRN_NEED_UPTIME|CRN_NEED_CAPACITY|CRN_NEED_GUARD); + choice = router_choose_random_node( + excludedsmartlist, excludedset, flags); } smartlist_free(excludednodes); if (!choice) { - if (strict) { - log_warn(LD_CIRC, "All preferred nodes were down when trying to choose " - "node, and the Strict[...]Nodes option is set. Failing."); - } else { - log_warn(LD_CIRC, - "No available nodes when trying to choose node. Failing."); - } + log_warn(LD_CIRC, + "No available nodes when trying to choose node. Failing."); } return choice; } @@ -2367,6 +2691,9 @@ extrainfo_free(extrainfo_t *extrainfo) static void signed_descriptor_free(signed_descriptor_t *sd) { + if (!sd) + return; + tor_free(sd->signed_descriptor_body); /* XXXX remove this once more bugs go away. */ @@ -2374,12 +2701,15 @@ signed_descriptor_free(signed_descriptor_t *sd) tor_free(sd); } -/** Extract a signed_descriptor_t from a routerinfo, and free the routerinfo. +/** Extract a signed_descriptor_t from a general routerinfo, and free the + * routerinfo. */ static signed_descriptor_t * signed_descriptor_from_routerinfo(routerinfo_t *ri) { - signed_descriptor_t *sd = tor_malloc_zero(sizeof(signed_descriptor_t)); + signed_descriptor_t *sd; + tor_assert(ri->purpose == ROUTER_PURPOSE_GENERAL); + sd = tor_malloc_zero(sizeof(signed_descriptor_t)); memcpy(sd, &(ri->cache_info), sizeof(signed_descriptor_t)); sd->routerlist_index = -1; ri->cache_info.signed_descriptor_body = NULL; @@ -2398,7 +2728,8 @@ _extrainfo_free(void *e) void routerlist_free(routerlist_t *rl) { - tor_assert(rl); + if (!rl) + return; rimap_free(rl->identity_map, NULL); sdmap_free(rl->desc_digest_map, NULL); sdmap_free(rl->desc_by_eid_map, NULL); @@ -2437,46 +2768,6 @@ dump_routerlist_mem_usage(int severity) "In %d old descriptors: "U64_FORMAT" bytes.", smartlist_len(routerlist->routers), U64_PRINTF_ARG(livedescs), smartlist_len(routerlist->old_routers), U64_PRINTF_ARG(olddescs)); - -#if 0 - { - const smartlist_t *networkstatus_v2_list = networkstatus_get_v2_list(); - networkstatus_t *consensus = networkstatus_get_latest_consensus(); - log(severity, LD_DIR, "Now let's look through old_descriptors!"); - SMARTLIST_FOREACH(routerlist->old_routers, signed_descriptor_t *, sd, { - int in_v2 = 0; - int in_v3 = 0; - char published[ISO_TIME_LEN+1]; - char last_valid_until[ISO_TIME_LEN+1]; - char last_served_at[ISO_TIME_LEN+1]; - char id[HEX_DIGEST_LEN+1]; - routerstatus_t *rs; - format_iso_time(published, sd->published_on); - format_iso_time(last_valid_until, sd->last_listed_as_valid_until); - format_iso_time(last_served_at, sd->last_served_at); - base16_encode(id, sizeof(id), sd->identity_digest, DIGEST_LEN); - SMARTLIST_FOREACH(networkstatus_v2_list, networkstatus_v2_t *, ns, - { - rs = networkstatus_v2_find_entry(ns, sd->identity_digest); - if (rs && tor_memeq(rs->descriptor_digest, - sd->signed_descriptor_digest, DIGEST_LEN)) { - in_v2 = 1; break; - } - }); - if (consensus) { - rs = networkstatus_vote_find_entry(consensus, sd->identity_digest); - if (rs && tor_memeq(rs->descriptor_digest, - sd->signed_descriptor_digest, DIGEST_LEN)) - in_v3 = 1; - } - log(severity, LD_DIR, - "Old descriptor for %s (published %s) %sin v2 ns, %sin v3 " - "consensus. Last valid until %s; last served at %s.", - id, published, in_v2 ? "" : "not ", in_v3 ? "" : "not ", - last_valid_until, last_served_at); - }); - } -#endif } /** Debugging helper: If <b>idx</b> is nonnegative, assert that <b>ri</b> is @@ -2509,6 +2800,7 @@ static void routerlist_insert(routerlist_t *rl, routerinfo_t *ri) { routerinfo_t *ri_old; + signed_descriptor_t *sd_old; { /* XXXX Remove if this slows us down. */ routerinfo_t *ri_generated = router_get_my_routerinfo(); @@ -2518,8 +2810,16 @@ routerlist_insert(routerlist_t *rl, routerinfo_t *ri) ri_old = rimap_set(rl->identity_map, ri->cache_info.identity_digest, ri); tor_assert(!ri_old); - sdmap_set(rl->desc_digest_map, ri->cache_info.signed_descriptor_digest, - &(ri->cache_info)); + + sd_old = sdmap_set(rl->desc_digest_map, + ri->cache_info.signed_descriptor_digest, + &(ri->cache_info)); + if (sd_old) { + rl->desc_store.bytes_dropped += sd_old->signed_descriptor_len; + sdmap_remove(rl->desc_by_eid_map, sd_old->extra_info_digest); + signed_descriptor_free(sd_old); + } + if (!tor_digest_is_zero(ri->cache_info.extra_info_digest)) sdmap_set(rl->desc_by_eid_map, ri->cache_info.extra_info_digest, &ri->cache_info); @@ -2738,6 +3038,7 @@ routerlist_replace(routerlist_t *rl, routerinfo_t *ri_old, routerinfo_t *ri_new) { int idx; + int same_descriptors; routerinfo_t *ri_tmp; extrainfo_t *ei_tmp; @@ -2782,8 +3083,15 @@ routerlist_replace(routerlist_t *rl, routerinfo_t *ri_old, &ri_new->cache_info); } + same_descriptors = tor_memeq(ri_old->cache_info.signed_descriptor_digest, + ri_new->cache_info.signed_descriptor_digest, + DIGEST_LEN); + if (should_cache_old_descriptors() && - ri_old->purpose == ROUTER_PURPOSE_GENERAL) { + ri_old->purpose == ROUTER_PURPOSE_GENERAL && + !same_descriptors) { + /* ri_old is going to become a signed_descriptor_t and go into + * old_routers */ signed_descriptor_t *sd = signed_descriptor_from_routerinfo(ri_old); smartlist_add(rl->old_routers, sd); sd->routerlist_index = smartlist_len(rl->old_routers)-1; @@ -2791,24 +3099,27 @@ routerlist_replace(routerlist_t *rl, routerinfo_t *ri_old, if (!tor_digest_is_zero(sd->extra_info_digest)) sdmap_set(rl->desc_by_eid_map, sd->extra_info_digest, sd); } else { - if (tor_memneq(ri_old->cache_info.signed_descriptor_digest, - ri_new->cache_info.signed_descriptor_digest, - DIGEST_LEN)) { - /* digests don't match; digestmap_set didn't replace */ + /* We're dropping ri_old. */ + if (!same_descriptors) { + /* digests don't match; The sdmap_set above didn't replace */ sdmap_remove(rl->desc_digest_map, ri_old->cache_info.signed_descriptor_digest); - } - ei_tmp = eimap_remove(rl->extra_info_map, - ri_old->cache_info.extra_info_digest); - if (ei_tmp) { - rl->extrainfo_store.bytes_dropped += - ei_tmp->cache_info.signed_descriptor_len; - extrainfo_free(ei_tmp); - } - if (!tor_digest_is_zero(ri_old->cache_info.extra_info_digest)) { - sdmap_remove(rl->desc_by_eid_map, - ri_old->cache_info.extra_info_digest); + if (tor_memneq(ri_old->cache_info.extra_info_digest, + ri_new->cache_info.extra_info_digest, DIGEST_LEN)) { + ei_tmp = eimap_remove(rl->extra_info_map, + ri_old->cache_info.extra_info_digest); + if (ei_tmp) { + rl->extrainfo_store.bytes_dropped += + ei_tmp->cache_info.signed_descriptor_len; + extrainfo_free(ei_tmp); + } + } + + if (!tor_digest_is_zero(ri_old->cache_info.extra_info_digest)) { + sdmap_remove(rl->desc_by_eid_map, + ri_old->cache_info.extra_info_digest); + } } rl->desc_store.bytes_dropped += ri_old->cache_info.signed_descriptor_len; routerinfo_free(ri_old); @@ -2846,8 +3157,7 @@ routerlist_reparse_old(routerlist_t *rl, signed_descriptor_t *sd) void routerlist_free_all(void) { - if (routerlist) - routerlist_free(routerlist); + routerlist_free(routerlist); routerlist = NULL; if (warned_nicknames) { SMARTLIST_FOREACH(warned_nicknames, char *, cp, tor_free(cp)); @@ -2940,7 +3250,8 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg, int from_cache, int from_fetch) { const char *id_digest; - int authdir = authdir_mode_handles_descs(get_options(), router->purpose); + or_options_t *options = get_options(); + int authdir = authdir_mode_handles_descs(options, router->purpose); int authdir_believes_valid = 0; routerinfo_t *old_router; networkstatus_t *consensus = networkstatus_get_latest_consensus(); @@ -2954,15 +3265,33 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg, id_digest = router->cache_info.identity_digest; + old_router = router_get_by_digest(id_digest); + /* Make sure that we haven't already got this exact descriptor. */ if (sdmap_get(routerlist->desc_digest_map, router->cache_info.signed_descriptor_digest)) { - log_info(LD_DIR, - "Dropping descriptor that we already have for router '%s'", - router->nickname); - *msg = "Router descriptor was not new."; - routerinfo_free(router); - return ROUTER_WAS_NOT_NEW; + /* If we have this descriptor already and the new descriptor is a bridge + * descriptor, replace it. If we had a bridge descriptor before and the + * new one is not a bridge descriptor, don't replace it. */ + + /* Only members of routerlist->identity_map can be bridges; we don't + * put bridges in old_routers. */ + const int was_bridge = old_router && + old_router->purpose == ROUTER_PURPOSE_BRIDGE; + + if (routerinfo_is_a_configured_bridge(router) && + router->purpose == ROUTER_PURPOSE_BRIDGE && + !was_bridge) { + log_info(LD_DIR, "Replacing non-bridge descriptor with bridge " + "descriptor for router '%s'", router->nickname); + } else { + log_info(LD_DIR, + "Dropping descriptor that we already have for router '%s'", + router->nickname); + *msg = "Router descriptor was not new."; + routerinfo_free(router); + return ROUTER_WAS_NOT_NEW; + } } if (authdir) { @@ -2999,15 +3328,14 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg, SMARTLIST_FOREACH(networkstatus_v2_list, networkstatus_v2_t *, ns, { routerstatus_t *rs = - networkstatus_v2_find_entry(ns, router->cache_info.identity_digest); + networkstatus_v2_find_entry(ns, id_digest); if (rs && tor_memeq(rs->descriptor_digest, router->cache_info.signed_descriptor_digest, DIGEST_LEN)) rs->need_to_mirror = 0; }); if (consensus) { - routerstatus_t *rs = networkstatus_vote_find_entry(consensus, - router->cache_info.identity_digest); + routerstatus_t *rs = networkstatus_vote_find_entry(consensus, id_digest); if (rs && tor_memeq(rs->descriptor_digest, router->cache_info.signed_descriptor_digest, DIGEST_LEN)) { @@ -3028,14 +3356,26 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg, return ROUTER_NOT_IN_CONSENSUS; } + /* If we're reading a bridge descriptor from our cache, and we don't + * recognize it as one of our currently configured bridges, drop the + * descriptor. Otherwise we could end up using it as one of our entry + * guards even if it isn't in our Bridge config lines. */ + if (router->purpose == ROUTER_PURPOSE_BRIDGE && from_cache && + !authdir_mode_bridge(options) && + !routerinfo_is_a_configured_bridge(router)) { + log_info(LD_DIR, "Dropping bridge descriptor for '%s' because we have " + "no bridge configured at that address.", router->nickname); + *msg = "Router descriptor was not a configured bridge."; + routerinfo_free(router); + return ROUTER_WAS_NOT_WANTED; + } + /* If we have a router with the same identity key, choose the newer one. */ - old_router = rimap_get(routerlist->identity_map, - router->cache_info.identity_digest); if (old_router) { if (!in_consensus && (router->cache_info.published_on <= old_router->cache_info.published_on)) { /* Same key, but old. This one is not listed in the consensus. */ - log_debug(LD_DIR, "Skipping not-new descriptor for router '%s'", + log_debug(LD_DIR, "Not-new descriptor for router '%s'", router->nickname); /* Only journal this desc if we'll be serving it. */ if (!from_cache && should_cache_old_descriptors()) @@ -3049,8 +3389,7 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg, log_debug(LD_DIR, "Replacing entry for router '%s/%s' [%s]", router->nickname, old_router->nickname, hex_str(id_digest,DIGEST_LEN)); - if (router->addr == old_router->addr && - router->or_port == old_router->or_port) { + if (routers_have_same_or_addr(router, old_router)) { /* these carry over when the address and orport are unchanged. */ router->last_reachable = old_router->last_reachable; router->testing_since = old_router->testing_since; @@ -3078,9 +3417,10 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg, /* We haven't seen a router with this identity before. Add it to the end of * the list. */ routerlist_insert(routerlist, router); - if (!from_cache) + if (!from_cache) { signed_desc_append_to_journal(&router->cache_info, &routerlist->desc_store); + } directory_set_dirty(); return ROUTER_ADDED_SUCCESSFULLY; } @@ -3096,7 +3436,7 @@ router_add_extrainfo_to_routerlist(extrainfo_t *ei, const char **msg, int inserted; (void)from_fetch; if (msg) *msg = NULL; - /*XXXX022 Do something with msg */ + /*XXXX023 Do something with msg */ inserted = extrainfo_insert(router_get_routerlist(), ei); @@ -3397,15 +3737,19 @@ routerlist_remove_old_routers(void) /** We just added a new set of descriptors. Take whatever extra steps * we need. */ -static void +void routerlist_descriptors_added(smartlist_t *sl, int from_cache) { tor_assert(sl); control_event_descriptors_changed(sl); - SMARTLIST_FOREACH(sl, routerinfo_t *, ri, + SMARTLIST_FOREACH_BEGIN(sl, routerinfo_t *, ri) { if (ri->purpose == ROUTER_PURPOSE_BRIDGE) learned_bridge_descriptor(ri, from_cache); - ); + if (ri->needs_retest_if_added) { + ri->needs_retest_if_added = 0; + dirserv_single_reachability_test(approx_time(), ri); + } + } SMARTLIST_FOREACH_END(ri); } /** @@ -3736,12 +4080,8 @@ add_trusted_dir_server(const char *nickname, const char *address, if (ent->or_port) ent->fake_status.version_supports_begindir = 1; -/* XX021 - wait until authorities are upgraded */ -#if 0 + ent->fake_status.version_supports_conditional_consensus = 1; -#else - ent->fake_status.version_supports_conditional_consensus = 0; -#endif smartlist_add(trusted_dir_servers, ent); router_dir_info_changed(); @@ -3756,10 +4096,8 @@ authority_cert_free(authority_cert_t *cert) return; tor_free(cert->cache_info.signed_descriptor_body); - if (cert->signing_key) - crypto_free_pk_env(cert->signing_key); - if (cert->identity_key) - crypto_free_pk_env(cert->identity_key); + crypto_free_pk_env(cert->signing_key); + crypto_free_pk_env(cert->identity_key); tor_free(cert); } @@ -3768,6 +4106,9 @@ authority_cert_free(authority_cert_t *cert) static void trusted_dir_server_free(trusted_dir_server_t *ds) { + if (!ds) + return; + tor_free(ds->nickname); tor_free(ds->description); tor_free(ds->address); @@ -3821,7 +4162,7 @@ list_pending_downloads(digestmap_t *result, const char *resource = TO_DIR_CONN(conn)->requested_resource; if (!strcmpstart(resource, prefix)) dir_split_resource_into_fingerprints(resource + p_len, - tmp, NULL, 1, 0); + tmp, NULL, DSR_HEX); } }); SMARTLIST_FOREACH(tmp, char *, d, @@ -3923,7 +4264,7 @@ client_would_use_router(routerstatus_t *rs, time_t now, or_options_t *options) * this number per server. */ #define MIN_DL_PER_REQUEST 4 /** To prevent a single screwy cache from confusing us by selective reply, - * try to split our requests into at least this this many requests. */ + * try to split our requests into at least this many requests. */ #define MIN_REQUESTS 3 /** If we want fewer than this many descriptors, wait until we * want more, or until MAX_CLIENT_INTERVAL_WITHOUT_REQUEST has @@ -3937,7 +4278,8 @@ client_would_use_router(routerstatus_t *rs, time_t now, or_options_t *options) * whether to delay fetching until we have more. If we don't want to delay, * launch one or more requests to the appropriate directory authorities. */ static void -launch_router_descriptor_downloads(smartlist_t *downloadable, time_t now) +launch_router_descriptor_downloads(smartlist_t *downloadable, + routerstatus_t *source, time_t now) { int should_delay = 0, n_downloadable; or_options_t *options = get_options(); @@ -3990,7 +4332,7 @@ launch_router_descriptor_downloads(smartlist_t *downloadable, time_t now) pds_flags |= PDS_NO_EXISTING_SERVERDESC_FETCH; } - n_per_request = (n_downloadable+MIN_REQUESTS-1) / MIN_REQUESTS; + n_per_request = CEIL_DIV(n_downloadable, MIN_REQUESTS); if (n_per_request > MAX_DL_PER_REQUEST) n_per_request = MAX_DL_PER_REQUEST; if (n_per_request < MIN_DL_PER_REQUEST) @@ -4003,11 +4345,11 @@ launch_router_descriptor_downloads(smartlist_t *downloadable, time_t now) log_info(LD_DIR, "Launching %d request%s for %d router%s, %d at a time", - (n_downloadable+n_per_request-1)/n_per_request, + CEIL_DIV(n_downloadable, n_per_request), req_plural, n_downloadable, rtr_plural, n_per_request); smartlist_sort_digests(downloadable); for (i=0; i < n_downloadable; i += n_per_request) { - initiate_descriptor_downloads(NULL, DIR_PURPOSE_FETCH_SERVERDESC, + initiate_descriptor_downloads(source, DIR_PURPOSE_FETCH_SERVERDESC, downloadable, i, i+n_per_request, pds_flags); } @@ -4163,18 +4505,18 @@ update_router_descriptor_cache_downloads_v2(time_t now) digestmap_free(map,NULL); } -/** For any descriptor that we want that's currently listed in the live - * consensus, download it as appropriate. */ -static void -update_consensus_router_descriptor_downloads(time_t now) +/** For any descriptor that we want that's currently listed in + * <b>consensus</b>, download it as appropriate. */ +void +update_consensus_router_descriptor_downloads(time_t now, int is_vote, + networkstatus_t *consensus) { or_options_t *options = get_options(); digestmap_t *map = NULL; smartlist_t *no_longer_old = smartlist_create(); smartlist_t *downloadable = smartlist_create(); + routerstatus_t *source = NULL; int authdir = authdir_mode(options); - networkstatus_t *consensus = - networkstatus_get_reasonably_live_consensus(now); int n_delayed=0, n_have=0, n_would_reject=0, n_wouldnt_use=0, n_inprogress=0, n_in_oldrouters=0; @@ -4183,10 +4525,24 @@ update_consensus_router_descriptor_downloads(time_t now) if (!consensus) goto done; + if (is_vote) { + /* where's it from, so we know whom to ask for descriptors */ + trusted_dir_server_t *ds; + networkstatus_voter_info_t *voter = smartlist_get(consensus->voters, 0); + tor_assert(voter); + ds = trusteddirserver_get_by_v3_auth_digest(voter->identity_digest); + if (ds) + source = &(ds->fake_status); + else + log_warn(LD_DIR, "couldn't lookup source from vote?"); + } + map = digestmap_new(); list_pending_descriptor_downloads(map, 0); - SMARTLIST_FOREACH(consensus->routerstatus_list, routerstatus_t *, rs, + SMARTLIST_FOREACH(consensus->routerstatus_list, void *, rsp, { + routerstatus_t *rs = + is_vote ? &(((vote_routerstatus_t *)rsp)->status) : rsp; signed_descriptor_t *sd; if ((sd = router_get_by_descriptor_digest(rs->descriptor_digest))) { routerinfo_t *ri; @@ -4221,6 +4577,18 @@ update_consensus_router_descriptor_downloads(time_t now) ++n_wouldnt_use; continue; /* We would never use it ourself. */ } + if (is_vote && source) { + char time_bufnew[ISO_TIME_LEN+1]; + char time_bufold[ISO_TIME_LEN+1]; + routerinfo_t *oldrouter = router_get_by_digest(rs->identity_digest); + format_iso_time(time_bufnew, rs->published_on); + if (oldrouter) + format_iso_time(time_bufold, oldrouter->cache_info.published_on); + log_info(LD_DIR, "Learned about %s (%s vs %s) from %s's vote (%s)", + rs->nickname, time_bufnew, + oldrouter ? time_bufold : "none", + source->nickname, oldrouter ? "known" : "unknown"); + } smartlist_add(downloadable, rs->descriptor_digest); }); @@ -4254,7 +4622,7 @@ update_consensus_router_descriptor_downloads(time_t now) smartlist_len(downloadable), n_delayed, n_have, n_in_oldrouters, n_would_reject, n_wouldnt_use, n_inprogress); - launch_router_descriptor_downloads(downloadable, now); + launch_router_descriptor_downloads(downloadable, source, now); digestmap_free(map, NULL); done: @@ -4264,7 +4632,7 @@ update_consensus_router_descriptor_downloads(time_t now) /** How often should we launch a server/authority request to be sure of getting * a guess for our IP? */ -/*XXXX021 this info should come from netinfo cells or something, or we should +/*XXXX023 this info should come from netinfo cells or something, or we should * do this only when we aren't seeing incoming data. see bug 652. */ #define DUMMY_DOWNLOAD_INTERVAL (20*60) @@ -4279,9 +4647,10 @@ update_router_descriptor_downloads(time_t now) if (directory_fetches_dir_info_early(options)) { update_router_descriptor_cache_downloads_v2(now); } - update_consensus_router_descriptor_downloads(now); + update_consensus_router_descriptor_downloads(now, 0, + networkstatus_get_reasonably_live_consensus(now)); - /* XXXX021 we could be smarter here; see notes on bug 652. */ + /* XXXX023 we could be smarter here; see notes on bug 652. */ /* If we're a server that doesn't have a configured address, we rely on * directory fetches to learn when our address changes. So if we haven't * tried to get any routerdescs in a long time, try a dummy fetch now. */ @@ -4416,16 +4785,21 @@ get_dir_info_status_string(void) /** Iterate over the servers listed in <b>consensus</b>, and count how many of * them seem like ones we'd use, and how many of <em>those</em> we have * descriptors for. Store the former in *<b>num_usable</b> and the latter in - * *<b>num_present</b>. */ + * *<b>num_present</b>. If <b>in_set</b> is non-NULL, only consider those + * routers in <b>in_set</b>. + */ static void count_usable_descriptors(int *num_present, int *num_usable, const networkstatus_t *consensus, - or_options_t *options, time_t now) + or_options_t *options, time_t now, + routerset_t *in_set) { *num_present = 0, *num_usable=0; SMARTLIST_FOREACH(consensus->routerstatus_list, routerstatus_t *, rs, { + if (in_set && ! routerset_contains_routerstatus(in_set, rs)) + continue; if (client_would_use_router(rs, now, options)) { ++*num_usable; /* the consensus says we want it. */ if (router_get_by_descriptor_digest(rs->descriptor_digest)) { @@ -4454,7 +4828,7 @@ count_loading_descriptors_progress(void) return 0; /* can't count descriptors if we have no list of them */ count_usable_descriptors(&num_present, &num_usable, - consensus, get_options(), now); + consensus, get_options(), now, NULL); if (num_usable == 0) return 0; /* don't div by 0 */ @@ -4498,22 +4872,39 @@ update_router_have_minimum_dir_info(void) goto done; } - count_usable_descriptors(&num_present, &num_usable, consensus, options, now); + count_usable_descriptors(&num_present, &num_usable, consensus, options, now, + NULL); if (num_present < num_usable/4) { tor_snprintf(dir_info_status, sizeof(dir_info_status), "We have only %d/%d usable descriptors.", num_present, num_usable); res = 0; control_event_bootstrap(BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS, 0); + goto done; } else if (num_present < 2) { tor_snprintf(dir_info_status, sizeof(dir_info_status), "Only %d descriptor%s here and believed reachable!", num_present, num_present ? "" : "s"); res = 0; - } else { - res = 1; + goto done; + } + + /* Check for entry nodes. */ + if (options->EntryNodes) { + count_usable_descriptors(&num_present, &num_usable, consensus, options, + now, options->EntryNodes); + + if (!num_usable || !num_present) { + tor_snprintf(dir_info_status, sizeof(dir_info_status), + "We have only %d/%d usable entry node descriptors.", + num_present, num_usable); + res = 0; + goto done; + } } + res = 1; + done: if (res && !have_min_dir_info) { log(LOG_NOTICE, LD_DIR, @@ -4526,6 +4917,13 @@ update_router_have_minimum_dir_info(void) log(quiet ? LOG_INFO : LOG_NOTICE, LD_DIR, "Our directory information is no longer up-to-date " "enough to build circuits: %s", dir_info_status); + + /* a) make us log when we next complete a circuit, so we know when Tor + * is back up and usable, and b) disable some activities that Tor + * should only do while circuits are working, like reachability tests + * and fetching bridge descriptors only over circuits. */ + can_complete_circuit = 0; + control_event_client_status(LOG_NOTICE, "NOT_ENOUGH_DIR_INFO"); } have_min_dir_info = res; @@ -4813,8 +5211,8 @@ esc_router_info(routerinfo_t *router) static char *info=NULL; char *esc_contact, *esc_platform; size_t len; - if (info) - tor_free(info); + tor_free(info); + if (!router) return NULL; /* we're exiting; just free the memory we use */ @@ -4949,9 +5347,8 @@ void routerset_refresh_countries(routerset_t *target) { int cc; - if (target->countries) { - bitarray_free(target->countries); - } + bitarray_free(target->countries); + if (!geoip_is_loaded()) { target->countries = NULL; target->n_countries = 0; @@ -5025,7 +5422,9 @@ routerset_parse(routerset_t *target, const char *s, const char *description) return r; } -/** DOCDOC */ +/** Called when we change a node set, or when we reload the geoip list: + * recompute all country info in all configuration node sets and in the + * routerlist. */ void refresh_all_country_info(void) { @@ -5076,7 +5475,7 @@ routerset_needs_geoip(const routerset_t *set) } /** Return true iff there are no entries in <b>set</b>. */ -static int +int routerset_is_empty(const routerset_t *set) { return !set || smartlist_len(set->list) == 0; @@ -5159,10 +5558,11 @@ routerset_contains_routerstatus(const routerset_t *set, routerstatus_t *rs) } /** Add every known routerinfo_t that is a member of <b>routerset</b> to - * <b>out</b>. If <b>running_only</b>, only add the running ones. */ + * <b>out</b>, but never add any that are part of <b>excludeset</b>. + * If <b>running_only</b>, only add the running ones. */ void routerset_get_all_routers(smartlist_t *out, const routerset_t *routerset, - int running_only) + const routerset_t *excludeset, int running_only) { tor_assert(out); if (!routerset || !routerset->list) @@ -5172,12 +5572,13 @@ routerset_get_all_routers(smartlist_t *out, const routerset_t *routerset, if (routerset_is_list(routerset)) { /* No routers are specified by type; all are given by name or digest. - * we can do a lookup in O(len(list)). */ + * we can do a lookup in O(len(routerset)). */ SMARTLIST_FOREACH(routerset->list, const char *, name, { routerinfo_t *router = router_get_by_nickname(name, 1); if (router) { if (!running_only || router->is_running) - smartlist_add(out, router); + if (!routerset_contains_router(excludeset, router)) + smartlist_add(out, router); } }); } else { @@ -5187,15 +5588,21 @@ routerset_get_all_routers(smartlist_t *out, const routerset_t *routerset, SMARTLIST_FOREACH(rl->routers, routerinfo_t *, router, { if (running_only && !router->is_running) continue; - if (routerset_contains_router(routerset, router)) + if (routerset_contains_router(routerset, router) && + !routerset_contains_router(excludeset, router)) smartlist_add(out, router); }); } } -/** Add to <b>target</b> every routerinfo_t from <b>source</b> that is in - * <b>include</b>, but not excluded in a more specific fashion by - * <b>exclude</b>. If <b>running_only</b>, only include running routers. +#if 0 +/** Add to <b>target</b> every routerinfo_t from <b>source</b> except: + * + * 1) Don't add it if <b>include</b> is non-empty and the relay isn't in + * <b>include</b>; and + * 2) Don't add it if <b>exclude</b> is non-empty and the relay is + * excluded in a more specific fashion by <b>exclude</b>. + * 3) If <b>running_only</b>, don't add non-running routers. */ void routersets_get_disjunction(smartlist_t *target, @@ -5219,6 +5626,7 @@ routersets_get_disjunction(smartlist_t *target, } }); } +#endif /** Remove every routerinfo_t from <b>lst</b> that is in <b>routerset</b>. */ void @@ -5250,10 +5658,15 @@ routerset_to_string(const routerset_t *set) int routerset_equal(const routerset_t *old, const routerset_t *new) { - if (old == NULL && new == NULL) + if (routerset_is_empty(old) && routerset_is_empty(new)) { + /* Two empty sets are equal */ return 1; - else if (old == NULL || new == NULL) + } else if (routerset_is_empty(old) || routerset_is_empty(new)) { + /* An empty set is equal to nothing else. */ return 0; + } + tor_assert(old != NULL); + tor_assert(new != NULL); if (smartlist_len(old->list) != smartlist_len(new->list)) return 0; @@ -5265,35 +5678,15 @@ routerset_equal(const routerset_t *old, const routerset_t *new) }); return 1; - -#if 0 - /* XXXX: This won't work if the names/digests are identical but in a - different order. Checking for exact equality would be heavy going, - is it worth it? -RH*/ - /* This code is totally bogus; sizeof doesn't work even remotely like this - * code seems to think. Let's revert to a string-based comparison for - * now. -NM*/ - if (sizeof(old->names) != sizeof(new->names)) - return 0; - - if (tor_memcmp(old->names,new->names,sizeof(new->names))) - return 0; - if (sizeof(old->digests) != sizeof(new->digests)) - return 0; - if (tor_memcmp(old->digests,new->digests,sizeof(new->digests))) - return 0; - if (sizeof(old->countries) != sizeof(new->countries)) - return 0; - if (tor_memcmp(old->countries,new->countries,sizeof(new->countries))) - return 0; - return 1; -#endif } /** Free all storage held in <b>routerset</b>. */ void routerset_free(routerset_t *routerset) { + if (!routerset) + return; + SMARTLIST_FOREACH(routerset->list, char *, cp, tor_free(cp)); smartlist_free(routerset->list); SMARTLIST_FOREACH(routerset->policies, addr_policy_t *, p, @@ -5304,8 +5697,7 @@ routerset_free(routerset_t *routerset) strmap_free(routerset->names, NULL); digestmap_free(routerset->digests, NULL); - if (routerset->countries) - bitarray_free(routerset->countries); + bitarray_free(routerset->countries); tor_free(routerset); } @@ -5336,7 +5728,6 @@ hid_serv_get_responsible_directories(smartlist_t *responsible_dirs, { int start, found, n_added = 0, i; networkstatus_t *c = networkstatus_get_latest_consensus(); - int use_begindir = get_options()->TunnelDirConns; if (!c || !smartlist_len(c->routerstatus_list)) { log_warn(LD_REND, "We don't have a consensus, so we can't perform v2 " "rendezvous operations."); @@ -5349,14 +5740,9 @@ hid_serv_get_responsible_directories(smartlist_t *responsible_dirs, do { routerstatus_t *r = smartlist_get(c->routerstatus_list, i); if (r->is_hs_dir) { - if (r->dir_port || use_begindir) - smartlist_add(responsible_dirs, r); - else - log_info(LD_REND, "Not adding router '%s' to list of responsible " - "hidden service directories, because we have no way of " - "reaching it.", r->nickname); + smartlist_add(responsible_dirs, r); if (++n_added == REND_NUMBER_OF_CONSECUTIVE_REPLICAS) - break; + return 0; } if (++i == smartlist_len(c->routerstatus_list)) i = 0; @@ -5373,8 +5759,6 @@ int hid_serv_acting_as_directory(void) { routerinfo_t *me = router_get_my_routerinfo(); - networkstatus_t *c; - routerstatus_t *rs; if (!me) return 0; if (!get_options()->HidServDirectoryV2) { @@ -5382,22 +5766,6 @@ hid_serv_acting_as_directory(void) "because we have not been configured as such."); return 0; } - if (!(c = networkstatus_get_latest_consensus())) { - log_info(LD_REND, "There's no consensus, so I can't tell if I'm a hidden " - "service directory"); - return 0; - } - rs = networkstatus_vote_find_entry(c, me->cache_info.identity_digest); - if (!rs) { - log_info(LD_REND, "We're not listed in the consensus, so we're not " - "being a hidden service directory."); - return 0; - } - if (!rs->is_hs_dir) { - log_info(LD_REND, "We're not listed as a hidden service directory in " - "the consensus, so we won't be one."); - return 0; - } return 1; } diff --git a/src/or/routerlist.h b/src/or/routerlist.h new file mode 100644 index 0000000000..fec18705b3 --- /dev/null +++ b/src/or/routerlist.h @@ -0,0 +1,200 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file routerlist.h + * \brief Header file for routerlist.c. + **/ + +#ifndef _TOR_ROUTERLIST_H +#define _TOR_ROUTERLIST_H + +int get_n_authorities(authority_type_t type); +int trusted_dirs_reload_certs(void); +int trusted_dirs_load_certs_from_string(const char *contents, int from_store, + int flush); +void trusted_dirs_flush_certs_to_disk(void); +authority_cert_t *authority_cert_get_newest_by_id(const char *id_digest); +authority_cert_t *authority_cert_get_by_sk_digest(const char *sk_digest); +authority_cert_t *authority_cert_get_by_digests(const char *id_digest, + const char *sk_digest); +void authority_cert_get_all(smartlist_t *certs_out); +void authority_cert_dl_failed(const char *id_digest, int status); +void authority_certs_fetch_missing(networkstatus_t *status, time_t now); +int router_reload_router_list(void); +int authority_cert_dl_looks_uncertain(const char *id_digest); +smartlist_t *router_get_trusted_dir_servers(void); + +routerstatus_t *router_pick_directory_server(authority_type_t type, int flags); +trusted_dir_server_t *router_get_trusteddirserver_by_digest(const char *d); +trusted_dir_server_t *trusteddirserver_get_by_v3_auth_digest(const char *d); +routerstatus_t *router_pick_trusteddirserver(authority_type_t type, int flags); +int router_get_my_share_of_directory_requests(double *v2_share_out, + double *v3_share_out); +void router_reset_status_download_failures(void); +void routerlist_add_family(smartlist_t *sl, routerinfo_t *router); +int routers_in_same_family(routerinfo_t *r1, routerinfo_t *r2); +int routers_have_same_or_addr(const routerinfo_t *r1, const routerinfo_t *r2); +void add_nickname_list_to_smartlist(smartlist_t *sl, const char *list, + int must_be_running); +int router_nickname_is_in_list(routerinfo_t *router, const char *list); +routerinfo_t *routerlist_find_my_routerinfo(void); +routerinfo_t *router_find_exact_exit_enclave(const char *address, + uint16_t port); +int router_is_unreliable(routerinfo_t *router, int need_uptime, + int need_capacity, int need_guard); +uint32_t router_get_advertised_bandwidth(routerinfo_t *router); +uint32_t router_get_advertised_bandwidth_capped(routerinfo_t *router); + +routerinfo_t *routerlist_sl_choose_by_bandwidth(smartlist_t *sl, + bandwidth_weight_rule_t rule); +routerstatus_t *routerstatus_sl_choose_by_bandwidth(smartlist_t *sl, + bandwidth_weight_rule_t rule); + +routerinfo_t *router_choose_random_node(smartlist_t *excludedsmartlist, + struct routerset_t *excludedset, + router_crn_flags_t flags); + +routerinfo_t *router_get_by_nickname(const char *nickname, + int warn_if_unnamed); +int router_digest_version_as_new_as(const char *digest, const char *cutoff); +int router_digest_is_trusted_dir_type(const char *digest, + authority_type_t type); +#define router_digest_is_trusted_dir(d) \ + router_digest_is_trusted_dir_type((d), NO_AUTHORITY) + +int router_addr_is_trusted_dir(uint32_t addr); +int hexdigest_to_digest(const char *hexdigest, char *digest); +routerinfo_t *router_get_by_hexdigest(const char *hexdigest); +routerinfo_t *router_get_by_digest(const char *digest); +signed_descriptor_t *router_get_by_descriptor_digest(const char *digest); +signed_descriptor_t *router_get_by_extrainfo_digest(const char *digest); +signed_descriptor_t *extrainfo_get_by_descriptor_digest(const char *digest); +const char *signed_descriptor_get_body(signed_descriptor_t *desc); +const char *signed_descriptor_get_annotations(signed_descriptor_t *desc); +routerlist_t *router_get_routerlist(void); +void routerinfo_free(routerinfo_t *router); +void extrainfo_free(extrainfo_t *extrainfo); +void routerlist_free(routerlist_t *rl); +void dump_routerlist_mem_usage(int severity); +void routerlist_remove(routerlist_t *rl, routerinfo_t *ri, int make_old, + time_t now); +void routerlist_free_all(void); +void routerlist_reset_warnings(void); +void router_set_status(const char *digest, int up); + +static int WRA_WAS_ADDED(was_router_added_t s); +static int WRA_WAS_OUTDATED(was_router_added_t s); +static int WRA_WAS_REJECTED(was_router_added_t s); +/** Return true iff the descriptor was added. It might still be necessary to + * check whether the descriptor generator should be notified. + */ +static INLINE int +WRA_WAS_ADDED(was_router_added_t s) { + return s == ROUTER_ADDED_SUCCESSFULLY || s == ROUTER_ADDED_NOTIFY_GENERATOR; +} +/** Return true iff the descriptor was not added because it was either: + * - not in the consensus + * - neither in the consensus nor in any networkstatus document + * - it was outdated. + */ +static INLINE int WRA_WAS_OUTDATED(was_router_added_t s) +{ + return (s == ROUTER_WAS_NOT_NEW || + s == ROUTER_NOT_IN_CONSENSUS || + s == ROUTER_NOT_IN_CONSENSUS_OR_NETWORKSTATUS); +} +static INLINE int WRA_WAS_REJECTED(was_router_added_t s) +{ + return (s == ROUTER_AUTHDIR_REJECTS); +} +was_router_added_t router_add_to_routerlist(routerinfo_t *router, + const char **msg, + int from_cache, + int from_fetch); +was_router_added_t router_add_extrainfo_to_routerlist( + extrainfo_t *ei, const char **msg, + int from_cache, int from_fetch); +void routerlist_descriptors_added(smartlist_t *sl, int from_cache); +void routerlist_remove_old_routers(void); +int router_load_single_router(const char *s, uint8_t purpose, int cache, + const char **msg); +int router_load_routers_from_string(const char *s, const char *eos, + saved_location_t saved_location, + smartlist_t *requested_fingerprints, + int descriptor_digests, + const char *prepend_annotations); +void router_load_extrainfo_from_string(const char *s, const char *eos, + saved_location_t saved_location, + smartlist_t *requested_fingerprints, + int descriptor_digests); + +void routerlist_retry_directory_downloads(time_t now); +int router_exit_policy_all_routers_reject(uint32_t addr, uint16_t port, + int need_uptime); +int router_exit_policy_rejects_all(routerinfo_t *router); +trusted_dir_server_t *add_trusted_dir_server(const char *nickname, + const char *address, + uint16_t dir_port, uint16_t or_port, + const char *digest, const char *v3_auth_digest, + authority_type_t type); +void authority_cert_free(authority_cert_t *cert); +void clear_trusted_dir_servers(void); +int any_trusted_dir_is_v1_authority(void); +void update_consensus_router_descriptor_downloads(time_t now, int is_vote, + networkstatus_t *consensus); +void update_router_descriptor_downloads(time_t now); +void update_extrainfo_downloads(time_t now); +int router_have_minimum_dir_info(void); +void router_dir_info_changed(void); +const char *get_dir_info_status_string(void); +int count_loading_descriptors_progress(void); +void router_reset_descriptor_download_failures(void); +int router_differences_are_cosmetic(routerinfo_t *r1, routerinfo_t *r2); +int routerinfo_incompatible_with_extrainfo(routerinfo_t *ri, extrainfo_t *ei, + signed_descriptor_t *sd, + const char **msg); + +void routerlist_assert_ok(routerlist_t *rl); +const char *esc_router_info(routerinfo_t *router); +void routers_sort_by_identity(smartlist_t *routers); + +routerset_t *routerset_new(void); +int routerset_parse(routerset_t *target, const char *s, + const char *description); +void routerset_union(routerset_t *target, const routerset_t *source); +int routerset_is_list(const routerset_t *set); +int routerset_needs_geoip(const routerset_t *set); +int routerset_is_empty(const routerset_t *set); +int routerset_contains_router(const routerset_t *set, routerinfo_t *ri); +int routerset_contains_routerstatus(const routerset_t *set, + routerstatus_t *rs); +int routerset_contains_extendinfo(const routerset_t *set, + const extend_info_t *ei); +void routerset_get_all_routers(smartlist_t *out, const routerset_t *routerset, + const routerset_t *excludeset, + int running_only); +#if 0 +void routersets_get_disjunction(smartlist_t *target, const smartlist_t *source, + const routerset_t *include, + const routerset_t *exclude, int running_only); +#endif +void routerset_subtract_routers(smartlist_t *out, + const routerset_t *routerset); +char *routerset_to_string(const routerset_t *routerset); +void routerset_refresh_countries(routerset_t *target); +int routerset_equal(const routerset_t *old, const routerset_t *new); +void routerset_free(routerset_t *routerset); +void routerinfo_set_country(routerinfo_t *ri); +void routerlist_refresh_countries(void); +void refresh_all_country_info(void); + +int hid_serv_get_responsible_directories(smartlist_t *responsible_dirs, + const char *id); +int hid_serv_acting_as_directory(void); +int hid_serv_responsible_for_desc_id(const char *id); + +#endif + diff --git a/src/or/routerparse.c b/src/or/routerparse.c index 19f9e38a9a..ce98a47b67 100644 --- a/src/or/routerparse.c +++ b/src/or/routerparse.c @@ -10,7 +10,21 @@ **/ #include "or.h" +#include "config.h" +#include "circuitbuild.h" +#include "dirserv.h" +#include "dirvote.h" +#include "policies.h" +#include "rendcommon.h" +#include "router.h" +#include "routerlist.h" #include "memarea.h" +#include "microdesc.h" +#include "networkstatus.h" +#include "rephist.h" +#include "routerparse.h" +#undef log +#include <math.h> /****************************************************************************/ @@ -55,6 +69,7 @@ typedef enum { K_S, K_V, K_W, + K_M, K_EVENTDNS, K_EXTRA_INFO, K_EXTRA_INFO_DIGEST, @@ -62,6 +77,31 @@ typedef enum { K_HIDDEN_SERVICE_DIR, K_ALLOW_SINGLE_HOP_EXITS, + K_DIRREQ_END, + K_DIRREQ_V2_IPS, + K_DIRREQ_V3_IPS, + K_DIRREQ_V2_REQS, + K_DIRREQ_V3_REQS, + K_DIRREQ_V2_SHARE, + K_DIRREQ_V3_SHARE, + K_DIRREQ_V2_RESP, + K_DIRREQ_V3_RESP, + K_DIRREQ_V2_DIR, + K_DIRREQ_V3_DIR, + K_DIRREQ_V2_TUN, + K_DIRREQ_V3_TUN, + K_ENTRY_END, + K_ENTRY_IPS, + K_CELL_END, + K_CELL_PROCESSED, + K_CELL_QUEUED, + K_CELL_TIME, + K_CELL_CIRCS, + K_EXIT_END, + K_EXIT_WRITTEN, + K_EXIT_READ, + K_EXIT_OPENED, + K_DIR_KEY_CERTIFICATE_VERSION, K_DIR_IDENTITY_KEY, K_DIR_KEY_PUBLISHED, @@ -78,13 +118,18 @@ typedef enum { K_KNOWN_FLAGS, K_PARAMS, + K_BW_WEIGHTS, K_VOTE_DIGEST, K_CONSENSUS_DIGEST, + K_ADDITIONAL_DIGEST, + K_ADDITIONAL_SIGNATURE, K_CONSENSUS_METHODS, K_CONSENSUS_METHOD, K_LEGACY_DIR_KEY, + K_DIRECTORY_FOOTER, A_PURPOSE, + A_LAST_LISTED, _A_UNKNOWN, R_RENDEZVOUS_SERVICE_DESCRIPTOR, @@ -122,7 +167,7 @@ typedef enum { * type. * * This structure is only allocated in memareas; do not allocate it on - * the heap, or token_free() won't work. + * the heap, or token_clear() won't work. */ typedef struct directory_token_t { directory_keyword tp; /**< Type of the token. */ @@ -258,6 +303,31 @@ static token_rule_t extrainfo_token_table[] = { T0N("opt", K_OPT, CONCAT_ARGS, OBJ_OK ), T01("read-history", K_READ_HISTORY, ARGS, NO_OBJ ), T01("write-history", K_WRITE_HISTORY, ARGS, NO_OBJ ), + T01("dirreq-stats-end", K_DIRREQ_END, ARGS, NO_OBJ ), + T01("dirreq-v2-ips", K_DIRREQ_V2_IPS, ARGS, NO_OBJ ), + T01("dirreq-v3-ips", K_DIRREQ_V3_IPS, ARGS, NO_OBJ ), + T01("dirreq-v2-reqs", K_DIRREQ_V2_REQS, ARGS, NO_OBJ ), + T01("dirreq-v3-reqs", K_DIRREQ_V3_REQS, ARGS, NO_OBJ ), + T01("dirreq-v2-share", K_DIRREQ_V2_SHARE, ARGS, NO_OBJ ), + T01("dirreq-v3-share", K_DIRREQ_V3_SHARE, ARGS, NO_OBJ ), + T01("dirreq-v2-resp", K_DIRREQ_V2_RESP, ARGS, NO_OBJ ), + T01("dirreq-v3-resp", K_DIRREQ_V3_RESP, ARGS, NO_OBJ ), + T01("dirreq-v2-direct-dl", K_DIRREQ_V2_DIR, ARGS, NO_OBJ ), + T01("dirreq-v3-direct-dl", K_DIRREQ_V3_DIR, ARGS, NO_OBJ ), + T01("dirreq-v2-tunneled-dl", K_DIRREQ_V2_TUN, ARGS, NO_OBJ ), + T01("dirreq-v3-tunneled-dl", K_DIRREQ_V3_TUN, ARGS, NO_OBJ ), + T01("entry-stats-end", K_ENTRY_END, ARGS, NO_OBJ ), + T01("entry-ips", K_ENTRY_IPS, ARGS, NO_OBJ ), + T01("cell-stats-end", K_CELL_END, ARGS, NO_OBJ ), + T01("cell-processed-cells", K_CELL_PROCESSED, ARGS, NO_OBJ ), + T01("cell-queued-cells", K_CELL_QUEUED, ARGS, NO_OBJ ), + T01("cell-time-in-queue", K_CELL_TIME, ARGS, NO_OBJ ), + T01("cell-circuits-per-decile", K_CELL_CIRCS, ARGS, NO_OBJ ), + T01("exit-stats-end", K_EXIT_END, ARGS, NO_OBJ ), + T01("exit-kibibytes-written", K_EXIT_WRITTEN, ARGS, NO_OBJ ), + T01("exit-kibibytes-read", K_EXIT_READ, ARGS, NO_OBJ ), + T01("exit-streams-opened", K_EXIT_OPENED, ARGS, NO_OBJ ), + T1_START( "extra-info", K_EXTRA_INFO, GE(2), NO_OBJ ), END_OF_TABLE @@ -267,10 +337,11 @@ static token_rule_t extrainfo_token_table[] = { * documents. */ static token_rule_t rtrstatus_token_table[] = { T01("p", K_P, CONCAT_ARGS, NO_OBJ ), - T1( "r", K_R, GE(8), NO_OBJ ), + T1( "r", K_R, GE(7), NO_OBJ ), T1( "s", K_S, ARGS, NO_OBJ ), T01("v", K_V, CONCAT_ARGS, NO_OBJ ), T01("w", K_W, ARGS, NO_OBJ ), + T0N("m", K_M, CONCAT_ARGS, NO_OBJ ), T0N("opt", K_OPT, CONCAT_ARGS, OBJ_OK ), END_OF_TABLE }; @@ -375,7 +446,7 @@ static token_rule_t client_keys_token_table[] = { /** List of tokens allowed in V3 networkstatus votes. */ static token_rule_t networkstatus_token_table[] = { - T1("network-status-version", K_NETWORK_STATUS_VERSION, + T1_START("network-status-version", K_NETWORK_STATUS_VERSION, GE(1), NO_OBJ ), T1("vote-status", K_VOTE_STATUS, GE(1), NO_OBJ ), T1("published", K_PUBLISHED, CONCAT_ARGS, NO_OBJ ), @@ -403,7 +474,7 @@ static token_rule_t networkstatus_token_table[] = { /** List of tokens allowed in V3 networkstatus consensuses. */ static token_rule_t networkstatus_consensus_token_table[] = { - T1("network-status-version", K_NETWORK_STATUS_VERSION, + T1_START("network-status-version", K_NETWORK_STATUS_VERSION, GE(1), NO_OBJ ), T1("vote-status", K_VOTE_STATUS, GE(1), NO_OBJ ), T1("valid-after", K_VALID_AFTER, CONCAT_ARGS, NO_OBJ ), @@ -430,17 +501,29 @@ static token_rule_t networkstatus_consensus_token_table[] = { /** List of tokens allowable in the footer of v1/v2 directory/networkstatus * footers. */ static token_rule_t networkstatus_vote_footer_token_table[] = { - T( "directory-signature", K_DIRECTORY_SIGNATURE, GE(2), NEED_OBJ ), + T01("directory-footer", K_DIRECTORY_FOOTER, NO_ARGS, NO_OBJ ), + T01("bandwidth-weights", K_BW_WEIGHTS, ARGS, NO_OBJ ), + T( "directory-signature", K_DIRECTORY_SIGNATURE, GE(2), NEED_OBJ ), END_OF_TABLE }; /** List of tokens allowable in detached networkstatus signature documents. */ static token_rule_t networkstatus_detached_signature_token_table[] = { T1_START("consensus-digest", K_CONSENSUS_DIGEST, GE(1), NO_OBJ ), + T("additional-digest", K_ADDITIONAL_DIGEST,GE(3), NO_OBJ ), T1("valid-after", K_VALID_AFTER, CONCAT_ARGS, NO_OBJ ), T1("fresh-until", K_FRESH_UNTIL, CONCAT_ARGS, NO_OBJ ), T1("valid-until", K_VALID_UNTIL, CONCAT_ARGS, NO_OBJ ), - T1N("directory-signature", K_DIRECTORY_SIGNATURE, GE(2), NEED_OBJ ), + T("additional-signature", K_ADDITIONAL_SIGNATURE, GE(4), NEED_OBJ ), + T1N("directory-signature", K_DIRECTORY_SIGNATURE, GE(2), NEED_OBJ ), + END_OF_TABLE +}; + +static token_rule_t microdesc_token_table[] = { + T1_START("onion-key", K_ONION_KEY, NO_ARGS, NEED_KEY_1024), + T01("family", K_FAMILY, ARGS, NO_OBJ ), + T01("p", K_P, CONCAT_ARGS, NO_OBJ ), + A01("@last-listed", A_LAST_LISTED, CONCAT_ARGS, NO_OBJ ), END_OF_TABLE }; @@ -453,9 +536,13 @@ static addr_policy_t *router_parse_addr_policy_private(directory_token_t *tok); static int router_get_hash_impl(const char *s, size_t s_len, char *digest, const char *start_str, const char *end_str, - char end_char); - -static void token_free(directory_token_t *tok); + char end_char, + digest_algorithm_t alg); +static int router_get_hashes_impl(const char *s, size_t s_len, + digests_t *digests, + const char *start_str, const char *end_str, + char end_char); +static void token_clear(directory_token_t *tok); static smartlist_t *find_all_exitpolicy(smartlist_t *s); static directory_token_t *_find_by_keyword(smartlist_t *s, directory_keyword keyword, @@ -479,6 +566,7 @@ static directory_token_t *get_next_token(memarea_t *area, #define CST_CHECK_AUTHORITY (1<<0) #define CST_NO_CHECK_OBJTYPE (1<<1) static int check_signature_token(const char *digest, + ssize_t digest_len, directory_token_t *tok, crypto_pk_env_t *pkey, int flags, @@ -499,6 +587,34 @@ static int tor_version_same_series(tor_version_t *a, tor_version_t *b); #define DUMP_AREA(a,name) STMT_NIL #endif +/** Last time we dumped a descriptor to disk. */ +static time_t last_desc_dumped = 0; + +/** For debugging purposes, dump unparseable descriptor *<b>desc</b> of + * type *<b>type</b> to file $DATADIR/unparseable-desc. Do not write more + * than one descriptor to disk per minute. If there is already such a + * file in the data directory, overwrite it. */ +static void +dump_desc(const char *desc, const char *type) +{ + time_t now = time(NULL); + tor_assert(desc); + tor_assert(type); + if (!last_desc_dumped || last_desc_dumped + 60 < now) { + char *debugfile = get_datadir_fname("unparseable-desc"); + size_t filelen = 50 + strlen(type) + strlen(desc); + char *content = tor_malloc_zero(filelen); + tor_snprintf(content, filelen, "Unable to parse descriptor of type " + "%s:\n%s", type, desc); + write_str_to_file(debugfile, content, 0); + log_info(LD_DIR, "Unable to parse descriptor of type %s. See file " + "unparseable-desc in data directory for details.", type); + tor_free(content); + tor_free(debugfile); + last_desc_dumped = now; + } +} + /** Set <b>digest</b> to the SHA-1 digest of the hash of the directory in * <b>s</b>. Return 0 on success, -1 on failure. */ @@ -506,7 +622,8 @@ int router_get_dir_hash(const char *s, char *digest) { return router_get_hash_impl(s, strlen(s), digest, - "signed-directory","\ndirectory-signature",'\n'); + "signed-directory","\ndirectory-signature",'\n', + DIGEST_SHA1); } /** Set <b>digest</b> to the SHA-1 digest of the hash of the first router in @@ -516,7 +633,8 @@ int router_get_router_hash(const char *s, size_t s_len, char *digest) { return router_get_hash_impl(s, s_len, digest, - "router ","\nrouter-signature", '\n'); + "router ","\nrouter-signature", '\n', + DIGEST_SHA1); } /** Set <b>digest</b> to the SHA-1 digest of the hash of the running-routers @@ -526,7 +644,8 @@ int router_get_runningrouters_hash(const char *s, char *digest) { return router_get_hash_impl(s, strlen(s), digest, - "network-status","\ndirectory-signature", '\n'); + "network-status","\ndirectory-signature", '\n', + DIGEST_SHA1); } /** Set <b>digest</b> to the SHA-1 digest of the hash of the network-status @@ -536,18 +655,31 @@ router_get_networkstatus_v2_hash(const char *s, char *digest) { return router_get_hash_impl(s, strlen(s), digest, "network-status-version","\ndirectory-signature", - '\n'); + '\n', + DIGEST_SHA1); +} + +/** Set <b>digests</b> to all the digests of the consensus document in + * <b>s</b> */ +int +router_get_networkstatus_v3_hashes(const char *s, digests_t *digests) +{ + return router_get_hashes_impl(s,strlen(s),digests, + "network-status-version", + "\ndirectory-signature", + ' '); } /** Set <b>digest</b> to the SHA-1 digest of the hash of the network-status * string in <b>s</b>. Return 0 on success, -1 on failure. */ int -router_get_networkstatus_v3_hash(const char *s, char *digest) +router_get_networkstatus_v3_hash(const char *s, char *digest, + digest_algorithm_t alg) { return router_get_hash_impl(s, strlen(s), digest, "network-status-version", "\ndirectory-signature", - ' '); + ' ', alg); } /** Set <b>digest</b> to the SHA-1 digest of the hash of the extrainfo @@ -556,7 +688,7 @@ int router_get_extrainfo_hash(const char *s, char *digest) { return router_get_hash_impl(s, strlen(s), digest, "extra-info", - "\nrouter-signature",'\n'); + "\nrouter-signature",'\n', DIGEST_SHA1); } /** Helper: used to generate signatures for routers, directories and @@ -568,16 +700,17 @@ router_get_extrainfo_hash(const char *s, char *digest) */ int router_append_dirobj_signature(char *buf, size_t buf_len, const char *digest, - crypto_pk_env_t *private_key) + size_t digest_len, crypto_pk_env_t *private_key) { char *signature; size_t i, keysize; + int siglen; keysize = crypto_pk_keysize(private_key); signature = tor_malloc(keysize); - if (crypto_pk_private_sign(private_key, signature, keysize, - digest, DIGEST_LEN) < 0) { - + siglen = crypto_pk_private_sign(private_key, signature, keysize, + digest, digest_len); + if (siglen < 0) { log_warn(LD_BUG,"Couldn't sign digest."); goto err; } @@ -585,7 +718,7 @@ router_append_dirobj_signature(char *buf, size_t buf_len, const char *digest, goto truncated; i = strlen(buf); - if (base64_encode(buf+i, buf_len-i, signature, 128) < 0) { + if (base64_encode(buf+i, buf_len-i, signature, siglen) < 0) { log_warn(LD_BUG,"couldn't base64-encode signature"); goto err; } @@ -692,7 +825,7 @@ router_parse_directory(const char *str) char digest[DIGEST_LEN]; time_t published_on; int r; - const char *end, *cp; + const char *end, *cp, *str_dup = str; smartlist_t *tokens = NULL; crypto_pk_env_t *declared_key = NULL; memarea_t *area = memarea_new(); @@ -728,11 +861,11 @@ router_parse_directory(const char *str) } declared_key = find_dir_signing_key(str, str+strlen(str)); note_crypto_pk_op(VERIFY_DIR); - if (check_signature_token(digest, tok, declared_key, + if (check_signature_token(digest, DIGEST_LEN, tok, declared_key, CST_CHECK_AUTHORITY, "directory")<0) goto err; - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_clear(tokens); memarea_clear(area); @@ -765,11 +898,12 @@ router_parse_directory(const char *str) r = 0; goto done; err: + dump_desc(str_dup, "v1 directory"); r = -1; done: if (declared_key) crypto_free_pk_env(declared_key); if (tokens) { - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); } if (area) { @@ -791,7 +925,7 @@ router_parse_runningrouters(const char *str) int r = -1; crypto_pk_env_t *declared_key = NULL; smartlist_t *tokens = NULL; - const char *eos = str + strlen(str); + const char *eos = str + strlen(str), *str_dup = str; memarea_t *area = NULL; if (router_get_runningrouters_hash(str, digest)) { @@ -820,7 +954,7 @@ router_parse_runningrouters(const char *str) } declared_key = find_dir_signing_key(str, eos); note_crypto_pk_op(VERIFY_DIR); - if (check_signature_token(digest, tok, declared_key, + if (check_signature_token(digest, DIGEST_LEN, tok, declared_key, CST_CHECK_AUTHORITY, "running-routers") < 0) goto err; @@ -832,9 +966,10 @@ router_parse_runningrouters(const char *str) r = 0; err: + dump_desc(str_dup, "v1 running-routers"); if (declared_key) crypto_free_pk_env(declared_key); if (tokens) { - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); } if (area) { @@ -884,7 +1019,7 @@ find_dir_signing_key(const char *str, const char *eos) } done: - if (tok) token_free(tok); + if (tok) token_clear(tok); if (area) { DUMP_AREA(area, "dir-signing-key token"); memarea_drop_all(area); @@ -920,6 +1055,7 @@ dir_signing_key_is_trusted(crypto_pk_env_t *key) */ static int check_signature_token(const char *digest, + ssize_t digest_len, directory_token_t *tok, crypto_pk_env_t *pkey, int flags, @@ -952,14 +1088,14 @@ check_signature_token(const char *digest, signed_digest = tor_malloc(keysize); if (crypto_pk_public_checksig(pkey, signed_digest, keysize, tok->object_body, tok->object_size) - != DIGEST_LEN) { + < digest_len) { log_warn(LD_DIR, "Error reading %s: invalid signature.", doctype); tor_free(signed_digest); return -1; } // log_debug(LD_DIR,"Signed %s hash starts %s", doctype, // hex_str(signed_digest,4)); - if (tor_memneq(digest, signed_digest, DIGEST_LEN)) { + if (tor_memneq(digest, signed_digest, digest_len)) { log_warn(LD_DIR, "Error reading %s: signature does not match.", doctype); tor_free(signed_digest); return -1; @@ -1145,7 +1281,7 @@ router_parse_entry_from_string(const char *s, const char *end, smartlist_t *tokens = NULL, *exit_policy_tokens = NULL; directory_token_t *tok; struct in_addr in; - const char *start_of_annotations, *cp; + const char *start_of_annotations, *cp, *s_dup = s; size_t prepend_len = prepend_annotations ? strlen(prepend_annotations) : 0; int ok = 1; memarea_t *area = NULL; @@ -1434,7 +1570,7 @@ router_parse_entry_from_string(const char *s, const char *end, verified_digests = digestmap_new(); digestmap_set(verified_digests, signed_digest, (void*)(uintptr_t)1); #endif - if (check_signature_token(digest, tok, router->identity_pkey, 0, + if (check_signature_token(digest, DIGEST_LEN, tok, router->identity_pkey, 0, "router descriptor") < 0) goto err; @@ -1452,16 +1588,15 @@ router_parse_entry_from_string(const char *s, const char *end, goto done; err: + dump_desc(s_dup, "router descriptor"); routerinfo_free(router); router = NULL; done: if (tokens) { - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); } - if (exit_policy_tokens) { - smartlist_free(exit_policy_tokens); - } + smartlist_free(exit_policy_tokens); if (area) { DUMP_AREA(area, "routerinfo"); memarea_drop_all(area); @@ -1486,6 +1621,7 @@ extrainfo_parse_entry_from_string(const char *s, const char *end, crypto_pk_env_t *key = NULL; routerinfo_t *router = NULL; memarea_t *area = NULL; + const char *s_dup = s; if (!end) { end = s + strlen(s); @@ -1560,7 +1696,8 @@ extrainfo_parse_entry_from_string(const char *s, const char *end, if (key) { note_crypto_pk_op(VERIFY_RTR); - if (check_signature_token(digest, tok, key, 0, "extra-info") < 0) + if (check_signature_token(digest, DIGEST_LEN, tok, key, 0, + "extra-info") < 0) goto err; if (router) @@ -1574,12 +1711,12 @@ extrainfo_parse_entry_from_string(const char *s, const char *end, goto done; err: - if (extrainfo) - extrainfo_free(extrainfo); + dump_desc(s_dup, "extra-info descriptor"); + extrainfo_free(extrainfo); extrainfo = NULL; done: if (tokens) { - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); } if (area) { @@ -1607,6 +1744,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) size_t len; int found; memarea_t *area = NULL; + const char *s_dup = s; s = eat_whitespace(s); eos = strstr(s, "\ndir-key-certification"); @@ -1637,7 +1775,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) goto err; } if (router_get_hash_impl(s, strlen(s), digest, "dir-key-certificate-version", - "\ndir-key-certification", '\n') < 0) + "\ndir-key-certification", '\n', DIGEST_SHA1) < 0) goto err; tok = smartlist_get(tokens, 0); if (tok->tp != K_DIR_KEY_CERTIFICATE_VERSION || strcmp(tok->args[0], "3")) { @@ -1685,7 +1823,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) struct in_addr in; char *address = NULL; tor_assert(tok->n_args); - /* XXX021 use tor_addr_port_parse() below instead. -RD */ + /* XXX023 use tor_addr_port_parse() below instead. -RD */ if (parse_addr_port(LOG_WARN, tok->args[0], &address, NULL, &cert->dir_port)<0 || tor_inet_aton(address, &in) == 0) { @@ -1730,7 +1868,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) } } if (!found) { - if (check_signature_token(digest, tok, cert->identity_key, 0, + if (check_signature_token(digest, DIGEST_LEN, tok, cert->identity_key, 0, "key certificate")) { goto err; } @@ -1739,6 +1877,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) /* XXXX Once all authorities generate cross-certified certificates, * make this field mandatory. */ if (check_signature_token(cert->cache_info.identity_digest, + DIGEST_LEN, tok, cert->signing_key, CST_NO_CHECK_OBJTYPE, @@ -1758,7 +1897,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) if (end_of_string) { *end_of_string = eat_whitespace(eos); } - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); if (area) { DUMP_AREA(area, "authority cert"); @@ -1766,8 +1905,9 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) } return cert; err: + dump_desc(s_dup, "authority cert"); authority_cert_free(cert); - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); if (area) { DUMP_AREA(area, "authority cert"); @@ -1778,23 +1918,28 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) /** Helper: given a string <b>s</b>, return the start of the next router-status * object (starting with "r " at the start of a line). If none is found, - * return the start of the next directory signature. If none is found, return - * the end of the string. */ + * return the start of the directory footer, or the next directory signature. + * If none is found, return the end of the string. */ static INLINE const char * find_start_of_next_routerstatus(const char *s) { - const char *eos = strstr(s, "\nr "); - if (eos) { - const char *eos2 = tor_memstr(s, eos-s, "\ndirectory-signature"); - if (eos2 && eos2 < eos) - return eos2; - else - return eos+1; - } else { - if ((eos = strstr(s, "\ndirectory-signature"))) - return eos+1; - return s + strlen(s); - } + const char *eos, *footer, *sig; + if ((eos = strstr(s, "\nr "))) + ++eos; + else + eos = s + strlen(s); + + footer = tor_memstr(s, eos-s, "\ndirectory-footer"); + sig = tor_memstr(s, eos-s, "\ndirectory-signature"); + + if (footer && sig) + return MIN(footer, sig) + 1; + else if (footer) + return footer+1; + else if (sig) + return sig+1; + else + return eos; } /** Given a string at *<b>s</b>, containing a routerstatus object, and an @@ -1808,22 +1953,29 @@ find_start_of_next_routerstatus(const char *s) * If <b>consensus_method</b> is nonzero, this routerstatus is part of a * consensus, and we should parse it according to the method used to * make that consensus. + * + * Parse according to the syntax used by the consensus flavor <b>flav</b>. **/ static routerstatus_t * routerstatus_parse_entry_from_string(memarea_t *area, const char **s, smartlist_t *tokens, networkstatus_t *vote, vote_routerstatus_t *vote_rs, - int consensus_method) + int consensus_method, + consensus_flavor_t flav) { - const char *eos; + const char *eos, *s_dup = *s; routerstatus_t *rs = NULL; directory_token_t *tok; char timebuf[ISO_TIME_LEN+1]; struct in_addr in; + int offset = 0; tor_assert(tokens); tor_assert(bool_eq(vote, vote_rs)); + if (!consensus_method) + flav = FLAV_NS; + eos = find_start_of_next_routerstatus(*s); if (tokenize_string(area,*s, eos, tokens, rtrstatus_token_table,0)) { @@ -1835,7 +1987,15 @@ routerstatus_parse_entry_from_string(memarea_t *area, goto err; } tok = find_by_keyword(tokens, K_R); - tor_assert(tok->n_args >= 8); + tor_assert(tok->n_args >= 7); + if (flav == FLAV_NS) { + if (tok->n_args < 8) { + log_warn(LD_DIR, "Too few arguments to r"); + goto err; + } + } else { + offset = -1; + } if (vote_rs) { rs = &vote_rs->status; } else { @@ -1856,29 +2016,34 @@ routerstatus_parse_entry_from_string(memarea_t *area, goto err; } - if (digest_from_base64(rs->descriptor_digest, tok->args[2])) { - log_warn(LD_DIR, "Error decoding descriptor digest %s", - escaped(tok->args[2])); - goto err; + if (flav == FLAV_NS) { + if (digest_from_base64(rs->descriptor_digest, tok->args[2])) { + log_warn(LD_DIR, "Error decoding descriptor digest %s", + escaped(tok->args[2])); + goto err; + } } if (tor_snprintf(timebuf, sizeof(timebuf), "%s %s", - tok->args[3], tok->args[4]) < 0 || + tok->args[3+offset], tok->args[4+offset]) < 0 || parse_iso_time(timebuf, &rs->published_on)<0) { - log_warn(LD_DIR, "Error parsing time '%s %s'", - tok->args[3], tok->args[4]); + log_warn(LD_DIR, "Error parsing time '%s %s' [%d %d]", + tok->args[3+offset], tok->args[4+offset], + offset, (int)flav); goto err; } - if (tor_inet_aton(tok->args[5], &in) == 0) { + if (tor_inet_aton(tok->args[5+offset], &in) == 0) { log_warn(LD_DIR, "Error parsing router address in network-status %s", - escaped(tok->args[5])); + escaped(tok->args[5+offset])); goto err; } rs->addr = ntohl(in.s_addr); - rs->or_port =(uint16_t) tor_parse_long(tok->args[6],10,0,65535,NULL,NULL); - rs->dir_port = (uint16_t) tor_parse_long(tok->args[7],10,0,65535,NULL,NULL); + rs->or_port = (uint16_t) tor_parse_long(tok->args[6+offset], + 10,0,65535,NULL,NULL); + rs->dir_port = (uint16_t) tor_parse_long(tok->args[7+offset], + 10,0,65535,NULL,NULL); tok = find_opt_by_keyword(tokens, K_S); if (tok && vote) { @@ -1964,6 +2129,17 @@ routerstatus_parse_entry_from_string(memarea_t *area, goto err; } rs->has_bandwidth = 1; + } else if (!strcmpstart(tok->args[i], "Measured=")) { + int ok; + rs->measured_bw = + (uint32_t)tor_parse_ulong(strchr(tok->args[i], '=')+1, + 10, 0, UINT32_MAX, &ok, NULL); + if (!ok) { + log_warn(LD_DIR, "Invalid Measured Bandwidth %s", + escaped(tok->args[i])); + goto err; + } + rs->has_measured_bw = 1; } } } @@ -1985,16 +2161,29 @@ routerstatus_parse_entry_from_string(memarea_t *area, rs->has_exitsummary = 1; } + if (vote_rs) { + SMARTLIST_FOREACH_BEGIN(tokens, directory_token_t *, t) { + if (t->tp == K_M && t->n_args) { + vote_microdesc_hash_t *line = + tor_malloc(sizeof(vote_microdesc_hash_t)); + line->next = vote_rs->microdesc; + line->microdesc_hash_line = tor_strdup(t->args[0]); + vote_rs->microdesc = line; + } + } SMARTLIST_FOREACH_END(t); + } + if (!strcasecmp(rs->nickname, UNNAMED_ROUTER_NICKNAME)) rs->is_named = 0; goto done; err: + dump_desc(s_dup, "routerstatus entry"); if (rs && !vote_rs) routerstatus_free(rs); rs = NULL; done: - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_clear(tokens); if (area) { DUMP_AREA(area, "routerstatus entry"); @@ -2006,8 +2195,8 @@ routerstatus_parse_entry_from_string(memarea_t *area, } /** Helper to sort a smartlist of pointers to routerstatus_t */ -static int -_compare_routerstatus_entries(const void **_a, const void **_b) +int +compare_routerstatus_entries(const void **_a, const void **_b) { const routerstatus_t *a = *_a, *b = *_b; return fast_memcmp(a->identity_digest, b->identity_digest, DIGEST_LEN); @@ -2031,7 +2220,7 @@ _free_duplicate_routerstatus_entry(void *e) networkstatus_v2_t * networkstatus_v2_parse_from_string(const char *s) { - const char *eos; + const char *eos, *s_dup = s; smartlist_t *tokens = smartlist_create(); smartlist_t *footer_tokens = smartlist_create(); networkstatus_v2_t *ns = NULL; @@ -2145,17 +2334,17 @@ networkstatus_v2_parse_from_string(const char *s) ns->entries = smartlist_create(); s = eos; - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_clear(tokens); memarea_clear(area); while (!strcmpstart(s, "r ")) { routerstatus_t *rs; if ((rs = routerstatus_parse_entry_from_string(area, &s, tokens, - NULL, NULL, 0))) + NULL, NULL, 0, 0))) smartlist_add(ns->entries, rs); } - smartlist_sort(ns->entries, _compare_routerstatus_entries); - smartlist_uniq(ns->entries, _compare_routerstatus_entries, + smartlist_sort(ns->entries, compare_routerstatus_entries); + smartlist_uniq(ns->entries, compare_routerstatus_entries, _free_duplicate_routerstatus_entry); if (tokenize_string(area,s, NULL, footer_tokens, dir_footer_token_table,0)) { @@ -2174,19 +2363,19 @@ networkstatus_v2_parse_from_string(const char *s) } note_crypto_pk_op(VERIFY_DIR); - if (check_signature_token(ns_digest, tok, ns->signing_key, 0, + if (check_signature_token(ns_digest, DIGEST_LEN, tok, ns->signing_key, 0, "network-status") < 0) goto err; goto done; err: - if (ns) - networkstatus_v2_free(ns); + dump_desc(s_dup, "v2 networkstatus"); + networkstatus_v2_free(ns); ns = NULL; done: - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); - SMARTLIST_FOREACH(footer_tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(footer_tokens, directory_token_t *, t, token_clear(t)); smartlist_free(footer_tokens); if (area) { DUMP_AREA(area, "v2 networkstatus"); @@ -2195,6 +2384,395 @@ networkstatus_v2_parse_from_string(const char *s) return ns; } +/** Verify the bandwidth weights of a network status document */ +int +networkstatus_verify_bw_weights(networkstatus_t *ns) +{ + int64_t weight_scale; + int64_t G=0, M=0, E=0, D=0, T=0; + double Wgg, Wgm, Wgd, Wmg, Wmm, Wme, Wmd, Weg, Wem, Wee, Wed; + double Gtotal=0, Mtotal=0, Etotal=0; + const char *casename = NULL; + int valid = 1; + + weight_scale = circuit_build_times_get_bw_scale(ns); + Wgg = networkstatus_get_bw_weight(ns, "Wgg", -1); + Wgm = networkstatus_get_bw_weight(ns, "Wgm", -1); + Wgd = networkstatus_get_bw_weight(ns, "Wgd", -1); + Wmg = networkstatus_get_bw_weight(ns, "Wmg", -1); + Wmm = networkstatus_get_bw_weight(ns, "Wmm", -1); + Wme = networkstatus_get_bw_weight(ns, "Wme", -1); + Wmd = networkstatus_get_bw_weight(ns, "Wmd", -1); + Weg = networkstatus_get_bw_weight(ns, "Weg", -1); + Wem = networkstatus_get_bw_weight(ns, "Wem", -1); + Wee = networkstatus_get_bw_weight(ns, "Wee", -1); + Wed = networkstatus_get_bw_weight(ns, "Wed", -1); + + if (Wgg<0 || Wgm<0 || Wgd<0 || Wmg<0 || Wmm<0 || Wme<0 || Wmd<0 || Weg<0 + || Wem<0 || Wee<0 || Wed<0) { + log_warn(LD_BUG, "No bandwidth weights produced in consensus!"); + return 0; + } + + // First, sanity check basic summing properties that hold for all cases + // We use > 1 as the check for these because they are computed as integers. + // Sometimes there are rounding errors. + if (fabs(Wmm - weight_scale) > 1) { + log_warn(LD_BUG, "Wmm=%lf != "I64_FORMAT, + Wmm, I64_PRINTF_ARG(weight_scale)); + valid = 0; + } + + if (fabs(Wem - Wee) > 1) { + log_warn(LD_BUG, "Wem=%lf != Wee=%lf", Wem, Wee); + valid = 0; + } + + if (fabs(Wgm - Wgg) > 1) { + log_warn(LD_BUG, "Wgm=%lf != Wgg=%lf", Wgm, Wgg); + valid = 0; + } + + if (fabs(Weg - Wed) > 1) { + log_warn(LD_BUG, "Wed=%lf != Weg=%lf", Wed, Weg); + valid = 0; + } + + if (fabs(Wgg + Wmg - weight_scale) > 0.001*weight_scale) { + log_warn(LD_BUG, "Wgg=%lf != "I64_FORMAT" - Wmg=%lf", Wgg, + I64_PRINTF_ARG(weight_scale), Wmg); + valid = 0; + } + + if (fabs(Wee + Wme - weight_scale) > 0.001*weight_scale) { + log_warn(LD_BUG, "Wee=%lf != "I64_FORMAT" - Wme=%lf", Wee, + I64_PRINTF_ARG(weight_scale), Wme); + valid = 0; + } + + if (fabs(Wgd + Wmd + Wed - weight_scale) > 0.001*weight_scale) { + log_warn(LD_BUG, "Wgd=%lf + Wmd=%lf + Wed=%lf != "I64_FORMAT, + Wgd, Wmd, Wed, I64_PRINTF_ARG(weight_scale)); + valid = 0; + } + + Wgg /= weight_scale; + Wgm /= weight_scale; + Wgd /= weight_scale; + + Wmg /= weight_scale; + Wmm /= weight_scale; + Wme /= weight_scale; + Wmd /= weight_scale; + + Weg /= weight_scale; + Wem /= weight_scale; + Wee /= weight_scale; + Wed /= weight_scale; + + // Then, gather G, M, E, D, T to determine case + SMARTLIST_FOREACH_BEGIN(ns->routerstatus_list, routerstatus_t *, rs) { + if (rs->has_bandwidth) { + T += rs->bandwidth; + if (rs->is_exit && rs->is_possible_guard) { + D += rs->bandwidth; + Gtotal += Wgd*rs->bandwidth; + Mtotal += Wmd*rs->bandwidth; + Etotal += Wed*rs->bandwidth; + } else if (rs->is_exit) { + E += rs->bandwidth; + Mtotal += Wme*rs->bandwidth; + Etotal += Wee*rs->bandwidth; + } else if (rs->is_possible_guard) { + G += rs->bandwidth; + Gtotal += Wgg*rs->bandwidth; + Mtotal += Wmg*rs->bandwidth; + } else { + M += rs->bandwidth; + Mtotal += Wmm*rs->bandwidth; + } + } else { + log_warn(LD_BUG, "Missing consensus bandwidth for router %s", + rs->nickname); + } + } SMARTLIST_FOREACH_END(rs); + + // Finally, check equality conditions depending upon case 1, 2 or 3 + // Full equality cases: 1, 3b + // Partial equality cases: 2b (E=G), 3a (M=E) + // Fully unknown: 2a + if (3*E >= T && 3*G >= T) { + // Case 1: Neither are scarce + casename = "Case 1"; + if (fabs(Etotal-Mtotal) > 0.01*MAX(Etotal,Mtotal)) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: Etotal %lf != Mtotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Etotal, Mtotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + if (fabs(Etotal-Gtotal) > 0.01*MAX(Etotal,Gtotal)) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: Etotal %lf != Gtotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Etotal, Gtotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + if (fabs(Gtotal-Mtotal) > 0.01*MAX(Gtotal,Mtotal)) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: Mtotal %lf != Gtotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Mtotal, Gtotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + } else if (3*E < T && 3*G < T) { + int64_t R = MIN(E, G); + int64_t S = MAX(E, G); + /* + * Case 2: Both Guards and Exits are scarce + * Balance D between E and G, depending upon + * D capacity and scarcity. Devote no extra + * bandwidth to middle nodes. + */ + if (R+D < S) { // Subcase a + double Rtotal, Stotal; + if (E < G) { + Rtotal = Etotal; + Stotal = Gtotal; + } else { + Rtotal = Gtotal; + Stotal = Etotal; + } + casename = "Case 2a"; + // Rtotal < Stotal + if (Rtotal > Stotal) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: Rtotal %lf > Stotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Rtotal, Stotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + // Rtotal < T/3 + if (3*Rtotal > T) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: 3*Rtotal %lf > T " + I64_FORMAT". G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT + " D="I64_FORMAT" T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Rtotal*3, I64_PRINTF_ARG(T), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + // Stotal < T/3 + if (3*Stotal > T) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: 3*Stotal %lf > T " + I64_FORMAT". G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT + " D="I64_FORMAT" T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Stotal*3, I64_PRINTF_ARG(T), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + // Mtotal > T/3 + if (3*Mtotal < T) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: 3*Mtotal %lf < T " + I64_FORMAT". " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Mtotal*3, I64_PRINTF_ARG(T), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + } else { // Subcase b: R+D > S + casename = "Case 2b"; + + /* Check the rare-M redirect case. */ + if (D != 0 && 3*M < T) { + casename = "Case 2b (balanced)"; + if (fabs(Etotal-Mtotal) > 0.01*MAX(Etotal,Mtotal)) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: Etotal %lf != Mtotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Etotal, Mtotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + if (fabs(Etotal-Gtotal) > 0.01*MAX(Etotal,Gtotal)) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: Etotal %lf != Gtotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Etotal, Gtotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + if (fabs(Gtotal-Mtotal) > 0.01*MAX(Gtotal,Mtotal)) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: Mtotal %lf != Gtotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Mtotal, Gtotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + } else { + if (fabs(Etotal-Gtotal) > 0.01*MAX(Etotal,Gtotal)) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: Etotal %lf != Gtotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Etotal, Gtotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + } + } + } else { // if (E < T/3 || G < T/3) { + int64_t S = MIN(E, G); + int64_t NS = MAX(E, G); + if (3*(S+D) < T) { // Subcase a: + double Stotal; + double NStotal; + if (G < E) { + casename = "Case 3a (G scarce)"; + Stotal = Gtotal; + NStotal = Etotal; + } else { // if (G >= E) { + casename = "Case 3a (E scarce)"; + NStotal = Gtotal; + Stotal = Etotal; + } + // Stotal < T/3 + if (3*Stotal > T) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: 3*Stotal %lf > T " + I64_FORMAT". G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT + " D="I64_FORMAT" T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Stotal*3, I64_PRINTF_ARG(T), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + if (NS >= M) { + if (fabs(NStotal-Mtotal) > 0.01*MAX(NStotal,Mtotal)) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: NStotal %lf != Mtotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, NStotal, Mtotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + } else { + // if NS < M, NStotal > T/3 because only one of G or E is scarce + if (3*NStotal < T) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: 3*NStotal %lf < T " + I64_FORMAT". G="I64_FORMAT" M="I64_FORMAT + " E="I64_FORMAT" D="I64_FORMAT" T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, NStotal*3, I64_PRINTF_ARG(T), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + } + } else { // Subcase b: S+D >= T/3 + casename = "Case 3b"; + if (fabs(Etotal-Mtotal) > 0.01*MAX(Etotal,Mtotal)) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: Etotal %lf != Mtotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Etotal, Mtotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + if (fabs(Etotal-Gtotal) > 0.01*MAX(Etotal,Gtotal)) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: Etotal %lf != Gtotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Etotal, Gtotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + if (fabs(Gtotal-Mtotal) > 0.01*MAX(Gtotal,Mtotal)) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: Mtotal %lf != Gtotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Mtotal, Gtotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + } + } + + if (valid) + log_notice(LD_DIR, "Bandwidth-weight %s is verified and valid.", + casename); + + return valid; +} + /** Parse a v3 networkstatus vote, opinion, or consensus (depending on * ns_type), from <b>s</b>, and return the result. Return NULL on failure. */ networkstatus_t * @@ -2205,19 +2783,21 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, smartlist_t *rs_tokens = NULL, *footer_tokens = NULL; networkstatus_voter_info_t *voter = NULL; networkstatus_t *ns = NULL; - char ns_digest[DIGEST_LEN]; - const char *cert, *end_of_header, *end_of_footer; + digests_t ns_digests; + const char *cert, *end_of_header, *end_of_footer, *s_dup = s; directory_token_t *tok; int ok; struct in_addr in; int i, inorder, n_signatures = 0; memarea_t *area = NULL, *rs_area = NULL; + consensus_flavor_t flav = FLAV_NS; + tor_assert(s); if (eos_out) *eos_out = NULL; - if (router_get_networkstatus_v3_hash(s, ns_digest)) { + if (router_get_networkstatus_v3_hashes(s, &ns_digests)) { log_warn(LD_DIR, "Unable to compute digest of network-status"); goto err; } @@ -2233,7 +2813,23 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, } ns = tor_malloc_zero(sizeof(networkstatus_t)); - memcpy(ns->networkstatus_digest, ns_digest, DIGEST_LEN); + memcpy(&ns->digests, &ns_digests, sizeof(ns_digests)); + + tok = find_by_keyword(tokens, K_NETWORK_STATUS_VERSION); + tor_assert(tok); + if (tok->n_args > 1) { + int flavor = networkstatus_parse_flavor_name(tok->args[1]); + if (flavor < 0) { + log_warn(LD_DIR, "Can't parse document with unknown flavor %s", + escaped(tok->args[2])); + goto err; + } + ns->flavor = flav = flavor; + } + if (flav != FLAV_NS && ns_type != NS_TYPE_CONSENSUS) { + log_warn(LD_DIR, "Flavor found on non-consensus networkstatus."); + goto err; + } if (ns_type != NS_TYPE_CONSENSUS) { const char *end_of_cert = NULL; @@ -2387,8 +2983,9 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, if (voter) smartlist_add(ns->voters, voter); voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t)); + voter->sigs = smartlist_create(); if (ns->type != NS_TYPE_CONSENSUS) - memcpy(voter->vote_digest, ns_digest, DIGEST_LEN); + memcpy(voter->vote_digest, ns_digests.d[DIGEST_SHA1], DIGEST_LEN); voter->nickname = tor_strdup(tok->args[0]); if (strlen(tok->args[1]) != HEX_DIGEST_LEN || @@ -2480,7 +3077,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, if (ns->type != NS_TYPE_CONSENSUS) { vote_routerstatus_t *rs = tor_malloc_zero(sizeof(vote_routerstatus_t)); if (routerstatus_parse_entry_from_string(rs_area, &s, rs_tokens, ns, - rs, 0)) + rs, 0, 0)) smartlist_add(ns->routerstatus_list, rs); else { tor_free(rs->version); @@ -2490,7 +3087,8 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, routerstatus_t *rs; if ((rs = routerstatus_parse_entry_from_string(rs_area, &s, rs_tokens, NULL, NULL, - ns->consensus_method))) + ns->consensus_method, + flav))) smartlist_add(ns->routerstatus_list, rs); } } @@ -2524,14 +3122,73 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, goto err; } - SMARTLIST_FOREACH(footer_tokens, directory_token_t *, _tok, { + int found_sig = 0; + SMARTLIST_FOREACH_BEGIN(footer_tokens, directory_token_t *, _tok) { + tok = _tok; + if (tok->tp == K_DIRECTORY_SIGNATURE) + found_sig = 1; + else if (found_sig) { + log_warn(LD_DIR, "Extraneous token after first directory-signature"); + goto err; + } + } SMARTLIST_FOREACH_END(_tok); + } + + if ((tok = find_opt_by_keyword(footer_tokens, K_DIRECTORY_FOOTER))) { + if (tok != smartlist_get(footer_tokens, 0)) { + log_warn(LD_DIR, "Misplaced directory-footer token"); + goto err; + } + } + + tok = find_opt_by_keyword(footer_tokens, K_BW_WEIGHTS); + if (tok) { + ns->weight_params = smartlist_create(); + for (i = 0; i < tok->n_args; ++i) { + int ok=0; + char *eq = strchr(tok->args[i], '='); + if (!eq) { + log_warn(LD_DIR, "Bad element '%s' in weight params", + escaped(tok->args[i])); + goto err; + } + tor_parse_long(eq+1, 10, INT32_MIN, INT32_MAX, &ok, NULL); + if (!ok) { + log_warn(LD_DIR, "Bad element '%s' in params", escaped(tok->args[i])); + goto err; + } + smartlist_add(ns->weight_params, tor_strdup(tok->args[i])); + } + } + + SMARTLIST_FOREACH_BEGIN(footer_tokens, directory_token_t *, _tok) { char declared_identity[DIGEST_LEN]; networkstatus_voter_info_t *v; + document_signature_t *sig; + const char *id_hexdigest = NULL; + const char *sk_hexdigest = NULL; + digest_algorithm_t alg = DIGEST_SHA1; tok = _tok; if (tok->tp != K_DIRECTORY_SIGNATURE) continue; tor_assert(tok->n_args >= 2); + if (tok->n_args == 2) { + id_hexdigest = tok->args[0]; + sk_hexdigest = tok->args[1]; + } else { + const char *algname = tok->args[0]; + int a; + id_hexdigest = tok->args[1]; + sk_hexdigest = tok->args[2]; + a = crypto_digest_algorithm_parse_name(algname); + if (a<0) { + log_warn(LD_DIR, "Unknown digest algorithm %s; skipping", + escaped(algname)); + continue; + } + alg = a; + } if (!tok->object_type || strcmp(tok->object_type, "SIGNATURE") || @@ -2540,11 +3197,11 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, goto err; } - if (strlen(tok->args[0]) != HEX_DIGEST_LEN || + if (strlen(id_hexdigest) != HEX_DIGEST_LEN || base16_decode(declared_identity, sizeof(declared_identity), - tok->args[0], HEX_DIGEST_LEN) < 0) { + id_hexdigest, HEX_DIGEST_LEN) < 0) { log_warn(LD_DIR, "Error decoding declared identity %s in " - "network-status vote.", escaped(tok->args[0])); + "network-status vote.", escaped(id_hexdigest)); goto err; } if (!(v = networkstatus_get_voter_by_id(ns, declared_identity))) { @@ -2552,11 +3209,15 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, "any declared directory source."); goto err; } - if (strlen(tok->args[1]) != HEX_DIGEST_LEN || - base16_decode(v->signing_key_digest, sizeof(v->signing_key_digest), - tok->args[1], HEX_DIGEST_LEN) < 0) { - log_warn(LD_DIR, "Error decoding declared digest %s in " - "network-status vote.", escaped(tok->args[1])); + sig = tor_malloc_zero(sizeof(document_signature_t)); + memcpy(sig->identity_digest, v->identity_digest, DIGEST_LEN); + sig->alg = alg; + if (strlen(sk_hexdigest) != HEX_DIGEST_LEN || + base16_decode(sig->signing_key_digest, sizeof(sig->signing_key_digest), + sk_hexdigest, HEX_DIGEST_LEN) < 0) { + log_warn(LD_DIR, "Error decoding declared signing key digest %s in " + "network-status vote.", escaped(sk_hexdigest)); + tor_free(sig); goto err; } @@ -2565,35 +3226,49 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, DIGEST_LEN)) { log_warn(LD_DIR, "Digest mismatch between declared and actual on " "network-status vote."); + tor_free(sig); goto err; } } + if (voter_get_sig_by_algorithm(v, sig->alg)) { + /* We already parsed a vote with this algorithm from this voter. Use the + first one. */ + log_fn(LOG_PROTOCOL_WARN, LD_DIR, "We received a networkstatus " + "that contains two votes from the same voter with the same " + "algorithm. Ignoring the second vote."); + tor_free(sig); + continue; + } + if (ns->type != NS_TYPE_CONSENSUS) { - if (check_signature_token(ns_digest, tok, ns->cert->signing_key, 0, - "network-status vote")) + if (check_signature_token(ns_digests.d[DIGEST_SHA1], DIGEST_LEN, + tok, ns->cert->signing_key, 0, + "network-status vote")) { + tor_free(sig); goto err; - v->good_signature = 1; + } + sig->good_signature = 1; } else { - if (tok->object_size >= INT_MAX || tok->object_size >= SIZE_T_CEILING) + if (tok->object_size >= INT_MAX || tok->object_size >= SIZE_T_CEILING) { + tor_free(sig); goto err; - /* We already parsed a vote from this voter. Use the first one. */ - if (v->signature) { - log_fn(LOG_PROTOCOL_WARN, LD_DIR, "We received a networkstatus " - "that contains two votes from the same voter. Ignoring " - "the second vote."); - continue; } - - v->signature = tor_memdup(tok->object_body, tok->object_size); - v->signature_len = (int) tok->object_size; + sig->signature = tor_memdup(tok->object_body, tok->object_size); + sig->signature_len = (int) tok->object_size; } + smartlist_add(v->sigs, sig); + ++n_signatures; - }); + } SMARTLIST_FOREACH_END(_tok); if (! n_signatures) { log_warn(LD_DIR, "No signatures on networkstatus vote."); goto err; + } else if (ns->type == NS_TYPE_VOTE && n_signatures != 1) { + log_warn(LD_DIR, "Received more than one signature on a " + "network-status vote."); + goto err; } if (eos_out) @@ -2601,27 +3276,31 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, goto done; err: - if (ns) - networkstatus_vote_free(ns); + dump_desc(s_dup, "v3 networkstatus"); + networkstatus_vote_free(ns); ns = NULL; done: if (tokens) { - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); } if (voter) { + if (voter->sigs) { + SMARTLIST_FOREACH(voter->sigs, document_signature_t *, sig, + document_signature_free(sig)); + smartlist_free(voter->sigs); + } tor_free(voter->nickname); tor_free(voter->address); tor_free(voter->contact); - tor_free(voter->signature); tor_free(voter); } if (rs_tokens) { - SMARTLIST_FOREACH(rs_tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(rs_tokens, directory_token_t *, t, token_clear(t)); smartlist_free(rs_tokens); } if (footer_tokens) { - SMARTLIST_FOREACH(footer_tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(footer_tokens, directory_token_t *, t, token_clear(t)); smartlist_free(footer_tokens); } if (area) { @@ -2634,6 +3313,35 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, return ns; } +/** Return the digests_t that holds the digests of the + * <b>flavor_name</b>-flavored networkstatus according to the detached + * signatures document <b>sigs</b>, allocating a new digests_t as neeeded. */ +static digests_t * +detached_get_digests(ns_detached_signatures_t *sigs, const char *flavor_name) +{ + digests_t *d = strmap_get(sigs->digests, flavor_name); + if (!d) { + d = tor_malloc_zero(sizeof(digests_t)); + strmap_set(sigs->digests, flavor_name, d); + } + return d; +} + +/** Return the list of signatures of the <b>flavor_name</b>-flavored + * networkstatus according to the detached signatures document <b>sigs</b>, + * allocating a new digests_t as neeeded. */ +static smartlist_t * +detached_get_signatures(ns_detached_signatures_t *sigs, + const char *flavor_name) +{ + smartlist_t *sl = strmap_get(sigs->signatures, flavor_name); + if (!sl) { + sl = smartlist_create(); + strmap_set(sigs->signatures, flavor_name, sl); + } + return sl; +} + /** Parse a detached v3 networkstatus signature document between <b>s</b> and * <b>eos</b> and return the result. Return -1 on failure. */ ns_detached_signatures_t * @@ -2643,10 +3351,13 @@ networkstatus_parse_detached_signatures(const char *s, const char *eos) * networkstatus_parse_vote_from_string(). */ directory_token_t *tok; memarea_t *area = NULL; + digests_t *digests; smartlist_t *tokens = smartlist_create(); ns_detached_signatures_t *sigs = tor_malloc_zero(sizeof(ns_detached_signatures_t)); + sigs->digests = strmap_new(); + sigs->signatures = strmap_new(); if (!eos) eos = s + strlen(s); @@ -2658,18 +3369,57 @@ networkstatus_parse_detached_signatures(const char *s, const char *eos) goto err; } - tok = find_by_keyword(tokens, K_CONSENSUS_DIGEST); - if (strlen(tok->args[0]) != HEX_DIGEST_LEN) { - log_warn(LD_DIR, "Wrong length on consensus-digest in detached " - "networkstatus signatures"); - goto err; - } - if (base16_decode(sigs->networkstatus_digest, DIGEST_LEN, - tok->args[0], strlen(tok->args[0])) < 0) { - log_warn(LD_DIR, "Bad encoding on on consensus-digest in detached " - "networkstatus signatures"); - goto err; - } + /* Grab all the digest-like tokens. */ + SMARTLIST_FOREACH_BEGIN(tokens, directory_token_t *, _tok) { + const char *algname; + digest_algorithm_t alg; + const char *flavor; + const char *hexdigest; + size_t expected_length; + + tok = _tok; + + if (tok->tp == K_CONSENSUS_DIGEST) { + algname = "sha1"; + alg = DIGEST_SHA1; + flavor = "ns"; + hexdigest = tok->args[0]; + } else if (tok->tp == K_ADDITIONAL_DIGEST) { + int a = crypto_digest_algorithm_parse_name(tok->args[1]); + if (a<0) { + log_warn(LD_DIR, "Unrecognized algorithm name %s", tok->args[0]); + continue; + } + alg = (digest_algorithm_t) a; + flavor = tok->args[0]; + algname = tok->args[1]; + hexdigest = tok->args[2]; + } else { + continue; + } + + expected_length = + (alg == DIGEST_SHA1) ? HEX_DIGEST_LEN : HEX_DIGEST256_LEN; + + if (strlen(hexdigest) != expected_length) { + log_warn(LD_DIR, "Wrong length on consensus-digest in detached " + "networkstatus signatures"); + goto err; + } + digests = detached_get_digests(sigs, flavor); + tor_assert(digests); + if (!tor_mem_is_zero(digests->d[alg], DIGEST256_LEN)) { + log_warn(LD_DIR, "Multiple digests for %s with %s on detached " + "signatures document", flavor, algname); + continue; + } + if (base16_decode(digests->d[alg], DIGEST256_LEN, + hexdigest, strlen(hexdigest)) < 0) { + log_warn(LD_DIR, "Bad encoding on consensus-digest in detached " + "networkstatus signatures"); + goto err; + } + } SMARTLIST_FOREACH_END(_tok); tok = find_by_keyword(tokens, K_VALID_AFTER); if (parse_iso_time(tok->args[0], &sigs->valid_after)) { @@ -2689,57 +3439,102 @@ networkstatus_parse_detached_signatures(const char *s, const char *eos) goto err; } - sigs->signatures = smartlist_create(); - SMARTLIST_FOREACH(tokens, directory_token_t *, _tok, - { - char id_digest[DIGEST_LEN]; - char sk_digest[DIGEST_LEN]; - networkstatus_voter_info_t *voter; + SMARTLIST_FOREACH_BEGIN(tokens, directory_token_t *, _tok) { + const char *id_hexdigest; + const char *sk_hexdigest; + const char *algname; + const char *flavor; + digest_algorithm_t alg; + + char id_digest[DIGEST_LEN]; + char sk_digest[DIGEST_LEN]; + smartlist_t *siglist; + document_signature_t *sig; + int is_duplicate; - tok = _tok; - if (tok->tp != K_DIRECTORY_SIGNATURE) - continue; + tok = _tok; + if (tok->tp == K_DIRECTORY_SIGNATURE) { tor_assert(tok->n_args >= 2); + flavor = "ns"; + algname = "sha1"; + id_hexdigest = tok->args[0]; + sk_hexdigest = tok->args[1]; + } else if (tok->tp == K_ADDITIONAL_SIGNATURE) { + tor_assert(tok->n_args >= 4); + flavor = tok->args[0]; + algname = tok->args[1]; + id_hexdigest = tok->args[2]; + sk_hexdigest = tok->args[3]; + } else { + continue; + } - if (!tok->object_type || - strcmp(tok->object_type, "SIGNATURE") || - tok->object_size < 128 || tok->object_size > 512) { - log_warn(LD_DIR, "Bad object type or length on directory-signature"); - goto err; + { + int a = crypto_digest_algorithm_parse_name(algname); + if (a<0) { + log_warn(LD_DIR, "Unrecognized algorithm name %s", algname); + continue; } + alg = (digest_algorithm_t) a; + } - if (strlen(tok->args[0]) != HEX_DIGEST_LEN || - base16_decode(id_digest, sizeof(id_digest), - tok->args[0], HEX_DIGEST_LEN) < 0) { - log_warn(LD_DIR, "Error decoding declared identity %s in " - "network-status vote.", escaped(tok->args[0])); - goto err; - } - if (strlen(tok->args[1]) != HEX_DIGEST_LEN || - base16_decode(sk_digest, sizeof(sk_digest), - tok->args[1], HEX_DIGEST_LEN) < 0) { - log_warn(LD_DIR, "Error decoding declared digest %s in " - "network-status vote.", escaped(tok->args[1])); - goto err; - } + if (!tok->object_type || + strcmp(tok->object_type, "SIGNATURE") || + tok->object_size < 128 || tok->object_size > 512) { + log_warn(LD_DIR, "Bad object type or length on directory-signature"); + goto err; + } - voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t)); - memcpy(voter->identity_digest, id_digest, DIGEST_LEN); - memcpy(voter->signing_key_digest, sk_digest, DIGEST_LEN); - if (tok->object_size >= INT_MAX || tok->object_size >= SIZE_T_CEILING) - goto err; - voter->signature = tor_memdup(tok->object_body, tok->object_size); - voter->signature_len = (int) tok->object_size; + if (strlen(id_hexdigest) != HEX_DIGEST_LEN || + base16_decode(id_digest, sizeof(id_digest), + id_hexdigest, HEX_DIGEST_LEN) < 0) { + log_warn(LD_DIR, "Error decoding declared identity %s in " + "network-status vote.", escaped(id_hexdigest)); + goto err; + } + if (strlen(sk_hexdigest) != HEX_DIGEST_LEN || + base16_decode(sk_digest, sizeof(sk_digest), + sk_hexdigest, HEX_DIGEST_LEN) < 0) { + log_warn(LD_DIR, "Error decoding declared signing key digest %s in " + "network-status vote.", escaped(sk_hexdigest)); + goto err; + } - smartlist_add(sigs->signatures, voter); + siglist = detached_get_signatures(sigs, flavor); + is_duplicate = 0; + SMARTLIST_FOREACH(siglist, document_signature_t *, s, { + if (s->alg == alg && + tor_memeq(id_digest, s->identity_digest, DIGEST_LEN) && + tor_memeq(sk_digest, s->signing_key_digest, DIGEST_LEN)) { + is_duplicate = 1; + } }); + if (is_duplicate) { + log_warn(LD_DIR, "Two signatures with identical keys and algorithm " + "found."); + continue; + } + + sig = tor_malloc_zero(sizeof(document_signature_t)); + sig->alg = alg; + memcpy(sig->identity_digest, id_digest, DIGEST_LEN); + memcpy(sig->signing_key_digest, sk_digest, DIGEST_LEN); + if (tok->object_size >= INT_MAX || tok->object_size >= SIZE_T_CEILING) { + tor_free(sig); + goto err; + } + sig->signature = tor_memdup(tok->object_body, tok->object_size); + sig->signature_len = (int) tok->object_size; + + smartlist_add(siglist, sig); + } SMARTLIST_FOREACH_END(_tok); goto done; err: ns_detached_signatures_free(sigs); sigs = NULL; done: - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); if (area) { DUMP_AREA(area, "detached signatures"); @@ -2795,7 +3590,7 @@ router_parse_addr_policy_item_from_string(const char *s, int assume_action) err: r = NULL; done: - token_free(tok); + token_clear(tok); if (area) { DUMP_AREA(area, "policy item"); memarea_drop_all(area); @@ -2918,9 +3713,8 @@ assert_addr_policy_ok(smartlist_t *lst) /** Free all resources allocated for <b>tok</b> */ static void -token_free(directory_token_t *tok) +token_clear(directory_token_t *tok) { - tor_assert(tok); if (tok->key) crypto_free_pk_env(tok->key); } @@ -2932,7 +3726,7 @@ token_free(directory_token_t *tok) #define RET_ERR(msg) \ STMT_BEGIN \ - if (tok) token_free(tok); \ + if (tok) token_clear(tok); \ tok = ALLOC_ZERO(sizeof(directory_token_t)); \ tok->tp = _ERR; \ tok->error = STRDUP(msg); \ @@ -3228,7 +4022,7 @@ tokenize_string(memarea_t *area, tok = get_next_token(area, s, end, table); if (tok->tp == _ERR) { log_warn(LD_DIR, "parse error: %s", tok->error); - token_free(tok); + token_clear(tok); return -1; } ++counts[tok->tp]; @@ -3342,17 +4136,11 @@ find_all_exitpolicy(smartlist_t *s) return out; } -/** Compute the SHA-1 digest of the substring of <b>s</b> taken from the first - * occurrence of <b>start_str</b> through the first instance of c after the - * first subsequent occurrence of <b>end_str</b>; store the 20-byte result in - * <b>digest</b>; return 0 on success. - * - * If no such substring exists, return -1. - */ static int -router_get_hash_impl(const char *s, size_t s_len, char *digest, +router_get_hash_impl_helper(const char *s, size_t s_len, const char *start_str, - const char *end_str, char end_c) + const char *end_str, char end_c, + const char **start_out, const char **end_out) { const char *start, *end; start = tor_memstr(s, s_len, start_str); @@ -3379,14 +4167,220 @@ router_get_hash_impl(const char *s, size_t s_len, char *digest, } ++end; - if (crypto_digest(digest, start, end-start)) { - log_warn(LD_BUG,"couldn't compute digest"); + *start_out = start; + *end_out = end; + return 0; +} + +/** Compute the digest of the substring of <b>s</b> taken from the first + * occurrence of <b>start_str</b> through the first instance of c after the + * first subsequent occurrence of <b>end_str</b>; store the 20-byte result in + * <b>digest</b>; return 0 on success. + * + * If no such substring exists, return -1. + */ +static int +router_get_hash_impl(const char *s, size_t s_len, char *digest, + const char *start_str, + const char *end_str, char end_c, + digest_algorithm_t alg) +{ + const char *start=NULL, *end=NULL; + if (router_get_hash_impl_helper(s,s_len,start_str,end_str,end_c, + &start,&end)<0) + return -1; + + if (alg == DIGEST_SHA1) { + if (crypto_digest(digest, start, end-start)) { + log_warn(LD_BUG,"couldn't compute digest"); + return -1; + } + } else { + if (crypto_digest256(digest, start, end-start, alg)) { + log_warn(LD_BUG,"couldn't compute digest"); + return -1; + } + } + + return 0; +} + +/** As router_get_hash_impl, but compute all hashes. */ +static int +router_get_hashes_impl(const char *s, size_t s_len, digests_t *digests, + const char *start_str, + const char *end_str, char end_c) +{ + const char *start=NULL, *end=NULL; + if (router_get_hash_impl_helper(s,s_len,start_str,end_str,end_c, + &start,&end)<0) + return -1; + + if (crypto_digest_all(digests, start, end-start)) { + log_warn(LD_BUG,"couldn't compute digests"); return -1; } return 0; } +/** Assuming that s starts with a microdesc, return the start of the + * *NEXT* one. Return NULL on "not found." */ +static const char * +find_start_of_next_microdesc(const char *s, const char *eos) +{ + int started_with_annotations; + s = eat_whitespace_eos(s, eos); + if (!s) + return NULL; + +#define CHECK_LENGTH() STMT_BEGIN \ + if (s+32 > eos) \ + return NULL; \ + STMT_END + +#define NEXT_LINE() STMT_BEGIN \ + s = memchr(s, '\n', eos-s); \ + if (!s || s+1 >= eos) \ + return NULL; \ + s++; \ + STMT_END + + CHECK_LENGTH(); + + started_with_annotations = (*s == '@'); + + if (started_with_annotations) { + /* Start by advancing to the first non-annotation line. */ + while (*s == '@') + NEXT_LINE(); + } + CHECK_LENGTH(); + + /* Now we should be pointed at an onion-key line. If we are, then skip + * it. */ + if (!strcmpstart(s, "onion-key")) + NEXT_LINE(); + + /* Okay, now we're pointed at the first line of the microdescriptor which is + not an annotation or onion-key. The next line that _is_ an annotation or + onion-key is the start of the next microdescriptor. */ + while (s+32 < eos) { + if (*s == '@' || !strcmpstart(s, "onion-key")) + return s; + NEXT_LINE(); + } + return NULL; + +#undef CHECK_LENGTH +#undef NEXT_LINE +} + +/** Parse as many microdescriptors as are found from the string starting at + * <b>s</b> and ending at <b>eos</b>. If allow_annotations is set, read any + * annotations we recognize and ignore ones we don't. If <b>copy_body</b> is + * true, then strdup the bodies of the microdescriptors. Return all newly + * parsed microdescriptors in a newly allocated smartlist_t. */ +smartlist_t * +microdescs_parse_from_string(const char *s, const char *eos, + int allow_annotations, int copy_body) +{ + smartlist_t *tokens; + smartlist_t *result; + microdesc_t *md = NULL; + memarea_t *area; + const char *start = s; + const char *start_of_next_microdesc; + int flags = allow_annotations ? TS_ANNOTATIONS_OK : 0; + + directory_token_t *tok; + + if (!eos) + eos = s + strlen(s); + + s = eat_whitespace_eos(s, eos); + area = memarea_new(); + result = smartlist_create(); + tokens = smartlist_create(); + + while (s < eos) { + start_of_next_microdesc = find_start_of_next_microdesc(s, eos); + if (!start_of_next_microdesc) + start_of_next_microdesc = eos; + + if (tokenize_string(area, s, start_of_next_microdesc, tokens, + microdesc_token_table, flags)) { + log_warn(LD_DIR, "Unparseable microdescriptor"); + goto next; + } + + md = tor_malloc_zero(sizeof(microdesc_t)); + { + const char *cp = tor_memstr(s, start_of_next_microdesc-s, + "onion-key"); + tor_assert(cp); + + md->bodylen = start_of_next_microdesc - cp; + if (copy_body) + md->body = tor_strndup(cp, md->bodylen); + else + md->body = (char*)cp; + md->off = cp - start; + } + + if ((tok = find_opt_by_keyword(tokens, A_LAST_LISTED))) { + if (parse_iso_time(tok->args[0], &md->last_listed)) { + log_warn(LD_DIR, "Bad last-listed time in microdescriptor"); + goto next; + } + } + + tok = find_by_keyword(tokens, K_ONION_KEY); + if (!crypto_pk_public_exponent_ok(tok->key)) { + log_warn(LD_DIR, + "Relay's onion key had invalid exponent."); + goto next; + } + md->onion_pkey = tok->key; + tok->key = NULL; + + if ((tok = find_opt_by_keyword(tokens, K_FAMILY))) { + int i; + md->family = smartlist_create(); + for (i=0;i<tok->n_args;++i) { + if (!is_legal_nickname_or_hexdigest(tok->args[i])) { + log_warn(LD_DIR, "Illegal nickname %s in family line", + escaped(tok->args[i])); + goto next; + } + smartlist_add(md->family, tor_strdup(tok->args[i])); + } + } + + if ((tok = find_opt_by_keyword(tokens, K_P))) { + md->exitsummary = tor_strdup(tok->args[0]); + } + + crypto_digest256(md->digest, md->body, md->bodylen, DIGEST_SHA256); + + smartlist_add(result, md); + + md = NULL; + next: + microdesc_free(md); + md = NULL; + + memarea_clear(area); + smartlist_clear(tokens); + s = start_of_next_microdesc; + } + + memarea_drop_all(area); + smartlist_free(tokens); + + return result; +} + /** Parse the Tor version of the platform string <b>platform</b>, * and compare it to the version in <b>cutoff</b>. Return 1 if * the router is at least as new as the cutoff, else return 0. @@ -3411,7 +4405,7 @@ tor_version_as_new_as(const char *platform, const char *cutoff) if (!*start) return 0; s = (char *)find_whitespace(start); /* also finds '\0', which is fine */ s2 = (char*)eat_whitespace(s); - if (!strcmpstart(s2, "(r")) + if (!strcmpstart(s2, "(r") || !strcmpstart(s2, "(git-")) s = (char*)find_whitespace(s2); if ((size_t)(s-start+1) >= sizeof(tmp)) /* too big, no */ @@ -3507,6 +4501,23 @@ tor_version_parse(const char *s, tor_version_t *out) if (!strcmpstart(cp, "(r")) { cp += 2; out->svn_revision = (int) strtol(cp,&eos,10); + } else if (!strcmpstart(cp, "(git-")) { + char *close_paren = strchr(cp, ')'); + int hexlen; + char digest[DIGEST_LEN]; + if (! close_paren) + return -1; + cp += 5; + if (close_paren-cp > HEX_DIGEST_LEN) + return -1; + hexlen = (int)(close_paren-cp); + memset(digest, 0, sizeof(digest)); + if ( hexlen == 0 || (hexlen % 2) == 1) + return -1; + if (base16_decode(digest, hexlen/2, cp, hexlen)) + return -1; + memcpy(out->git_tag, digest, hexlen/2); + out->git_tag_len = hexlen/2; } return 0; @@ -3532,8 +4543,14 @@ tor_version_compare(tor_version_t *a, tor_version_t *b) return i; else if ((i = strcmp(a->status_tag, b->status_tag))) return i; + else if ((i = a->svn_revision - b->svn_revision)) + return i; + else if ((i = a->git_tag_len - b->git_tag_len)) + return i; + else if (a->git_tag_len) + return fast_memcmp(a->git_tag, b->git_tag, a->git_tag_len); else - return a->svn_revision - b->svn_revision; + return 0; } /** Return true iff versions <b>a</b> and <b>b</b> belong to the same series. @@ -3622,7 +4639,7 @@ rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out, /* Compute descriptor hash for later validation. */ if (router_get_hash_impl(desc, strlen(desc), desc_hash, "rendezvous-service-descriptor ", - "\nsignature", '\n') < 0) { + "\nsignature", '\n', DIGEST_SHA1) < 0) { log_warn(LD_REND, "Couldn't compute descriptor hash."); goto err; } @@ -3633,12 +4650,12 @@ rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out, else eos = eos + 1; /* Check length. */ - if (strlen(desc) > REND_DESC_MAX_SIZE) { + if (eos-desc > REND_DESC_MAX_SIZE) { /* XXX023 If we are parsing this descriptor as a server, this * should be a protocol warning. */ - log_warn(LD_REND, "Descriptor length is %i which exceeds " - "maximum rendezvous descriptor size of %i bytes.", - (int)strlen(desc), REND_DESC_MAX_SIZE); + log_warn(LD_REND, "Descriptor length is %d which exceeds " + "maximum rendezvous descriptor size of %d bytes.", + (int)(eos-desc), REND_DESC_MAX_SIZE); goto err; } /* Tokenize descriptor. */ @@ -3743,7 +4760,7 @@ rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out, /* Parse and verify signature. */ tok = find_by_keyword(tokens, R_SIGNATURE); note_crypto_pk_op(VERIFY_RTR); - if (check_signature_token(desc_hash, tok, result->pk, 0, + if (check_signature_token(desc_hash, DIGEST_LEN, tok, result->pk, 0, "v2 rendezvous service descriptor") < 0) goto err; /* Verify that descriptor ID belongs to public key and secret ID part. */ @@ -3757,12 +4774,11 @@ rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out, } goto done; err: - if (result) - rend_service_descriptor_free(result); + rend_service_descriptor_free(result); result = NULL; done: if (tokens) { - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); } if (area) @@ -3920,7 +4936,7 @@ rend_parse_introduction_points(rend_service_descriptor_t *parsed, eos = eos+1; tor_assert(eos <= intro_points_encoded+intro_points_encoded_size); /* Free tokens and clear token list. */ - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_clear(tokens); memarea_clear(area); /* Tokenize string. */ @@ -4005,7 +5021,7 @@ rend_parse_introduction_points(rend_service_descriptor_t *parsed, done: /* Free tokens and clear token list. */ - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); if (area) memarea_drop_all(area); @@ -4044,7 +5060,7 @@ rend_parse_client_keys(strmap_t *parsed_clients, const char *ckstr) else eos = eos + 1; /* Free tokens and clear token list. */ - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_clear(tokens); memarea_clear(area); /* Tokenize string. */ @@ -4116,7 +5132,7 @@ rend_parse_client_keys(strmap_t *parsed_clients, const char *ckstr) result = -1; done: /* Free tokens and clear token list. */ - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); if (area) memarea_drop_all(area); diff --git a/src/or/routerparse.h b/src/or/routerparse.h new file mode 100644 index 0000000000..8b8cde25f6 --- /dev/null +++ b/src/or/routerparse.h @@ -0,0 +1,86 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file routerparse.h + * \brief Header file for routerparse.c. + **/ + +#ifndef _TOR_ROUTERPARSE_H +#define _TOR_ROUTERPARSE_H + +int router_get_router_hash(const char *s, size_t s_len, char *digest); +int router_get_dir_hash(const char *s, char *digest); +int router_get_runningrouters_hash(const char *s, char *digest); +int router_get_networkstatus_v2_hash(const char *s, char *digest); +int router_get_networkstatus_v3_hash(const char *s, char *digest, + digest_algorithm_t algorithm); +int router_get_networkstatus_v3_hashes(const char *s, digests_t *digests); +int router_get_extrainfo_hash(const char *s, char *digest); +#define DIROBJ_MAX_SIG_LEN 256 +int router_append_dirobj_signature(char *buf, size_t buf_len, + const char *digest, + size_t digest_len, + crypto_pk_env_t *private_key); +int router_parse_list_from_string(const char **s, const char *eos, + smartlist_t *dest, + saved_location_t saved_location, + int is_extrainfo, + int allow_annotations, + const char *prepend_annotations); +int router_parse_runningrouters(const char *str); +int router_parse_directory(const char *str); + +routerinfo_t *router_parse_entry_from_string(const char *s, const char *end, + int cache_copy, + int allow_annotations, + const char *prepend_annotations); +extrainfo_t *extrainfo_parse_entry_from_string(const char *s, const char *end, + int cache_copy, struct digest_ri_map_t *routermap); +addr_policy_t *router_parse_addr_policy_item_from_string(const char *s, + int assume_action); +version_status_t tor_version_is_obsolete(const char *myversion, + const char *versionlist); +int tor_version_parse(const char *s, tor_version_t *out); +int tor_version_as_new_as(const char *platform, const char *cutoff); +int tor_version_compare(tor_version_t *a, tor_version_t *b); +void sort_version_list(smartlist_t *lst, int remove_duplicates); +void assert_addr_policy_ok(smartlist_t *t); +void dump_distinct_digest_count(int severity); + +int compare_routerstatus_entries(const void **_a, const void **_b); +networkstatus_v2_t *networkstatus_v2_parse_from_string(const char *s); +int networkstatus_verify_bw_weights(networkstatus_t *ns); +networkstatus_t *networkstatus_parse_vote_from_string(const char *s, + const char **eos_out, + networkstatus_type_t ns_type); +ns_detached_signatures_t *networkstatus_parse_detached_signatures( + const char *s, const char *eos); + +smartlist_t *microdescs_parse_from_string(const char *s, const char *eos, + int allow_annotations, + int copy_body); + +authority_cert_t *authority_cert_parse_from_string(const char *s, + const char **end_of_string); +int rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out, + char *desc_id_out, + char **intro_points_encrypted_out, + size_t *intro_points_encrypted_size_out, + size_t *encoded_size_out, + const char **next_out, const char *desc); +int rend_decrypt_introduction_points(char **ipos_decrypted, + size_t *ipos_decrypted_size, + const char *descriptor_cookie, + const char *ipos_encrypted, + size_t ipos_encrypted_size); +int rend_parse_introduction_points(rend_service_descriptor_t *parsed, + const char *intro_points_encoded, + size_t intro_points_encoded_size); +int rend_parse_client_keys(strmap_t *parsed_clients, const char *str); + +#endif + diff --git a/src/or/test.c b/src/or/test.c deleted file mode 100644 index dc71db4e1d..0000000000 --- a/src/or/test.c +++ /dev/null @@ -1,4973 +0,0 @@ -/* Copyright (c) 2001-2004, Roger Dingledine. - * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2011, The Tor Project, Inc. */ -/* See LICENSE for licensing information */ - -/* Ordinarily defined in tor_main.c; this bit is just here to provide one - * since we're not linking to tor_main.c */ -const char tor_svn_revision[] = ""; - -/** - * \file test.c - * \brief Unit tests for many pieces of the lower level Tor modules. - **/ - -#include "orconfig.h" - -#include <stdio.h> -#ifdef HAVE_FCNTL_H -#include <fcntl.h> -#endif - -#ifdef MS_WINDOWS -/* For mkdir() */ -#include <direct.h> -#else -#include <dirent.h> -#endif - -/* These macros pull in declarations for some functions and structures that - * are typically file-private. */ -#define BUFFERS_PRIVATE -#define CONFIG_PRIVATE -#define CONTROL_PRIVATE -#define CRYPTO_PRIVATE -#define DIRSERV_PRIVATE -#define DIRVOTE_PRIVATE -#define GEOIP_PRIVATE -#define MEMPOOL_PRIVATE -#define ROUTER_PRIVATE - -#include "or.h" -#include "test.h" -#include "torgzip.h" -#include "mempool.h" -#include "memarea.h" -#include "di_ops.h" - -#ifdef USE_DMALLOC -#include <dmalloc.h> -#include <openssl/crypto.h> -#endif - -/** Set to true if any unit test has failed. Mostly, this is set by the macros - * in test.h */ -int have_failed = 0; - -/** Temporary directory (set up by setup_directory) under which we store all - * our files during testing. */ -static char temp_dir[256]; - -/** Select and create the temporary directory we'll use to run our unit tests. - * Store it in <b>temp_dir</b>. Exit immediately if we can't create it. - * idempotent. */ -static void -setup_directory(void) -{ - static int is_setup = 0; - int r; - if (is_setup) return; - -#ifdef MS_WINDOWS - // XXXX - tor_snprintf(temp_dir, sizeof(temp_dir), - "c:\\windows\\temp\\tor_test_%d", (int)getpid()); - r = mkdir(temp_dir); -#else - tor_snprintf(temp_dir, sizeof(temp_dir), "/tmp/tor_test_%d", (int) getpid()); - r = mkdir(temp_dir, 0700); -#endif - if (r) { - fprintf(stderr, "Can't create directory %s:", temp_dir); - perror(""); - exit(1); - } - is_setup = 1; -} - -/** Return a filename relative to our testing temporary directory */ -static const char * -get_fname(const char *name) -{ - static char buf[1024]; - setup_directory(); - tor_snprintf(buf,sizeof(buf),"%s/%s",temp_dir,name); - return buf; -} - -/** Remove all files stored under the temporary directory, and the directory - * itself. */ -static void -remove_directory(void) -{ - smartlist_t *elements = tor_listdir(temp_dir); - if (elements) { - SMARTLIST_FOREACH(elements, const char *, cp, - { - size_t len = strlen(cp)+strlen(temp_dir)+16; - char *tmp = tor_malloc(len); - tor_snprintf(tmp, len, "%s"PATH_SEPARATOR"%s", temp_dir, cp); - unlink(tmp); - tor_free(tmp); - }); - SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp)); - smartlist_free(elements); - } - rmdir(temp_dir); -} - -/** Define this if unit tests spend too much time generating public keys*/ -#undef CACHE_GENERATED_KEYS - -static crypto_pk_env_t *pregen_keys[5] = {NULL, NULL, NULL, NULL, NULL}; -#define N_PREGEN_KEYS ((int)(sizeof(pregen_keys)/sizeof(pregen_keys[0]))) - -/** Generate and return a new keypair for use in unit tests. If we're using - * the key cache optimization, we might reuse keys: we only guarantee that - * keys made with distinct values for <b>idx</b> are different. The value of - * <b>idx</b> must be at least 0, and less than N_PREGEN_KEYS. */ -static crypto_pk_env_t * -pk_generate(int idx) -{ -#ifdef CACHE_GENERATED_KEYS - tor_assert(idx < N_PREGEN_KEYS); - if (! pregen_keys[idx]) { - pregen_keys[idx] = crypto_new_pk_env(); - tor_assert(!crypto_pk_generate_key(pregen_keys[idx])); - } - return crypto_pk_dup_key(pregen_keys[idx]); -#else - crypto_pk_env_t *result; - (void) idx; - result = crypto_new_pk_env(); - tor_assert(!crypto_pk_generate_key(result)); - return result; -#endif -} - -/** Free all storage used for the cached key optimization. */ -static void -free_pregenerated_keys(void) -{ - unsigned idx; - for (idx = 0; idx < N_PREGEN_KEYS; ++idx) { - if (pregen_keys[idx]) { - crypto_free_pk_env(pregen_keys[idx]); - pregen_keys[idx] = NULL; - } - } -} - -/** Run unit tests for buffers.c */ -static void -test_buffers(void) -{ - char str[256]; - char str2[256]; - - buf_t *buf = NULL, *buf2 = NULL; - const char *cp; - - int j; - size_t r; - - /**** - * buf_new - ****/ - if (!(buf = buf_new())) - test_fail(); - - //test_eq(buf_capacity(buf), 4096); - test_eq(buf_datalen(buf), 0); - - /**** - * General pointer frobbing - */ - for (j=0;j<256;++j) { - str[j] = (char)j; - } - write_to_buf(str, 256, buf); - write_to_buf(str, 256, buf); - test_eq(buf_datalen(buf), 512); - fetch_from_buf(str2, 200, buf); - test_memeq(str, str2, 200); - test_eq(buf_datalen(buf), 312); - memset(str2, 0, sizeof(str2)); - - fetch_from_buf(str2, 256, buf); - test_memeq(str+200, str2, 56); - test_memeq(str, str2+56, 200); - test_eq(buf_datalen(buf), 56); - memset(str2, 0, sizeof(str2)); - /* Okay, now we should be 512 bytes into the 4096-byte buffer. If we add - * another 3584 bytes, we hit the end. */ - for (j=0;j<15;++j) { - write_to_buf(str, 256, buf); - } - assert_buf_ok(buf); - test_eq(buf_datalen(buf), 3896); - fetch_from_buf(str2, 56, buf); - test_eq(buf_datalen(buf), 3840); - test_memeq(str+200, str2, 56); - for (j=0;j<15;++j) { - memset(str2, 0, sizeof(str2)); - fetch_from_buf(str2, 256, buf); - test_memeq(str, str2, 256); - } - test_eq(buf_datalen(buf), 0); - buf_free(buf); - buf = NULL; - - /* Okay, now make sure growing can work. */ - buf = buf_new_with_capacity(16); - //test_eq(buf_capacity(buf), 16); - write_to_buf(str+1, 255, buf); - //test_eq(buf_capacity(buf), 256); - fetch_from_buf(str2, 254, buf); - test_memeq(str+1, str2, 254); - //test_eq(buf_capacity(buf), 256); - assert_buf_ok(buf); - write_to_buf(str, 32, buf); - //test_eq(buf_capacity(buf), 256); - assert_buf_ok(buf); - write_to_buf(str, 256, buf); - assert_buf_ok(buf); - //test_eq(buf_capacity(buf), 512); - test_eq(buf_datalen(buf), 33+256); - fetch_from_buf(str2, 33, buf); - test_eq(*str2, str[255]); - - test_memeq(str2+1, str, 32); - //test_eq(buf_capacity(buf), 512); - test_eq(buf_datalen(buf), 256); - fetch_from_buf(str2, 256, buf); - test_memeq(str, str2, 256); - - /* now try shrinking: case 1. */ - buf_free(buf); - buf = buf_new_with_capacity(33668); - for (j=0;j<67;++j) { - write_to_buf(str,255, buf); - } - //test_eq(buf_capacity(buf), 33668); - test_eq(buf_datalen(buf), 17085); - for (j=0; j < 40; ++j) { - fetch_from_buf(str2, 255,buf); - test_memeq(str2, str, 255); - } - - /* now try shrinking: case 2. */ - buf_free(buf); - buf = buf_new_with_capacity(33668); - for (j=0;j<67;++j) { - write_to_buf(str,255, buf); - } - for (j=0; j < 20; ++j) { - fetch_from_buf(str2, 255,buf); - test_memeq(str2, str, 255); - } - for (j=0;j<80;++j) { - write_to_buf(str,255, buf); - } - //test_eq(buf_capacity(buf),33668); - for (j=0; j < 120; ++j) { - fetch_from_buf(str2, 255,buf); - test_memeq(str2, str, 255); - } - - /* Move from buf to buf. */ - buf_free(buf); - buf = buf_new_with_capacity(4096); - buf2 = buf_new_with_capacity(4096); - for (j=0;j<100;++j) - write_to_buf(str, 255, buf); - test_eq(buf_datalen(buf), 25500); - for (j=0;j<100;++j) { - r = 10; - move_buf_to_buf(buf2, buf, &r); - test_eq(r, 0); - } - test_eq(buf_datalen(buf), 24500); - test_eq(buf_datalen(buf2), 1000); - for (j=0;j<3;++j) { - fetch_from_buf(str2, 255, buf2); - test_memeq(str2, str, 255); - } - r = 8192; /*big move*/ - move_buf_to_buf(buf2, buf, &r); - test_eq(r, 0); - r = 30000; /* incomplete move */ - move_buf_to_buf(buf2, buf, &r); - test_eq(r, 13692); - for (j=0;j<97;++j) { - fetch_from_buf(str2, 255, buf2); - test_memeq(str2, str, 255); - } - buf_free(buf); - buf_free(buf2); - buf = buf2 = NULL; - - buf = buf_new_with_capacity(5); - cp = "Testing. This is a moderately long Testing string."; - for (j = 0; cp[j]; j++) - write_to_buf(cp+j, 1, buf); - test_eq(0, buf_find_string_offset(buf, "Testing", 7)); - test_eq(1, buf_find_string_offset(buf, "esting", 6)); - test_eq(1, buf_find_string_offset(buf, "est", 3)); - test_eq(39, buf_find_string_offset(buf, "ing str", 7)); - test_eq(35, buf_find_string_offset(buf, "Testing str", 11)); - test_eq(32, buf_find_string_offset(buf, "ng ", 3)); - test_eq(43, buf_find_string_offset(buf, "string.", 7)); - test_eq(-1, buf_find_string_offset(buf, "shrdlu", 6)); - test_eq(-1, buf_find_string_offset(buf, "Testing thing", 13)); - test_eq(-1, buf_find_string_offset(buf, "ngx", 3)); - buf_free(buf); - buf = NULL; - -#if 0 - { - int s; - int eof; - int i; - buf_t *buf2; - /**** - * read_to_buf - ****/ - s = open(get_fname("data"), O_WRONLY|O_CREAT|O_TRUNC, 0600); - write(s, str, 256); - close(s); - - s = open(get_fname("data"), O_RDONLY, 0); - eof = 0; - errno = 0; /* XXXX */ - i = read_to_buf(s, 10, buf, &eof); - printf("%s\n", strerror(errno)); - test_eq(i, 10); - test_eq(eof, 0); - //test_eq(buf_capacity(buf), 4096); - test_eq(buf_datalen(buf), 10); - - test_memeq(str, (char*)_buf_peek_raw_buffer(buf), 10); - - /* Test reading 0 bytes. */ - i = read_to_buf(s, 0, buf, &eof); - //test_eq(buf_capacity(buf), 512*1024); - test_eq(buf_datalen(buf), 10); - test_eq(eof, 0); - test_eq(i, 0); - - /* Now test when buffer is filled exactly. */ - buf2 = buf_new_with_capacity(6); - i = read_to_buf(s, 6, buf2, &eof); - //test_eq(buf_capacity(buf2), 6); - test_eq(buf_datalen(buf2), 6); - test_eq(eof, 0); - test_eq(i, 6); - test_memeq(str+10, (char*)_buf_peek_raw_buffer(buf2), 6); - buf_free(buf2); - buf2 = NULL; - - /* Now test when buffer is filled with more data to read. */ - buf2 = buf_new_with_capacity(32); - i = read_to_buf(s, 128, buf2, &eof); - //test_eq(buf_capacity(buf2), 128); - test_eq(buf_datalen(buf2), 32); - test_eq(eof, 0); - test_eq(i, 32); - buf_free(buf2); - buf2 = NULL; - - /* Now read to eof. */ - test_assert(buf_capacity(buf) > 256); - i = read_to_buf(s, 1024, buf, &eof); - test_eq(i, (256-32-10-6)); - test_eq(buf_capacity(buf), MAX_BUF_SIZE); - test_eq(buf_datalen(buf), 256-6-32); - test_memeq(str, (char*)_buf_peek_raw_buffer(buf), 10); /* XXX Check rest. */ - test_eq(eof, 0); - - i = read_to_buf(s, 1024, buf, &eof); - test_eq(i, 0); - test_eq(buf_capacity(buf), MAX_BUF_SIZE); - test_eq(buf_datalen(buf), 256-6-32); - test_eq(eof, 1); - } -#endif - - done: - if (buf) - buf_free(buf); - if (buf2) - buf_free(buf2); -} - -/** Run unit tests for Diffie-Hellman functionality. */ -static void -test_crypto_dh(void) -{ - crypto_dh_env_t *dh1 = crypto_dh_new(DH_TYPE_CIRCUIT); - crypto_dh_env_t *dh2 = crypto_dh_new(DH_TYPE_CIRCUIT); - char p1[DH_BYTES]; - char p2[DH_BYTES]; - char s1[DH_BYTES]; - char s2[DH_BYTES]; - ssize_t s1len, s2len; - - test_eq(crypto_dh_get_bytes(dh1), DH_BYTES); - test_eq(crypto_dh_get_bytes(dh2), DH_BYTES); - - memset(p1, 0, DH_BYTES); - memset(p2, 0, DH_BYTES); - test_memeq(p1, p2, DH_BYTES); - test_assert(! crypto_dh_get_public(dh1, p1, DH_BYTES)); - test_memneq(p1, p2, DH_BYTES); - test_assert(! crypto_dh_get_public(dh2, p2, DH_BYTES)); - test_memneq(p1, p2, DH_BYTES); - - memset(s1, 0, DH_BYTES); - memset(s2, 0xFF, DH_BYTES); - s1len = crypto_dh_compute_secret(dh1, p2, DH_BYTES, s1, 50); - s2len = crypto_dh_compute_secret(dh2, p1, DH_BYTES, s2, 50); - test_assert(s1len > 0); - test_eq(s1len, s2len); - test_memeq(s1, s2, s1len); - - { - /* XXXX Now fabricate some bad values and make sure they get caught, - * Check 0, 1, N-1, >= N, etc. - */ - } - - done: - crypto_dh_free(dh1); - crypto_dh_free(dh2); -} - -/** Run unit tests for our random number generation function and its wrappers. - */ -static void -test_crypto_rng(void) -{ - int i, j, allok; - char data1[100], data2[100]; - - /* Try out RNG. */ - test_assert(! crypto_seed_rng(0)); - crypto_rand(data1, 100); - crypto_rand(data2, 100); - test_memneq(data1,data2,100); - allok = 1; - for (i = 0; i < 100; ++i) { - uint64_t big; - char *host; - j = crypto_rand_int(100); - if (i < 0 || i >= 100) - allok = 0; - big = crypto_rand_uint64(U64_LITERAL(1)<<40); - if (big >= (U64_LITERAL(1)<<40)) - allok = 0; - big = crypto_rand_uint64(U64_LITERAL(5)); - if (big >= 5) - allok = 0; - host = crypto_random_hostname(3,8,"www.",".onion"); - if (strcmpstart(host,"www.") || - strcmpend(host,".onion") || - strlen(host) < 13 || - strlen(host) > 18) - allok = 0; - tor_free(host); - } - test_assert(allok); - done: - ; -} - -/** Run unit tests for our AES functionality */ -static void -test_crypto_aes(void) -{ - char *data1 = NULL, *data2 = NULL, *data3 = NULL; - crypto_cipher_env_t *env1 = NULL, *env2 = NULL; - int i, j; - - data1 = tor_malloc(1024); - data2 = tor_malloc(1024); - data3 = tor_malloc(1024); - - /* Now, test encryption and decryption with stream cipher. */ - data1[0]='\0'; - for (i = 1023; i>0; i -= 35) - strncat(data1, "Now is the time for all good onions", i); - - memset(data2, 0, 1024); - memset(data3, 0, 1024); - env1 = crypto_new_cipher_env(); - test_neq(env1, 0); - env2 = crypto_new_cipher_env(); - test_neq(env2, 0); - j = crypto_cipher_generate_key(env1); - crypto_cipher_set_key(env2, crypto_cipher_get_key(env1)); - crypto_cipher_encrypt_init_cipher(env1); - crypto_cipher_decrypt_init_cipher(env2); - - /* Try encrypting 512 chars. */ - crypto_cipher_encrypt(env1, data2, data1, 512); - crypto_cipher_decrypt(env2, data3, data2, 512); - test_memeq(data1, data3, 512); - test_memneq(data1, data2, 512); - - /* Now encrypt 1 at a time, and get 1 at a time. */ - for (j = 512; j < 560; ++j) { - crypto_cipher_encrypt(env1, data2+j, data1+j, 1); - } - for (j = 512; j < 560; ++j) { - crypto_cipher_decrypt(env2, data3+j, data2+j, 1); - } - test_memeq(data1, data3, 560); - /* Now encrypt 3 at a time, and get 5 at a time. */ - for (j = 560; j < 1024-5; j += 3) { - crypto_cipher_encrypt(env1, data2+j, data1+j, 3); - } - for (j = 560; j < 1024-5; j += 5) { - crypto_cipher_decrypt(env2, data3+j, data2+j, 5); - } - test_memeq(data1, data3, 1024-5); - /* Now make sure that when we encrypt with different chunk sizes, we get - the same results. */ - crypto_free_cipher_env(env2); - env2 = NULL; - - memset(data3, 0, 1024); - env2 = crypto_new_cipher_env(); - test_neq(env2, 0); - crypto_cipher_set_key(env2, crypto_cipher_get_key(env1)); - crypto_cipher_encrypt_init_cipher(env2); - for (j = 0; j < 1024-16; j += 17) { - crypto_cipher_encrypt(env2, data3+j, data1+j, 17); - } - for (j= 0; j < 1024-16; ++j) { - if (data2[j] != data3[j]) { - printf("%d: %d\t%d\n", j, (int) data2[j], (int) data3[j]); - } - } - test_memeq(data2, data3, 1024-16); - crypto_free_cipher_env(env1); - env1 = NULL; - crypto_free_cipher_env(env2); - env2 = NULL; - - /* NIST test vector for aes. */ - env1 = crypto_new_cipher_env(); /* IV starts at 0 */ - crypto_cipher_set_key(env1, "\x80\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00"); - crypto_cipher_encrypt_init_cipher(env1); - crypto_cipher_encrypt(env1, data1, - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00", 16); - test_memeq_hex(data1, "0EDD33D3C621E546455BD8BA1418BEC8"); - - /* Now test rollover. All these values are originally from a python - * script. */ - crypto_cipher_set_iv(env1, "\x00\x00\x00\x00\x00\x00\x00\x00" - "\xff\xff\xff\xff\xff\xff\xff\xff"); - memset(data2, 0, 1024); - crypto_cipher_encrypt(env1, data1, data2, 32); - test_memeq_hex(data1, "335fe6da56f843199066c14a00a40231" - "cdd0b917dbc7186908a6bfb5ffd574d3"); - - crypto_cipher_set_iv(env1, "\x00\x00\x00\x00\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff"); - memset(data2, 0, 1024); - crypto_cipher_encrypt(env1, data1, data2, 32); - test_memeq_hex(data1, "e627c6423fa2d77832a02b2794094b73" - "3e63c721df790d2c6469cc1953a3ffac"); - - crypto_cipher_set_iv(env1, "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff"); - memset(data2, 0, 1024); - crypto_cipher_encrypt(env1, data1, data2, 32); - test_memeq_hex(data1, "2aed2bff0de54f9328efd070bf48f70a" - "0EDD33D3C621E546455BD8BA1418BEC8"); - - /* Now check rollover on inplace cipher. */ - crypto_cipher_set_iv(env1, "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff"); - crypto_cipher_crypt_inplace(env1, data2, 64); - test_memeq_hex(data2, "2aed2bff0de54f9328efd070bf48f70a" - "0EDD33D3C621E546455BD8BA1418BEC8" - "93e2c5243d6839eac58503919192f7ae" - "1908e67cafa08d508816659c2e693191"); - crypto_cipher_set_iv(env1, "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff"); - crypto_cipher_crypt_inplace(env1, data2, 64); - test_assert(tor_mem_is_zero(data2, 64)); - - done: - if (env1) - crypto_free_cipher_env(env1); - if (env2) - crypto_free_cipher_env(env2); - tor_free(data1); - tor_free(data2); - tor_free(data3); -} - -/** Run unit tests for our SHA-1 functionality */ -static void -test_crypto_sha(void) -{ - crypto_digest_env_t *d1 = NULL, *d2 = NULL; - int i; - char key[80]; - char digest[20]; - char data[50]; - char d_out1[DIGEST_LEN], d_out2[DIGEST_LEN]; - - /* Test SHA-1 with a test vector from the specification. */ - i = crypto_digest(data, "abc", 3); - test_memeq_hex(data, "A9993E364706816ABA3E25717850C26C9CD0D89D"); - - /* Test HMAC-SHA-1 with test cases from RFC2202. */ - - /* Case 1. */ - memset(key, 0x0b, 20); - crypto_hmac_sha1(digest, key, 20, "Hi There", 8); - test_streq(hex_str(digest, 20), - "B617318655057264E28BC0B6FB378C8EF146BE00"); - /* Case 2. */ - crypto_hmac_sha1(digest, "Jefe", 4, "what do ya want for nothing?", 28); - test_streq(hex_str(digest, 20), - "EFFCDF6AE5EB2FA2D27416D5F184DF9C259A7C79"); - - /* Case 4. */ - base16_decode(key, 25, - "0102030405060708090a0b0c0d0e0f10111213141516171819", 50); - memset(data, 0xcd, 50); - crypto_hmac_sha1(digest, key, 25, data, 50); - test_streq(hex_str(digest, 20), - "4C9007F4026250C6BC8414F9BF50C86C2D7235DA"); - - /* Case . */ - memset(key, 0xaa, 80); - crypto_hmac_sha1(digest, key, 80, - "Test Using Larger Than Block-Size Key - Hash Key First", - 54); - test_streq(hex_str(digest, 20), - "AA4AE5E15272D00E95705637CE8A3B55ED402112"); - - /* Incremental digest code. */ - d1 = crypto_new_digest_env(); - test_assert(d1); - crypto_digest_add_bytes(d1, "abcdef", 6); - d2 = crypto_digest_dup(d1); - test_assert(d2); - crypto_digest_add_bytes(d2, "ghijkl", 6); - crypto_digest_get_digest(d2, d_out1, sizeof(d_out1)); - crypto_digest(d_out2, "abcdefghijkl", 12); - test_memeq(d_out1, d_out2, DIGEST_LEN); - crypto_digest_assign(d2, d1); - crypto_digest_add_bytes(d2, "mno", 3); - crypto_digest_get_digest(d2, d_out1, sizeof(d_out1)); - crypto_digest(d_out2, "abcdefmno", 9); - test_memeq(d_out1, d_out2, DIGEST_LEN); - crypto_digest_get_digest(d1, d_out1, sizeof(d_out1)); - crypto_digest(d_out2, "abcdef", 6); - test_memeq(d_out1, d_out2, DIGEST_LEN); - - done: - if (d1) - crypto_free_digest_env(d1); - if (d2) - crypto_free_digest_env(d2); -} - -/** Run unit tests for our public key crypto functions */ -static void -test_crypto_pk(void) -{ - crypto_pk_env_t *pk1 = NULL, *pk2 = NULL; - char *encoded = NULL; - char data1[1024], data2[1024], data3[1024]; - size_t size; - int i, j, p, len; - - /* Public-key ciphers */ - pk1 = pk_generate(0); - pk2 = crypto_new_pk_env(); - test_assert(pk1 && pk2); - test_assert(! crypto_pk_write_public_key_to_string(pk1, &encoded, &size)); - test_assert(! crypto_pk_read_public_key_from_string(pk2, encoded, size)); - test_eq(0, crypto_pk_cmp_keys(pk1, pk2)); - - test_eq(128, crypto_pk_keysize(pk1)); - test_eq(128, crypto_pk_keysize(pk2)); - - test_eq(128, crypto_pk_public_encrypt(pk2, data1, sizeof(data1), - "Hello whirled.", 15, - PK_PKCS1_OAEP_PADDING)); - test_eq(128, crypto_pk_public_encrypt(pk1, data2, sizeof(data2), - "Hello whirled.", 15, - PK_PKCS1_OAEP_PADDING)); - /* oaep padding should make encryption not match */ - test_memneq(data1, data2, 128); - test_eq(15, crypto_pk_private_decrypt(pk1, data3, sizeof(data3), data1, 128, - PK_PKCS1_OAEP_PADDING,1)); - test_streq(data3, "Hello whirled."); - memset(data3, 0, 1024); - test_eq(15, crypto_pk_private_decrypt(pk1, data3, sizeof(data3), data2, 128, - PK_PKCS1_OAEP_PADDING,1)); - test_streq(data3, "Hello whirled."); - /* Can't decrypt with public key. */ - test_eq(-1, crypto_pk_private_decrypt(pk2, data3, sizeof(data3), data2, 128, - PK_PKCS1_OAEP_PADDING,1)); - /* Try again with bad padding */ - memcpy(data2+1, "XYZZY", 5); /* This has fails ~ once-in-2^40 */ - test_eq(-1, crypto_pk_private_decrypt(pk1, data3, sizeof(data3), data2, 128, - PK_PKCS1_OAEP_PADDING,1)); - - /* File operations: save and load private key */ - test_assert(! crypto_pk_write_private_key_to_filename(pk1, - get_fname("pkey1"))); - /* failing case for read: can't read. */ - test_assert(crypto_pk_read_private_key_from_filename(pk2, - get_fname("xyzzy")) < 0); - write_str_to_file(get_fname("xyzzy"), "foobar", 6); - /* Failing case for read: no key. */ - test_assert(crypto_pk_read_private_key_from_filename(pk2, - get_fname("xyzzy")) < 0); - test_assert(! crypto_pk_read_private_key_from_filename(pk2, - get_fname("pkey1"))); - test_eq(15, crypto_pk_private_decrypt(pk2, data3, sizeof(data3), data1, 128, - PK_PKCS1_OAEP_PADDING,1)); - - /* Now try signing. */ - strlcpy(data1, "Ossifrage", 1024); - test_eq(128, crypto_pk_private_sign(pk1, data2, sizeof(data2), data1, 10)); - test_eq(10, crypto_pk_public_checksig(pk1, data3, sizeof(data3), data2, 128)); - test_streq(data3, "Ossifrage"); - /* Try signing digests. */ - test_eq(128, crypto_pk_private_sign_digest(pk1, data2, sizeof(data2), - data1, 10)); - test_eq(20, crypto_pk_public_checksig(pk1, data3, sizeof(data1), data2, 128)); - test_eq(0, crypto_pk_public_checksig_digest(pk1, data1, - 10, data2, 128)); - test_eq(-1, crypto_pk_public_checksig_digest(pk1, data1, - 11, data2, 128)); - /*XXXX test failed signing*/ - - /* Try encoding */ - crypto_free_pk_env(pk2); - pk2 = NULL; - i = crypto_pk_asn1_encode(pk1, data1, 1024); - test_assert(i>0); - pk2 = crypto_pk_asn1_decode(data1, i); - test_assert(crypto_pk_cmp_keys(pk1,pk2) == 0); - - /* Try with hybrid encryption wrappers. */ - crypto_rand(data1, 1024); - for (i = 0; i < 3; ++i) { - for (j = 85; j < 140; ++j) { - memset(data2,0,1024); - memset(data3,0,1024); - if (i == 0 && j < 129) - continue; - p = (i==0)?PK_NO_PADDING: - (i==1)?PK_PKCS1_PADDING:PK_PKCS1_OAEP_PADDING; - len = crypto_pk_public_hybrid_encrypt(pk1,data2,sizeof(data2), - data1,j,p,0); - test_assert(len>=0); - len = crypto_pk_private_hybrid_decrypt(pk1,data3,sizeof(data3), - data2,len,p,1); - test_eq(len,j); - test_memeq(data1,data3,j); - } - } - - /* Try copy_full */ - crypto_free_pk_env(pk2); - pk2 = crypto_pk_copy_full(pk1); - test_assert(pk2 != NULL); - test_neq_ptr(pk1, pk2); - test_assert(crypto_pk_cmp_keys(pk1,pk2) == 0); - - done: - if (pk1) - crypto_free_pk_env(pk1); - if (pk2) - crypto_free_pk_env(pk2); - tor_free(encoded); -} - -/** Run unit tests for misc crypto functionality. */ -static void -test_crypto(void) -{ - char *data1 = NULL, *data2 = NULL, *data3 = NULL; - int i, j, idx; - - data1 = tor_malloc(1024); - data2 = tor_malloc(1024); - data3 = tor_malloc(1024); - test_assert(data1 && data2 && data3); - - /* Base64 tests */ - memset(data1, 6, 1024); - for (idx = 0; idx < 10; ++idx) { - i = base64_encode(data2, 1024, data1, idx); - test_assert(i >= 0); - j = base64_decode(data3, 1024, data2, i); - test_eq(j,idx); - test_memeq(data3, data1, idx); - } - - strlcpy(data1, "Test string that contains 35 chars.", 1024); - strlcat(data1, " 2nd string that contains 35 chars.", 1024); - - i = base64_encode(data2, 1024, data1, 71); - j = base64_decode(data3, 1024, data2, i); - test_eq(j, 71); - test_streq(data3, data1); - test_assert(data2[i] == '\0'); - - crypto_rand(data1, DIGEST_LEN); - memset(data2, 100, 1024); - digest_to_base64(data2, data1); - test_eq(BASE64_DIGEST_LEN, strlen(data2)); - test_eq(100, data2[BASE64_DIGEST_LEN+2]); - memset(data3, 99, 1024); - test_eq(digest_from_base64(data3, data2), 0); - test_memeq(data1, data3, DIGEST_LEN); - test_eq(99, data3[DIGEST_LEN+1]); - - test_assert(digest_from_base64(data3, "###") < 0); - - /* Base32 tests */ - strlcpy(data1, "5chrs", 1024); - /* bit pattern is: [35 63 68 72 73] -> - * [00110101 01100011 01101000 01110010 01110011] - * By 5s: [00110 10101 10001 10110 10000 11100 10011 10011] - */ - base32_encode(data2, 9, data1, 5); - test_streq(data2, "gvrwq4tt"); - - strlcpy(data1, "\xFF\xF5\x6D\x44\xAE\x0D\x5C\xC9\x62\xC4", 1024); - base32_encode(data2, 30, data1, 10); - test_streq(data2, "772w2rfobvomsywe"); - - /* Base16 tests */ - strlcpy(data1, "6chrs\xff", 1024); - base16_encode(data2, 13, data1, 6); - test_streq(data2, "3663687273FF"); - - strlcpy(data1, "f0d678affc000100", 1024); - i = base16_decode(data2, 8, data1, 16); - test_eq(i,0); - test_memeq(data2, "\xf0\xd6\x78\xaf\xfc\x00\x01\x00",8); - - /* now try some failing base16 decodes */ - test_eq(-1, base16_decode(data2, 8, data1, 15)); /* odd input len */ - test_eq(-1, base16_decode(data2, 7, data1, 16)); /* dest too short */ - strlcpy(data1, "f0dz!8affc000100", 1024); - test_eq(-1, base16_decode(data2, 8, data1, 16)); - - tor_free(data1); - tor_free(data2); - tor_free(data3); - - /* Add spaces to fingerprint */ - { - data1 = tor_strdup("ABCD1234ABCD56780000ABCD1234ABCD56780000"); - test_eq(strlen(data1), 40); - data2 = tor_malloc(FINGERPRINT_LEN+1); - add_spaces_to_fp(data2, FINGERPRINT_LEN+1, data1); - test_streq(data2, "ABCD 1234 ABCD 5678 0000 ABCD 1234 ABCD 5678 0000"); - tor_free(data1); - tor_free(data2); - } - - /* Check fingerprint */ - { - test_assert(crypto_pk_check_fingerprint_syntax( - "ABCD 1234 ABCD 5678 0000 ABCD 1234 ABCD 5678 0000")); - test_assert(!crypto_pk_check_fingerprint_syntax( - "ABCD 1234 ABCD 5678 0000 ABCD 1234 ABCD 5678 000")); - test_assert(!crypto_pk_check_fingerprint_syntax( - "ABCD 1234 ABCD 5678 0000 ABCD 1234 ABCD 5678 00000")); - test_assert(!crypto_pk_check_fingerprint_syntax( - "ABCD 1234 ABCD 5678 0000 ABCD1234 ABCD 5678 0000")); - test_assert(!crypto_pk_check_fingerprint_syntax( - "ABCD 1234 ABCD 5678 0000 ABCD1234 ABCD 5678 00000")); - test_assert(!crypto_pk_check_fingerprint_syntax( - "ACD 1234 ABCD 5678 0000 ABCD 1234 ABCD 5678 00000")); - } - - done: - tor_free(data1); - tor_free(data2); - tor_free(data3); -} - -/** Run unit tests for our secret-to-key passphrase hashing functionality. */ -static void -test_crypto_s2k(void) -{ - char buf[29]; - char buf2[29]; - char *buf3 = NULL; - int i; - - memset(buf, 0, sizeof(buf)); - memset(buf2, 0, sizeof(buf2)); - buf3 = tor_malloc(65536); - memset(buf3, 0, 65536); - - secret_to_key(buf+9, 20, "", 0, buf); - crypto_digest(buf2+9, buf3, 1024); - test_memeq(buf, buf2, 29); - - memcpy(buf,"vrbacrda",8); - memcpy(buf2,"vrbacrda",8); - buf[8] = 96; - buf2[8] = 96; - secret_to_key(buf+9, 20, "12345678", 8, buf); - for (i = 0; i < 65536; i += 16) { - memcpy(buf3+i, "vrbacrda12345678", 16); - } - crypto_digest(buf2+9, buf3, 65536); - test_memeq(buf, buf2, 29); - - done: - tor_free(buf3); -} - -/** Helper: return a tristate based on comparing the strings in *<b>a</b> and - * *<b>b</b>. */ -static int -_compare_strs(const void **a, const void **b) -{ - const char *s1 = *a, *s2 = *b; - return strcmp(s1, s2); -} - -/** Helper: return a tristate based on comparing the strings in *<b>a</b> and - * *<b>b</b>, excluding a's first character, and ignoring case. */ -static int -_compare_without_first_ch(const void *a, const void **b) -{ - const char *s1 = a, *s2 = *b; - return strcasecmp(s1+1, s2); -} - -/** Test basic utility functionality. */ -static void -test_util(void) -{ - struct timeval start, end; - struct tm a_time; - char timestr[RFC1123_TIME_LEN+1]; - char buf[1024]; - time_t t_res; - int i; - uint32_t u32; - uint16_t u16; - char *cp, *k, *v; - const char *str; - - start.tv_sec = 5; - start.tv_usec = 5000; - - end.tv_sec = 5; - end.tv_usec = 5000; - - test_eq(0L, tv_udiff(&start, &end)); - - end.tv_usec = 7000; - - test_eq(2000L, tv_udiff(&start, &end)); - - end.tv_sec = 6; - - test_eq(1002000L, tv_udiff(&start, &end)); - - end.tv_usec = 0; - - test_eq(995000L, tv_udiff(&start, &end)); - - end.tv_sec = 4; - - test_eq(-1005000L, tv_udiff(&start, &end)); - - end.tv_usec = 999990; - start.tv_sec = 1; - start.tv_usec = 500; - - /* The test values here are confirmed to be correct on a platform - * with a working timegm. */ - a_time.tm_year = 2003-1900; - a_time.tm_mon = 7; - a_time.tm_mday = 30; - a_time.tm_hour = 6; - a_time.tm_min = 14; - a_time.tm_sec = 55; - test_eq((time_t) 1062224095UL, tor_timegm(&a_time)); - a_time.tm_year = 2004-1900; /* Try a leap year, after feb. */ - test_eq((time_t) 1093846495UL, tor_timegm(&a_time)); - a_time.tm_mon = 1; /* Try a leap year, in feb. */ - a_time.tm_mday = 10; - test_eq((time_t) 1076393695UL, tor_timegm(&a_time)); - - format_rfc1123_time(timestr, 0); - test_streq("Thu, 01 Jan 1970 00:00:00 GMT", timestr); - format_rfc1123_time(timestr, (time_t)1091580502UL); - test_streq("Wed, 04 Aug 2004 00:48:22 GMT", timestr); - - t_res = 0; - i = parse_rfc1123_time(timestr, &t_res); - test_eq(i,0); - test_eq(t_res, (time_t)1091580502UL); - test_eq(-1, parse_rfc1123_time("Wed, zz Aug 2004 99-99x99 GMT", &t_res)); - tor_gettimeofday(&start); - - /* Tests for corner cases of strl operations */ - test_eq(5, strlcpy(buf, "Hello", 0)); - strlcpy(buf, "Hello", sizeof(buf)); - test_eq(10, strlcat(buf, "Hello", 5)); - - /* Test tor_strstrip() */ - strlcpy(buf, "Testing 1 2 3", sizeof(buf)); - tor_strstrip(buf, ",!"); - test_streq(buf, "Testing 1 2 3"); - strlcpy(buf, "!Testing 1 2 3?", sizeof(buf)); - tor_strstrip(buf, "!? "); - test_streq(buf, "Testing123"); - - /* Test parse_addr_port */ - cp = NULL; u32 = 3; u16 = 3; - test_assert(!parse_addr_port(LOG_WARN, "1.2.3.4", &cp, &u32, &u16)); - test_streq(cp, "1.2.3.4"); - test_eq(u32, 0x01020304u); - test_eq(u16, 0); - tor_free(cp); - test_assert(!parse_addr_port(LOG_WARN, "4.3.2.1:99", &cp, &u32, &u16)); - test_streq(cp, "4.3.2.1"); - test_eq(u32, 0x04030201u); - test_eq(u16, 99); - tor_free(cp); - test_assert(!parse_addr_port(LOG_WARN, "nonexistent.address:4040", - &cp, NULL, &u16)); - test_streq(cp, "nonexistent.address"); - test_eq(u16, 4040); - tor_free(cp); - test_assert(!parse_addr_port(LOG_WARN, "localhost:9999", &cp, &u32, &u16)); - test_streq(cp, "localhost"); - test_eq(u32, 0x7f000001u); - test_eq(u16, 9999); - tor_free(cp); - u32 = 3; - test_assert(!parse_addr_port(LOG_WARN, "localhost", NULL, &u32, &u16)); - test_eq(cp, NULL); - test_eq(u32, 0x7f000001u); - test_eq(u16, 0); - tor_free(cp); - test_eq(0, addr_mask_get_bits(0x0u)); - test_eq(32, addr_mask_get_bits(0xFFFFFFFFu)); - test_eq(16, addr_mask_get_bits(0xFFFF0000u)); - test_eq(31, addr_mask_get_bits(0xFFFFFFFEu)); - test_eq(1, addr_mask_get_bits(0x80000000u)); - - /* Test tor_parse_long. */ - test_eq(10L, tor_parse_long("10",10,0,100,NULL,NULL)); - test_eq(0L, tor_parse_long("10",10,50,100,NULL,NULL)); - test_eq(-50L, tor_parse_long("-50",10,-100,100,NULL,NULL)); - - /* Test tor_parse_ulong */ - test_eq(10UL, tor_parse_ulong("10",10,0,100,NULL,NULL)); - test_eq(0UL, tor_parse_ulong("10",10,50,100,NULL,NULL)); - - /* Test tor_parse_uint64. */ - test_assert(U64_LITERAL(10) == tor_parse_uint64("10 x",10,0,100, &i, &cp)); - test_assert(i == 1); - test_streq(cp, " x"); - test_assert(U64_LITERAL(12345678901) == - tor_parse_uint64("12345678901",10,0,UINT64_MAX, &i, &cp)); - test_assert(i == 1); - test_streq(cp, ""); - test_assert(U64_LITERAL(0) == - tor_parse_uint64("12345678901",10,500,INT32_MAX, &i, &cp)); - test_assert(i == 0); - - /* Test failing snprintf cases */ - test_eq(-1, tor_snprintf(buf, 0, "Foo")); - test_eq(-1, tor_snprintf(buf, 2, "Foo")); - - /* Test printf with uint64 */ - tor_snprintf(buf, sizeof(buf), "x!"U64_FORMAT"!x", - U64_PRINTF_ARG(U64_LITERAL(12345678901))); - test_streq(buf, "x!12345678901!x"); - - /* Test parse_config_line_from_str */ - strlcpy(buf, "k v\n" " key value with spaces \n" "keykey val\n" - "k2\n" - "k3 \n" "\n" " \n" "#comment\n" - "k4#a\n" "k5#abc\n" "k6 val #with comment\n" - "kseven \"a quoted 'string\"\n" - "k8 \"a \\x71uoted\\n\\\"str\\\\ing\\t\\001\\01\\1\\\"\"\n" - , sizeof(buf)); - str = buf; - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "k"); - test_streq(v, "v"); - tor_free(k); tor_free(v); - test_assert(!strcmpstart(str, "key value with")); - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "key"); - test_streq(v, "value with spaces"); - tor_free(k); tor_free(v); - test_assert(!strcmpstart(str, "keykey")); - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "keykey"); - test_streq(v, "val"); - tor_free(k); tor_free(v); - test_assert(!strcmpstart(str, "k2\n")); - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "k2"); - test_streq(v, ""); - tor_free(k); tor_free(v); - test_assert(!strcmpstart(str, "k3 \n")); - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "k3"); - test_streq(v, ""); - tor_free(k); tor_free(v); - test_assert(!strcmpstart(str, "#comment")); - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "k4"); - test_streq(v, ""); - tor_free(k); tor_free(v); - test_assert(!strcmpstart(str, "k5#abc")); - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "k5"); - test_streq(v, ""); - tor_free(k); tor_free(v); - test_assert(!strcmpstart(str, "k6")); - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "k6"); - test_streq(v, "val"); - tor_free(k); tor_free(v); - test_assert(!strcmpstart(str, "kseven")); - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "kseven"); - test_streq(v, "a quoted \'string"); - tor_free(k); tor_free(v); - test_assert(!strcmpstart(str, "k8 ")); - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "k8"); - test_streq(v, "a quoted\n\"str\\ing\t\x01\x01\x01\""); - tor_free(k); tor_free(v); - test_streq(str, ""); - - /* Test for strcmpstart and strcmpend. */ - test_assert(strcmpstart("abcdef", "abcdef")==0); - test_assert(strcmpstart("abcdef", "abc")==0); - test_assert(strcmpstart("abcdef", "abd")<0); - test_assert(strcmpstart("abcdef", "abb")>0); - test_assert(strcmpstart("ab", "abb")<0); - - test_assert(strcmpend("abcdef", "abcdef")==0); - test_assert(strcmpend("abcdef", "def")==0); - test_assert(strcmpend("abcdef", "deg")<0); - test_assert(strcmpend("abcdef", "dee")>0); - test_assert(strcmpend("ab", "abb")<0); - - test_assert(strcasecmpend("AbcDEF", "abcdef")==0); - test_assert(strcasecmpend("abcdef", "dEF")==0); - test_assert(strcasecmpend("abcDEf", "deg")<0); - test_assert(strcasecmpend("abcdef", "DEE")>0); - test_assert(strcasecmpend("ab", "abB")<0); - - /* Test mem_is_zero */ - memset(buf,0,128); - buf[128] = 'x'; - test_assert(tor_digest_is_zero(buf)); - test_assert(tor_mem_is_zero(buf, 10)); - test_assert(tor_mem_is_zero(buf, 20)); - test_assert(tor_mem_is_zero(buf, 128)); - test_assert(!tor_mem_is_zero(buf, 129)); - buf[60] = (char)255; - test_assert(!tor_mem_is_zero(buf, 128)); - buf[0] = (char)1; - test_assert(!tor_mem_is_zero(buf, 10)); - - /* Test inet_ntop */ - { - char tmpbuf[TOR_ADDR_BUF_LEN]; - const char *ip = "176.192.208.224"; - struct in_addr in; - tor_inet_pton(AF_INET, ip, &in); - tor_inet_ntop(AF_INET, &in, tmpbuf, sizeof(tmpbuf)); - test_streq(tmpbuf, ip); - } - - /* Test 'escaped' */ - test_streq("\"\"", escaped("")); - test_streq("\"abcd\"", escaped("abcd")); - test_streq("\"\\\\\\n\\r\\t\\\"\\'\"", escaped("\\\n\r\t\"\'")); - test_streq("\"z\\001abc\\277d\"", escaped("z\001abc\277d")); - test_assert(NULL == escaped(NULL)); - - /* Test strndup and memdup */ - { - const char *s = "abcdefghijklmnopqrstuvwxyz"; - cp = tor_strndup(s, 30); - test_streq(cp, s); /* same string, */ - test_neq(cp, s); /* but different pointers. */ - tor_free(cp); - - cp = tor_strndup(s, 5); - test_streq(cp, "abcde"); - tor_free(cp); - - s = "a\0b\0c\0d\0e\0"; - cp = tor_memdup(s,10); - test_memeq(cp, s, 10); /* same ram, */ - test_neq(cp, s); /* but different pointers. */ - tor_free(cp); - } - - /* Test str-foo functions */ - cp = tor_strdup("abcdef"); - test_assert(tor_strisnonupper(cp)); - cp[3] = 'D'; - test_assert(!tor_strisnonupper(cp)); - tor_strupper(cp); - test_streq(cp, "ABCDEF"); - test_assert(tor_strisprint(cp)); - cp[3] = 3; - test_assert(!tor_strisprint(cp)); - tor_free(cp); - - /* Test eat_whitespace. */ - { - const char *s = " \n a"; - test_eq_ptr(eat_whitespace(s), s+4); - s = "abcd"; - test_eq_ptr(eat_whitespace(s), s); - s = "#xyz\nab"; - test_eq_ptr(eat_whitespace(s), s+5); - } - - /* Test memmem and memstr */ - { - const char *haystack = "abcde"; - tor_assert(!tor_memmem(haystack, 5, "ef", 2)); - test_eq_ptr(tor_memmem(haystack, 5, "cd", 2), haystack + 2); - test_eq_ptr(tor_memmem(haystack, 5, "cde", 3), haystack + 2); - haystack = "ababcad"; - test_eq_ptr(tor_memmem(haystack, 7, "abc", 3), haystack + 2); - test_eq_ptr(tor_memstr(haystack, 7, "abc"), haystack + 2); - test_assert(!tor_memstr(haystack, 7, "fe")); - test_assert(!tor_memstr(haystack, 7, "longerthantheoriginal")); - } - - /* Test wrap_string */ - { - smartlist_t *sl = smartlist_create(); - wrap_string(sl, "This is a test of string wrapping functionality: woot.", - 10, "", ""); - cp = smartlist_join_strings(sl, "", 0, NULL); - test_streq(cp, - "This is a\ntest of\nstring\nwrapping\nfunctional\nity: woot.\n"); - tor_free(cp); - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_clear(sl); - - wrap_string(sl, "This is a test of string wrapping functionality: woot.", - 16, "### ", "# "); - cp = smartlist_join_strings(sl, "", 0, NULL); - test_streq(cp, - "### This is a\n# test of string\n# wrapping\n# functionality:\n" - "# woot.\n"); - - tor_free(cp); - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_free(sl); - } - - tor_gettimeofday(&start); - /* now make sure time works. */ - tor_gettimeofday(&end); - /* We might've timewarped a little. */ - test_assert(tv_udiff(&start, &end) >= -5000); - - /* Test tor_log2(). */ - test_eq(tor_log2(64), 6); - test_eq(tor_log2(65), 6); - test_eq(tor_log2(63), 5); - test_eq(tor_log2(1), 0); - test_eq(tor_log2(2), 1); - test_eq(tor_log2(3), 1); - test_eq(tor_log2(4), 2); - test_eq(tor_log2(5), 2); - test_eq(tor_log2(U64_LITERAL(40000000000000000)), 55); - test_eq(tor_log2(UINT64_MAX), 63); - - /* Test round_to_power_of_2 */ - test_eq(round_to_power_of_2(120), 128); - test_eq(round_to_power_of_2(128), 128); - test_eq(round_to_power_of_2(130), 128); - test_eq(round_to_power_of_2(U64_LITERAL(40000000000000000)), - U64_LITERAL(1)<<55); - test_eq(round_to_power_of_2(0), 2); - - done: - ; -} - -static void -test_util_di_ops(void) -{ -#define LT -1 -#define GT 1 -#define EQ 0 - const struct { - const char *a; int want_sign; const char *b; - } examples[] = { - { "Foo", EQ, "Foo" }, - { "foo", GT, "bar", }, - { "foobar", EQ ,"foobar" }, - { "foobar", LT, "foobaw" }, - { "foobar", GT, "f00bar" }, - { "foobar", GT, "boobar" }, - { "", EQ, "" }, - { NULL, 0, NULL }, - }; - - int i; - - for (i = 0; examples[i].a; ++i) { - size_t len = strlen(examples[i].a); - int eq1, eq2, neq1, neq2, cmp1, cmp2; - test_eq(len, strlen(examples[i].b)); - /* We do all of the operations, with operands in both orders. */ - eq1 = tor_memeq(examples[i].a, examples[i].b, len); - eq2 = tor_memeq(examples[i].b, examples[i].a, len); - neq1 = tor_memneq(examples[i].a, examples[i].b, len); - neq2 = tor_memneq(examples[i].b, examples[i].a, len); - cmp1 = tor_memcmp(examples[i].a, examples[i].b, len); - cmp2 = tor_memcmp(examples[i].b, examples[i].a, len); - - /* Check for correctness of cmp1 */ - if (cmp1 < 0 && examples[i].want_sign != LT) - test_fail(); - else if (cmp1 > 0 && examples[i].want_sign != GT) - test_fail(); - else if (cmp1 == 0 && examples[i].want_sign != EQ) - test_fail(); - - /* Check for consistency of everything else with cmp1 */ - test_eq(eq1, eq2); - test_eq(neq1, neq2); - test_eq(cmp1, -cmp2); - test_eq(eq1, cmp1 == 0); - test_eq(neq1, !eq1); - } - - done: - ; -} - -/** Helper: assert that IPv6 addresses <b>a</b> and <b>b</b> are the same. On - * failure, reports an error, describing the addresses as <b>e1</b> and - * <b>e2</b>, and reporting the line number as <b>line</b>. */ -static void -_test_eq_ip6(struct in6_addr *a, struct in6_addr *b, const char *e1, - const char *e2, int line) -{ - int i; - int ok = 1; - for (i = 0; i < 16; ++i) { - if (a->s6_addr[i] != b->s6_addr[i]) { - ok = 0; - break; - } - } - if (ok) { - printf("."); fflush(stdout); - } else { - char buf1[128], *cp1; - char buf2[128], *cp2; - have_failed = 1; - cp1 = buf1; cp2 = buf2; - for (i=0; i<16; ++i) { - tor_snprintf(cp1, sizeof(buf1)-(cp1-buf1), "%02x", a->s6_addr[i]); - tor_snprintf(cp2, sizeof(buf2)-(cp2-buf2), "%02x", b->s6_addr[i]); - cp1 += 2; cp2 += 2; - if ((i%2)==1 && i != 15) { - *cp1++ = ':'; - *cp2++ = ':'; - } - } - *cp1 = *cp2 = '\0'; - printf("Line %d: assertion failed: (%s == %s)\n" - " %s != %s\n", line, e1, e2, buf1, buf2); - fflush(stdout); - } -} - -/** Helper: Assert that two strings both decode as IPv6 addresses with - * tor_inet_pton(), and both decode to the same address. */ -#define test_pton6_same(a,b) STMT_BEGIN \ - test_eq(tor_inet_pton(AF_INET6, a, &a1), 1); \ - test_eq(tor_inet_pton(AF_INET6, b, &a2), 1); \ - _test_eq_ip6(&a1,&a2,#a,#b,__LINE__); \ - STMT_END - -/** Helper: Assert that <b>a</b> is recognized as a bad IPv6 address by - * tor_inet_pton(). */ -#define test_pton6_bad(a) \ - test_eq(0, tor_inet_pton(AF_INET6, a, &a1)) - -/** Helper: assert that <b>a</b>, when parsed by tor_inet_pton() and displayed - * with tor_inet_ntop(), yields <b>b</b>. Also assert that <b>b</b> parses to - * the same value as <b>a</b>. */ -#define test_ntop6_reduces(a,b) STMT_BEGIN \ - test_eq(tor_inet_pton(AF_INET6, a, &a1), 1); \ - test_streq(tor_inet_ntop(AF_INET6, &a1, buf, sizeof(buf)), b); \ - test_eq(tor_inet_pton(AF_INET6, b, &a2), 1); \ - _test_eq_ip6(&a1, &a2, a, b, __LINE__); \ - STMT_END - -/** Helper: assert that <b>a</b> parses by tor_inet_pton() into a address that - * passes tor_addr_is_internal() with <b>for_listening</b>. */ -#define test_internal_ip(a,for_listening) STMT_BEGIN \ - test_eq(tor_inet_pton(AF_INET6, a, &t1.addr.in6_addr), 1); \ - t1.family = AF_INET6; \ - if (!tor_addr_is_internal(&t1, for_listening)) \ - test_fail_msg( a "was not internal."); \ - STMT_END - -/** Helper: assert that <b>a</b> parses by tor_inet_pton() into a address that - * does not pass tor_addr_is_internal() with <b>for_listening</b>. */ -#define test_external_ip(a,for_listening) STMT_BEGIN \ - test_eq(tor_inet_pton(AF_INET6, a, &t1.addr.in6_addr), 1); \ - t1.family = AF_INET6; \ - if (tor_addr_is_internal(&t1, for_listening)) \ - test_fail_msg(a "was not external."); \ - STMT_END - -/** Helper: Assert that <b>a</b> and <b>b</b>, when parsed by - * tor_inet_pton(), give addresses that compare in the order defined by - * <b>op</b> with tor_addr_compare(). */ -#define test_addr_compare(a, op, b) STMT_BEGIN \ - test_eq(tor_inet_pton(AF_INET6, a, &t1.addr.in6_addr), 1); \ - test_eq(tor_inet_pton(AF_INET6, b, &t2.addr.in6_addr), 1); \ - t1.family = t2.family = AF_INET6; \ - r = tor_addr_compare(&t1,&t2,CMP_SEMANTIC); \ - if (!(r op 0)) \ - test_fail_msg("failed: tor_addr_compare("a","b") "#op" 0"); \ - STMT_END - -/** Helper: Assert that <b>a</b> and <b>b</b>, when parsed by - * tor_inet_pton(), give addresses that compare in the order defined by - * <b>op</b> with tor_addr_compare_masked() with <b>m</b> masked. */ -#define test_addr_compare_masked(a, op, b, m) STMT_BEGIN \ - test_eq(tor_inet_pton(AF_INET6, a, &t1.addr.in6_addr), 1); \ - test_eq(tor_inet_pton(AF_INET6, b, &t2.addr.in6_addr), 1); \ - t1.family = t2.family = AF_INET6; \ - r = tor_addr_compare_masked(&t1,&t2,m,CMP_SEMANTIC); \ - if (!(r op 0)) \ - test_fail_msg("failed: tor_addr_compare_masked("a","b","#m") "#op" 0"); \ - STMT_END - -/** Helper: assert that <b>xx</b> is parseable as a masked IPv6 address with - * ports by tor_parse_mask_addr_ports(), with family <b>f</b>, IP address - * as 4 32-bit words <b>ip1...ip4</b>, mask bits as <b>mm</b>, and port range - * as <b>pt1..pt2</b>. */ -#define test_addr_mask_ports_parse(xx, f, ip1, ip2, ip3, ip4, mm, pt1, pt2) \ - STMT_BEGIN \ - test_eq(tor_addr_parse_mask_ports(xx, &t1, &mask, &port1, &port2), f); \ - p1=tor_inet_ntop(AF_INET6, &t1.addr.in6_addr, bug, sizeof(bug)); \ - test_eq(htonl(ip1), tor_addr_to_in6_addr32(&t1)[0]); \ - test_eq(htonl(ip2), tor_addr_to_in6_addr32(&t1)[1]); \ - test_eq(htonl(ip3), tor_addr_to_in6_addr32(&t1)[2]); \ - test_eq(htonl(ip4), tor_addr_to_in6_addr32(&t1)[3]); \ - test_eq(mask, mm); \ - test_eq(port1, pt1); \ - test_eq(port2, pt2); \ - STMT_END - -/** Run unit tests for IPv6 encoding/decoding/manipulation functions. */ -static void -test_util_ip6_helpers(void) -{ - char buf[TOR_ADDR_BUF_LEN], bug[TOR_ADDR_BUF_LEN]; - struct in6_addr a1, a2; - tor_addr_t t1, t2; - int r, i; - uint16_t port1, port2; - maskbits_t mask; - const char *p1; - struct sockaddr_storage sa_storage; - struct sockaddr_in *sin; - struct sockaddr_in6 *sin6; - - // struct in_addr b1, b2; - /* Test tor_inet_ntop and tor_inet_pton: IPv6 */ - - /* ==== Converting to and from sockaddr_t. */ - sin = (struct sockaddr_in *)&sa_storage; - sin->sin_family = AF_INET; - sin->sin_port = 9090; - sin->sin_addr.s_addr = htonl(0x7f7f0102); /*127.127.1.2*/ - tor_addr_from_sockaddr(&t1, (struct sockaddr *)sin, NULL); - test_eq(tor_addr_family(&t1), AF_INET); - test_eq(tor_addr_to_ipv4h(&t1), 0x7f7f0102); - - memset(&sa_storage, 0, sizeof(sa_storage)); - test_eq(sizeof(struct sockaddr_in), - tor_addr_to_sockaddr(&t1, 1234, (struct sockaddr *)&sa_storage, - sizeof(sa_storage))); - test_eq(1234, ntohs(sin->sin_port)); - test_eq(0x7f7f0102, ntohl(sin->sin_addr.s_addr)); - - memset(&sa_storage, 0, sizeof(sa_storage)); - sin6 = (struct sockaddr_in6 *)&sa_storage; - sin6->sin6_family = AF_INET6; - sin6->sin6_port = htons(7070); - sin6->sin6_addr.s6_addr[0] = 128; - tor_addr_from_sockaddr(&t1, (struct sockaddr *)sin6, NULL); - test_eq(tor_addr_family(&t1), AF_INET6); - p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 0); - test_streq(p1, "8000::"); - - memset(&sa_storage, 0, sizeof(sa_storage)); - test_eq(sizeof(struct sockaddr_in6), - tor_addr_to_sockaddr(&t1, 9999, (struct sockaddr *)&sa_storage, - sizeof(sa_storage))); - test_eq(AF_INET6, sin6->sin6_family); - test_eq(9999, ntohs(sin6->sin6_port)); - test_eq(0x80000000, ntohl(S6_ADDR32(sin6->sin6_addr)[0])); - - /* ==== tor_addr_lookup: static cases. (Can't test dns without knowing we - * have a good resolver. */ - test_eq(0, tor_addr_lookup("127.128.129.130", AF_UNSPEC, &t1)); - test_eq(AF_INET, tor_addr_family(&t1)); - test_eq(tor_addr_to_ipv4h(&t1), 0x7f808182); - - test_eq(0, tor_addr_lookup("9000::5", AF_UNSPEC, &t1)); - test_eq(AF_INET6, tor_addr_family(&t1)); - test_eq(0x90, tor_addr_to_in6_addr8(&t1)[0]); - test_assert(tor_mem_is_zero((char*)tor_addr_to_in6_addr8(&t1)+1, 14)); - test_eq(0x05, tor_addr_to_in6_addr8(&t1)[15]); - - /* === Test pton: valid af_inet6 */ - /* Simple, valid parsing. */ - r = tor_inet_pton(AF_INET6, - "0102:0304:0506:0708:090A:0B0C:0D0E:0F10", &a1); - test_assert(r==1); - for (i=0;i<16;++i) { test_eq(i+1, (int)a1.s6_addr[i]); } - /* ipv4 ending. */ - test_pton6_same("0102:0304:0506:0708:090A:0B0C:0D0E:0F10", - "0102:0304:0506:0708:090A:0B0C:13.14.15.16"); - /* shortened words. */ - test_pton6_same("0001:0099:BEEF:0000:0123:FFFF:0001:0001", - "1:99:BEEF:0:0123:FFFF:1:1"); - /* zeros at the beginning */ - test_pton6_same("0000:0000:0000:0000:0009:C0A8:0001:0001", - "::9:c0a8:1:1"); - test_pton6_same("0000:0000:0000:0000:0009:C0A8:0001:0001", - "::9:c0a8:0.1.0.1"); - /* zeros in the middle. */ - test_pton6_same("fe80:0000:0000:0000:0202:1111:0001:0001", - "fe80::202:1111:1:1"); - /* zeros at the end. */ - test_pton6_same("1000:0001:0000:0007:0000:0000:0000:0000", - "1000:1:0:7::"); - - /* === Test ntop: af_inet6 */ - test_ntop6_reduces("0:0:0:0:0:0:0:0", "::"); - - test_ntop6_reduces("0001:0099:BEEF:0006:0123:FFFF:0001:0001", - "1:99:beef:6:123:ffff:1:1"); - - //test_ntop6_reduces("0:0:0:0:0:0:c0a8:0101", "::192.168.1.1"); - test_ntop6_reduces("0:0:0:0:0:ffff:c0a8:0101", "::ffff:192.168.1.1"); - test_ntop6_reduces("002:0:0000:0:3::4", "2::3:0:0:4"); - test_ntop6_reduces("0:0::1:0:3", "::1:0:3"); - test_ntop6_reduces("008:0::0", "8::"); - test_ntop6_reduces("0:0:0:0:0:ffff::1", "::ffff:0.0.0.1"); - test_ntop6_reduces("abcd:0:0:0:0:0:7f00::", "abcd::7f00:0"); - test_ntop6_reduces("0000:0000:0000:0000:0009:C0A8:0001:0001", - "::9:c0a8:1:1"); - test_ntop6_reduces("fe80:0000:0000:0000:0202:1111:0001:0001", - "fe80::202:1111:1:1"); - test_ntop6_reduces("1000:0001:0000:0007:0000:0000:0000:0000", - "1000:1:0:7::"); - - /* === Test pton: invalid in6. */ - test_pton6_bad("foobar."); - test_pton6_bad("55555::"); - test_pton6_bad("9:-60::"); - test_pton6_bad("1:2:33333:4:0002:3::"); - //test_pton6_bad("1:2:3333:4:00002:3::");// BAD, but glibc doesn't say so. - test_pton6_bad("1:2:3333:4:fish:3::"); - test_pton6_bad("1:2:3:4:5:6:7:8:9"); - test_pton6_bad("1:2:3:4:5:6:7"); - test_pton6_bad("1:2:3:4:5:6:1.2.3.4.5"); - test_pton6_bad("1:2:3:4:5:6:1.2.3"); - test_pton6_bad("::1.2.3"); - test_pton6_bad("::1.2.3.4.5"); - test_pton6_bad("99"); - test_pton6_bad(""); - test_pton6_bad("1::2::3:4"); - test_pton6_bad("a:::b:c"); - test_pton6_bad(":::a:b:c"); - test_pton6_bad("a:b:c:::"); - - /* test internal checking */ - test_external_ip("fbff:ffff::2:7", 0); - test_internal_ip("fc01::2:7", 0); - test_internal_ip("fdff:ffff::f:f", 0); - test_external_ip("fe00::3:f", 0); - - test_external_ip("fe7f:ffff::2:7", 0); - test_internal_ip("fe80::2:7", 0); - test_internal_ip("febf:ffff::f:f", 0); - - test_internal_ip("fec0::2:7:7", 0); - test_internal_ip("feff:ffff::e:7:7", 0); - test_external_ip("ff00::e:7:7", 0); - - test_internal_ip("::", 0); - test_internal_ip("::1", 0); - test_internal_ip("::1", 1); - test_internal_ip("::", 0); - test_external_ip("::", 1); - test_external_ip("::2", 0); - test_external_ip("2001::", 0); - test_external_ip("ffff::", 0); - - test_external_ip("::ffff:0.0.0.0", 1); - test_internal_ip("::ffff:0.0.0.0", 0); - test_internal_ip("::ffff:0.255.255.255", 0); - test_external_ip("::ffff:1.0.0.0", 0); - - test_external_ip("::ffff:9.255.255.255", 0); - test_internal_ip("::ffff:10.0.0.0", 0); - test_internal_ip("::ffff:10.255.255.255", 0); - test_external_ip("::ffff:11.0.0.0", 0); - - test_external_ip("::ffff:126.255.255.255", 0); - test_internal_ip("::ffff:127.0.0.0", 0); - test_internal_ip("::ffff:127.255.255.255", 0); - test_external_ip("::ffff:128.0.0.0", 0); - - test_external_ip("::ffff:172.15.255.255", 0); - test_internal_ip("::ffff:172.16.0.0", 0); - test_internal_ip("::ffff:172.31.255.255", 0); - test_external_ip("::ffff:172.32.0.0", 0); - - test_external_ip("::ffff:192.167.255.255", 0); - test_internal_ip("::ffff:192.168.0.0", 0); - test_internal_ip("::ffff:192.168.255.255", 0); - test_external_ip("::ffff:192.169.0.0", 0); - - test_external_ip("::ffff:169.253.255.255", 0); - test_internal_ip("::ffff:169.254.0.0", 0); - test_internal_ip("::ffff:169.254.255.255", 0); - test_external_ip("::ffff:169.255.0.0", 0); - test_assert(is_internal_IP(0x7f000001, 0)); - - /* tor_addr_compare(tor_addr_t x2) */ - test_addr_compare("ffff::", ==, "ffff::0"); - test_addr_compare("0::3:2:1", <, "0::ffff:0.3.2.1"); - test_addr_compare("0::2:2:1", <, "0::ffff:0.3.2.1"); - test_addr_compare("0::ffff:0.3.2.1", >, "0::0:0:0"); - test_addr_compare("0::ffff:5.2.2.1", <, "::ffff:6.0.0.0"); /* XXXX wrong. */ - tor_addr_parse_mask_ports("[::ffff:2.3.4.5]", &t1, NULL, NULL, NULL); - tor_addr_parse_mask_ports("2.3.4.5", &t2, NULL, NULL, NULL); - test_assert(tor_addr_compare(&t1, &t2, CMP_SEMANTIC) == 0); - tor_addr_parse_mask_ports("[::ffff:2.3.4.4]", &t1, NULL, NULL, NULL); - tor_addr_parse_mask_ports("2.3.4.5", &t2, NULL, NULL, NULL); - test_assert(tor_addr_compare(&t1, &t2, CMP_SEMANTIC) < 0); - - /* test compare_masked */ - test_addr_compare_masked("ffff::", ==, "ffff::0", 128); - test_addr_compare_masked("ffff::", ==, "ffff::0", 64); - test_addr_compare_masked("0::2:2:1", <, "0::8000:2:1", 81); - test_addr_compare_masked("0::2:2:1", ==, "0::8000:2:1", 80); - - /* Test decorated addr_to_string. */ - test_eq(AF_INET6, tor_addr_from_str(&t1, "[123:45:6789::5005:11]")); - p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 1); - test_streq(p1, "[123:45:6789::5005:11]"); - test_eq(AF_INET, tor_addr_from_str(&t1, "18.0.0.1")); - p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 1); - test_streq(p1, "18.0.0.1"); - - /* Test tor_addr_parse_reverse_lookup_name */ - i = tor_addr_parse_reverse_lookup_name(&t1, "Foobar.baz", AF_UNSPEC, 0); - test_eq(0, i); - i = tor_addr_parse_reverse_lookup_name(&t1, "Foobar.baz", AF_UNSPEC, 1); - test_eq(0, i); - i = tor_addr_parse_reverse_lookup_name(&t1, "1.0.168.192.in-addr.arpa", - AF_UNSPEC, 1); - test_eq(1, i); - test_eq(tor_addr_family(&t1), AF_INET); - p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 1); - test_streq(p1, "192.168.0.1"); - i = tor_addr_parse_reverse_lookup_name(&t1, "192.168.0.99", AF_UNSPEC, 0); - test_eq(0, i); - i = tor_addr_parse_reverse_lookup_name(&t1, "192.168.0.99", AF_UNSPEC, 1); - test_eq(1, i); - p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 1); - test_streq(p1, "192.168.0.99"); - memset(&t1, 0, sizeof(t1)); - i = tor_addr_parse_reverse_lookup_name(&t1, - "0.1.2.3.4.5.6.7.8.9.a.b.c.d.e.f." - "f.e.e.b.1.e.b.e.e.f.f.e.e.e.d.9." - "ip6.ARPA", - AF_UNSPEC, 0); - test_eq(1, i); - p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 1); - test_streq(p1, "[9dee:effe:ebe1:beef:fedc:ba98:7654:3210]"); - /* Failing cases. */ - i = tor_addr_parse_reverse_lookup_name(&t1, - "6.7.8.9.a.b.c.d.e.f." - "f.e.e.b.1.e.b.e.e.f.f.e.e.e.d.9." - "ip6.ARPA", - AF_UNSPEC, 0); - test_eq(i, -1); - i = tor_addr_parse_reverse_lookup_name(&t1, - "6.7.8.9.a.b.c.d.e.f.a.b.c.d.e.f.0." - "f.e.e.b.1.e.b.e.e.f.f.e.e.e.d.9." - "ip6.ARPA", - AF_UNSPEC, 0); - test_eq(i, -1); - i = tor_addr_parse_reverse_lookup_name(&t1, - "6.7.8.9.a.b.c.d.e.f.X.0.0.0.0.9." - "f.e.e.b.1.e.b.e.e.f.f.e.e.e.d.9." - "ip6.ARPA", - AF_UNSPEC, 0); - test_eq(i, -1); - i = tor_addr_parse_reverse_lookup_name(&t1, "32.1.1.in-addr.arpa", - AF_UNSPEC, 0); - test_eq(i, -1); - i = tor_addr_parse_reverse_lookup_name(&t1, ".in-addr.arpa", - AF_UNSPEC, 0); - test_eq(i, -1); - i = tor_addr_parse_reverse_lookup_name(&t1, "1.2.3.4.5.in-addr.arpa", - AF_UNSPEC, 0); - test_eq(i, -1); - i = tor_addr_parse_reverse_lookup_name(&t1, "1.2.3.4.5.in-addr.arpa", - AF_INET6, 0); - test_eq(i, -1); - i = tor_addr_parse_reverse_lookup_name(&t1, - "6.7.8.9.a.b.c.d.e.f.a.b.c.d.e.0." - "f.e.e.b.1.e.b.e.e.f.f.e.e.e.d.9." - "ip6.ARPA", - AF_INET, 0); - test_eq(i, -1); - - /* test tor_addr_parse_mask_ports */ - test_addr_mask_ports_parse("[::f]/17:47-95", AF_INET6, - 0, 0, 0, 0x0000000f, 17, 47, 95); - //test_addr_parse("[::fefe:4.1.1.7/120]:999-1000"); - //test_addr_parse_check("::fefe:401:107", 120, 999, 1000); - test_addr_mask_ports_parse("[::ffff:4.1.1.7]/120:443", AF_INET6, - 0, 0, 0x0000ffff, 0x04010107, 120, 443, 443); - test_addr_mask_ports_parse("[abcd:2::44a:0]:2-65000", AF_INET6, - 0xabcd0002, 0, 0, 0x044a0000, 128, 2, 65000); - - r=tor_addr_parse_mask_ports("[fefef::]/112", &t1, NULL, NULL, NULL); - test_assert(r == -1); - r=tor_addr_parse_mask_ports("efef::/112", &t1, NULL, NULL, NULL); - test_assert(r == -1); - r=tor_addr_parse_mask_ports("[f:f:f:f:f:f:f:f::]", &t1, NULL, NULL, NULL); - test_assert(r == -1); - r=tor_addr_parse_mask_ports("[::f:f:f:f:f:f:f:f]", &t1, NULL, NULL, NULL); - test_assert(r == -1); - r=tor_addr_parse_mask_ports("[f:f:f:f:f:f:f:f:f]", &t1, NULL, NULL, NULL); - test_assert(r == -1); - /* Test for V4-mapped address with mask < 96. (arguably not valid) */ - r=tor_addr_parse_mask_ports("[::ffff:1.1.2.2/33]", &t1, &mask, NULL, NULL); - test_assert(r == -1); - r=tor_addr_parse_mask_ports("1.1.2.2/33", &t1, &mask, NULL, NULL); - test_assert(r == -1); - r=tor_addr_parse_mask_ports("1.1.2.2/31", &t1, &mask, NULL, NULL); - test_assert(r == AF_INET); - r=tor_addr_parse_mask_ports("[efef::]/112", &t1, &mask, &port1, &port2); - test_assert(r == AF_INET6); - test_assert(port1 == 1); - test_assert(port2 == 65535); - - /* make sure inet address lengths >= max */ - test_assert(INET_NTOA_BUF_LEN >= sizeof("255.255.255.255")); - test_assert(TOR_ADDR_BUF_LEN >= - sizeof("ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255")); - - test_assert(sizeof(tor_addr_t) >= sizeof(struct in6_addr)); - - /* get interface addresses */ - r = get_interface_address6(LOG_DEBUG, AF_INET, &t1); - i = get_interface_address6(LOG_DEBUG, AF_INET6, &t2); -#if 0 - tor_inet_ntop(AF_INET, &t1.sa.sin_addr, buf, sizeof(buf)); - printf("\nv4 address: %s (family=%i)", buf, IN_FAMILY(&t1)); - tor_inet_ntop(AF_INET6, &t2.sa6.sin6_addr, buf, sizeof(buf)); - printf("\nv6 address: %s (family=%i)", buf, IN_FAMILY(&t2)); -#endif - - done: - ; -} - -/** Run unit tests for basic dynamic-sized array functionality. */ -static void -test_util_smartlist_basic(void) -{ - smartlist_t *sl; - - /* XXXX test sort_digests, uniq_strings, uniq_digests */ - - /* Test smartlist add, del_keeporder, insert, get. */ - sl = smartlist_create(); - smartlist_add(sl, (void*)1); - smartlist_add(sl, (void*)2); - smartlist_add(sl, (void*)3); - smartlist_add(sl, (void*)4); - smartlist_del_keeporder(sl, 1); - smartlist_insert(sl, 1, (void*)22); - smartlist_insert(sl, 0, (void*)0); - smartlist_insert(sl, 5, (void*)555); - test_eq_ptr((void*)0, smartlist_get(sl,0)); - test_eq_ptr((void*)1, smartlist_get(sl,1)); - test_eq_ptr((void*)22, smartlist_get(sl,2)); - test_eq_ptr((void*)3, smartlist_get(sl,3)); - test_eq_ptr((void*)4, smartlist_get(sl,4)); - test_eq_ptr((void*)555, smartlist_get(sl,5)); - /* Try deleting in the middle. */ - smartlist_del(sl, 1); - test_eq_ptr((void*)555, smartlist_get(sl, 1)); - /* Try deleting at the end. */ - smartlist_del(sl, 4); - test_eq(4, smartlist_len(sl)); - - /* test isin. */ - test_assert(smartlist_isin(sl, (void*)3)); - test_assert(!smartlist_isin(sl, (void*)99)); - - done: - smartlist_free(sl); -} - -/** Run unit tests for smartlist-of-strings functionality. */ -static void -test_util_smartlist_strings(void) -{ - smartlist_t *sl = smartlist_create(); - char *cp=NULL, *cp_alloc=NULL; - size_t sz; - - /* Test split and join */ - test_eq(0, smartlist_len(sl)); - smartlist_split_string(sl, "abc", ":", 0, 0); - test_eq(1, smartlist_len(sl)); - test_streq("abc", smartlist_get(sl, 0)); - smartlist_split_string(sl, "a::bc::", "::", 0, 0); - test_eq(4, smartlist_len(sl)); - test_streq("a", smartlist_get(sl, 1)); - test_streq("bc", smartlist_get(sl, 2)); - test_streq("", smartlist_get(sl, 3)); - cp_alloc = smartlist_join_strings(sl, "", 0, NULL); - test_streq(cp_alloc, "abcabc"); - tor_free(cp_alloc); - cp_alloc = smartlist_join_strings(sl, "!", 0, NULL); - test_streq(cp_alloc, "abc!a!bc!"); - tor_free(cp_alloc); - cp_alloc = smartlist_join_strings(sl, "XY", 0, NULL); - test_streq(cp_alloc, "abcXYaXYbcXY"); - tor_free(cp_alloc); - cp_alloc = smartlist_join_strings(sl, "XY", 1, NULL); - test_streq(cp_alloc, "abcXYaXYbcXYXY"); - tor_free(cp_alloc); - cp_alloc = smartlist_join_strings(sl, "", 1, NULL); - test_streq(cp_alloc, "abcabc"); - tor_free(cp_alloc); - - smartlist_split_string(sl, "/def/ /ghijk", "/", 0, 0); - test_eq(8, smartlist_len(sl)); - test_streq("", smartlist_get(sl, 4)); - test_streq("def", smartlist_get(sl, 5)); - test_streq(" ", smartlist_get(sl, 6)); - test_streq("ghijk", smartlist_get(sl, 7)); - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_clear(sl); - - smartlist_split_string(sl, "a,bbd,cdef", ",", SPLIT_SKIP_SPACE, 0); - test_eq(3, smartlist_len(sl)); - test_streq("a", smartlist_get(sl,0)); - test_streq("bbd", smartlist_get(sl,1)); - test_streq("cdef", smartlist_get(sl,2)); - smartlist_split_string(sl, " z <> zhasd <> <> bnud<> ", "<>", - SPLIT_SKIP_SPACE, 0); - test_eq(8, smartlist_len(sl)); - test_streq("z", smartlist_get(sl,3)); - test_streq("zhasd", smartlist_get(sl,4)); - test_streq("", smartlist_get(sl,5)); - test_streq("bnud", smartlist_get(sl,6)); - test_streq("", smartlist_get(sl,7)); - - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_clear(sl); - - smartlist_split_string(sl, " ab\tc \td ef ", NULL, - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - test_eq(4, smartlist_len(sl)); - test_streq("ab", smartlist_get(sl,0)); - test_streq("c", smartlist_get(sl,1)); - test_streq("d", smartlist_get(sl,2)); - test_streq("ef", smartlist_get(sl,3)); - smartlist_split_string(sl, "ghi\tj", NULL, - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - test_eq(6, smartlist_len(sl)); - test_streq("ghi", smartlist_get(sl,4)); - test_streq("j", smartlist_get(sl,5)); - - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_clear(sl); - - cp_alloc = smartlist_join_strings(sl, "XY", 0, NULL); - test_streq(cp_alloc, ""); - tor_free(cp_alloc); - cp_alloc = smartlist_join_strings(sl, "XY", 1, NULL); - test_streq(cp_alloc, "XY"); - tor_free(cp_alloc); - - smartlist_split_string(sl, " z <> zhasd <> <> bnud<> ", "<>", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - test_eq(3, smartlist_len(sl)); - test_streq("z", smartlist_get(sl, 0)); - test_streq("zhasd", smartlist_get(sl, 1)); - test_streq("bnud", smartlist_get(sl, 2)); - smartlist_split_string(sl, " z <> zhasd <> <> bnud<> ", "<>", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 2); - test_eq(5, smartlist_len(sl)); - test_streq("z", smartlist_get(sl, 3)); - test_streq("zhasd <> <> bnud<>", smartlist_get(sl, 4)); - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_clear(sl); - - smartlist_split_string(sl, "abcd\n", "\n", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - test_eq(1, smartlist_len(sl)); - test_streq("abcd", smartlist_get(sl, 0)); - smartlist_split_string(sl, "efgh", "\n", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - test_eq(2, smartlist_len(sl)); - test_streq("efgh", smartlist_get(sl, 1)); - - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_clear(sl); - - /* Test swapping, shuffling, and sorting. */ - smartlist_split_string(sl, "the,onion,router,by,arma,and,nickm", ",", 0, 0); - test_eq(7, smartlist_len(sl)); - smartlist_sort(sl, _compare_strs); - cp_alloc = smartlist_join_strings(sl, ",", 0, NULL); - test_streq(cp_alloc,"and,arma,by,nickm,onion,router,the"); - tor_free(cp_alloc); - smartlist_swap(sl, 1, 5); - cp_alloc = smartlist_join_strings(sl, ",", 0, NULL); - test_streq(cp_alloc,"and,router,by,nickm,onion,arma,the"); - tor_free(cp_alloc); - smartlist_shuffle(sl); - test_eq(7, smartlist_len(sl)); - test_assert(smartlist_string_isin(sl, "and")); - test_assert(smartlist_string_isin(sl, "router")); - test_assert(smartlist_string_isin(sl, "by")); - test_assert(smartlist_string_isin(sl, "nickm")); - test_assert(smartlist_string_isin(sl, "onion")); - test_assert(smartlist_string_isin(sl, "arma")); - test_assert(smartlist_string_isin(sl, "the")); - - /* Test bsearch. */ - smartlist_sort(sl, _compare_strs); - test_streq("nickm", smartlist_bsearch(sl, "zNicKM", - _compare_without_first_ch)); - test_streq("and", smartlist_bsearch(sl, " AND", _compare_without_first_ch)); - test_eq_ptr(NULL, smartlist_bsearch(sl, " ANz", _compare_without_first_ch)); - - /* Test bsearch_idx */ - { - int f; - test_eq(0, smartlist_bsearch_idx(sl," aaa",_compare_without_first_ch,&f)); - test_eq(f, 0); - test_eq(0, smartlist_bsearch_idx(sl," and",_compare_without_first_ch,&f)); - test_eq(f, 1); - test_eq(1, smartlist_bsearch_idx(sl," arm",_compare_without_first_ch,&f)); - test_eq(f, 0); - test_eq(1, smartlist_bsearch_idx(sl," arma",_compare_without_first_ch,&f)); - test_eq(f, 1); - test_eq(2, smartlist_bsearch_idx(sl," armb",_compare_without_first_ch,&f)); - test_eq(f, 0); - test_eq(7, smartlist_bsearch_idx(sl," zzzz",_compare_without_first_ch,&f)); - test_eq(f, 0); - } - - /* Test reverse() and pop_last() */ - smartlist_reverse(sl); - cp_alloc = smartlist_join_strings(sl, ",", 0, NULL); - test_streq(cp_alloc,"the,router,onion,nickm,by,arma,and"); - tor_free(cp_alloc); - cp_alloc = smartlist_pop_last(sl); - test_streq(cp_alloc, "and"); - tor_free(cp_alloc); - test_eq(smartlist_len(sl), 6); - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_clear(sl); - cp_alloc = smartlist_pop_last(sl); - test_eq(cp_alloc, NULL); - - /* Test uniq() */ - smartlist_split_string(sl, - "50,noon,radar,a,man,a,plan,a,canal,panama,radar,noon,50", - ",", 0, 0); - smartlist_sort(sl, _compare_strs); - smartlist_uniq(sl, _compare_strs, _tor_free); - cp_alloc = smartlist_join_strings(sl, ",", 0, NULL); - test_streq(cp_alloc, "50,a,canal,man,noon,panama,plan,radar"); - tor_free(cp_alloc); - - /* Test string_isin and isin_case and num_isin */ - test_assert(smartlist_string_isin(sl, "noon")); - test_assert(!smartlist_string_isin(sl, "noonoon")); - test_assert(smartlist_string_isin_case(sl, "nOOn")); - test_assert(!smartlist_string_isin_case(sl, "nooNooN")); - test_assert(smartlist_string_num_isin(sl, 50)); - test_assert(!smartlist_string_num_isin(sl, 60)); - - /* Test smartlist_choose */ - { - int i; - int allsame = 1; - int allin = 1; - void *first = smartlist_choose(sl); - test_assert(smartlist_isin(sl, first)); - for (i = 0; i < 100; ++i) { - void *second = smartlist_choose(sl); - if (second != first) - allsame = 0; - if (!smartlist_isin(sl, second)) - allin = 0; - } - test_assert(!allsame); - test_assert(allin); - } - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_clear(sl); - - /* Test string_remove and remove and join_strings2 */ - smartlist_split_string(sl, - "Some say the Earth will end in ice and some in fire", - " ", 0, 0); - cp = smartlist_get(sl, 4); - test_streq(cp, "will"); - smartlist_add(sl, cp); - smartlist_remove(sl, cp); - tor_free(cp); - cp_alloc = smartlist_join_strings(sl, ",", 0, NULL); - test_streq(cp_alloc, "Some,say,the,Earth,fire,end,in,ice,and,some,in"); - tor_free(cp_alloc); - smartlist_string_remove(sl, "in"); - cp_alloc = smartlist_join_strings2(sl, "+XX", 1, 0, &sz); - test_streq(cp_alloc, "Some+say+the+Earth+fire+end+some+ice+and"); - test_eq((int)sz, 40); - - done: - - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_free(sl); - tor_free(cp_alloc); -} - -/** Run unit tests for smartlist set manipulation functions. */ -static void -test_util_smartlist_overlap(void) -{ - smartlist_t *sl = smartlist_create(); - smartlist_t *ints = smartlist_create(); - smartlist_t *odds = smartlist_create(); - smartlist_t *evens = smartlist_create(); - smartlist_t *primes = smartlist_create(); - int i; - for (i=1; i < 10; i += 2) - smartlist_add(odds, (void*)(uintptr_t)i); - for (i=0; i < 10; i += 2) - smartlist_add(evens, (void*)(uintptr_t)i); - - /* add_all */ - smartlist_add_all(ints, odds); - smartlist_add_all(ints, evens); - test_eq(smartlist_len(ints), 10); - - smartlist_add(primes, (void*)2); - smartlist_add(primes, (void*)3); - smartlist_add(primes, (void*)5); - smartlist_add(primes, (void*)7); - - /* overlap */ - test_assert(smartlist_overlap(ints, odds)); - test_assert(smartlist_overlap(odds, primes)); - test_assert(smartlist_overlap(evens, primes)); - test_assert(!smartlist_overlap(odds, evens)); - - /* intersect */ - smartlist_add_all(sl, odds); - smartlist_intersect(sl, primes); - test_eq(smartlist_len(sl), 3); - test_assert(smartlist_isin(sl, (void*)3)); - test_assert(smartlist_isin(sl, (void*)5)); - test_assert(smartlist_isin(sl, (void*)7)); - - /* subtract */ - smartlist_add_all(sl, primes); - smartlist_subtract(sl, odds); - test_eq(smartlist_len(sl), 1); - test_assert(smartlist_isin(sl, (void*)2)); - - done: - smartlist_free(odds); - smartlist_free(evens); - smartlist_free(ints); - smartlist_free(primes); - smartlist_free(sl); -} - -/** Run unit tests for smartlist-of-digests functions. */ -static void -test_util_smartlist_digests(void) -{ - smartlist_t *sl = smartlist_create(); - - /* digest_isin. */ - smartlist_add(sl, tor_memdup("AAAAAAAAAAAAAAAAAAAA", DIGEST_LEN)); - smartlist_add(sl, tor_memdup("\00090AAB2AAAAaasdAAAAA", DIGEST_LEN)); - smartlist_add(sl, tor_memdup("\00090AAB2AAAAaasdAAAAA", DIGEST_LEN)); - test_eq(0, smartlist_digest_isin(NULL, "AAAAAAAAAAAAAAAAAAAA")); - test_assert(smartlist_digest_isin(sl, "AAAAAAAAAAAAAAAAAAAA")); - test_assert(smartlist_digest_isin(sl, "\00090AAB2AAAAaasdAAAAA")); - test_eq(0, smartlist_digest_isin(sl, "\00090AAB2AAABaasdAAAAA")); - - /* sort digests */ - smartlist_sort_digests(sl); - test_memeq(smartlist_get(sl, 0), "\00090AAB2AAAAaasdAAAAA", DIGEST_LEN); - test_memeq(smartlist_get(sl, 1), "\00090AAB2AAAAaasdAAAAA", DIGEST_LEN); - test_memeq(smartlist_get(sl, 2), "AAAAAAAAAAAAAAAAAAAA", DIGEST_LEN); - test_eq(3, smartlist_len(sl)); - - /* uniq_digests */ - smartlist_uniq_digests(sl); - test_eq(2, smartlist_len(sl)); - test_memeq(smartlist_get(sl, 0), "\00090AAB2AAAAaasdAAAAA", DIGEST_LEN); - test_memeq(smartlist_get(sl, 1), "AAAAAAAAAAAAAAAAAAAA", DIGEST_LEN); - - done: - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_free(sl); -} - -/** Run unit tests for concatenate-a-smartlist-of-strings functions. */ -static void -test_util_smartlist_join(void) -{ - smartlist_t *sl = smartlist_create(); - smartlist_t *sl2 = smartlist_create(), *sl3 = smartlist_create(), - *sl4 = smartlist_create(); - char *joined=NULL; - /* unique, sorted. */ - smartlist_split_string(sl, - "Abashments Ambush Anchorman Bacon Banks Borscht " - "Bunks Inhumane Insurance Knish Know Manners " - "Maraschinos Stamina Sunbonnets Unicorns Wombats", - " ", 0, 0); - /* non-unique, sorted. */ - smartlist_split_string(sl2, - "Ambush Anchorman Anchorman Anemias Anemias Bacon " - "Crossbowmen Inhumane Insurance Knish Know Manners " - "Manners Maraschinos Wombats Wombats Work", - " ", 0, 0); - SMARTLIST_FOREACH_JOIN(sl, char *, cp1, - sl2, char *, cp2, - strcmp(cp1,cp2), - smartlist_add(sl3, cp2)) { - test_streq(cp1, cp2); - smartlist_add(sl4, cp1); - } SMARTLIST_FOREACH_JOIN_END(cp1, cp2); - - SMARTLIST_FOREACH(sl3, const char *, cp, - test_assert(smartlist_isin(sl2, cp) && - !smartlist_string_isin(sl, cp))); - SMARTLIST_FOREACH(sl4, const char *, cp, - test_assert(smartlist_isin(sl, cp) && - smartlist_string_isin(sl2, cp))); - joined = smartlist_join_strings(sl3, ",", 0, NULL); - test_streq(joined, "Anemias,Anemias,Crossbowmen,Work"); - tor_free(joined); - joined = smartlist_join_strings(sl4, ",", 0, NULL); - test_streq(joined, "Ambush,Anchorman,Anchorman,Bacon,Inhumane,Insurance," - "Knish,Know,Manners,Manners,Maraschinos,Wombats,Wombats"); - tor_free(joined); - - done: - smartlist_free(sl4); - smartlist_free(sl3); - SMARTLIST_FOREACH(sl2, char *, cp, tor_free(cp)); - smartlist_free(sl2); - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_free(sl); - tor_free(joined); -} - -/** Run unit tests for bitarray code */ -static void -test_util_bitarray(void) -{ - bitarray_t *ba = NULL; - int i, j, ok=1; - - ba = bitarray_init_zero(1); - test_assert(ba); - test_assert(! bitarray_is_set(ba, 0)); - bitarray_set(ba, 0); - test_assert(bitarray_is_set(ba, 0)); - bitarray_clear(ba, 0); - test_assert(! bitarray_is_set(ba, 0)); - bitarray_free(ba); - - ba = bitarray_init_zero(1023); - for (i = 1; i < 64; ) { - for (j = 0; j < 1023; ++j) { - if (j % i) - bitarray_set(ba, j); - else - bitarray_clear(ba, j); - } - for (j = 0; j < 1023; ++j) { - if (!bool_eq(bitarray_is_set(ba, j), j%i)) - ok = 0; - } - test_assert(ok); - if (i < 7) - ++i; - else if (i == 28) - i = 32; - else - i += 7; - } - - done: - if (ba) - bitarray_free(ba); -} - -/** Run unit tests for digest set code (implemented as a hashtable or as a - * bloom filter) */ -static void -test_util_digestset(void) -{ - smartlist_t *included = smartlist_create(); - char d[DIGEST_LEN]; - int i; - int ok = 1; - int false_positives = 0; - digestset_t *set = NULL; - - for (i = 0; i < 1000; ++i) { - crypto_rand(d, DIGEST_LEN); - smartlist_add(included, tor_memdup(d, DIGEST_LEN)); - } - set = digestset_new(1000); - SMARTLIST_FOREACH(included, const char *, cp, - if (digestset_isin(set, cp)) - ok = 0); - test_assert(ok); - SMARTLIST_FOREACH(included, const char *, cp, - digestset_add(set, cp)); - SMARTLIST_FOREACH(included, const char *, cp, - if (!digestset_isin(set, cp)) - ok = 0); - test_assert(ok); - for (i = 0; i < 1000; ++i) { - crypto_rand(d, DIGEST_LEN); - if (digestset_isin(set, d)) - ++false_positives; - } - test_assert(false_positives < 50); /* Should be far lower. */ - - done: - if (set) - digestset_free(set); - SMARTLIST_FOREACH(included, char *, cp, tor_free(cp)); - smartlist_free(included); -} - -/** mutex for thread test to stop the threads hitting data at the same time. */ -static tor_mutex_t *_thread_test_mutex = NULL; -/** mutexes for the thread test to make sure that the threads have to - * interleave somewhat. */ -static tor_mutex_t *_thread_test_start1 = NULL, - *_thread_test_start2 = NULL; -/** Shared strmap for the thread test. */ -static strmap_t *_thread_test_strmap = NULL; -/** The name of thread1 for the thread test */ -static char *_thread1_name = NULL; -/** The name of thread2 for the thread test */ -static char *_thread2_name = NULL; - -static void _thread_test_func(void* _s) ATTR_NORETURN; - -/** How many iterations have the threads in the unit test run? */ -static int t1_count = 0, t2_count = 0; - -/** Helper function for threading unit tests: This function runs in a - * subthread. It grabs its own mutex (start1 or start2) to make sure that it - * should start, then it repeatedly alters _test_thread_strmap protected by - * _thread_test_mutex. */ -static void -_thread_test_func(void* _s) -{ - char *s = _s; - int i, *count; - tor_mutex_t *m; - char buf[64]; - char **cp; - if (!strcmp(s, "thread 1")) { - m = _thread_test_start1; - cp = &_thread1_name; - count = &t1_count; - } else { - m = _thread_test_start2; - cp = &_thread2_name; - count = &t2_count; - } - tor_mutex_acquire(m); - - tor_snprintf(buf, sizeof(buf), "%lu", tor_get_thread_id()); - *cp = tor_strdup(buf); - - for (i=0; i<10000; ++i) { - tor_mutex_acquire(_thread_test_mutex); - strmap_set(_thread_test_strmap, "last to run", *cp); - ++*count; - tor_mutex_release(_thread_test_mutex); - } - tor_mutex_acquire(_thread_test_mutex); - strmap_set(_thread_test_strmap, s, *cp); - tor_mutex_release(_thread_test_mutex); - - tor_mutex_release(m); - - spawn_exit(); -} - -/** Run unit tests for threading logic. */ -static void -test_util_threads(void) -{ - char *s1 = NULL, *s2 = NULL; - int done = 0, timedout = 0; - time_t started; -#ifndef MS_WINDOWS - struct timeval tv; - tv.tv_sec=0; - tv.tv_usec=10; -#endif -#ifndef TOR_IS_MULTITHREADED - /* Skip this test if we aren't threading. We should be threading most - * everywhere by now. */ - if (1) - return; -#endif - _thread_test_mutex = tor_mutex_new(); - _thread_test_start1 = tor_mutex_new(); - _thread_test_start2 = tor_mutex_new(); - _thread_test_strmap = strmap_new(); - s1 = tor_strdup("thread 1"); - s2 = tor_strdup("thread 2"); - tor_mutex_acquire(_thread_test_start1); - tor_mutex_acquire(_thread_test_start2); - spawn_func(_thread_test_func, s1); - spawn_func(_thread_test_func, s2); - tor_mutex_release(_thread_test_start2); - tor_mutex_release(_thread_test_start1); - started = time(NULL); - while (!done) { - tor_mutex_acquire(_thread_test_mutex); - strmap_assert_ok(_thread_test_strmap); - if (strmap_get(_thread_test_strmap, "thread 1") && - strmap_get(_thread_test_strmap, "thread 2")) { - done = 1; - } else if (time(NULL) > started + 25) { - timedout = done = 1; - } - tor_mutex_release(_thread_test_mutex); -#ifndef MS_WINDOWS - /* Prevent the main thread from starving the worker threads. */ - select(0, NULL, NULL, NULL, &tv); -#endif - } - - tor_mutex_acquire(_thread_test_start1); - tor_mutex_release(_thread_test_start1); - tor_mutex_acquire(_thread_test_start2); - tor_mutex_release(_thread_test_start2); - - tor_mutex_free(_thread_test_mutex); - - if (timedout) { - printf("\nTimed out: %d %d", t1_count, t2_count); - test_assert(strmap_get(_thread_test_strmap, "thread 1")); - test_assert(strmap_get(_thread_test_strmap, "thread 2")); - test_assert(!timedout); - } - - /* different thread IDs. */ - test_assert(strcmp(strmap_get(_thread_test_strmap, "thread 1"), - strmap_get(_thread_test_strmap, "thread 2"))); - test_assert(!strcmp(strmap_get(_thread_test_strmap, "thread 1"), - strmap_get(_thread_test_strmap, "last to run")) || - !strcmp(strmap_get(_thread_test_strmap, "thread 2"), - strmap_get(_thread_test_strmap, "last to run"))); - - done: - tor_free(s1); - tor_free(s2); - tor_free(_thread1_name); - tor_free(_thread2_name); - if (_thread_test_strmap) - strmap_free(_thread_test_strmap, NULL); - if (_thread_test_start1) - tor_mutex_free(_thread_test_start1); - if (_thread_test_start2) - tor_mutex_free(_thread_test_start2); -} - -/** Helper: return a tristate based on comparing two strings. */ -static int -_compare_strings_for_pqueue(const void *s1, const void *s2) -{ - return strcmp((const char*)s1, (const char*)s2); -} - -/** Run unit tests for heap-based priority queue functions. */ -static void -test_util_pqueue(void) -{ - smartlist_t *sl = smartlist_create(); - int (*cmp)(const void *, const void*); -#define OK() smartlist_pqueue_assert_ok(sl, cmp) - - cmp = _compare_strings_for_pqueue; - - smartlist_pqueue_add(sl, cmp, (char*)"cows"); - smartlist_pqueue_add(sl, cmp, (char*)"zebras"); - smartlist_pqueue_add(sl, cmp, (char*)"fish"); - smartlist_pqueue_add(sl, cmp, (char*)"frogs"); - smartlist_pqueue_add(sl, cmp, (char*)"apples"); - smartlist_pqueue_add(sl, cmp, (char*)"squid"); - smartlist_pqueue_add(sl, cmp, (char*)"daschunds"); - smartlist_pqueue_add(sl, cmp, (char*)"eggplants"); - smartlist_pqueue_add(sl, cmp, (char*)"weissbier"); - smartlist_pqueue_add(sl, cmp, (char*)"lobsters"); - smartlist_pqueue_add(sl, cmp, (char*)"roquefort"); - - OK(); - - test_eq(smartlist_len(sl), 11); - test_streq(smartlist_get(sl, 0), "apples"); - test_streq(smartlist_pqueue_pop(sl, cmp), "apples"); - test_eq(smartlist_len(sl), 10); - OK(); - test_streq(smartlist_pqueue_pop(sl, cmp), "cows"); - test_streq(smartlist_pqueue_pop(sl, cmp), "daschunds"); - smartlist_pqueue_add(sl, cmp, (char*)"chinchillas"); - OK(); - smartlist_pqueue_add(sl, cmp, (char*)"fireflies"); - OK(); - test_streq(smartlist_pqueue_pop(sl, cmp), "chinchillas"); - test_streq(smartlist_pqueue_pop(sl, cmp), "eggplants"); - test_streq(smartlist_pqueue_pop(sl, cmp), "fireflies"); - OK(); - test_streq(smartlist_pqueue_pop(sl, cmp), "fish"); - test_streq(smartlist_pqueue_pop(sl, cmp), "frogs"); - test_streq(smartlist_pqueue_pop(sl, cmp), "lobsters"); - test_streq(smartlist_pqueue_pop(sl, cmp), "roquefort"); - OK(); - test_eq(smartlist_len(sl), 3); - test_streq(smartlist_pqueue_pop(sl, cmp), "squid"); - test_streq(smartlist_pqueue_pop(sl, cmp), "weissbier"); - test_streq(smartlist_pqueue_pop(sl, cmp), "zebras"); - test_eq(smartlist_len(sl), 0); - OK(); -#undef OK - - done: - - smartlist_free(sl); -} - -/** Run unit tests for compression functions */ -static void -test_util_gzip(void) -{ - char *buf1=NULL, *buf2=NULL, *buf3=NULL, *cp1, *cp2; - const char *ccp2; - size_t len1, len2; - tor_zlib_state_t *state = NULL; - - buf1 = tor_strdup("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ"); - test_assert(detect_compression_method(buf1, strlen(buf1)) == UNKNOWN_METHOD); - if (is_gzip_supported()) { - test_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1, - GZIP_METHOD)); - test_assert(buf2); - test_assert(!memcmp(buf2, "\037\213", 2)); /* Gzip magic. */ - test_assert(detect_compression_method(buf2, len1) == GZIP_METHOD); - - test_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1, - GZIP_METHOD, 1, LOG_INFO)); - test_assert(buf3); - test_streq(buf1,buf3); - - tor_free(buf2); - tor_free(buf3); - } - - test_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1, - ZLIB_METHOD)); - test_assert(buf2); - test_assert(!memcmp(buf2, "\x78\xDA", 2)); /* deflate magic. */ - test_assert(detect_compression_method(buf2, len1) == ZLIB_METHOD); - - test_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1, - ZLIB_METHOD, 1, LOG_INFO)); - test_assert(buf3); - test_streq(buf1,buf3); - - /* Check whether we can uncompress concatenated, compressed strings. */ - tor_free(buf3); - buf2 = tor_realloc(buf2, len1*2); - memcpy(buf2+len1, buf2, len1); - test_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1*2, - ZLIB_METHOD, 1, LOG_INFO)); - test_eq(len2, (strlen(buf1)+1)*2); - test_memeq(buf3, - "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ\0" - "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ\0", - (strlen(buf1)+1)*2); - - tor_free(buf1); - tor_free(buf2); - tor_free(buf3); - - /* Check whether we can uncompress partial strings. */ - buf1 = - tor_strdup("String with low redundancy that won't be compressed much."); - test_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1, - ZLIB_METHOD)); - tor_assert(len1>16); - /* when we allow an incomplete string, we should succeed.*/ - tor_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1-16, - ZLIB_METHOD, 0, LOG_INFO)); - buf3[len2]='\0'; - tor_assert(len2 > 5); - tor_assert(!strcmpstart(buf1, buf3)); - - /* when we demand a complete string, this must fail. */ - tor_free(buf3); - tor_assert(tor_gzip_uncompress(&buf3, &len2, buf2, len1-16, - ZLIB_METHOD, 1, LOG_INFO)); - tor_assert(!buf3); - - /* Now, try streaming compression. */ - tor_free(buf1); - tor_free(buf2); - tor_free(buf3); - state = tor_zlib_new(1, ZLIB_METHOD); - tor_assert(state); - cp1 = buf1 = tor_malloc(1024); - len1 = 1024; - ccp2 = "ABCDEFGHIJABCDEFGHIJ"; - len2 = 21; - test_assert(tor_zlib_process(state, &cp1, &len1, &ccp2, &len2, 0) - == TOR_ZLIB_OK); - test_eq(len2, 0); /* Make sure we compressed it all. */ - test_assert(cp1 > buf1); - - len2 = 0; - cp2 = cp1; - test_assert(tor_zlib_process(state, &cp1, &len1, &ccp2, &len2, 1) - == TOR_ZLIB_DONE); - test_eq(len2, 0); - test_assert(cp1 > cp2); /* Make sure we really added something. */ - - tor_assert(!tor_gzip_uncompress(&buf3, &len2, buf1, 1024-len1, - ZLIB_METHOD, 1, LOG_WARN)); - test_streq(buf3, "ABCDEFGHIJABCDEFGHIJ"); /*Make sure it compressed right.*/ - - done: - if (state) - tor_zlib_free(state); - tor_free(buf2); - tor_free(buf3); - tor_free(buf1); -} - -/** Run unit tests for string-to-void* map functions */ -static void -test_util_strmap(void) -{ - strmap_t *map; - strmap_iter_t *iter; - const char *k; - void *v; - char *visited = NULL; - smartlist_t *found_keys = NULL; - - map = strmap_new(); - test_assert(map); - test_eq(strmap_size(map), 0); - test_assert(strmap_isempty(map)); - v = strmap_set(map, "K1", (void*)99); - test_eq(v, NULL); - test_assert(!strmap_isempty(map)); - v = strmap_set(map, "K2", (void*)101); - test_eq(v, NULL); - v = strmap_set(map, "K1", (void*)100); - test_eq(v, (void*)99); - test_eq_ptr(strmap_get(map,"K1"), (void*)100); - test_eq_ptr(strmap_get(map,"K2"), (void*)101); - test_eq_ptr(strmap_get(map,"K-not-there"), NULL); - strmap_assert_ok(map); - - v = strmap_remove(map,"K2"); - strmap_assert_ok(map); - test_eq_ptr(v, (void*)101); - test_eq_ptr(strmap_get(map,"K2"), NULL); - test_eq_ptr(strmap_remove(map,"K2"), NULL); - - strmap_set(map, "K2", (void*)101); - strmap_set(map, "K3", (void*)102); - strmap_set(map, "K4", (void*)103); - test_eq(strmap_size(map), 4); - strmap_assert_ok(map); - strmap_set(map, "K5", (void*)104); - strmap_set(map, "K6", (void*)105); - strmap_assert_ok(map); - - /* Test iterator. */ - iter = strmap_iter_init(map); - found_keys = smartlist_create(); - while (!strmap_iter_done(iter)) { - strmap_iter_get(iter,&k,&v); - smartlist_add(found_keys, tor_strdup(k)); - test_eq_ptr(v, strmap_get(map, k)); - - if (!strcmp(k, "K2")) { - iter = strmap_iter_next_rmv(map,iter); - } else { - iter = strmap_iter_next(map,iter); - } - } - - /* Make sure we removed K2, but not the others. */ - test_eq_ptr(strmap_get(map, "K2"), NULL); - test_eq_ptr(strmap_get(map, "K5"), (void*)104); - /* Make sure we visited everyone once */ - smartlist_sort_strings(found_keys); - visited = smartlist_join_strings(found_keys, ":", 0, NULL); - test_streq(visited, "K1:K2:K3:K4:K5:K6"); - - strmap_assert_ok(map); - /* Clean up after ourselves. */ - strmap_free(map, NULL); - map = NULL; - - /* Now try some lc functions. */ - map = strmap_new(); - strmap_set_lc(map,"Ab.C", (void*)1); - test_eq_ptr(strmap_get(map,"ab.c"), (void*)1); - strmap_assert_ok(map); - test_eq_ptr(strmap_get_lc(map,"AB.C"), (void*)1); - test_eq_ptr(strmap_get(map,"AB.C"), NULL); - test_eq_ptr(strmap_remove_lc(map,"aB.C"), (void*)1); - strmap_assert_ok(map); - test_eq_ptr(strmap_get_lc(map,"AB.C"), NULL); - - done: - if (map) - strmap_free(map,NULL); - if (found_keys) { - SMARTLIST_FOREACH(found_keys, char *, cp, tor_free(cp)); - smartlist_free(found_keys); - } - tor_free(visited); -} - -/** Run unit tests for mmap() wrapper functionality. */ -static void -test_util_mmap(void) -{ - char *fname1 = tor_strdup(get_fname("mapped_1")); - char *fname2 = tor_strdup(get_fname("mapped_2")); - char *fname3 = tor_strdup(get_fname("mapped_3")); - const size_t buflen = 17000; - char *buf = tor_malloc(17000); - tor_mmap_t *mapping = NULL; - - crypto_rand(buf, buflen); - - mapping = tor_mmap_file(fname1); - test_assert(! mapping); - - write_str_to_file(fname1, "Short file.", 1); - write_bytes_to_file(fname2, buf, buflen, 1); - write_bytes_to_file(fname3, buf, 16384, 1); - - mapping = tor_mmap_file(fname1); - test_assert(mapping); - test_eq(mapping->size, strlen("Short file.")); - test_streq(mapping->data, "Short file."); -#ifdef MS_WINDOWS - tor_munmap_file(mapping); - mapping = NULL; - test_assert(unlink(fname1) == 0); -#else - /* make sure we can unlink. */ - test_assert(unlink(fname1) == 0); - test_streq(mapping->data, "Short file."); - tor_munmap_file(mapping); - mapping = NULL; -#endif - - /* Now a zero-length file. */ - write_str_to_file(fname1, "", 1); - mapping = tor_mmap_file(fname1); - test_eq(mapping, NULL); - test_eq(ERANGE, errno); - unlink(fname1); - - /* Make sure that we fail to map a no-longer-existent file. */ - mapping = tor_mmap_file(fname1); - test_assert(mapping == NULL); - - /* Now try a big file that stretches across a few pages and isn't aligned */ - mapping = tor_mmap_file(fname2); - test_assert(mapping); - test_eq(mapping->size, buflen); - test_memeq(mapping->data, buf, buflen); - tor_munmap_file(mapping); - mapping = NULL; - - /* Now try a big aligned file. */ - mapping = tor_mmap_file(fname3); - test_assert(mapping); - test_eq(mapping->size, 16384); - test_memeq(mapping->data, buf, 16384); - tor_munmap_file(mapping); - mapping = NULL; - - done: - unlink(fname1); - unlink(fname2); - unlink(fname3); - - tor_free(fname1); - tor_free(fname2); - tor_free(fname3); - tor_free(buf); - - if (mapping) - tor_munmap_file(mapping); -} - -/** Run unit tests for escaping/unescaping data for use by controllers. */ -static void -test_util_control_formats(void) -{ - char *out = NULL; - const char *inp = - "..This is a test\r\nof the emergency \nbroadcast\r\n..system.\r\nZ.\r\n"; - size_t sz; - - sz = read_escaped_data(inp, strlen(inp), &out); - test_streq(out, - ".This is a test\nof the emergency \nbroadcast\n.system.\nZ.\n"); - test_eq(sz, strlen(out)); - - done: - tor_free(out); -} - -static void -test_util_sscanf(void) -{ - unsigned u1, u2, u3; - char s1[10], s2[10], s3[10], ch; - int r; - - r = tor_sscanf("hello world", "hello world"); /* String match: success */ - test_eq(r, 0); - r = tor_sscanf("hello world 3", "hello worlb %u", &u1); /* String fail */ - test_eq(r, 0); - r = tor_sscanf("12345", "%u", &u1); /* Simple number */ - test_eq(r, 1); - test_eq(u1, 12345u); - r = tor_sscanf("", "%u", &u1); /* absent number */ - test_eq(r, 0); - r = tor_sscanf("A", "%u", &u1); /* bogus number */ - test_eq(r, 0); - r = tor_sscanf("4294967295", "%u", &u1); /* UINT32_MAX should work. */ - test_eq(r, 1); - test_eq(u1, 4294967295u); - r = tor_sscanf("4294967296", "%u", &u1); /* Always say -1 at 32 bits. */ - test_eq(r, 0); - r = tor_sscanf("123456", "%2u%u", &u1, &u2); /* Width */ - test_eq(r, 2); - test_eq(u1, 12u); - test_eq(u2, 3456u); - r = tor_sscanf("!12:3:456", "!%2u:%2u:%3u", &u1, &u2, &u3); /* separators */ - test_eq(r, 3); - test_eq(u1, 12u); - test_eq(u2, 3u); - test_eq(u3, 456u); - r = tor_sscanf("12:3:045", "%2u:%2u:%3u", &u1, &u2, &u3); /* 0s */ - test_eq(r, 3); - test_eq(u1, 12u); - test_eq(u2, 3u); - test_eq(u3, 45u); - /* %u does not match space.*/ - r = tor_sscanf("12:3: 45", "%2u:%2u:%3u", &u1, &u2, &u3); - test_eq(r, 2); - /* %u does not match negative numbers. */ - r = tor_sscanf("12:3:-4", "%2u:%2u:%3u", &u1, &u2, &u3); - test_eq(r, 2); - /* Arbitrary amounts of 0-padding are okay */ - r = tor_sscanf("12:03:000000000000000099", "%2u:%2u:%u", &u1, &u2, &u3); - test_eq(r, 3); - test_eq(u1, 12u); - test_eq(u2, 3u); - test_eq(u3, 99u); - - r = tor_sscanf("99% fresh", "%3u%% fresh", &u1); /* percents are scannable.*/ - test_eq(r, 1); - test_eq(u1, 99); - - r = tor_sscanf("hello", "%s", s1); /* %s needs a number. */ - test_eq(r, -1); - - r = tor_sscanf("hello", "%3s%7s", s1, s2); /* %s matches characters. */ - test_eq(r, 2); - test_streq(s1, "hel"); - test_streq(s2, "lo"); - r = tor_sscanf("WD40", "%2s%u", s3, &u1); /* %s%u */ - test_eq(r, 2); - test_streq(s3, "WD"); - test_eq(u1, 40); - r = tor_sscanf("76trombones", "%6u%9s", &u1, s1); /* %u%s */ - test_eq(r, 2); - test_eq(u1, 76); - test_streq(s1, "trombones"); - r = tor_sscanf("hello world", "%9s %9s", s1, s2); /* %s doesn't eat space. */ - test_eq(r, 2); - test_streq(s1, "hello"); - test_streq(s2, "world"); - r = tor_sscanf("hi", "%9s%9s%3s", s1, s2, s3); /* %s can be empty. */ - test_eq(r, 3); - test_streq(s1, "hi"); - test_streq(s2, ""); - test_streq(s3, ""); - - r = tor_sscanf("1.2.3", "%u.%u.%u%c", &u1, &u2, &u3, &ch); - test_eq(r, 3); - r = tor_sscanf("1.2.3 foobar", "%u.%u.%u%c", &u1, &u2, &u3, &ch); - test_eq(r, 4); - - done: - ; -} - -/** Run unit tests for the onion handshake code. */ -static void -test_onion_handshake(void) -{ - /* client-side */ - crypto_dh_env_t *c_dh = NULL; - char c_buf[ONIONSKIN_CHALLENGE_LEN]; - char c_keys[40]; - - /* server-side */ - char s_buf[ONIONSKIN_REPLY_LEN]; - char s_keys[40]; - - /* shared */ - crypto_pk_env_t *pk = NULL; - - pk = pk_generate(0); - - /* client handshake 1. */ - memset(c_buf, 0, ONIONSKIN_CHALLENGE_LEN); - test_assert(! onion_skin_create(pk, &c_dh, c_buf)); - - /* server handshake */ - memset(s_buf, 0, ONIONSKIN_REPLY_LEN); - memset(s_keys, 0, 40); - test_assert(! onion_skin_server_handshake(c_buf, pk, NULL, - s_buf, s_keys, 40)); - - /* client handshake 2 */ - memset(c_keys, 0, 40); - test_assert(! onion_skin_client_handshake(c_dh, s_buf, c_keys, 40)); - - if (memcmp(c_keys, s_keys, 40)) { - puts("Aiiiie"); - exit(1); - } - test_memeq(c_keys, s_keys, 40); - memset(s_buf, 0, 40); - test_memneq(c_keys, s_buf, 40); - - done: - if (c_dh) - crypto_dh_free(c_dh); - if (pk) - crypto_free_pk_env(pk); -} - -/** Run unit tests for router descriptor generation logic. */ -static void -test_dir_format(void) -{ - char buf[8192], buf2[8192]; - char platform[256]; - char fingerprint[FINGERPRINT_LEN+1]; - char *pk1_str = NULL, *pk2_str = NULL, *pk3_str = NULL, *cp; - size_t pk1_str_len, pk2_str_len, pk3_str_len; - routerinfo_t *r1=NULL, *r2=NULL; - crypto_pk_env_t *pk1 = NULL, *pk2 = NULL, *pk3 = NULL; - routerinfo_t *rp1 = NULL; - addr_policy_t *ex1, *ex2; - routerlist_t *dir1 = NULL, *dir2 = NULL; - tor_version_t ver1; - - pk1 = pk_generate(0); - pk2 = pk_generate(1); - pk3 = pk_generate(2); - - test_assert( is_legal_nickname("a")); - test_assert(!is_legal_nickname("")); - test_assert(!is_legal_nickname("abcdefghijklmnopqrst")); /* 20 chars */ - test_assert(!is_legal_nickname("hyphen-")); /* bad char */ - test_assert( is_legal_nickname("abcdefghijklmnopqrs")); /* 19 chars */ - test_assert(!is_legal_nickname("$AAAAAAAA01234AAAAAAAAAAAAAAAAAAAAAAAAAAA")); - /* valid */ - test_assert( is_legal_nickname_or_hexdigest( - "$AAAAAAAA01234AAAAAAAAAAAAAAAAAAAAAAAAAAA")); - test_assert( is_legal_nickname_or_hexdigest( - "$AAAAAAAA01234AAAAAAAAAAAAAAAAAAAAAAAAAAA=fred")); - test_assert( is_legal_nickname_or_hexdigest( - "$AAAAAAAA01234AAAAAAAAAAAAAAAAAAAAAAAAAAA~fred")); - /* too short */ - test_assert(!is_legal_nickname_or_hexdigest( - "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")); - /* illegal char */ - test_assert(!is_legal_nickname_or_hexdigest( - "$AAAAAAzAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")); - /* hex part too long */ - test_assert(!is_legal_nickname_or_hexdigest( - "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")); - test_assert(!is_legal_nickname_or_hexdigest( - "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=fred")); - /* Bad nickname */ - test_assert(!is_legal_nickname_or_hexdigest( - "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=")); - test_assert(!is_legal_nickname_or_hexdigest( - "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~")); - test_assert(!is_legal_nickname_or_hexdigest( - "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~hyphen-")); - test_assert(!is_legal_nickname_or_hexdigest( - "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~" - "abcdefghijklmnoppqrst")); - /* Bad extra char. */ - test_assert(!is_legal_nickname_or_hexdigest( - "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA!")); - test_assert(is_legal_nickname_or_hexdigest("xyzzy")); - test_assert(is_legal_nickname_or_hexdigest("abcdefghijklmnopqrs")); - test_assert(!is_legal_nickname_or_hexdigest("abcdefghijklmnopqrst")); - - get_platform_str(platform, sizeof(platform)); - r1 = tor_malloc_zero(sizeof(routerinfo_t)); - r1->address = tor_strdup("18.244.0.1"); - r1->addr = 0xc0a80001u; /* 192.168.0.1 */ - r1->cache_info.published_on = 0; - r1->or_port = 9000; - r1->dir_port = 9003; - r1->onion_pkey = crypto_pk_dup_key(pk1); - r1->identity_pkey = crypto_pk_dup_key(pk2); - r1->bandwidthrate = 1000; - r1->bandwidthburst = 5000; - r1->bandwidthcapacity = 10000; - r1->exit_policy = NULL; - r1->nickname = tor_strdup("Magri"); - r1->platform = tor_strdup(platform); - - ex1 = tor_malloc_zero(sizeof(addr_policy_t)); - ex2 = tor_malloc_zero(sizeof(addr_policy_t)); - ex1->policy_type = ADDR_POLICY_ACCEPT; - tor_addr_from_ipv4h(&ex1->addr, 0); - ex1->maskbits = 0; - ex1->prt_min = ex1->prt_max = 80; - ex2->policy_type = ADDR_POLICY_REJECT; - tor_addr_from_ipv4h(&ex2->addr, 18<<24); - ex2->maskbits = 8; - ex2->prt_min = ex2->prt_max = 24; - r2 = tor_malloc_zero(sizeof(routerinfo_t)); - r2->address = tor_strdup("1.1.1.1"); - r2->addr = 0x0a030201u; /* 10.3.2.1 */ - r2->platform = tor_strdup(platform); - r2->cache_info.published_on = 5; - r2->or_port = 9005; - r2->dir_port = 0; - r2->onion_pkey = crypto_pk_dup_key(pk2); - r2->identity_pkey = crypto_pk_dup_key(pk1); - r2->bandwidthrate = r2->bandwidthburst = r2->bandwidthcapacity = 3000; - r2->exit_policy = smartlist_create(); - smartlist_add(r2->exit_policy, ex2); - smartlist_add(r2->exit_policy, ex1); - r2->nickname = tor_strdup("Fred"); - - test_assert(!crypto_pk_write_public_key_to_string(pk1, &pk1_str, - &pk1_str_len)); - test_assert(!crypto_pk_write_public_key_to_string(pk2 , &pk2_str, - &pk2_str_len)); - test_assert(!crypto_pk_write_public_key_to_string(pk3 , &pk3_str, - &pk3_str_len)); - - memset(buf, 0, 2048); - test_assert(router_dump_router_to_string(buf, 2048, r1, pk2)>0); - - strlcpy(buf2, "router Magri 18.244.0.1 9000 0 9003\n" - "platform Tor "VERSION" on ", sizeof(buf2)); - strlcat(buf2, get_uname(), sizeof(buf2)); - strlcat(buf2, "\n" - "opt protocols Link 1 2 Circuit 1\n" - "published 1970-01-01 00:00:00\n" - "opt fingerprint ", sizeof(buf2)); - test_assert(!crypto_pk_get_fingerprint(pk2, fingerprint, 1)); - strlcat(buf2, fingerprint, sizeof(buf2)); - strlcat(buf2, "\nuptime 0\n" - /* XXX the "0" above is hard-coded, but even if we made it reflect - * uptime, that still wouldn't make it right, because the two - * descriptors might be made on different seconds... hm. */ - "bandwidth 1000 5000 10000\n" - "opt extra-info-digest 0000000000000000000000000000000000000000\n" - "onion-key\n", sizeof(buf2)); - strlcat(buf2, pk1_str, sizeof(buf2)); - strlcat(buf2, "signing-key\n", sizeof(buf2)); - strlcat(buf2, pk2_str, sizeof(buf2)); - strlcat(buf2, "opt hidden-service-dir\n", sizeof(buf2)); - strlcat(buf2, "reject *:*\nrouter-signature\n", sizeof(buf2)); - buf[strlen(buf2)] = '\0'; /* Don't compare the sig; it's never the same - * twice */ - - test_streq(buf, buf2); - - test_assert(router_dump_router_to_string(buf, 2048, r1, pk2)>0); - cp = buf; - rp1 = router_parse_entry_from_string((const char*)cp,NULL,1,0,NULL); - test_assert(rp1); - test_streq(rp1->address, r1->address); - test_eq(rp1->or_port, r1->or_port); - //test_eq(rp1->dir_port, r1->dir_port); - test_eq(rp1->bandwidthrate, r1->bandwidthrate); - test_eq(rp1->bandwidthburst, r1->bandwidthburst); - test_eq(rp1->bandwidthcapacity, r1->bandwidthcapacity); - test_assert(crypto_pk_cmp_keys(rp1->onion_pkey, pk1) == 0); - test_assert(crypto_pk_cmp_keys(rp1->identity_pkey, pk2) == 0); - //test_assert(rp1->exit_policy == NULL); - -#if 0 - /* XXX Once we have exit policies, test this again. XXX */ - strlcpy(buf2, "router tor.tor.tor 9005 0 0 3000\n", sizeof(buf2)); - strlcat(buf2, pk2_str, sizeof(buf2)); - strlcat(buf2, "signing-key\n", sizeof(buf2)); - strlcat(buf2, pk1_str, sizeof(buf2)); - strlcat(buf2, "accept *:80\nreject 18.*:24\n\n", sizeof(buf2)); - test_assert(router_dump_router_to_string(buf, 2048, &r2, pk2)>0); - test_streq(buf, buf2); - - cp = buf; - rp2 = router_parse_entry_from_string(&cp,1); - test_assert(rp2); - test_streq(rp2->address, r2.address); - test_eq(rp2->or_port, r2.or_port); - test_eq(rp2->dir_port, r2.dir_port); - test_eq(rp2->bandwidth, r2.bandwidth); - test_assert(crypto_pk_cmp_keys(rp2->onion_pkey, pk2) == 0); - test_assert(crypto_pk_cmp_keys(rp2->identity_pkey, pk1) == 0); - test_eq(rp2->exit_policy->policy_type, EXIT_POLICY_ACCEPT); - test_streq(rp2->exit_policy->string, "accept *:80"); - test_streq(rp2->exit_policy->address, "*"); - test_streq(rp2->exit_policy->port, "80"); - test_eq(rp2->exit_policy->next->policy_type, EXIT_POLICY_REJECT); - test_streq(rp2->exit_policy->next->string, "reject 18.*:24"); - test_streq(rp2->exit_policy->next->address, "18.*"); - test_streq(rp2->exit_policy->next->port, "24"); - test_assert(rp2->exit_policy->next->next == NULL); - - /* Okay, now for the directories. */ - { - fingerprint_list = smartlist_create(); - crypto_pk_get_fingerprint(pk2, buf, 1); - add_fingerprint_to_dir("Magri", buf, fingerprint_list); - crypto_pk_get_fingerprint(pk1, buf, 1); - add_fingerprint_to_dir("Fred", buf, fingerprint_list); - } - - { - char d[DIGEST_LEN]; - const char *m; - /* XXXX NM re-enable. */ - /* Make sure routers aren't too far in the past any more. */ - r1->cache_info.published_on = time(NULL); - r2->cache_info.published_on = time(NULL)-3*60*60; - test_assert(router_dump_router_to_string(buf, 2048, r1, pk2)>0); - test_eq(dirserv_add_descriptor(buf,&m,""), ROUTER_ADDED_NOTIFY_GENERATOR); - test_assert(router_dump_router_to_string(buf, 2048, r2, pk1)>0); - test_eq(dirserv_add_descriptor(buf,&m,""), ROUTER_ADDED_NOTIFY_GENERATOR); - get_options()->Nickname = tor_strdup("DirServer"); - test_assert(!dirserv_dump_directory_to_string(&cp,pk3, 0)); - crypto_pk_get_digest(pk3, d); - test_assert(!router_parse_directory(cp)); - test_eq(2, smartlist_len(dir1->routers)); - tor_free(cp); - } -#endif - dirserv_free_fingerprint_list(); - - /* Try out version parsing functionality */ - test_eq(0, tor_version_parse("0.3.4pre2-cvs", &ver1)); - test_eq(0, ver1.major); - test_eq(3, ver1.minor); - test_eq(4, ver1.micro); - test_eq(VER_PRE, ver1.status); - test_eq(2, ver1.patchlevel); - test_eq(0, tor_version_parse("0.3.4rc1", &ver1)); - test_eq(0, ver1.major); - test_eq(3, ver1.minor); - test_eq(4, ver1.micro); - test_eq(VER_RC, ver1.status); - test_eq(1, ver1.patchlevel); - test_eq(0, tor_version_parse("1.3.4", &ver1)); - test_eq(1, ver1.major); - test_eq(3, ver1.minor); - test_eq(4, ver1.micro); - test_eq(VER_RELEASE, ver1.status); - test_eq(0, ver1.patchlevel); - test_eq(0, tor_version_parse("1.3.4.999", &ver1)); - test_eq(1, ver1.major); - test_eq(3, ver1.minor); - test_eq(4, ver1.micro); - test_eq(VER_RELEASE, ver1.status); - test_eq(999, ver1.patchlevel); - test_eq(0, tor_version_parse("0.1.2.4-alpha", &ver1)); - test_eq(0, ver1.major); - test_eq(1, ver1.minor); - test_eq(2, ver1.micro); - test_eq(4, ver1.patchlevel); - test_eq(VER_RELEASE, ver1.status); - test_streq("alpha", ver1.status_tag); - test_eq(0, tor_version_parse("0.1.2.4", &ver1)); - test_eq(0, ver1.major); - test_eq(1, ver1.minor); - test_eq(2, ver1.micro); - test_eq(4, ver1.patchlevel); - test_eq(VER_RELEASE, ver1.status); - test_streq("", ver1.status_tag); - -#define test_eq_vs(vs1, vs2) test_eq_type(version_status_t, "%d", (vs1), (vs2)) -#define test_v_i_o(val, ver, lst) \ - test_eq_vs(val, tor_version_is_obsolete(ver, lst)) - - /* make sure tor_version_is_obsolete() works */ - test_v_i_o(VS_OLD, "0.0.1", "Tor 0.0.2"); - test_v_i_o(VS_OLD, "0.0.1", "0.0.2, Tor 0.0.3"); - test_v_i_o(VS_OLD, "0.0.1", "0.0.2,Tor 0.0.3"); - test_v_i_o(VS_OLD, "0.0.1","0.0.3,BetterTor 0.0.1"); - test_v_i_o(VS_RECOMMENDED, "0.0.2", "Tor 0.0.2,Tor 0.0.3"); - test_v_i_o(VS_NEW_IN_SERIES, "0.0.2", "Tor 0.0.2pre1,Tor 0.0.3"); - test_v_i_o(VS_OLD, "0.0.2", "Tor 0.0.2.1,Tor 0.0.3"); - test_v_i_o(VS_NEW, "0.1.0", "Tor 0.0.2,Tor 0.0.3"); - test_v_i_o(VS_RECOMMENDED, "0.0.7rc2", "0.0.7,Tor 0.0.7rc2,Tor 0.0.8"); - test_v_i_o(VS_OLD, "0.0.5.0", "0.0.5.1-cvs"); - test_v_i_o(VS_NEW_IN_SERIES, "0.0.5.1-cvs", "0.0.5, 0.0.6"); - /* Not on list, but newer than any in same series. */ - test_v_i_o(VS_NEW_IN_SERIES, "0.1.0.3", - "Tor 0.1.0.2,Tor 0.0.9.5,Tor 0.1.1.0"); - /* Series newer than any on list. */ - test_v_i_o(VS_NEW, "0.1.2.3", "Tor 0.1.0.2,Tor 0.0.9.5,Tor 0.1.1.0"); - /* Series older than any on list. */ - test_v_i_o(VS_OLD, "0.0.1.3", "Tor 0.1.0.2,Tor 0.0.9.5,Tor 0.1.1.0"); - /* Not on list, not newer than any on same series. */ - test_v_i_o(VS_UNRECOMMENDED, "0.1.0.1", - "Tor 0.1.0.2,Tor 0.0.9.5,Tor 0.1.1.0"); - /* On list, not newer than any on same series. */ - test_v_i_o(VS_UNRECOMMENDED, - "0.1.0.1", "Tor 0.1.0.2,Tor 0.0.9.5,Tor 0.1.1.0"); - test_eq(0, tor_version_as_new_as("Tor 0.0.5", "0.0.9pre1-cvs")); - test_eq(1, tor_version_as_new_as( - "Tor 0.0.8 on Darwin 64-121-192-100.c3-0." - "sfpo-ubr1.sfrn-sfpo.ca.cable.rcn.com Power Macintosh", - "0.0.8rc2")); - test_eq(0, tor_version_as_new_as( - "Tor 0.0.8 on Darwin 64-121-192-100.c3-0." - "sfpo-ubr1.sfrn-sfpo.ca.cable.rcn.com Power Macintosh", "0.0.8.2")); - - /* Now try svn revisions. */ - test_eq(1, tor_version_as_new_as("Tor 0.2.1.0-dev (r100)", - "Tor 0.2.1.0-dev (r99)")); - test_eq(1, tor_version_as_new_as("Tor 0.2.1.0-dev (r100) on Banana Jr", - "Tor 0.2.1.0-dev (r99) on Hal 9000")); - test_eq(1, tor_version_as_new_as("Tor 0.2.1.0-dev (r100)", - "Tor 0.2.1.0-dev on Colossus")); - test_eq(0, tor_version_as_new_as("Tor 0.2.1.0-dev (r99)", - "Tor 0.2.1.0-dev (r100)")); - test_eq(0, tor_version_as_new_as("Tor 0.2.1.0-dev (r99) on MCP", - "Tor 0.2.1.0-dev (r100) on AM")); - test_eq(0, tor_version_as_new_as("Tor 0.2.1.0-dev", - "Tor 0.2.1.0-dev (r99)")); - test_eq(1, tor_version_as_new_as("Tor 0.2.1.1", - "Tor 0.2.1.0-dev (r99)")); - done: - if (r1) - routerinfo_free(r1); - if (r2) - routerinfo_free(r2); - - tor_free(pk1_str); - tor_free(pk2_str); - tor_free(pk3_str); - if (pk1) crypto_free_pk_env(pk1); - if (pk2) crypto_free_pk_env(pk2); - if (pk3) crypto_free_pk_env(pk3); - if (rp1) routerinfo_free(rp1); - tor_free(dir1); /* XXXX And more !*/ - tor_free(dir2); /* And more !*/ -} - -/** Run unit tests for misc directory functions. */ -static void -test_dirutil(void) -{ - smartlist_t *sl = smartlist_create(); - fp_pair_t *pair; - - dir_split_resource_into_fingerprint_pairs( - /* Two pairs, out of order, with one duplicate. */ - "73656372657420646174612E0000000000FFFFFF-" - "557365204145532d32353620696e73746561642e+" - "73656372657420646174612E0000000000FFFFFF-" - "557365204145532d32353620696e73746561642e+" - "48657861646563696d616c2069736e277420736f-" - "676f6f6420666f7220686964696e6720796f7572.z", sl); - - test_eq(smartlist_len(sl), 2); - pair = smartlist_get(sl, 0); - test_memeq(pair->first, "Hexadecimal isn't so", DIGEST_LEN); - test_memeq(pair->second, "good for hiding your", DIGEST_LEN); - pair = smartlist_get(sl, 1); - test_memeq(pair->first, "secret data.\0\0\0\0\0\xff\xff\xff", DIGEST_LEN); - test_memeq(pair->second, "Use AES-256 instead.", DIGEST_LEN); - - done: - SMARTLIST_FOREACH(sl, fp_pair_t *, pair, tor_free(pair)); - smartlist_free(sl); -} - -extern const char AUTHORITY_CERT_1[]; -extern const char AUTHORITY_SIGNKEY_1[]; -extern const char AUTHORITY_CERT_2[]; -extern const char AUTHORITY_SIGNKEY_2[]; -extern const char AUTHORITY_CERT_3[]; -extern const char AUTHORITY_SIGNKEY_3[]; - -/** Helper: Test that two networkstatus_voter_info_t do in fact represent the - * same voting authority, and that they do in fact have all the same - * information. */ -static void -test_same_voter(networkstatus_voter_info_t *v1, - networkstatus_voter_info_t *v2) -{ - test_streq(v1->nickname, v2->nickname); - test_memeq(v1->identity_digest, v2->identity_digest, DIGEST_LEN); - test_streq(v1->address, v2->address); - test_eq(v1->addr, v2->addr); - test_eq(v1->dir_port, v2->dir_port); - test_eq(v1->or_port, v2->or_port); - test_streq(v1->contact, v2->contact); - test_memeq(v1->vote_digest, v2->vote_digest, DIGEST_LEN); - done: - ; -} - -/** Run unit tests for getting the median of a list. */ -static void -test_util_order_functions(void) -{ - int lst[25], n = 0; - // int a=12,b=24,c=25,d=60,e=77; - -#define median() median_int(lst, n) - - lst[n++] = 12; - test_eq(12, median()); /* 12 */ - lst[n++] = 77; - //smartlist_shuffle(sl); - test_eq(12, median()); /* 12, 77 */ - lst[n++] = 77; - //smartlist_shuffle(sl); - test_eq(77, median()); /* 12, 77, 77 */ - lst[n++] = 24; - test_eq(24, median()); /* 12,24,77,77 */ - lst[n++] = 60; - lst[n++] = 12; - lst[n++] = 25; - //smartlist_shuffle(sl); - test_eq(25, median()); /* 12,12,24,25,60,77,77 */ -#undef median - - done: - ; -} - -/** Helper: Make a new routerinfo containing the right information for a - * given vote_routerstatus_t. */ -static routerinfo_t * -generate_ri_from_rs(const vote_routerstatus_t *vrs) -{ - routerinfo_t *r; - const routerstatus_t *rs = &vrs->status; - static time_t published = 0; - - r = tor_malloc_zero(sizeof(routerinfo_t)); - memcpy(r->cache_info.identity_digest, rs->identity_digest, DIGEST_LEN); - memcpy(r->cache_info.signed_descriptor_digest, rs->descriptor_digest, - DIGEST_LEN); - r->cache_info.do_not_cache = 1; - r->cache_info.routerlist_index = -1; - r->cache_info.signed_descriptor_body = - tor_strdup("123456789012345678901234567890123"); - r->cache_info.signed_descriptor_len = - strlen(r->cache_info.signed_descriptor_body); - r->exit_policy = smartlist_create(); - r->cache_info.published_on = ++published + time(NULL); - return r; -} - -/** Run unit tests for generating and parsing V3 consensus networkstatus - * documents. */ -static void -test_v3_networkstatus(void) -{ - authority_cert_t *cert1=NULL, *cert2=NULL, *cert3=NULL; - crypto_pk_env_t *sign_skey_1=NULL, *sign_skey_2=NULL, *sign_skey_3=NULL; - crypto_pk_env_t *sign_skey_leg1=NULL; - const char *msg=NULL; - - time_t now = time(NULL); - networkstatus_voter_info_t *voter; - networkstatus_t *vote=NULL, *v1=NULL, *v2=NULL, *v3=NULL, *con=NULL; - vote_routerstatus_t *vrs; - routerstatus_t *rs; - char *v1_text=NULL, *v2_text=NULL, *v3_text=NULL, *consensus_text=NULL, *cp; - smartlist_t *votes = smartlist_create(); - - /* For generating the two other consensuses. */ - char *detached_text1=NULL, *detached_text2=NULL; - char *consensus_text2=NULL, *consensus_text3=NULL; - networkstatus_t *con2=NULL, *con3=NULL; - ns_detached_signatures_t *dsig1=NULL, *dsig2=NULL; - - /* Parse certificates and keys. */ - cert1 = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); - test_assert(cert1); - test_assert(cert1->is_cross_certified); - cert2 = authority_cert_parse_from_string(AUTHORITY_CERT_2, NULL); - test_assert(cert2); - cert3 = authority_cert_parse_from_string(AUTHORITY_CERT_3, NULL); - test_assert(cert3); - sign_skey_1 = crypto_new_pk_env(); - sign_skey_2 = crypto_new_pk_env(); - sign_skey_3 = crypto_new_pk_env(); - sign_skey_leg1 = pk_generate(4); - - test_assert(!crypto_pk_read_private_key_from_string(sign_skey_1, - AUTHORITY_SIGNKEY_1,-1)); - test_assert(!crypto_pk_read_private_key_from_string(sign_skey_2, - AUTHORITY_SIGNKEY_2,-1)); - test_assert(!crypto_pk_read_private_key_from_string(sign_skey_3, - AUTHORITY_SIGNKEY_3,-1)); - - test_assert(!crypto_pk_cmp_keys(sign_skey_1, cert1->signing_key)); - test_assert(!crypto_pk_cmp_keys(sign_skey_2, cert2->signing_key)); - - /* - * Set up a vote; generate it; try to parse it. - */ - vote = tor_malloc_zero(sizeof(networkstatus_t)); - vote->type = NS_TYPE_VOTE; - vote->published = now; - vote->valid_after = now+1000; - vote->fresh_until = now+2000; - vote->valid_until = now+3000; - vote->vote_seconds = 100; - vote->dist_seconds = 200; - vote->supported_methods = smartlist_create(); - smartlist_split_string(vote->supported_methods, "1 2 3", NULL, 0, -1); - vote->client_versions = tor_strdup("0.1.2.14,0.1.2.15"); - vote->server_versions = tor_strdup("0.1.2.14,0.1.2.15,0.1.2.16"); - vote->known_flags = smartlist_create(); - smartlist_split_string(vote->known_flags, - "Authority Exit Fast Guard Running Stable V2Dir Valid", - 0, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - vote->voters = smartlist_create(); - voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t)); - voter->nickname = tor_strdup("Voter1"); - voter->address = tor_strdup("1.2.3.4"); - voter->addr = 0x01020304; - voter->dir_port = 80; - voter->or_port = 9000; - voter->contact = tor_strdup("voter@example.com"); - crypto_pk_get_digest(cert1->identity_key, voter->identity_digest); - smartlist_add(vote->voters, voter); - vote->cert = authority_cert_dup(cert1); - vote->routerstatus_list = smartlist_create(); - /* add the first routerstatus. */ - vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); - rs = &vrs->status; - vrs->version = tor_strdup("0.1.2.14"); - rs->published_on = now-1500; - strlcpy(rs->nickname, "router2", sizeof(rs->nickname)); - memset(rs->identity_digest, 3, DIGEST_LEN); - memset(rs->descriptor_digest, 78, DIGEST_LEN); - rs->addr = 0x99008801; - rs->or_port = 443; - rs->dir_port = 8000; - /* all flags but running cleared */ - rs->is_running = 1; - smartlist_add(vote->routerstatus_list, vrs); - test_assert(router_add_to_routerlist(generate_ri_from_rs(vrs), &msg,0,0)>=0); - - /* add the second routerstatus. */ - vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); - rs = &vrs->status; - vrs->version = tor_strdup("0.2.0.5"); - rs->published_on = now-1000; - strlcpy(rs->nickname, "router1", sizeof(rs->nickname)); - memset(rs->identity_digest, 5, DIGEST_LEN); - memset(rs->descriptor_digest, 77, DIGEST_LEN); - rs->addr = 0x99009901; - rs->or_port = 443; - rs->dir_port = 0; - rs->is_exit = rs->is_stable = rs->is_fast = rs->is_running = - rs->is_valid = rs->is_v2_dir = rs->is_possible_guard = 1; - smartlist_add(vote->routerstatus_list, vrs); - test_assert(router_add_to_routerlist(generate_ri_from_rs(vrs), &msg,0,0)>=0); - - /* add the third routerstatus. */ - vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); - rs = &vrs->status; - vrs->version = tor_strdup("0.1.0.3"); - rs->published_on = now-1000; - strlcpy(rs->nickname, "router3", sizeof(rs->nickname)); - memset(rs->identity_digest, 33, DIGEST_LEN); - memset(rs->descriptor_digest, 79, DIGEST_LEN); - rs->addr = 0xAA009901; - rs->or_port = 400; - rs->dir_port = 9999; - rs->is_authority = rs->is_exit = rs->is_stable = rs->is_fast = - rs->is_running = rs->is_valid = rs->is_v2_dir = rs->is_possible_guard = 1; - smartlist_add(vote->routerstatus_list, vrs); - test_assert(router_add_to_routerlist(generate_ri_from_rs(vrs), &msg,0,0)>=0); - - /* add a fourth routerstatus that is not running. */ - vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); - rs = &vrs->status; - vrs->version = tor_strdup("0.1.6.3"); - rs->published_on = now-1000; - strlcpy(rs->nickname, "router4", sizeof(rs->nickname)); - memset(rs->identity_digest, 34, DIGEST_LEN); - memset(rs->descriptor_digest, 48, DIGEST_LEN); - rs->addr = 0xC0000203; - rs->or_port = 500; - rs->dir_port = 1999; - /* Running flag (and others) cleared */ - smartlist_add(vote->routerstatus_list, vrs); - test_assert(router_add_to_routerlist(generate_ri_from_rs(vrs), &msg,0,0)>=0); - - /* dump the vote and try to parse it. */ - v1_text = format_networkstatus_vote(sign_skey_1, vote); - test_assert(v1_text); - v1 = networkstatus_parse_vote_from_string(v1_text, NULL, NS_TYPE_VOTE); - test_assert(v1); - - /* Make sure the parsed thing was right. */ - test_eq(v1->type, NS_TYPE_VOTE); - test_eq(v1->published, vote->published); - test_eq(v1->valid_after, vote->valid_after); - test_eq(v1->fresh_until, vote->fresh_until); - test_eq(v1->valid_until, vote->valid_until); - test_eq(v1->vote_seconds, vote->vote_seconds); - test_eq(v1->dist_seconds, vote->dist_seconds); - test_streq(v1->client_versions, vote->client_versions); - test_streq(v1->server_versions, vote->server_versions); - test_assert(v1->voters && smartlist_len(v1->voters)); - voter = smartlist_get(v1->voters, 0); - test_streq(voter->nickname, "Voter1"); - test_streq(voter->address, "1.2.3.4"); - test_eq(voter->addr, 0x01020304); - test_eq(voter->dir_port, 80); - test_eq(voter->or_port, 9000); - test_streq(voter->contact, "voter@example.com"); - test_assert(v1->cert); - test_assert(!crypto_pk_cmp_keys(sign_skey_1, v1->cert->signing_key)); - cp = smartlist_join_strings(v1->known_flags, ":", 0, NULL); - test_streq(cp, "Authority:Exit:Fast:Guard:Running:Stable:V2Dir:Valid"); - tor_free(cp); - test_eq(smartlist_len(v1->routerstatus_list), 4); - /* Check the first routerstatus. */ - vrs = smartlist_get(v1->routerstatus_list, 0); - rs = &vrs->status; - test_streq(vrs->version, "0.1.2.14"); - test_eq(rs->published_on, now-1500); - test_streq(rs->nickname, "router2"); - test_memeq(rs->identity_digest, - "\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3", - DIGEST_LEN); - test_memeq(rs->descriptor_digest, "NNNNNNNNNNNNNNNNNNNN", DIGEST_LEN); - test_eq(rs->addr, 0x99008801); - test_eq(rs->or_port, 443); - test_eq(rs->dir_port, 8000); - test_eq(vrs->flags, U64_LITERAL(16)); // no flags except "running" - /* Check the second routerstatus. */ - vrs = smartlist_get(v1->routerstatus_list, 1); - rs = &vrs->status; - test_streq(vrs->version, "0.2.0.5"); - test_eq(rs->published_on, now-1000); - test_streq(rs->nickname, "router1"); - test_memeq(rs->identity_digest, - "\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5", - DIGEST_LEN); - test_memeq(rs->descriptor_digest, "MMMMMMMMMMMMMMMMMMMM", DIGEST_LEN); - test_eq(rs->addr, 0x99009901); - test_eq(rs->or_port, 443); - test_eq(rs->dir_port, 0); - test_eq(vrs->flags, U64_LITERAL(254)); // all flags except "authority." - - /* Generate second vote. It disagrees on some of the times, - * and doesn't list versions, and knows some crazy flags */ - vote->published = now+1; - vote->fresh_until = now+3005; - vote->dist_seconds = 300; - authority_cert_free(vote->cert); - vote->cert = authority_cert_dup(cert2); - tor_free(vote->client_versions); - tor_free(vote->server_versions); - voter = smartlist_get(vote->voters, 0); - tor_free(voter->nickname); - tor_free(voter->address); - voter->nickname = tor_strdup("Voter2"); - voter->address = tor_strdup("2.3.4.5"); - voter->addr = 0x02030405; - crypto_pk_get_digest(cert2->identity_key, voter->identity_digest); - smartlist_add(vote->known_flags, tor_strdup("MadeOfCheese")); - smartlist_add(vote->known_flags, tor_strdup("MadeOfTin")); - smartlist_sort_strings(vote->known_flags); - vrs = smartlist_get(vote->routerstatus_list, 2); - smartlist_del_keeporder(vote->routerstatus_list, 2); - tor_free(vrs->version); - tor_free(vrs); - vrs = smartlist_get(vote->routerstatus_list, 0); - vrs->status.is_fast = 1; - /* generate and parse. */ - v2_text = format_networkstatus_vote(sign_skey_2, vote); - test_assert(v2_text); - v2 = networkstatus_parse_vote_from_string(v2_text, NULL, NS_TYPE_VOTE); - test_assert(v2); - /* Check that flags come out right.*/ - cp = smartlist_join_strings(v2->known_flags, ":", 0, NULL); - test_streq(cp, "Authority:Exit:Fast:Guard:MadeOfCheese:MadeOfTin:" - "Running:Stable:V2Dir:Valid"); - tor_free(cp); - vrs = smartlist_get(v2->routerstatus_list, 1); - /* 1023 - authority(1) - madeofcheese(16) - madeoftin(32) */ - test_eq(vrs->flags, U64_LITERAL(974)); - - /* Generate the third vote. */ - vote->published = now; - vote->fresh_until = now+2003; - vote->dist_seconds = 250; - authority_cert_free(vote->cert); - vote->cert = authority_cert_dup(cert3); - smartlist_add(vote->supported_methods, tor_strdup("4")); - vote->client_versions = tor_strdup("0.1.2.14,0.1.2.17"); - vote->server_versions = tor_strdup("0.1.2.10,0.1.2.15,0.1.2.16"); - voter = smartlist_get(vote->voters, 0); - tor_free(voter->nickname); - tor_free(voter->address); - voter->nickname = tor_strdup("Voter3"); - voter->address = tor_strdup("3.4.5.6"); - voter->addr = 0x03040506; - crypto_pk_get_digest(cert3->identity_key, voter->identity_digest); - /* This one has a legacy id. */ - memset(voter->legacy_id_digest, (int)'A', DIGEST_LEN); - vrs = smartlist_get(vote->routerstatus_list, 0); - smartlist_del_keeporder(vote->routerstatus_list, 0); - tor_free(vrs->version); - tor_free(vrs); - vrs = smartlist_get(vote->routerstatus_list, 0); - memset(vrs->status.descriptor_digest, (int)'Z', DIGEST_LEN); - test_assert(router_add_to_routerlist(generate_ri_from_rs(vrs), &msg,0,0)>=0); - - v3_text = format_networkstatus_vote(sign_skey_3, vote); - test_assert(v3_text); - - v3 = networkstatus_parse_vote_from_string(v3_text, NULL, NS_TYPE_VOTE); - test_assert(v3); - - /* Compute a consensus as voter 3. */ - smartlist_add(votes, v3); - smartlist_add(votes, v1); - smartlist_add(votes, v2); - consensus_text = networkstatus_compute_consensus(votes, 3, - cert3->identity_key, - sign_skey_3, - "AAAAAAAAAAAAAAAAAAAA", - sign_skey_leg1); - test_assert(consensus_text); - con = networkstatus_parse_vote_from_string(consensus_text, NULL, - NS_TYPE_CONSENSUS); - test_assert(con); - //log_notice(LD_GENERAL, "<<%s>>\n<<%s>>\n<<%s>>\n", - // v1_text, v2_text, v3_text); - - /* Check consensus contents. */ - test_assert(con->type == NS_TYPE_CONSENSUS); - test_eq(con->published, 0); /* this field only appears in votes. */ - test_eq(con->valid_after, now+1000); - test_eq(con->fresh_until, now+2003); /* median */ - test_eq(con->valid_until, now+3000); - test_eq(con->vote_seconds, 100); - test_eq(con->dist_seconds, 250); /* median */ - test_streq(con->client_versions, "0.1.2.14"); - test_streq(con->server_versions, "0.1.2.15,0.1.2.16"); - cp = smartlist_join_strings(v2->known_flags, ":", 0, NULL); - test_streq(cp, "Authority:Exit:Fast:Guard:MadeOfCheese:MadeOfTin:" - "Running:Stable:V2Dir:Valid"); - tor_free(cp); - test_eq(4, smartlist_len(con->voters)); /*3 voters, 1 legacy key.*/ - /* The voter id digests should be in this order. */ - test_assert(memcmp(cert2->cache_info.identity_digest, - cert1->cache_info.identity_digest,DIGEST_LEN)<0); - test_assert(memcmp(cert1->cache_info.identity_digest, - cert3->cache_info.identity_digest,DIGEST_LEN)<0); - test_same_voter(smartlist_get(con->voters, 1), - smartlist_get(v2->voters, 0)); - test_same_voter(smartlist_get(con->voters, 2), - smartlist_get(v1->voters, 0)); - test_same_voter(smartlist_get(con->voters, 3), - smartlist_get(v3->voters, 0)); - - test_assert(!con->cert); - test_eq(2, smartlist_len(con->routerstatus_list)); - /* There should be two listed routers: one with identity 3, one with - * identity 5. */ - /* This one showed up in 2 digests. */ - rs = smartlist_get(con->routerstatus_list, 0); - test_memeq(rs->identity_digest, - "\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3", - DIGEST_LEN); - test_memeq(rs->descriptor_digest, "NNNNNNNNNNNNNNNNNNNN", DIGEST_LEN); - test_assert(!rs->is_authority); - test_assert(!rs->is_exit); - test_assert(!rs->is_fast); - test_assert(!rs->is_possible_guard); - test_assert(!rs->is_stable); - test_assert(rs->is_running); /* If it wasn't running it wouldn't be here */ - test_assert(!rs->is_v2_dir); - test_assert(!rs->is_valid); - test_assert(!rs->is_named); - /* XXXX check version */ - - rs = smartlist_get(con->routerstatus_list, 1); - /* This one showed up in 3 digests. Twice with ID 'M', once with 'Z'. */ - test_memeq(rs->identity_digest, - "\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5", - DIGEST_LEN); - test_streq(rs->nickname, "router1"); - test_memeq(rs->descriptor_digest, "MMMMMMMMMMMMMMMMMMMM", DIGEST_LEN); - test_eq(rs->published_on, now-1000); - test_eq(rs->addr, 0x99009901); - test_eq(rs->or_port, 443); - test_eq(rs->dir_port, 0); - test_assert(!rs->is_authority); - test_assert(rs->is_exit); - test_assert(rs->is_fast); - test_assert(rs->is_possible_guard); - test_assert(rs->is_stable); - test_assert(rs->is_running); - test_assert(rs->is_v2_dir); - test_assert(rs->is_valid); - test_assert(!rs->is_named); - /* XXXX check version */ - // x231 - // x213 - - /* Check signatures. the first voter is a pseudo-entry with a legacy key. - * The second one hasn't signed. The fourth one has signed: validate it. */ - voter = smartlist_get(con->voters, 1); - test_assert(!voter->signature); - test_assert(!voter->good_signature); - test_assert(!voter->bad_signature); - - voter = smartlist_get(con->voters, 3); - test_assert(voter->signature); - test_assert(!voter->good_signature); - test_assert(!voter->bad_signature); - test_assert(!networkstatus_check_voter_signature(con, - smartlist_get(con->voters, 3), - cert3)); - test_assert(voter->signature); - test_assert(voter->good_signature); - test_assert(!voter->bad_signature); - - { - const char *msg=NULL; - /* Compute the other two signed consensuses. */ - smartlist_shuffle(votes); - consensus_text2 = networkstatus_compute_consensus(votes, 3, - cert2->identity_key, - sign_skey_2, NULL,NULL); - smartlist_shuffle(votes); - consensus_text3 = networkstatus_compute_consensus(votes, 3, - cert1->identity_key, - sign_skey_1, NULL,NULL); - test_assert(consensus_text2); - test_assert(consensus_text3); - con2 = networkstatus_parse_vote_from_string(consensus_text2, NULL, - NS_TYPE_CONSENSUS); - con3 = networkstatus_parse_vote_from_string(consensus_text3, NULL, - NS_TYPE_CONSENSUS); - test_assert(con2); - test_assert(con3); - - /* All three should have the same digest. */ - test_memeq(con->networkstatus_digest, con2->networkstatus_digest, - DIGEST_LEN); - test_memeq(con->networkstatus_digest, con3->networkstatus_digest, - DIGEST_LEN); - - /* Extract a detached signature from con3. */ - detached_text1 = networkstatus_get_detached_signatures(con3); - tor_assert(detached_text1); - /* Try to parse it. */ - dsig1 = networkstatus_parse_detached_signatures(detached_text1, NULL); - tor_assert(dsig1); - - /* Are parsed values as expected? */ - test_eq(dsig1->valid_after, con3->valid_after); - test_eq(dsig1->fresh_until, con3->fresh_until); - test_eq(dsig1->valid_until, con3->valid_until); - test_memeq(dsig1->networkstatus_digest, con3->networkstatus_digest, - DIGEST_LEN); - test_eq(1, smartlist_len(dsig1->signatures)); - voter = smartlist_get(dsig1->signatures, 0); - test_memeq(voter->identity_digest, cert1->cache_info.identity_digest, - DIGEST_LEN); - - /* Try adding it to con2. */ - detached_text2 = networkstatus_get_detached_signatures(con2); - test_eq(1, networkstatus_add_detached_signatures(con2, dsig1, &msg)); - tor_free(detached_text2); - detached_text2 = networkstatus_get_detached_signatures(con2); - //printf("\n<%s>\n", detached_text2); - dsig2 = networkstatus_parse_detached_signatures(detached_text2, NULL); - test_assert(dsig2); - /* - printf("\n"); - SMARTLIST_FOREACH(dsig2->signatures, networkstatus_voter_info_t *, vi, { - char hd[64]; - base16_encode(hd, sizeof(hd), vi->identity_digest, DIGEST_LEN); - printf("%s\n", hd); - }); - */ - test_eq(2, smartlist_len(dsig2->signatures)); - - /* Try adding to con2 twice; verify that nothing changes. */ - test_eq(0, networkstatus_add_detached_signatures(con2, dsig1, &msg)); - - /* Add to con. */ - test_eq(2, networkstatus_add_detached_signatures(con, dsig2, &msg)); - /* Check signatures */ - test_assert(!networkstatus_check_voter_signature(con, - smartlist_get(con->voters, 1), - cert2)); - test_assert(!networkstatus_check_voter_signature(con, - smartlist_get(con->voters, 2), - cert1)); - - } - - done: - smartlist_free(votes); - tor_free(v1_text); - tor_free(v2_text); - tor_free(v3_text); - tor_free(consensus_text); - - if (vote) - networkstatus_vote_free(vote); - if (v1) - networkstatus_vote_free(v1); - if (v2) - networkstatus_vote_free(v2); - if (v3) - networkstatus_vote_free(v3); - if (con) - networkstatus_vote_free(con); - if (sign_skey_1) - crypto_free_pk_env(sign_skey_1); - if (sign_skey_2) - crypto_free_pk_env(sign_skey_2); - if (sign_skey_3) - crypto_free_pk_env(sign_skey_3); - if (sign_skey_leg1) - crypto_free_pk_env(sign_skey_leg1); - if (cert1) - authority_cert_free(cert1); - if (cert2) - authority_cert_free(cert2); - if (cert3) - authority_cert_free(cert3); - - tor_free(consensus_text2); - tor_free(consensus_text3); - tor_free(detached_text1); - tor_free(detached_text2); - if (con2) - networkstatus_vote_free(con2); - if (con3) - networkstatus_vote_free(con3); - if (dsig1) - ns_detached_signatures_free(dsig1); - if (dsig2) - ns_detached_signatures_free(dsig2); -} - -/** Helper: Parse the exit policy string in <b>policy_str</b>, and make sure - * that policies_summarize() produces the string <b>expected_summary</b> from - * it. */ -static void -test_policy_summary_helper(const char *policy_str, - const char *expected_summary) -{ - config_line_t line; - smartlist_t *policy = smartlist_create(); - char *summary = NULL; - int r; - - line.key = (char*)"foo"; - line.value = (char *)policy_str; - line.next = NULL; - - r = policies_parse_exit_policy(&line, &policy, 0, NULL); - test_eq(r, 0); - summary = policy_summarize(policy); - - test_assert(summary != NULL); - test_streq(summary, expected_summary); - - done: - tor_free(summary); - if (policy) - addr_policy_list_free(policy); -} - -/** Run unit tests for generating summary lines of exit policies */ -static void -test_policies(void) -{ - int i; - smartlist_t *policy = NULL, *policy2 = NULL; - addr_policy_t *p; - tor_addr_t tar; - config_line_t line; - smartlist_t *sm = NULL; - char *policy_str = NULL; - - policy = smartlist_create(); - - p = router_parse_addr_policy_item_from_string("reject 192.168.0.0/16:*",-1); - test_assert(p != NULL); - test_eq(ADDR_POLICY_REJECT, p->policy_type); - tor_addr_from_ipv4h(&tar, 0xc0a80000u); - test_eq(0, tor_addr_compare(&p->addr, &tar, CMP_EXACT)); - test_eq(16, p->maskbits); - test_eq(1, p->prt_min); - test_eq(65535, p->prt_max); - - smartlist_add(policy, p); - - test_assert(ADDR_POLICY_ACCEPTED == - compare_addr_to_addr_policy(0x01020304u, 2, policy)); - test_assert(ADDR_POLICY_PROBABLY_ACCEPTED == - compare_addr_to_addr_policy(0, 2, policy)); - test_assert(ADDR_POLICY_REJECTED == - compare_addr_to_addr_policy(0xc0a80102, 2, policy)); - - policy2 = NULL; - test_assert(0 == policies_parse_exit_policy(NULL, &policy2, 1, NULL)); - test_assert(policy2); - - test_assert(!exit_policy_is_general_exit(policy)); - test_assert(exit_policy_is_general_exit(policy2)); - test_assert(!exit_policy_is_general_exit(NULL)); - - test_assert(cmp_addr_policies(policy, policy2)); - test_assert(cmp_addr_policies(policy, NULL)); - test_assert(!cmp_addr_policies(policy2, policy2)); - test_assert(!cmp_addr_policies(NULL, NULL)); - - test_assert(!policy_is_reject_star(policy2)); - test_assert(policy_is_reject_star(policy)); - test_assert(policy_is_reject_star(NULL)); - - addr_policy_list_free(policy); - policy = NULL; - - /* make sure compacting logic works. */ - policy = NULL; - line.key = (char*)"foo"; - line.value = (char*)"accept *:80,reject private:*,reject *:*"; - line.next = NULL; - test_assert(0 == policies_parse_exit_policy(&line, &policy, 0, NULL)); - test_assert(policy); - //test_streq(policy->string, "accept *:80"); - //test_streq(policy->next->string, "reject *:*"); - test_eq(smartlist_len(policy), 2); - - /* test policy summaries */ - /* check if we properly ignore private IP addresses */ - test_policy_summary_helper("reject 192.168.0.0/16:*," - "reject 0.0.0.0/8:*," - "reject 10.0.0.0/8:*," - "accept *:10-30," - "accept *:90," - "reject *:*", - "accept 10-30,90"); - /* check all accept policies, and proper counting of rejects */ - test_policy_summary_helper("reject 11.0.0.0/9:80," - "reject 12.0.0.0/9:80," - "reject 13.0.0.0/9:80," - "reject 14.0.0.0/9:80," - "accept *:*", "accept 1-65535"); - test_policy_summary_helper("reject 11.0.0.0/9:80," - "reject 12.0.0.0/9:80," - "reject 13.0.0.0/9:80," - "reject 14.0.0.0/9:80," - "reject 15.0.0.0:81," - "accept *:*", "accept 1-65535"); - test_policy_summary_helper("reject 11.0.0.0/9:80," - "reject 12.0.0.0/9:80," - "reject 13.0.0.0/9:80," - "reject 14.0.0.0/9:80," - "reject 15.0.0.0:80," - "accept *:*", - "reject 80"); - /* no exits */ - test_policy_summary_helper("accept 11.0.0.0/9:80," - "reject *:*", - "reject 1-65535"); - /* port merging */ - test_policy_summary_helper("accept *:80," - "accept *:81," - "accept *:100-110," - "accept *:111," - "reject *:*", - "accept 80-81,100-111"); - /* border ports */ - test_policy_summary_helper("accept *:1," - "accept *:3," - "accept *:65535," - "reject *:*", - "accept 1,3,65535"); - /* holes */ - test_policy_summary_helper("accept *:1," - "accept *:3," - "accept *:5," - "accept *:7," - "reject *:*", - "accept 1,3,5,7"); - test_policy_summary_helper("reject *:1," - "reject *:3," - "reject *:5," - "reject *:7," - "accept *:*", - "reject 1,3,5,7"); - - /* truncation ports */ - sm = smartlist_create(); - for (i=1; i<2000; i+=2) { - char buf[POLICY_BUF_LEN]; - tor_snprintf(buf, sizeof(buf), "reject *:%d", i); - smartlist_add(sm, tor_strdup(buf)); - } - smartlist_add(sm, tor_strdup("accept *:*")); - policy_str = smartlist_join_strings(sm, ",", 0, NULL); - test_policy_summary_helper( policy_str, - "accept 2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44," - "46,48,50,52,54,56,58,60,62,64,66,68,70,72,74,76,78,80,82,84,86,88,90," - "92,94,96,98,100,102,104,106,108,110,112,114,116,118,120,122,124,126,128," - "130,132,134,136,138,140,142,144,146,148,150,152,154,156,158,160,162,164," - "166,168,170,172,174,176,178,180,182,184,186,188,190,192,194,196,198,200," - "202,204,206,208,210,212,214,216,218,220,222,224,226,228,230,232,234,236," - "238,240,242,244,246,248,250,252,254,256,258,260,262,264,266,268,270,272," - "274,276,278,280,282,284,286,288,290,292,294,296,298,300,302,304,306,308," - "310,312,314,316,318,320,322,324,326,328,330,332,334,336,338,340,342,344," - "346,348,350,352,354,356,358,360,362,364,366,368,370,372,374,376,378,380," - "382,384,386,388,390,392,394,396,398,400,402,404,406,408,410,412,414,416," - "418,420,422,424,426,428,430,432,434,436,438,440,442,444,446,448,450,452," - "454,456,458,460,462,464,466,468,470,472,474,476,478,480,482,484,486,488," - "490,492,494,496,498,500,502,504,506,508,510,512,514,516,518,520,522"); - - done: - if (policy) - addr_policy_list_free(policy); - if (policy2) - addr_policy_list_free(policy2); - tor_free(policy_str); - if (sm) { - SMARTLIST_FOREACH(sm, char *, s, tor_free(s)); - smartlist_free(sm); - } -} - -/** Run unit tests for basic rendezvous functions. */ -static void -test_rend_fns(void) -{ - char address1[] = "fooaddress.onion"; - char address2[] = "aaaaaaaaaaaaaaaa.onion"; - char address3[] = "fooaddress.exit"; - char address4[] = "www.torproject.org"; - rend_service_descriptor_t *d1 = - tor_malloc_zero(sizeof(rend_service_descriptor_t)); - rend_service_descriptor_t *d2 = NULL; - char *encoded = NULL; - size_t len; - time_t now; - int i; - crypto_pk_env_t *pk1 = pk_generate(0), *pk2 = pk_generate(1); - - /* Test unversioned (v0) descriptor */ - d1->pk = crypto_pk_dup_key(pk1); - now = time(NULL); - d1->timestamp = now; - d1->version = 0; - d1->intro_nodes = smartlist_create(); - for (i = 0; i < 3; i++) { - rend_intro_point_t *intro = tor_malloc_zero(sizeof(rend_intro_point_t)); - intro->extend_info = tor_malloc_zero(sizeof(extend_info_t)); - crypto_rand(intro->extend_info->identity_digest, DIGEST_LEN); - intro->extend_info->nickname[0] = '$'; - base16_encode(intro->extend_info->nickname+1, HEX_DIGEST_LEN+1, - intro->extend_info->identity_digest, DIGEST_LEN); - smartlist_add(d1->intro_nodes, intro); - } - test_assert(! rend_encode_service_descriptor(d1, pk1, &encoded, &len)); - d2 = rend_parse_service_descriptor(encoded, len); - test_assert(d2); - - test_assert(!crypto_pk_cmp_keys(d1->pk, d2->pk)); - test_eq(d2->timestamp, now); - test_eq(d2->version, 0); - test_eq(d2->protocols, 1<<2); - test_eq(smartlist_len(d2->intro_nodes), 3); - for (i = 0; i < 3; i++) { - rend_intro_point_t *intro1 = smartlist_get(d1->intro_nodes, i); - rend_intro_point_t *intro2 = smartlist_get(d2->intro_nodes, i); - test_streq(intro1->extend_info->nickname, - intro2->extend_info->nickname); - } - - test_assert(BAD_HOSTNAME == parse_extended_hostname(address1)); - test_assert(ONION_HOSTNAME == parse_extended_hostname(address2)); - test_assert(EXIT_HOSTNAME == parse_extended_hostname(address3)); - test_assert(NORMAL_HOSTNAME == parse_extended_hostname(address4)); - - crypto_free_pk_env(pk1); - crypto_free_pk_env(pk2); - pk1 = pk2 = NULL; - rend_service_descriptor_free(d1); - rend_service_descriptor_free(d2); - d1 = d2 = NULL; - - done: - if (pk1) - crypto_free_pk_env(pk1); - if (pk2) - crypto_free_pk_env(pk2); - if (d1) - rend_service_descriptor_free(d1); - if (d2) - rend_service_descriptor_free(d2); - tor_free(encoded); -} - -/** Run AES performance benchmarks. */ -static void -bench_aes(void) -{ - int len, i; - char *b1, *b2; - crypto_cipher_env_t *c; - struct timeval start, end; - const int iters = 100000; - uint64_t nsec; - c = crypto_new_cipher_env(); - crypto_cipher_generate_key(c); - crypto_cipher_encrypt_init_cipher(c); - for (len = 1; len <= 8192; len *= 2) { - b1 = tor_malloc_zero(len); - b2 = tor_malloc_zero(len); - tor_gettimeofday(&start); - for (i = 0; i < iters; ++i) { - crypto_cipher_encrypt(c, b1, b2, len); - } - tor_gettimeofday(&end); - tor_free(b1); - tor_free(b2); - nsec = (uint64_t) tv_udiff(&start,&end); - nsec *= 1000; - nsec /= (iters*len); - printf("%d bytes: "U64_FORMAT" nsec per byte\n", len, - U64_PRINTF_ARG(nsec)); - } - crypto_free_cipher_env(c); -} - -/** Run digestmap_t performance benchmarks. */ -static void -bench_dmap(void) -{ - smartlist_t *sl = smartlist_create(); - smartlist_t *sl2 = smartlist_create(); - struct timeval start, end, pt2, pt3, pt4; - const int iters = 10000; - const int elts = 4000; - const int fpostests = 1000000; - char d[20]; - int i,n=0, fp = 0; - digestmap_t *dm = digestmap_new(); - digestset_t *ds = digestset_new(elts); - - for (i = 0; i < elts; ++i) { - crypto_rand(d, 20); - smartlist_add(sl, tor_memdup(d, 20)); - } - for (i = 0; i < elts; ++i) { - crypto_rand(d, 20); - smartlist_add(sl2, tor_memdup(d, 20)); - } - printf("nbits=%d\n", ds->mask+1); - - tor_gettimeofday(&start); - for (i = 0; i < iters; ++i) { - SMARTLIST_FOREACH(sl, const char *, cp, digestmap_set(dm, cp, (void*)1)); - } - tor_gettimeofday(&pt2); - for (i = 0; i < iters; ++i) { - SMARTLIST_FOREACH(sl, const char *, cp, digestmap_get(dm, cp)); - SMARTLIST_FOREACH(sl2, const char *, cp, digestmap_get(dm, cp)); - } - tor_gettimeofday(&pt3); - for (i = 0; i < iters; ++i) { - SMARTLIST_FOREACH(sl, const char *, cp, digestset_add(ds, cp)); - } - tor_gettimeofday(&pt4); - for (i = 0; i < iters; ++i) { - SMARTLIST_FOREACH(sl, const char *, cp, n += digestset_isin(ds, cp)); - SMARTLIST_FOREACH(sl2, const char *, cp, n += digestset_isin(ds, cp)); - } - tor_gettimeofday(&end); - - for (i = 0; i < fpostests; ++i) { - crypto_rand(d, 20); - if (digestset_isin(ds, d)) ++fp; - } - - printf("%ld\n",(unsigned long)tv_udiff(&start, &pt2)); - printf("%ld\n",(unsigned long)tv_udiff(&pt2, &pt3)); - printf("%ld\n",(unsigned long)tv_udiff(&pt3, &pt4)); - printf("%ld\n",(unsigned long)tv_udiff(&pt4, &end)); - printf("-- %d\n", n); - printf("++ %f\n", fp/(double)fpostests); - digestmap_free(dm, NULL); - digestset_free(ds); - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - SMARTLIST_FOREACH(sl2, char *, cp, tor_free(cp)); - smartlist_free(sl); - smartlist_free(sl2); -} - -/** Run unittests for memory pool allocator */ -static void -test_util_mempool(void) -{ - mp_pool_t *pool = NULL; - smartlist_t *allocated = NULL; - int i; - - pool = mp_pool_new(1, 100); - test_assert(pool); - test_assert(pool->new_chunk_capacity >= 100); - test_assert(pool->item_alloc_size >= sizeof(void*)+1); - mp_pool_destroy(pool); - pool = NULL; - - pool = mp_pool_new(241, 2500); - test_assert(pool); - test_assert(pool->new_chunk_capacity >= 10); - test_assert(pool->item_alloc_size >= sizeof(void*)+241); - test_eq(pool->item_alloc_size & 0x03, 0); - test_assert(pool->new_chunk_capacity < 60); - - allocated = smartlist_create(); - for (i = 0; i < 20000; ++i) { - if (smartlist_len(allocated) < 20 || crypto_rand_int(2)) { - void *m = mp_pool_get(pool); - memset(m, 0x09, 241); - smartlist_add(allocated, m); - //printf("%d: %p\n", i, m); - //mp_pool_assert_ok(pool); - } else { - int idx = crypto_rand_int(smartlist_len(allocated)); - void *m = smartlist_get(allocated, idx); - //printf("%d: free %p\n", i, m); - smartlist_del(allocated, idx); - mp_pool_release(m); - //mp_pool_assert_ok(pool); - } - if (crypto_rand_int(777)==0) - mp_pool_clean(pool, 1, 1); - - if (i % 777) - mp_pool_assert_ok(pool); - } - - done: - if (allocated) { - SMARTLIST_FOREACH(allocated, void *, m, mp_pool_release(m)); - mp_pool_assert_ok(pool); - mp_pool_clean(pool, 0, 0); - mp_pool_assert_ok(pool); - smartlist_free(allocated); - } - - if (pool) - mp_pool_destroy(pool); -} - -/** Run unittests for memory area allocator */ -static void -test_util_memarea(void) -{ - memarea_t *area = memarea_new(); - char *p1, *p2, *p3, *p1_orig; - void *malloced_ptr = NULL; - int i; - - test_assert(area); - - p1_orig = p1 = memarea_alloc(area,64); - p2 = memarea_alloc_zero(area,52); - p3 = memarea_alloc(area,11); - - test_assert(memarea_owns_ptr(area, p1)); - test_assert(memarea_owns_ptr(area, p2)); - test_assert(memarea_owns_ptr(area, p3)); - /* Make sure we left enough space. */ - test_assert(p1+64 <= p2); - test_assert(p2+52 <= p3); - /* Make sure we aligned. */ - test_eq(((uintptr_t)p1) % sizeof(void*), 0); - test_eq(((uintptr_t)p2) % sizeof(void*), 0); - test_eq(((uintptr_t)p3) % sizeof(void*), 0); - test_assert(!memarea_owns_ptr(area, p3+8192)); - test_assert(!memarea_owns_ptr(area, p3+30)); - test_assert(tor_mem_is_zero(p2, 52)); - /* Make sure we don't overalign. */ - p1 = memarea_alloc(area, 1); - p2 = memarea_alloc(area, 1); - test_eq(p1+sizeof(void*), p2); - { - malloced_ptr = tor_malloc(64); - test_assert(!memarea_owns_ptr(area, malloced_ptr)); - tor_free(malloced_ptr); - } - - /* memarea_memdup */ - { - malloced_ptr = tor_malloc(64); - crypto_rand((char*)malloced_ptr, 64); - p1 = memarea_memdup(area, malloced_ptr, 64); - test_assert(p1 != malloced_ptr); - test_memeq(p1, malloced_ptr, 64); - tor_free(malloced_ptr); - } - - /* memarea_strdup. */ - p1 = memarea_strdup(area,""); - p2 = memarea_strdup(area, "abcd"); - test_assert(p1); - test_assert(p2); - test_streq(p1, ""); - test_streq(p2, "abcd"); - - /* memarea_strndup. */ - { - const char *s = "Ad ogni porta batte la morte e grida: il nome!"; - /* (From Turandot, act 3.) */ - size_t len = strlen(s); - p1 = memarea_strndup(area, s, 1000); - p2 = memarea_strndup(area, s, 10); - test_streq(p1, s); - test_assert(p2 >= p1 + len + 1); - test_memeq(s, p2, 10); - test_eq(p2[10], '\0'); - p3 = memarea_strndup(area, s, len); - test_streq(p3, s); - p3 = memarea_strndup(area, s, len-1); - test_memeq(s, p3, len-1); - test_eq(p3[len-1], '\0'); - } - - memarea_clear(area); - p1 = memarea_alloc(area, 1); - test_eq(p1, p1_orig); - memarea_clear(area); - - /* Check for running over an area's size. */ - for (i = 0; i < 512; ++i) { - p1 = memarea_alloc(area, crypto_rand_int(5)+1); - test_assert(memarea_owns_ptr(area, p1)); - } - memarea_assert_ok(area); - /* Make sure we can allocate a too-big object. */ - p1 = memarea_alloc_zero(area, 9000); - p2 = memarea_alloc_zero(area, 16); - test_assert(memarea_owns_ptr(area, p1)); - test_assert(memarea_owns_ptr(area, p2)); - - done: - memarea_drop_all(area); - tor_free(malloced_ptr); -} - -/** Run unit tests for utility functions to get file names relative to - * the data directory. */ -static void -test_util_datadir(void) -{ - char buf[1024]; - char *f = NULL; - - f = get_datadir_fname(NULL); - test_streq(f, temp_dir); - tor_free(f); - f = get_datadir_fname("state"); - tor_snprintf(buf, sizeof(buf), "%s"PATH_SEPARATOR"state", temp_dir); - test_streq(f, buf); - tor_free(f); - f = get_datadir_fname2("cache", "thingy"); - tor_snprintf(buf, sizeof(buf), - "%s"PATH_SEPARATOR"cache"PATH_SEPARATOR"thingy", temp_dir); - test_streq(f, buf); - tor_free(f); - f = get_datadir_fname2_suffix("cache", "thingy", ".foo"); - tor_snprintf(buf, sizeof(buf), - "%s"PATH_SEPARATOR"cache"PATH_SEPARATOR"thingy.foo", temp_dir); - test_streq(f, buf); - tor_free(f); - f = get_datadir_fname_suffix("cache", ".foo"); - tor_snprintf(buf, sizeof(buf), "%s"PATH_SEPARATOR"cache.foo", - temp_dir); - test_streq(f, buf); - - done: - tor_free(f); -} - -/** Test AES-CTR encryption and decryption with IV. */ -static void -test_crypto_aes_iv(void) -{ - crypto_cipher_env_t *cipher; - char *plain, *encrypted1, *encrypted2, *decrypted1, *decrypted2; - char plain_1[1], plain_15[15], plain_16[16], plain_17[17]; - char key1[16], key2[16]; - ssize_t encrypted_size, decrypted_size; - - plain = tor_malloc(4095); - encrypted1 = tor_malloc(4095 + 1 + 16); - encrypted2 = tor_malloc(4095 + 1 + 16); - decrypted1 = tor_malloc(4095 + 1); - decrypted2 = tor_malloc(4095 + 1); - - crypto_rand(plain, 4095); - crypto_rand(key1, 16); - crypto_rand(key2, 16); - crypto_rand(plain_1, 1); - crypto_rand(plain_15, 15); - crypto_rand(plain_16, 16); - crypto_rand(plain_17, 17); - key1[0] = key2[0] + 128; /* Make sure that contents are different. */ - /* Encrypt and decrypt with the same key. */ - cipher = crypto_create_init_cipher(key1, 1); - encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted1, 16 + 4095, - plain, 4095); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(encrypted_size, 16 + 4095); - tor_assert(encrypted_size > 0); /* This is obviously true, since 4111 is - * greater than 0, but its truth is not - * obvious to all analysis tools. */ - cipher = crypto_create_init_cipher(key1, 0); - decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 4095, - encrypted1, encrypted_size); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(decrypted_size, 4095); - tor_assert(decrypted_size > 0); - test_memeq(plain, decrypted1, 4095); - /* Encrypt a second time (with a new random initialization vector). */ - cipher = crypto_create_init_cipher(key1, 1); - encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted2, 16 + 4095, - plain, 4095); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(encrypted_size, 16 + 4095); - tor_assert(encrypted_size > 0); - cipher = crypto_create_init_cipher(key1, 0); - decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted2, 4095, - encrypted2, encrypted_size); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(decrypted_size, 4095); - tor_assert(decrypted_size > 0); - test_memeq(plain, decrypted2, 4095); - test_memneq(encrypted1, encrypted2, encrypted_size); - /* Decrypt with the wrong key. */ - cipher = crypto_create_init_cipher(key2, 0); - decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted2, 4095, - encrypted1, encrypted_size); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_memneq(plain, decrypted2, encrypted_size); - /* Alter the initialization vector. */ - encrypted1[0] += 42; - cipher = crypto_create_init_cipher(key1, 0); - decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 4095, - encrypted1, encrypted_size); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_memneq(plain, decrypted2, 4095); - /* Special length case: 1. */ - cipher = crypto_create_init_cipher(key1, 1); - encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted1, 16 + 1, - plain_1, 1); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(encrypted_size, 16 + 1); - tor_assert(encrypted_size > 0); - cipher = crypto_create_init_cipher(key1, 0); - decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 1, - encrypted1, encrypted_size); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(decrypted_size, 1); - tor_assert(decrypted_size > 0); - test_memeq(plain_1, decrypted1, 1); - /* Special length case: 15. */ - cipher = crypto_create_init_cipher(key1, 1); - encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted1, 16 + 15, - plain_15, 15); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(encrypted_size, 16 + 15); - tor_assert(encrypted_size > 0); - cipher = crypto_create_init_cipher(key1, 0); - decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 15, - encrypted1, encrypted_size); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(decrypted_size, 15); - tor_assert(decrypted_size > 0); - test_memeq(plain_15, decrypted1, 15); - /* Special length case: 16. */ - cipher = crypto_create_init_cipher(key1, 1); - encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted1, 16 + 16, - plain_16, 16); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(encrypted_size, 16 + 16); - tor_assert(encrypted_size > 0); - cipher = crypto_create_init_cipher(key1, 0); - decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 16, - encrypted1, encrypted_size); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(decrypted_size, 16); - tor_assert(decrypted_size > 0); - test_memeq(plain_16, decrypted1, 16); - /* Special length case: 17. */ - cipher = crypto_create_init_cipher(key1, 1); - encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted1, 16 + 17, - plain_17, 17); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(encrypted_size, 16 + 17); - tor_assert(encrypted_size > 0); - cipher = crypto_create_init_cipher(key1, 0); - decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 17, - encrypted1, encrypted_size); - test_eq(decrypted_size, 17); - tor_assert(decrypted_size > 0); - test_memeq(plain_17, decrypted1, 17); - - done: - /* Free memory. */ - tor_free(plain); - tor_free(encrypted1); - tor_free(encrypted2); - tor_free(decrypted1); - tor_free(decrypted2); - if (cipher) - crypto_free_cipher_env(cipher); -} - -/** Test base32 decoding. */ -static void -test_crypto_base32_decode(void) -{ - char plain[60], encoded[96 + 1], decoded[60]; - int res; - crypto_rand(plain, 60); - /* Encode and decode a random string. */ - base32_encode(encoded, 96 + 1, plain, 60); - res = base32_decode(decoded, 60, encoded, 96); - test_eq(res, 0); - test_memeq(plain, decoded, 60); - /* Encode, uppercase, and decode a random string. */ - base32_encode(encoded, 96 + 1, plain, 60); - tor_strupper(encoded); - res = base32_decode(decoded, 60, encoded, 96); - test_eq(res, 0); - test_memeq(plain, decoded, 60); - /* Change encoded string and decode. */ - if (encoded[0] == 'A' || encoded[0] == 'a') - encoded[0] = 'B'; - else - encoded[0] = 'A'; - res = base32_decode(decoded, 60, encoded, 96); - test_eq(res, 0); - test_memneq(plain, decoded, 60); - /* Bad encodings. */ - encoded[0] = '!'; - res = base32_decode(decoded, 60, encoded, 96); - test_assert(res < 0); - - done: - ; -} - -/** Test encoding and parsing of v2 rendezvous service descriptors. */ -static void -test_rend_fns_v2(void) -{ - rend_service_descriptor_t *generated = NULL, *parsed = NULL; - char service_id[DIGEST_LEN]; - char service_id_base32[REND_SERVICE_ID_LEN_BASE32+1]; - const char *next_desc; - smartlist_t *descs = smartlist_create(); - char computed_desc_id[DIGEST_LEN]; - char parsed_desc_id[DIGEST_LEN]; - crypto_pk_env_t *pk1 = NULL, *pk2 = NULL; - time_t now; - char *intro_points_encrypted = NULL; - size_t intro_points_size; - size_t encoded_size; - int i; - pk1 = pk_generate(0); - pk2 = pk_generate(1); - generated = tor_malloc_zero(sizeof(rend_service_descriptor_t)); - generated->pk = crypto_pk_dup_key(pk1); - crypto_pk_get_digest(generated->pk, service_id); - base32_encode(service_id_base32, REND_SERVICE_ID_LEN_BASE32+1, - service_id, REND_SERVICE_ID_LEN); - now = time(NULL); - generated->timestamp = now; - generated->version = 2; - generated->protocols = 42; - generated->intro_nodes = smartlist_create(); - - for (i = 0; i < 3; i++) { - rend_intro_point_t *intro = tor_malloc_zero(sizeof(rend_intro_point_t)); - crypto_pk_env_t *okey = pk_generate(2 + i); - intro->extend_info = tor_malloc_zero(sizeof(extend_info_t)); - intro->extend_info->onion_key = okey; - crypto_pk_get_digest(intro->extend_info->onion_key, - intro->extend_info->identity_digest); - //crypto_rand(info->identity_digest, DIGEST_LEN); /* Would this work? */ - intro->extend_info->nickname[0] = '$'; - base16_encode(intro->extend_info->nickname + 1, - sizeof(intro->extend_info->nickname) - 1, - intro->extend_info->identity_digest, DIGEST_LEN); - /* Does not cover all IP addresses. */ - tor_addr_from_ipv4h(&intro->extend_info->addr, crypto_rand_int(65536)); - intro->extend_info->port = crypto_rand_int(65536); - intro->intro_key = crypto_pk_dup_key(pk2); - smartlist_add(generated->intro_nodes, intro); - } - test_assert(rend_encode_v2_descriptors(descs, generated, now, 0, - REND_NO_AUTH, NULL, NULL) > 0); - test_assert(rend_compute_v2_desc_id(computed_desc_id, service_id_base32, - NULL, now, 0) == 0); - test_memeq(((rend_encoded_v2_service_descriptor_t *) - smartlist_get(descs, 0))->desc_id, computed_desc_id, DIGEST_LEN); - test_assert(rend_parse_v2_service_descriptor(&parsed, parsed_desc_id, - &intro_points_encrypted, - &intro_points_size, - &encoded_size, - &next_desc, - ((rend_encoded_v2_service_descriptor_t *) - smartlist_get(descs, 0))->desc_str) == 0); - test_assert(parsed); - test_memeq(((rend_encoded_v2_service_descriptor_t *) - smartlist_get(descs, 0))->desc_id, parsed_desc_id, DIGEST_LEN); - test_eq(rend_parse_introduction_points(parsed, intro_points_encrypted, - intro_points_size), 3); - test_assert(!crypto_pk_cmp_keys(generated->pk, parsed->pk)); - test_eq(parsed->timestamp, now); - test_eq(parsed->version, 2); - test_eq(parsed->protocols, 42); - test_eq(smartlist_len(parsed->intro_nodes), 3); - for (i = 0; i < smartlist_len(parsed->intro_nodes); i++) { - rend_intro_point_t *par_intro = smartlist_get(parsed->intro_nodes, i), - *gen_intro = smartlist_get(generated->intro_nodes, i); - extend_info_t *par_info = par_intro->extend_info; - extend_info_t *gen_info = gen_intro->extend_info; - test_assert(!crypto_pk_cmp_keys(gen_info->onion_key, par_info->onion_key)); - test_memeq(gen_info->identity_digest, par_info->identity_digest, - DIGEST_LEN); - test_streq(gen_info->nickname, par_info->nickname); - test_assert(tor_addr_eq(&gen_info->addr, &par_info->addr)); - test_eq(gen_info->port, par_info->port); - } - - rend_service_descriptor_free(parsed); - rend_service_descriptor_free(generated); - parsed = generated = NULL; - - done: - if (descs) { - for (i = 0; i < smartlist_len(descs); i++) - rend_encoded_v2_service_descriptor_free(smartlist_get(descs, i)); - smartlist_free(descs); - } - if (parsed) - rend_service_descriptor_free(parsed); - if (generated) - rend_service_descriptor_free(generated); - if (pk1) - crypto_free_pk_env(pk1); - if (pk2) - crypto_free_pk_env(pk2); - tor_free(intro_points_encrypted); -} - -/** Run unit tests for GeoIP code. */ -static void -test_geoip(void) -{ - int i, j; - time_t now = time(NULL); - char *s = NULL; - - /* Populate the DB a bit. Add these in order, since we can't do the final - * 'sort' step. These aren't very good IP addresses, but they're perfectly - * fine uint32_t values. */ - test_eq(0, geoip_parse_entry("10,50,AB")); - test_eq(0, geoip_parse_entry("52,90,XY")); - test_eq(0, geoip_parse_entry("95,100,AB")); - test_eq(0, geoip_parse_entry("\"105\",\"140\",\"ZZ\"")); - test_eq(0, geoip_parse_entry("\"150\",\"190\",\"XY\"")); - test_eq(0, geoip_parse_entry("\"200\",\"250\",\"AB\"")); - - /* We should have 3 countries: ab, xy, zz. */ - test_eq(3, geoip_get_n_countries()); - /* Make sure that country ID actually works. */ -#define NAMEFOR(x) geoip_get_country_name(geoip_get_country_by_ip(x)) - test_streq("ab", NAMEFOR(32)); - test_streq("??", NAMEFOR(5)); - test_streq("??", NAMEFOR(51)); - test_streq("xy", NAMEFOR(150)); - test_streq("xy", NAMEFOR(190)); - test_streq("??", NAMEFOR(2000)); -#undef NAMEFOR - - get_options()->BridgeRelay = 1; - get_options()->BridgeRecordUsageByCountry = 1; - /* Put 9 observations in AB... */ - for (i=32; i < 40; ++i) - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, i, now-7200); - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, 225, now-7200); - /* and 3 observations in XY, several times. */ - for (j=0; j < 10; ++j) - for (i=52; i < 55; ++i) - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, i, now-3600); - /* and 17 observations in ZZ... */ - for (i=110; i < 127; ++i) - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, i, now); - s = geoip_get_client_history(now+5*24*60*60, GEOIP_CLIENT_CONNECT); - test_assert(s); - test_streq("zz=24,ab=16,xy=8", s); - tor_free(s); - - /* Now clear out all the AB observations. */ - geoip_remove_old_clients(now-6000); - s = geoip_get_client_history(now+5*24*60*60, GEOIP_CLIENT_CONNECT); - test_assert(s); - test_streq("zz=24,xy=8", s); - - done: - tor_free(s); -} - -/** For test_array. Declare an CLI-invocable off-by-default function in the - * unit tests, with function name and user-visible name <b>x</b>*/ -#define DISABLED(x) { #x, x, 0, 0, 0 } -/** For test_array. Declare an CLI-invocable unit test function, with function - * name test_<b>x</b>(), and user-visible name <b>x</b> */ -#define ENT(x) { #x, test_ ## x, 0, 0, 1 } -/** For test_array. Declare an CLI-invocable unit test function, with function - * name test_<b>x</b>_<b>y</b>(), and user-visible name - * <b>x</b>/<b>y</b>. This function will be treated as a subentry of <b>x</b>, - * so that invoking <b>x</b> from the CLI invokes this test too. */ -#define SUBENT(x,y) { #x "/" #y, test_ ## x ## _ ## y, 1, 0, 1 } - -/** An array of functions and information for all the unit tests we can run. */ -static struct { - const char *test_name; /**< How does the user refer to this test from the - * command line? */ - void (*test_fn)(void); /**< What function is called to run this test? */ - int is_subent; /**< Is this a subentry of a bigger set of related tests? */ - int selected; /**< Are we planning to run this one? */ - int is_default; /**< If the user doesn't say what tests they want, do they - * get this function by default? */ -} test_array[] = { - ENT(buffers), - ENT(crypto), - SUBENT(crypto, rng), - SUBENT(crypto, aes), - SUBENT(crypto, sha), - SUBENT(crypto, pk), - SUBENT(crypto, dh), - SUBENT(crypto, s2k), - SUBENT(crypto, aes_iv), - SUBENT(crypto, base32_decode), - ENT(util), - SUBENT(util, di_ops), - SUBENT(util, ip6_helpers), - SUBENT(util, gzip), - SUBENT(util, datadir), - SUBENT(util, smartlist_basic), - SUBENT(util, smartlist_strings), - SUBENT(util, smartlist_overlap), - SUBENT(util, smartlist_digests), - SUBENT(util, smartlist_join), - SUBENT(util, bitarray), - SUBENT(util, digestset), - SUBENT(util, mempool), - SUBENT(util, memarea), - SUBENT(util, strmap), - SUBENT(util, control_formats), - SUBENT(util, pqueue), - SUBENT(util, mmap), - SUBENT(util, threads), - SUBENT(util, order_functions), - SUBENT(util, sscanf), - ENT(onion_handshake), - ENT(dir_format), - ENT(dirutil), - ENT(v3_networkstatus), - ENT(policies), - ENT(rend_fns), - SUBENT(rend_fns, v2), - ENT(geoip), - - DISABLED(bench_aes), - DISABLED(bench_dmap), - { NULL, NULL, 0, 0, 0 }, -}; - -static void syntax(void) ATTR_NORETURN; - -/** Print a syntax usage message, and exit.*/ -static void -syntax(void) -{ - int i; - printf("Syntax:\n" - " test [-v|--verbose] [--warn|--notice|--info|--debug]\n" - " [testname...]\n" - "Recognized tests are:\n"); - for (i = 0; test_array[i].test_name; ++i) { - printf(" %s\n", test_array[i].test_name); - } - - exit(0); -} - -/** Main entry point for unit test code: parse the command line, and run - * some unit tests. */ -int -main(int c, char**v) -{ - or_options_t *options; - char *errmsg = NULL; - int i; - int verbose = 0, any_selected = 0; - int loglevel = LOG_ERR; - -#ifdef USE_DMALLOC - { - int r = CRYPTO_set_mem_ex_functions(_tor_malloc, _tor_realloc, _tor_free); - tor_assert(r); - } -#endif - - update_approx_time(time(NULL)); - options = options_new(); - tor_threads_init(); - init_logging(); - - for (i = 1; i < c; ++i) { - if (!strcmp(v[i], "-v") || !strcmp(v[i], "--verbose")) - verbose++; - else if (!strcmp(v[i], "--warn")) - loglevel = LOG_WARN; - else if (!strcmp(v[i], "--notice")) - loglevel = LOG_NOTICE; - else if (!strcmp(v[i], "--info")) - loglevel = LOG_INFO; - else if (!strcmp(v[i], "--debug")) - loglevel = LOG_DEBUG; - else if (!strcmp(v[i], "--help") || !strcmp(v[i], "-h") || v[i][0] == '-') - syntax(); - else { - int j, found=0; - for (j = 0; test_array[j].test_name; ++j) { - if (!strcmp(v[i], test_array[j].test_name) || - (test_array[j].is_subent && - !strcmpstart(test_array[j].test_name, v[i]) && - test_array[j].test_name[strlen(v[i])] == '/') || - (v[i][0] == '=' && !strcmp(v[i]+1, test_array[j].test_name))) { - test_array[j].selected = 1; - any_selected = 1; - found = 1; - } - } - if (!found) { - printf("Unknown test: %s\n", v[i]); - syntax(); - } - } - } - - if (!any_selected) { - for (i = 0; test_array[i].test_name; ++i) { - test_array[i].selected = test_array[i].is_default; - } - } - - { - log_severity_list_t s; - memset(&s, 0, sizeof(s)); - set_log_severity_config(loglevel, LOG_ERR, &s); - add_stream_log(&s, "", fileno(stdout)); - } - - options->command = CMD_RUN_UNITTESTS; - crypto_global_init(0); - rep_hist_init(); - network_init(); - setup_directory(); - options_init(options); - options->DataDirectory = tor_strdup(temp_dir); - if (set_options(options, &errmsg) < 0) { - printf("Failed to set initial options: %s\n", errmsg); - tor_free(errmsg); - return 1; - } - - crypto_seed_rng(1); - - atexit(remove_directory); - - printf("Running Tor unit tests on %s\n", get_uname()); - - for (i = 0; test_array[i].test_name; ++i) { - if (!test_array[i].selected) - continue; - if (!test_array[i].is_subent) { - printf("\n============================== %s\n",test_array[i].test_name); - } else if (test_array[i].is_subent && verbose) { - printf("\n%s", test_array[i].test_name); - } - test_array[i].test_fn(); - } - puts(""); - - free_pregenerated_keys(); -#ifdef USE_DMALLOC - tor_free_all(0); - dmalloc_log_unfreed(); -#endif - - if (have_failed) - return 1; - else - return 0; -} - diff --git a/src/or/test_data.c b/src/or/test_data.c deleted file mode 100644 index fc85857615..0000000000 --- a/src/or/test_data.c +++ /dev/null @@ -1,173 +0,0 @@ -/* Copyright 2001-2004 Roger Dingledine. - * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2011, The Tor Project, Inc. */ -/* See LICENSE for licensing information */ - -/** First of 3 example authority certificates for unit testing. */ -const char AUTHORITY_CERT_1[] = -"dir-key-certificate-version 3\n" -"fingerprint D867ACF56A9D229B35C25F0090BC9867E906BE69\n" -"dir-key-published 2008-12-12 18:07:24\n" -"dir-key-expires 2009-12-12 18:07:24\n" -"dir-identity-key\n" -"-----BEGIN RSA PUBLIC KEY-----\n" -"MIIBigKCAYEAveMpKlw8oD1YqFqpJchuwSR82BDhutbqgHiez3QO9FmzOctJpV+Y\n" -"mpTYIJLS/qC+4GBKFF1VK0C4SoBrS3zri0qdXdE+vBGcyrxrjMklpxoqSKRY2011\n" -"4eqYPghKlo5RzuqteBclGCHyNxWjUJeRKDWgvh+U/gr2uYM6fRm5q0fCzg4aECE7\n" -"VP6fDGZrMbQI8jHpiMSoC9gkUASNEa6chLInlnP8/H5qUEW4TB9CN/q095pefuwL\n" -"P+F+1Nz5hnM7fa5XmeMB8iM4RriUmOQlLBZgpQBMpEfWMIPcR9F1Gh3MxERqqUcH\n" -"tmij+IZdeXg9OkCXykcabaYIhZD3meErn9Tax4oA/THduLfgli9zM0ExwzH1OooN\n" -"L8rIcJ+2eBo3bQiQUbdYW71sl9w7nSPtircbJUa1mUvWYLPWQxFliPiQSetgJLMj\n" -"VQqtPmV2hvN2Xk3lLfJO50qMTK7w7Gsaw8UtV4YDM1Hcjp/hQaIB1xfwhXgl+eUU\n" -"btUa4c+cUTjHAgMBAAE=\n" -"-----END RSA PUBLIC KEY-----\n" -"dir-signing-key\n" -"-----BEGIN RSA PUBLIC KEY-----\n" -"MIGJAoGBALPSUInyuEu6NV3NjozplaniIEBzQXEjv1x9/+mqnwZABpYVmuy9A8nx\n" -"eoyY3sZFsnYwNW/IZjAgG23pEmevu3F+L4myMjjaa6ORl3MgRYQ4gmuFqpefrGdm\n" -"ywRCleh2JerkQ4VxOuq10dn/abITzLyaZzMw30KXWp5pxKXOLtxFAgMBAAE=\n" -"-----END RSA PUBLIC KEY-----\n" -"dir-key-crosscert\n" -"-----BEGIN ID SIGNATURE-----\n" -"FTBJNR/Hlt4T53yUMp1r/QCSMCpkHJCbYBT0R0pvYqhqFfYN5qHRSICRXaFFImIF\n" -"0DGWmwRza6DxPKNzkm5/b7I0de9zJW1jNNdQAQK5xppAtQcAafRdu8cBonnmh9KX\n" -"k1NrAK/X00FYywju3yl/SxCn1GddVNkHYexEudmJMPM=\n" -"-----END ID SIGNATURE-----\n" -"dir-key-certification\n" -"-----BEGIN SIGNATURE-----\n" -"pjWguLFBfELZDc6DywL6Do21SCl7LcutfpM92MEn4WYeSNcTXNR6lRX7reOEJk4e\n" -"NwEaMt+Hl7slgeR5wjnW3OmMmRPZK9bquNWbfD+sAOV9bRFZTpXIdleAQFPlwvMF\n" -"z/Gzwspzn4i2Yh6hySShrctMmW8YL3OM8LsBXzBhp/rG2uHlsxmIsc13DA6HWt61\n" -"ffY72uNE6KckDGsQ4wPGP9q69y6g+X+TNio1KPbsILbePv6EjbO+rS8FiS4njPlg\n" -"SPYry1RaUvxzxTkswIzdE1tjJrUiqpbWlTGxrH9N4OszoLm45Pc784KLULrjKIoi\n" -"Q+vRsGrcMBAa+kDowWU6H1ryKR7KOhzRTcf2uqLE/W3ezaRwmOG+ETmoVFwbhk2X\n" -"OlbXEM9fWP+INvFkr6Z93VYL2jGkCjV7e3xXmre/Lb92fUcYi6t5dwzfV8gJnIoG\n" -"eCHd0K8NrQK0ipVk/7zcPDKOPeo9Y5aj/f6X/pDHtb+Dd5sT+l82G/Tqy4DIYUYR\n" -"-----END SIGNATURE-----\n"; - -/** The private signing key for AUTHORITY_CERT_1 */ -const char AUTHORITY_SIGNKEY_1[] = -"-----BEGIN RSA PRIVATE KEY-----\n" -"MIICWwIBAAKBgQCz0lCJ8rhLujVdzY6M6ZWp4iBAc0FxI79cff/pqp8GQAaWFZrs\n" -"vQPJ8XqMmN7GRbJ2MDVvyGYwIBtt6RJnr7txfi+JsjI42mujkZdzIEWEOIJrhaqX\n" -"n6xnZssEQpXodiXq5EOFcTrqtdHZ/2myE8y8mmczMN9Cl1qeacSlzi7cRQIDAQAB\n" -"AoGASpzUkDinIbzU0eQt5ugxEnliOnvYRpK3nzAk1JbYPyan1PSIAPz4qn1JBTeV\n" -"EB3xS7r7ITO8uvFHkFZqLZ2sH1uE6e4sAytJGO+kyqnlkiDTPEXpcGe99j8PH1yj\n" -"xUOrHRlAYWjG8NEkQi+APA+HZkswE3L/viFwR2AARoE2ac0CQQDsOLdNJa+mqn6N\n" -"1L76nEl/YgXHtKUks+beOR3IgknKEjcsJJEUHyiu0wjbXZV6gTtyQvcAePglUUD1\n" -"R2OkOOADAkEAwuCxvHEAPeQbVt8fSvxw74vqew6LITP2Utb1dQK0E26IRPF36BsJ\n" -"buO/gqMZv6ALq+/KxpA/pUsApbgog9uUFwJAYvHCvbrKX1pM1iXFtP1fv86UMzlU\n" -"bxI34t8zvXftZonIuGG8rxv6E3hr3k7NvNmCx/KKuZTyA9eMCPFVKEV2dwJACn8j\n" -"06yagLrqphE6lEVop953cM1lvRIZcHjXm8fbfzhy6pO/C6d5KJnn1NeIKYQrXMV7\n" -"vJpEc1jI3iQ/Omr3XQJAEBIt5MlP2wlrX9om7B+32XBygUssY3cw/bXybZrtSU0/\n" -"Yx4lqK0ca5IkTp3HevwnlWaJgbaOTUspCVshzJBhDA==\n" -"-----END RSA PRIVATE KEY-----\n"; - -/** Second of 3 example authority certificates for unit testing. */ -const char AUTHORITY_CERT_2[] = -"dir-key-certificate-version 3\n" -"fingerprint 4D44AE0470B9E88FD4558EFEC82698FB33715400\n" -"dir-key-published 2007-06-13 16:52:32\n" -"dir-key-expires 2008-06-13 16:52:32\n" -"dir-identity-key\n" -"-----BEGIN RSA PUBLIC KEY-----\n" -"MIIBigKCAYEAqukDwQRm1Oy1pPY+7GNRnRNFJzEVPUBfJwC4tBH19tkvdRQPuIGI\n" -"2jiTy/rmZ6CLcl1G0oulSgxfKEX75QdptOasZu+rKUrRRSxx0QrXhs9a7up0rpXh\n" -"13fw3mh1Vl/As3rJYF30Hjk01BTOJMxi/HY2y0ALQytFWjiMGY74A9Y6+uDcHkB2\n" -"KflBjxIl8zpCsXsTTnUhN5kXqaOOnK46XaUShSpXsyOxTMJXuJEtgLz9XCyA8XjW\n" -"d75QLHucEnlTqxUAdI5YSN2KIlIJiySCVnAorDpJey2mE9VncpHQWMCv/FPFdnSU\n" -"EMMPUc4bBShcoNFf0mMJeV2sv+dBkgKAL0GLM19PuJIThJhfN/B6+YQTxw4HEpPV\n" -"plfUqYRN0fYC+5hCTS6rroO/uCfDR7NBtoeDNm9dQrvjfk3b/Mywah1rdWNjnVqs\n" -"tPJaz3fc/CVBOUUexhmyktgLuwSNEYIQilQ+BydkWN/4RObhV+YSV5BgekEDVaoS\n" -"RHw4IbYBDHVxAgMBAAE=\n" -"-----END RSA PUBLIC KEY-----\n" -"dir-signing-key\n" -"-----BEGIN RSA PUBLIC KEY-----\n" -"MIGJAoGBAOu3dgrQth3iqvi/UzfywaANw0bBUuMOBhnMBeiLEcRLneJHUJkVvrpR\n" -"/EDQkdMov1e7CX6aqBKygVnbDNYjJ+bcQej8MKpuuW+zIknnz5lfnAVZO5uAmo3Y\n" -"DpG574oQ2FFMdkWHSBloIRxSj/E4Jn1M2qJjElBXP0E33Ka/Noo7AgMBAAE=\n" -"-----END RSA PUBLIC KEY-----\n" -"dir-key-certification\n" -"-----BEGIN SIGNATURE-----\n" -"Fv0Li68QUdAiChY3OklZOakHzwXAUfCzDNxkqe+HLC0n6ZECE9ZCvLVo69XmgVhH\n" -"L5qYr2rxT6QpF+9yuOHbN9gWn8EsDcli06MlhX9TUt/IYVxHa/9tJwNoTfEw2w2D\n" -"tyHhWm94IfOK7/Sea6jHnjckl80X+kk0ZNtAGs3/6fP4iltKNGXnvBwfgLpEgW7X\n" -"NpDl0OLeDuA79zem2GogwQZQdoDbePByU0TJVx9jYi2Bzx2Nb2H0hRTPP6+dY0HQ\n" -"MHb7yyyTQRad5iAUnExKhhyt22p7X3a6lgkAhq4YrNn/zVPkpnT2dzjsOydTHOW8\n" -"2BQs33QlGNe095i47pJBDYsUgmJaXfqB/RG6dFg7jwIsc3/7dZcvcqfxY7wKcD/T\n" -"wtogCIKxDvWbZn7f0hqYkT6uQC8Zom8bcnedmyzufOZCyA2SqQ2wvio6lznR4RIB\n" -"a8qDHR0tPS9/VkqTPcvUWCZeY3UiDeWPjoK1nea1pz6DHDWglKPx86a0amjjayZQ\n" -"-----END SIGNATURE-----\n"; - -/** The private signing key for AUTHORITY_CERT_2 */ -const char AUTHORITY_SIGNKEY_2[] = -"-----BEGIN RSA PRIVATE KEY-----\n" -"MIICXgIBAAKBgQDrt3YK0LYd4qr4v1M38sGgDcNGwVLjDgYZzAXoixHES53iR1CZ\n" -"Fb66UfxA0JHTKL9Xuwl+mqgSsoFZ2wzWIyfm3EHo/DCqbrlvsyJJ58+ZX5wFWTub\n" -"gJqN2A6Rue+KENhRTHZFh0gZaCEcUo/xOCZ9TNqiYxJQVz9BN9ymvzaKOwIDAQAB\n" -"AoGAJ+I9/ex8tCfTSA2PdisEKiHKBeHWNYb870Z/RW6qje1BhLUOZSixwfL3XLwt\n" -"wG3nml+SZrKid69uhZaz4FPIf0tqCgURf6dDrF5vuzzr7VLVqkZHYSBp0vE6bu0R\n" -"Sgc5QNxI2talgc4bsp0O0C+Zd4n3Yto0pXl/I6NHVAxlFBECQQD2mahkY+QEHWPV\n" -"yRY3w3HhRmWBcrkY2zVyvPpqfn/sdHRPYW/yj4Xr/d1CO9VyFmEs4k324lIvu6LT\n" -"WDdpPlcJAkEA9LOZv5aNeAm8ckvvXH7iv8KiONiSz0n9wlisxMhNYTEkOCo1g7jG\n" -"AX5ZknRC9s4sWCPOBpMhloUvemdQ5FCEIwJBAMqCFwoSCf7jD8hRcUBr7QodoF/0\n" -"kVJ7OeI2lMJ9jZnlbFp/3snn2Qeam2e38SnWfQi582KKKwnt4eIDMMXpntkCQQDI\n" -"v1Lh11wl3y7nQZ6T7lCNatp08k+2mQgCWYcbRQweMRd6sD4I2xwt+372ZETPfyLo\n" -"CC+sOyYx+v+RVpMJS3irAkEA6l98nMteZKmhOgyKSjdolP+ahpZunb+WnCdAtP97\n" -"rjZyXmEZS3oe7TRCDD28GAGMmxSDvNfOOpyn14ishEs5AQ==\n" -"-----END RSA PRIVATE KEY-----\n"; - -/** Third of 3 example authority certificates for unit testing. */ -const char AUTHORITY_CERT_3[] = -"dir-key-certificate-version 3\n" -"fingerprint ED3719BF554DE9D7D59F5CA5A4F5AD121D020ED9\n" -"dir-key-published 2007-06-13 16:52:40\n" -"dir-key-expires 2008-06-13 16:52:40\n" -"dir-identity-key\n" -"-----BEGIN RSA PUBLIC KEY-----\n" -"MIIBigKCAYEAtB+yw4BNxtZAG4cPaedkhWNmeij7IuNWmXjh58ZYEGurvGyHs1w4\n" -"QlwNYI2UftSIeIGdWZ5fJ17h9P3xvO6eeJuOt4KPrNOxUbSGrELEx1Lje1fDAJ1X\n" -"SvN+dvptusxtyFUr8afgTPrFIvYuazQ6q/Rw+NDagjmDx3h/A/enihpBnjwzeH8j\n" -"Xzu7b+HKnzFnNfveTDdvSy0NSC6tCOnrfXo31XbXRXtlesnMIpbJClUcAv55eyai\n" -"/PrVPCCUz8mk0sQnn2Xhv1YJmwOlQTGMfg0a0kWLmh+UWcHsGQ4VWxBZJcuzgFHG\n" -"hu2/Fz6DXSpX5Q6B9HKoGmnH1oBh24l0kUW1jL8BxPY4YDU1Lt5t3qgcDn9dXYcI\n" -"o8VvyI0ecSc26Q2PYFWX1hpN4VIBZ8uGaW3IpyTdNiRq0g3iMGRFEXcDlWuyMB9E\n" -"EbSM7m/79V/z7SjDd75EP8Z0qDPESEVB8a8LbuSJtzFVE0KHd7RzkIEN5sorXspZ\n" -"/THukftSmkIvAgMBAAE=\n" -"-----END RSA PUBLIC KEY-----\n" -"dir-signing-key\n" -"-----BEGIN RSA PUBLIC KEY-----\n" -"MIGJAoGBANrSZlUq38Boz3iuUOydYTJV57rTbq1bz805FP2QG2Z+2bwpgKIOZag/\n" -"gN2A1ySJaIYLgZIg9irxrLkqlY/UAjC23y6V9fJXP1S3TXoqLmHleW8PsaDLuwTo\n" -"hCWaR61Mx9WG7IXcodn2Z7RiCfZpSW4Rztbk5WtjQa5jPXSFOuBJAgMBAAE=\n" -"-----END RSA PUBLIC KEY-----\n" -"dir-key-certification\n" -"-----BEGIN SIGNATURE-----\n" -"UNXZy+4OQ8iat+gw+vg2ynvKj2BYbqZt+EAZAV3rmw6gux44U9TLRECRd6LsA08N\n" -"4+Vz01TU81xqMgfrUy94ei2YvcfpO8art9/muWHTP9SmOX8S1uqDqLWA+n723C9A\n" -"HyVXn4aINncO2081gJcIW5+Ul8WTCeZe/n3LVPTCKbTdqxvmrPUdCWlJTQUmb19M\n" -"T+kcCjaEfgQGLC+Y2MHqYe/nxz+aBKqpjiWUDdjc35va6r/2e3c0jGi1B1xRZxN1\n" -"xThPZ+CifjDoWBxJdDGlIfZRK1lMnOCJY9w9ibTXQ1UnvE4whFvmB55/t9/XLq4q\n" -"3pnZz0H7funey3+ilmTxDohoAYT1GX+4a+3xYH07UmAFqlTzqKClj84XEHn+Cer7\n" -"Nun9kJlJFuBgUpQjwCkzedFZKKLOHgB2h7trJfnqcBpAM8Rup1Bb5u/RcBx9gy1q\n" -"pMc65FviIrc/Q5TUku6NNbCbnGll1599PvWuUzkG42lJ17V6psKHIsqGtVdHlCUc\n" -"-----END SIGNATURE-----\n"; - -/** The private signing key for AUTHORITY_CERT_3 */ -const char AUTHORITY_SIGNKEY_3[] = -"-----BEGIN RSA PRIVATE KEY-----\n" -"MIICXgIBAAKBgQDa0mZVKt/AaM94rlDsnWEyVee6026tW8/NORT9kBtmftm8KYCi\n" -"DmWoP4DdgNckiWiGC4GSIPYq8ay5KpWP1AIwtt8ulfXyVz9Ut016Ki5h5XlvD7Gg\n" -"y7sE6IQlmketTMfVhuyF3KHZ9me0Ygn2aUluEc7W5OVrY0GuYz10hTrgSQIDAQAB\n" -"AoGBAIyoeG1AnQmildKeQpiGZackf0uhg2BeRwpFKg//5Q0Sd0Wza+M/2+q1v1Ei\n" -"86ihxxV7KfPTykk6hmuUSwVkI28Z+5J9NYTr35EzPiUlqpo0iclTkFqrlbqSPULx\n" -"9fQhvcOGv1c0m5CnYrHsM8eu3tagLg+6OE4abLOYX4Az5pkxAkEA/NwHhVaVJrXH\n" -"lGDrRAfGtaD5Tzeeg1H9DNZi5lmFiSNR0O11sgDLkiZNP5oM8knyqo8Gq08hwxEb\n" -"yqMXM3XtJQJBAN2KJbFhOjDIkvJyYvbmcP6P7vV2c9j+oUTKkFMF7vvfWunxMi9j\n" -"ghbdUKgl7tU0VFpw7ufDDD0pkN6sua3gp1UCQQCvNzTK861U7p/GtMYyFQVf9JTt\n" -"jMf9jYHBNInBvwTme6AFG5bz6tMlif77dJ9GAXHzODrR2Hq3thJA/3RjR3M1AkBg\n" -"+6M4ncmtpYC+5lhwob0Bk90WU/6vFflfdhXsYoKWfNb95vsDR9qhS82Nbt25NClh\n" -"VmMfzoFDHTkwYgj/F4PpAkEA+RaaSRP7BmbvFNqvlm8J/m0RVdAH4+p/Q5Z5u6Yo\n" -"N7xC/gFi0qFPGKsDvD2CncAYmt+KNsd8S0JGDN4eieKn+Q==\n" -"-----END RSA PRIVATE KEY-----\n"; - diff --git a/src/or/tor_main.c b/src/or/tor_main.c index 8fd3804b29..1ce14ab768 100644 --- a/src/or/tor_main.c +++ b/src/or/tor_main.c @@ -7,7 +7,7 @@ * built from. This string is generated by a bit of shell kludging int * src/or/Makefile.am, and is usually right. */ -const char tor_svn_revision[] = +const char tor_git_revision[] = #ifndef _MSC_VER #include "micro-revision.i" #endif |